diff --git a/app/public/dict/band-0.json.gz b/app/public/dict/band-0.json.gz new file mode 100644 index 0000000..c072f9a Binary files /dev/null and b/app/public/dict/band-0.json.gz differ diff --git a/app/public/dict/band-1.json.gz b/app/public/dict/band-1.json.gz new file mode 100644 index 0000000..d53e157 Binary files /dev/null and b/app/public/dict/band-1.json.gz differ diff --git a/app/public/dict/band-2.json.gz b/app/public/dict/band-2.json.gz new file mode 100644 index 0000000..f6a4f81 Binary files /dev/null and b/app/public/dict/band-2.json.gz differ diff --git a/app/public/dict/band-3.json.gz b/app/public/dict/band-3.json.gz new file mode 100644 index 0000000..c7b7582 Binary files /dev/null and b/app/public/dict/band-3.json.gz differ diff --git a/app/public/dict/band-4.json.gz b/app/public/dict/band-4.json.gz new file mode 100644 index 0000000..62ffff7 Binary files /dev/null and b/app/public/dict/band-4.json.gz differ diff --git a/app/public/dict/band-5.json.gz b/app/public/dict/band-5.json.gz new file mode 100644 index 0000000..c904f69 Binary files /dev/null and b/app/public/dict/band-5.json.gz differ diff --git a/app/public/dict/band-6.json.gz b/app/public/dict/band-6.json.gz new file mode 100644 index 0000000..5acf3a7 Binary files /dev/null and b/app/public/dict/band-6.json.gz differ diff --git a/app/public/dict/manifest.json b/app/public/dict/manifest.json new file mode 100644 index 0000000..e995947 --- /dev/null +++ b/app/public/dict/manifest.json @@ -0,0 +1,81 @@ +{ + "builtWith": { + "dictionary": "kaikki", + "dictionaryEntries": 33419, + "frequencyForms": 688129 + }, + "totals": { + "lemmas": 30520, + "surfaces": 43879 + }, + "bands": [ + { + "band": 0, + "file": "band-0.json.gz", + "lemmas": 801, + "surfaces": 1176, + "bytes": 24941, + "sha256": "4173cde20d7434c3e6a9a5062f2704884920ad59f3c156451408c40a4b5d8134", + "reference": false + }, + { + "band": 1, + "file": "band-1.json.gz", + "lemmas": 1240, + "surfaces": 2335, + "bytes": 54275, + "sha256": "4161561294ecf7214b2f2dd674ebcd5aeb8ae085bbca90aa7cd52ee40afda2ce", + "reference": false + }, + { + "band": 2, + "file": "band-2.json.gz", + "lemmas": 1450, + "surfaces": 2590, + "bytes": 61269, + "sha256": "f3cc9662acbb90d4e1af918445c12ffef50c708ef0e6091a38e7e97b56e5bc10", + "reference": false + }, + { + "band": 3, + "file": "band-3.json.gz", + "lemmas": 1972, + "surfaces": 3373, + "bytes": 81310, + "sha256": "3b981e160ecc97b54006424106bf9b8ca6495f0c2d24a801e1a53496ca0d867d", + "reference": false + }, + { + "band": 4, + "file": "band-4.json.gz", + "lemmas": 2987, + "surfaces": 4775, + "bytes": 119241, + "sha256": "37843887a427579c41eeb02ddd907e5ce066d75f422aee019c8e7a76d3452f66", + "reference": false + }, + { + "band": 5, + "file": "band-5.json.gz", + "lemmas": 6963, + "surfaces": 10347, + "bytes": 270967, + "sha256": "baf78213827168cc9074724b2ab1e339d15843b77401645d90931e99e332b2d5", + "reference": false + }, + { + "band": 6, + "file": "band-6.json.gz", + "lemmas": 15107, + "surfaces": 19283, + "bytes": 551959, + "sha256": "b3bf35f53f8f591609075609612442cc3b1ee898f3ad304416f8e49cc1718e7a", + "reference": true + } + ], + "attribution": [ + "English Wiktionary via kaikki.org / wiktextract — CC BY-SA 3.0 + GFDL", + "hermitdave/FrequencyWords, OpenSubtitles2018 — CC BY-SA 4.0" + ], + "notice": "See NOTICE.md. Share-alike applies to this dictionary data, not to the app code." +} diff --git a/app/public/dict/seed.sqlite3 b/app/public/dict/seed.sqlite3 new file mode 100644 index 0000000..ed1e362 Binary files /dev/null and b/app/public/dict/seed.sqlite3 differ diff --git a/shared/bands.mjs b/shared/bands.mjs new file mode 100644 index 0000000..32e35a7 --- /dev/null +++ b/shared/bands.mjs @@ -0,0 +1,69 @@ +/* The vocabulary bands — the mechanism that makes the gate scale. + + PORT.md's idea: ship only the bands the learner has reached, and let + buildGate()'s vocabQuery hook replace 371 hand-typed words with a query. + One band per phase, widening as he goes. + + Two signals decide which band a word lands in: + + - the dictionary's own curated learner level (초급 / 중급 / 고급), which + is human-graded for exactly this purpose; + - frequency rank, as the tiebreaker inside a level. + + Level leads because a frequency list built from subtitles is a list of + SURFACE forms, and a lemma-keyed dictionary joins onto it badly — see + tools/dict/freq-forms.mjs for how the ranks are recovered at all. When a + source carries no level, bandOf() degrades to frequency alone. + + PHASE 1 ADMITS NO FREQUENCY BAND AT ALL. During the writing-system phase + every word must be phonologically legal for the unit reached — a rank + ceiling would hand the learner a 겹받침 during unit 1.4. Phase 1 gets the + curated words and nothing else; see shared/phonology.mjs. */ + +/** Bands 0-5 are gated vocabulary, one per curriculum phase. */ +export const BANDS = [ + { band: 0, phase: 1, maxFreq: 0, levels: [] }, + { band: 1, phase: 2, maxFreq: 1500, levels: ["초급"] }, + { band: 2, phase: 3, maxFreq: 3000, levels: ["초급"] }, + { band: 3, phase: 4, maxFreq: 5000, levels: ["초급", "중급"] }, + { band: 4, phase: 5, maxFreq: 8000, levels: ["초급", "중급"] }, + { band: 5, phase: 6, maxFreq: 15000, levels: ["초급", "중급", "고급"] }, +]; + +/** + * Everything the dictionary knows that no band admits. Shipped so the word + * rail can gloss an unfamiliar word the learner meets in the wild, but never + * returned by vocabQuery — the tutor cannot reach it. + */ +export const REFERENCE_BAND = 6; + +/** Sources that are always available, whatever the frequency data says. */ +export const ALWAYS_AVAILABLE = new Set(["curated", "grammar", "sentence", "sfx"]); + +/** Curriculum phase (1-6) → band. */ +export const bandForPhase = (phase) => + Math.max(0, Math.min(BANDS.length - 1, Math.round(phase) - 1)); + +/** Unit id ("3.4") → band. */ +export const bandForUnit = (unitId) => bandForPhase(Number.parseInt(String(unitId), 10) || 1); + +/** The frequency ceiling a learner at this band may draw on. */ +export const ceilingForBand = (band) => BANDS[Math.max(0, Math.min(BANDS.length - 1, band))].maxFreq; + +/** + * The lowest band that admits this word, or REFERENCE_BAND if none does. + * `level` may be null when the source does not grade its entries. + */ +export function bandOf({ source, freqRank, level }) { + if (ALWAYS_AVAILABLE.has(source)) return 0; + + for (const b of BANDS) { + if (b.band === 0) continue; // phase 1 takes curated words only + if (freqRank == null || freqRank > b.maxFreq) continue; + // A graded source must also clear the level gate; an ungraded one is + // admitted on frequency alone. + if (level && b.levels.length && !b.levels.includes(level)) continue; + return b.band; + } + return REFERENCE_BAND; +} diff --git a/shared/phonology.mjs b/shared/phonology.mjs new file mode 100644 index 0000000..940c586 --- /dev/null +++ b/shared/phonology.mjs @@ -0,0 +1,109 @@ +/* The phonological ladder of Phase 1, as a reusable filter. + + validate.mjs check 1 verifies that no curriculum word uses a sound + phenomenon its unit has not reached yet — plain vs tense vs aspirated + consonants, basic vs compound vowels, single vs double batchim, and + whether a word creates a liaison or nasalisation context before those + units exist. That check is the reason Phase 1 is correct by construction, + and it reports 0 violations today. + + The same rule has to apply to vocabulary that comes from the DICTIONARY, + not just the hand-listed curriculum: a frequency band would happily hand + the learner 괜찮다 during unit 1.4, when he can read one final consonant + and no compound vowels. So the ladder is lifted here and used to filter + vocabQuery's results while the learner is still in Phase 1. + + validate.mjs itself is untouched — it ships verbatim and keeps its own + copy. This is a second reader of the same rule, not a refactor of it. */ + +const TENSE = "ㄲㄸㅃㅆㅉ"; +const ASPIRATED = "ㅋㅌㅍㅊ"; +const COMPOUND_VOWEL = "ㅐㅔㅒㅖㅘㅙㅚㅝㅞㅟㅢ"; +const DOUBLE_FINAL = "ㄳㄵㄶㄺㄻㄼㄽㄾㄿㅀㅄ"; +const STOPS = "ㄱㄷㅂㅅㅈㅊㅌㅍㅋ"; +const NASALS = "ㄴㅁ"; + +const CHO = "ㄱㄲㄴㄷㄸㄹㅁㅂㅃㅅㅆㅇㅈㅉㅊㅋㅌㅍㅎ"; +const JUNG = "ㅏㅐㅑㅒㅓㅔㅕㅖㅗㅘㅙㅚㅛㅜㅝㅞㅟㅠㅡㅢㅣ"; +const JONG = " ㄱㄲㄳㄴㄵㄶㄷㄹㄺㄻㄼㄽㄾㄿㅀㅁㅂㅄㅅㅆㅇㅈㅊㅋㅌㅍㅎ"; + +/** [initial, medial, final] as jamo, or null if not a syllable block. */ +function jamo(ch) { + const c = ch.codePointAt(0) - 0xac00; + if (c < 0 || c > 11171) return null; + const f = JONG[c % 28]; + return [CHO[Math.floor(c / 588)], JUNG[Math.floor((c % 588) / 28)], f === " " ? "" : f]; +} + +/** The phenomena, in the order Phase 1 teaches them. */ +export const ORDER = [ + "basic", + "compV", + "tense", + "batchim", + "liaison", + "nasal", + "double", + "allsound", +]; + +/** Which Phase 1 unit introduces which phenomenon. */ +export const FEATURE_UNIT = { + 1.1: "basic", + 1.2: "compV", + 1.3: "tense", + 1.4: "batchim", + 1.5: "liaison", + 1.6: "nasal", + 1.7: "double", + 1.8: "allsound", +}; + +/** + * How far up the ladder a learner standing at `unitIndex` has climbed. + * `indexOfUnit` maps a unit id to its position in the flattened list. + * Returns -1 before the first sound unit, ORDER.length-1 once past them. + */ +export function featureLevel(unitIndex, indexOfUnit) { + let level = -1; + for (const [id, feature] of Object.entries(FEATURE_UNIT)) { + const at = indexOfUnit(id); + if (at >= 0 && at <= unitIndex) level = Math.max(level, ORDER.indexOf(feature)); + } + return level; +} + +const hasFeature = (level, feature) => level >= ORDER.indexOf(feature); + +/** + * Why this word is unreadable at this rung of the ladder. Empty means it is + * fine. Mirrors validate.mjs check 1 exactly. + */ +export function phonologyViolations(word, level) { + const out = []; + const blocks = [...String(word)].map(jamo).filter(Boolean); + + for (const [c, v, f] of blocks) { + if (!hasFeature(level, "tense") && (TENSE.includes(c) || ASPIRATED.includes(c))) + out.push(`tense/aspirated ${c}`); + if (!hasFeature(level, "compV") && COMPOUND_VOWEL.includes(v)) out.push(`compound vowel ${v}`); + if (f && !hasFeature(level, "batchim")) out.push(`batchim ${f}`); + if (f && DOUBLE_FINAL.includes(f) && !hasFeature(level, "double")) out.push(`double batchim ${f}`); + } + + for (let i = 0; i < blocks.length - 1; i++) { + const a = blocks[i]; + const b = blocks[i + 1]; + if (a[2] && b[0] === "ㅇ" && !hasFeature(level, "liaison")) out.push("liaison context"); + if (a[2] && STOPS.includes(a[2]) && NASALS.includes(b[0]) && !hasFeature(level, "nasal")) + out.push("nasalisation context"); + } + + return out; +} + +/** True when every sound in the word has already been taught. */ +export const isReadableAt = (word, level) => phonologyViolations(word, level).length === 0; + +/** Past the sound phase the ladder is fully climbed and the filter is a no-op. */ +export const LADDER_COMPLETE = ORDER.length - 1; diff --git a/test/domain/bands.test.ts b/test/domain/bands.test.ts new file mode 100644 index 0000000..93af053 --- /dev/null +++ b/test/domain/bands.test.ts @@ -0,0 +1,129 @@ +/* The vocabulary bands and the phonological ladder — the two things that + decide what the tutor is allowed to reach for. */ + +import { describe, it, expect } from "vitest"; +import { BANDS, REFERENCE_BAND, bandForPhase, bandForUnit, bandOf, ceilingForBand } from "@shared/bands.mjs"; +import { featureLevel, isReadableAt, phonologyViolations, LADDER_COMPLETE } from "@shared/phonology.mjs"; +import { flatten } from "@lib/gate.js"; +import curriculum from "@data/curriculum.json"; +import type { Curriculum } from "@lib/gate.js"; + +const units = flatten(curriculum as unknown as Curriculum); +const indexOf = (id: string) => units.findIndex((u) => u.id === id); + +describe("bands", () => { + it("gives one band per curriculum phase, widening as it goes", () => { + expect(BANDS).toHaveLength(6); + const ceilings = BANDS.map((b) => b.maxFreq); + for (let i = 1; i < ceilings.length; i++) { + expect(ceilings[i]!, `band ${i} must not be narrower than band ${i - 1}`).toBeGreaterThan( + ceilings[i - 1]!, + ); + } + }); + + it("maps phases and unit ids onto bands", () => { + expect(bandForPhase(1)).toBe(0); + expect(bandForPhase(6)).toBe(5); + expect(bandForUnit("1.1")).toBe(0); + expect(bandForUnit("3.4")).toBe(2); + expect(bandForUnit("6.8")).toBe(5); + }); + + it("clamps a phase outside the curriculum rather than throwing", () => { + expect(bandForPhase(0)).toBe(0); + expect(bandForPhase(99)).toBe(5); + }); + + it("admits phase 1 to curated words only — no frequency band at all", () => { + expect(ceilingForBand(0)).toBe(0); + // Even the single most frequent word in Korean is not admitted by rank. + expect(bandOf({ source: "kaikki", freqRank: 1, level: null })).toBeGreaterThan(0); + // But a curated word is always available. + expect(bandOf({ source: "curated", freqRank: null, level: null })).toBe(0); + expect(bandOf({ source: "grammar", freqRank: null, level: null })).toBe(0); + }); + + it("puts a common word in an early band and a rare one late", () => { + const common = bandOf({ source: "kaikki", freqRank: 200, level: null }); + const mid = bandOf({ source: "kaikki", freqRank: 4000, level: null }); + const rare = bandOf({ source: "kaikki", freqRank: 12000, level: null }); + expect(common).toBeLessThan(mid); + expect(mid).toBeLessThan(rare); + }); + + it("lets the curated level override raw frequency", () => { + // Same rank, different graded level — the graded one lands later. + const beginner = bandOf({ source: "krdict", freqRank: 900, level: "초급" }); + const advanced = bandOf({ source: "krdict", freqRank: 900, level: "고급" }); + expect(beginner).toBeLessThan(advanced); + }); + + it("sends unranked and very rare words to the reference band", () => { + expect(bandOf({ source: "kaikki", freqRank: null, level: null })).toBe(REFERENCE_BAND); + expect(bandOf({ source: "kaikki", freqRank: 999_999, level: null })).toBe(REFERENCE_BAND); + }); + + it("keeps the reference band above every gated band, so it can never leak", () => { + for (const b of BANDS) expect(REFERENCE_BAND).toBeGreaterThan(b.band); + }); +}); + +describe("the phonological ladder", () => { + it("agrees with validate.mjs — no curriculum word breaks its own unit", () => { + const violations: string[] = []; + units.forEach((u, i) => { + const level = featureLevel(i, indexOf); + for (const w of u.words ?? []) { + const v = phonologyViolations(w, level); + if (v.length) violations.push(`${u.id} ${w}: ${v.join(", ")}`); + } + }); + expect(violations).toEqual([]); + }); + + it("climbs monotonically through phase 1", () => { + const levels = ["1.1", "1.2", "1.3", "1.4", "1.5", "1.6", "1.7", "1.8"].map((id) => + featureLevel(indexOf(id), indexOf), + ); + for (let i = 1; i < levels.length; i++) { + expect(levels[i]!, `unit 1.${i + 1}`).toBeGreaterThan(levels[i - 1]!); + } + }); + + it("blocks a sound the learner has not reached", () => { + const at11 = featureLevel(indexOf("1.1"), indexOf); + // Unit 1.1 is batchim-free and uses only the ten basic vowels. + expect(isReadableAt("가", at11)).toBe(true); + expect(isReadableAt("밥", at11)).toBe(false); // final consonant + expect(isReadableAt("개", at11)).toBe(false); // compound vowel + expect(isReadableAt("까", at11)).toBe(false); // tense consonant + }); + + it("admits a double batchim only once 1.7 has been reached", () => { + const before = featureLevel(indexOf("1.6"), indexOf); + const after = featureLevel(indexOf("1.7"), indexOf); + expect(isReadableAt("값", before)).toBe(false); + expect(isReadableAt("값", after)).toBe(true); + }); + + it("admits a liaison context only once 1.5 has been reached", () => { + const before = featureLevel(indexOf("1.4"), indexOf); + const after = featureLevel(indexOf("1.5"), indexOf); + expect(phonologyViolations("음악", before)).toContain("liaison context"); + expect(isReadableAt("음악", after)).toBe(true); + }); + + it("is a no-op once the sound phase is over", () => { + const at61 = featureLevel(indexOf("6.1"), indexOf); + expect(at61).toBe(LADDER_COMPLETE); + for (const w of ["괜찮다", "값", "읽었어", "많이"]) { + expect(isReadableAt(w, at61), w).toBe(true); + } + }); + + it("ignores non-Hangul characters rather than choking on them", () => { + expect(phonologyViolations("ABC 123", 0)).toEqual([]); + expect(phonologyViolations("", 0)).toEqual([]); + }); +}); diff --git a/test/domain/freq-forms.test.ts b/test/domain/freq-forms.test.ts new file mode 100644 index 0000000..bc2151a --- /dev/null +++ b/test/domain/freq-forms.test.ts @@ -0,0 +1,116 @@ +/* The inverted frequency join. + + A subtitle frequency list holds surface forms; the dictionary holds + lemmas. Joining them on the headword gives verbs a frequency of roughly + zero, because the -다 citation form barely occurs in running text. These + tests pin the expansion that fixes it. */ + +import { describe, it, expect } from "vitest"; +import { frequencyForms, hasBatchim, rankByFrequency } from "../../tools/dict/freq-forms.mjs"; + +describe("hasBatchim", () => { + it("detects a final consonant", () => { + expect(hasBatchim("밥")).toBe(true); + expect(hasBatchim("학교")).toBe(false); + expect(hasBatchim("값")).toBe(true); + expect(hasBatchim("나")).toBe(false); + }); +}); + +describe("frequencyForms", () => { + it("expands a verb into the forms it actually appears as", () => { + const forms = frequencyForms("먹다", "verb"); + // The tested generator's output must be in there. + for (const f of ["먹어", "먹어요", "먹었어"]) expect(forms.has(f), f).toBe(true); + // Plus the high-yield endings the citation form hides. + for (const f of ["먹고", "먹지", "먹으면", "먹는", "먹습니다"]) expect(forms.has(f), f).toBe(true); + expect(forms.has("먹다")).toBe(true); + }); + + it("picks the right allomorph for an open stem", () => { + const forms = frequencyForms("가다", "verb"); + expect(forms.has("간다")).toBe(true); // 가 + ㄴ다, fused into one block + expect(forms.has("갑니다")).toBe(true); // 가 + ㅂ니다 + expect(forms.has("가면")).toBe(true); // no 으 after a vowel + }); + + it("attaches particles to a noun, allomorph by 받침", () => { + const withFinal = frequencyForms("밥", "noun"); + expect(withFinal.has("밥이")).toBe(true); + expect(withFinal.has("밥을")).toBe(true); + expect(withFinal.has("밥은")).toBe(true); + expect(withFinal.has("밥가")).toBe(false); + + const openStem = frequencyForms("학교", "noun"); + expect(openStem.has("학교가")).toBe(true); + expect(openStem.has("학교를")).toBe(true); + expect(openStem.has("학교는")).toBe(true); + expect(openStem.has("학교이")).toBe(false); + }); + + it("leaves a particle or an ending alone — they take nothing", () => { + expect([...frequencyForms("은", "particle")]).toEqual(["은"]); + expect([...frequencyForms("습니다", "ending")]).toEqual(["습니다"]); + }); +}); + +describe("rankByFrequency", () => { + it("recovers a verb that the naive headword join would score at zero", () => { + const counts = new Map([ + ["하다", 1], // the citation form barely occurs … + ["해", 5000], // … while its real mass sits here + ["했어", 3000], + ["하고", 2000], + ["밥", 400], + ]); + const ranks = rankByFrequency( + [ + { headword: "하다", pos: "verb" }, + { headword: "밥", pos: "noun" }, + ], + counts, + ); + // 하다 sums to ~10,000 against 밥's 400, so it must rank first. + expect(ranks.get("하다 verb")).toBe(1); + expect(ranks.get("밥 noun")).toBe(2); + }); + + it("drops a form two lemmas both claim, rather than double-counting it", () => { + // A real collision: 가다's 해체 form is 가, which is also the noun 가. + // Neither lemma may claim that huge count. + const counts = new Map([ + ["가", 100_000], // claimed by both — must be ignored + ["갔어", 700], // 가다 alone + ["가를", 3], // the noun alone + ]); + const ranks = rankByFrequency( + [ + { headword: "가다", pos: "verb" }, + { headword: "가", pos: "noun" }, + ], + counts, + ); + // Each scores only on its unambiguous forms, so the verb wins on 700 + // rather than either of them inheriting 100,000. + expect(ranks.get("가다 verb")).toBe(1); + expect(ranks.get("가 noun")).toBe(2); + }); + + it("omits a lemma that matches nothing at all", () => { + const ranks = rankByFrequency([{ headword: "없는말", pos: "noun" }], new Map([["밥", 10]])); + expect(ranks.size).toBe(0); + }); + + it("ranks densely from 1, so band ceilings mean what they say", () => { + const counts = new Map([ + ["가", 300], + ["나", 200], + ["다", 100], + ]); + const ranks = rankByFrequency( + ["가", "나", "다"].map((headword) => ({ headword, pos: "adv" })), + counts, + ); + expect([...ranks.values()].sort()).toEqual([1, 2, 3]); + }); +}); diff --git a/tools/dict/assert-roadmap.mjs b/tools/dict/assert-roadmap.mjs new file mode 100644 index 0000000..5854102 --- /dev/null +++ b/tools/dict/assert-roadmap.mjs @@ -0,0 +1,124 @@ +/* Every roadmap word must resolve. Build-time, blocking. + + REVIEW.md §2: 167 of the 371 words the curriculum tells the tutor it may + use had no lexicon entry, so the word rail silently showed nothing. The + dictionary import is what fixes that; this is what stops it regressing. + + Runs against the committed band files, so CI needs no network and no + vendored dictionary. Exits non-zero on any miss. + + Run: npm run dict:assert */ + +import { readFile } from "node:fs/promises"; +import { gunzipSync } from "node:zlib"; +import { fileURLToPath } from "node:url"; + +import { flatten } from "../../lib/gate.js"; + +const ROOT = fileURLToPath(new URL("../../", import.meta.url)); +const DICT = `${ROOT}app/public/dict/`; + +const readJson = async (p) => JSON.parse(await readFile(p, "utf8")); + +async function loadLexicon() { + let manifest; + try { + manifest = await readJson(`${DICT}manifest.json`); + } catch { + console.error("No dictionary found. Run `npm run dict:build` first."); + process.exit(2); + } + + const lemmas = new Map(); // headword -> [pos] + const surfaces = new Map(); // form -> analysis + + for (const band of manifest.bands) { + const raw = gunzipSync(await readFile(DICT + band.file)); + const data = JSON.parse(raw.toString("utf8")); + // Columns are positional to keep the files small; see build.mjs. + const L = data.columns.lemma; + const S = data.columns.surface; + const hw = L.indexOf("headword"); + const pos = L.indexOf("pos"); + const form = S.indexOf("form"); + const analysis = S.indexOf("analysis"); + + for (const row of data.lemmas) { + const w = row[hw]; + if (!lemmas.has(w)) lemmas.set(w, []); + lemmas.get(w).push(row[pos]); + } + for (const row of data.surfaces) { + if (!surfaces.has(row[form])) surfaces.set(row[form], row[analysis]); + } + } + + return { manifest, lemmas, surfaces }; +} + +const resolves = (lex, word) => lex.lemmas.has(word) || lex.surfaces.has(word); + +async function main() { + const lex = await loadLexicon(); + const curriculum = await readJson(`${ROOT}data/curriculum.json`); + const deck = await readJson(`${ROOT}data/deck.json`); + const units = flatten(curriculum); + + const failures = []; + const check = (kind, where, word) => { + if (!resolves(lex, word)) failures.push({ kind, where, word }); + }; + + /* 1. the roadmap itself — the assertion REVIEW.md asked for */ + let roadmapWords = 0; + for (const u of units) { + for (const w of u.words ?? []) { + roadmapWords++; + check("roadmap", u.id, w); + } + } + + /* 2. the spiral targets, which the gate renders as their own instruction */ + let revisits = 0; + for (const u of units) { + for (const r of u.revisits ?? []) { + revisits++; + check("revisit", u.id, r.word); + } + } + + /* 3. the curated deck, which the vocabulary tab renders directly */ + const deckWords = Object.values(deck.topics).flat(); + for (const [w] of deckWords) check("deck", "deck.json", w); + + /* ── report ── */ + const src = lex.manifest.builtWith.dictionary; + console.log(`Dictionary: ${src} — ${lex.manifest.totals.lemmas} lemmas, ` + + `${lex.manifest.totals.surfaces} surface forms`); + console.log(`Resolvable: ${lex.lemmas.size} headwords, ${lex.surfaces.size} forms\n`); + + const byKind = (k) => failures.filter((f) => f.kind === k); + const report = (label, total, kind) => { + const bad = byKind(kind); + console.log(`${label}: ${total - bad.length}/${total} resolve` + (bad.length ? " ✗" : " ✓")); + const grouped = {}; + for (const f of bad) (grouped[f.where] ??= []).push(f.word); + for (const [where, words] of Object.entries(grouped)) { + console.log(` ${where}: ${words.join(" · ")}`); + } + }; + + report("Roadmap words", roadmapWords, "roadmap"); + report("Spiral targets", revisits, "revisit"); + report("Deck words", deckWords.length, "deck"); + + if (failures.length) { + console.log(`\nFAIL — ${failures.length} words cannot be glossed.`); + console.log("Add them to tools/dict/grammar-lexicon.json, or check the dictionary source."); + process.exit(1); + } + + console.log("\nPASS — every roadmap word, spiral target and deck word resolves."); +} + +await main(); diff --git a/tools/dict/build.mjs b/tools/dict/build.mjs new file mode 100644 index 0000000..2f8072c Binary files /dev/null and b/tools/dict/build.mjs differ diff --git a/tools/dict/fetch.mjs b/tools/dict/fetch.mjs new file mode 100644 index 0000000..b005f2b --- /dev/null +++ b/tools/dict/fetch.mjs @@ -0,0 +1,92 @@ +/* Fetch what can be fetched into vendor/. + + Two of the three sources are a plain download. The third is not: + + KRDICT (한국어기초사전) is the preferred dictionary — it is the only source + that carries curated learner glosses in English AND a human-graded + difficulty level (초급/중급/고급), which is what the vocabulary bands are + built on. It is free and needs no login, but the download page is a + JavaScript form behind anti-bot protection, so a script cannot get at it. + Fetch it once by hand and drop the ZIP in vendor/; build.mjs picks it up + automatically and prefers it over kaikki from then on. + + https://krdict.korean.go.kr/download/downloadPopup + → 사전 내려받기 → XML + + Everything here is cached: a source already in vendor/ is left alone. */ + +import { createWriteStream } from "node:fs"; +import { mkdir, stat } from "node:fs/promises"; +import { Readable } from "node:stream"; +import { pipeline } from "node:stream/promises"; +import { createGzip } from "node:zlib"; +import { fileURLToPath } from "node:url"; + +const VENDOR = fileURLToPath(new URL("../../vendor/", import.meta.url)); + +export const SOURCES = { + kaikki: { + file: "kaikki-Korean.jsonl.gz", + // The ENGLISH-edition Wiktionary, filtered to Korean words — 57,252 of + // them, with English glosses. Not to be confused with + // /dictionary/downloads/ko/, which is the Korean-EDITION Wiktionary: + // words of every language, glossed in Korean. Wrong direction entirely. + url: "https://kaikki.org/dictionary/Korean/kaikki.org-dictionary-Korean.jsonl", + // Served uncompressed at ~200 MB; gzipped on the way in. + gzip: true, + note: "English Wiktionary via kaikki.org / wiktextract — CC BY-SA 3.0 + GFDL", + }, + frequency: { + file: "ko_full.txt", + // The full list, not ko_50k. Korean inflection scatters a lemma's mass + // across dozens of surface forms, so the long tail is where most of a + // lemma's count actually lives — with the 50k list only a quarter of the + // dictionary got any rank at all, and the upper bands came out empty. + url: "https://raw.githubusercontent.com/hermitdave/FrequencyWords/master/content/2018/ko/ko_full.txt", + note: "hermitdave/FrequencyWords, OpenSubtitles2018 — CC BY-SA 4.0", + }, +}; + +const MB = (n) => `${(n / 1024 / 1024).toFixed(1)} MB`; + +async function sizeOf(path) { + try { + return (await stat(path)).size; + } catch { + return null; + } +} + +export async function fetchSource(key) { + const src = SOURCES[key]; + const dest = VENDOR + src.file; + + const have = await sizeOf(dest); + if (have) { + console.log(` ${src.file} — cached (${MB(have)})`); + return dest; + } + + console.log(` ${src.file} — downloading…`); + const res = await fetch(src.url); + if (!res.ok) throw new Error(`${src.url} → HTTP ${res.status}`); + const stages = [Readable.fromWeb(res.body)]; + if (src.gzip) stages.push(createGzip()); + stages.push(createWriteStream(dest)); + await pipeline(...stages); + console.log(` ${src.file} — ${MB(await sizeOf(dest))} (${src.note})`); + return dest; +} + +export async function ensureVendor() { + await mkdir(VENDOR, { recursive: true }); + return VENDOR; +} + +if (import.meta.url === `file://${process.argv[1]}`) { + await ensureVendor(); + console.log("Fetching dictionary sources into vendor/ …"); + for (const key of Object.keys(SOURCES)) await fetchSource(key); + console.log("\nDone. KRDICT, if you want it, is a manual download — see the"); + console.log("comment at the top of tools/dict/fetch.mjs."); +} diff --git a/tools/dict/freq-forms.mjs b/tools/dict/freq-forms.mjs new file mode 100644 index 0000000..192b08d --- /dev/null +++ b/tools/dict/freq-forms.mjs @@ -0,0 +1,195 @@ +/* Recovering lemma frequency from a surface-form frequency list. + + THE PROBLEM + + The frequency list is Korean as it is actually written; a dictionary is + keyed on lemmas. Korean is agglutinative with obligatory particles, and + the verb citation form (-다) essentially never occurs in running text. So + joining `frequency.word = lemma.headword` would: + + - give ZERO frequency to every verb and adjective, the most important + class in a learner dictionary, because the citation form barely + appears while its mass is spread across a dozen inflected forms; + - understate every noun, whose count is split across the bare form and + each of its particle-attached forms. + + THE FIX + + Invert the join. Expand each lemma into the surface forms it plausibly + takes, then SUM the counts over them. No runtime analyser and no + build-image dependency — just the rules, applied at build time. + + Two honest limitations, both of which cost recall and not correctness: + + - the generated endings are regular, so an irregular stem produces some + forms that do not exist. A form that is not real simply matches + nothing in the frequency list. + - a form claimed by more than one lemma is DROPPED rather than split, + so homographs do not inherit each other's mass. Dropping is the + conservative choice. + + These forms are for FREQUENCY SCORING ONLY. The `surface` table the app + actually queries is populated strictly from lib/conjugation.js + surfaceForms() plus the headword — see build.mjs. */ + +import { surfaceForms, haeche, past, polite } from "../../lib/conjugation.js"; + +/** True when the word ends in a consonant. Pure arithmetic on the syllable. */ +export function hasBatchim(word) { + const cp = word.codePointAt(word.length - 1); + if (cp === undefined) return false; + const i = cp - 0xac00; + return i >= 0 && i <= 11171 && i % 28 !== 0; +} + +/** Particles that attach to a noun; allomorph chosen by the final consonant. */ +const NOUN_PARTICLES = [ + ["이", "가"], // subject + ["을", "를"], // object + ["은", "는"], // topic + ["과", "와"], // and / with + ["이나", "나"], // or + ["이랑", "랑"], // and, casual + ["으로", "로"], // direction / means + ["아", "야"], // vocative +]; + +/** Particles with a single form, whatever the stem ends in. */ +const INVARIANT_PARTICLES = [ + "에", // 에 + "에서", // 에서 + "도", // 도 + "의", // 의 + "만", // 만 + "까지", // 까지 + "부터", // 부터 + "처럼", // 처럼 + "보다", // 보다 + "한테", // 한테 + "에게", // 에게 + "께", // 께 + "마다", // 마다 +]; + +/** Endings attached straight to a stem. Regular forms only. */ +const STEM_ENDINGS = [ + "고", // 고 + "지", // 지 + "다가", // 다가 + "면서", // 면서 + "네", // 네 + "자", // 자 + "는", // 는, adnominal present +]; + +/** Endings whose shape depends on whether the stem has a final consonant. */ +const STEM_ENDINGS_BATCHIM = [ + ["은", "ㄴ"], // 은 / ㄴ adnominal past, adjectival + ["을", "ㄹ"], // 을 / ㄹ adnominal prospective + ["으면", "면"], // 으면 / 면 + ["으니까", "니까"], // 으니까 / 니까 + ["습니다", "ㅂ니다"], // 습니다 / ㅂ니다 + ["는다", "ㄴ다"], // 는다 / ㄴ다 +]; + +const JONG = " ㄱㄲㄳㄴㄵㄶㄷㄹㄺㄻㄼㄽㄾㄿㅀㅁㅂㅄㅅㅆㅇㅈㅊㅋㅌㅍㅎ"; + +/** Glue a bare-jamo ending onto an open syllable, so the two become one block. */ +function fuseFinal(stem, ending) { + const idx = JONG.indexOf(ending[0]); + if (idx <= 0) return stem + ending; + const cp = stem.codePointAt(stem.length - 1); + const i = cp - 0xac00; + if (i < 0 || i > 11171 || i % 28 !== 0) return stem + ending; // needs an open syllable + return stem.slice(0, -1) + String.fromCodePoint(0xac00 + i + idx) + ending.slice(1); +} + +const DICT_SUFFIX = "다"; // 다 + +/** + * Every surface form this lemma plausibly takes, for summing frequency. + * Deliberately over-generates: a form that does not exist matches nothing. + */ +export function frequencyForms(headword, pos) { + const forms = new Set([headword]); + + if (pos === "verb" || pos === "adj") { + // The real, tested generator first. + for (const s of surfaceForms(headword, "")) forms.add(s.form); + const present = haeche(headword); + if (present) { + forms.add(present); + forms.add(polite(present)); + const p = past(present); + if (p) { + forms.add(p); + forms.add(polite(p)); + } + } + + if (headword.endsWith(DICT_SUFFIX)) { + const stem = headword.slice(0, -1); + if (stem) { + for (const e of STEM_ENDINGS) forms.add(stem + e); + const batchim = hasBatchim(stem); + for (const [withB, withoutB] of STEM_ENDINGS_BATCHIM) { + forms.add(batchim ? stem + withB : fuseFinal(stem, withoutB)); + } + } + } + return forms; + } + + // Nouns, pronouns, numerals, counters and determiners take particles. + if (["noun", "pron", "num", "counter", "det"].includes(pos)) { + const batchim = hasBatchim(headword); + for (const [withB, withoutB] of NOUN_PARTICLES) { + forms.add(headword + (batchim ? withB : withoutB)); + } + for (const p of INVARIANT_PARTICLES) forms.add(headword + p); + } + + return forms; +} + +/** + * Rank every headword by summed frequency. + * + * @param entries [{ headword, pos }] + * @param counts Map from the frequency list + * @returns Map<"headword pos", rank> — rank 1 is the most frequent + */ +export function rankByFrequency(entries, counts) { + const claims = new Map(); // form -> the lemmas that generated it + const formsOf = new Map(); + + for (const e of entries) { + const key = `${e.headword} ${e.pos}`; + if (formsOf.has(key)) continue; + const forms = frequencyForms(e.headword, e.pos); + formsOf.set(key, forms); + for (const f of forms) { + let owners = claims.get(f); + if (!owners) claims.set(f, (owners = new Set())); + owners.add(key); + } + } + + const totals = []; + for (const [key, forms] of formsOf) { + let total = 0; + for (const f of forms) { + const n = counts.get(f); + if (!n) continue; + // Ambiguous forms are dropped, not split — see the header. + if (claims.get(f).size > 1) continue; + total += n; + } + if (total > 0) totals.push([key, total]); + } + + totals.sort((a, b) => b[1] - a[1]); + const ranks = new Map(); + totals.forEach(([key], i) => ranks.set(key, i + 1)); + return ranks; +} diff --git a/tools/dict/grammar-lexicon.json b/tools/dict/grammar-lexicon.json new file mode 100644 index 0000000..b7e6a9c --- /dev/null +++ b/tools/dict/grammar-lexicon.json @@ -0,0 +1,87 @@ +{ + "note": "Grammatical detail for function words, layered ON TOP of data/gloss-extra.json. gloss-extra supplies a plain English gloss for all 167 roadmap words the deck does not cover; this file adds what a gloss cannot carry — a part of speech (particle / ending / contraction / bound noun), a Korean gloss, and the allomorph rule (\uc740 after a consonant, \ub294 after a vowel). Merged LAST, so these win for the (headword, pos) pairs they define. Entries always land in band 0: function words are available from the start because the curriculum introduces them explicitly.", + "license": "Written for Hankan. No third-party dictionary content.", + "entries": [ + { "headword": "은", "pos": "particle", "gloss_en": "topic marker", "gloss_ko": "주제", "note": "after a consonant; 는 after a vowel" }, + { "headword": "는", "pos": "particle", "gloss_en": "topic marker", "gloss_ko": "주제", "note": "after a vowel; 은 after a consonant" }, + { "headword": "이", "pos": "particle", "gloss_en": "subject marker", "gloss_ko": "주격", "note": "after a consonant; 가 after a vowel" }, + { "headword": "가", "pos": "particle", "gloss_en": "subject marker", "gloss_ko": "주격", "note": "after a vowel; 이 after a consonant" }, + { "headword": "을", "pos": "particle", "gloss_en": "object marker", "gloss_ko": "목적격", "note": "after a consonant; 를 after a vowel" }, + { "headword": "를", "pos": "particle", "gloss_en": "object marker", "gloss_ko": "목적격", "note": "after a vowel; 을 after a consonant" }, + { "headword": "에", "pos": "particle", "gloss_en": "at, to, in — a place or a time", "gloss_ko": "장소·시간", "note": "where something IS or is going" }, + { "headword": "에서", "pos": "particle", "gloss_en": "at, from — where an action happens", "gloss_ko": "행동의 장소", "note": "contrast with 에: 에서 is where you DO something" }, + { "headword": "도", "pos": "particle", "gloss_en": "also, too, even", "gloss_ko": "역시", "note": "replaces 은/는 and 이/가 rather than stacking" }, + { "headword": "만", "pos": "particle", "gloss_en": "only, just", "gloss_ko": "오직", "note": "" }, + { "headword": "의", "pos": "particle", "gloss_en": "of, 's — possession", "gloss_ko": "소유", "note": "often dropped in speech" }, + { "headword": "로", "pos": "particle", "gloss_en": "by, with, toward", "gloss_ko": "방향·수단", "note": "after a vowel or ㄹ; 으로 after another consonant" }, + { "headword": "으로", "pos": "particle", "gloss_en": "by, with, toward", "gloss_ko": "방향·수단", "note": "after a consonant; 로 after a vowel or ㄹ" }, + { "headword": "와", "pos": "particle", "gloss_en": "and, with", "gloss_ko": "그리고", "note": "after a vowel; 과 after a consonant. Written register" }, + { "headword": "과", "pos": "particle", "gloss_en": "and, with", "gloss_ko": "그리고", "note": "after a consonant; 와 after a vowel. Written register" }, + { "headword": "하고", "pos": "particle", "gloss_en": "and, with", "gloss_ko": "그리고", "note": "spoken; same job as 와/과" }, + { "headword": "랑", "pos": "particle", "gloss_en": "and, with", "gloss_ko": "그리고", "note": "the most casual of 와/과 · 하고 · 랑; 이랑 after a consonant" }, + { "headword": "이랑", "pos": "particle", "gloss_en": "and, with", "gloss_ko": "그리고", "note": "after a consonant; 랑 after a vowel" }, + { "headword": "부터", "pos": "particle", "gloss_en": "from — a starting point", "gloss_ko": "시작", "note": "pairs with 까지" }, + { "headword": "까지", "pos": "particle", "gloss_en": "until, as far as, up to", "gloss_ko": "끝", "note": "pairs with 부터" }, + { "headword": "처럼", "pos": "particle", "gloss_en": "like, as", "gloss_ko": "같이", "note": "" }, + { "headword": "한테", "pos": "particle", "gloss_en": "to, for — a person", "gloss_ko": "사람에게", "note": "spoken; 에게 in writing" }, + { "headword": "에게", "pos": "particle", "gloss_en": "to, for — a person", "gloss_ko": "사람에게", "note": "written; 한테 in speech" }, + { "headword": "께", "pos": "particle", "gloss_en": "to — honorific", "gloss_ko": "높임", "note": "the honorific form of 한테 / 에게" }, + { "headword": "보다", "pos": "particle", "gloss_en": "than", "gloss_ko": "비교", "note": "comparison; distinct from the verb 보다 'to see'" }, + { "headword": "마다", "pos": "particle", "gloss_en": "every, each", "gloss_ko": "각각", "note": "" }, + { "headword": "밖에", "pos": "particle", "gloss_en": "nothing but, only", "gloss_ko": "오직", "note": "always followed by a negative" }, + + { "headword": "습니다", "pos": "ending", "gloss_en": "formal declarative ending", "gloss_ko": "합쇼체", "note": "after a consonant stem; ㅂ니다 after a vowel" }, + { "headword": "ㅂ니다", "pos": "ending", "gloss_en": "formal declarative ending", "gloss_ko": "합쇼체", "note": "after a vowel stem; 습니다 after a consonant" }, + { "headword": "입니다", "pos": "ending", "gloss_en": "is, am, are — formal", "gloss_ko": "이다의 합쇼체", "note": "the formal form of 이다" }, + { "headword": "그렇습니다", "pos": "phrase", "gloss_en": "that is so, yes — formal", "gloss_ko": "그렇다의 합쇼체", "note": "formal 그래" }, + { "headword": "고", "pos": "ending", "gloss_en": "and — links two clauses", "gloss_ko": "연결", "note": "" }, + { "headword": "지", "pos": "ending", "gloss_en": "isn't it, right? — also the base for 지 않다", "gloss_ko": "확인", "note": "" }, + { "headword": "면", "pos": "ending", "gloss_en": "if, when", "gloss_ko": "조건", "note": "after a vowel; 으면 after a consonant" }, + { "headword": "으면", "pos": "ending", "gloss_en": "if, when", "gloss_ko": "조건", "note": "after a consonant; 면 after a vowel" }, + { "headword": "니까", "pos": "ending", "gloss_en": "because, since", "gloss_ko": "이유", "note": "after a vowel; 으니까 after a consonant" }, + { "headword": "어서", "pos": "ending", "gloss_en": "and so, because", "gloss_ko": "이유·순서", "note": "아서 after a bright vowel" }, + { "headword": "는데", "pos": "ending", "gloss_en": "but, and — sets up background", "gloss_ko": "배경", "note": "very common in dialogue" }, + { "headword": "지만", "pos": "ending", "gloss_en": "but, although", "gloss_ko": "대조", "note": "" }, + { "headword": "라고", "pos": "ending", "gloss_en": "quoting — \"that …\"", "gloss_ko": "인용", "note": "marks reported speech; 이라고 after a consonant" }, + { "headword": "대", "pos": "ending", "gloss_en": "they say that …", "gloss_ko": "인용 축약", "note": "contracted from 다고 해" }, + { "headword": "래", "pos": "ending", "gloss_en": "he says, she says — reported", "gloss_ko": "인용 축약", "note": "contracted from 라고 해" }, + { "headword": "냬", "pos": "ending", "gloss_en": "asks whether — a reported question", "gloss_ko": "인용 축약", "note": "contracted from 냐고 해" }, + { "headword": "재", "pos": "ending", "gloss_en": "suggests that — a reported proposal", "gloss_ko": "인용 축약", "note": "contracted from 자고 해" }, + + { "headword": "한다", "pos": "form", "gloss_en": "does — plain written style", "gloss_ko": "하다의 해라체", "note": "from 하다; the register of narration and manhwa captions" }, + { "headword": "했다", "pos": "form", "gloss_en": "did — plain written style", "gloss_ko": "하다의 과거 해라체", "note": "from 하다" }, + { "headword": "였다", "pos": "form", "gloss_en": "was — plain written style", "gloss_ko": "이다의 과거 해라체", "note": "from 이다; 이었다 after a consonant" }, + + { "headword": "것", "pos": "noun", "gloss_en": "thing, one — a bound noun", "gloss_ko": "사물", "note": "needs a modifier in front; 거 in speech" }, + { "headword": "거", "pos": "noun", "gloss_en": "thing, one", "gloss_ko": "것의 준말", "note": "the spoken form of 것" }, + { "headword": "수", "pos": "noun", "gloss_en": "way, possibility — as in 할 수 있다 \"can\"", "gloss_ko": "가능성", "note": "bound noun; almost always with 있다 / 없다" }, + { "headword": "때", "pos": "noun", "gloss_en": "time, when", "gloss_ko": "시간", "note": "bound noun after a modifier" }, + { "headword": "곳", "pos": "noun", "gloss_en": "place", "gloss_ko": "장소", "note": "bound noun after a modifier" }, + { "headword": "적", "pos": "noun", "gloss_en": "occasion, the experience of — as in 한 적 있다", "gloss_ko": "경험", "note": "bound noun" }, + { "headword": "뿐", "pos": "noun", "gloss_en": "only, nothing but", "gloss_ko": "오직", "note": "bound noun" }, + { "headword": "줄", "pos": "noun", "gloss_en": "how to, the fact that — as in 할 줄 알다", "gloss_ko": "방법", "note": "bound noun" }, + + { "headword": "난", "pos": "contraction", "gloss_en": "I — 나 + 는", "gloss_ko": "나는", "note": "" }, + { "headword": "넌", "pos": "contraction", "gloss_en": "you — 너 + 는", "gloss_ko": "너는", "note": "" }, + { "headword": "건", "pos": "contraction", "gloss_en": "the thing — 것 + 은", "gloss_ko": "것은", "note": "" }, + { "headword": "이건", "pos": "contraction", "gloss_en": "this thing — 이것 + 은", "gloss_ko": "이것은", "note": "" }, + { "headword": "그건", "pos": "contraction", "gloss_en": "that thing — 그것 + 은", "gloss_ko": "그것은", "note": "" }, + { "headword": "뭘", "pos": "contraction", "gloss_en": "what — 무엇 + 을", "gloss_ko": "무엇을", "note": "" }, + { "headword": "절", "pos": "contraction", "gloss_en": "me — 저 + 를, humble", "gloss_ko": "저를", "note": "" }, + { "headword": "날", "pos": "contraction", "gloss_en": "me — 나 + 를", "gloss_ko": "나를", "note": "" }, + { "headword": "제가", "pos": "contraction", "gloss_en": "I — 저 + 가, humble", "gloss_ko": "저가", "note": "the humble counterpart of 내가" }, + { "headword": "내가", "pos": "contraction", "gloss_en": "I — 나 + 가", "gloss_ko": "나가", "note": "" }, + + { "headword": "드리다", "pos": "verb", "gloss_en": "to give — humble", "gloss_ko": "주다의 겸양", "note": "used when the speaker gives to someone higher" }, + { "headword": "계시다", "pos": "verb", "gloss_en": "to be, to stay — honorific", "gloss_ko": "있다의 높임", "note": "the honorific of 있다, for people" }, + { "headword": "주무시다", "pos": "verb", "gloss_en": "to sleep — honorific", "gloss_ko": "자다의 높임", "note": "the honorific of 자다" }, + { "headword": "앉히다", "pos": "verb", "gloss_en": "to seat someone, to sit someone down", "gloss_ko": "앉게 하다", "note": "causative of 앉다" }, + + { "headword": "나중에", "pos": "adv", "gloss_en": "later, afterwards", "gloss_ko": "이따가", "note": "" }, + { "headword": "신경 쓰다", "pos": "phrase", "gloss_en": "to care about, to be bothered by", "gloss_ko": "마음을 쓰다", "note": "신경 'nerve' + 쓰다 'to use'" }, + { "headword": "몇", "pos": "det", "gloss_en": "how many, a few", "gloss_ko": "얼마나", "note": "takes a counter after it" }, + { "headword": "명", "pos": "counter", "gloss_en": "counter for people", "gloss_ko": "사람 세는 말", "note": "" }, + { "headword": "몇 명", "pos": "phrase", "gloss_en": "how many people", "gloss_ko": "사람이 얼마나", "note": "몇 + the counter 명" }, + { "headword": "신라", "pos": "noun", "gloss_en": "Silla — the ancient Korean kingdom", "gloss_ko": "옛 나라 이름", "note": "read [실라]; a standing example of ㄴ+ㄹ becoming ㄹㄹ" } + ] +} diff --git a/tools/dict/sources/frequency.mjs b/tools/dict/sources/frequency.mjs new file mode 100644 index 0000000..29c3211 --- /dev/null +++ b/tools/dict/sources/frequency.mjs @@ -0,0 +1,33 @@ +/* hermitdave/FrequencyWords — Korean, OpenSubtitles2018. + + `word count` per line, descending. 50k lines, 695 KB, plain fetch. + Licence: CC BY-SA 4.0 for the list content. See NOTICE.md, and note the + deliberate decision recorded there to keep this data in its own column + rather than merging it into the dictionary content. */ + +import { readFile } from "node:fs/promises"; + +/** + * Surface form -> occurrence count. These are SURFACE forms from subtitles, + * not lemmas — see freq-forms.mjs for why that matters and how the ranks + * are recovered. + */ +export async function readFrequency(path) { + const text = await readFile(path, "utf8"); + const counts = new Map(); + + for (const line of text.split("\n")) { + const sp = line.indexOf(" "); + if (sp < 1) continue; + const word = line.slice(0, sp); + const n = Number.parseInt(line.slice(sp + 1), 10); + if (!Number.isFinite(n)) continue; + // Later duplicates would only ever be smaller; keep the first. + if (!counts.has(word)) counts.set(word, n); + } + + return counts; +} + +export const FREQUENCY_ATTRIBUTION = + "hermitdave/FrequencyWords, OpenSubtitles2018 — CC BY-SA 4.0"; diff --git a/tools/dict/sources/kaikki.mjs b/tools/dict/sources/kaikki.mjs new file mode 100644 index 0000000..9e83cf1 --- /dev/null +++ b/tools/dict/sources/kaikki.mjs @@ -0,0 +1,125 @@ +/* kaikki.org adapter — English Wiktionary, filtered to Korean. + + ~64k records / ~34k distinct Hangul headwords, with English glosses written + by and for English speakers. Automatable: a stable URL, no signup. + + What it does NOT have is a curated learner level, so bands fall back to + frequency alone for entries from this source (see shared/bands.mjs). That + is the main reason KRDICT is preferred when it has been vendored. + + Licence: CC BY-SA 3.0 + GFDL, inherited from Wiktionary. See NOTICE.md. */ + +import { createReadStream } from "node:fs"; +import { createGunzip } from "node:zlib"; +import { createInterface } from "node:readline"; + +/** kaikki part-of-speech → ours. Anything absent is skipped. */ +const POS = { + noun: "noun", + verb: "verb", + adj: "adj", + adv: "adv", + pron: "pron", + det: "det", + num: "num", + conj: "conj", + intj: "interj", + particle: "particle", + postp: "particle", + counter: "counter", + suffix: "suffix", + prefix: "prefix", + phrase: "phrase", + proverb: "phrase", + contraction: "contraction", +}; + +/* Hanja, syllable stubs, proper nouns and romanisations are not words the + learner is reading manhwa to decode. */ +const SKIP_POS = new Set([ + "character", + "syllable", + "name", + "root", + "symbol", + "romanization", + "punct", + "affix", + "interfix", +]); + +/** Senses worth ignoring when a better one exists. */ +const WEAK_TAGS = new Set([ + "obsolete", + "archaic", + "rare", + "dialectal", + "North-Korea", + "dated", + "historical", +]); + +const HANGUL_ONLY = /^[가-힣]+(?: [가-힣]+)*$/; +const isEnglish = (s) => s && !/[가-힣]/.test(s); + +/** Pick the most useful English gloss, preferring a plain modern sense. */ +function bestGloss(senses) { + const usable = []; + for (const s of senses ?? []) { + const glosses = (s.glosses ?? []).filter(isEnglish); + if (!glosses.length) continue; + // "form-of" senses describe an inflected form; we generate those + // ourselves from surfaceForms(), so they add nothing here. + const tags = new Set(s.tags ?? []); + if (tags.has("form-of") || s.form_of) continue; + usable.push({ text: glosses.join("; "), weak: [...tags].some((t) => WEAK_TAGS.has(t)) }); + } + if (!usable.length) return ""; + const strong = usable.filter((u) => !u.weak); + const pick = (strong.length ? strong : usable).slice(0, 2); + return pick.map((p) => p.text).join("; ").slice(0, 240); +} + +/** + * Yields { headword, pos, gloss_en, gloss_ko, level, source } for every + * usable Korean entry. Streams — the file does not fit comfortably in memory. + */ +export async function* readKaikki(path) { + const rl = createInterface({ + input: createReadStream(path).pipe(createGunzip()), + crlfDelay: Infinity, + }); + + for await (const line of rl) { + if (!line) continue; + let o; + try { + o = JSON.parse(line); + } catch { + continue; // a truncated tail line is not worth failing the build over + } + + if (o.lang_code !== "ko") continue; + if (SKIP_POS.has(o.pos)) continue; + const pos = POS[o.pos]; + if (!pos) continue; + + const headword = o.word; + if (!headword || !HANGUL_ONLY.test(headword)) continue; + + const gloss = bestGloss(o.senses); + if (!gloss) continue; + + yield { + headword, + pos, + gloss_en: gloss, + gloss_ko: "", + level: null, // kaikki does not grade entries + source: "kaikki", + }; + } +} + +export const KAIKKI_ATTRIBUTION = + "English Wiktionary via kaikki.org / wiktextract — CC BY-SA 3.0 + GFDL"; diff --git a/tools/dict/sources/krdict.mjs b/tools/dict/sources/krdict.mjs new file mode 100644 index 0000000..845e632 --- /dev/null +++ b/tools/dict/sources/krdict.mjs @@ -0,0 +1,204 @@ +/* KRDICT (한국어기초사전) adapter — the preferred dictionary source. + + Why it is preferred over kaikki: + + - curated learner glosses in English, written for exactly this audience; + - a human-graded difficulty level per entry (초급 / 중급 / 고급), which is + what shared/bands.mjs would rather build the vocabulary bands on than + a subtitle frequency count; + - Korean definitions as well as English ones, so the word rail can show + both. + + ACQUISITION IS MANUAL, ON PURPOSE. The dataset is free and needs no login, + but the download page is a JavaScript form behind anti-bot protection, so + no build script can fetch it. Download once by hand and drop the result in + vendor/ — build.mjs then prefers it automatically and records the choice + in the manifest: + + https://krdict.korean.go.kr/download/downloadPopup → 사전 내려받기 → XML + + Either the .zip or its unpacked .xml files will do; a .zip is unpacked + with the system `unzip`, which is the only external tool this needs. + + Licence: CC BY-SA 2.0 KR. Attribution is required and share-alike applies + to the derived dictionary data — see NOTICE.md. The XML references audio + under dicmedia.korean.go.kr; that media is EXCLUDED from the open licence, + so the URLs are ignored here and nothing is mirrored. + + NOTE: this adapter is written against the documented LMF structure but has + not been run — no KRDICT file has been vendored yet. The kaikki path is + the one currently exercised by the build and by CI. */ + +import { createReadStream } from "node:fs"; +import { readdir, stat } from "node:fs/promises"; +import { spawn } from "node:child_process"; +import { createInterface } from "node:readline"; +import path from "node:path"; + +/** KRDICT 품사 → ours. Anything absent is skipped. */ +const POS = { + 명사: "noun", + 동사: "verb", + 형용사: "adj", + 부사: "adv", + 대명사: "pron", + 관형사: "det", + 수사: "num", + 감탄사: "interj", + 조사: "particle", + 접사: "suffix", + 의존명사: "counter", + 어미: "ending", +}; + +const LEVELS = new Set(["초급", "중급", "고급"]); +const ENGLISH = "영어"; + +const tag = (xml, name) => { + const m = xml.match(new RegExp(`<${name}[^>]*>([\\s\\S]*?)`)); + return m ? decode(m[1].trim()) : ""; +}; + +const attr = (xml, name, key) => { + const m = xml.match(new RegExp(`<${name}\\b[^>]*\\b${key}="([^"]*)"`)); + return m ? decode(m[1]) : ""; +}; + +function decode(s) { + return s + .replace(//g, "$1") + .replace(/</g, "<") + .replace(/>/g, ">") + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/&/g, "&") + .replace(/<[^>]+>/g, "") + .trim(); +} + +/** + * Read a style value, which is how LMF carries + * most of its fields. Falls back to an element of the same name. + */ +function feat(xml, att) { + const m = xml.match(new RegExp(`]*\\batt="${att}"[^>]*\\bval="([^"]*)"`)); + if (m) return decode(m[1]); + const alt = xml.match(new RegExp(`]*\\bval="([^"]*)"[^>]*\\batt="${att}"`)); + return alt ? decode(alt[1]) : tag(xml, att); +} + +/** The English Equivalent block, if the entry has one. */ +function englishEquivalent(senseXml) { + const blocks = senseXml.match(//g) ?? []; + for (const b of blocks) { + const lang = feat(b, "language") || attr(b, "Equivalent", "language"); + if (lang && lang !== ENGLISH) continue; + const lemma = feat(b, "lemma"); + const definition = feat(b, "definition"); + if (lemma || definition) return { lemma, definition }; + } + return null; +} + +function parseEntry(xml) { + const headword = feat(xml, "writtenForm") || tag(xml, "writtenForm"); + if (!headword || !/^[가-힣]+(?: [가-힣]+)*$/.test(headword)) return null; + + const posKo = feat(xml, "partOfSpeech"); + const pos = POS[posKo]; + if (!pos) return null; + + const levelRaw = feat(xml, "vocabularyLevel"); + const level = LEVELS.has(levelRaw) ? levelRaw : null; + + const senses = xml.match(//g) ?? []; + let glossEn = ""; + let glossKo = ""; + + for (const s of senses) { + if (!glossKo) glossKo = feat(s, "definition"); + const eq = englishEquivalent(s); + if (eq) { + const text = [eq.lemma, eq.definition].filter(Boolean).join("; "); + glossEn = glossEn ? `${glossEn}; ${text}` : text; + } + if (glossEn.length > 200) break; + } + + if (!glossEn && !glossKo) return null; + + return { + headword, + pos, + gloss_en: glossEn.slice(0, 240), + gloss_ko: glossKo.slice(0, 240), + level, + source: "krdict", + }; +} + +/** Every .xml under a directory, or the file itself. */ +async function xmlFiles(target) { + const s = await stat(target); + if (s.isFile()) return [target]; + const names = await readdir(target); + return names + .filter((n) => n.toLowerCase().endsWith(".xml")) + .sort() + .map((n) => path.join(target, n)); +} + +/** Line stream for either a plain .xml or a member of a .zip. */ +function lineStream(file) { + if (file.toLowerCase().endsWith(".zip")) { + const proc = spawn("unzip", ["-p", file], { stdio: ["ignore", "pipe", "inherit"] }); + return createInterface({ input: proc.stdout, crlfDelay: Infinity }); + } + return createInterface({ input: createReadStream(file), crlfDelay: Infinity }); +} + +/** + * Yields { headword, pos, gloss_en, gloss_ko, level, source } per entry. + * Accumulates one at a time so a multi-hundred-MB file never + * lands in memory. + */ +export async function* readKrdict(target) { + const files = target.toLowerCase().endsWith(".zip") ? [target] : await xmlFiles(target); + + for (const file of files) { + let buffer = ""; + let inside = false; + + for await (const line of lineStream(file)) { + if (!inside && line.includes("")) { + const entry = parseEntry(buffer); + buffer = ""; + inside = false; + if (entry) yield entry; + } + } + } +} + +/** The .zip or directory to read, if the user has vendored one. */ +export async function findKrdict(vendorDir) { + let names; + try { + names = await readdir(vendorDir); + } catch { + return null; + } + const zip = names.find((n) => /krdict|기초사전/i.test(n) && n.toLowerCase().endsWith(".zip")); + if (zip) return path.join(vendorDir, zip); + const dir = names.find((n) => /krdict/i.test(n) && !n.includes(".")); + if (dir) return path.join(vendorDir, dir); + const xml = names.find((n) => /krdict/i.test(n) && n.toLowerCase().endsWith(".xml")); + return xml ? path.join(vendorDir, xml) : null; +} + +export const KRDICT_ATTRIBUTION = + "한국어기초사전, 국립국어원 (National Institute of Korean Language) — CC BY-SA 2.0 KR"; diff --git a/types/shared/bands.mjs.d.ts b/types/shared/bands.mjs.d.ts new file mode 100644 index 0000000..c2436b4 --- /dev/null +++ b/types/shared/bands.mjs.d.ts @@ -0,0 +1,23 @@ +/* Declarations for shared/bands.mjs — imported by both the app (TypeScript) + and the build pipeline (plain Node ESM), so the band table exists once. */ + +export interface BandSpec { + band: number; + phase: number; + maxFreq: number; + levels: string[]; +} + +export const BANDS: BandSpec[]; +export const REFERENCE_BAND: number; +export const ALWAYS_AVAILABLE: Set; + +export function bandForPhase(phase: number): number; +export function bandForUnit(unitId: string): number; +export function ceilingForBand(band: number): number; + +export function bandOf(word: { + source: string; + freqRank: number | null; + level?: string | null; +}): number; diff --git a/types/shared/phonology.mjs.d.ts b/types/shared/phonology.mjs.d.ts new file mode 100644 index 0000000..dfd5899 --- /dev/null +++ b/types/shared/phonology.mjs.d.ts @@ -0,0 +1,9 @@ +/* Declarations for shared/phonology.mjs. */ + +export const ORDER: string[]; +export const FEATURE_UNIT: Record; +export const LADDER_COMPLETE: number; + +export function featureLevel(unitIndex: number, indexOfUnit: (id: string) => number): number; +export function phonologyViolations(word: string, level: number): string[]; +export function isReadableAt(word: string, level: number): boolean;