fabi/Hankan
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(dict): every roadmap word is a card, tagged with its unit

Browse Source 
The reworked app's learner model hangs off cards: recall evidence, the
phase-review checklist, the practice set, and the gate's "words he has
met". So every word a unit introduces has to be studiable — and 16 of the
371 were not. Eleven existed only as sentence chunks or dictionary rows
outside the review deck, and five (봐 읽어 갔어 봤어 먹었어) nowhere at all.

The build now marks exactly one reviewable lemma per roadmap word with the
unit that introduces it. Where the deck has the word, its row is chosen
deterministically (deck order, then source, then part of speech) — the
artifact tagged whichever card came last, which put the evidence for 이, 눈
and 저 on the wrong meaning. The sixteen get a `curriculum` lemma of their
own, glossed from the curated verb they conjugate (자 is "sleep", the 반말
of 자다 — not the dictionary's "ruler"), else from the sentence that uses
them, else the dictionary.

Lemmas also carry their topic, which the vocabulary filters need.
dict:assert gains the guarantee: 371/371 roadmap words as one card each.
Migration 7 adds the two columns; the rows arrive with the dictionary
reload a changed build now triggers on its own.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
MechaCat02
2026-09-16 20:03:09 +02:00
parent f8183d786f
commit 48987b96ae
17 changed files with 205 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
tools/dict/assert-roadmap.mjs
View File

@@ -31,6 +31,7 @@ async function loadLexicon() {
const lemmas = new Map(); // headword -> [pos]
const surfaces = new Map(); // form -> analysis
const cards = new Map(); // roadmap word -> [{ unit, source }], from unit_id
for (const band of manifest.bands) {
const raw = gunzipSync(await readFile(DICT + band.file));
@@ -40,6 +41,8 @@ async function loadLexicon() {
const S = data.columns.surface;
const hw = L.indexOf("headword");
const pos = L.indexOf("pos");
const unit = L.indexOf("unit_id");
const source = L.indexOf("source");
const form = S.indexOf("form");
const analysis = S.indexOf("analysis");
@@ -47,13 +50,17 @@ async function loadLexicon() {
const w = row[hw];
if (!lemmas.has(w)) lemmas.set(w, []);
lemmas.get(w).push(row[pos]);
if (unit !== -1 && row[unit]) {
if (!cards.has(w)) cards.set(w, []);
cards.get(w).push({ unit: row[unit], source: row[source] });
}
}
for (const row of data.surfaces) {
if (!surfaces.has(row[form])) surfaces.set(row[form], row[analysis]);
}
}
return { manifest, lemmas, surfaces };
return { manifest, lemmas, surfaces, cards };
}
const resolves = (lex, word) => lex.lemmas.has(word) || lex.surfaces.has(word);
@@ -91,6 +98,21 @@ async function main() {
const deckWords = Object.values(deck.topics).flat();
for (const [w] of deckWords) check("deck", "deck.json", w);
/* 4. every roadmap word is exactly ONE reviewable card, tagged with the
unit that introduces it — the recall evidence, the phase review and
the gate's met words all hang off that card */
const REVIEWABLE = new Set(["curated", "grammar", "sfx", "curriculum"]);
let cardWords = 0;
for (const u of units) {
for (const w of u.words ?? []) {
cardWords++;
const tagged = lex.cards.get(w) ?? [];
if (tagged.length !== 1 || tagged[0].unit !== u.id || !REVIEWABLE.has(tagged[0].source)) {
failures.push({ kind: "card", where: u.id, word: `${w} (${tagged.length} tagged)` });
}
}
}
/* ── report ── */
const src = lex.manifest.builtWith.dictionary;
console.log(`Dictionary: ${src} — ${lex.manifest.totals.lemmas} lemmas, ` +
@@ -111,6 +133,7 @@ async function main() {
report("Roadmap words", roadmapWords, "roadmap");
report("Spiral targets", revisits, "revisit");
report("Deck words", deckWords.length, "deck");
report("Roadmap words as one card each", cardWords, "card");
if (failures.length) {
console.log(`\nFAIL — ${failures.length} words cannot be glossed.`);

111
tools/dict/build.mjs
View File

@@ -25,7 +25,7 @@ import { createHash } from "node:crypto";
import { fileURLToPath } from "node:url";
import { DatabaseSync } from "node:sqlite";
import { surfaceForms } from "../../lib/conjugation.js";
import { surfaceForms, haeche, past } from "../../lib/conjugation.js";
import { flatten } from "../../lib/gate.js";
import { bandOf, bandForUnit, REFERENCE_BAND, BANDS } from "../../shared/bands.mjs";
import { lemmaId } from "../../shared/lemma-id.mjs";
@@ -153,11 +153,13 @@ async function loadGlossExtra(lex) {
}
/** deck.json — 386 curated words. These glosses beat the dictionary's. */
async function loadDeck(lex) {
async function loadDeck(lex, deckOrder) {
const deck = await readJson(`${ROOT}data/deck.json`);
let n = 0;
for (const [topic, rows] of Object.entries(deck.topics)) {
for (const [headword, , gloss, pos] of rows) {
const key = lex.key(headword, pos === "phrase" ? "phrase" : pos);
if (!deckOrder.has(key)) deckOrder.set(key, deckOrder.size);
lex.add({
headword,
pos: pos === "phrase" ? "phrase" : pos,
@@ -228,6 +230,89 @@ async function loadGrammar(lex) {
return g.entries.length;
}
/* ── curriculum words are cards ────────────────────────────────────────
Every word a unit introduces must be studiable: the recall evidence, the
phase-review checklist, the practice set and the gate's "met words" all
hang off a card. So each roadmap word gets exactly ONE reviewable lemma
marked with the unit that introduces it.
Sixteen roadmap words have no reviewable row — inflected forms (먹어,
봤어) and words the dictionary only knows as something else (자 as
"ruler", where unit 2.3 means the 반말 of 자다). Those get a lemma of
their own, glossed from the curated verb they conjugate, else from the
sentence that uses them, else from the dictionary. */
/** Sources the review deck draws from. */
const REVIEWABLE = new Set(["curated", "grammar", "sfx", "curriculum"]);
/** A deterministic preference among several reviewable rows for one word. */
const POS_ORDER = ["noun", "verb", "adj", "adv", "pron", "num", "det", "particle", "ending", "phrase", "word"];
const posRank = (pos) => {
const i = POS_ORDER.indexOf(pos);
return i === -1 ? POS_ORDER.length : i;
};
function markCurriculum(lex, units, deckOrder) {
/* 반말 forms of the curated verbs: 봐 is 보다, 먹었어 is 먹다 in the past. */
const formGloss = new Map();
for (const e of lex.entries()) {
if ((e.pos !== "verb" && e.pos !== "adj") || !REVIEWABLE.has(e.source) || !e.gloss_en) continue;
const g = String(e.gloss_en).replace(/^to be /, "").replace(/^to /, "");
const present = haeche(e.headword);
if (!present) continue;
if (!formGloss.has(present)) formGloss.set(present, { gloss: g, note: `반말, from ${e.headword}` });
const was = past(present);
if (was && !formGloss.has(was)) formGloss.set(was, { gloss: `${g} (past)`, note: `반말 past, from ${e.headword}` });
}
let chosen = 0;
let made = 0;
const seen = new Set();
for (const u of units) {
for (const w of u.words ?? []) {
if (seen.has(w)) continue;
seen.add(w);
const topic = `수업 ${u.id} ${u.ko}`;
const rows = lex.forHeadword(w);
const reviewable = rows
.filter((e) => REVIEWABLE.has(e.source))
.sort(
(a, b) =>
(deckOrder.get(lex.key(a.headword, a.pos)) ?? Infinity) -
(deckOrder.get(lex.key(b.headword, b.pos)) ?? Infinity) ||
["curated", "grammar", "sfx", "curriculum"].indexOf(a.source) -
["curated", "grammar", "sfx", "curriculum"].indexOf(b.source) ||
posRank(a.pos) - posRank(b.pos) ||
a.pos.localeCompare(b.pos),
);
if (reviewable.length) {
const e = reviewable[0];
lex.add({ ...e, unit_id: u.id, topic: e.topic ?? topic });
chosen++;
continue;
}
const form = formGloss.get(w);
const chunk = rows.find((e) => e.source === "sentence" && e.gloss_en);
const other = rows.find((e) => e.gloss_en);
lex.add({
headword: w,
pos: form ? "form" : "word",
gloss_en: form?.gloss ?? chunk?.gloss_en ?? other?.gloss_en ?? "",
gloss_ko: "",
note: form?.note ?? "",
level: null,
source: "curriculum",
topic,
unit_id: u.id,
});
made++;
}
}
log(` curriculum: ${chosen + made} roadmap words as cards (${made} given a lemma of their own)`);
}
/* ── build ────────────────────────────────────────────────────────── */
async function main() {
@@ -236,11 +321,16 @@ async function main() {
await mkdir(OUT, { recursive: true });
const lex = new Lexicon();
const deckOrder = new Map();
const dict = await loadDictionary(lex);
await loadGlossExtra(lex);
await loadDeck(lex);
await loadDeck(lex, deckOrder);
await loadSentencesAndSfx(lex);
await loadGrammar(lex);
const curriculum = await readJson(`${ROOT}data/curriculum.json`);
const units = flatten(curriculum);
markCurriculum(lex, units, deckOrder);
log(` merged: ${lex.size} distinct (headword, pos)\n`);
/* Frequency, via the inverted join — see freq-forms.mjs. */
@@ -252,8 +342,6 @@ async function main() {
/* Which unit introduces a word, so curriculum vocabulary is admitted by
the band of its own unit rather than by its raw frequency. */
const curriculum = await readJson(`${ROOT}data/curriculum.json`);
const units = flatten(curriculum);
const introducedIn = new Map();
for (const u of units) {
for (const w of u.words ?? []) if (!introducedIn.has(w)) introducedIn.set(w, u.id);
@@ -297,6 +385,8 @@ async function main() {
gloss_ko: e.gloss_ko ?? "",
unit_band: band,
source: e.source,
topic: e.topic ?? null,
unit_id: e.unit_id ?? null,
});
/* The surface index. Strictly surfaceForms() plus the headword itself —
@@ -341,11 +431,11 @@ async function main() {
band,
format: 2,
columns: {
lemma: ["headword", "pos", "freq_rank", "level", "gloss_en", "gloss_ko", "unit_band", "source"],
lemma: ["headword", "pos", "freq_rank", "level", "gloss_en", "gloss_ko", "unit_band", "source", "topic", "unit_id"],
surface: ["form", "lemma", "analysis"],
},
lemmas: data.lemmas.map((l) => [
l.headword, l.pos, l.freq_rank, l.level, l.gloss_en, l.gloss_ko, l.unit_band, l.source,
l.headword, l.pos, l.freq_rank, l.level, l.gloss_en, l.gloss_ko, l.unit_band, l.source, l.topic, l.unit_id,
]),
surfaces: data.surfaces.map((s) => [s.form, rowOf.get(s.lemma_id), s.analysis]),
};
@@ -379,17 +469,18 @@ async function main() {
seed.exec(`
CREATE TABLE lemma (id INTEGER PRIMARY KEY, headword TEXT NOT NULL, pos TEXT NOT NULL,
freq_rank INTEGER, level TEXT, gloss_en TEXT NOT NULL DEFAULT '',
gloss_ko TEXT NOT NULL DEFAULT '', unit_band INTEGER NOT NULL DEFAULT 0, source TEXT NOT NULL);
gloss_ko TEXT NOT NULL DEFAULT '', unit_band INTEGER NOT NULL DEFAULT 0, source TEXT NOT NULL,
topic TEXT, unit_id TEXT);
CREATE TABLE surface (form TEXT NOT NULL, lemma_id INTEGER NOT NULL, analysis TEXT NOT NULL,
PRIMARY KEY (form, lemma_id)) WITHOUT ROWID;
CREATE INDEX surface_form ON surface(form);
`);
const band0 = byBand.get(0) ?? { lemmas: [], surfaces: [] };
const li = seed.prepare("INSERT INTO lemma VALUES (?,?,?,?,?,?,?,?,?)");
const li = seed.prepare("INSERT INTO lemma VALUES (?,?,?,?,?,?,?,?,?,?,?)");
const si = seed.prepare("INSERT OR IGNORE INTO surface VALUES (?,?,?)");
seed.exec("BEGIN");
for (const l of band0.lemmas)
li.run(l.id, l.headword, l.pos, l.freq_rank, l.level, l.gloss_en, l.gloss_ko, l.unit_band, l.source);
li.run(l.id, l.headword, l.pos, l.freq_rank, l.level, l.gloss_en, l.gloss_ko, l.unit_band, l.source, l.topic, l.unit_id);
for (const s of band0.surfaces) si.run(s.form, s.lemma_id, s.analysis);
seed.exec("COMMIT");
seed.close();