diff --git a/app/public/dict/band-0.json.gz b/app/public/dict/band-0.json.gz index 93281af..7685791 100644 Binary files a/app/public/dict/band-0.json.gz and b/app/public/dict/band-0.json.gz differ diff --git a/app/public/dict/band-1.json.gz b/app/public/dict/band-1.json.gz index 41a89fc..2f58d0d 100644 Binary files a/app/public/dict/band-1.json.gz and b/app/public/dict/band-1.json.gz differ diff --git a/app/public/dict/band-2.json.gz b/app/public/dict/band-2.json.gz index 332eeb0..6196eae 100644 Binary files a/app/public/dict/band-2.json.gz and b/app/public/dict/band-2.json.gz differ diff --git a/app/public/dict/band-3.json.gz b/app/public/dict/band-3.json.gz index cd4d9a6..991e4ad 100644 Binary files a/app/public/dict/band-3.json.gz and b/app/public/dict/band-3.json.gz differ diff --git a/app/public/dict/band-4.json.gz b/app/public/dict/band-4.json.gz index a1576e2..551d62c 100644 Binary files a/app/public/dict/band-4.json.gz and b/app/public/dict/band-4.json.gz differ diff --git a/app/public/dict/band-5.json.gz b/app/public/dict/band-5.json.gz index 1cfb71c..2d6690b 100644 Binary files a/app/public/dict/band-5.json.gz and b/app/public/dict/band-5.json.gz differ diff --git a/app/public/dict/band-6.json.gz b/app/public/dict/band-6.json.gz index aaee92d..d620ea1 100644 Binary files a/app/public/dict/band-6.json.gz and b/app/public/dict/band-6.json.gz differ diff --git a/app/public/dict/manifest.json b/app/public/dict/manifest.json index b9ccd6e..fc2147c 100644 --- a/app/public/dict/manifest.json +++ b/app/public/dict/manifest.json @@ -6,53 +6,53 @@ "frequencyForms": 688129 }, "totals": { - "lemmas": 30520, - "surfaces": 43879 + "lemmas": 30536, + "surfaces": 43895 }, "bands": [ { "band": 0, "file": "band-0.json.gz", - "lemmas": 801, - "surfaces": 1176, - "bytes": 21332, - "sha256": "e631e95e0263995c23ee58a5a3ed7cadf11d7b10566449192d644ccb2be7fe04", + "lemmas": 802, + "surfaces": 1177, + "bytes": 24358, + "sha256": "ec95a68245700a6dd9aee519378b9d86713484efe888f8450578f5c9315a48c6", "reference": false }, { "band": 1, "file": "band-1.json.gz", - "lemmas": 1240, - "surfaces": 2335, - "bytes": 48215, - "sha256": "f4ab15154ed2cefd0aa86346d874c6f3b54c8455e6c58f1b20cd83750b8402e0", + "lemmas": 1246, + "surfaces": 2341, + "bytes": 48984, + "sha256": "7f3aa9965d3c893efb3c6ffa96f68cce219b00c1bd146a9b847f4dbc25ec203b", "reference": false }, { "band": 2, "file": "band-2.json.gz", - "lemmas": 1450, - "surfaces": 2590, - "bytes": 54698, - "sha256": "2a48e603909cd969423cc444cfac215e7a2e51c6129a80e0779cbeed0c81ebd1", + "lemmas": 1456, + "surfaces": 2596, + "bytes": 55436, + "sha256": "4420ee704d1e16de6d9e5e721b3e9656482f7fd9f7930b9276ca1a9299031532", "reference": false }, { "band": 3, "file": "band-3.json.gz", - "lemmas": 1972, - "surfaces": 3373, - "bytes": 72713, - "sha256": "1b5bf666475b301930a99b2c23a8d7a9e585a3cbd40a4787feeebdf96e6ad2af", + "lemmas": 1974, + "surfaces": 3375, + "bytes": 73309, + "sha256": "64789873ede0cb9876fc01d45b89c144a69c28fa87bd6e7734d26936626a87b7", "reference": false }, { "band": 4, "file": "band-4.json.gz", - "lemmas": 2987, - "surfaces": 4775, - "bytes": 106990, - "sha256": "a1ec6eb219b3b1c303c26296d47e5341329e9773b7a30432729288edbda91957", + "lemmas": 2988, + "surfaces": 4776, + "bytes": 107822, + "sha256": "ec8c31ddf5080f7a6f0a1c92ac4067e764815122d9bdc244bd6b717d8a228d5e", "reference": false }, { @@ -60,8 +60,8 @@ "file": "band-5.json.gz", "lemmas": 6963, "surfaces": 10347, - "bytes": 245251, - "sha256": "1f8576b0d5686b67aece0a5b7a2bf291e5680f3fb7f43ec95a49cf7839d5730d", + "bytes": 247327, + "sha256": "de9b3c2daec3d5086a5711e6384a7239036fb3fd9e69ce4bfeea89458698731d", "reference": false }, { @@ -69,8 +69,8 @@ "file": "band-6.json.gz", "lemmas": 15107, "surfaces": 19283, - "bytes": 499732, - "sha256": "f9e2ed69ddb4515192d6e24a8abe863656c993414f6e28948ef59d78c6d0056c", + "bytes": 507133, + "sha256": "2a893ac64e46af015db7a3a6a29043848284e989e3b92c6f026059d6c3f11d51", "reference": true } ], diff --git a/app/public/dict/seed.sqlite3 b/app/public/dict/seed.sqlite3 index 649f414..fa9f046 100644 Binary files a/app/public/dict/seed.sqlite3 and b/app/public/dict/seed.sqlite3 differ diff --git a/app/src/db/migrations.ts b/app/src/db/migrations.ts index 8fc76f1..3e83c05 100644 --- a/app/src/db/migrations.ts +++ b/app/src/db/migrations.ts @@ -177,6 +177,19 @@ export const MIGRATIONS: Migration[] = [ `, run: rekeyLemmas, }, + { + id: 7, + name: "curriculum words are cards — a lemma knows its topic and the unit that introduces it", + sql: /* sql */ ` + -- Reference data from the band files, like the rest of lemma: not + -- synced, no updated_at. The build marks exactly one reviewable lemma + -- per roadmap word with its unit. The rows arrive with the next band + -- load, which a changed dictionary triggers on its own. + ALTER TABLE lemma ADD COLUMN topic TEXT; + ALTER TABLE lemma ADD COLUMN unit_id TEXT; + CREATE INDEX IF NOT EXISTS lemma_unit ON lemma(unit_id); + `, + }, ]; /** diff --git a/app/src/db/writes.ts b/app/src/db/writes.ts index 9bee094..f631898 100644 --- a/app/src/db/writes.ts +++ b/app/src/db/writes.ts @@ -385,6 +385,10 @@ export interface LemmaRow { gloss_ko: string; unit_band: number; source: string; + /** The deck topic, or "수업 " for a curriculum word. */ + topic?: string | null; + /** Set on exactly one lemma per roadmap word: the unit that introduces it. */ + unit_id?: string | null; } export interface SurfaceRow { @@ -414,10 +418,10 @@ export async function insertBand( surfaces: SurfaceRow[], ): Promise { await db.tx(async (tx) => { - const lemmaChunk = chunkFor(9); + const lemmaChunk = chunkFor(11); for (let i = 0; i < lemmas.length; i += lemmaChunk) { const slice = lemmas.slice(i, i + lemmaChunk); - const values = slice.map(() => "(?,?,?,?,?,?,?,?,?)").join(","); + const values = slice.map(() => "(?,?,?,?,?,?,?,?,?,?,?)").join(","); const params: Params = slice.flatMap((l) => [ l.id, l.headword, @@ -428,10 +432,12 @@ export async function insertBand( l.gloss_ko, l.unit_band, l.source, + l.topic ?? null, + l.unit_id ?? null, ]); await tx.run( `INSERT OR REPLACE INTO lemma - (id, headword, pos, freq_rank, level, gloss_en, gloss_ko, unit_band, source) + (id, headword, pos, freq_rank, level, gloss_en, gloss_ko, unit_band, source, topic, unit_id) VALUES ${values}`, params, ); diff --git a/app/src/domain/cards.ts b/app/src/domain/cards.ts index b232130..5d607fa 100644 --- a/app/src/domain/cards.ts +++ b/app/src/domain/cards.ts @@ -20,6 +20,9 @@ export interface DeckEntry { pos: string; glossEn: string; source: string; + topic: string | null; + /** The unit that introduces this word, for a roadmap word's card. */ + unitId: string | null; card: Card | null; status: CardStatus; } @@ -44,7 +47,10 @@ function toCard(row: Record): Card | null { */ // 'custom' is here so the learner's own words are reviewable like any // other — adding a word you cannot then study would be pointless. -const REVIEWABLE = "('curated', 'sfx', 'grammar', 'custom')"; +// 'curriculum' is a roadmap word the curated deck does not hold (먹어, 봤어): +// every word a unit introduces is a card, because the recall evidence and +// the phase review both hang off one. +const REVIEWABLE = "('curated', 'sfx', 'grammar', 'custom', 'curriculum')"; const SENTENCE_SOURCE = "('sentence')"; export interface DeckOptions { @@ -63,7 +69,8 @@ function sourceClause(opts: DeckOptions): string { export async function deck(db: Db, opts: DeckOptions = {}): Promise { const rows = await db.all>( - `SELECT l.id AS lemmaId, l.headword, l.pos, l.gloss_en AS glossEn, l.source, ${CARD_COLUMNS} + `SELECT l.id AS lemmaId, l.headword, l.pos, l.gloss_en AS glossEn, l.source, + l.topic, l.unit_id AS unitId, ${CARD_COLUMNS} FROM lemma l LEFT JOIN card c ON c.lemma_id = l.id WHERE ${sourceClause(opts)} ORDER BY l.headword`, @@ -77,6 +84,8 @@ export async function deck(db: Db, opts: DeckOptions = {}): Promise pos: r.pos as string, glossEn: r.glossEn as string, source: r.source as string, + topic: (r.topic as string | null) ?? null, + unitId: (r.unitId as string | null) ?? null, card, status: statusOf(card), }; diff --git a/app/src/domain/lexicon.ts b/app/src/domain/lexicon.ts index fd2ce8c..f5e2997 100644 --- a/app/src/domain/lexicon.ts +++ b/app/src/domain/lexicon.ts @@ -39,7 +39,8 @@ const SELECT_VIA_SURFACE = ` /** Prefer a curated gloss, then a common word, then anything. */ const RANKED = ` ORDER BY CASE l.source WHEN 'curated' THEN 0 WHEN 'grammar' THEN 1 - WHEN 'sentence' THEN 2 WHEN 'sfx' THEN 3 ELSE 4 END, + WHEN 'curriculum' THEN 2 WHEN 'sentence' THEN 3 + WHEN 'sfx' THEN 4 ELSE 5 END, l.freq_rank IS NULL, l.freq_rank`; /** diff --git a/app/src/ui/tutor/TutorTab.tsx b/app/src/ui/tutor/TutorTab.tsx index ee29643..4ab8d3c 100644 --- a/app/src/ui/tutor/TutorTab.tsx +++ b/app/src/ui/tutor/TutorTab.tsx @@ -80,7 +80,7 @@ async function readBandWords(db: Db, band: number, ceiling: number): Promise { }); }); +describe("curriculum words", () => { + it("arrive as studiable cards, each tagged with the unit that introduces it", async () => { + serve(shipped); + const { ensureBands } = await loader(); + await ensureBands(db, 1); + const { deck } = await import("@app/domain/cards.js"); + const entries = await deck(db); + + const eat = entries.find((e) => e.headword === "먹어"); + expect(eat).toMatchObject({ source: "curriculum", unitId: "2.3", glossEn: "eat" }); + expect(eat!.lemmaId).toBe(lemmaId("먹어", "form")); + + const tagged = entries.filter((e) => e.unitId?.startsWith("1.")); + const phase1 = (await import("@app/domain/gate.js")).UNITS.filter((u) => u.phase === 1).flatMap((u) => u.words ?? []); + expect(tagged.map((e) => e.headword).sort()).toEqual([...new Set(phase1)].sort()); + }); +}); + describe("a rebuilt dictionary", () => { it("reloads, and every card and custom word still names its word", async () => { serve(shipped); diff --git a/tools/dict/assert-roadmap.mjs b/tools/dict/assert-roadmap.mjs index 5854102..f1eef65 100644 --- a/tools/dict/assert-roadmap.mjs +++ b/tools/dict/assert-roadmap.mjs @@ -31,6 +31,7 @@ async function loadLexicon() { const lemmas = new Map(); // headword -> [pos] const surfaces = new Map(); // form -> analysis + const cards = new Map(); // roadmap word -> [{ unit, source }], from unit_id for (const band of manifest.bands) { const raw = gunzipSync(await readFile(DICT + band.file)); @@ -40,6 +41,8 @@ async function loadLexicon() { const S = data.columns.surface; const hw = L.indexOf("headword"); const pos = L.indexOf("pos"); + const unit = L.indexOf("unit_id"); + const source = L.indexOf("source"); const form = S.indexOf("form"); const analysis = S.indexOf("analysis"); @@ -47,13 +50,17 @@ async function loadLexicon() { const w = row[hw]; if (!lemmas.has(w)) lemmas.set(w, []); lemmas.get(w).push(row[pos]); + if (unit !== -1 && row[unit]) { + if (!cards.has(w)) cards.set(w, []); + cards.get(w).push({ unit: row[unit], source: row[source] }); + } } for (const row of data.surfaces) { if (!surfaces.has(row[form])) surfaces.set(row[form], row[analysis]); } } - return { manifest, lemmas, surfaces }; + return { manifest, lemmas, surfaces, cards }; } const resolves = (lex, word) => lex.lemmas.has(word) || lex.surfaces.has(word); @@ -91,6 +98,21 @@ async function main() { const deckWords = Object.values(deck.topics).flat(); for (const [w] of deckWords) check("deck", "deck.json", w); + /* 4. every roadmap word is exactly ONE reviewable card, tagged with the + unit that introduces it — the recall evidence, the phase review and + the gate's met words all hang off that card */ + const REVIEWABLE = new Set(["curated", "grammar", "sfx", "curriculum"]); + let cardWords = 0; + for (const u of units) { + for (const w of u.words ?? []) { + cardWords++; + const tagged = lex.cards.get(w) ?? []; + if (tagged.length !== 1 || tagged[0].unit !== u.id || !REVIEWABLE.has(tagged[0].source)) { + failures.push({ kind: "card", where: u.id, word: `${w} (${tagged.length} tagged)` }); + } + } + } + /* ── report ── */ const src = lex.manifest.builtWith.dictionary; console.log(`Dictionary: ${src} — ${lex.manifest.totals.lemmas} lemmas, ` + @@ -111,6 +133,7 @@ async function main() { report("Roadmap words", roadmapWords, "roadmap"); report("Spiral targets", revisits, "revisit"); report("Deck words", deckWords.length, "deck"); + report("Roadmap words as one card each", cardWords, "card"); if (failures.length) { console.log(`\nFAIL — ${failures.length} words cannot be glossed.`); diff --git a/tools/dict/build.mjs b/tools/dict/build.mjs index 859cfa9..fc935b4 100644 --- a/tools/dict/build.mjs +++ b/tools/dict/build.mjs @@ -25,7 +25,7 @@ import { createHash } from "node:crypto"; import { fileURLToPath } from "node:url"; import { DatabaseSync } from "node:sqlite"; -import { surfaceForms } from "../../lib/conjugation.js"; +import { surfaceForms, haeche, past } from "../../lib/conjugation.js"; import { flatten } from "../../lib/gate.js"; import { bandOf, bandForUnit, REFERENCE_BAND, BANDS } from "../../shared/bands.mjs"; import { lemmaId } from "../../shared/lemma-id.mjs"; @@ -153,11 +153,13 @@ async function loadGlossExtra(lex) { } /** deck.json — 386 curated words. These glosses beat the dictionary's. */ -async function loadDeck(lex) { +async function loadDeck(lex, deckOrder) { const deck = await readJson(`${ROOT}data/deck.json`); let n = 0; for (const [topic, rows] of Object.entries(deck.topics)) { for (const [headword, , gloss, pos] of rows) { + const key = lex.key(headword, pos === "phrase" ? "phrase" : pos); + if (!deckOrder.has(key)) deckOrder.set(key, deckOrder.size); lex.add({ headword, pos: pos === "phrase" ? "phrase" : pos, @@ -228,6 +230,89 @@ async function loadGrammar(lex) { return g.entries.length; } +/* ── curriculum words are cards ──────────────────────────────────────── + Every word a unit introduces must be studiable: the recall evidence, the + phase-review checklist, the practice set and the gate's "met words" all + hang off a card. So each roadmap word gets exactly ONE reviewable lemma + marked with the unit that introduces it. + + Sixteen roadmap words have no reviewable row — inflected forms (먹어, + 봤어) and words the dictionary only knows as something else (자 as + "ruler", where unit 2.3 means the 반말 of 자다). Those get a lemma of + their own, glossed from the curated verb they conjugate, else from the + sentence that uses them, else from the dictionary. */ + +/** Sources the review deck draws from. */ +const REVIEWABLE = new Set(["curated", "grammar", "sfx", "curriculum"]); + +/** A deterministic preference among several reviewable rows for one word. */ +const POS_ORDER = ["noun", "verb", "adj", "adv", "pron", "num", "det", "particle", "ending", "phrase", "word"]; +const posRank = (pos) => { + const i = POS_ORDER.indexOf(pos); + return i === -1 ? POS_ORDER.length : i; +}; + +function markCurriculum(lex, units, deckOrder) { + /* 반말 forms of the curated verbs: 봐 is 보다, 먹었어 is 먹다 in the past. */ + const formGloss = new Map(); + for (const e of lex.entries()) { + if ((e.pos !== "verb" && e.pos !== "adj") || !REVIEWABLE.has(e.source) || !e.gloss_en) continue; + const g = String(e.gloss_en).replace(/^to be /, "").replace(/^to /, ""); + const present = haeche(e.headword); + if (!present) continue; + if (!formGloss.has(present)) formGloss.set(present, { gloss: g, note: `반말, from ${e.headword}` }); + const was = past(present); + if (was && !formGloss.has(was)) formGloss.set(was, { gloss: `${g} (past)`, note: `반말 past, from ${e.headword}` }); + } + + let chosen = 0; + let made = 0; + const seen = new Set(); + for (const u of units) { + for (const w of u.words ?? []) { + if (seen.has(w)) continue; + seen.add(w); + const topic = `수업 ${u.id} ${u.ko}`; + const rows = lex.forHeadword(w); + const reviewable = rows + .filter((e) => REVIEWABLE.has(e.source)) + .sort( + (a, b) => + (deckOrder.get(lex.key(a.headword, a.pos)) ?? Infinity) - + (deckOrder.get(lex.key(b.headword, b.pos)) ?? Infinity) || + ["curated", "grammar", "sfx", "curriculum"].indexOf(a.source) - + ["curated", "grammar", "sfx", "curriculum"].indexOf(b.source) || + posRank(a.pos) - posRank(b.pos) || + a.pos.localeCompare(b.pos), + ); + + if (reviewable.length) { + const e = reviewable[0]; + lex.add({ ...e, unit_id: u.id, topic: e.topic ?? topic }); + chosen++; + continue; + } + + const form = formGloss.get(w); + const chunk = rows.find((e) => e.source === "sentence" && e.gloss_en); + const other = rows.find((e) => e.gloss_en); + lex.add({ + headword: w, + pos: form ? "form" : "word", + gloss_en: form?.gloss ?? chunk?.gloss_en ?? other?.gloss_en ?? "", + gloss_ko: "", + note: form?.note ?? "", + level: null, + source: "curriculum", + topic, + unit_id: u.id, + }); + made++; + } + } + log(` curriculum: ${chosen + made} roadmap words as cards (${made} given a lemma of their own)`); +} + /* ── build ────────────────────────────────────────────────────────── */ async function main() { @@ -236,11 +321,16 @@ async function main() { await mkdir(OUT, { recursive: true }); const lex = new Lexicon(); + const deckOrder = new Map(); const dict = await loadDictionary(lex); await loadGlossExtra(lex); - await loadDeck(lex); + await loadDeck(lex, deckOrder); await loadSentencesAndSfx(lex); await loadGrammar(lex); + + const curriculum = await readJson(`${ROOT}data/curriculum.json`); + const units = flatten(curriculum); + markCurriculum(lex, units, deckOrder); log(` merged: ${lex.size} distinct (headword, pos)\n`); /* Frequency, via the inverted join — see freq-forms.mjs. */ @@ -252,8 +342,6 @@ async function main() { /* Which unit introduces a word, so curriculum vocabulary is admitted by the band of its own unit rather than by its raw frequency. */ - const curriculum = await readJson(`${ROOT}data/curriculum.json`); - const units = flatten(curriculum); const introducedIn = new Map(); for (const u of units) { for (const w of u.words ?? []) if (!introducedIn.has(w)) introducedIn.set(w, u.id); @@ -297,6 +385,8 @@ async function main() { gloss_ko: e.gloss_ko ?? "", unit_band: band, source: e.source, + topic: e.topic ?? null, + unit_id: e.unit_id ?? null, }); /* The surface index. Strictly surfaceForms() plus the headword itself — @@ -341,11 +431,11 @@ async function main() { band, format: 2, columns: { - lemma: ["headword", "pos", "freq_rank", "level", "gloss_en", "gloss_ko", "unit_band", "source"], + lemma: ["headword", "pos", "freq_rank", "level", "gloss_en", "gloss_ko", "unit_band", "source", "topic", "unit_id"], surface: ["form", "lemma", "analysis"], }, lemmas: data.lemmas.map((l) => [ - l.headword, l.pos, l.freq_rank, l.level, l.gloss_en, l.gloss_ko, l.unit_band, l.source, + l.headword, l.pos, l.freq_rank, l.level, l.gloss_en, l.gloss_ko, l.unit_band, l.source, l.topic, l.unit_id, ]), surfaces: data.surfaces.map((s) => [s.form, rowOf.get(s.lemma_id), s.analysis]), }; @@ -379,17 +469,18 @@ async function main() { seed.exec(` CREATE TABLE lemma (id INTEGER PRIMARY KEY, headword TEXT NOT NULL, pos TEXT NOT NULL, freq_rank INTEGER, level TEXT, gloss_en TEXT NOT NULL DEFAULT '', - gloss_ko TEXT NOT NULL DEFAULT '', unit_band INTEGER NOT NULL DEFAULT 0, source TEXT NOT NULL); + gloss_ko TEXT NOT NULL DEFAULT '', unit_band INTEGER NOT NULL DEFAULT 0, source TEXT NOT NULL, + topic TEXT, unit_id TEXT); CREATE TABLE surface (form TEXT NOT NULL, lemma_id INTEGER NOT NULL, analysis TEXT NOT NULL, PRIMARY KEY (form, lemma_id)) WITHOUT ROWID; CREATE INDEX surface_form ON surface(form); `); const band0 = byBand.get(0) ?? { lemmas: [], surfaces: [] }; - const li = seed.prepare("INSERT INTO lemma VALUES (?,?,?,?,?,?,?,?,?)"); + const li = seed.prepare("INSERT INTO lemma VALUES (?,?,?,?,?,?,?,?,?,?,?)"); const si = seed.prepare("INSERT OR IGNORE INTO surface VALUES (?,?,?)"); seed.exec("BEGIN"); for (const l of band0.lemmas) - li.run(l.id, l.headword, l.pos, l.freq_rank, l.level, l.gloss_en, l.gloss_ko, l.unit_band, l.source); + li.run(l.id, l.headword, l.pos, l.freq_rank, l.level, l.gloss_en, l.gloss_ko, l.unit_band, l.source, l.topic, l.unit_id); for (const s of band0.surfaces) si.run(s.form, s.lemma_id, s.analysis); seed.exec("COMMIT"); seed.close();