feat(dict): stable lemma ids — a card names its word, not a build position

Cards point at lemmas by id, and an id was the entry's position in the
sorted build. One word added near the top of the dictionary would have
moved every card below it onto a different word — silently, because loaded
bands were recorded by number and a rebuilt dictionary never reached an
existing install anyway. A custom word took max(id)+1 on whichever device
added it, so the same id meant different words on a phone and a laptop.

An id is now lemmaId(headword, pos), a 53-bit hash defined once in
shared/ and used by the build, the loader, custom words and the migration.
The build asserts all 30,520 entries are collision-free, and a test pins the
function itself, since changing it re-keys every card.

Band files are format 2: they carry no ids at all. The loader derives each
id from the word, and a surface names its lemma by row index in the same
file. Writing hashed ids out cost 0.5 MB of incompressible digits; leaving
them out makes the files smaller than before (1.1 MB -> 1.0 MB).

The loaded dictionary is now versioned by its band hashes, so a rebuild
reloads on the next boot — safe only now that a reload cannot move a card.

Migration 6 re-keys an existing install without stamping anything: cards
and their tombstones move through the lemma rows still loaded, custom words
become custom_word rows (the learner's data, which can travel) carrying the
time their card was made, and the dictionary is dropped to reload.

Sync is paused until the protocol that replaces it lands: the server still
holds rows under the old ids, and exchanging them would plant cards that
name no word.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

tools/dict/build.mjs

@@ -28,6 +28,7 @@ import { DatabaseSync } from "node:sqlite";
 import { surfaceForms } from "../../lib/conjugation.js";
 import { flatten } from "../../lib/gate.js";
 import { bandOf, bandForUnit, REFERENCE_BAND, BANDS } from "../../shared/bands.mjs";
+import { lemmaId } from "../../shared/lemma-id.mjs";
 
 import { readKaikki, KAIKKI_ATTRIBUTION } from "./sources/kaikki.mjs";
 import { readKrdict, findKrdict, KRDICT_ATTRIBUTION } from "./sources/krdict.mjs";
@@ -258,17 +259,28 @@ async function main() {
     for (const w of u.words ?? []) if (!introducedIn.has(w)) introducedIn.set(w, u.id);
   }
 
-  /* Stable ids: sort first so a rebuild produces byte-identical output. */
+  /* Sorted, so a rebuild produces byte-identical output. The ids do NOT come
+     from this order: an id is a hash of (headword, pos) — see
+     shared/lemma-id.mjs — so a card keeps its word however the dictionary
+     changes around it. */
   entries.sort((a, b) =>
     a.headword === b.headword ? a.pos.localeCompare(b.pos) : a.headword.localeCompare(b.headword),
   );
 
   const lemmas = [];
   const surfaces = [];
-  let id = 0;
+  const idOwner = new Map();
 
   for (const e of entries) {
-    id++;
+    const id = lemmaId(e.headword, e.pos);
+    const clash = idOwner.get(id);
+    if (clash) {
+      throw new Error(
+        `lemma id collision: ${clash} and ${e.headword}/${e.pos} both hash to ${id}. ` +
+          "Two words would share every card and review — change the hash before shipping.",
+      );
+    }
+    idOwner.set(id, `${e.headword}/${e.pos}`);
     const rank = ranks.get(`${e.headword} ${e.pos}`) ?? null;
     let band = bandOf({ source: e.source, freqRank: rank, level: e.level });
 
@@ -318,17 +330,24 @@ async function main() {
 
   for (const band of allBands) {
     const data = byBand.get(band) ?? { lemmas: [], surfaces: [] };
-    // Arrays, not objects: the key names would otherwise be ~60% of the file.
+    const rowOf = new Map(data.lemmas.map((l, i) => [l.id, i]));
+    /* Arrays, not objects: the key names would otherwise be ~60% of the file.
+
+       Format 2 ships no ids at all. An id is lemmaId(headword, pos), which the
+       app computes as it loads, and a surface names its lemma by row index in
+       this same file — a surface always travels in its lemma's band. Hashed
+       ids written out in full cost 0.5 MB of incompressible digits. */
     const payload = {
       band,
+      format: 2,
       columns: {
-        lemma: ["id", "headword", "pos", "freq_rank", "level", "gloss_en", "gloss_ko", "unit_band", "source"],
-        surface: ["form", "lemma_id", "analysis"],
+        lemma: ["headword", "pos", "freq_rank", "level", "gloss_en", "gloss_ko", "unit_band", "source"],
+        surface: ["form", "lemma", "analysis"],
       },
       lemmas: data.lemmas.map((l) => [
-        l.id, l.headword, l.pos, l.freq_rank, l.level, l.gloss_en, l.gloss_ko, l.unit_band, l.source,
+        l.headword, l.pos, l.freq_rank, l.level, l.gloss_en, l.gloss_ko, l.unit_band, l.source,
       ]),
-      surfaces: data.surfaces.map((s) => [s.form, s.lemma_id, s.analysis]),
+      surfaces: data.surfaces.map((s) => [s.form, rowOf.get(s.lemma_id), s.analysis]),
     };
 
     const gz = gzipSync(Buffer.from(JSON.stringify(payload), "utf8"), { level: 9 });
@@ -376,6 +395,7 @@ async function main() {
   seed.close();
 
   const manifest = {
+    format: 2,
     builtWith: {
       dictionary: dict.name,
       dictionaryEntries: dict.entries,