feat(dict): stable lemma ids — a card names its word, not a build position

Cards point at lemmas by id, and an id was the entry's position in the
sorted build. One word added near the top of the dictionary would have
moved every card below it onto a different word — silently, because loaded
bands were recorded by number and a rebuilt dictionary never reached an
existing install anyway. A custom word took max(id)+1 on whichever device
added it, so the same id meant different words on a phone and a laptop.

An id is now lemmaId(headword, pos), a 53-bit hash defined once in
shared/ and used by the build, the loader, custom words and the migration.
The build asserts all 30,520 entries are collision-free, and a test pins the
function itself, since changing it re-keys every card.

Band files are format 2: they carry no ids at all. The loader derives each
id from the word, and a surface names its lemma by row index in the same
file. Writing hashed ids out cost 0.5 MB of incompressible digits; leaving
them out makes the files smaller than before (1.1 MB -> 1.0 MB).

The loaded dictionary is now versioned by its band hashes, so a rebuild
reloads on the next boot — safe only now that a reload cannot move a card.

Migration 6 re-keys an existing install without stamping anything: cards
and their tombstones move through the lemma rows still loaded, custom words
become custom_word rows (the learner's data, which can travel) carrying the
time their card was made, and the dictionary is dropped to reload.

Sync is paused until the protocol that replaces it lands: the server still
holds rows under the old ids, and exchanging them would plant cards that
name no word.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

test/db/conformance.ts

@@ -23,7 +23,7 @@ import {
 import { newCard, grade, GOOD } from "@lib/srs.js";
 
 /** Every table that carries user data, and therefore a write timestamp. */
-const SYNCABLE = ["card", "progress", "chat", "meta", "study_log", "peek"] as const;
+const SYNCABLE = ["card", "progress", "chat", "meta", "study_log", "peek", "custom_word"] as const;
 
 export function conformanceSuite(name: string, open: () => Promise<Db>): void {
   describe(`Db conformance — ${name}`, () => {

test/db/lemma-id.test.ts

@@ -0,0 +1,62 @@
+/* A lemma's id is a hash of the word, not its position in the build.
+
+   Cards point at lemmas by id. When ids were positions, adding one entry near
+   the top of the dictionary moved every card below it onto a different word,
+   and a custom word's id meant different words on different devices. */
+
+import { describe, it, expect } from "vitest";
+import { readFileSync } from "node:fs";
+import { gunzipSync } from "node:zlib";
+import { lemmaId } from "@shared/lemma-id.mjs";
+
+describe("lemmaId", () => {
+  /* PINNED. Changing the hash re-keys every card on every device; if this
+     test has to change, a migration has to move the cards with it. */
+  it("is exactly this function", () => {
+    expect(lemmaId("밥", "noun")).toBe(55855054575946);
+    expect(lemmaId("가다", "verb")).toBe(1434356355562999);
+  });
+
+  it("depends on the part of speech as well as the headword", () => {
+    expect(lemmaId("밥", "noun")).not.toBe(lemmaId("밥", "verb"));
+  });
+
+  it("is a positive safe integer, which SQLite stores as INTEGER PRIMARY KEY", () => {
+    for (const [h, p] of [["밥", "noun"], ["", ""], ["신경 쓰다", "verb"], ["x".repeat(500), "noun"]]) {
+      const id = lemmaId(h!, p!);
+      expect(Number.isSafeInteger(id)).toBe(true);
+      expect(id).toBeGreaterThan(0);
+    }
+  });
+});
+
+describe("the shipped dictionary", () => {
+  const manifest = JSON.parse(
+    readFileSync(new URL("../../app/public/dict/manifest.json", import.meta.url), "utf8"),
+  ) as { format: number; totals: { lemmas: number }; bands: { file: string }[] };
+
+  it("is format 2 — no ids in the files, derived as they load", () => {
+    expect(manifest.format).toBe(2);
+  });
+
+  it("gives every (headword, pos) its own id", () => {
+    const owner = new Map<number, string>();
+    let n = 0;
+    for (const band of manifest.bands) {
+      const data = JSON.parse(
+        gunzipSync(readFileSync(new URL(`../../app/public/dict/${band.file}`, import.meta.url))).toString("utf8"),
+      ) as { format: number; columns: { lemma: string[] }; lemmas: unknown[][] };
+      expect(data.format).toBe(2);
+      const hw = data.columns.lemma.indexOf("headword");
+      const pos = data.columns.lemma.indexOf("pos");
+      for (const row of data.lemmas) {
+        const key = `${row[hw]}/${row[pos]}`;
+        const id = lemmaId(row[hw] as string, row[pos] as string);
+        expect(owner.get(id) ?? key, `${key} collides`).toBe(key);
+        owner.set(id, key);
+        n++;
+      }
+    }
+    expect(n).toBe(manifest.totals.lemmas);
+  });
+});

test/db/migration-6.test.ts

@@ -0,0 +1,89 @@
+/* Migration 6 — cards move to stable lemma ids.
+
+   A database as the previous build left it: positional lemma ids, a custom
+   word at the old reserved offset, a tombstone naming a card by its old id.
+   After the migration every reference names the same word under its hashed
+   id, and nothing has been stamped — the rows are the same rows, renamed. */
+
+import { describe, it, expect, beforeEach, afterEach } from "vitest";
+import { SqliteWasmDb } from "@app/db/sqlite-wasm-core.js";
+import { migrate } from "@app/db/migrate.js";
+import { lemmaId } from "@shared/lemma-id.mjs";
+import type { Db } from "@app/db/types.js";
+
+let db: Db;
+beforeEach(async () => {
+  db = await SqliteWasmDb.open({ memory: true });
+  await migrate(db, 5);
+  await db.exec(`
+    INSERT INTO lemma (id, headword, pos, gloss_en, unit_band, source) VALUES
+      (1, '물', 'noun', 'water', 0, 'curated'),
+      (2, '밥', 'noun', 'rice', 0, 'curated'),
+      (3, '책', 'noun', 'book', 0, 'curated'),
+      (10000000, '던전', 'noun', 'dungeon', 0, 'custom');
+    INSERT INTO surface (form, lemma_id, analysis) VALUES
+      ('밥', 2, 'headword, noun'),
+      ('던전', 10000000, 'headword, custom');
+    INSERT INTO card (lemma_id, state, interval, due, reps, updated_at) VALUES
+      (2, 2, 21, 100, 5, 111),
+      (10000000, 1, 1, 90, 1, 222),
+      (424242, 1, 1, 90, 1, 250);
+    INSERT INTO tombstone (tbl, pk, updated_at) VALUES ('card', '3', 333);
+    INSERT INTO meta (k, v, updated_at) VALUES ('dict.loadedBands', '[0]', 0);
+  `);
+  await migrate(db);
+});
+afterEach(async () => {
+  await db.close();
+});
+
+describe("migration 6", () => {
+  it("moves each card to its word's stable id, keeping its schedule and stamp", async () => {
+    const rice = await db.get<{ interval: number; reps: number; updated_at: number }>(
+      "SELECT interval, reps, updated_at FROM card WHERE lemma_id = ?",
+      [lemmaId("밥", "noun")],
+    );
+    expect(rice).toEqual({ interval: 21, reps: 5, updated_at: 111 });
+    expect(await db.get("SELECT 1 FROM card WHERE lemma_id = 2")).toBeUndefined();
+  });
+
+  it("leaves a card it cannot name exactly as it was", async () => {
+    expect(await db.get<{ updated_at: number }>("SELECT updated_at FROM card WHERE lemma_id = 424242")).toEqual({
+      updated_at: 250,
+    });
+  });
+
+  it("renames a card's tombstone with it", async () => {
+    const graves = await db.all<{ pk: string; updated_at: number }>(
+      "SELECT pk, updated_at FROM tombstone WHERE tbl = 'card'",
+    );
+    expect(graves).toEqual([{ pk: String(lemmaId("책", "noun")), updated_at: 333 }]);
+  });
+
+  it("turns a custom lemma into a custom_word, stamped when its card was made", async () => {
+    expect(await db.all("SELECT headword, pos, gloss, updated_at FROM custom_word")).toEqual([
+      { headword: "던전", pos: "noun", gloss: "dungeon", updated_at: 222 },
+    ]);
+    const id = lemmaId("던전", "noun");
+    expect(await db.get("SELECT headword, source FROM lemma WHERE id = ?", [id])).toEqual({
+      headword: "던전",
+      source: "custom",
+    });
+    expect(await db.get("SELECT form FROM surface WHERE lemma_id = ?", [id])).toEqual({ form: "던전" });
+    expect(await db.get<{ updated_at: number }>("SELECT updated_at FROM card WHERE lemma_id = ?", [id])).toEqual({
+      updated_at: 222,
+    });
+  });
+
+  it("drops the shipped dictionary, so the bands reload under the new ids", async () => {
+    expect(await db.all("SELECT headword FROM lemma WHERE source <> 'custom'")).toEqual([]);
+    expect(await db.get("SELECT v FROM meta WHERE k = 'dict.loadedBands'")).toBeUndefined();
+  });
+
+  it("stamps nothing", async () => {
+    for (const tbl of ["card", "custom_word", "tombstone", "meta"]) {
+      const row = await db.get<{ top: number | null }>(`SELECT max(updated_at) AS top FROM ${tbl}`);
+      expect(row?.top ?? 0, tbl).toBeLessThanOrEqual(333);
+    }
+  });
+});

test/domain/dictionary.test.ts

@@ -0,0 +1,97 @@
+/* Loading the shipped dictionary, and reloading it when it changes.
+
+   Loaded bands used to be recorded by number alone, so a rebuilt dictionary
+   never reached an existing install. Reloading only became safe once an id
+   named a word instead of a position — these pin both halves. */
+
+import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
+import { readFileSync } from "node:fs";
+import { SqliteWasmDb } from "@app/db/sqlite-wasm-core.js";
+import { migrate } from "@app/db/migrate.js";
+import { editCard, editAddCustomWord } from "@app/db/writes.js";
+import { markKnown } from "@lib/srs.js";
+import { lemmaId } from "@shared/lemma-id.mjs";
+import type { Db } from "@app/db/types.js";
+
+const DICT = new URL("../../app/public/dict/", import.meta.url);
+const shipped = JSON.parse(readFileSync(new URL("manifest.json", DICT), "utf8")) as {
+  bands: { band: number; file: string; sha256: string; lemmas: number }[];
+};
+
+/** Serve the committed files; `manifest` lets a test pretend to be a rebuild. */
+function serve(manifest: unknown) {
+  vi.stubGlobal("fetch", async (input: string) => {
+    const name = String(input).split("/dict/")[1] ?? "";
+    if (name === "manifest.json") return new Response(JSON.stringify(manifest));
+    return new Response(readFileSync(new URL(name, DICT)));
+  });
+}
+
+async function loader() {
+  vi.resetModules(); // the manifest is cached per module instance
+  return import("@app/domain/dictionary.js");
+}
+
+let db: Db;
+beforeEach(async () => {
+  db = await SqliteWasmDb.open({ memory: true });
+  await migrate(db);
+});
+afterEach(async () => {
+  vi.unstubAllGlobals();
+  await db.close();
+});
+
+const count = async (sql: string, params: (string | number)[] = []) =>
+  (await db.get<{ n: number }>(sql, params))!.n;
+
+describe("loading a band", () => {
+  it("derives every id from the word, so surfaces join their lemmas", async () => {
+    serve(shipped);
+    const { ensureBands } = await loader();
+    await ensureBands(db, 0);
+
+    expect(await count("SELECT count(*) AS n FROM lemma")).toBe(shipped.bands[0]!.lemmas);
+    const rice = await db.get<{ id: number }>("SELECT id FROM lemma WHERE headword = '밥' AND pos = 'noun'");
+    expect(rice?.id).toBe(lemmaId("밥", "noun"));
+    expect(await count("SELECT count(*) AS n FROM surface s LEFT JOIN lemma l ON l.id = s.lemma_id WHERE l.id IS NULL")).toBe(0);
+  });
+
+  it("does not load a band twice", async () => {
+    serve(shipped);
+    const { ensureBands } = await loader();
+    expect(await ensureBands(db, 0)).toEqual([0]);
+    expect(await ensureBands(db, 0)).toEqual([]);
+  });
+});
+
+describe("a rebuilt dictionary", () => {
+  it("reloads, and every card and custom word still names its word", async () => {
+    serve(shipped);
+    let dict = await loader();
+    await dict.ensureBands(db, 0);
+    const rice = lemmaId("밥", "noun");
+    await editCard(db, rice, markKnown(100));
+    const custom = await editAddCustomWord(db, { headword: "던전돌", gloss: "dungeon stone", pos: "noun" });
+
+    // A rebuild: band 0's file hash changes.
+    const rebuilt = structuredClone(shipped);
+    rebuilt.bands[0]!.sha256 = "f".repeat(64);
+    serve(rebuilt);
+    dict = await loader();
+    await db.run("DELETE FROM lemma WHERE headword = '물'"); // prove the reload really re-inserts
+    expect(await dict.ensureBands(db, 0)).toEqual([0]);
+
+    expect(await count("SELECT count(*) AS n FROM lemma WHERE headword = '물'")).toBeGreaterThan(0);
+    const card = await db.get<{ headword: string }>(
+      "SELECT l.headword FROM card c JOIN lemma l ON l.id = c.lemma_id WHERE c.lemma_id = ?",
+      [rice],
+    );
+    expect(card?.headword).toBe("밥");
+    const mine = await db.get<{ headword: string; source: string }>(
+      "SELECT headword, source FROM lemma WHERE id = ?",
+      [custom.lemmaId],
+    );
+    expect(mine).toEqual({ headword: "던전돌", source: "custom" });
+  });
+});

test/domain/reset.test.ts

@@ -10,6 +10,7 @@ import { describe, it, expect, beforeEach, afterEach } from "vitest";
 import { SqliteWasmDb } from "@app/db/sqlite-wasm-core.js";
 import { migrate } from "@app/db/migrate.js";
 import {
+  editAddCustomWord,
   editCard,
   editChatTurn,
   editMeta,
@@ -96,6 +97,20 @@ describe("editReset('everything')", () => {
     expect(tables).toContain("study_log");
   });
 
+  it("removes the learner's own words, lemma and all, and tombstones them", async () => {
+    await aLearnerWithHistory();
+    const { lemmaId } = await editAddCustomWord(db, { headword: "던전", gloss: "dungeon", pos: "noun" });
+    await editReset(db, "everything");
+
+    expect(await count("custom_word")).toBe(0);
+    expect(await db.get("SELECT 1 FROM lemma WHERE id = ?", [lemmaId])).toBeUndefined();
+    expect(await db.get("SELECT 1 FROM surface WHERE lemma_id = ?", [lemmaId])).toBeUndefined();
+    expect(await db.get("SELECT pk FROM tombstone WHERE tbl = 'custom_word'")).toEqual({
+      pk: JSON.stringify(["던전", "noun"]),
+    });
+    expect(await count("lemma")).toBe(3); // the shipped words stay
+  });
+
   it("does not survive a reboot: nothing reseeds the deck", async () => {
     await aLearnerWithHistory();
     await editReset(db, "everything");
@@ -124,9 +139,10 @@ describe("migration 5 — the known-words seed is gone", () => {
     await editCard(db, 2, newCard()); // something the learner actually did
     await db.run("INSERT INTO meta (k, v, updated_at) VALUES ('seed.known', '47', 0)");
 
-    // Wind the recorded version back to 4 so migrate() really runs 5.
+    // Wind the recorded version back to 4 so migrate() really runs 5 — and
+    // only 5: migration 6 re-keys cards, which is not what this pins.
     await db.run("INSERT OR REPLACE INTO meta (k, v, updated_at) VALUES ('schema_version','4',0)");
-    const { from, to } = await migrate(db);
+    const { from, to } = await migrate(db, 5);
     expect(from).toBe(4);
     expect(to).toBe(5);