Cards point at lemmas by id, and an id was the entry's position in the sorted build. One word added near the top of the dictionary would have moved every card below it onto a different word — silently, because loaded bands were recorded by number and a rebuilt dictionary never reached an existing install anyway. A custom word took max(id)+1 on whichever device added it, so the same id meant different words on a phone and a laptop. An id is now lemmaId(headword, pos), a 53-bit hash defined once in shared/ and used by the build, the loader, custom words and the migration. The build asserts all 30,520 entries are collision-free, and a test pins the function itself, since changing it re-keys every card. Band files are format 2: they carry no ids at all. The loader derives each id from the word, and a surface names its lemma by row index in the same file. Writing hashed ids out cost 0.5 MB of incompressible digits; leaving them out makes the files smaller than before (1.1 MB -> 1.0 MB). The loaded dictionary is now versioned by its band hashes, so a rebuild reloads on the next boot — safe only now that a reload cannot move a card. Migration 6 re-keys an existing install without stamping anything: cards and their tombstones move through the lemma rows still loaded, custom words become custom_word rows (the learner's data, which can travel) carrying the time their card was made, and the dictionary is dropped to reload. Sync is paused until the protocol that replaces it lands: the server still holds rows under the old ids, and exchanging them would plant cards that name no word. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
98 lines
3.8 KiB
TypeScript
98 lines
3.8 KiB
TypeScript
/* Loading the shipped dictionary, and reloading it when it changes.
|
|
|
|
Loaded bands used to be recorded by number alone, so a rebuilt dictionary
|
|
never reached an existing install. Reloading only became safe once an id
|
|
named a word instead of a position — these pin both halves. */
|
|
|
|
import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
|
|
import { readFileSync } from "node:fs";
|
|
import { SqliteWasmDb } from "@app/db/sqlite-wasm-core.js";
|
|
import { migrate } from "@app/db/migrate.js";
|
|
import { editCard, editAddCustomWord } from "@app/db/writes.js";
|
|
import { markKnown } from "@lib/srs.js";
|
|
import { lemmaId } from "@shared/lemma-id.mjs";
|
|
import type { Db } from "@app/db/types.js";
|
|
|
|
const DICT = new URL("../../app/public/dict/", import.meta.url);
|
|
const shipped = JSON.parse(readFileSync(new URL("manifest.json", DICT), "utf8")) as {
|
|
bands: { band: number; file: string; sha256: string; lemmas: number }[];
|
|
};
|
|
|
|
/** Serve the committed files; `manifest` lets a test pretend to be a rebuild. */
|
|
function serve(manifest: unknown) {
|
|
vi.stubGlobal("fetch", async (input: string) => {
|
|
const name = String(input).split("/dict/")[1] ?? "";
|
|
if (name === "manifest.json") return new Response(JSON.stringify(manifest));
|
|
return new Response(readFileSync(new URL(name, DICT)));
|
|
});
|
|
}
|
|
|
|
async function loader() {
|
|
vi.resetModules(); // the manifest is cached per module instance
|
|
return import("@app/domain/dictionary.js");
|
|
}
|
|
|
|
let db: Db;
|
|
beforeEach(async () => {
|
|
db = await SqliteWasmDb.open({ memory: true });
|
|
await migrate(db);
|
|
});
|
|
afterEach(async () => {
|
|
vi.unstubAllGlobals();
|
|
await db.close();
|
|
});
|
|
|
|
const count = async (sql: string, params: (string | number)[] = []) =>
|
|
(await db.get<{ n: number }>(sql, params))!.n;
|
|
|
|
describe("loading a band", () => {
|
|
it("derives every id from the word, so surfaces join their lemmas", async () => {
|
|
serve(shipped);
|
|
const { ensureBands } = await loader();
|
|
await ensureBands(db, 0);
|
|
|
|
expect(await count("SELECT count(*) AS n FROM lemma")).toBe(shipped.bands[0]!.lemmas);
|
|
const rice = await db.get<{ id: number }>("SELECT id FROM lemma WHERE headword = '밥' AND pos = 'noun'");
|
|
expect(rice?.id).toBe(lemmaId("밥", "noun"));
|
|
expect(await count("SELECT count(*) AS n FROM surface s LEFT JOIN lemma l ON l.id = s.lemma_id WHERE l.id IS NULL")).toBe(0);
|
|
});
|
|
|
|
it("does not load a band twice", async () => {
|
|
serve(shipped);
|
|
const { ensureBands } = await loader();
|
|
expect(await ensureBands(db, 0)).toEqual([0]);
|
|
expect(await ensureBands(db, 0)).toEqual([]);
|
|
});
|
|
});
|
|
|
|
describe("a rebuilt dictionary", () => {
|
|
it("reloads, and every card and custom word still names its word", async () => {
|
|
serve(shipped);
|
|
let dict = await loader();
|
|
await dict.ensureBands(db, 0);
|
|
const rice = lemmaId("밥", "noun");
|
|
await editCard(db, rice, markKnown(100));
|
|
const custom = await editAddCustomWord(db, { headword: "던전돌", gloss: "dungeon stone", pos: "noun" });
|
|
|
|
// A rebuild: band 0's file hash changes.
|
|
const rebuilt = structuredClone(shipped);
|
|
rebuilt.bands[0]!.sha256 = "f".repeat(64);
|
|
serve(rebuilt);
|
|
dict = await loader();
|
|
await db.run("DELETE FROM lemma WHERE headword = '물'"); // prove the reload really re-inserts
|
|
expect(await dict.ensureBands(db, 0)).toEqual([0]);
|
|
|
|
expect(await count("SELECT count(*) AS n FROM lemma WHERE headword = '물'")).toBeGreaterThan(0);
|
|
const card = await db.get<{ headword: string }>(
|
|
"SELECT l.headword FROM card c JOIN lemma l ON l.id = c.lemma_id WHERE c.lemma_id = ?",
|
|
[rice],
|
|
);
|
|
expect(card?.headword).toBe("밥");
|
|
const mine = await db.get<{ headword: string; source: string }>(
|
|
"SELECT headword, source FROM lemma WHERE id = ?",
|
|
[custom.lemmaId],
|
|
);
|
|
expect(mine).toEqual({ headword: "던전돌", source: "custom" });
|
|
});
|
|
});
|