feat(sync): tombstones, the wire protocol, and the client loop

Row-level last-write-wins on updated_at, cursor-based on a server-assigned
change_seq. The schema was built for this in step 1, so the work here is
the three things it did not yet have.

Tombstones (migration 4). Row-level sync cannot express a delete: with the
row gone there is nothing to compare timestamps against, so the other
device pushes its still-live copy back and the row silently returns. Every
delete path now writes a tombstone inside the same transaction.

The wire format lives in shared/sync-protocol.mjs and is imported by both
sides, so there is one definition rather than two that drift. It carries
the syncable-meta allowlist, which is the load-bearing part: meta mixes the
learner's preferences with bookkeeping that describes one install, and
replicating dict.loadedBands would tell a phone that had loaded bands 0-2
it holds every row the desktop has — the word rail would then fail to find
words it believes are present.

The sync loop pushes first, then pages the pull. Two details it would be
easy to get wrong, both commented at their site:

- The pull cursor advances only as rows are applied, never from the push
  response. The server's newest change_seq includes rows this device has
  not seen; adopting it skips them permanently, and nothing ever asks for
  that range again.
- Pulled rows advance sync.pushedAt too, bounded by the instant the sync
  started. Otherwise they look like local edits and get pushed straight
  back, and an edit made during the sync is not swept up with them.

Seeded rows carry updated_at = 0, so a fresh device is never dirty and can
never win a conflict — the artifact's clobbering bug stays unrepresentable.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
MechaCat02
2026-09-08 19:57:32 +02:00
parent f49ed388f4
commit f447f881c0
8 changed files with 643 additions and 8 deletions

View File

@@ -116,6 +116,23 @@ export const MIGRATIONS: Migration[] = [
);
`,
},
{
id: 4,
name: "tombstones — row-level sync cannot propagate a delete without them",
sql: /* sql */ `
-- A deleted row leaves nothing behind to compare timestamps against, so
-- last-write-wins would silently resurrect it from the other device on
-- the next pull. The tombstone is the delete, expressed as a row.
CREATE TABLE IF NOT EXISTS tombstone (
tbl TEXT NOT NULL, -- 'card' | 'chat' | 'peek' | ...
pk TEXT NOT NULL, -- primary key, as text
updated_at INTEGER NOT NULL DEFAULT 0,
PRIMARY KEY (tbl, pk)
) WITHOUT ROWID;
CREATE INDEX IF NOT EXISTS tombstone_updated ON tombstone(updated_at);
`,
},
];
export const SCHEMA_VERSION = MIGRATIONS[MIGRATIONS.length - 1]!.id;

View File

@@ -34,6 +34,22 @@ import type { Card, Grade } from "@lib/srs.js";
/** Wall-clock milliseconds, for stamping a genuine user edit. */
export const now = (): number => Date.now();
/**
* Record that a row was deleted.
*
* A delete leaves nothing for last-write-wins to compare against, so without
* this the other device would push its still-live copy back and the row
* would silently return. Always written inside the same transaction as the
* delete it describes.
*/
async function tombstone(db: Db, tbl: string, pk: string | number): Promise<void> {
await db.run(
`INSERT INTO tombstone (tbl, pk, updated_at) VALUES (?, ?, ?)
ON CONFLICT(tbl, pk) DO UPDATE SET updated_at = excluded.updated_at`,
[tbl, String(pk), now()],
);
}
/* ═════════════════════════════════════════════════════════════════════
SEED WRITES — defaults, first-run state, anything the user did not do.
None of these may set updated_at; the column default of 0 is the point.
@@ -88,7 +104,10 @@ export async function editCard(db: Db, lemmaId: number, card: Card): Promise<voi
/** Forgetting a card back to new, from the vocabulary tab. */
export async function editCardReset(db: Db, lemmaId: number): Promise<void> {
await db.run("DELETE FROM card WHERE lemma_id = ?", [lemmaId]);
await db.tx(async (tx) => {
await tx.run("DELETE FROM card WHERE lemma_id = ?", [lemmaId]);
await tombstone(tx, "card", lemmaId);
});
}
/** The tutor's ::progress read, or the learner moving the unit by hand. */
@@ -141,15 +160,27 @@ export async function editChatTurn(db: Db, role: string, body: string): Promise<
/** Wipe the transcript. Progress and cards are untouched. */
export async function editChatClear(db: Db): Promise<void> {
await db.run("DELETE FROM chat");
await db.tx(async (tx) => {
const rows = await tx.all<{ id: number }>("SELECT id FROM chat");
await tx.run("DELETE FROM chat");
for (const r of rows) await tombstone(tx, "chat", r.id);
});
}
/** Trim the transcript. The artifact kept the last 26 turns. */
export async function editChatTrim(db: Db, keep: number): Promise<void> {
await db.run(
`DELETE FROM chat WHERE id NOT IN (SELECT id FROM chat ORDER BY id DESC LIMIT ?)`,
[keep],
);
await db.tx(async (tx) => {
const doomed = await tx.all<{ id: number }>(
`SELECT id FROM chat WHERE id NOT IN (SELECT id FROM chat ORDER BY id DESC LIMIT ?)`,
[keep],
);
if (!doomed.length) return;
await tx.run(
`DELETE FROM chat WHERE id NOT IN (SELECT id FROM chat ORDER BY id DESC LIMIT ?)`,
[keep],
);
for (const r of doomed) await tombstone(tx, "chat", r.id);
});
}
/** Study happened today: reviews, correct answers, drill answers. */
@@ -272,6 +303,7 @@ export async function editRemoveCustomWord(db: Db, lemmaId: number): Promise<voi
await tx.run("DELETE FROM card WHERE lemma_id = ?", [lemmaId]);
await tx.run("DELETE FROM surface WHERE lemma_id = ?", [lemmaId]);
await tx.run("DELETE FROM lemma WHERE id = ?", [lemmaId]);
await tombstone(tx, "card", lemmaId);
});
}
@@ -288,10 +320,24 @@ export type ResetScope = "roadmap" | "everything";
export async function editReset(db: Db, scope: ResetScope): Promise<void> {
await db.tx(async (tx) => {
// Tombstone before deleting, while the keys are still readable — a reset
// must propagate, or the next pull restores everything it just erased.
for (const r of await tx.all<{ unit_id: string }>("SELECT unit_id FROM progress"))
await tombstone(tx, "progress", r.unit_id);
for (const r of await tx.all<{ id: number }>("SELECT id FROM chat"))
await tombstone(tx, "chat", r.id);
await tx.run("DELETE FROM progress");
await tx.run("DELETE FROM chat");
if (scope === "everything") {
for (const r of await tx.all<{ lemma_id: number }>("SELECT lemma_id FROM card"))
await tombstone(tx, "card", r.lemma_id);
for (const r of await tx.all<{ day: number }>("SELECT day FROM study_log"))
await tombstone(tx, "study_log", r.day);
for (const r of await tx.all<{ form: string }>("SELECT form FROM peek"))
await tombstone(tx, "peek", r.form);
await tx.run("DELETE FROM card");
await tx.run("DELETE FROM study_log");
await tx.run("DELETE FROM peek");

View File

@@ -0,0 +1,43 @@
/* Where the Pi is, if there is one.
Kept in `meta` under keys the sync allowlist excludes, because this is
device-local and one of the two values is a secret. Syncing a token to
every device through the very endpoint it authenticates would be
circular, and syncing a base URL would break a device on a different
network.
With nothing configured the app uses the local stub and never touches the
network — which is the whole offline-first guarantee, expressed as a
default rather than a mode. */
import type { Db } from "../db/types.js";
import { editMeta } from "../db/writes.js";
const URL_KEY = "server.url";
const TOKEN_KEY = "server.token";
export interface ServerConfig {
baseUrl: string;
token: string;
}
export async function readServerConfig(db: Db): Promise<ServerConfig | null> {
const rows = await db.all<{ k: string; v: string }>(
"SELECT k, v FROM meta WHERE k IN (?, ?)",
[URL_KEY, TOKEN_KEY],
);
const map = new Map(rows.map((r) => [r.k, r.v]));
const baseUrl = (map.get(URL_KEY) ?? "").trim();
const token = (map.get(TOKEN_KEY) ?? "").trim();
return baseUrl && token ? { baseUrl, token } : null;
}
export async function writeServerConfig(db: Db, cfg: ServerConfig): Promise<void> {
await editMeta(db, URL_KEY, cfg.baseUrl.trim());
await editMeta(db, TOKEN_KEY, cfg.token.trim());
}
export async function clearServerConfig(db: Db): Promise<void> {
await editMeta(db, URL_KEY, "");
await editMeta(db, TOKEN_KEY, "");
}

View File

@@ -25,6 +25,13 @@ import { bandForUnit } from "@shared/bands.mjs";
import { ensureBands, loadManifest, recordProvenance, type DictManifest } from "../domain/dictionary.js";
import { initProgress, readProgress } from "../domain/progress.js";
import { seedKnownWords } from "../domain/seed-known.js";
import {
clearServerConfig,
readServerConfig,
writeServerConfig,
type ServerConfig,
} from "../domain/server-config.js";
import { trySync, type SyncResult } from "../sync/client.js";
import type { ProgressState } from "@lib/gate.js";
import type { FocusMode } from "../domain/gate.js";
@@ -89,6 +96,20 @@ export interface Store {
/** Bump to tell tabs that card or log data changed underneath them. */
revision: number;
invalidate: () => void;
/** The Pi, when one is configured. Null means fully local. */
server: ServerConfig | null;
setServer: (cfg: ServerConfig | null) => Promise<void>;
/** Last sync outcome, for the settings panel. Null until one has run. */
syncState: SyncState;
syncNow: () => Promise<void>;
}
export interface SyncState {
at: number | null;
result: SyncResult | null;
error: string | null;
running: boolean;
}
const StoreContext = createContext<Store | null>(null);
@@ -123,6 +144,13 @@ export function StoreProvider({
const [prefs, setPrefs] = useState<Prefs>(DEFAULT_PREFS);
const [revision, setRevision] = useState(0);
const [today, setToday] = useState(() => dayNumber());
const [server, setServerState] = useState<ServerConfig | null>(null);
const [syncState, setSyncState] = useState<SyncState>({
at: null,
result: null,
error: null,
running: false,
});
const started = useRef(false);
useEffect(() => {
@@ -159,6 +187,7 @@ export function StoreProvider({
setStore({ db, dbInfo: db.info, manifest: await loadManifest() });
setProgress(stored);
setPrefs(loaded);
setServerState(await readServerConfig(db));
setBoot({ phase: "ready", detail: "" });
} catch (err) {
const error = err instanceof Error ? err : new Error(String(err));
@@ -200,12 +229,86 @@ export function StoreProvider({
const invalidate = useCallback(() => setRevision((r) => r + 1), []);
const setServer = useCallback(
async (cfg: ServerConfig | null) => {
if (!store) return;
if (cfg) await writeServerConfig(store.db, cfg);
else await clearServerConfig(store.db);
setServerState(cfg);
},
[store],
);
/* Sync is opportunistic and never blocks anything. A failure is recorded
for the settings panel and otherwise ignored — the app has to stay
fully usable with the Pi unreachable. */
const running = useRef(false);
const syncNow = useCallback(async () => {
if (!store || !server || running.current) return;
running.current = true;
setSyncState((s) => ({ ...s, running: true }));
const result = await trySync(store.db, server);
running.current = false;
setSyncState({
at: Date.now(),
result,
error: result ? null : "Could not reach the server",
running: false,
});
if (result && (result.pulled > 0 || result.pushed > 0)) {
await readProgress(store.db).then(setProgress);
setRevision((r) => r + 1);
}
}, [server, store]);
/* Once on connect, then every few minutes. Deliberately not on every
write: a tutor turn should never wait on the network. */
useEffect(() => {
if (!store || !server) return;
void syncNow();
const timer = setInterval(() => void syncNow(), 5 * 60_000);
const onVisible = () => {
if (document.visibilityState === "visible") void syncNow();
};
document.addEventListener("visibilitychange", onVisible);
return () => {
clearInterval(timer);
document.removeEventListener("visibilitychange", onVisible);
};
}, [server, store, syncNow]);
const value = useMemo<Store | null>(
() =>
store && progress
? { ...store, progress, refreshProgress, prefs, setPref, today, revision, invalidate }
? {
...store,
progress,
refreshProgress,
prefs,
setPref,
today,
revision,
invalidate,
server,
setServer,
syncState,
syncNow,
}
: null,
[store, progress, refreshProgress, prefs, setPref, today, revision, invalidate],
[
store,
progress,
refreshProgress,
prefs,
setPref,
today,
revision,
invalidate,
server,
setServer,
syncState,
syncNow,
],
);
if (!value) return <>{fallback(boot)}</>;

258
app/src/sync/client.ts Normal file
View File

@@ -0,0 +1,258 @@
/* The sync loop: push what changed here, pull what changed there, apply.
Offline-first is not negotiable. Sync runs only when a server is
configured, and every failure is non-fatal — the app must stay completely
usable with the Pi unreachable, which is the acceptance criterion the
whole port was built around. Nothing in here may throw into the UI. */
import type { Db } from "../db/types.js";
import {
PAGE_SIZE,
SYNC_TABLES,
SYNC_TABLE_NAMES,
incomingWins,
isSyncableMetaKey,
} from "@shared/sync-protocol.mjs";
/** A row on the wire: its table, its columns, and when it last changed. */
export interface SyncRow {
tbl: string;
data: Record<string, string | number | null>;
updated_at: number;
/** Present and true when this row is a delete. */
deleted?: boolean;
}
export interface PushRequest {
rows: SyncRow[];
}
export interface PushResponse {
cursor: number;
}
export interface PullResponse {
rows: SyncRow[];
cursor: number;
more: boolean;
}
export interface SyncConfig {
baseUrl: string;
token: string;
}
export interface SyncResult {
pushed: number;
pulled: number;
cursor: number;
}
const CURSOR_KEY = "sync.cursor";
const PUSHED_KEY = "sync.pushedAt";
/* ── local bookkeeping ────────────────────────────────────────────────
These are device state, not user data. They are written with
updated_at = 0 and excluded from the syncable-meta allowlist, so they
never travel. */
async function readNum(db: Db, key: string): Promise<number> {
const row = await db.get<{ v: string }>("SELECT v FROM meta WHERE k = ?", [key]);
const n = row ? Number(row.v) : 0;
return Number.isFinite(n) ? n : 0;
}
async function writeNum(db: Db, key: string, value: number): Promise<void> {
await db.run("INSERT OR REPLACE INTO meta (k, v, updated_at) VALUES (?, ?, 0)", [
key,
String(value),
]);
}
/* ── collect ─────────────────────────────────────────────────────────── */
/** Everything edited on this device since the last successful push. */
export async function collectDirty(db: Db, pushedAt: number): Promise<SyncRow[]> {
const out: SyncRow[] = [];
for (const tbl of SYNC_TABLE_NAMES) {
const spec = SYNC_TABLES[tbl]!;
const cols = [...spec.cols, "updated_at"].join(", ");
const rows = await db.all<Record<string, string | number>>(
`SELECT ${cols} FROM ${tbl} WHERE updated_at > ? ORDER BY updated_at LIMIT ?`,
[pushedAt, PAGE_SIZE],
);
for (const row of rows) {
// The allowlist is applied at the point of collection, so a
// device-local meta key cannot leave this machine even by accident.
if (tbl === "meta" && !isSyncableMetaKey(String(row.k))) continue;
const { updated_at, ...data } = row;
out.push({ tbl, data, updated_at: Number(updated_at) });
}
}
const graves = await db.all<{ tbl: string; pk: string; updated_at: number }>(
"SELECT tbl, pk, updated_at FROM tombstone WHERE updated_at > ? ORDER BY updated_at LIMIT ?",
[pushedAt, PAGE_SIZE],
);
for (const g of graves) {
out.push({ tbl: g.tbl, data: { pk: g.pk }, updated_at: Number(g.updated_at), deleted: true });
}
return out;
}
/* ── apply ───────────────────────────────────────────────────────────── */
/** Apply one incoming row, last-write-wins against what is already here. */
async function applyRow(db: Db, row: SyncRow): Promise<boolean> {
const spec = SYNC_TABLES[row.tbl];
if (!spec) return false; // unknown table — a newer server; ignore rather than fail
if (row.deleted) {
const pk = String(row.data.pk);
const where = spec.pk.map((c) => `${c} = ?`).join(" AND ");
const parts = pk.split(" ");
const local = await db.get<{ updated_at: number }>(
`SELECT updated_at FROM ${row.tbl} WHERE ${where}`,
parts,
);
if (local && !incomingWins(row.updated_at, local.updated_at)) return false;
await db.tx(async (tx) => {
await tx.run(`DELETE FROM ${row.tbl} WHERE ${where}`, parts);
await tx.run(
`INSERT INTO tombstone (tbl, pk, updated_at) VALUES (?, ?, ?)
ON CONFLICT(tbl, pk) DO UPDATE SET updated_at = excluded.updated_at`,
[row.tbl, pk, row.updated_at],
);
});
return true;
}
if (row.tbl === "meta" && !isSyncableMetaKey(String(row.data.k))) return false;
const where = spec.pk.map((c) => `${c} = ?`).join(" AND ");
const keys = spec.pk.map((c) => row.data[c] as string | number);
const local = await db.get<{ updated_at: number }>(
`SELECT updated_at FROM ${row.tbl} WHERE ${where}`,
keys,
);
if (local && !incomingWins(row.updated_at, local.updated_at)) return false;
const cols = [...spec.cols, "updated_at"];
const holes = cols.map(() => "?").join(", ");
const values = [...spec.cols.map((c) => row.data[c] ?? null), row.updated_at];
await db.run(`INSERT OR REPLACE INTO ${row.tbl} (${cols.join(", ")}) VALUES (${holes})`, values);
return true;
}
/* ── transport ───────────────────────────────────────────────────────── */
async function call<T>(
cfg: SyncConfig,
path: string,
init: RequestInit,
signal?: AbortSignal,
): Promise<T> {
const res = await fetch(`${cfg.baseUrl.replace(/\/$/, "")}${path}`, {
...init,
signal,
headers: {
"content-type": "application/json",
authorization: `Bearer ${cfg.token}`,
...(init.headers ?? {}),
},
});
if (!res.ok) throw new Error(`sync ${path}: HTTP ${res.status}`);
return (await res.json()) as T;
}
/* ── the loop ────────────────────────────────────────────────────────── */
/**
* One full exchange. Push first so local edits are never lost to an
* incoming row that would have overwritten them; then pull, paging until
* the server has nothing newer.
*/
export async function syncOnce(
db: Db,
cfg: SyncConfig,
signal?: AbortSignal,
): Promise<SyncResult> {
// Anything edited after this instant stays dirty, whatever the pull
// brings back. That is what keeps an edit made *during* a sync from being
// marked clean by a remote row that happens to carry a later timestamp.
const startedAt = Date.now();
const pushedAt = await readNum(db, PUSHED_KEY);
const dirty = await collectDirty(db, pushedAt);
let watermark = pushedAt;
if (dirty.length) {
const body: PushRequest = { rows: dirty };
await call<PushResponse>(
cfg,
"/api/sync",
{ method: "POST", body: JSON.stringify(body) },
signal,
);
// Only advance once the server has them. A failed push throws before
// this line and the rows stay dirty, which is the safe direction.
watermark = dirty.reduce((m, r) => Math.max(m, r.updated_at), watermark);
await writeNum(db, PUSHED_KEY, watermark);
}
/* The pull cursor advances ONLY as rows are applied.
It must never be taken from the push response: the server's newest
change_seq includes rows this device has not seen, and adopting it
would skip them permanently — a silent, unrecoverable data loss, since
nothing would ever ask for that range again. */
let cursor = await readNum(db, CURSOR_KEY);
let pulled = 0;
for (;;) {
const page = await call<PullResponse>(
cfg,
`/api/sync?cursor=${cursor}`,
{ method: "GET" },
signal,
);
for (const row of page.rows) {
if (await applyRow(db, row)) pulled++;
// A row we just received is not a local edit. Marking it clean stops
// the next sync pushing it straight back. Bounded by startedAt so a
// concurrent local edit is never swept up with it.
if (row.updated_at <= startedAt) watermark = Math.max(watermark, row.updated_at);
}
cursor = page.cursor;
await writeNum(db, CURSOR_KEY, cursor);
if (!page.more) break;
}
if (watermark > pushedAt) await writeNum(db, PUSHED_KEY, watermark);
return { pushed: dirty.length, pulled, cursor };
}
/**
* Sync if configured, and swallow anything that goes wrong.
*
* Callers are UI code on a timer; a sync failure is a diagnostic, never an
* interruption. Returns null when there is nothing configured or the
* exchange failed.
*/
export async function trySync(
db: Db,
cfg: SyncConfig | null,
signal?: AbortSignal,
): Promise<SyncResult | null> {
if (!cfg?.baseUrl || !cfg.token) return null;
try {
return await syncOnce(db, cfg, signal);
} catch (err) {
console.warn("[sync] skipped:", err instanceof Error ? err.message : err);
return null;
}
}

View File

@@ -228,6 +228,87 @@ function About() {
);
}
/* The Pi, if there is one. Everything works without it; this is what turns
on the real tutor and syncing between devices. */
function ServerPanel() {
const { server, setServer, syncState, syncNow } = useStore();
const [url, setUrl] = useState(server?.baseUrl ?? "");
const [token, setToken] = useState(server?.token ?? "");
const ago =
syncState.at === null
? "never"
: `${Math.max(0, Math.round((Date.now() - syncState.at) / 1000))}s ago`;
return (
<div className="panel">
<div className="panel-h">
<h2></h2>
<span className="note">
{server ? "Connected — the real 선생님, and sync" : "Not set — everything stays on this device"}
</span>
</div>
<div className="panel-b set-grid">
<label className="set-row">
<span>Server URL</span>
<input
type="url"
value={url}
placeholder="https://hankan.example.com"
onChange={(e) => setUrl(e.target.value)}
/>
</label>
<label className="set-row">
<span>Token</span>
<input
type="password"
value={token}
placeholder="HANKAN_TOKEN"
onChange={(e) => setToken(e.target.value)}
/>
</label>
<div className="toolbar" style={{ gridColumn: "1 / -1" }}>
<button
className="btn primary"
disabled={!url.trim() || !token.trim()}
onClick={() => void setServer({ baseUrl: url.trim(), token: token.trim() })}
>
{server ? "Update" : "Connect"}
</button>
{server && (
<>
<button className="btn" disabled={syncState.running} onClick={() => void syncNow()}>
{syncState.running ? "Syncing…" : "Sync now"}
</button>
<button
className="btn"
onClick={() => {
setUrl("");
setToken("");
void setServer(null);
}}
>
Disconnect
</button>
</>
)}
</div>
{server && (
<p className="add-note" style={{ gridColumn: "1 / -1" }}>
{syncState.error
? `Last sync failed (${ago}) — ${syncState.error}. Your work is safe here and will go up when the server is reachable.`
: syncState.result
? `Last sync ${ago}: sent ${syncState.result.pushed}, received ${syncState.result.pulled}.`
: "Not synced yet."}
</p>
)}
</div>
</div>
);
}
export function TodayTab({ onGoTo }: { onGoTo: (tab: "lesson" | "sent") => void }) {
const { db, progress, prefs, today, revision } = useStore();
const { start } = useReview();
@@ -353,6 +434,7 @@ export function TodayTab({ onGoTo }: { onGoTo: (tab: "lesson" | "sent") => void
<Heatmap rows={log} today={today} />
<Settings />
<ServerPanel />
<About />
<DangerZone />
</>