feat(gate): one resolver, the words he has met, and the audit through the port
The bundle's audit-gate.mjs measures the word gate against 54 real tutor
messages. Scored the same way, the port's gate was not the gate that was
measured:
port's allowed set 13 · 2 over-fires on words he had met
port's dictionary resolver 4 · 2 too weak: 마셔 → 마시다 again
lib/lexicon.js, DB behind it 7 · 2 the shipped gate
domain/resolver.ts makes lib/lexicon.js the one resolver for word taps and
the gate, built from the shipped data in the audit's order — roadmap words
first, so a scheduled form cannot inherit its stem's permission — with the
dictionary consulted only where lexicon.js has no route. That fallback is
what the port adds over the artifact: a word the curated data does not know
is still recognised as a real word.
The allowed set gains every word he has met (any card out of "new"), as
the shipped gate has it; the frequency band stays on top from Phase 2, as
PORT.md specifies. The words the tutor is TOLD it may use and the words it
is CHECKED against are one set, so it is never refused for a word it was
offered.
test/domain/gate-audit.test.ts reproduces the audit through the app's own
code — database, loader, resolver — and gets the shipped 149-word allowed
set and the same 7 and 2 messages, word for word.
Also: {{VARIETY}} and {{FOCUS}} take the artifact's wording and know the
fifth exercise type, recall; and the prompt's maintainer header, which
explains the placeholders, is no longer sent to the model.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
112
test/domain/gate-audit.test.ts
Normal file
112
test/domain/gate-audit.test.ts
Normal file
@@ -0,0 +1,112 @@
|
||||
/* audit-gate.mjs, reproduced through the PORT's own code.
|
||||
|
||||
run-checks.sh measures lib/ on its own. This measures what the app
|
||||
actually does: the allowed set built from its database, the resolver
|
||||
that falls back to its dictionary, parse() as the tutor tab sees it. If
|
||||
the two ever disagree, the port is not running the gate that was measured.
|
||||
|
||||
Numbers from audit-gate.mjs, and why they are what they are, are in that
|
||||
file: 7 of 41 earlier messages, 2 of 13 from the live 1.10 review. */
|
||||
|
||||
import { describe, it, expect, beforeAll, afterAll, vi } from "vitest";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { SqliteWasmDb } from "@app/db/sqlite-wasm-core.js";
|
||||
import { migrate } from "@app/db/migrate.js";
|
||||
import { editCard } from "@app/db/writes.js";
|
||||
import type { Db } from "@app/db/types.js";
|
||||
import { parse } from "@lib/blocks.js";
|
||||
import { proseIsKorean, scanTask, taskMaterial } from "@lib/gate.js";
|
||||
import { markKnown } from "@lib/srs.js";
|
||||
import { allowedSet, courseScaffold, gateFor, metWords, UNITS, unitOf } from "@app/domain/gate.js";
|
||||
import { headsFor } from "@app/domain/resolver.js";
|
||||
|
||||
const read = (f: string) => JSON.parse(readFileSync(new URL(`../../${f}`, import.meta.url), "utf8"));
|
||||
const DICT = new URL("../../app/public/dict/", import.meta.url);
|
||||
|
||||
interface Snapshot {
|
||||
road: { done: Record<string, number>; unit: string };
|
||||
deckWordsMet: string[];
|
||||
allowedWordsExpected: string[];
|
||||
}
|
||||
const snapshot = read("fixtures/progress-snapshot.json") as Snapshot;
|
||||
|
||||
let db: Db;
|
||||
|
||||
beforeAll(async () => {
|
||||
vi.stubGlobal("fetch", async (input: string) => {
|
||||
const name = String(input).split("/dict/")[1] ?? "";
|
||||
return new Response(readFileSync(new URL(name, DICT)));
|
||||
});
|
||||
db = await SqliteWasmDb.open({ memory: true });
|
||||
await migrate(db);
|
||||
const { ensureBands } = await import("@app/domain/dictionary.js");
|
||||
await ensureBands(db, 6);
|
||||
|
||||
// The student's state in the snapshot: the deck words he has met.
|
||||
for (const w of snapshot.deckWordsMet) {
|
||||
const lemma = await db.get<{ id: number }>(
|
||||
`SELECT id FROM lemma WHERE headword = ?
|
||||
ORDER BY unit_id IS NULL, source <> 'curated', id LIMIT 1`,
|
||||
[w],
|
||||
);
|
||||
if (lemma) await editCard(db, lemma.id, markKnown(20_000));
|
||||
}
|
||||
}, 60_000);
|
||||
|
||||
afterAll(async () => {
|
||||
vi.unstubAllGlobals();
|
||||
await db.close();
|
||||
});
|
||||
|
||||
const progress = () => ({
|
||||
current: snapshot.road.unit,
|
||||
done: Object.fromEntries(Object.keys(snapshot.road.done).map((k) => [k, true])),
|
||||
confidence: {},
|
||||
});
|
||||
|
||||
describe("the allowed set", () => {
|
||||
it("is the one the shipped app computed from this state — 149 words", async () => {
|
||||
const met = await metWords(db);
|
||||
const done = new Set(Object.keys(snapshot.road.done));
|
||||
const base = new Set([...UNITS.filter((u) => done.has(u.id)).flatMap((u) => u.words ?? []), ...met]);
|
||||
const expected = new Set(snapshot.allowedWordsExpected);
|
||||
expect([...expected].filter((w) => !base.has(w)), "missing here").toEqual([]);
|
||||
expect([...base].filter((w) => !expected.has(w)), "extra here").toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
async function audit(file: string): Promise<string[]> {
|
||||
const gate = gateFor({ progress: progress(), met: await metWords(db) });
|
||||
const allowed = allowedSet(gate);
|
||||
const scaffold = courseScaffold();
|
||||
const { messages } = read(file) as { messages: string[] };
|
||||
const flagged: string[] = [];
|
||||
|
||||
for (const [i, m] of messages.entries()) {
|
||||
const parsed = parse(m);
|
||||
const tokens = taskMaterial(parsed.task).join(" ").match(/[가-힣]+/g) ?? [];
|
||||
const heads = await headsFor(db, tokens);
|
||||
const found = scanTask(parsed, { allowed, scaffold, heads, unitOf }).map((f) => f.word);
|
||||
if (proseIsKorean(m)) found.push("«KO»");
|
||||
if (found.length) flagged.push(`#${i} ${found.join(" ")}`);
|
||||
}
|
||||
return flagged;
|
||||
}
|
||||
|
||||
describe("the audit, through the port", () => {
|
||||
it("flags 7 of the 41 earlier messages — the shipped gate's count", async () => {
|
||||
expect(await audit("fixtures/tutor-messages.json")).toEqual([
|
||||
"#10 마셔",
|
||||
"#14 «KO»",
|
||||
"#18 마셔",
|
||||
"#20 마셔",
|
||||
"#27 마셔",
|
||||
"#30 살 이야",
|
||||
"#32 살 이야",
|
||||
]);
|
||||
});
|
||||
|
||||
it("flags 2 of the 13 from the live 1.10 review — both 아파, from unit 3.2", async () => {
|
||||
expect(await audit("fixtures/tutor-messages-live.json")).toEqual(["#1 아파", "#3 아파"]);
|
||||
});
|
||||
});
|
||||
@@ -1,7 +1,8 @@
|
||||
/* The gate — the guarantee that the tutor cannot teach out of order.
|
||||
|
||||
These test the three refinements the app layers inside vocabQuery, plus
|
||||
the prompt assembly and the confidence clamp. */
|
||||
the prompt assembly. The clamp on the tutor's progress reports is tested
|
||||
against a real database in learner.test.ts. */
|
||||
|
||||
import { describe, it, expect } from "vitest";
|
||||
import { readFileSync } from "node:fs";
|
||||
@@ -16,7 +17,6 @@ import {
|
||||
focusLine,
|
||||
type BandQuery,
|
||||
} from "@app/domain/gate.js";
|
||||
import { MAX_DELTA_PER_TURN, READY_AT } from "@app/domain/progress.js";
|
||||
import { featureLevel, isReadableAt } from "@shared/phonology.mjs";
|
||||
import type { ProgressState } from "@lib/gate.js";
|
||||
|
||||
@@ -200,8 +200,10 @@ describe("the prompt", () => {
|
||||
const gate = gateFor({ progress: at("2.1"), bandQuery: bandOffering("학교") });
|
||||
const out = assemblePrompt({ template: SHIPPED, gate, recent: [], focus: "auto" });
|
||||
|
||||
// The header must survive intact, with its backticked mentions.
|
||||
expect(out).toContain("`{{GATE}}` is `renderGate()`");
|
||||
// The header paragraph is for whoever maintains the app, and is not
|
||||
// sent: the prompt the model reads starts below its --- rule.
|
||||
expect(out).not.toContain("Assembled per turn");
|
||||
expect(out.startsWith("You are 선생님")).toBe(true);
|
||||
|
||||
// And the gate must land after the profile and before the pre-flight
|
||||
// check, which is where the shipped file puts the slot.
|
||||
@@ -240,44 +242,15 @@ describe("the prompt", () => {
|
||||
expect(out).toContain(gate.forbidden.near[0]!);
|
||||
});
|
||||
|
||||
it("asks for a different exercise type than the last few", () => {
|
||||
it("asks for a type not used recently, naming all five", () => {
|
||||
const line = varietyLine(["translate", "match"]);
|
||||
expect(line).toContain("translate");
|
||||
expect(line.includes("build") || line.includes("choice")).toBe(true);
|
||||
expect(line).toContain("most recent first: match, translate.");
|
||||
expect(line).toContain("You have not used recall or build or choice recently");
|
||||
expect(line).toContain("recall is the one that proves memory");
|
||||
});
|
||||
|
||||
it("says nothing on auto focus", () => {
|
||||
it("says nothing on auto focus, and marks a chosen focus as his choice", () => {
|
||||
expect(focusLine("auto")).toBe("");
|
||||
expect(focusLine("manhwa")).not.toBe("");
|
||||
});
|
||||
});
|
||||
|
||||
describe("the confidence clamp", () => {
|
||||
/* The artifact wrote the model's number straight into the advancement
|
||||
gate, so one hallucinated ::progress 95 could skip a unit. */
|
||||
it("limits how far one turn may climb", () => {
|
||||
expect(MAX_DELTA_PER_TURN).toBeLessThan(READY_AT);
|
||||
const before = 20;
|
||||
const asked = 95;
|
||||
const stored = asked <= before ? asked : Math.min(asked, before + MAX_DELTA_PER_TURN);
|
||||
expect(stored).toBe(before + MAX_DELTA_PER_TURN);
|
||||
expect(stored).toBeLessThan(READY_AT);
|
||||
});
|
||||
|
||||
it("takes several honest turns to reach the advancement threshold", () => {
|
||||
let c = 0;
|
||||
let turns = 0;
|
||||
while (c < READY_AT && turns < 50) {
|
||||
c = Math.min(100, c + MAX_DELTA_PER_TURN);
|
||||
turns++;
|
||||
}
|
||||
expect(turns).toBeGreaterThan(5);
|
||||
});
|
||||
|
||||
it("honours a downward correction in full", () => {
|
||||
const before = 80;
|
||||
const asked = 30;
|
||||
const stored = asked <= before ? asked : Math.min(asked, before + MAX_DELTA_PER_TURN);
|
||||
expect(stored).toBe(30);
|
||||
expect(focusLine("manhwa")).toMatch(/^He has manually set the focus to this/);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user