Hankan/test/domain/marking.test.ts

/* Which word each mark is for — against three real gpt-oss-20b sessions:
   one that wrote ::result English prompt first, one that wrote no ::result
   at all and marked in prose, and one that fenced every block in ``` and put
   the right answer where the slip belongs. Captured verbatim through the
   server. */

import { describe, it, expect, beforeAll, afterAll, vi } from "vitest";
import { readFileSync } from "node:fs";
import type { Db } from "@app/db/types.js";
import { parseMessage } from "@app/domain/gloss.js";
import { readMarks, readResults } from "@app/domain/marking.js";
import { expectedFor } from "@app/domain/letters.js";
import { applyReply } from "@app/domain/turn.js";
import { readEvidence } from "@app/domain/evidence.js";
import { isExerciseAnswer, noteAnswer, readProgress } from "@app/domain/progress.js";
import { dictionaryDb } from "../helpers/dict-db.js";

const round = JSON.parse(
  readFileSync(new URL("../fixtures/gpt-oss-20b-recall-round.json", import.meta.url), "utf8"),
) as { answered: string; answer: string; marking: string };

const answered = parseMessage(round.answered);
const marking = parseMessage(round.marking);

describe("reading the marks", () => {
  it("attaches the model's English-first rows to the words they mark", () => {
    expect(readResults(marking.results, answered)).toEqual([
      { item: "나", ok: true, mistakenFor: "" },
      { item: "너", ok: true, mistakenFor: "" },
      { item: "우리", ok: false, mistakenFor: "우라" },
      { item: "거기", ok: true, mistakenFor: "" },
      { item: "하나", ok: true, mistakenFor: "" },
    ]);
  });

  it("leaves rows written as the prompt asks exactly as they are", () => {
    const rows = [
      { item: "닭", ok: true, mistakenFor: "" },
      { item: "여덟", ok: false, mistakenFor: "여덜" },
    ];
    expect(readResults(rows, answered)).toEqual(rows);
  });

  it("takes the word out of quotation marks, and what he wrote out of an arrow", () => {
    expect(readResults([{ item: "“우리”", ok: false, mistakenFor: "우라 → 우리" }], null)).toEqual([
      { item: "우리", ok: false, mistakenFor: "우라" },
    ]);
  });

  it("does not guess which word of a sentence a mark is for", () => {
    const rows = [{ item: "머리 아파", ok: false, mistakenFor: "read 머리 as leg" }];
    expect(readResults(rows, answered)).toEqual(rows);
  });

  it("does not attach an English row that is not one of the exercise's prompts", () => {
    const rows = [{ item: "the sea", ok: false, mistakenFor: "" }];
    expect(readResults(rows, answered)).toEqual(rows);
  });
});

describe("the round, recorded", () => {
  let db: Db;
  beforeAll(async () => {
    db = await dictionaryDb(1);
  });
  afterAll(async () => {
    vi.unstubAllGlobals();
    await db.close();
  });

  it("puts every mark from the live round into the evidence", async () => {
    const applied = await applyReply(db, marking, { lookups: [], today: 20_000, answered });
    expect(applied.results?.recorded).toEqual(["나", "너", "우리", "거기", "하나"]);
    expect(applied.results?.ignored).toEqual([]);

    const evidence = await readEvidence(db, ["우리", "나"]);
    expect(evidence.get("우리")).toMatchObject({ ok: 0, wrong: 1 });
    expect(evidence.get("나")).toMatchObject({ ok: 1, wrong: 0 });
    expect(await db.get("SELECT mistook FROM confusion WHERE word = '우리'")).toEqual({ mistook: "우라" });
  });
});

/* The second session: no ::result at all, marks in prose, and a ::progress
   line on messages that answered nothing. The whole transcript, verbatim. */
const session = JSON.parse(
  readFileSync(new URL("../fixtures/gpt-oss-20b-prose-marks.json", import.meta.url), "utf8"),
) as { turns: { role: "user" | "assistant"; body: string }[] };
const turn = (i: number) => parseMessage(session.turns[i]!.body);

describe("marks written in prose", () => {
  it("reads ✓ and ✗ lines through the prompts of the exercise he answered", () => {
    expect(turn(3).results).toBeNull();
    expect(readMarks(turn(3), turn(1))).toEqual([
      { item: "나", ok: true, mistakenFor: "" },
      { item: "너", ok: true, mistakenFor: "" },
      { item: "이", ok: true, mistakenFor: "" },
      { item: "그", ok: true, mistakenFor: "" },
      { item: "저", ok: false, mistakenFor: "나" },
    ]);
  });

  it("tells the prompt a line is about from a shorter one inside it", () => {
    // "I (humble)" contains "I"; the longer prompt is the one meant.
    expect(readMarks(turn(7), turn(5))).toEqual([
      { item: "바다", ok: true, mistakenFor: "" },
      { item: "다리", ok: false, mistakenFor: "나" },
      { item: "저", ok: false, mistakenFor: "나" },
      { item: "너", ok: true, mistakenFor: "" },
      { item: "하나", ok: true, mistakenFor: "" },
    ]);
  });

  it("marks nothing the answered exercise did not ask for", () => {
    const stray = parseMessage("✓ 바다 | the sea\n✗ 소리 | sound → 소라");
    expect(readMarks(stray, turn(1))).toBeNull();
  });

  it("prefers the ::result block when there is one", () => {
    expect(readMarks(marking, answered)?.map((r) => r.item)).toEqual(["나", "너", "우리", "거기", "하나"]);
  });
});

describe("the second session, replayed", () => {
  let db: Db;
  beforeAll(async () => {
    db = await dictionaryDb(1);
  });
  afterAll(async () => {
    vi.unstubAllGlobals();
    await db.close();
  });

  it("records every round, and moves progress only in reply to an answer", async () => {
    const stored: number[] = [];
    let exercise: ReturnType<typeof parseMessage> | null = null;
    for (let i = 1; i < session.turns.length; i++) {
      const { role, body } = session.turns[i]!;
      if (role === "user") {
        await noteAnswer(db, await readProgress(db), body);
        continue;
      }
      const reply = parseMessage(body);
      const asked = session.turns[i - 1]!;
      await applyReply(db, reply, {
        lookups: [],
        today: 20_000,
        answered: exercise,
        afterAnswer: asked.role === "user" && isExerciseAnswer(asked.body),
      });
      if (reply.task) exercise = reply;
      stored.push((await readProgress(db)).confidence["1.1"] ?? 0);
    }
    // The model reported 80, 60, 65, 70, 75, 80. Clamped, and read only
    // after an answer: 80 → 25, 60 → 50, 70 → 70; the requests change nothing.
    expect(stored).toEqual([0, 25, 25, 50, 50, 70, 70, 70]);

    const evidence = await readEvidence(db, ["저", "거기", "나무"]);
    expect(evidence.get("저")).toMatchObject({ ok: 0, wrong: 2 });
    expect(evidence.get("거기")).toMatchObject({ wrong: 1 });
    expect(evidence.get("나무")).toMatchObject({ ok: 1 });
    expect(await db.get("SELECT mistook FROM confusion WHERE word = '거기'")).toEqual({ mistook: "여기" });
  });
});

/* The third session: every block fenced in ```, the right answer where the
   slip belongs, and qualified prompts — "I (humble)" beside "I". */
const fenced = JSON.parse(
  readFileSync(new URL("../fixtures/gpt-oss-20b-fenced-blocks.json", import.meta.url), "utf8"),
) as { turns: { role: "user" | "assistant"; body: string }[] };
const fencedTurn = (i: number) => parseMessage(fenced.turns[i]!.body);

describe("fenced blocks and qualified prompts", () => {
  it("reads the blocks inside the fences, and leaves no fence in the prose", () => {
    const m = fencedTurn(3);
    expect(m.results).toHaveLength(17);
    expect(m.task?.type).toBe("recall");
    expect(m.body).not.toContain("```");
  });

  it("tells I (humble) from I, and there from over there", () => {
    const { words } = fencedTurn(3);
    expect(expectedFor("I (humble)", words)).toBe("저");
    expect(expectedFor("I", words)).toBe("나");
    expect(expectedFor("There (there)", words)).toBe("거기");
  });

  it("drops every hint that is one of the message's own words", () => {
    const task = fencedTurn(3).task;
    if (task?.type !== "recall") throw new Error("expected recall");
    expect(task.items.map((i) => i.hint)).toEqual(["", "", "", "", "", ""]);
  });

  it("takes what he mistook a word for from what he wrote, when the mark gives the right answer", () => {
    expect(readMarks(fencedTurn(3), fencedTurn(1), fenced.turns[2]!.body)?.find((r) => r.item === "우리")).toEqual({
      item: "우리",
      ok: false,
      mistakenFor: "우라",
    });
  });
});

describe("the third session, replayed", () => {
  let db: Db;
  beforeAll(async () => {
    db = await dictionaryDb(1);
  });
  afterAll(async () => {
    vi.unstubAllGlobals();
    await db.close();
  });

  it("files every mark under the word it is for", async () => {
    let exercise: ReturnType<typeof parseMessage> | null = null;
    for (let i = 1; i < fenced.turns.length; i++) {
      const { role, body } = fenced.turns[i]!;
      if (role === "user") {
        await noteAnswer(db, await readProgress(db), body);
        continue;
      }
      const asked = fenced.turns[i - 1]!;
      const reply = parseMessage(body);
      await applyReply(db, reply, {
        lookups: [],
        today: 20_000,
        answered: exercise,
        answer: asked.body,
        afterAnswer: asked.role === "user" && isExerciseAnswer(asked.body),
      });
      if (reply.task) exercise = reply;
    }

    const evidence = await readEvidence(db, ["저", "나", "거기", "우리"]);
    // "I (humble) → 나" was marked wrong in round two and right in round
    // three; it used to be filed under 나, and dropped as a repeat.
    expect(evidence.get("저")).toMatchObject({ ok: 1, wrong: 1 });
    expect(evidence.get("나")).toMatchObject({ ok: 2, wrong: 0 });
    expect(evidence.get("거기")).toMatchObject({ ok: 1, wrong: 1 });
    expect(evidence.get("우리")).toMatchObject({ ok: 0, wrong: 1 });

    const confusions = await db.all<{ word: string; mistook: string }>(
      "SELECT word, mistook FROM confusion ORDER BY word",
    );
    expect(confusions).toEqual([
      { word: "거기", mistook: "여기" },
      { word: "우리", mistook: "우라" },
      { word: "저", mistook: "나" },
    ]);
  });
});