Hankan/test/domain/block-leak.test.ts

/* Blocks leaking into the message body.

   Reported from the app, twice: raw exercise markup appeared as prose above
   the exercise it had already been rendered into.

   It is lib/blocks.js, not the model. parse() deletes each block with a
   regex whose terminator (?:\n::|$) is part of the match, so deleting one
   block also deletes the "::" that opens the next. That block is then no
   longer a block, and stays in the body as a bare word plus its rows. The
   16 Sep lib changed which block survives — it now strips every kind in
   turn instead of truncating at ::words — but not the defect.

   Order is what decides it. The stub tutor emits ::words before ::task and
   is therefore fine; a real model emitted ::task first. Nothing in the
   prompt requires either order, so this was always reachable.

   The fixture is a verbatim capture from gpt-oss-20b through the server. */

import { describe, it, expect } from "vitest";
import { readFileSync } from "node:fs";
import { parse } from "@lib/blocks.js";
import { parseMessage, proseOf, glossContent } from "@app/domain/gloss.js";

import type { ParsedMessage } from "@lib/blocks.js";

/** Narrow the Task union; only translate tasks appear in these fixtures. */
function translateItems(task: ParsedMessage["task"]): { q: string }[] {
  if (!task || task.type !== "translate") throw new Error(`expected a translate task, got ${task?.type}`);
  return task.items;
}

const CAPTURED = "::task translate\n\ub098 \uac00\ub2e4  \n\ub108 \uba39\ub2e4  \n\uc6b0\ub9ac \ub9c8\uc2dc\ub2e4  \n\uc774 \uc790\ub2e4  \n\uadf8 \ub098\ubb34 \uc11c\ub2e4  \n\n::words\n\ub098 | I, me (casual) | pron  \n\uac00\ub2e4 | to go | verb (plain)  \n\ub108 | you (casual) | pron  \n\uba39\ub2e4 | to eat | verb (plain)  \n\uc6b0\ub9ac | we, our | pron  \n\ub9c8\uc2dc\ub2e4 | to drink | verb (plain)  \n\uc774 | this | det  \n\uc790\ub2e4 | to sleep | verb (plain)  \n\uadf8 | that (near you) | det  \n\ub098\ubb34 | tree | noun  \n\uc11c\ub2e4 | to stand | verb (plain)  \n\n::progress 20";

describe("lib/blocks.js — the leak, pinned as it is", () => {
  it("leaves the de-coloned ::words block in body when ::task precedes ::words", () => {
    const r = parse(CAPTURED);
    // The blocks themselves parse correctly...
    expect(r.task?.type).toBe("translate");
    expect(translateItems(r.task)).toHaveLength(5);
    expect(r.words).toHaveLength(11);
    // ...and the body carries the words block, its "::" eaten by the task's
    // terminator: not even recognisable as markup any more.
    expect(r.body).toMatch(/^words\n/);
    expect(r.body).toContain("나 | I, me (casual) | pron");
    expect(r.body).not.toContain("::");
  });

  it("is fine in the other order, which is why the stub never showed it", () => {
    const swapped = "Here you go.\n\n::words\n나 | I | pron\n\n::task translate\n나 가다\n";
    expect(parse(swapped).body).not.toContain("::task");
  });
});

describe("parseMessage — the workaround", () => {
  it("keeps the blocks and drops the leaked markup", () => {
    const r = parseMessage(CAPTURED);
    expect(r.body).not.toContain("::");
    expect(r.body).not.toContain("나 가다");
    // Nothing the blocks needed was lost.
    expect(r.task?.type).toBe("translate");
    expect(translateItems(r.task)).toHaveLength(5);
    expect(r.words).toHaveLength(11);
    expect(r.progress?.score).toBe(20);
  });

  it("keeps prose that comes before the blocks", () => {
    const r = parseMessage("좋아요. Try these.\n\n::task translate\n나 가다\n\n::words\n나 | I | pron\n");
    expect(r.body).toBe("좋아요. Try these.");
    expect(translateItems(r.task)).toHaveLength(1);
  });

  it("still handles the order the stub uses", () => {
    const r = parseMessage("Here you go.\n\n::words\n나 | I | pron\n\n::task translate\n나 가다\n");
    expect(r.body).toBe("Here you go.");
    expect(translateItems(r.task)).toHaveLength(1);
  });

  it("leaves an ordinary message alone", () => {
    const r = parseMessage("좋아요!\n\nThat is the shape of it.");
    expect(r.body).toBe("좋아요!\n\nThat is the shape of it.");
  });
});

describe("proseOf", () => {
  it("does nothing without a directive", () => {
    expect(proseOf("a\n\nb")).toBe("a\n\nb");
  });

  it("drops every block, not merely from the first one on", () => {
    expect(proseOf("keep\n::task translate\ndrop\n::words\ndrop | x")).toBe("keep");
  });

  it("resumes the prose after a block closes — gloss blocks sit inline", () => {
    expect(proseOf("before\n::gloss\n물 | N | water |\n::\nafter")).toBe("before\nafter");
  });

  it("does not trip on a colon that is not at the start of a line", () => {
    expect(proseOf("see:: this")).toBe("see:: this");
  });

  it("ignores the trailing whitespace this model puts on every line", () => {
    // markdown hard line breaks: "::task translate  "
    expect(proseOf("keep\n::task translate  \n나 바다  \n::words  \n물 | water")).toBe("keep");
  });

  it("treats ::progress, which has no closing ::, as running to the end", () => {
    expect(proseOf("keep\n::progress 40 | going well")).toBe("keep");
  });
});

/* A verbatim capture of gpt-oss-20b answering "Start unit 1.1." against the
   real assembled prompt — the reply behind the second report of raw markup
   showing above the exercise.

   It differs from the first capture in the way that mattered: here ::gloss
   comes first, and removing it took the "::" off the ::task that followed,
   so what leaked was the bare word "task translate" with its five sentences
   under it. A rule that looked for lines beginning "::" could not see that,
   which is why the body is now read from the raw text instead. */
describe("gpt-oss-20b, unit 1.1 — the reported reply", () => {
  const RAW = readFileSync(
    new URL("../fixtures/gpt-oss-20b-unit-1.1.txt", import.meta.url),
    "utf8",
  );

  it("puts the blocks in the order that breaks lib/blocks.js", () => {
    const order = RAW.split("\n")
      .filter((l) => /^\s*::\w/.test(l))
      .map((l) => l.trim().split(/\s/)[0]);
    expect(order).toEqual(["::gloss", "::task", "::words", "::progress"]);
  });

  it("renders the exercise", () => {
    expect(parseMessage(RAW).task?.type).toBe("translate");
  });

  it("leaves no directive, and no de-colonised directive, in the prose", () => {
    const body = parseMessage(RAW).body;
    expect(body).not.toMatch(/(^|\n)\s*::/);
    // The exact shape of this leak: the marker stripped, the word left.
    expect(body).not.toMatch(/(^|\n)\s*(task|words|gloss|progress)\b/);
  });

  it("does not repeat the exercise as prose above it", () => {
    const parsed = parseMessage(RAW);
    const task = parsed.task;
    // Task is a union; only the non-match kinds carry `items`.
    const raw = task && "items" in task ? (task.items as { q?: string }[]) : [];
    const items = raw.map((i) => i.q).filter((q): q is string => Boolean(q));
    expect(items.length).toBeGreaterThan(1);

    /* Not "no item appears": this reply uses 나 바다 as a worked example in
       the prose AND as the first exercise line, which is legitimate. A leak
       drags in every line, so that is what to measure. */
    const echoed = items.filter((q) => parsed.body.includes(q));
    expect(echoed.length).toBeLessThan(items.length);
  });

  it("keeps the teaching prose that came before the blocks", () => {
    expect(parseMessage(RAW).body).toContain("Korean syllables are written as blocks");
  });
});

/* A gloss block the model forgot to close. Every local model tested does
   this, and lib/blocks.js then reads the following paragraph as gloss
   parts — two sentences of English rendered as Korean example text inside
   the card, at example-line size. */
describe("an unclosed ::gloss block", () => {
  const RAW = [
    "Here is an example.",
    "::gloss",
    "나 | S | I",
    "바다 | O | sea",
    "= I sea",
    "",
    "(Here 바다 is two blocks.)",
    "Now practice these.",
    "::task translate",
    "나 바다",
  ].join("\n");

  it("lib alone reads the following prose as gloss parts — pinned as it is", () => {
    const parts = (parse(RAW).gloss ?? []).flatMap((g) => g.parts.map((p) => p.ko));
    expect(parts).toContain("Now practice these.");
  });

  it("ends the block where its rows end", () => {
    expect(glossContent("나 | S | I\n= I sea\n\nprose here\nmore prose")).toBe("나 | S | I\n= I sea\n");
  });

  it("keeps only the real gloss rows in the card", () => {
    const gloss = parseMessage(RAW).gloss ?? [];
    expect(gloss).toHaveLength(1);
    expect(gloss[0]!.parts.map((p) => p.ko)).toEqual(["나", "바다"]);
    expect(gloss[0]!.en).toBe("I sea");
  });

  it("returns the stranded prose to the message", () => {
    const body = parseMessage(RAW).body;
    expect(body).toContain("Here is an example.");
    expect(body).toContain("(Here 바다 is two blocks.)");
    expect(body).toContain("Now practice these.");
    // and still no exercise text or markup
    expect(body).not.toContain("나 바다");
    expect(body).not.toMatch(/::/);
  });

  it("finds a second gloss block written straight after the first", () => {
    // Both blocks share one terminator, so lib alone never sees the second.
    const src = "::gloss\n나 | S | I\n= I.\n::gloss\n밥 | O | rice\n= Rice.\n::";
    expect(parse(src).gloss).toHaveLength(1);
    expect(parseMessage(src).gloss!.map((g) => g.en)).toEqual(["I.", "Rice."]);
    expect(parseMessage(src).body).toBe("");
  });

  it("leaves a properly closed block exactly as lib parses it", () => {
    const closed = "::gloss\n물 | N | water\n= Water.\n::\nAfter.";
    const g = parseMessage(closed).gloss ?? [];
    expect(g).toHaveLength(1);
    expect(g[0]!.parts.map((p) => p.ko)).toEqual(["물"]);
    expect(parseMessage(closed).body).toBe("After.");
  });
});

/* A recall round with gpt-oss-20b, captured verbatim through the server:
   the exercise, his answer, and the marking. The model closed none of its
   blocks, and each unclosed block swallowed what followed it. */
describe("gpt-oss-20b, a recall round — blocks it never closed", () => {
  const round = JSON.parse(
    readFileSync(new URL("../fixtures/gpt-oss-20b-recall-round.json", import.meta.url), "utf8"),
  ) as { answered: string; marking: string; reply: string };

  const recall = (task: ParsedMessage["task"]) => {
    if (!task || task.type !== "recall") throw new Error(`expected a recall task, got ${task?.type}`);
    return task.items;
  };

  it("shows the feedback written after ::result, instead of reading it as marks", () => {
    const m = parseMessage(round.marking);
    expect(m.body).toContain("You wrote an extra line");
    expect(m.body).toContain("your score for this unit is about");
    expect(m.body).not.toMatch(/\n{3,}|---$/);
    expect(m.results?.map((r) => r.item)).toEqual(["I", "you", "we", "there", "one"]);
  });

  it("never makes a markdown rule an exercise item", () => {
    expect(recall(parseMessage(round.answered).task).map((i) => i.q)).toEqual([
      "I",
      "you",
      "we",
      "there",
      "one",
    ]);
    expect(parse(round.answered).task?.rows).toContain("---"); // lib, unaided
  });

  it("drops a recall hint that is the answer itself", () => {
    const items = recall(parseMessage(round.marking).task);
    expect(items.map((i) => [i.q, i.hint])).toEqual([
      ["two", ""],
      ["that", ""],
      ["I (humble)", ""],
    ]);
  });

  it("keeps only the Korean line of a translate row", () => {
    expect(translateItems(parseMessage(round.reply).task).map((i) => i.q)).toEqual([
      "나",
      "너",
      "우리",
      "거기",
      "하나",
    ]);
  });

  it("keeps the closing remark after ::words as prose", () => {
    expect(parseMessage(round.answered).body).toContain("You have just started");
  });
});