Hankan/test/domain/block-leak.test.ts

/* The ::task block leaking into the message body.

   Reported from the app: the raw text "::task translate" and its five
   sentences appeared as prose above the exercise those same lines had
   already been rendered into.

   It is lib/blocks.js, not the model. parse() deletes each block with a
   regex whose terminator (?:\n::|$) is part of the match, so deleting one
   block also deletes the "::" that opens the next. That block is then no
   longer a block, and stays in the body as a bare word plus its rows. The
   16 Sep lib changed which block survives — it now strips every kind in
   turn instead of truncating at ::words — but not the defect.

   Order is what decides it. The stub tutor emits ::words before ::task and
   is therefore fine; a real model emitted ::task first. Nothing in the
   prompt requires either order, so this was always reachable.

   The fixture is a verbatim capture from gpt-oss-20b through the server. */

import { describe, it, expect } from "vitest";
import { readFileSync } from "node:fs";
import { parse } from "@lib/blocks.js";
import { parseMessage, proseOf, glossContent } from "@app/domain/gloss.js";

import type { ParsedMessage } from "@lib/blocks.js";

/** Narrow the Task union; only translate tasks appear in these fixtures. */
function translateItems(task: ParsedMessage["task"]): { q: string }[] {
  if (!task || task.type !== "translate") throw new Error(`expected a translate task, got ${task?.type}`);
  return task.items;
}

const CAPTURED = "::task translate\n\ub098 \uac00\ub2e4  \n\ub108 \uba39\ub2e4  \n\uc6b0\ub9ac \ub9c8\uc2dc\ub2e4  \n\uc774 \uc790\ub2e4  \n\uadf8 \ub098\ubb34 \uc11c\ub2e4  \n\n::words\n\ub098 | I, me (casual) | pron  \n\uac00\ub2e4 | to go | verb (plain)  \n\ub108 | you (casual) | pron  \n\uba39\ub2e4 | to eat | verb (plain)  \n\uc6b0\ub9ac | we, our | pron  \n\ub9c8\uc2dc\ub2e4 | to drink | verb (plain)  \n\uc774 | this | det  \n\uc790\ub2e4 | to sleep | verb (plain)  \n\uadf8 | that (near you) | det  \n\ub098\ubb34 | tree | noun  \n\uc11c\ub2e4 | to stand | verb (plain)  \n\n::progress 20";

describe("lib/blocks.js — the leak, pinned as it is", () => {
  it("leaves the de-coloned ::words block in body when ::task precedes ::words", () => {
    const r = parse(CAPTURED);
    // The blocks themselves parse correctly...
    expect(r.task?.type).toBe("translate");
    expect(translateItems(r.task)).toHaveLength(5);
    expect(r.words).toHaveLength(11);
    // ...and the body carries the words block, its "::" eaten by the task's
    // terminator: not even recognisable as markup any more.
    expect(r.body).toMatch(/^words\n/);
    expect(r.body).toContain("나 | I, me (casual) | pron");
    expect(r.body).not.toContain("::");
  });

  it("is fine in the other order, which is why the stub never showed it", () => {
    const swapped = "Here you go.\n\n::words\n나 | I | pron\n\n::task translate\n나 가다\n";
    expect(parse(swapped).body).not.toContain("::task");
  });
});

describe("parseMessage — the workaround", () => {
  it("keeps the blocks and drops the leaked markup", () => {
    const r = parseMessage(CAPTURED);
    expect(r.body).not.toContain("::");
    expect(r.body).not.toContain("나 가다");
    // Nothing the blocks needed was lost.
    expect(r.task?.type).toBe("translate");
    expect(translateItems(r.task)).toHaveLength(5);
    expect(r.words).toHaveLength(11);
    expect(r.progress?.score).toBe(20);
  });

  it("keeps prose that comes before the blocks", () => {
    const r = parseMessage("좋아요. Try these.\n\n::task translate\n나 가다\n\n::words\n나 | I | pron\n");
    expect(r.body).toBe("좋아요. Try these.");
    expect(translateItems(r.task)).toHaveLength(1);
  });

  it("still handles the order the stub uses", () => {
    const r = parseMessage("Here you go.\n\n::words\n나 | I | pron\n\n::task translate\n나 가다\n");
    expect(r.body).toBe("Here you go.");
    expect(translateItems(r.task)).toHaveLength(1);
  });

  it("leaves an ordinary message alone", () => {
    const r = parseMessage("좋아요!\n\nThat is the shape of it.");
    expect(r.body).toBe("좋아요!\n\nThat is the shape of it.");
  });
});

describe("proseOf", () => {
  it("does nothing without a directive", () => {
    expect(proseOf("a\n\nb")).toBe("a\n\nb");
  });

  it("drops every block, not merely from the first one on", () => {
    expect(proseOf("keep\n::task translate\ndrop\n::words\ndrop")).toBe("keep");
  });

  it("resumes the prose after a block closes — gloss blocks sit inline", () => {
    expect(proseOf("before\n::gloss\n물 | N | water |\n::\nafter")).toBe("before\nafter");
  });

  it("does not trip on a colon that is not at the start of a line", () => {
    expect(proseOf("see:: this")).toBe("see:: this");
  });

  it("ignores the trailing whitespace this model puts on every line", () => {
    // markdown hard line breaks: "::task translate  "
    expect(proseOf("keep\n::task translate  \n나 바다  \n::words  \n물 | water")).toBe("keep");
  });

  it("treats ::progress, which has no closing ::, as running to the end", () => {
    expect(proseOf("keep\n::progress 40 | going well")).toBe("keep");
  });
});

/* A verbatim capture of gpt-oss-20b answering "Start unit 1.1." against the
   real assembled prompt — the reply behind the second report of raw markup
   showing above the exercise.

   It differs from the first capture in the way that mattered: here ::gloss
   comes first, and removing it took the "::" off the ::task that followed,
   so what leaked was the bare word "task translate" with its five sentences
   under it. A rule that looked for lines beginning "::" could not see that,
   which is why the body is now read from the raw text instead. */
describe("gpt-oss-20b, unit 1.1 — the reported reply", () => {
  const RAW = readFileSync(
    new URL("../fixtures/gpt-oss-20b-unit-1.1.txt", import.meta.url),
    "utf8",
  );

  it("puts the blocks in the order that breaks lib/blocks.js", () => {
    const order = RAW.split("\n")
      .filter((l) => /^\s*::\w/.test(l))
      .map((l) => l.trim().split(/\s/)[0]);
    expect(order).toEqual(["::gloss", "::task", "::words", "::progress"]);
  });

  it("renders the exercise", () => {
    expect(parseMessage(RAW).task?.type).toBe("translate");
  });

  it("leaves no directive, and no de-colonised directive, in the prose", () => {
    const body = parseMessage(RAW).body;
    expect(body).not.toMatch(/(^|\n)\s*::/);
    // The exact shape of this leak: the marker stripped, the word left.
    expect(body).not.toMatch(/(^|\n)\s*(task|words|gloss|progress)\b/);
  });

  it("does not repeat the exercise as prose above it", () => {
    const parsed = parseMessage(RAW);
    const task = parsed.task;
    // Task is a union; only the non-match kinds carry `items`.
    const raw = task && "items" in task ? (task.items as { q?: string }[]) : [];
    const items = raw.map((i) => i.q).filter((q): q is string => Boolean(q));
    expect(items.length).toBeGreaterThan(1);

    /* Not "no item appears": this reply uses 나 바다 as a worked example in
       the prose AND as the first exercise line, which is legitimate. A leak
       drags in every line, so that is what to measure. */
    const echoed = items.filter((q) => parsed.body.includes(q));
    expect(echoed.length).toBeLessThan(items.length);
  });

  it("keeps the teaching prose that came before the blocks", () => {
    expect(parseMessage(RAW).body).toContain("Korean syllables are written as blocks");
  });
});

/* A gloss block the model forgot to close. Every local model tested does
   this, and lib/blocks.js then reads the following paragraph as gloss
   parts — two sentences of English rendered as Korean example text inside
   the card, at example-line size. */
describe("an unclosed ::gloss block", () => {
  const RAW = [
    "Here is an example.",
    "::gloss",
    "나 | S | I",
    "바다 | O | sea",
    "= I sea",
    "",
    "(Here 바다 is two blocks.)",
    "Now practice these.",
    "::task translate",
    "나 바다",
  ].join("\n");

  it("ends the block where its rows end", () => {
    expect(glossContent("나 | S | I\n= I sea\n\nprose here\nmore prose")).toBe("나 | S | I\n= I sea\n");
  });

  it("keeps only the real gloss rows in the card", () => {
    const gloss = parseMessage(RAW).gloss ?? [];
    expect(gloss).toHaveLength(1);
    expect(gloss[0]!.parts.map((p) => p.ko)).toEqual(["나", "바다"]);
    expect(gloss[0]!.en).toBe("I sea");
  });

  it("returns the stranded prose to the message", () => {
    const body = parseMessage(RAW).body;
    expect(body).toContain("Here is an example.");
    expect(body).toContain("(Here 바다 is two blocks.)");
    expect(body).toContain("Now practice these.");
    // and still no exercise text or markup
    expect(body).not.toContain("나 바다");
    expect(body).not.toMatch(/::/);
  });

  it("leaves a properly closed block exactly as lib parses it", () => {
    const closed = "::gloss\n물 | N | water\n= Water.\n::\nAfter.";
    const g = parseMessage(closed).gloss ?? [];
    expect(g).toHaveLength(1);
    expect(g[0]!.parts.map((p) => p.ko)).toEqual(["물"]);
    expect(parseMessage(closed).body).toBe("After.");
  });
});