/* The ::task block leaking into the message body. Reported from the app: the raw text "::task translate" and its five sentences appeared as prose above the exercise those same lines had already been rendered into. It is lib/blocks.js, not the model. parse() deletes each block with a regex whose terminator (?:\n::|$) is part of the match, so deleting one block also deletes the "::" that opens the next. That block is then no longer a block, and stays in the body as a bare word plus its rows. The 16 Sep lib changed which block survives — it now strips every kind in turn instead of truncating at ::words — but not the defect. Order is what decides it. The stub tutor emits ::words before ::task and is therefore fine; a real model emitted ::task first. Nothing in the prompt requires either order, so this was always reachable. The fixture is a verbatim capture from gpt-oss-20b through the server. */ import { describe, it, expect } from "vitest"; import { readFileSync } from "node:fs"; import { parse } from "@lib/blocks.js"; import { parseMessage, proseOf, glossContent } from "@app/domain/gloss.js"; import type { ParsedMessage } from "@lib/blocks.js"; /** Narrow the Task union; only translate tasks appear in these fixtures. */ function translateItems(task: ParsedMessage["task"]): { q: string }[] { if (!task || task.type !== "translate") throw new Error(`expected a translate task, got ${task?.type}`); return task.items; } const CAPTURED = "::task translate\n\ub098 \uac00\ub2e4 \n\ub108 \uba39\ub2e4 \n\uc6b0\ub9ac \ub9c8\uc2dc\ub2e4 \n\uc774 \uc790\ub2e4 \n\uadf8 \ub098\ubb34 \uc11c\ub2e4 \n\n::words\n\ub098 | I, me (casual) | pron \n\uac00\ub2e4 | to go | verb (plain) \n\ub108 | you (casual) | pron \n\uba39\ub2e4 | to eat | verb (plain) \n\uc6b0\ub9ac | we, our | pron \n\ub9c8\uc2dc\ub2e4 | to drink | verb (plain) \n\uc774 | this | det \n\uc790\ub2e4 | to sleep | verb (plain) \n\uadf8 | that (near you) | det \n\ub098\ubb34 | tree | noun \n\uc11c\ub2e4 | to stand | verb (plain) \n\n::progress 20"; describe("lib/blocks.js — the leak, pinned as it is", () => { it("leaves the de-coloned ::words block in body when ::task precedes ::words", () => { const r = parse(CAPTURED); // The blocks themselves parse correctly... expect(r.task?.type).toBe("translate"); expect(translateItems(r.task)).toHaveLength(5); expect(r.words).toHaveLength(11); // ...and the body carries the words block, its "::" eaten by the task's // terminator: not even recognisable as markup any more. expect(r.body).toMatch(/^words\n/); expect(r.body).toContain("나 | I, me (casual) | pron"); expect(r.body).not.toContain("::"); }); it("is fine in the other order, which is why the stub never showed it", () => { const swapped = "Here you go.\n\n::words\n나 | I | pron\n\n::task translate\n나 가다\n"; expect(parse(swapped).body).not.toContain("::task"); }); }); describe("parseMessage — the workaround", () => { it("keeps the blocks and drops the leaked markup", () => { const r = parseMessage(CAPTURED); expect(r.body).not.toContain("::"); expect(r.body).not.toContain("나 가다"); // Nothing the blocks needed was lost. expect(r.task?.type).toBe("translate"); expect(translateItems(r.task)).toHaveLength(5); expect(r.words).toHaveLength(11); expect(r.progress?.score).toBe(20); }); it("keeps prose that comes before the blocks", () => { const r = parseMessage("좋아요. Try these.\n\n::task translate\n나 가다\n\n::words\n나 | I | pron\n"); expect(r.body).toBe("좋아요. Try these."); expect(translateItems(r.task)).toHaveLength(1); }); it("still handles the order the stub uses", () => { const r = parseMessage("Here you go.\n\n::words\n나 | I | pron\n\n::task translate\n나 가다\n"); expect(r.body).toBe("Here you go."); expect(translateItems(r.task)).toHaveLength(1); }); it("leaves an ordinary message alone", () => { const r = parseMessage("좋아요!\n\nThat is the shape of it."); expect(r.body).toBe("좋아요!\n\nThat is the shape of it."); }); }); describe("proseOf", () => { it("does nothing without a directive", () => { expect(proseOf("a\n\nb")).toBe("a\n\nb"); }); it("drops every block, not merely from the first one on", () => { expect(proseOf("keep\n::task translate\ndrop\n::words\ndrop")).toBe("keep"); }); it("resumes the prose after a block closes — gloss blocks sit inline", () => { expect(proseOf("before\n::gloss\n물 | N | water |\n::\nafter")).toBe("before\nafter"); }); it("does not trip on a colon that is not at the start of a line", () => { expect(proseOf("see:: this")).toBe("see:: this"); }); it("ignores the trailing whitespace this model puts on every line", () => { // markdown hard line breaks: "::task translate " expect(proseOf("keep\n::task translate \n나 바다 \n::words \n물 | water")).toBe("keep"); }); it("treats ::progress, which has no closing ::, as running to the end", () => { expect(proseOf("keep\n::progress 40 | going well")).toBe("keep"); }); }); /* A verbatim capture of gpt-oss-20b answering "Start unit 1.1." against the real assembled prompt — the reply behind the second report of raw markup showing above the exercise. It differs from the first capture in the way that mattered: here ::gloss comes first, and removing it took the "::" off the ::task that followed, so what leaked was the bare word "task translate" with its five sentences under it. A rule that looked for lines beginning "::" could not see that, which is why the body is now read from the raw text instead. */ describe("gpt-oss-20b, unit 1.1 — the reported reply", () => { const RAW = readFileSync( new URL("../fixtures/gpt-oss-20b-unit-1.1.txt", import.meta.url), "utf8", ); it("puts the blocks in the order that breaks lib/blocks.js", () => { const order = RAW.split("\n") .filter((l) => /^\s*::\w/.test(l)) .map((l) => l.trim().split(/\s/)[0]); expect(order).toEqual(["::gloss", "::task", "::words", "::progress"]); }); it("renders the exercise", () => { expect(parseMessage(RAW).task?.type).toBe("translate"); }); it("leaves no directive, and no de-colonised directive, in the prose", () => { const body = parseMessage(RAW).body; expect(body).not.toMatch(/(^|\n)\s*::/); // The exact shape of this leak: the marker stripped, the word left. expect(body).not.toMatch(/(^|\n)\s*(task|words|gloss|progress)\b/); }); it("does not repeat the exercise as prose above it", () => { const parsed = parseMessage(RAW); const task = parsed.task; // Task is a union; only the non-match kinds carry `items`. const raw = task && "items" in task ? (task.items as { q?: string }[]) : []; const items = raw.map((i) => i.q).filter((q): q is string => Boolean(q)); expect(items.length).toBeGreaterThan(1); /* Not "no item appears": this reply uses 나 바다 as a worked example in the prose AND as the first exercise line, which is legitimate. A leak drags in every line, so that is what to measure. */ const echoed = items.filter((q) => parsed.body.includes(q)); expect(echoed.length).toBeLessThan(items.length); }); it("keeps the teaching prose that came before the blocks", () => { expect(parseMessage(RAW).body).toContain("Korean syllables are written as blocks"); }); }); /* A gloss block the model forgot to close. Every local model tested does this, and lib/blocks.js then reads the following paragraph as gloss parts — two sentences of English rendered as Korean example text inside the card, at example-line size. */ describe("an unclosed ::gloss block", () => { const RAW = [ "Here is an example.", "::gloss", "나 | S | I", "바다 | O | sea", "= I sea", "", "(Here 바다 is two blocks.)", "Now practice these.", "::task translate", "나 바다", ].join("\n"); it("ends the block where its rows end", () => { expect(glossContent("나 | S | I\n= I sea\n\nprose here\nmore prose")).toBe("나 | S | I\n= I sea\n"); }); it("keeps only the real gloss rows in the card", () => { const gloss = parseMessage(RAW).gloss ?? []; expect(gloss).toHaveLength(1); expect(gloss[0]!.parts.map((p) => p.ko)).toEqual(["나", "바다"]); expect(gloss[0]!.en).toBe("I sea"); }); it("returns the stranded prose to the message", () => { const body = parseMessage(RAW).body; expect(body).toContain("Here is an example."); expect(body).toContain("(Here 바다 is two blocks.)"); expect(body).toContain("Now practice these."); // and still no exercise text or markup expect(body).not.toContain("나 바다"); expect(body).not.toMatch(/::/); }); it("leaves a properly closed block exactly as lib parses it", () => { const closed = "::gloss\n물 | N | water\n= Water.\n::\nAfter."; const g = parseMessage(closed).gloss ?? []; expect(g).toHaveLength(1); expect(g[0]!.parts.map((p) => p.ko)).toEqual(["물"]); expect(parseMessage(closed).body).toBe("After."); }); });