Hankan/app/src/domain/gloss.ts

/* Where a tutor message's blocks END — a workaround for lib/blocks.js,
   which ships unchanged.

   The 16 Sep lib fixed the two defects this module used to work around: a
   gloss block now closes at each "=" line, and every block is collected
   rather than only the first. What remains is how a block ends, and every
   one of these reaches the student.

   ── 1. Removing one block takes the next block's colons ────────────────

   parse() builds `body` by deleting each block with a regex whose
   terminator `(?:\n::|$)` is INSIDE the match. Deleting one block therefore
   deletes the "::" that opens the block after it, which is then no longer
   recognisable as a block at all and stays behind as prose:

       ::task translate            ->   words
       나 가다                          나 | I, me (casual) | pron
       ::words
       나 | I, me (casual) | pron

   Which block leaks depends on the order the model chose, and nothing in
   the prompt fixes one. So the prose is read from the RAW text, where every
   marker is still intact. That is not a repair of parse()'s body — it is
   just reading.

   ── 2. A block runs to the next "::", or to the end ────────────────────

   The prompt closes every block with "::". Every local model tested leaves
   it off, and lib then reads whatever follows as rows of the block:

     · after a gloss block, the paragraph that follows becomes gloss PARTS,
       so two sentences of English render as Korean example text;
     · after ::result, the tutor's own feedback — "you wrote an extra line",
       "your score for this unit is about 5%" — became result rows, so the
       student never saw it;
     · a markdown rule, "---", became an exercise item, and he was asked to
       write the 한글 for "---".

   So every block's extent is decided here, from the raw text, and lib is
   handed the blocks properly closed — the row formats stay entirely lib's.
   A block ends where its rows end: at "::", at the next block, at a rule
   line, at a blank line that nothing row-like follows, and — for the
   blocks whose rows always carry a "|" — at the first line without one. A
   gloss row is `한글 | role | gloss` and its translation starts with "=".

   ── 3. Rows that give the exercise away ────────────────────────────────

   Measured on gpt-oss-20b: translate rows arrived with their meaning
   appended ("나 | I"), and recall hints were the answer itself ("two | 이").
   A translate row is one Korean line, so a "|" tail is not part of it; a
   recall hint that is the expected word, or any word the message declares,
   is dropped. Neither loosens anything — the exercise is only ever made
   harder to cheat.

   And in one session every block came wrapped in a markdown code fence,
   which left empty "```" lines in the message. A fence line is dropped. */

import { parse } from "@lib/blocks.js";
import type { ParsedMessage, Task, WordEntry } from "@lib/blocks.js";
import { expectedFor } from "./letters.js";

/** A markdown rule: never a row, and never the start of one. */
const RULE = /^\s*(?:-{3,}|\*{3,}|_{3,})\s*$/;

/** A code fence. The prompt forbids them; one session wrapped every block
    in one. Not prose, not a row: dropped, and it ends a block. */
const FENCE = /^\s*```/;

/** Blocks whose every row carries a "|". */
const PIPED = new Set(["words", "result", "task match", "task build", "task choice"]);

/**
 * Is this a line of ::gloss content?
 *
 * A gloss row is `한글 | role | English` and a translation line starts with
 * `=`. Nothing else belongs, so the first line that is neither ends the
 * block whether or not the model closed it.
 */
function isGlossContent(line: string): boolean {
  const t = line.trim();
  return t === "" || t.includes("|") || t.startsWith("=");
}

/** Where a gloss block's content really ends, closed properly or not. */
export function glossContent(body: string): string {
  const lines = body.split("\n");
  const end = lines.findIndex((l) => !isGlossContent(l));
  return (end === -1 ? lines : lines.slice(0, end)).join("\n");
}

interface Block {
  /** The opening line, as written: "::task recall", "::progress 40 | note". */
  head: string;
  /** "gloss", "words", "task recall", "progress" … */
  kind: string;
  lines: string[];
}

interface Scanned {
  prose: string[];
  blocks: Block[];
}

function kindOf(head: string): string {
  const [name = "", sub = ""] = head.slice(2).trim().split(/\s+/);
  return name === "task" ? `task ${sub}` : name;
}

/**
 * One pass over the raw text.
 *
 * A line of exactly "::" closes a block; any other line beginning "::"
 * opens one (::gloss, ::task translate, ::words, ::progress 40 | note); a
 * ::progress line is a whole block by itself. The rest of the rules are at
 * the top of this file. Trailing whitespace is common — one model ends every
 * line with markdown's two spaces — so nothing here compares exact strings.
 */
function scan(raw: string): Scanned {
  const prose: string[] = [];
  const blocks: Block[] = [];
  const lines = raw.split("\n");
  let block: Block | null = null;

  const close = () => {
    if (block) blocks.push(block);
    block = null;
  };

  for (let i = 0; i < lines.length; i++) {
    const line = lines[i]!;
    const t = line.trim();

    if (FENCE.test(line)) {
      close();
      continue;
    }
    if (t === "::") {
      close();
      continue;
    }
    if (t.startsWith("::")) {
      close();
      const opened: Block = { head: t, kind: kindOf(t), lines: [] };
      if (opened.kind === "progress") blocks.push(opened);
      else block = opened;
      continue;
    }

    const open = block as Block | null;
    if (open) {
      if (open.kind === "gloss") {
        // An unclosed gloss ends where its rows do; what follows is prose.
        if (!isGlossContent(line)) close();
      } else if (RULE.test(line)) {
        close();
      } else if (t === "") {
        // A blank line inside a block is only a gap if a row comes next.
        const next = lines.slice(i + 1).find((l) => l.trim() !== "")?.trim() ?? "";
        if (!next.startsWith("::") && !(next.includes("|") && !RULE.test(next))) close();
      } else if (PIPED.has(open.kind) && !t.includes("|")) {
        close();
      }
    }

    const current = block as Block | null;
    if (current) {
      if (t !== "") current.lines.push(line);
    } else {
      prose.push(line);
    }
  }
  close();

  return { prose, blocks };
}

/** The prose of a message: every line that is not inside a block. */
export function proseOf(raw: string): string {
  return tidyProse(scan(raw).prose);
}

/**
 * The prose, without the separators a closed block leaves behind: runs of
 * blank lines, and "---" rules with nothing left to separate at either end.
 */
function tidyProse(lines: string[]): string {
  const out = [...lines];
  const edge = (l: string | undefined) => l !== undefined && (l.trim() === "" || RULE.test(l));
  while (edge(out[0])) out.shift();
  while (edge(out[out.length - 1])) out.pop();
  return out.join("\n").replace(/\n{3,}/g, "\n\n");
}

/** Has at least one letter — Hangul or Latin — and so can be an item. */
const hasLetter = (s: string) => /\p{L}/u.test(s);

/** Section 3 of the header: rows that give the exercise away, and non-rows. */
function tidyTask(task: Task | null, words: WordEntry[] | null): Task | null {
  if (!task) return task;
  const rows = task.rows?.filter(hasLetter);
  switch (task.type) {
    case "translate":
      return {
        ...task,
        rows,
        items: task.items.map((it) => ({ q: it.q.split("|")[0]!.trim() })).filter((it) => hasLetter(it.q)),
      };
    case "recall":
      return {
        ...task,
        rows,
        items: task.items
          .filter((it) => hasLetter(it.q))
          .map((it) => {
            // A hint holding the answer, or any word the message declares,
            // is the answer given away. Real hints are about the word:
            // "double batchim".
            const answer = expectedFor(it.q, words);
            const hintWords = it.hint.match(/[가-힣]+/g) ?? [];
            const gives =
              (answer && it.hint.includes(answer)) || hintWords.some((h) => words?.some((w) => w.ko === h));
            return gives ? { ...it, hint: "" } : it;
          }),
      };
    default:
      return { ...task, rows };
  }
}

/**
 * parse(), with every block's extent and the prose decided from the raw
 * text. Use this everywhere instead of calling parse() directly.
 */
export function parseMessage(text: string): ParsedMessage {
  const { prose, blocks } = scan(text);
  const closed = blocks.map((b) => [b.head, ...b.lines, "::"].join("\n")).join("\n");
  const parsed = parse(closed);
  return {
    ...parsed,
    body: tidyProse(prose),
    task: tidyTask(parsed.task, parsed.words),
  };
}

export type { ParsedMessage };