467d9d1d7c
Three unit-1.1 lessons with gpt-oss-20b through LM Studio and the server —
the first real model on the reworked turn. The letter-level check went out
right, and the +25 clamp held: a reported 80 on the first answer was stored
as 25. What failed was how the model wrote its blocks, a different way each
session. All three transcripts are in test/fixtures/, verbatim, and each
failure below is a test against them.
Marks lost. The prompt asks for `여덟 | wrong | 여덜`. The first session wrote
`we | wrong | 우라 → 우리`, English prompt first; the second wrote no ::result
at all and marked only in prose, `✗ 나 | I (humble) → 저`. evidence.ts keys on
the first field of a ::result row, so nothing was ever recorded — no
evidence, no schedule, no confusions, and a 다지기 review that could never
close. The artifact would have lost them the same way. domain/marking.ts
attaches each mark to its word only where that is unambiguous: one Korean
word first, or through a prompt of the exercise he answered, read via that
exercise's ::words as the letter check reads it. With no ::result block the
✓/✗ lines are read on the same terms, so a mark can never name a word the
exercise did not ask for; a mark on a whole sentence is still dropped. What
he mistook a word for is taken from what he actually wrote whenever the mark
itself gives no other word — the third session put the right answer there.
Its third session, marked through all of this: 20 evidence rows, 20 cards.
Progress on requests. The prompt allows marks, ::confirmed and ::progress
only in reply to an answer. The model wrote ::progress on every message, and
three requests for a new exercise took the unit from 50% to 80% with nothing
answered. A reply to anything but an answer now changes none of them.
Feedback swallowed. The model closed no blocks, so lib read what followed
each one as rows: "your score is about 5%" became a result row the student
never saw, and a "---" became a recall item he was asked to write in 한글.
Another session fenced every block in ```. gloss.ts now decides every
block's extent from the raw text — at "::", the next block, a rule or fence
line, a blank line with no row after it, or for the piped blocks the first
line without a "|" — and hands lib the blocks properly closed. The gate
audit, now run through the parser the lesson uses, still flags 7 and 2.
Answers given away. Translate rows came with their meanings ("나 | I") and
recall hints were the answers ("two | 이"). A translate row keeps only its
Korean line, and a recall hint that is the expected word, or any word the
message declares, is dropped.
Also: the spelling a recall prompt expects now keeps its qualifiers. With 나
"I, me (casual)" and 저 "I, me (humble)" in one list, "I (humble)" matched
나: no letter check was sent, and the mark for 저 was filed under 나.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
92 lines
3.3 KiB
TypeScript
92 lines
3.3 KiB
TypeScript
/* The letter-level check for a recall answer.
|
|
|
|
A Hangul syllable reaches the model as one character, so it cannot see
|
|
the letters inside it — and asked which letter a student got wrong, it
|
|
invents a plausible answer. It told a student 빫다 was wrong in its ㄼ,
|
|
which was identical in both words; the slip was the initial ㅉ→ㅃ.
|
|
|
|
So the app computes the comparison (lib/hangul.js letterCheck) and the
|
|
prompt forbids the tutor from working one out. A recall prompt is
|
|
English, and the spelling it expects comes from the same message's
|
|
::words block — the artifact's own rule for finding it. */
|
|
|
|
import type { RecallTask, WordEntry } from "@lib/blocks.js";
|
|
import { letterCheck, type LetterRow } from "@lib/hangul.js";
|
|
|
|
/** "The Sea (noun)" → "the sea" */
|
|
const normal = (s: string): string =>
|
|
s
|
|
.toLowerCase()
|
|
.replace(/\([^)]*\)/g, " ")
|
|
.replace(/[^a-z]+/g, " ")
|
|
.replace(/\s+/g, " ")
|
|
.trim();
|
|
|
|
/** Case and spacing only — "I (humble)" stays distinct from "I". */
|
|
const verbatim = (s: string): string => s.toLowerCase().replace(/\s+/g, " ").trim();
|
|
|
|
const FILLER = new Set(["a", "an", "the", "to", "be"]);
|
|
|
|
/** The words of a meaning, qualifiers included: "I, me (humble)" → i, me, humble. */
|
|
const wordsOf = (s: string): Set<string> =>
|
|
new Set(
|
|
s
|
|
.toLowerCase()
|
|
.split(/[^a-z]+/)
|
|
.filter((w) => w && !FILLER.has(w)),
|
|
);
|
|
|
|
/**
|
|
* The Korean a recall prompt is asking for, from the ::words entries:
|
|
*
|
|
* 1. the meaning that is the prompt word for word;
|
|
* 2. the closest meaning holding every word of the prompt, qualifiers
|
|
* included — "I (humble)" is in "I, me (humble)" and not in
|
|
* "I, me (casual)";
|
|
* 3. the meaning that matches once qualifiers are dropped, else one that
|
|
* contains the prompt or is contained by it.
|
|
*
|
|
* "" when nothing matches — and then no check is sent, which the prompt
|
|
* tells the tutor how to handle.
|
|
*
|
|
* Measured on gpt-oss-20b: with 나 "I, me (casual)" and 저 "I, me (humble)"
|
|
* in one list, dropping the qualifiers first matched "I (humble)" to 나. The
|
|
* letter check was never sent, and the tutor's mark for 저 was filed under 나.
|
|
*/
|
|
export function expectedFor(prompt: string, words: WordEntry[] | null): string {
|
|
const q = normal(prompt);
|
|
if (!q || !words?.length) return "";
|
|
const literal = words.find((w) => verbatim(w.gloss) === verbatim(prompt));
|
|
if (literal) return literal.ko;
|
|
|
|
const want = wordsOf(prompt);
|
|
let closest: WordEntry | null = null;
|
|
let extra = Infinity;
|
|
for (const w of words) {
|
|
const have = wordsOf(w.gloss);
|
|
if (want.size && [...want].every((x) => have.has(x)) && have.size - want.size < extra) {
|
|
closest = w;
|
|
extra = have.size - want.size;
|
|
}
|
|
}
|
|
if (closest) return closest.ko;
|
|
|
|
const exact = words.find((w) => normal(w.gloss) === q);
|
|
if (exact) return exact.ko;
|
|
const near = words.find((w) => {
|
|
const g = normal(w.gloss);
|
|
return g && (g.includes(q) || q.includes(g));
|
|
});
|
|
return near?.ko ?? "";
|
|
}
|
|
|
|
/** The LETTER-LEVEL CHECK block for a recall answer, or "". */
|
|
export function recallLetterBlock(task: RecallTask, answers: string[], words: WordEntry[] | null): string {
|
|
const rows: LetterRow[] = task.items.map((item, i) => ({
|
|
prompt: item.q,
|
|
expected: expectedFor(item.q, words),
|
|
written: (answers[i] ?? "").trim(),
|
|
}));
|
|
return letterCheck(rows);
|
|
}
|