feat(tutor): the real endpoint, streamed, with the system prompt cached

POST /api/tutor takes the assembled system prompt, the transcript the
client owns, and the new message, and streams tokens back. It holds
nothing between requests, so a dropped connection costs one turn rather
than the conversation.

The seam moved first: Sample took the assembled prompt as messages[0] with
role user. It now has an explicit system field, which is what lets the
backend put it in the API's system parameter as a cached block. The gate
is ~12k characters and is byte-identical for as long as the learner stays
in one unit, so every turn after the first reads the prefix at a fraction
of the input price. That is the single biggest cost lever in the design,
and it was unreachable through the old shape.

prompt/tutor-system.md still ships unchanged; only where the string is
placed changed.

SSE has three rules that are silent when broken, and all three are
handled: every event ends with a blank line, payloads are JSON-encoded
because a raw newline in Korean text would break the framing, and a `:`
heartbeat every 15s keeps intermediaries from timing the stream out.
Cache-Control is set on the returned Response rather than inside
streamSSE, which writes its own and would overwrite it; `no-transform` is
there because compression, not buffering, is what usually makes SSE look
like it hangs behind a proxy.

The client uses fetch + getReader, not EventSource — EventSource cannot
POST, and the body is {system, history, message}. Aborting closes the
connection, the server aborts upstream, and a cancelled turn stops
billing. With no server configured the app falls back to the stub, so the
offline build is untouched.

backends/anthropic.ts is the default. backends/agent-sdk.ts is deliberately
unimplemented and documents why the plain API was chosen over PORT.md's
Agent SDK — chiefly that its prompt accepts only user-role messages, so
the transcript would have to be flattened into a single turn.

The endpoint's own tests use a mock backend and need no API key.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
MechaCat02
2026-09-08 19:58:01 +02:00
parent c2e1fc23fe
commit 66f92247d2
8 changed files with 582 additions and 6 deletions

View File

@@ -30,12 +30,27 @@ export interface SampleOptions {
onText?: (update: { text: string }) => void;
}
/**
* One turn's request.
*
* `system` is separate from `messages` on purpose. The artifact had no
* system role available and smuggled the whole prompt in as messages[0];
* carrying that forward would have meant the ~12k-character gate could not
* be given a cache breakpoint, and would be re-billed in full on every
* turn. Keeping it its own field is what makes prompt caching possible on
* the server.
*/
export interface SampleRequest {
system: string;
messages: SampleMessage[];
}
export interface SampleResult {
text: string;
truncated?: boolean;
}
export type Sample = (messages: SampleMessage[], opts?: SampleOptions) => Promise<SampleResult>;
export type Sample = (req: SampleRequest, opts?: SampleOptions) => Promise<SampleResult>;
export class SampleError extends Error {
code: string;
@@ -169,7 +184,7 @@ export interface StubOptions {
export function makeStubTutor(context: () => StubContext, opts: StubOptions = {}): Sample {
const delay = opts.chunkDelay ?? 18;
return async (_messages, options = {}) => {
return async (_req, options = {}) => {
const { signal, onText } = options;
const full = composeReply(context());

View File

@@ -0,0 +1,125 @@
/* The real tutor, over SSE.
Implements the same Sample contract as the stub, so swapping one for the
other touches nothing in the UI.
Not EventSource: it can only GET, and a turn is a POST carrying the
system prompt and the transcript. fetch + a reader gives us the body we
need and an AbortController for free — and the abort is load-bearing,
because it is what stops a cancelled turn from being billed. */
import {
SampleError,
type Sample,
type SampleOptions,
type SampleRequest,
type SampleResult,
} from "./stub-tutor.js";
export interface TutorEndpoint {
baseUrl: string;
token: string;
}
interface WireEvent {
type: "delta" | "done" | "error";
text?: string;
code?: string;
message?: string;
}
/**
* Split an SSE byte stream into events.
*
* Events are separated by a blank line and may span reads, so a partial
* event has to be carried over rather than parsed early. Lines beginning
* with ":" are heartbeat comments and carry no data.
*/
async function* readEvents(body: ReadableStream<Uint8Array>): AsyncGenerator<WireEvent> {
const reader = body.getReader();
const decoder = new TextDecoder();
let buffer = "";
try {
for (;;) {
const { done, value } = await reader.read();
if (done) break;
buffer += decoder.decode(value, { stream: true });
let split: number;
while ((split = buffer.indexOf("\n\n")) !== -1) {
const frame = buffer.slice(0, split);
buffer = buffer.slice(split + 2);
const data = frame
.split("\n")
.filter((l) => l.startsWith("data:"))
.map((l) => l.slice(5).trim())
.join("");
if (!data) continue; // heartbeat, or an event with an empty payload
try {
yield JSON.parse(data) as WireEvent;
} catch {
// A frame we cannot parse is not worth killing the turn over.
}
}
}
} finally {
// Release the stream whatever happened. No `return` here: a return
// inside `finally` would swallow an in-flight exception.
reader.cancel().catch(() => undefined);
}
}
/** A Sample backed by the Pi. */
export function makeRemoteTutor(endpoint: TutorEndpoint): Sample {
return async (req: SampleRequest, opts: SampleOptions = {}): Promise<SampleResult> => {
const { signal, onText } = opts;
const history = req.messages.slice(0, -1);
const last = req.messages[req.messages.length - 1];
let res: Response;
try {
res = await fetch(`${endpoint.baseUrl.replace(/\/$/, "")}/api/tutor`, {
method: "POST",
signal,
headers: {
"content-type": "application/json",
authorization: `Bearer ${endpoint.token}`,
},
body: JSON.stringify({
system: req.system,
history,
message: last?.content ?? "",
}),
});
} catch (err) {
if (signal?.aborted) throw new SampleError("cancelled");
throw new SampleError("offline", err instanceof Error ? err.message : "Could not reach 선생님.");
}
if (res.status === 401) throw new SampleError("unauthorized", "The tutor token was rejected.");
if (!res.ok) throw new SampleError("upstream", `Tutor endpoint returned HTTP ${res.status}.`);
if (!res.body) throw new SampleError("upstream", "Tutor endpoint returned no body.");
// The UI wants cumulative text; the wire carries deltas.
let text = "";
for await (const event of readEvents(res.body)) {
if (event.type === "delta" && event.text) {
text += event.text;
onText?.({ text });
} else if (event.type === "error") {
// Keep whatever streamed before the failure — a partial lesson is
// worth more than an empty bubble.
throw new SampleError(event.code ?? "upstream", event.message, text);
} else if (event.type === "done") {
break;
}
}
if (signal?.aborted) throw new SampleError("cancelled", "stopped", text);
return { text };
};
}

View File

@@ -27,6 +27,7 @@ import {
import type { FocusMode } from "../../domain/gate.js";
import { applyProgressReport, currentUnit } from "../../domain/progress.js";
import { makeStubTutor, SampleError, type Sample, type StubWord } from "../../domain/stub-tutor.js";
import { makeRemoteTutor } from "../../domain/tutor-client.js";
import { lookupMany } from "../../domain/lexicon.js";
import { REFERENCE_BAND, bandForUnit, ceilingForBand } from "@shared/bands.mjs";
import type { Db } from "../../db/types.js";
@@ -87,7 +88,7 @@ async function readBandWords(db: Db, band: number, ceiling: number): Promise<str
/* ── the tab ─────────────────────────────────────────────────────── */
export function TutorTab() {
const { db, progress, prefs, setPref, refreshProgress } = useStore();
const { db, progress, prefs, setPref, server, refreshProgress } = useStore();
const [turns, setTurns] = useState<Turn[]>([]);
const [streaming, setStreaming] = useState<string | null>(null);
const [busy, setBusy] = useState(false);
@@ -148,6 +149,10 @@ export function TutorTab() {
/* ── the responder ── */
const sample: Sample = useMemo(() => {
// A configured server means the real 선생님; otherwise the local stand-in,
// so the app is complete offline rather than degraded.
if (server) return makeRemoteTutor(server);
return makeStubTutor(() => {
const words: StubWord[] = railVocabulary.current;
return {
@@ -157,7 +162,7 @@ export function TutorTab() {
confidence: progress.confidence?.[progress.current] ?? 0,
};
});
}, [gate, turns, progress]);
}, [gate, turns, progress, server]);
/* The unit's own new words, glossed — what the stub builds exercises from
and what the real tutor would be told it may introduce. */
@@ -205,7 +210,10 @@ export function TutorTab() {
try {
const result = await sample(
[{ role: "user", content: systemPrompt }, ...history, { role: "user", content: body }],
{
system: systemPrompt,
messages: [...history, { role: "user", content: body }],
},
{ signal: controller.signal, onText: ({ text }) => setStreaming(text) },
);
@@ -350,7 +358,8 @@ export function TutorTab() {
</select>
</label>
<span className="note">
{gate.vocabulary.length} words unlocked · {gate.newWords.length} new this unit
{server ? "connected" : "local stand-in"} · {gate.vocabulary.length} words unlocked ·{" "}
{gate.newWords.length} new this unit
</span>
</div>

View File

@@ -0,0 +1,49 @@
/* The Claude Agent SDK backend — deliberately not implemented.
*
* PORT.md originally specified this path, for one good reason: a
* CLAUDE_CODE_OAUTH_TOKEN bills against a Claude subscription rather than
* per token. If that matters more than the trade-offs below, this is the
* file to fill in; nothing else changes, because the route only knows the
* TutorBackend interface.
*
* What it would cost, so the decision can be made with the facts:
*
* 1. THE CONVERSATION SHAPE DOES NOT FIT. query()'s `prompt` accepts only
* user-role messages. There is no way to hand it prior *assistant*
* turns, so the transcript has to be flattened into one user turn —
* losing the role structure the tutor's marking behaviour depends on.
* The alternative, `resume: sessionId`, puts session state back on the
* server, which is precisely what PORT.md says to avoid.
*
* 2. IT IS NOT A LIBRARY. It supervises a `claude` CLI subprocess, ~215 MB
* unpacked, with hosting guidance of roughly 1 GiB RAM and a CPU per
* concurrent agent. That is heavy for a Pi already running Postgres
* and Caddy. Note also that `npm ci --omit=optional` yields no binary
* and fails at runtime with "Claude Code not found".
*
* 3. NO PROMPT CACHE HANDLE. The gate prompt cannot be given a
* cache_control breakpoint, only the coarser systemPrompt snapshot —
* so the cost saving that makes the API path cheap per turn is not
* available to offset the subscription's rate limits.
*
* 4. A POLICY QUESTION. Anthropic's docs say third-party developers may
* not offer claude.ai login or subscription rate limits for their
* products, including agents built on the Agent SDK, and point to API
* key auth instead. A personal, single-user, self-hosted endpoint is
* not addressed either way. Unsettled rather than permitted.
*
* If implemented, it would need: tools: [] (allowedTools only auto-approves,
* it does not restrict), settingSources: [], persistSession: false,
* maxTurns: 1, CLAUDE_CODE_DISABLE_AUTO_MEMORY=1, and
* includePartialMessages: true to get token deltas. ANTHROPIC_API_KEY must
* be unset in the environment or it silently outranks the OAuth token.
*/
import type { TutorBackend } from "./types.ts";
export function agentSdkBackend(): TutorBackend {
throw new Error(
"The Agent SDK backend is not implemented — see the note at the top of " +
"server/src/backends/agent-sdk.ts. Use HANKAN_TUTOR_BACKEND=anthropic.",
);
}

View File

@@ -0,0 +1,81 @@
/* The Claude API backend.
One streaming completion per turn, no tools, no server-side session. The
request shape maps 1:1 onto what the tutor tab already sends.
The system prompt is cached. It is ~12k characters of gate — the taught
list, the forbidden list, the vocabulary — and it is byte-identical for
as long as the learner stays in one unit, which is many turns. Caching it
means every turn after the first reads the prefix at a fraction of the
input price instead of re-billing the whole thing. This is the single
biggest cost lever in the design, and it is the reason the prompt moved
out of messages[0] and into `system`. */
import Anthropic from "@anthropic-ai/sdk";
import type { TutorBackend, TutorEvent, TutorRequest } from "./types.ts";
const MODEL = "claude-opus-5";
/* A tutor turn is a lesson intro at most — the prompt caps it at 250-450
words. This is a ceiling that prevents a runaway, not an allocation. */
const MAX_TOKENS = 16_000;
export function anthropicBackend(client = new Anthropic()): TutorBackend {
return {
name: "anthropic",
async *stream(req: TutorRequest): AsyncIterable<TutorEvent> {
const stream = client.messages.stream(
{
model: MODEL,
max_tokens: MAX_TOKENS,
// Stable for the whole unit, so it caches; the messages after it
// are what vary per turn.
system: [{ type: "text", text: req.system, cache_control: { type: "ephemeral" } }],
messages: [
...req.history.map((t) => ({ role: t.role, content: t.content })),
{ role: "user" as const, content: req.message },
],
},
{ signal: req.signal },
);
try {
for await (const event of stream) {
if (
event.type === "content_block_delta" &&
event.delta.type === "text_delta" &&
event.delta.text
) {
yield { type: "delta", text: event.delta.text };
}
}
const final = await stream.finalMessage();
// Safety classifiers can decline a request and still return 200.
// Reading content[0] without checking this is how that surfaces as
// a confusing empty reply rather than something the UI can explain.
if (final.stop_reason === "refusal") {
yield {
type: "error",
code: "refused",
message: "선생님 declined that one. Try rephrasing it.",
};
return;
}
yield { type: "done", stopReason: final.stop_reason, usage: final.usage };
} catch (err) {
if (req.signal.aborted) {
// The learner pressed stop. Not a failure, and the upstream
// request is already cancelled, so billing stops here.
yield { type: "done", stopReason: "cancelled" };
return;
}
const message = err instanceof Error ? err.message : String(err);
yield { type: "error", code: "upstream", message };
}
},
};
}

View File

@@ -0,0 +1,34 @@
/* What the tutor route needs from a model, and nothing more.
Two backends are possible. The Claude API one is built and is the
default. The Claude Agent SDK one — which PORT.md originally specified,
for its subscription billing — is a documented stub, because it cannot
express this conversation shape without a compromise: its prompt accepts
only user-role messages, so prior assistant turns have to be flattened
into text. See backends/agent-sdk.ts. */
export interface TutorTurn {
role: "user" | "assistant";
content: string;
}
export interface TutorRequest {
/** The assembled system prompt: tutor-system.md with {{GATE}} filled in. */
system: string;
/** The transcript so far. The client owns it and sends it every turn. */
history: TutorTurn[];
/** What the learner just said. */
message: string;
signal: AbortSignal;
}
/** Emitted as the model produces them. Text chunks are deltas, not cumulative. */
export type TutorEvent =
| { type: "delta"; text: string }
| { type: "done"; stopReason: string | null; usage?: unknown }
| { type: "error"; code: string; message: string };
export interface TutorBackend {
readonly name: string;
stream(req: TutorRequest): AsyncIterable<TutorEvent>;
}

103
server/src/tutor.ts Normal file
View File

@@ -0,0 +1,103 @@
/* POST /api/tutor — one turn in, a token stream out.
No session state. The client posts the assembled system prompt, the
transcript it owns, and the new message; the server holds nothing between
requests, so a dropped connection costs one turn rather than the
conversation.
SSE framing has three rules that are easy to get wrong and silent when
broken: every event ends with a blank line, a `data:` field may not
contain a raw newline (Korean tutor replies are full of them, so payloads
are JSON-encoded), and a comment line starting with `:` is a heartbeat
that keeps intermediaries from timing the stream out. */
import { Hono } from "hono";
import { streamSSE } from "hono/streaming";
import { anthropicBackend } from "./backends/anthropic.ts";
import type { TutorBackend, TutorTurn } from "./backends/types.ts";
const HEARTBEAT_MS = 15_000;
/** Bound the transcript the client may send, so one request cannot be huge. */
const MAX_HISTORY = 40;
function pickBackend(): TutorBackend {
const name = process.env.HANKAN_TUTOR_BACKEND ?? "anthropic";
if (name === "anthropic") return anthropicBackend();
throw new Error(`unknown HANKAN_TUTOR_BACKEND: ${name}`);
}
export function tutorRoute(backend?: TutorBackend) {
const app = new Hono();
// Constructed lazily so the process can start — and serve sync — without
// an API key present.
let resolved: TutorBackend | null = backend ?? null;
app.post("/", async (c) => {
const body = (await c.req.json()) as {
system?: string;
history?: TutorTurn[];
message?: string;
};
if (!body.system || !body.message) {
return c.json({ error: "system and message are required" }, 400);
}
const history = (Array.isArray(body.history) ? body.history : [])
.filter((t) => t && (t.role === "user" || t.role === "assistant") && typeof t.content === "string")
.slice(-MAX_HISTORY);
try {
resolved ??= pickBackend();
} catch (err) {
return c.json({ error: err instanceof Error ? err.message : "no backend" }, 503);
}
const res = streamSSE(c, async (stream) => {
const abort = new AbortController();
// The client going away is the cancel signal: abort upstream so a
// cancelled turn stops billing.
stream.onAbort(() => abort.abort());
const beat = setInterval(() => {
void stream.writeSSE({ data: "", event: "ping" });
}, HEARTBEAT_MS);
try {
for await (const event of resolved!.stream({
system: body.system!,
history,
message: body.message!,
signal: abort.signal,
})) {
await stream.writeSSE({ event: event.type, data: JSON.stringify(event) });
if (event.type === "done" || event.type === "error") break;
}
} catch (err) {
await stream.writeSSE({
event: "error",
data: JSON.stringify({
type: "error",
code: "server",
message: err instanceof Error ? err.message : String(err),
}),
});
} finally {
clearInterval(beat);
}
});
/* Set after streamSSE, which writes its own Cache-Control and would
overwrite these. `no-transform` tells intermediaries not to re-encode
the body — compression is the usual reason SSE appears to hang behind
a proxy, because an unfinished compression frame holds the events.
X-Accel-Buffering is nginx's opt-out and harmless elsewhere. */
res.headers.set("Cache-Control", "no-cache, no-transform");
res.headers.set("X-Accel-Buffering", "no");
return res;
});
return app;
}

View File

@@ -0,0 +1,160 @@
/* The tutor endpoint's wire behaviour, against a mock backend.
Testable with no API key, which matters: the framing and the abort path
are where this breaks, and neither needs a real model to exercise. A
raw newline inside a `data:` field would silently truncate every Korean
reply — that is the bug this file exists to catch. */
import { describe, it, expect } from "vitest";
import { tutorRoute } from "../../server/src/tutor.ts";
import type { TutorBackend, TutorEvent } from "../../server/src/backends/types.ts";
/** Emits the chunks it is given, then done. */
function mockBackend(chunks: string[], opts: { hang?: boolean } = {}): TutorBackend {
return {
name: "mock",
async *stream(req): AsyncIterable<TutorEvent> {
for (const text of chunks) {
if (req.signal.aborted) return;
yield { type: "delta", text };
}
if (opts.hang) {
// Stay open until aborted, so the abort path can be observed.
await new Promise<void>((resolve) => {
if (req.signal.aborted) return resolve();
req.signal.addEventListener("abort", () => resolve(), { once: true });
});
return;
}
yield { type: "done", stopReason: "end_turn" };
},
};
}
const post = (app: ReturnType<typeof tutorRoute>, body: unknown) =>
app.request("/", {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify(body),
});
/** Parse an SSE body into its events, the way the client does. */
function parseSSE(text: string): { event: string; data: string }[] {
return text
.split("\n\n")
.filter((f) => f.trim())
.map((frame) => {
const lines = frame.split("\n");
return {
event: lines.find((l) => l.startsWith("event:"))?.slice(6).trim() ?? "",
data: lines
.filter((l) => l.startsWith("data:"))
.map((l) => l.slice(5).trim())
.join(""),
};
});
}
describe("tutor endpoint", () => {
it("rejects a request with no system prompt or message", async () => {
const app = tutorRoute(mockBackend(["hi"]));
expect((await post(app, { message: "안녕" })).status).toBe(400);
expect((await post(app, { system: "S" })).status).toBe(400);
});
it("streams deltas as SSE and closes with done", async () => {
const app = tutorRoute(mockBackend(["안녕", "하세요"]));
const res = await post(app, { system: "S", history: [], message: "안녕" });
expect(res.status).toBe(200);
expect(res.headers.get("content-type")).toContain("text/event-stream");
// Compression or a caching proxy would break the stream; both are
// refused explicitly.
expect(res.headers.get("cache-control")).toContain("no-transform");
const events = parseSSE(await res.text());
const deltas = events.filter((e) => e.event === "delta").map((e) => JSON.parse(e.data).text);
expect(deltas).toEqual(["안녕", "하세요"]);
expect(events.at(-1)?.event).toBe("done");
});
/* The framing bug this guards against: a tutor reply is full of newlines
— every ::words row, every task line — and a raw newline inside a
`data:` field ends the event early, truncating the turn. */
it("survives newlines in the payload, which every real reply has", async () => {
const reply = "좋아.\n\n::task translate\n밥 먹어\n::\n::words\n밥 | rice\n::";
const app = tutorRoute(mockBackend([reply]));
const res = await post(app, { system: "S", history: [], message: "네" });
const events = parseSSE(await res.text());
const delta = events.find((e) => e.event === "delta");
expect(JSON.parse(delta!.data).text).toBe(reply);
});
it("passes the transcript through and caps its length", async () => {
let seen: { history: unknown[]; system: string } | null = null;
const spy: TutorBackend = {
name: "spy",
async *stream(req) {
seen = { history: req.history, system: req.system };
yield { type: "done", stopReason: "end_turn" };
},
};
const app = tutorRoute(spy);
const history: { role: "user" | "assistant"; content: string }[] = Array.from(
{ length: 60 },
(_, i) => ({ role: i % 2 ? "assistant" : "user", content: `turn ${i}` }),
);
const res = await post(app, { system: "GATE", history, message: "다음" });
await res.text();
expect(seen!.system).toBe("GATE");
expect(seen!.history.length).toBe(40); // MAX_HISTORY
expect((seen!.history.at(-1) as { content: string }).content).toBe("turn 59");
});
it("reports a backend error as an error event rather than a dead stream", async () => {
const failing: TutorBackend = {
name: "failing",
// eslint-disable-next-line require-yield
async *stream() {
throw new Error("upstream exploded");
},
};
const res = await post(tutorRoute(failing), { system: "S", history: [], message: "x" });
const events = parseSSE(await res.text());
expect(events.at(-1)?.event).toBe("error");
expect(JSON.parse(events.at(-1)!.data).message).toContain("upstream exploded");
});
/* The abort path is what makes "Stop" stop billing: the client going away
has to reach the backend's signal, not just close the socket. */
it("aborts the backend when the client goes away", async () => {
let seen: AbortSignal | null = null;
const holding: TutorBackend = {
name: "holding",
async *stream(req) {
seen = req.signal;
yield { type: "delta", text: "부" };
await new Promise<void>((resolve) =>
req.signal.addEventListener("abort", () => resolve(), { once: true }),
);
},
};
const res = await post(tutorRoute(holding), { system: "S", history: [], message: "x" });
const reader = res.body!.getReader();
await reader.read(); // the turn is live and the backend holds the signal
expect(seen).not.toBeNull();
expect(seen!.aborted).toBe(false);
await reader.cancel(); // the client goes away
for (let i = 0; i < 50 && !seen!.aborted; i++) {
await new Promise((r) => setTimeout(r, 20));
}
expect(seen!.aborted, "the backend must see the client disconnect").toBe(true);
});
});