diff --git a/app/src/domain/stub-tutor.ts b/app/src/domain/stub-tutor.ts index cdbe67b..ed82b4e 100644 --- a/app/src/domain/stub-tutor.ts +++ b/app/src/domain/stub-tutor.ts @@ -30,12 +30,27 @@ export interface SampleOptions { onText?: (update: { text: string }) => void; } +/** + * One turn's request. + * + * `system` is separate from `messages` on purpose. The artifact had no + * system role available and smuggled the whole prompt in as messages[0]; + * carrying that forward would have meant the ~12k-character gate could not + * be given a cache breakpoint, and would be re-billed in full on every + * turn. Keeping it its own field is what makes prompt caching possible on + * the server. + */ +export interface SampleRequest { + system: string; + messages: SampleMessage[]; +} + export interface SampleResult { text: string; truncated?: boolean; } -export type Sample = (messages: SampleMessage[], opts?: SampleOptions) => Promise; +export type Sample = (req: SampleRequest, opts?: SampleOptions) => Promise; export class SampleError extends Error { code: string; @@ -169,7 +184,7 @@ export interface StubOptions { export function makeStubTutor(context: () => StubContext, opts: StubOptions = {}): Sample { const delay = opts.chunkDelay ?? 18; - return async (_messages, options = {}) => { + return async (_req, options = {}) => { const { signal, onText } = options; const full = composeReply(context()); diff --git a/app/src/domain/tutor-client.ts b/app/src/domain/tutor-client.ts new file mode 100644 index 0000000..b16eb7e --- /dev/null +++ b/app/src/domain/tutor-client.ts @@ -0,0 +1,125 @@ +/* The real tutor, over SSE. + + Implements the same Sample contract as the stub, so swapping one for the + other touches nothing in the UI. + + Not EventSource: it can only GET, and a turn is a POST carrying the + system prompt and the transcript. fetch + a reader gives us the body we + need and an AbortController for free — and the abort is load-bearing, + because it is what stops a cancelled turn from being billed. */ + +import { + SampleError, + type Sample, + type SampleOptions, + type SampleRequest, + type SampleResult, +} from "./stub-tutor.js"; + +export interface TutorEndpoint { + baseUrl: string; + token: string; +} + +interface WireEvent { + type: "delta" | "done" | "error"; + text?: string; + code?: string; + message?: string; +} + +/** + * Split an SSE byte stream into events. + * + * Events are separated by a blank line and may span reads, so a partial + * event has to be carried over rather than parsed early. Lines beginning + * with ":" are heartbeat comments and carry no data. + */ +async function* readEvents(body: ReadableStream): AsyncGenerator { + const reader = body.getReader(); + const decoder = new TextDecoder(); + let buffer = ""; + + try { + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + buffer += decoder.decode(value, { stream: true }); + + let split: number; + while ((split = buffer.indexOf("\n\n")) !== -1) { + const frame = buffer.slice(0, split); + buffer = buffer.slice(split + 2); + + const data = frame + .split("\n") + .filter((l) => l.startsWith("data:")) + .map((l) => l.slice(5).trim()) + .join(""); + if (!data) continue; // heartbeat, or an event with an empty payload + + try { + yield JSON.parse(data) as WireEvent; + } catch { + // A frame we cannot parse is not worth killing the turn over. + } + } + } + } finally { + // Release the stream whatever happened. No `return` here: a return + // inside `finally` would swallow an in-flight exception. + reader.cancel().catch(() => undefined); + } +} + +/** A Sample backed by the Pi. */ +export function makeRemoteTutor(endpoint: TutorEndpoint): Sample { + return async (req: SampleRequest, opts: SampleOptions = {}): Promise => { + const { signal, onText } = opts; + const history = req.messages.slice(0, -1); + const last = req.messages[req.messages.length - 1]; + + let res: Response; + try { + res = await fetch(`${endpoint.baseUrl.replace(/\/$/, "")}/api/tutor`, { + method: "POST", + signal, + headers: { + "content-type": "application/json", + authorization: `Bearer ${endpoint.token}`, + }, + body: JSON.stringify({ + system: req.system, + history, + message: last?.content ?? "", + }), + }); + } catch (err) { + if (signal?.aborted) throw new SampleError("cancelled"); + throw new SampleError("offline", err instanceof Error ? err.message : "Could not reach 선생님."); + } + + if (res.status === 401) throw new SampleError("unauthorized", "The tutor token was rejected."); + if (!res.ok) throw new SampleError("upstream", `Tutor endpoint returned HTTP ${res.status}.`); + if (!res.body) throw new SampleError("upstream", "Tutor endpoint returned no body."); + + // The UI wants cumulative text; the wire carries deltas. + let text = ""; + + for await (const event of readEvents(res.body)) { + if (event.type === "delta" && event.text) { + text += event.text; + onText?.({ text }); + } else if (event.type === "error") { + // Keep whatever streamed before the failure — a partial lesson is + // worth more than an empty bubble. + throw new SampleError(event.code ?? "upstream", event.message, text); + } else if (event.type === "done") { + break; + } + } + + if (signal?.aborted) throw new SampleError("cancelled", "stopped", text); + return { text }; + }; +} diff --git a/app/src/ui/tutor/TutorTab.tsx b/app/src/ui/tutor/TutorTab.tsx index faab426..8ea2c8f 100644 --- a/app/src/ui/tutor/TutorTab.tsx +++ b/app/src/ui/tutor/TutorTab.tsx @@ -27,6 +27,7 @@ import { import type { FocusMode } from "../../domain/gate.js"; import { applyProgressReport, currentUnit } from "../../domain/progress.js"; import { makeStubTutor, SampleError, type Sample, type StubWord } from "../../domain/stub-tutor.js"; +import { makeRemoteTutor } from "../../domain/tutor-client.js"; import { lookupMany } from "../../domain/lexicon.js"; import { REFERENCE_BAND, bandForUnit, ceilingForBand } from "@shared/bands.mjs"; import type { Db } from "../../db/types.js"; @@ -87,7 +88,7 @@ async function readBandWords(db: Db, band: number, ceiling: number): Promise([]); const [streaming, setStreaming] = useState(null); const [busy, setBusy] = useState(false); @@ -148,6 +149,10 @@ export function TutorTab() { /* ── the responder ── */ const sample: Sample = useMemo(() => { + // A configured server means the real 선생님; otherwise the local stand-in, + // so the app is complete offline rather than degraded. + if (server) return makeRemoteTutor(server); + return makeStubTutor(() => { const words: StubWord[] = railVocabulary.current; return { @@ -157,7 +162,7 @@ export function TutorTab() { confidence: progress.confidence?.[progress.current] ?? 0, }; }); - }, [gate, turns, progress]); + }, [gate, turns, progress, server]); /* The unit's own new words, glossed — what the stub builds exercises from and what the real tutor would be told it may introduce. */ @@ -205,7 +210,10 @@ export function TutorTab() { try { const result = await sample( - [{ role: "user", content: systemPrompt }, ...history, { role: "user", content: body }], + { + system: systemPrompt, + messages: [...history, { role: "user", content: body }], + }, { signal: controller.signal, onText: ({ text }) => setStreaming(text) }, ); @@ -350,7 +358,8 @@ export function TutorTab() { - {gate.vocabulary.length} words unlocked · {gate.newWords.length} new this unit + {server ? "connected" : "local stand-in"} · {gate.vocabulary.length} words unlocked ·{" "} + {gate.newWords.length} new this unit diff --git a/server/src/backends/agent-sdk.ts b/server/src/backends/agent-sdk.ts new file mode 100644 index 0000000..5ad6d4a --- /dev/null +++ b/server/src/backends/agent-sdk.ts @@ -0,0 +1,49 @@ +/* The Claude Agent SDK backend — deliberately not implemented. + * + * PORT.md originally specified this path, for one good reason: a + * CLAUDE_CODE_OAUTH_TOKEN bills against a Claude subscription rather than + * per token. If that matters more than the trade-offs below, this is the + * file to fill in; nothing else changes, because the route only knows the + * TutorBackend interface. + * + * What it would cost, so the decision can be made with the facts: + * + * 1. THE CONVERSATION SHAPE DOES NOT FIT. query()'s `prompt` accepts only + * user-role messages. There is no way to hand it prior *assistant* + * turns, so the transcript has to be flattened into one user turn — + * losing the role structure the tutor's marking behaviour depends on. + * The alternative, `resume: sessionId`, puts session state back on the + * server, which is precisely what PORT.md says to avoid. + * + * 2. IT IS NOT A LIBRARY. It supervises a `claude` CLI subprocess, ~215 MB + * unpacked, with hosting guidance of roughly 1 GiB RAM and a CPU per + * concurrent agent. That is heavy for a Pi already running Postgres + * and Caddy. Note also that `npm ci --omit=optional` yields no binary + * and fails at runtime with "Claude Code not found". + * + * 3. NO PROMPT CACHE HANDLE. The gate prompt cannot be given a + * cache_control breakpoint, only the coarser systemPrompt snapshot — + * so the cost saving that makes the API path cheap per turn is not + * available to offset the subscription's rate limits. + * + * 4. A POLICY QUESTION. Anthropic's docs say third-party developers may + * not offer claude.ai login or subscription rate limits for their + * products, including agents built on the Agent SDK, and point to API + * key auth instead. A personal, single-user, self-hosted endpoint is + * not addressed either way. Unsettled rather than permitted. + * + * If implemented, it would need: tools: [] (allowedTools only auto-approves, + * it does not restrict), settingSources: [], persistSession: false, + * maxTurns: 1, CLAUDE_CODE_DISABLE_AUTO_MEMORY=1, and + * includePartialMessages: true to get token deltas. ANTHROPIC_API_KEY must + * be unset in the environment or it silently outranks the OAuth token. + */ + +import type { TutorBackend } from "./types.ts"; + +export function agentSdkBackend(): TutorBackend { + throw new Error( + "The Agent SDK backend is not implemented — see the note at the top of " + + "server/src/backends/agent-sdk.ts. Use HANKAN_TUTOR_BACKEND=anthropic.", + ); +} diff --git a/server/src/backends/anthropic.ts b/server/src/backends/anthropic.ts new file mode 100644 index 0000000..2d568d7 --- /dev/null +++ b/server/src/backends/anthropic.ts @@ -0,0 +1,81 @@ +/* The Claude API backend. + + One streaming completion per turn, no tools, no server-side session. The + request shape maps 1:1 onto what the tutor tab already sends. + + The system prompt is cached. It is ~12k characters of gate — the taught + list, the forbidden list, the vocabulary — and it is byte-identical for + as long as the learner stays in one unit, which is many turns. Caching it + means every turn after the first reads the prefix at a fraction of the + input price instead of re-billing the whole thing. This is the single + biggest cost lever in the design, and it is the reason the prompt moved + out of messages[0] and into `system`. */ + +import Anthropic from "@anthropic-ai/sdk"; +import type { TutorBackend, TutorEvent, TutorRequest } from "./types.ts"; + +const MODEL = "claude-opus-5"; + +/* A tutor turn is a lesson intro at most — the prompt caps it at 250-450 + words. This is a ceiling that prevents a runaway, not an allocation. */ +const MAX_TOKENS = 16_000; + +export function anthropicBackend(client = new Anthropic()): TutorBackend { + return { + name: "anthropic", + + async *stream(req: TutorRequest): AsyncIterable { + const stream = client.messages.stream( + { + model: MODEL, + max_tokens: MAX_TOKENS, + // Stable for the whole unit, so it caches; the messages after it + // are what vary per turn. + system: [{ type: "text", text: req.system, cache_control: { type: "ephemeral" } }], + messages: [ + ...req.history.map((t) => ({ role: t.role, content: t.content })), + { role: "user" as const, content: req.message }, + ], + }, + { signal: req.signal }, + ); + + try { + for await (const event of stream) { + if ( + event.type === "content_block_delta" && + event.delta.type === "text_delta" && + event.delta.text + ) { + yield { type: "delta", text: event.delta.text }; + } + } + + const final = await stream.finalMessage(); + + // Safety classifiers can decline a request and still return 200. + // Reading content[0] without checking this is how that surfaces as + // a confusing empty reply rather than something the UI can explain. + if (final.stop_reason === "refusal") { + yield { + type: "error", + code: "refused", + message: "선생님 declined that one. Try rephrasing it.", + }; + return; + } + + yield { type: "done", stopReason: final.stop_reason, usage: final.usage }; + } catch (err) { + if (req.signal.aborted) { + // The learner pressed stop. Not a failure, and the upstream + // request is already cancelled, so billing stops here. + yield { type: "done", stopReason: "cancelled" }; + return; + } + const message = err instanceof Error ? err.message : String(err); + yield { type: "error", code: "upstream", message }; + } + }, + }; +} diff --git a/server/src/backends/types.ts b/server/src/backends/types.ts new file mode 100644 index 0000000..698bc99 --- /dev/null +++ b/server/src/backends/types.ts @@ -0,0 +1,34 @@ +/* What the tutor route needs from a model, and nothing more. + + Two backends are possible. The Claude API one is built and is the + default. The Claude Agent SDK one — which PORT.md originally specified, + for its subscription billing — is a documented stub, because it cannot + express this conversation shape without a compromise: its prompt accepts + only user-role messages, so prior assistant turns have to be flattened + into text. See backends/agent-sdk.ts. */ + +export interface TutorTurn { + role: "user" | "assistant"; + content: string; +} + +export interface TutorRequest { + /** The assembled system prompt: tutor-system.md with {{GATE}} filled in. */ + system: string; + /** The transcript so far. The client owns it and sends it every turn. */ + history: TutorTurn[]; + /** What the learner just said. */ + message: string; + signal: AbortSignal; +} + +/** Emitted as the model produces them. Text chunks are deltas, not cumulative. */ +export type TutorEvent = + | { type: "delta"; text: string } + | { type: "done"; stopReason: string | null; usage?: unknown } + | { type: "error"; code: string; message: string }; + +export interface TutorBackend { + readonly name: string; + stream(req: TutorRequest): AsyncIterable; +} diff --git a/server/src/tutor.ts b/server/src/tutor.ts new file mode 100644 index 0000000..f0b7240 --- /dev/null +++ b/server/src/tutor.ts @@ -0,0 +1,103 @@ +/* POST /api/tutor — one turn in, a token stream out. + + No session state. The client posts the assembled system prompt, the + transcript it owns, and the new message; the server holds nothing between + requests, so a dropped connection costs one turn rather than the + conversation. + + SSE framing has three rules that are easy to get wrong and silent when + broken: every event ends with a blank line, a `data:` field may not + contain a raw newline (Korean tutor replies are full of them, so payloads + are JSON-encoded), and a comment line starting with `:` is a heartbeat + that keeps intermediaries from timing the stream out. */ + +import { Hono } from "hono"; +import { streamSSE } from "hono/streaming"; + +import { anthropicBackend } from "./backends/anthropic.ts"; +import type { TutorBackend, TutorTurn } from "./backends/types.ts"; + +const HEARTBEAT_MS = 15_000; + +/** Bound the transcript the client may send, so one request cannot be huge. */ +const MAX_HISTORY = 40; + +function pickBackend(): TutorBackend { + const name = process.env.HANKAN_TUTOR_BACKEND ?? "anthropic"; + if (name === "anthropic") return anthropicBackend(); + throw new Error(`unknown HANKAN_TUTOR_BACKEND: ${name}`); +} + +export function tutorRoute(backend?: TutorBackend) { + const app = new Hono(); + // Constructed lazily so the process can start — and serve sync — without + // an API key present. + let resolved: TutorBackend | null = backend ?? null; + + app.post("/", async (c) => { + const body = (await c.req.json()) as { + system?: string; + history?: TutorTurn[]; + message?: string; + }; + + if (!body.system || !body.message) { + return c.json({ error: "system and message are required" }, 400); + } + + const history = (Array.isArray(body.history) ? body.history : []) + .filter((t) => t && (t.role === "user" || t.role === "assistant") && typeof t.content === "string") + .slice(-MAX_HISTORY); + + try { + resolved ??= pickBackend(); + } catch (err) { + return c.json({ error: err instanceof Error ? err.message : "no backend" }, 503); + } + + const res = streamSSE(c, async (stream) => { + const abort = new AbortController(); + // The client going away is the cancel signal: abort upstream so a + // cancelled turn stops billing. + stream.onAbort(() => abort.abort()); + + const beat = setInterval(() => { + void stream.writeSSE({ data: "", event: "ping" }); + }, HEARTBEAT_MS); + + try { + for await (const event of resolved!.stream({ + system: body.system!, + history, + message: body.message!, + signal: abort.signal, + })) { + await stream.writeSSE({ event: event.type, data: JSON.stringify(event) }); + if (event.type === "done" || event.type === "error") break; + } + } catch (err) { + await stream.writeSSE({ + event: "error", + data: JSON.stringify({ + type: "error", + code: "server", + message: err instanceof Error ? err.message : String(err), + }), + }); + } finally { + clearInterval(beat); + } + }); + + /* Set after streamSSE, which writes its own Cache-Control and would + overwrite these. `no-transform` tells intermediaries not to re-encode + the body — compression is the usual reason SSE appears to hang behind + a proxy, because an unfinished compression frame holds the events. + X-Accel-Buffering is nginx's opt-out and harmless elsewhere. */ + res.headers.set("Cache-Control", "no-cache, no-transform"); + res.headers.set("X-Accel-Buffering", "no"); + return res; + }); + + return app; +} diff --git a/test/server/tutor-sse.test.ts b/test/server/tutor-sse.test.ts new file mode 100644 index 0000000..afbbfd3 --- /dev/null +++ b/test/server/tutor-sse.test.ts @@ -0,0 +1,160 @@ +/* The tutor endpoint's wire behaviour, against a mock backend. + + Testable with no API key, which matters: the framing and the abort path + are where this breaks, and neither needs a real model to exercise. A + raw newline inside a `data:` field would silently truncate every Korean + reply — that is the bug this file exists to catch. */ + +import { describe, it, expect } from "vitest"; +import { tutorRoute } from "../../server/src/tutor.ts"; +import type { TutorBackend, TutorEvent } from "../../server/src/backends/types.ts"; + +/** Emits the chunks it is given, then done. */ +function mockBackend(chunks: string[], opts: { hang?: boolean } = {}): TutorBackend { + return { + name: "mock", + async *stream(req): AsyncIterable { + for (const text of chunks) { + if (req.signal.aborted) return; + yield { type: "delta", text }; + } + if (opts.hang) { + // Stay open until aborted, so the abort path can be observed. + await new Promise((resolve) => { + if (req.signal.aborted) return resolve(); + req.signal.addEventListener("abort", () => resolve(), { once: true }); + }); + return; + } + yield { type: "done", stopReason: "end_turn" }; + }, + }; +} + +const post = (app: ReturnType, body: unknown) => + app.request("/", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + }); + +/** Parse an SSE body into its events, the way the client does. */ +function parseSSE(text: string): { event: string; data: string }[] { + return text + .split("\n\n") + .filter((f) => f.trim()) + .map((frame) => { + const lines = frame.split("\n"); + return { + event: lines.find((l) => l.startsWith("event:"))?.slice(6).trim() ?? "", + data: lines + .filter((l) => l.startsWith("data:")) + .map((l) => l.slice(5).trim()) + .join(""), + }; + }); +} + +describe("tutor endpoint", () => { + it("rejects a request with no system prompt or message", async () => { + const app = tutorRoute(mockBackend(["hi"])); + expect((await post(app, { message: "안녕" })).status).toBe(400); + expect((await post(app, { system: "S" })).status).toBe(400); + }); + + it("streams deltas as SSE and closes with done", async () => { + const app = tutorRoute(mockBackend(["안녕", "하세요"])); + const res = await post(app, { system: "S", history: [], message: "안녕" }); + + expect(res.status).toBe(200); + expect(res.headers.get("content-type")).toContain("text/event-stream"); + // Compression or a caching proxy would break the stream; both are + // refused explicitly. + expect(res.headers.get("cache-control")).toContain("no-transform"); + + const events = parseSSE(await res.text()); + const deltas = events.filter((e) => e.event === "delta").map((e) => JSON.parse(e.data).text); + expect(deltas).toEqual(["안녕", "하세요"]); + expect(events.at(-1)?.event).toBe("done"); + }); + + /* The framing bug this guards against: a tutor reply is full of newlines + — every ::words row, every task line — and a raw newline inside a + `data:` field ends the event early, truncating the turn. */ + it("survives newlines in the payload, which every real reply has", async () => { + const reply = "좋아.\n\n::task translate\n밥 먹어\n::\n::words\n밥 | rice\n::"; + const app = tutorRoute(mockBackend([reply])); + const res = await post(app, { system: "S", history: [], message: "네" }); + + const events = parseSSE(await res.text()); + const delta = events.find((e) => e.event === "delta"); + expect(JSON.parse(delta!.data).text).toBe(reply); + }); + + it("passes the transcript through and caps its length", async () => { + let seen: { history: unknown[]; system: string } | null = null; + const spy: TutorBackend = { + name: "spy", + async *stream(req) { + seen = { history: req.history, system: req.system }; + yield { type: "done", stopReason: "end_turn" }; + }, + }; + const app = tutorRoute(spy); + const history: { role: "user" | "assistant"; content: string }[] = Array.from( + { length: 60 }, + (_, i) => ({ role: i % 2 ? "assistant" : "user", content: `turn ${i}` }), + ); + + const res = await post(app, { system: "GATE", history, message: "다음" }); + await res.text(); + + expect(seen!.system).toBe("GATE"); + expect(seen!.history.length).toBe(40); // MAX_HISTORY + expect((seen!.history.at(-1) as { content: string }).content).toBe("turn 59"); + }); + + it("reports a backend error as an error event rather than a dead stream", async () => { + const failing: TutorBackend = { + name: "failing", + // eslint-disable-next-line require-yield + async *stream() { + throw new Error("upstream exploded"); + }, + }; + const res = await post(tutorRoute(failing), { system: "S", history: [], message: "x" }); + const events = parseSSE(await res.text()); + expect(events.at(-1)?.event).toBe("error"); + expect(JSON.parse(events.at(-1)!.data).message).toContain("upstream exploded"); + }); + + /* The abort path is what makes "Stop" stop billing: the client going away + has to reach the backend's signal, not just close the socket. */ + it("aborts the backend when the client goes away", async () => { + let seen: AbortSignal | null = null; + const holding: TutorBackend = { + name: "holding", + async *stream(req) { + seen = req.signal; + yield { type: "delta", text: "부" }; + await new Promise((resolve) => + req.signal.addEventListener("abort", () => resolve(), { once: true }), + ); + }, + }; + + const res = await post(tutorRoute(holding), { system: "S", history: [], message: "x" }); + const reader = res.body!.getReader(); + await reader.read(); // the turn is live and the backend holds the signal + + expect(seen).not.toBeNull(); + expect(seen!.aborted).toBe(false); + + await reader.cancel(); // the client goes away + + for (let i = 0; i < 50 && !seen!.aborted; i++) { + await new Promise((r) => setTimeout(r, 20)); + } + expect(seen!.aborted, "the backend must see the client disconnect").toBe(true); + }); +});