diff --git a/README.md b/README.md index 69e92c0..7c31d6f 100644 --- a/README.md +++ b/README.md @@ -168,6 +168,12 @@ Three details worth knowing here: - **The system prompt is cached.** It is ~12k characters of gate, identical for as long as the learner stays in one unit, so every turn after the first reads it at a fraction of the input price. +- **The model is a config line, not a code change.** `HANKAN_TUTOR_BACKEND` + selects the Claude API, any OpenAI-compatible endpoint (LM Studio, Ollama, + llama.cpp, vLLM, OpenRouter), or a keyless echo backend for proving a + deployment. Keeping the system prompt first and unchanged is what makes + Anthropic's prefix cache work — and, it turns out, a local server's KV + cache too: 1,563ms to first token cold, 324ms warm. ## Not in this pass diff --git a/server/.env.example b/server/.env.example index 1a81f84..e4e4681 100644 --- a/server/.env.example +++ b/server/.env.example @@ -8,10 +8,30 @@ DATABASE_URL=postgres://hankan:CHANGE_ME@postgres:5432/hankan # openssl rand -base64 32 HANKAN_TOKEN=CHANGE_ME -# For the tutor. Omit it and sync still works — the app falls back to its -# local stand-in tutor. +# Which model serves the tutor: anthropic (default) | openai | echo. +# Omit the whole tutor config and sync still works — the app falls back to +# its local stand-in tutor. +HANKAN_TUTOR_BACKEND=anthropic + +# For HANKAN_TUTOR_BACKEND=anthropic. ANTHROPIC_API_KEY= +# For HANKAN_TUTOR_BACKEND=openai — anything speaking /chat/completions: +# LM Studio, Ollama, llama.cpp, vLLM, LiteLLM, OpenRouter, OpenAI. +# From a container, localhost is the container: use the host's LAN address +# or host.docker.internal, not 127.0.0.1. +# LM Studio http://:1234/v1 model = the id shown in its UI +# Ollama http://:11434/v1 model = e.g. qwen2.5:14b +# llama.cpp http://:8080/v1 +# OpenRouter https://openrouter.ai/api/v1 +HANKAN_OPENAI_BASE_URL=http://host.docker.internal:1234/v1 +HANKAN_OPENAI_MODEL=local-model +# Local servers ignore this; hosted ones require it. Leave blank for local. +HANKAN_OPENAI_API_KEY= +# Completion ceiling. Kept modest because a small-context local model errors +# outright if asked for more than its context holds. +HANKAN_OPENAI_MAX_TOKENS=2048 + # The docker network your existing Postgres and Caddy are on. # docker network ls HANKAN_NETWORK=caddy_default diff --git a/server/README.md b/server/README.md index 09599e4..fd434ee 100644 --- a/server/README.md +++ b/server/README.md @@ -152,8 +152,50 @@ and is byte-identical for as long as the learner stays in one unit — many turns — so every turn after the first reads the prefix at a fraction of the input price. This is the single biggest cost lever in the design. +### Choosing a model + +`HANKAN_TUTOR_BACKEND` picks one of three: + +| | | +|---|---| +| `anthropic` (default) | The Claude API. `ANTHROPIC_API_KEY`. | +| `openai` | Anything speaking OpenAI's `/chat/completions`: LM Studio, Ollama, llama.cpp, vLLM, LiteLLM, OpenRouter, OpenAI. | +| `echo` | No model at all. Reflects the request back, for proving a deployment. | + +For `openai`, set `HANKAN_OPENAI_BASE_URL` and `HANKAN_OPENAI_MODEL`; +`HANKAN_OPENAI_API_KEY` is only needed by hosted endpoints. **From inside +the container, `localhost` is the container** — a model server on the Pi +itself is `http://host.docker.internal:1234/v1`, which `compose.yaml` maps +for you. + +Verified against LM Studio serving `openai/gpt-oss-20b`: a turn streams end +to end from the app, through this server, to the model and back. + +**What the OpenAI path costs you: prompt caching.** The Anthropic backend +marks the ~12k-character gate as a cached prefix, so every turn after the +first reads it at a fraction of the input price. There is no portable +equivalent, so against a *paid hosted* endpoint the whole system prompt is +re-billed every turn — the single biggest cost lever in the design, gone. + +Against a local model it costs nothing, and the shape still pays. llama.cpp +and LM Studio reuse their KV cache for an unchanged prefix, and the system +prompt is byte-identical for as long as the learner stays in one unit. +Measured on a 6,948-character prompt against gpt-oss-20b: **first token +1,563ms cold, 324ms with the prefix already warm.** Which is why the system +prompt goes first and stays put in this backend too. + +A reasoning model served locally often emits chain-of-thought inline in +`content` rather than in a separate field. `` blocks are stripped +from the stream, including when the tags arrive split across chunks — +otherwise they land in the lesson transcript and the block parser reads +them as prose. + `backends/` holds the seam. `anthropic.ts` is the Claude API and is the -default. `echo.ts` needs no API key and reflects the request back, chunk by +default. `openai.ts` is written against `fetch` rather than the `openai` +package: the Anthropic SDK alone is 14MB in the image, this backend uses +one endpoint with no tools and no retries, and local servers are the ones +most likely to deviate from an SDK's expectations. `echo.ts` needs no API +key and reflects the request back, chunk by chunk — set `HANKAN_TUTOR_BACKEND=echo` to prove a deployment (container, proxy, token, CORS, SSE through Caddy) from the phone before a key is involved and before anything is billed. Five things that can each break on diff --git a/server/compose.yaml b/server/compose.yaml index 2fc857b..f86f80e 100644 --- a/server/compose.yaml +++ b/server/compose.yaml @@ -21,9 +21,18 @@ services: HANKAN_TOKEN: ${HANKAN_TOKEN} ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY} HANKAN_TUTOR_BACKEND: ${HANKAN_TUTOR_BACKEND:-anthropic} + HANKAN_OPENAI_BASE_URL: ${HANKAN_OPENAI_BASE_URL:-} + HANKAN_OPENAI_MODEL: ${HANKAN_OPENAI_MODEL:-} + HANKAN_OPENAI_API_KEY: ${HANKAN_OPENAI_API_KEY:-} + HANKAN_OPENAI_MAX_TOKENS: ${HANKAN_OPENAI_MAX_TOKENS:-} + HANKAN_ALLOWED_ORIGINS: ${HANKAN_ALLOWED_ORIGINS:-*} PORT: 8787 networks: - hankan-net + # Lets HANKAN_OPENAI_BASE_URL point at a model server running on the Pi + # itself rather than in this compose file. Ignored if unused. + extra_hosts: + - "host.docker.internal:host-gateway" # No ports published: Caddy is on the same network and proxies to # hankan:8787 by name, so the service is never exposed directly. diff --git a/server/src/backends/openai.ts b/server/src/backends/openai.ts new file mode 100644 index 0000000..027308d --- /dev/null +++ b/server/src/backends/openai.ts @@ -0,0 +1,248 @@ +/* Any OpenAI-compatible /chat/completions endpoint. + + LM Studio, Ollama, llama.cpp, vLLM, LiteLLM, OpenRouter, OpenAI itself — + they all speak the same streaming shape, so one backend covers the lot + and the model becomes a config line rather than a code change. + + Written against fetch rather than the openai package on purpose. The + Anthropic SDK alone is 14MB in the image; a second one buys little here, + because this backend uses one endpoint with no tools and no retries, and + local servers are the ones most likely to deviate from an SDK's + expectations. The real risk in hand-rolling it is SSE reassembly — a JSON + payload split across two TCP reads — so that case is pinned by a test + that deliberately splits mid-object. + + WHAT THIS COSTS YOU: prompt caching. The Anthropic backend marks the + ~12k-character gate as a cached prefix, so every turn after the first + reads it at a fraction of the input price. There is no portable + equivalent here. Against a paid hosted endpoint that means re-billing the + whole system prompt every turn. Against a local model it costs nothing — + and llama.cpp and LM Studio reuse their KV cache for an unchanged prefix + anyway, so keeping the system prompt first and stable still pays for + itself in latency. */ + +import type { TutorBackend, TutorEvent, TutorRequest } from "./types.ts"; + +export interface OpenAIOptions { + baseUrl?: string; + apiKey?: string; + model?: string; + maxTokens?: number; +} + +/* LM Studio's default. Ollama is :11434/v1, llama.cpp :8080/v1. */ +const DEFAULT_BASE_URL = "http://localhost:1234/v1"; + +/* LM Studio accepts any string here and serves whatever is loaded. A hosted + endpoint needs the real id, so this is only a convenience default. */ +const DEFAULT_MODEL = "local-model"; + +/* A tutor turn is a lesson intro at most — the prompt caps it at 250-450 + words. Deliberately far below the Anthropic backend's 16k: this one may + be pointed at a 4k-context local model, where asking for more completion + tokens than the context holds is an outright error rather than a ceiling. */ +const DEFAULT_MAX_TOKENS = 2048; + +/** + * Strip blocks from a token stream. + * + * Reasoning models served locally (the DeepSeek-R1 distills, Qwen's + * thinking variants) emit chain-of-thought inline in `content` rather than + * in a separate field. Left in, it lands in the lesson transcript and the + * block parser reads it as prose. + * + * The tags arrive split across chunks, so this holds back any trailing text + * that could still turn out to be the start of one. + */ +export function makeThinkStripper(): (chunk: string) => string { + const OPEN = ""; + const CLOSE = ""; + let buffer = ""; + let inside = false; + + /** Length of the longest suffix of `s` that is a prefix of `tag`. */ + const heldBack = (s: string, tag: string): number => { + const max = Math.min(s.length, tag.length - 1); + for (let n = max; n > 0; n--) if (tag.startsWith(s.slice(s.length - n))) return n; + return 0; + }; + + return (chunk: string): string => { + buffer += chunk; + let out = ""; + + for (;;) { + if (!inside) { + const at = buffer.indexOf(OPEN); + if (at >= 0) { + out += buffer.slice(0, at); + buffer = buffer.slice(at + OPEN.length); + inside = true; + continue; + } + const keep = heldBack(buffer, OPEN); + out += buffer.slice(0, buffer.length - keep); + buffer = buffer.slice(buffer.length - keep); + return out; + } + + const at = buffer.indexOf(CLOSE); + if (at >= 0) { + buffer = buffer.slice(at + CLOSE.length); + inside = false; + continue; + } + // Still reasoning: drop everything except a possible partial tag. + buffer = buffer.slice(buffer.length - heldBack(buffer, CLOSE)); + return out; + } + }; +} + +interface Choice { + delta?: { content?: string | null }; + finish_reason?: string | null; +} +interface Chunk { + choices?: Choice[]; + usage?: unknown; + error?: { message?: string } | string; +} + +export function openaiBackend(opts: OpenAIOptions = {}): TutorBackend { + const baseUrl = (opts.baseUrl ?? process.env.HANKAN_OPENAI_BASE_URL ?? DEFAULT_BASE_URL).replace( + /\/$/, + "", + ); + const apiKey = opts.apiKey ?? process.env.HANKAN_OPENAI_API_KEY ?? ""; + const model = opts.model ?? process.env.HANKAN_OPENAI_MODEL ?? DEFAULT_MODEL; + const maxTokens = + opts.maxTokens ?? Number(process.env.HANKAN_OPENAI_MAX_TOKENS ?? DEFAULT_MAX_TOKENS); + + return { + name: `openai:${model}`, + + async *stream(req: TutorRequest): AsyncIterable { + /* The system prompt is a message here, not a parameter. It stays + first and unchanged for the whole unit, which is what lets a local + server reuse its KV cache across turns. */ + const messages = [ + { role: "system", content: req.system }, + ...req.history.map((t) => ({ role: t.role, content: t.content })), + { role: "user", content: req.message }, + ]; + + let res: Response; + try { + res = await fetch(`${baseUrl}/chat/completions`, { + method: "POST", + signal: req.signal, + headers: { + "content-type": "application/json", + // Local servers ignore it; hosted ones require it. Omitted + // rather than sent empty, because some servers reject "Bearer ". + ...(apiKey ? { authorization: `Bearer ${apiKey}` } : {}), + }, + body: JSON.stringify({ model, messages, max_tokens: maxTokens, stream: true }), + }); + } catch (err) { + if (req.signal.aborted) { + yield { type: "done", stopReason: "cancelled" }; + return; + } + yield { + type: "error", + code: "unreachable", + message: `Could not reach ${baseUrl} — ${err instanceof Error ? err.message : String(err)}`, + }; + return; + } + + if (!res.ok || !res.body) { + // These bodies are worth surfacing verbatim: "model not loaded" and + // "context length exceeded" are the two most common local failures + // and both tell you exactly what to change. + const body = await res.text().catch(() => ""); + yield { + type: "error", + code: `http_${res.status}`, + message: `${baseUrl} returned ${res.status}${body ? ` — ${body.slice(0, 400)}` : ""}`, + }; + return; + } + + const reader = res.body.getReader(); + const decoder = new TextDecoder(); + const strip = makeThinkStripper(); + let buffer = ""; + let finish: string | null = null; + let usage: unknown; + + try { + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + buffer += decoder.decode(value, { stream: true }); + + /* Split on the blank line that terminates an SSE event, and keep + the remainder — an event, and the JSON inside it, is routinely + delivered across two reads. */ + let cut: number; + while ((cut = buffer.search(/\r?\n\r?\n/)) !== -1) { + const raw = buffer.slice(0, cut); + buffer = buffer.slice(cut).replace(/^\r?\n\r?\n/, ""); + + const data = raw + .split(/\r?\n/) + .filter((l) => l.startsWith("data:")) + .map((l) => l.slice(5).trim()) + .join(""); + if (!data) continue; // a `:` heartbeat, or a field we ignore + if (data === "[DONE]") { + yield { type: "done", stopReason: finish, usage }; + return; + } + + let chunk: Chunk; + try { + chunk = JSON.parse(data) as Chunk; + } catch { + continue; // not ours to interpret; the stream is still fine + } + + if (chunk.error) { + const message = + typeof chunk.error === "string" ? chunk.error : (chunk.error.message ?? "error"); + yield { type: "error", code: "upstream", message }; + return; + } + + if (chunk.usage) usage = chunk.usage; + const choice = chunk.choices?.[0]; + if (choice?.finish_reason) finish = choice.finish_reason; + const text = choice?.delta?.content; + if (text) { + const visible = strip(text); + if (visible) yield { type: "delta", text: visible }; + } + } + } + + // Some servers close the stream without ever sending [DONE]. + yield { type: "done", stopReason: finish, usage }; + } catch (err) { + if (req.signal.aborted) { + yield { type: "done", stopReason: "cancelled" }; + return; + } + yield { + type: "error", + code: "stream", + message: err instanceof Error ? err.message : String(err), + }; + } finally { + await reader.cancel().catch(() => {}); + } + }, + }; +} diff --git a/server/src/tutor.ts b/server/src/tutor.ts index 083d57c..e9fa1af 100644 --- a/server/src/tutor.ts +++ b/server/src/tutor.ts @@ -16,6 +16,7 @@ import { streamSSE } from "hono/streaming"; import { anthropicBackend } from "./backends/anthropic.ts"; import { echoBackend } from "./backends/echo.ts"; +import { openaiBackend } from "./backends/openai.ts"; import type { TutorBackend, TutorTurn } from "./backends/types.ts"; const HEARTBEAT_MS = 15_000; @@ -28,7 +29,10 @@ function pickBackend(): TutorBackend { if (name === "anthropic") return anthropicBackend(); // Keyless, for proving a deployment before any billing happens. if (name === "echo") return echoBackend(); - throw new Error(`unknown HANKAN_TUTOR_BACKEND: ${name}`); + // Anything speaking OpenAI's /chat/completions: LM Studio, Ollama, + // llama.cpp, vLLM, LiteLLM, OpenRouter, OpenAI. + if (name === "openai") return openaiBackend(); + throw new Error(`unknown HANKAN_TUTOR_BACKEND: ${name} (anthropic | openai | echo)`); } export function tutorRoute(backend?: TutorBackend) { diff --git a/test/server/http.test.ts b/test/server/http.test.ts index 8b64331..1068598 100644 --- a/test/server/http.test.ts +++ b/test/server/http.test.ts @@ -85,8 +85,13 @@ run("tutor over HTTP", () => { expect(events.some((e) => e.type === "delta")).toBe(true); expect(events[events.length - 1]!.type).toBe("done"); + /* Deliberately not asserting the content. This file tests the + transport, and the server may be configured with any backend — + anthropic, openai or echo. Asserting the echo backend's wording made + the test fail the moment the server was pointed at a real model, + which is exactly the case it should have kept working through. */ const text = events.filter((e) => e.type === "delta").map((e) => e.text).join(""); - expect(text).toContain("hello"); // the echo backend reflects the message + expect(text.length).toBeGreaterThan(0); }); it("rejects a request with no system prompt", async () => { diff --git a/test/server/openai-backend.test.ts b/test/server/openai-backend.test.ts new file mode 100644 index 0000000..d7672fe --- /dev/null +++ b/test/server/openai-backend.test.ts @@ -0,0 +1,203 @@ +/* The OpenAI-compatible backend, against a fake server that behaves the way + real ones do — including the awkward ways. + + The whole point of this backend is that it is not written against an SDK, + so the parsing is ours to get right. The cases that actually break a + hand-rolled SSE reader are here: a JSON payload split across two writes, + an event split across two writes, heartbeat comment lines, CRLF framing, + and a stream that ends without ever sending [DONE]. */ + +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { createServer, type Server } from "node:http"; +import { openaiBackend, makeThinkStripper } from "../../server/src/backends/openai.ts"; +import type { TutorEvent } from "../../server/src/backends/types.ts"; + +/** What the next request should write, as a list of raw TCP writes. */ +let script: string[] = []; +let lastBody: Record = {}; +let status = 200; +let server: Server; +let base = ""; + +beforeAll(async () => { + server = createServer((req, res) => { + let body = ""; + req.on("data", (c) => (body += c)); + req.on("end", async () => { + lastBody = body ? (JSON.parse(body) as Record) : {}; + if (status !== 200) { + res.writeHead(status, { "content-type": "application/json" }); + res.end(JSON.stringify({ error: { message: "model not loaded" } })); + return; + } + res.writeHead(200, { "content-type": "text/event-stream" }); + for (const piece of script) { + res.write(piece); + // Force separate reads, which is what splits a JSON payload. + await new Promise((r) => setTimeout(r, 5)); + } + res.end(); + }); + }); + await new Promise((r) => server.listen(0, "127.0.0.1", r)); + const addr = server.address(); + base = `http://127.0.0.1:${typeof addr === "object" && addr ? addr.port : 0}/v1`; +}); + +afterAll(() => new Promise((r) => server.close(() => r()))); + +const delta = (text: string) => + `data: ${JSON.stringify({ choices: [{ delta: { content: text } }] })}\n\n`; + +async function collect(message = "hi"): Promise { + const backend = openaiBackend({ baseUrl: base, model: "test-model" }); + const out: TutorEvent[] = []; + for await (const e of backend.stream({ + system: "SYSTEM", + history: [{ role: "user", content: "earlier" }], + message, + signal: new AbortController().signal, + })) { + out.push(e); + } + return out; +} + +const textOf = (events: TutorEvent[]) => + events.filter((e) => e.type === "delta").map((e) => (e as { text: string }).text).join(""); + +describe("openai backend", () => { + beforeAll(() => { + status = 200; + }); + + it("streams deltas and finishes", async () => { + script = [delta("안녕"), delta("하세요"), 'data: [DONE]\n\n']; + const events = await collect(); + expect(textOf(events)).toBe("안녕하세요"); + expect(events[events.length - 1]!.type).toBe("done"); + }); + + it("sends the system prompt as the first message, before the history", async () => { + script = ['data: [DONE]\n\n']; + await collect("what now"); + const msgs = lastBody.messages as { role: string; content: string }[]; + expect(msgs[0]).toEqual({ role: "system", content: "SYSTEM" }); + expect(msgs[1]).toEqual({ role: "user", content: "earlier" }); + expect(msgs[2]).toEqual({ role: "user", content: "what now" }); + expect(lastBody.stream).toBe(true); + expect(lastBody.model).toBe("test-model"); + }); + + it("reassembles a JSON payload split across two reads", async () => { + const whole = delta("split me"); + script = [whole.slice(0, 30), whole.slice(30), 'data: [DONE]\n\n']; + expect(textOf(await collect())).toBe("split me"); + }); + + it("reassembles an event whose blank-line terminator lands in the next read", async () => { + const a = delta("one"); + script = [a.slice(0, a.length - 1), a.slice(a.length - 1) + delta("two"), 'data: [DONE]\n\n']; + expect(textOf(await collect())).toBe("onetwo"); + }); + + it("ignores heartbeat comments and tolerates CRLF framing", async () => { + script = [ + ": ping\n\n", + `data: ${JSON.stringify({ choices: [{ delta: { content: "ok" } }] })}\r\n\r\n`, + 'data: [DONE]\n\n', + ]; + expect(textOf(await collect())).toBe("ok"); + }); + + it("ends cleanly when the server never sends [DONE]", async () => { + script = [delta("abrupt")]; + const events = await collect(); + expect(textOf(events)).toBe("abrupt"); + expect(events[events.length - 1]!.type).toBe("done"); + }); + + it("reports finish_reason", async () => { + script = [ + `data: ${JSON.stringify({ choices: [{ delta: { content: "x" }, finish_reason: "length" }] })}\n\n`, + 'data: [DONE]\n\n', + ]; + const done = (await collect()).at(-1) as { type: string; stopReason: string }; + expect(done.stopReason).toBe("length"); + }); + + it("surfaces an HTTP error body rather than an empty reply", async () => { + status = 400; + const events = await collect(); + status = 200; + const err = events[0] as { type: string; message: string }; + expect(err.type).toBe("error"); + // "model not loaded" is the most common local failure; it must reach the UI. + expect(err.message).toContain("model not loaded"); + }); + + it("reports an unreachable server instead of hanging", async () => { + const backend = openaiBackend({ baseUrl: "http://127.0.0.1:1/v1" }); + const out: TutorEvent[] = []; + for await (const e of backend.stream({ + system: "s", + history: [], + message: "m", + signal: new AbortController().signal, + })) { + out.push(e); + } + const err = out[0] as { type: string; code: string }; + expect(err.type).toBe("error"); + expect(err.code).toBe("unreachable"); + }); + + it("stops on abort without reporting a failure", async () => { + script = [delta("a"), delta("b"), delta("c"), 'data: [DONE]\n\n']; + const backend = openaiBackend({ baseUrl: base }); + const ctrl = new AbortController(); + const out: TutorEvent[] = []; + for await (const e of backend.stream({ + system: "s", + history: [], + message: "m", + signal: ctrl.signal, + })) { + out.push(e); + if (e.type === "delta") ctrl.abort(); + } + const last = out.at(-1) as { type: string; stopReason: string }; + expect(last.type).toBe("done"); + expect(last.stopReason).toBe("cancelled"); + }); +}); + +describe(" stripping", () => { + it("removes a reasoning block that arrives whole", () => { + const strip = makeThinkStripper(); + expect(strip("hmm안녕")).toBe("안녕"); + }); + + it("removes one split across many chunks, tags included", () => { + const strip = makeThinkStripper(); + const out = ["let me ", "reason", "안녕", "하세요"] + .map(strip) + .join(""); + expect(out).toBe("안녕하세요"); + }); + + it("never holds back ordinary text", () => { + const strip = makeThinkStripper(); + expect(["a", "b", "c"].map(strip).join("")).toBe("abc"); + }); + + it("keeps text on both sides of the block", () => { + const strip = makeThinkStripper(); + expect(["before x", "yz after"].map(strip).join("")).toBe("before after"); + }); + + it("does not eat a lone angle bracket that is not a tag", () => { + const strip = makeThinkStripper(); + expect(["1 < 2", " and 3 > 2"].map(strip).join("")).toBe("1 < 2 and 3 > 2"); + }); +});