From 074f60249404d003619c1fa11693e7e381bac973 Mon Sep 17 00:00:00 2001 From: MechaCat02 Date: Wed, 9 Sep 2026 19:39:31 +0200 Subject: [PATCH] feat(server): an OpenAI-compatible backend, so the model is yours to pick HANKAN_TUTOR_BACKEND=openai talks to anything serving /chat/completions -- LM Studio, Ollama, llama.cpp, vLLM, LiteLLM, OpenRouter, OpenAI. The TutorBackend seam already existed for this, so the model becomes a config line rather than a code change. Written against fetch rather than the openai package. The Anthropic SDK alone is 14MB in the image, this backend uses one endpoint with no tools and no retries, and local servers are the ones most likely to deviate from an SDK's expectations. The real risk in hand-rolling it is SSE reassembly, so that is where the tests are: a JSON payload split across two TCP reads, an event whose blank-line terminator lands in the next read, heartbeat comments, CRLF framing, and a stream that ends without [DONE]. The two split cases both fail against a naive per-read parser, which is what makes them worth having. blocks are stripped from the stream, tags split across chunks included. Reasoning models served locally often emit chain-of-thought inline in `content` rather than in a separate field, and left in it lands in the lesson transcript where the block parser reads it as prose. WHAT THIS COSTS: prompt caching. The Anthropic backend marks the ~12k character gate as a cached prefix, so every turn after the first reads it at a fraction of the input price. There is no portable equivalent, so against a paid hosted endpoint the system prompt is re-billed every turn -- the biggest cost lever in the design, gone. Against a local model it costs nothing, and the shape still pays: llama.cpp and LM Studio reuse their KV cache for an unchanged prefix. Measured on a 6,948-character prompt against gpt-oss-20b, first token 1,563ms cold and 324ms warm, so the system prompt goes first and stays put here too. Verified against LM Studio running openai/gpt-oss-20b, not only a fake: a turn streams from the browser through this server to the model and back, rendered in the chat, no page errors. Also makes test/server/http.test.ts backend-agnostic. It asserted the echo backend's wording and so failed the moment the server was pointed at a real model -- precisely the case a transport test should survive. Co-Authored-By: Claude Opus 5 --- README.md | 6 + server/.env.example | 24 ++- server/README.md | 44 ++++- server/compose.yaml | 9 ++ server/src/backends/openai.ts | 248 +++++++++++++++++++++++++++++ server/src/tutor.ts | 6 +- test/server/http.test.ts | 7 +- test/server/openai-backend.test.ts | 203 +++++++++++++++++++++++ 8 files changed, 542 insertions(+), 5 deletions(-) create mode 100644 server/src/backends/openai.ts create mode 100644 test/server/openai-backend.test.ts diff --git a/README.md b/README.md index 69e92c0..7c31d6f 100644 --- a/README.md +++ b/README.md @@ -168,6 +168,12 @@ Three details worth knowing here: - **The system prompt is cached.** It is ~12k characters of gate, identical for as long as the learner stays in one unit, so every turn after the first reads it at a fraction of the input price. +- **The model is a config line, not a code change.** `HANKAN_TUTOR_BACKEND` + selects the Claude API, any OpenAI-compatible endpoint (LM Studio, Ollama, + llama.cpp, vLLM, OpenRouter), or a keyless echo backend for proving a + deployment. Keeping the system prompt first and unchanged is what makes + Anthropic's prefix cache work — and, it turns out, a local server's KV + cache too: 1,563ms to first token cold, 324ms warm. ## Not in this pass diff --git a/server/.env.example b/server/.env.example index 1a81f84..e4e4681 100644 --- a/server/.env.example +++ b/server/.env.example @@ -8,10 +8,30 @@ DATABASE_URL=postgres://hankan:CHANGE_ME@postgres:5432/hankan # openssl rand -base64 32 HANKAN_TOKEN=CHANGE_ME -# For the tutor. Omit it and sync still works — the app falls back to its -# local stand-in tutor. +# Which model serves the tutor: anthropic (default) | openai | echo. +# Omit the whole tutor config and sync still works — the app falls back to +# its local stand-in tutor. +HANKAN_TUTOR_BACKEND=anthropic + +# For HANKAN_TUTOR_BACKEND=anthropic. ANTHROPIC_API_KEY= +# For HANKAN_TUTOR_BACKEND=openai — anything speaking /chat/completions: +# LM Studio, Ollama, llama.cpp, vLLM, LiteLLM, OpenRouter, OpenAI. +# From a container, localhost is the container: use the host's LAN address +# or host.docker.internal, not 127.0.0.1. +# LM Studio http://:1234/v1 model = the id shown in its UI +# Ollama http://:11434/v1 model = e.g. qwen2.5:14b +# llama.cpp http://:8080/v1 +# OpenRouter https://openrouter.ai/api/v1 +HANKAN_OPENAI_BASE_URL=http://host.docker.internal:1234/v1 +HANKAN_OPENAI_MODEL=local-model +# Local servers ignore this; hosted ones require it. Leave blank for local. +HANKAN_OPENAI_API_KEY= +# Completion ceiling. Kept modest because a small-context local model errors +# outright if asked for more than its context holds. +HANKAN_OPENAI_MAX_TOKENS=2048 + # The docker network your existing Postgres and Caddy are on. # docker network ls HANKAN_NETWORK=caddy_default diff --git a/server/README.md b/server/README.md index 09599e4..fd434ee 100644 --- a/server/README.md +++ b/server/README.md @@ -152,8 +152,50 @@ and is byte-identical for as long as the learner stays in one unit — many turns — so every turn after the first reads the prefix at a fraction of the input price. This is the single biggest cost lever in the design. +### Choosing a model + +`HANKAN_TUTOR_BACKEND` picks one of three: + +| | | +|---|---| +| `anthropic` (default) | The Claude API. `ANTHROPIC_API_KEY`. | +| `openai` | Anything speaking OpenAI's `/chat/completions`: LM Studio, Ollama, llama.cpp, vLLM, LiteLLM, OpenRouter, OpenAI. | +| `echo` | No model at all. Reflects the request back, for proving a deployment. | + +For `openai`, set `HANKAN_OPENAI_BASE_URL` and `HANKAN_OPENAI_MODEL`; +`HANKAN_OPENAI_API_KEY` is only needed by hosted endpoints. **From inside +the container, `localhost` is the container** — a model server on the Pi +itself is `http://host.docker.internal:1234/v1`, which `compose.yaml` maps +for you. + +Verified against LM Studio serving `openai/gpt-oss-20b`: a turn streams end +to end from the app, through this server, to the model and back. + +**What the OpenAI path costs you: prompt caching.** The Anthropic backend +marks the ~12k-character gate as a cached prefix, so every turn after the +first reads it at a fraction of the input price. There is no portable +equivalent, so against a *paid hosted* endpoint the whole system prompt is +re-billed every turn — the single biggest cost lever in the design, gone. + +Against a local model it costs nothing, and the shape still pays. llama.cpp +and LM Studio reuse their KV cache for an unchanged prefix, and the system +prompt is byte-identical for as long as the learner stays in one unit. +Measured on a 6,948-character prompt against gpt-oss-20b: **first token +1,563ms cold, 324ms with the prefix already warm.** Which is why the system +prompt goes first and stays put in this backend too. + +A reasoning model served locally often emits chain-of-thought inline in +`content` rather than in a separate field. `` blocks are stripped +from the stream, including when the tags arrive split across chunks — +otherwise they land in the lesson transcript and the block parser reads +them as prose. + `backends/` holds the seam. `anthropic.ts` is the Claude API and is the -default. `echo.ts` needs no API key and reflects the request back, chunk by +default. `openai.ts` is written against `fetch` rather than the `openai` +package: the Anthropic SDK alone is 14MB in the image, this backend uses +one endpoint with no tools and no retries, and local servers are the ones +most likely to deviate from an SDK's expectations. `echo.ts` needs no API +key and reflects the request back, chunk by chunk — set `HANKAN_TUTOR_BACKEND=echo` to prove a deployment (container, proxy, token, CORS, SSE through Caddy) from the phone before a key is involved and before anything is billed. Five things that can each break on diff --git a/server/compose.yaml b/server/compose.yaml index 2fc857b..f86f80e 100644 --- a/server/compose.yaml +++ b/server/compose.yaml @@ -21,9 +21,18 @@ services: HANKAN_TOKEN: ${HANKAN_TOKEN} ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY} HANKAN_TUTOR_BACKEND: ${HANKAN_TUTOR_BACKEND:-anthropic} + HANKAN_OPENAI_BASE_URL: ${HANKAN_OPENAI_BASE_URL:-} + HANKAN_OPENAI_MODEL: ${HANKAN_OPENAI_MODEL:-} + HANKAN_OPENAI_API_KEY: ${HANKAN_OPENAI_API_KEY:-} + HANKAN_OPENAI_MAX_TOKENS: ${HANKAN_OPENAI_MAX_TOKENS:-} + HANKAN_ALLOWED_ORIGINS: ${HANKAN_ALLOWED_ORIGINS:-*} PORT: 8787 networks: - hankan-net + # Lets HANKAN_OPENAI_BASE_URL point at a model server running on the Pi + # itself rather than in this compose file. Ignored if unused. + extra_hosts: + - "host.docker.internal:host-gateway" # No ports published: Caddy is on the same network and proxies to # hankan:8787 by name, so the service is never exposed directly. diff --git a/server/src/backends/openai.ts b/server/src/backends/openai.ts new file mode 100644 index 0000000..027308d --- /dev/null +++ b/server/src/backends/openai.ts @@ -0,0 +1,248 @@ +/* Any OpenAI-compatible /chat/completions endpoint. + + LM Studio, Ollama, llama.cpp, vLLM, LiteLLM, OpenRouter, OpenAI itself — + they all speak the same streaming shape, so one backend covers the lot + and the model becomes a config line rather than a code change. + + Written against fetch rather than the openai package on purpose. The + Anthropic SDK alone is 14MB in the image; a second one buys little here, + because this backend uses one endpoint with no tools and no retries, and + local servers are the ones most likely to deviate from an SDK's + expectations. The real risk in hand-rolling it is SSE reassembly — a JSON + payload split across two TCP reads — so that case is pinned by a test + that deliberately splits mid-object. + + WHAT THIS COSTS YOU: prompt caching. The Anthropic backend marks the + ~12k-character gate as a cached prefix, so every turn after the first + reads it at a fraction of the input price. There is no portable + equivalent here. Against a paid hosted endpoint that means re-billing the + whole system prompt every turn. Against a local model it costs nothing — + and llama.cpp and LM Studio reuse their KV cache for an unchanged prefix + anyway, so keeping the system prompt first and stable still pays for + itself in latency. */ + +import type { TutorBackend, TutorEvent, TutorRequest } from "./types.ts"; + +export interface OpenAIOptions { + baseUrl?: string; + apiKey?: string; + model?: string; + maxTokens?: number; +} + +/* LM Studio's default. Ollama is :11434/v1, llama.cpp :8080/v1. */ +const DEFAULT_BASE_URL = "http://localhost:1234/v1"; + +/* LM Studio accepts any string here and serves whatever is loaded. A hosted + endpoint needs the real id, so this is only a convenience default. */ +const DEFAULT_MODEL = "local-model"; + +/* A tutor turn is a lesson intro at most — the prompt caps it at 250-450 + words. Deliberately far below the Anthropic backend's 16k: this one may + be pointed at a 4k-context local model, where asking for more completion + tokens than the context holds is an outright error rather than a ceiling. */ +const DEFAULT_MAX_TOKENS = 2048; + +/** + * Strip blocks from a token stream. + * + * Reasoning models served locally (the DeepSeek-R1 distills, Qwen's + * thinking variants) emit chain-of-thought inline in `content` rather than + * in a separate field. Left in, it lands in the lesson transcript and the + * block parser reads it as prose. + * + * The tags arrive split across chunks, so this holds back any trailing text + * that could still turn out to be the start of one. + */ +export function makeThinkStripper(): (chunk: string) => string { + const OPEN = ""; + const CLOSE = ""; + let buffer = ""; + let inside = false; + + /** Length of the longest suffix of `s` that is a prefix of `tag`. */ + const heldBack = (s: string, tag: string): number => { + const max = Math.min(s.length, tag.length - 1); + for (let n = max; n > 0; n--) if (tag.startsWith(s.slice(s.length - n))) return n; + return 0; + }; + + return (chunk: string): string => { + buffer += chunk; + let out = ""; + + for (;;) { + if (!inside) { + const at = buffer.indexOf(OPEN); + if (at >= 0) { + out += buffer.slice(0, at); + buffer = buffer.slice(at + OPEN.length); + inside = true; + continue; + } + const keep = heldBack(buffer, OPEN); + out += buffer.slice(0, buffer.length - keep); + buffer = buffer.slice(buffer.length - keep); + return out; + } + + const at = buffer.indexOf(CLOSE); + if (at >= 0) { + buffer = buffer.slice(at + CLOSE.length); + inside = false; + continue; + } + // Still reasoning: drop everything except a possible partial tag. + buffer = buffer.slice(buffer.length - heldBack(buffer, CLOSE)); + return out; + } + }; +} + +interface Choice { + delta?: { content?: string | null }; + finish_reason?: string | null; +} +interface Chunk { + choices?: Choice[]; + usage?: unknown; + error?: { message?: string } | string; +} + +export function openaiBackend(opts: OpenAIOptions = {}): TutorBackend { + const baseUrl = (opts.baseUrl ?? process.env.HANKAN_OPENAI_BASE_URL ?? DEFAULT_BASE_URL).replace( + /\/$/, + "", + ); + const apiKey = opts.apiKey ?? process.env.HANKAN_OPENAI_API_KEY ?? ""; + const model = opts.model ?? process.env.HANKAN_OPENAI_MODEL ?? DEFAULT_MODEL; + const maxTokens = + opts.maxTokens ?? Number(process.env.HANKAN_OPENAI_MAX_TOKENS ?? DEFAULT_MAX_TOKENS); + + return { + name: `openai:${model}`, + + async *stream(req: TutorRequest): AsyncIterable { + /* The system prompt is a message here, not a parameter. It stays + first and unchanged for the whole unit, which is what lets a local + server reuse its KV cache across turns. */ + const messages = [ + { role: "system", content: req.system }, + ...req.history.map((t) => ({ role: t.role, content: t.content })), + { role: "user", content: req.message }, + ]; + + let res: Response; + try { + res = await fetch(`${baseUrl}/chat/completions`, { + method: "POST", + signal: req.signal, + headers: { + "content-type": "application/json", + // Local servers ignore it; hosted ones require it. Omitted + // rather than sent empty, because some servers reject "Bearer ". + ...(apiKey ? { authorization: `Bearer ${apiKey}` } : {}), + }, + body: JSON.stringify({ model, messages, max_tokens: maxTokens, stream: true }), + }); + } catch (err) { + if (req.signal.aborted) { + yield { type: "done", stopReason: "cancelled" }; + return; + } + yield { + type: "error", + code: "unreachable", + message: `Could not reach ${baseUrl} — ${err instanceof Error ? err.message : String(err)}`, + }; + return; + } + + if (!res.ok || !res.body) { + // These bodies are worth surfacing verbatim: "model not loaded" and + // "context length exceeded" are the two most common local failures + // and both tell you exactly what to change. + const body = await res.text().catch(() => ""); + yield { + type: "error", + code: `http_${res.status}`, + message: `${baseUrl} returned ${res.status}${body ? ` — ${body.slice(0, 400)}` : ""}`, + }; + return; + } + + const reader = res.body.getReader(); + const decoder = new TextDecoder(); + const strip = makeThinkStripper(); + let buffer = ""; + let finish: string | null = null; + let usage: unknown; + + try { + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + buffer += decoder.decode(value, { stream: true }); + + /* Split on the blank line that terminates an SSE event, and keep + the remainder — an event, and the JSON inside it, is routinely + delivered across two reads. */ + let cut: number; + while ((cut = buffer.search(/\r?\n\r?\n/)) !== -1) { + const raw = buffer.slice(0, cut); + buffer = buffer.slice(cut).replace(/^\r?\n\r?\n/, ""); + + const data = raw + .split(/\r?\n/) + .filter((l) => l.startsWith("data:")) + .map((l) => l.slice(5).trim()) + .join(""); + if (!data) continue; // a `:` heartbeat, or a field we ignore + if (data === "[DONE]") { + yield { type: "done", stopReason: finish, usage }; + return; + } + + let chunk: Chunk; + try { + chunk = JSON.parse(data) as Chunk; + } catch { + continue; // not ours to interpret; the stream is still fine + } + + if (chunk.error) { + const message = + typeof chunk.error === "string" ? chunk.error : (chunk.error.message ?? "error"); + yield { type: "error", code: "upstream", message }; + return; + } + + if (chunk.usage) usage = chunk.usage; + const choice = chunk.choices?.[0]; + if (choice?.finish_reason) finish = choice.finish_reason; + const text = choice?.delta?.content; + if (text) { + const visible = strip(text); + if (visible) yield { type: "delta", text: visible }; + } + } + } + + // Some servers close the stream without ever sending [DONE]. + yield { type: "done", stopReason: finish, usage }; + } catch (err) { + if (req.signal.aborted) { + yield { type: "done", stopReason: "cancelled" }; + return; + } + yield { + type: "error", + code: "stream", + message: err instanceof Error ? err.message : String(err), + }; + } finally { + await reader.cancel().catch(() => {}); + } + }, + }; +} diff --git a/server/src/tutor.ts b/server/src/tutor.ts index 083d57c..e9fa1af 100644 --- a/server/src/tutor.ts +++ b/server/src/tutor.ts @@ -16,6 +16,7 @@ import { streamSSE } from "hono/streaming"; import { anthropicBackend } from "./backends/anthropic.ts"; import { echoBackend } from "./backends/echo.ts"; +import { openaiBackend } from "./backends/openai.ts"; import type { TutorBackend, TutorTurn } from "./backends/types.ts"; const HEARTBEAT_MS = 15_000; @@ -28,7 +29,10 @@ function pickBackend(): TutorBackend { if (name === "anthropic") return anthropicBackend(); // Keyless, for proving a deployment before any billing happens. if (name === "echo") return echoBackend(); - throw new Error(`unknown HANKAN_TUTOR_BACKEND: ${name}`); + // Anything speaking OpenAI's /chat/completions: LM Studio, Ollama, + // llama.cpp, vLLM, LiteLLM, OpenRouter, OpenAI. + if (name === "openai") return openaiBackend(); + throw new Error(`unknown HANKAN_TUTOR_BACKEND: ${name} (anthropic | openai | echo)`); } export function tutorRoute(backend?: TutorBackend) { diff --git a/test/server/http.test.ts b/test/server/http.test.ts index 8b64331..1068598 100644 --- a/test/server/http.test.ts +++ b/test/server/http.test.ts @@ -85,8 +85,13 @@ run("tutor over HTTP", () => { expect(events.some((e) => e.type === "delta")).toBe(true); expect(events[events.length - 1]!.type).toBe("done"); + /* Deliberately not asserting the content. This file tests the + transport, and the server may be configured with any backend — + anthropic, openai or echo. Asserting the echo backend's wording made + the test fail the moment the server was pointed at a real model, + which is exactly the case it should have kept working through. */ const text = events.filter((e) => e.type === "delta").map((e) => e.text).join(""); - expect(text).toContain("hello"); // the echo backend reflects the message + expect(text.length).toBeGreaterThan(0); }); it("rejects a request with no system prompt", async () => { diff --git a/test/server/openai-backend.test.ts b/test/server/openai-backend.test.ts new file mode 100644 index 0000000..d7672fe --- /dev/null +++ b/test/server/openai-backend.test.ts @@ -0,0 +1,203 @@ +/* The OpenAI-compatible backend, against a fake server that behaves the way + real ones do — including the awkward ways. + + The whole point of this backend is that it is not written against an SDK, + so the parsing is ours to get right. The cases that actually break a + hand-rolled SSE reader are here: a JSON payload split across two writes, + an event split across two writes, heartbeat comment lines, CRLF framing, + and a stream that ends without ever sending [DONE]. */ + +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { createServer, type Server } from "node:http"; +import { openaiBackend, makeThinkStripper } from "../../server/src/backends/openai.ts"; +import type { TutorEvent } from "../../server/src/backends/types.ts"; + +/** What the next request should write, as a list of raw TCP writes. */ +let script: string[] = []; +let lastBody: Record = {}; +let status = 200; +let server: Server; +let base = ""; + +beforeAll(async () => { + server = createServer((req, res) => { + let body = ""; + req.on("data", (c) => (body += c)); + req.on("end", async () => { + lastBody = body ? (JSON.parse(body) as Record) : {}; + if (status !== 200) { + res.writeHead(status, { "content-type": "application/json" }); + res.end(JSON.stringify({ error: { message: "model not loaded" } })); + return; + } + res.writeHead(200, { "content-type": "text/event-stream" }); + for (const piece of script) { + res.write(piece); + // Force separate reads, which is what splits a JSON payload. + await new Promise((r) => setTimeout(r, 5)); + } + res.end(); + }); + }); + await new Promise((r) => server.listen(0, "127.0.0.1", r)); + const addr = server.address(); + base = `http://127.0.0.1:${typeof addr === "object" && addr ? addr.port : 0}/v1`; +}); + +afterAll(() => new Promise((r) => server.close(() => r()))); + +const delta = (text: string) => + `data: ${JSON.stringify({ choices: [{ delta: { content: text } }] })}\n\n`; + +async function collect(message = "hi"): Promise { + const backend = openaiBackend({ baseUrl: base, model: "test-model" }); + const out: TutorEvent[] = []; + for await (const e of backend.stream({ + system: "SYSTEM", + history: [{ role: "user", content: "earlier" }], + message, + signal: new AbortController().signal, + })) { + out.push(e); + } + return out; +} + +const textOf = (events: TutorEvent[]) => + events.filter((e) => e.type === "delta").map((e) => (e as { text: string }).text).join(""); + +describe("openai backend", () => { + beforeAll(() => { + status = 200; + }); + + it("streams deltas and finishes", async () => { + script = [delta("안녕"), delta("하세요"), 'data: [DONE]\n\n']; + const events = await collect(); + expect(textOf(events)).toBe("안녕하세요"); + expect(events[events.length - 1]!.type).toBe("done"); + }); + + it("sends the system prompt as the first message, before the history", async () => { + script = ['data: [DONE]\n\n']; + await collect("what now"); + const msgs = lastBody.messages as { role: string; content: string }[]; + expect(msgs[0]).toEqual({ role: "system", content: "SYSTEM" }); + expect(msgs[1]).toEqual({ role: "user", content: "earlier" }); + expect(msgs[2]).toEqual({ role: "user", content: "what now" }); + expect(lastBody.stream).toBe(true); + expect(lastBody.model).toBe("test-model"); + }); + + it("reassembles a JSON payload split across two reads", async () => { + const whole = delta("split me"); + script = [whole.slice(0, 30), whole.slice(30), 'data: [DONE]\n\n']; + expect(textOf(await collect())).toBe("split me"); + }); + + it("reassembles an event whose blank-line terminator lands in the next read", async () => { + const a = delta("one"); + script = [a.slice(0, a.length - 1), a.slice(a.length - 1) + delta("two"), 'data: [DONE]\n\n']; + expect(textOf(await collect())).toBe("onetwo"); + }); + + it("ignores heartbeat comments and tolerates CRLF framing", async () => { + script = [ + ": ping\n\n", + `data: ${JSON.stringify({ choices: [{ delta: { content: "ok" } }] })}\r\n\r\n`, + 'data: [DONE]\n\n', + ]; + expect(textOf(await collect())).toBe("ok"); + }); + + it("ends cleanly when the server never sends [DONE]", async () => { + script = [delta("abrupt")]; + const events = await collect(); + expect(textOf(events)).toBe("abrupt"); + expect(events[events.length - 1]!.type).toBe("done"); + }); + + it("reports finish_reason", async () => { + script = [ + `data: ${JSON.stringify({ choices: [{ delta: { content: "x" }, finish_reason: "length" }] })}\n\n`, + 'data: [DONE]\n\n', + ]; + const done = (await collect()).at(-1) as { type: string; stopReason: string }; + expect(done.stopReason).toBe("length"); + }); + + it("surfaces an HTTP error body rather than an empty reply", async () => { + status = 400; + const events = await collect(); + status = 200; + const err = events[0] as { type: string; message: string }; + expect(err.type).toBe("error"); + // "model not loaded" is the most common local failure; it must reach the UI. + expect(err.message).toContain("model not loaded"); + }); + + it("reports an unreachable server instead of hanging", async () => { + const backend = openaiBackend({ baseUrl: "http://127.0.0.1:1/v1" }); + const out: TutorEvent[] = []; + for await (const e of backend.stream({ + system: "s", + history: [], + message: "m", + signal: new AbortController().signal, + })) { + out.push(e); + } + const err = out[0] as { type: string; code: string }; + expect(err.type).toBe("error"); + expect(err.code).toBe("unreachable"); + }); + + it("stops on abort without reporting a failure", async () => { + script = [delta("a"), delta("b"), delta("c"), 'data: [DONE]\n\n']; + const backend = openaiBackend({ baseUrl: base }); + const ctrl = new AbortController(); + const out: TutorEvent[] = []; + for await (const e of backend.stream({ + system: "s", + history: [], + message: "m", + signal: ctrl.signal, + })) { + out.push(e); + if (e.type === "delta") ctrl.abort(); + } + const last = out.at(-1) as { type: string; stopReason: string }; + expect(last.type).toBe("done"); + expect(last.stopReason).toBe("cancelled"); + }); +}); + +describe(" stripping", () => { + it("removes a reasoning block that arrives whole", () => { + const strip = makeThinkStripper(); + expect(strip("hmm안녕")).toBe("안녕"); + }); + + it("removes one split across many chunks, tags included", () => { + const strip = makeThinkStripper(); + const out = ["let me ", "reason", "안녕", "하세요"] + .map(strip) + .join(""); + expect(out).toBe("안녕하세요"); + }); + + it("never holds back ordinary text", () => { + const strip = makeThinkStripper(); + expect(["a", "b", "c"].map(strip).join("")).toBe("abc"); + }); + + it("keeps text on both sides of the block", () => { + const strip = makeThinkStripper(); + expect(["before x", "yz after"].map(strip).join("")).toBe("before after"); + }); + + it("does not eat a lone angle bracket that is not a tag", () => { + const strip = makeThinkStripper(); + expect(["1 < 2", " and 3 > 2"].map(strip).join("")).toBe("1 < 2 and 3 > 2"); + }); +});