feat(server): an OpenAI-compatible backend, so the model is yours to pick
HANKAN_TUTOR_BACKEND=openai talks to anything serving /chat/completions -- LM Studio, Ollama, llama.cpp, vLLM, LiteLLM, OpenRouter, OpenAI. The TutorBackend seam already existed for this, so the model becomes a config line rather than a code change. Written against fetch rather than the openai package. The Anthropic SDK alone is 14MB in the image, this backend uses one endpoint with no tools and no retries, and local servers are the ones most likely to deviate from an SDK's expectations. The real risk in hand-rolling it is SSE reassembly, so that is where the tests are: a JSON payload split across two TCP reads, an event whose blank-line terminator lands in the next read, heartbeat comments, CRLF framing, and a stream that ends without [DONE]. The two split cases both fail against a naive per-read parser, which is what makes them worth having. <think> blocks are stripped from the stream, tags split across chunks included. Reasoning models served locally often emit chain-of-thought inline in `content` rather than in a separate field, and left in it lands in the lesson transcript where the block parser reads it as prose. WHAT THIS COSTS: prompt caching. The Anthropic backend marks the ~12k character gate as a cached prefix, so every turn after the first reads it at a fraction of the input price. There is no portable equivalent, so against a paid hosted endpoint the system prompt is re-billed every turn -- the biggest cost lever in the design, gone. Against a local model it costs nothing, and the shape still pays: llama.cpp and LM Studio reuse their KV cache for an unchanged prefix. Measured on a 6,948-character prompt against gpt-oss-20b, first token 1,563ms cold and 324ms warm, so the system prompt goes first and stays put here too. Verified against LM Studio running openai/gpt-oss-20b, not only a fake: a turn streams from the browser through this server to the model and back, rendered in the chat, no page errors. Also makes test/server/http.test.ts backend-agnostic. It asserted the echo backend's wording and so failed the moment the server was pointed at a real model -- precisely the case a transport test should survive. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
MechaCat02
@@ -85,8 +85,13 @@ run("tutor over HTTP", () => {
|
||||
expect(events.some((e) => e.type === "delta")).toBe(true);
|
||||
expect(events[events.length - 1]!.type).toBe("done");
|
||||
|
||||
/* Deliberately not asserting the content. This file tests the
|
||||
transport, and the server may be configured with any backend —
|
||||
anthropic, openai or echo. Asserting the echo backend's wording made
|
||||
the test fail the moment the server was pointed at a real model,
|
||||
which is exactly the case it should have kept working through. */
|
||||
const text = events.filter((e) => e.type === "delta").map((e) => e.text).join("");
|
||||
expect(text).toContain("hello"); // the echo backend reflects the message
|
||||
expect(text.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("rejects a request with no system prompt", async () => {
|
||||
|
||||
203
test/server/openai-backend.test.ts
Normal file
203
test/server/openai-backend.test.ts
Normal file
@@ -0,0 +1,203 @@
|
||||
/* The OpenAI-compatible backend, against a fake server that behaves the way
|
||||
real ones do — including the awkward ways.
|
||||
|
||||
The whole point of this backend is that it is not written against an SDK,
|
||||
so the parsing is ours to get right. The cases that actually break a
|
||||
hand-rolled SSE reader are here: a JSON payload split across two writes,
|
||||
an event split across two writes, heartbeat comment lines, CRLF framing,
|
||||
and a stream that ends without ever sending [DONE]. */
|
||||
|
||||
import { describe, it, expect, beforeAll, afterAll } from "vitest";
|
||||
import { createServer, type Server } from "node:http";
|
||||
import { openaiBackend, makeThinkStripper } from "../../server/src/backends/openai.ts";
|
||||
import type { TutorEvent } from "../../server/src/backends/types.ts";
|
||||
|
||||
/** What the next request should write, as a list of raw TCP writes. */
|
||||
let script: string[] = [];
|
||||
let lastBody: Record<string, unknown> = {};
|
||||
let status = 200;
|
||||
let server: Server;
|
||||
let base = "";
|
||||
|
||||
beforeAll(async () => {
|
||||
server = createServer((req, res) => {
|
||||
let body = "";
|
||||
req.on("data", (c) => (body += c));
|
||||
req.on("end", async () => {
|
||||
lastBody = body ? (JSON.parse(body) as Record<string, unknown>) : {};
|
||||
if (status !== 200) {
|
||||
res.writeHead(status, { "content-type": "application/json" });
|
||||
res.end(JSON.stringify({ error: { message: "model not loaded" } }));
|
||||
return;
|
||||
}
|
||||
res.writeHead(200, { "content-type": "text/event-stream" });
|
||||
for (const piece of script) {
|
||||
res.write(piece);
|
||||
// Force separate reads, which is what splits a JSON payload.
|
||||
await new Promise((r) => setTimeout(r, 5));
|
||||
}
|
||||
res.end();
|
||||
});
|
||||
});
|
||||
await new Promise<void>((r) => server.listen(0, "127.0.0.1", r));
|
||||
const addr = server.address();
|
||||
base = `http://127.0.0.1:${typeof addr === "object" && addr ? addr.port : 0}/v1`;
|
||||
});
|
||||
|
||||
afterAll(() => new Promise<void>((r) => server.close(() => r())));
|
||||
|
||||
const delta = (text: string) =>
|
||||
`data: ${JSON.stringify({ choices: [{ delta: { content: text } }] })}\n\n`;
|
||||
|
||||
async function collect(message = "hi"): Promise<TutorEvent[]> {
|
||||
const backend = openaiBackend({ baseUrl: base, model: "test-model" });
|
||||
const out: TutorEvent[] = [];
|
||||
for await (const e of backend.stream({
|
||||
system: "SYSTEM",
|
||||
history: [{ role: "user", content: "earlier" }],
|
||||
message,
|
||||
signal: new AbortController().signal,
|
||||
})) {
|
||||
out.push(e);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const textOf = (events: TutorEvent[]) =>
|
||||
events.filter((e) => e.type === "delta").map((e) => (e as { text: string }).text).join("");
|
||||
|
||||
describe("openai backend", () => {
|
||||
beforeAll(() => {
|
||||
status = 200;
|
||||
});
|
||||
|
||||
it("streams deltas and finishes", async () => {
|
||||
script = [delta("안녕"), delta("하세요"), 'data: [DONE]\n\n'];
|
||||
const events = await collect();
|
||||
expect(textOf(events)).toBe("안녕하세요");
|
||||
expect(events[events.length - 1]!.type).toBe("done");
|
||||
});
|
||||
|
||||
it("sends the system prompt as the first message, before the history", async () => {
|
||||
script = ['data: [DONE]\n\n'];
|
||||
await collect("what now");
|
||||
const msgs = lastBody.messages as { role: string; content: string }[];
|
||||
expect(msgs[0]).toEqual({ role: "system", content: "SYSTEM" });
|
||||
expect(msgs[1]).toEqual({ role: "user", content: "earlier" });
|
||||
expect(msgs[2]).toEqual({ role: "user", content: "what now" });
|
||||
expect(lastBody.stream).toBe(true);
|
||||
expect(lastBody.model).toBe("test-model");
|
||||
});
|
||||
|
||||
it("reassembles a JSON payload split across two reads", async () => {
|
||||
const whole = delta("split me");
|
||||
script = [whole.slice(0, 30), whole.slice(30), 'data: [DONE]\n\n'];
|
||||
expect(textOf(await collect())).toBe("split me");
|
||||
});
|
||||
|
||||
it("reassembles an event whose blank-line terminator lands in the next read", async () => {
|
||||
const a = delta("one");
|
||||
script = [a.slice(0, a.length - 1), a.slice(a.length - 1) + delta("two"), 'data: [DONE]\n\n'];
|
||||
expect(textOf(await collect())).toBe("onetwo");
|
||||
});
|
||||
|
||||
it("ignores heartbeat comments and tolerates CRLF framing", async () => {
|
||||
script = [
|
||||
": ping\n\n",
|
||||
`data: ${JSON.stringify({ choices: [{ delta: { content: "ok" } }] })}\r\n\r\n`,
|
||||
'data: [DONE]\n\n',
|
||||
];
|
||||
expect(textOf(await collect())).toBe("ok");
|
||||
});
|
||||
|
||||
it("ends cleanly when the server never sends [DONE]", async () => {
|
||||
script = [delta("abrupt")];
|
||||
const events = await collect();
|
||||
expect(textOf(events)).toBe("abrupt");
|
||||
expect(events[events.length - 1]!.type).toBe("done");
|
||||
});
|
||||
|
||||
it("reports finish_reason", async () => {
|
||||
script = [
|
||||
`data: ${JSON.stringify({ choices: [{ delta: { content: "x" }, finish_reason: "length" }] })}\n\n`,
|
||||
'data: [DONE]\n\n',
|
||||
];
|
||||
const done = (await collect()).at(-1) as { type: string; stopReason: string };
|
||||
expect(done.stopReason).toBe("length");
|
||||
});
|
||||
|
||||
it("surfaces an HTTP error body rather than an empty reply", async () => {
|
||||
status = 400;
|
||||
const events = await collect();
|
||||
status = 200;
|
||||
const err = events[0] as { type: string; message: string };
|
||||
expect(err.type).toBe("error");
|
||||
// "model not loaded" is the most common local failure; it must reach the UI.
|
||||
expect(err.message).toContain("model not loaded");
|
||||
});
|
||||
|
||||
it("reports an unreachable server instead of hanging", async () => {
|
||||
const backend = openaiBackend({ baseUrl: "http://127.0.0.1:1/v1" });
|
||||
const out: TutorEvent[] = [];
|
||||
for await (const e of backend.stream({
|
||||
system: "s",
|
||||
history: [],
|
||||
message: "m",
|
||||
signal: new AbortController().signal,
|
||||
})) {
|
||||
out.push(e);
|
||||
}
|
||||
const err = out[0] as { type: string; code: string };
|
||||
expect(err.type).toBe("error");
|
||||
expect(err.code).toBe("unreachable");
|
||||
});
|
||||
|
||||
it("stops on abort without reporting a failure", async () => {
|
||||
script = [delta("a"), delta("b"), delta("c"), 'data: [DONE]\n\n'];
|
||||
const backend = openaiBackend({ baseUrl: base });
|
||||
const ctrl = new AbortController();
|
||||
const out: TutorEvent[] = [];
|
||||
for await (const e of backend.stream({
|
||||
system: "s",
|
||||
history: [],
|
||||
message: "m",
|
||||
signal: ctrl.signal,
|
||||
})) {
|
||||
out.push(e);
|
||||
if (e.type === "delta") ctrl.abort();
|
||||
}
|
||||
const last = out.at(-1) as { type: string; stopReason: string };
|
||||
expect(last.type).toBe("done");
|
||||
expect(last.stopReason).toBe("cancelled");
|
||||
});
|
||||
});
|
||||
|
||||
describe("<think> stripping", () => {
|
||||
it("removes a reasoning block that arrives whole", () => {
|
||||
const strip = makeThinkStripper();
|
||||
expect(strip("<think>hmm</think>안녕")).toBe("안녕");
|
||||
});
|
||||
|
||||
it("removes one split across many chunks, tags included", () => {
|
||||
const strip = makeThinkStripper();
|
||||
const out = ["<thi", "nk>let me ", "reason</thi", "nk>", "안녕", "하세요"]
|
||||
.map(strip)
|
||||
.join("");
|
||||
expect(out).toBe("안녕하세요");
|
||||
});
|
||||
|
||||
it("never holds back ordinary text", () => {
|
||||
const strip = makeThinkStripper();
|
||||
expect(["a", "b", "c"].map(strip).join("")).toBe("abc");
|
||||
});
|
||||
|
||||
it("keeps text on both sides of the block", () => {
|
||||
const strip = makeThinkStripper();
|
||||
expect(["before <think>x", "yz</think> after"].map(strip).join("")).toBe("before after");
|
||||
});
|
||||
|
||||
it("does not eat a lone angle bracket that is not a tag", () => {
|
||||
const strip = makeThinkStripper();
|
||||
expect(["1 < 2", " and 3 > 2"].map(strip).join("")).toBe("1 < 2 and 3 > 2");
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user