feat(server): an OpenAI-compatible backend, so the model is yours to pick
HANKAN_TUTOR_BACKEND=openai talks to anything serving /chat/completions -- LM Studio, Ollama, llama.cpp, vLLM, LiteLLM, OpenRouter, OpenAI. The TutorBackend seam already existed for this, so the model becomes a config line rather than a code change. Written against fetch rather than the openai package. The Anthropic SDK alone is 14MB in the image, this backend uses one endpoint with no tools and no retries, and local servers are the ones most likely to deviate from an SDK's expectations. The real risk in hand-rolling it is SSE reassembly, so that is where the tests are: a JSON payload split across two TCP reads, an event whose blank-line terminator lands in the next read, heartbeat comments, CRLF framing, and a stream that ends without [DONE]. The two split cases both fail against a naive per-read parser, which is what makes them worth having. <think> blocks are stripped from the stream, tags split across chunks included. Reasoning models served locally often emit chain-of-thought inline in `content` rather than in a separate field, and left in it lands in the lesson transcript where the block parser reads it as prose. WHAT THIS COSTS: prompt caching. The Anthropic backend marks the ~12k character gate as a cached prefix, so every turn after the first reads it at a fraction of the input price. There is no portable equivalent, so against a paid hosted endpoint the system prompt is re-billed every turn -- the biggest cost lever in the design, gone. Against a local model it costs nothing, and the shape still pays: llama.cpp and LM Studio reuse their KV cache for an unchanged prefix. Measured on a 6,948-character prompt against gpt-oss-20b, first token 1,563ms cold and 324ms warm, so the system prompt goes first and stays put here too. Verified against LM Studio running openai/gpt-oss-20b, not only a fake: a turn streams from the browser through this server to the model and back, rendered in the chat, no page errors. Also makes test/server/http.test.ts backend-agnostic. It asserted the echo backend's wording and so failed the moment the server was pointed at a real model -- precisely the case a transport test should survive. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
MechaCat02
248
server/src/backends/openai.ts
Normal file
248
server/src/backends/openai.ts
Normal file
@@ -0,0 +1,248 @@
|
||||
/* Any OpenAI-compatible /chat/completions endpoint.
|
||||
|
||||
LM Studio, Ollama, llama.cpp, vLLM, LiteLLM, OpenRouter, OpenAI itself —
|
||||
they all speak the same streaming shape, so one backend covers the lot
|
||||
and the model becomes a config line rather than a code change.
|
||||
|
||||
Written against fetch rather than the openai package on purpose. The
|
||||
Anthropic SDK alone is 14MB in the image; a second one buys little here,
|
||||
because this backend uses one endpoint with no tools and no retries, and
|
||||
local servers are the ones most likely to deviate from an SDK's
|
||||
expectations. The real risk in hand-rolling it is SSE reassembly — a JSON
|
||||
payload split across two TCP reads — so that case is pinned by a test
|
||||
that deliberately splits mid-object.
|
||||
|
||||
WHAT THIS COSTS YOU: prompt caching. The Anthropic backend marks the
|
||||
~12k-character gate as a cached prefix, so every turn after the first
|
||||
reads it at a fraction of the input price. There is no portable
|
||||
equivalent here. Against a paid hosted endpoint that means re-billing the
|
||||
whole system prompt every turn. Against a local model it costs nothing —
|
||||
and llama.cpp and LM Studio reuse their KV cache for an unchanged prefix
|
||||
anyway, so keeping the system prompt first and stable still pays for
|
||||
itself in latency. */
|
||||
|
||||
import type { TutorBackend, TutorEvent, TutorRequest } from "./types.ts";
|
||||
|
||||
export interface OpenAIOptions {
|
||||
baseUrl?: string;
|
||||
apiKey?: string;
|
||||
model?: string;
|
||||
maxTokens?: number;
|
||||
}
|
||||
|
||||
/* LM Studio's default. Ollama is :11434/v1, llama.cpp :8080/v1. */
|
||||
const DEFAULT_BASE_URL = "http://localhost:1234/v1";
|
||||
|
||||
/* LM Studio accepts any string here and serves whatever is loaded. A hosted
|
||||
endpoint needs the real id, so this is only a convenience default. */
|
||||
const DEFAULT_MODEL = "local-model";
|
||||
|
||||
/* A tutor turn is a lesson intro at most — the prompt caps it at 250-450
|
||||
words. Deliberately far below the Anthropic backend's 16k: this one may
|
||||
be pointed at a 4k-context local model, where asking for more completion
|
||||
tokens than the context holds is an outright error rather than a ceiling. */
|
||||
const DEFAULT_MAX_TOKENS = 2048;
|
||||
|
||||
/**
|
||||
* Strip <think> blocks from a token stream.
|
||||
*
|
||||
* Reasoning models served locally (the DeepSeek-R1 distills, Qwen's
|
||||
* thinking variants) emit chain-of-thought inline in `content` rather than
|
||||
* in a separate field. Left in, it lands in the lesson transcript and the
|
||||
* block parser reads it as prose.
|
||||
*
|
||||
* The tags arrive split across chunks, so this holds back any trailing text
|
||||
* that could still turn out to be the start of one.
|
||||
*/
|
||||
export function makeThinkStripper(): (chunk: string) => string {
|
||||
const OPEN = "<think>";
|
||||
const CLOSE = "</think>";
|
||||
let buffer = "";
|
||||
let inside = false;
|
||||
|
||||
/** Length of the longest suffix of `s` that is a prefix of `tag`. */
|
||||
const heldBack = (s: string, tag: string): number => {
|
||||
const max = Math.min(s.length, tag.length - 1);
|
||||
for (let n = max; n > 0; n--) if (tag.startsWith(s.slice(s.length - n))) return n;
|
||||
return 0;
|
||||
};
|
||||
|
||||
return (chunk: string): string => {
|
||||
buffer += chunk;
|
||||
let out = "";
|
||||
|
||||
for (;;) {
|
||||
if (!inside) {
|
||||
const at = buffer.indexOf(OPEN);
|
||||
if (at >= 0) {
|
||||
out += buffer.slice(0, at);
|
||||
buffer = buffer.slice(at + OPEN.length);
|
||||
inside = true;
|
||||
continue;
|
||||
}
|
||||
const keep = heldBack(buffer, OPEN);
|
||||
out += buffer.slice(0, buffer.length - keep);
|
||||
buffer = buffer.slice(buffer.length - keep);
|
||||
return out;
|
||||
}
|
||||
|
||||
const at = buffer.indexOf(CLOSE);
|
||||
if (at >= 0) {
|
||||
buffer = buffer.slice(at + CLOSE.length);
|
||||
inside = false;
|
||||
continue;
|
||||
}
|
||||
// Still reasoning: drop everything except a possible partial tag.
|
||||
buffer = buffer.slice(buffer.length - heldBack(buffer, CLOSE));
|
||||
return out;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
interface Choice {
|
||||
delta?: { content?: string | null };
|
||||
finish_reason?: string | null;
|
||||
}
|
||||
interface Chunk {
|
||||
choices?: Choice[];
|
||||
usage?: unknown;
|
||||
error?: { message?: string } | string;
|
||||
}
|
||||
|
||||
export function openaiBackend(opts: OpenAIOptions = {}): TutorBackend {
|
||||
const baseUrl = (opts.baseUrl ?? process.env.HANKAN_OPENAI_BASE_URL ?? DEFAULT_BASE_URL).replace(
|
||||
/\/$/,
|
||||
"",
|
||||
);
|
||||
const apiKey = opts.apiKey ?? process.env.HANKAN_OPENAI_API_KEY ?? "";
|
||||
const model = opts.model ?? process.env.HANKAN_OPENAI_MODEL ?? DEFAULT_MODEL;
|
||||
const maxTokens =
|
||||
opts.maxTokens ?? Number(process.env.HANKAN_OPENAI_MAX_TOKENS ?? DEFAULT_MAX_TOKENS);
|
||||
|
||||
return {
|
||||
name: `openai:${model}`,
|
||||
|
||||
async *stream(req: TutorRequest): AsyncIterable<TutorEvent> {
|
||||
/* The system prompt is a message here, not a parameter. It stays
|
||||
first and unchanged for the whole unit, which is what lets a local
|
||||
server reuse its KV cache across turns. */
|
||||
const messages = [
|
||||
{ role: "system", content: req.system },
|
||||
...req.history.map((t) => ({ role: t.role, content: t.content })),
|
||||
{ role: "user", content: req.message },
|
||||
];
|
||||
|
||||
let res: Response;
|
||||
try {
|
||||
res = await fetch(`${baseUrl}/chat/completions`, {
|
||||
method: "POST",
|
||||
signal: req.signal,
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
// Local servers ignore it; hosted ones require it. Omitted
|
||||
// rather than sent empty, because some servers reject "Bearer ".
|
||||
...(apiKey ? { authorization: `Bearer ${apiKey}` } : {}),
|
||||
},
|
||||
body: JSON.stringify({ model, messages, max_tokens: maxTokens, stream: true }),
|
||||
});
|
||||
} catch (err) {
|
||||
if (req.signal.aborted) {
|
||||
yield { type: "done", stopReason: "cancelled" };
|
||||
return;
|
||||
}
|
||||
yield {
|
||||
type: "error",
|
||||
code: "unreachable",
|
||||
message: `Could not reach ${baseUrl} — ${err instanceof Error ? err.message : String(err)}`,
|
||||
};
|
||||
return;
|
||||
}
|
||||
|
||||
if (!res.ok || !res.body) {
|
||||
// These bodies are worth surfacing verbatim: "model not loaded" and
|
||||
// "context length exceeded" are the two most common local failures
|
||||
// and both tell you exactly what to change.
|
||||
const body = await res.text().catch(() => "");
|
||||
yield {
|
||||
type: "error",
|
||||
code: `http_${res.status}`,
|
||||
message: `${baseUrl} returned ${res.status}${body ? ` — ${body.slice(0, 400)}` : ""}`,
|
||||
};
|
||||
return;
|
||||
}
|
||||
|
||||
const reader = res.body.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
const strip = makeThinkStripper();
|
||||
let buffer = "";
|
||||
let finish: string | null = null;
|
||||
let usage: unknown;
|
||||
|
||||
try {
|
||||
for (;;) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
buffer += decoder.decode(value, { stream: true });
|
||||
|
||||
/* Split on the blank line that terminates an SSE event, and keep
|
||||
the remainder — an event, and the JSON inside it, is routinely
|
||||
delivered across two reads. */
|
||||
let cut: number;
|
||||
while ((cut = buffer.search(/\r?\n\r?\n/)) !== -1) {
|
||||
const raw = buffer.slice(0, cut);
|
||||
buffer = buffer.slice(cut).replace(/^\r?\n\r?\n/, "");
|
||||
|
||||
const data = raw
|
||||
.split(/\r?\n/)
|
||||
.filter((l) => l.startsWith("data:"))
|
||||
.map((l) => l.slice(5).trim())
|
||||
.join("");
|
||||
if (!data) continue; // a `:` heartbeat, or a field we ignore
|
||||
if (data === "[DONE]") {
|
||||
yield { type: "done", stopReason: finish, usage };
|
||||
return;
|
||||
}
|
||||
|
||||
let chunk: Chunk;
|
||||
try {
|
||||
chunk = JSON.parse(data) as Chunk;
|
||||
} catch {
|
||||
continue; // not ours to interpret; the stream is still fine
|
||||
}
|
||||
|
||||
if (chunk.error) {
|
||||
const message =
|
||||
typeof chunk.error === "string" ? chunk.error : (chunk.error.message ?? "error");
|
||||
yield { type: "error", code: "upstream", message };
|
||||
return;
|
||||
}
|
||||
|
||||
if (chunk.usage) usage = chunk.usage;
|
||||
const choice = chunk.choices?.[0];
|
||||
if (choice?.finish_reason) finish = choice.finish_reason;
|
||||
const text = choice?.delta?.content;
|
||||
if (text) {
|
||||
const visible = strip(text);
|
||||
if (visible) yield { type: "delta", text: visible };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Some servers close the stream without ever sending [DONE].
|
||||
yield { type: "done", stopReason: finish, usage };
|
||||
} catch (err) {
|
||||
if (req.signal.aborted) {
|
||||
yield { type: "done", stopReason: "cancelled" };
|
||||
return;
|
||||
}
|
||||
yield {
|
||||
type: "error",
|
||||
code: "stream",
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
} finally {
|
||||
await reader.cancel().catch(() => {});
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -16,6 +16,7 @@ import { streamSSE } from "hono/streaming";
|
||||
|
||||
import { anthropicBackend } from "./backends/anthropic.ts";
|
||||
import { echoBackend } from "./backends/echo.ts";
|
||||
import { openaiBackend } from "./backends/openai.ts";
|
||||
import type { TutorBackend, TutorTurn } from "./backends/types.ts";
|
||||
|
||||
const HEARTBEAT_MS = 15_000;
|
||||
@@ -28,7 +29,10 @@ function pickBackend(): TutorBackend {
|
||||
if (name === "anthropic") return anthropicBackend();
|
||||
// Keyless, for proving a deployment before any billing happens.
|
||||
if (name === "echo") return echoBackend();
|
||||
throw new Error(`unknown HANKAN_TUTOR_BACKEND: ${name}`);
|
||||
// Anything speaking OpenAI's /chat/completions: LM Studio, Ollama,
|
||||
// llama.cpp, vLLM, LiteLLM, OpenRouter, OpenAI.
|
||||
if (name === "openai") return openaiBackend();
|
||||
throw new Error(`unknown HANKAN_TUTOR_BACKEND: ${name} (anthropic | openai | echo)`);
|
||||
}
|
||||
|
||||
export function tutorRoute(backend?: TutorBackend) {
|
||||
|
||||
Reference in New Issue
Block a user