/* POST /api/tutor — one turn in, a token stream out. No session state. The client posts the assembled system prompt, the transcript it owns, and the new message; the server holds nothing between requests, so a dropped connection costs one turn rather than the conversation. SSE framing has three rules that are easy to get wrong and silent when broken: every event ends with a blank line, a `data:` field may not contain a raw newline (Korean tutor replies are full of them, so payloads are JSON-encoded), and a comment line starting with `:` is a heartbeat that keeps intermediaries from timing the stream out. */ import { Hono } from "hono"; import { streamSSE } from "hono/streaming"; import { anthropicBackend } from "./backends/anthropic.ts"; import { echoBackend } from "./backends/echo.ts"; import { openaiBackend } from "./backends/openai.ts"; import type { TutorBackend, TutorTurn } from "./backends/types.ts"; const HEARTBEAT_MS = 15_000; /** Bound the transcript the client may send, so one request cannot be huge. */ const MAX_HISTORY = 40; function pickBackend(): TutorBackend { const name = process.env.HANKAN_TUTOR_BACKEND ?? "anthropic"; if (name === "anthropic") return anthropicBackend(); // Keyless, for proving a deployment before any billing happens. if (name === "echo") return echoBackend(); // Anything speaking OpenAI's /chat/completions: LM Studio, Ollama, // llama.cpp, vLLM, LiteLLM, OpenRouter, OpenAI. if (name === "openai") return openaiBackend(); throw new Error(`unknown HANKAN_TUTOR_BACKEND: ${name} (anthropic | openai | echo)`); } export function tutorRoute(backend?: TutorBackend) { const app = new Hono(); // Constructed lazily so the process can start — and serve sync — without // an API key present. let resolved: TutorBackend | null = backend ?? null; app.post("/", async (c) => { const body = (await c.req.json()) as { system?: string; systemTail?: string; history?: TutorTurn[]; message?: string; }; if (!body.system || !body.message) { return c.json({ error: "system and message are required" }, 400); } const history = (Array.isArray(body.history) ? body.history : []) .filter((t) => t && (t.role === "user" || t.role === "assistant") && typeof t.content === "string") .slice(-MAX_HISTORY); try { resolved ??= pickBackend(); } catch (err) { return c.json({ error: err instanceof Error ? err.message : "no backend" }, 503); } const res = streamSSE(c, async (stream) => { const abort = new AbortController(); // The client going away is the cancel signal: abort upstream so a // cancelled turn stops billing. stream.onAbort(() => abort.abort()); const beat = setInterval(() => { void stream.writeSSE({ data: "", event: "ping" }); }, HEARTBEAT_MS); try { for await (const event of resolved!.stream({ system: body.system!, systemTail: typeof body.systemTail === "string" ? body.systemTail : undefined, history, message: body.message!, signal: abort.signal, })) { await stream.writeSSE({ event: event.type, data: JSON.stringify(event) }); if (event.type === "done" || event.type === "error") break; } } catch (err) { await stream.writeSSE({ event: "error", data: JSON.stringify({ type: "error", code: "server", message: err instanceof Error ? err.message : String(err), }), }); } finally { clearInterval(beat); } }); /* Set after streamSSE, which writes its own Cache-Control and would overwrite these. `no-transform` tells intermediaries not to re-encode the body — compression is the usual reason SSE appears to hang behind a proxy, because an unfinished compression frame holds the events. X-Accel-Buffering is nginx's opt-out and harmless elsewhere. */ res.headers.set("Cache-Control", "no-cache, no-transform"); res.headers.set("X-Accel-Buffering", "no"); return res; }); return app; }