diff --git a/server/.env.example b/server/.env.example index e4e4681..e98d83f 100644 --- a/server/.env.example +++ b/server/.env.example @@ -28,9 +28,11 @@ HANKAN_OPENAI_BASE_URL=http://host.docker.internal:1234/v1 HANKAN_OPENAI_MODEL=local-model # Local servers ignore this; hosted ones require it. Leave blank for local. HANKAN_OPENAI_API_KEY= -# Completion ceiling. Kept modest because a small-context local model errors -# outright if asked for more than its context holds. -HANKAN_OPENAI_MAX_TOKENS=2048 +# Completion ceiling. Do not lower this much: on a reasoning model the +# reasoning comes out of the same budget and goes first, so a small value +# truncates the actual lesson away. gpt-oss-20b returns an empty string at +# 1400 and a broken half-Korean fragment at 2048. +HANKAN_OPENAI_MAX_TOKENS=8192 # The docker network your existing Postgres and Caddy are on. # docker network ls diff --git a/server/README.md b/server/README.md index fd434ee..64579a3 100644 --- a/server/README.md +++ b/server/README.md @@ -184,6 +184,15 @@ Measured on a 6,948-character prompt against gpt-oss-20b: **first token 1,563ms cold, 324ms with the prefix already warm.** Which is why the system prompt goes first and stays put in this backend too. +**Give a reasoning model room.** Its reasoning is spent from the same +`max_tokens` budget and goes first, so too small a value truncates the +lesson away entirely — and it fails silently, looking exactly like a model +that cannot follow instructions. Measured on gpt-oss-20b with the real +~3.5k-token system prompt: an empty string at 1,400, a broken half-Korean +fragment with none of the required blocks at 2,048, and a correct English +lesson with all three at 8,000. `HANKAN_OPENAI_MAX_TOKENS` defaults to +8192 for that reason. + A reasoning model served locally often emits chain-of-thought inline in `content` rather than in a separate field. `` blocks are stripped from the stream, including when the tags arrive split across chunks — diff --git a/server/src/backends/openai.ts b/server/src/backends/openai.ts index 027308d..7456a77 100644 --- a/server/src/backends/openai.ts +++ b/server/src/backends/openai.ts @@ -38,10 +38,20 @@ const DEFAULT_BASE_URL = "http://localhost:1234/v1"; const DEFAULT_MODEL = "local-model"; /* A tutor turn is a lesson intro at most — the prompt caps it at 250-450 - words. Deliberately far below the Anthropic backend's 16k: this one may - be pointed at a 4k-context local model, where asking for more completion - tokens than the context holds is an outright error rather than a ceiling. */ -const DEFAULT_MAX_TOKENS = 2048; + words, so roughly 1,200 tokens of visible text. + + This is 8k anyway, because on a reasoning model the reasoning is spent + from the SAME budget and it goes first. Measured on gpt-oss-20b with the + real ~3.5k-token system prompt: at 1,400 it returned an empty string, at + 2,048 a truncated fragment of Korean with none of the required blocks, + and at 8,000 a correct English lesson with all three. The failure is + silent and looks exactly like a model that cannot follow instructions, + which is the expensive way to debug it. + + An earlier note here worried about 4k-context models. That was wrong: + the system prompt alone is ~3.5k tokens, so such a model cannot run this + app at all and there is nothing to protect. */ +const DEFAULT_MAX_TOKENS = 8192; /** * Strip blocks from a token stream.