fix(analysis): use json_schema response_format + surface vision error bodies
The worker hardcoded response_format={type:json_object}, which LM Studio
rejects with 400 ('must be json_schema or text'); it also leaves
reasoning models (Gemma) with an empty `content`. Fix:
- Default to response_format=json_schema with the real analysis schema
(prompt::output_json_schema), which LM Studio accepts and which forces
clean JSON into message.content. Configurable via ANALYSIS_RESPONSE_FORMAT
= json_schema (default) | json_object | none (config::ResponseFormat).
- build_request_body is now a pure, unit-tested function.
- Vision errors now include the server's response body (status + text)
instead of a bare status, so a 400's reason is visible in logs.
Tests: response-format body shapes (none/json_object/json_schema), schema
shape + round-trip with the DTO, config parsing/defaults.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -15,7 +15,7 @@ use serde_json::json;
|
||||
use crate::analysis::prompt::{
|
||||
self, MAX_OCR_PIECES, MAX_OCR_TEXT_CHARS, MAX_SCENE_CHARS, MAX_TAGS,
|
||||
};
|
||||
use crate::config::AnalysisConfig;
|
||||
use crate::config::{AnalysisConfig, ResponseFormat};
|
||||
use crate::domain::page_analysis::{ContentWarning, VisionAnalysis};
|
||||
|
||||
/// Vision client built from [`AnalysisConfig`]. Cheap to clone (holds a
|
||||
@@ -28,6 +28,7 @@ pub struct VisionClient {
|
||||
api_key: Option<String>,
|
||||
max_tokens: u32,
|
||||
max_image_dim: u32,
|
||||
response_format: ResponseFormat,
|
||||
}
|
||||
|
||||
impl VisionClient {
|
||||
@@ -39,6 +40,7 @@ impl VisionClient {
|
||||
api_key: cfg.api_key.clone(),
|
||||
max_tokens: cfg.max_tokens,
|
||||
max_image_dim: cfg.max_image_dim,
|
||||
response_format: cfg.response_format,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -52,35 +54,73 @@ impl VisionClient {
|
||||
let b64 = base64::engine::general_purpose::STANDARD.encode(&bytes);
|
||||
let data_url = format!("data:{mime};base64,{b64}");
|
||||
|
||||
let body = json!({
|
||||
"model": self.model,
|
||||
"temperature": 0,
|
||||
"max_tokens": self.max_tokens,
|
||||
"response_format": { "type": "json_object" },
|
||||
"messages": [
|
||||
{ "role": "system", "content": prompt::SYSTEM_PROMPT },
|
||||
{ "role": "user", "content": [
|
||||
{ "type": "image_url", "image_url": { "url": data_url } }
|
||||
]}
|
||||
]
|
||||
});
|
||||
let body = build_request_body(
|
||||
&self.model,
|
||||
self.max_tokens,
|
||||
&data_url,
|
||||
self.response_format,
|
||||
);
|
||||
|
||||
let mut req = self.http.post(&self.endpoint).json(&body);
|
||||
if let Some(key) = &self.api_key {
|
||||
req = req.bearer_auth(key);
|
||||
}
|
||||
let resp = req
|
||||
.send()
|
||||
.await
|
||||
.context("vision request failed")?
|
||||
.error_for_status()
|
||||
.context("vision endpoint returned an error status")?;
|
||||
let resp = req.send().await.context("vision request failed")?;
|
||||
let status = resp.status();
|
||||
if !status.is_success() {
|
||||
// Surface the server's error body — different OpenAI-compatible
|
||||
// servers reject different fields (e.g. LM Studio rejects
|
||||
// response_format=json_object), and the body says which.
|
||||
let body = resp.text().await.unwrap_or_default();
|
||||
anyhow::bail!(
|
||||
"vision endpoint returned {status}: {}",
|
||||
body.trim().chars().take(800).collect::<String>()
|
||||
);
|
||||
}
|
||||
let value: serde_json::Value =
|
||||
resp.json().await.context("vision response was not JSON")?;
|
||||
parse_chat_completion(&value)
|
||||
}
|
||||
}
|
||||
|
||||
/// Build the chat/completions request body. Pure so the per-format shape is
|
||||
/// unit-tested without a live server. The image is sent as an `image_url`
|
||||
/// content part with a base64 data URL (OpenAI vision convention).
|
||||
pub fn build_request_body(
|
||||
model: &str,
|
||||
max_tokens: u32,
|
||||
data_url: &str,
|
||||
response_format: ResponseFormat,
|
||||
) -> serde_json::Value {
|
||||
let mut body = json!({
|
||||
"model": model,
|
||||
"temperature": 0,
|
||||
"max_tokens": max_tokens,
|
||||
"messages": [
|
||||
{ "role": "system", "content": prompt::SYSTEM_PROMPT },
|
||||
{ "role": "user", "content": [
|
||||
{ "type": "image_url", "image_url": { "url": data_url } }
|
||||
]}
|
||||
]
|
||||
});
|
||||
let rf = match response_format {
|
||||
ResponseFormat::None => None,
|
||||
ResponseFormat::JsonObject => Some(json!({ "type": "json_object" })),
|
||||
ResponseFormat::JsonSchema => Some(json!({
|
||||
"type": "json_schema",
|
||||
"json_schema": {
|
||||
"name": "page_analysis",
|
||||
"strict": true,
|
||||
"schema": prompt::output_json_schema()
|
||||
}
|
||||
})),
|
||||
};
|
||||
if let Some(rf) = rf {
|
||||
body["response_format"] = rf;
|
||||
}
|
||||
body
|
||||
}
|
||||
|
||||
/// Downscale `bytes` so its longest edge is `max_dim`, re-encoding as JPEG.
|
||||
/// Returns `None` (use the original bytes) when the image already fits or
|
||||
/// can't be decoded — a non-decodable page is passed through untouched and
|
||||
@@ -277,4 +317,40 @@ mod tests {
|
||||
fn downscale_passes_through_undecodable_bytes() {
|
||||
assert!(downscale(b"not an image", 512).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn request_body_carries_model_image_and_prompt() {
|
||||
let body = build_request_body("m", 100, "data:image/png;base64,AA", ResponseFormat::None);
|
||||
assert_eq!(body["model"], "m");
|
||||
assert_eq!(body["max_tokens"], 100);
|
||||
// System prompt + a user image_url part.
|
||||
assert_eq!(body["messages"][0]["role"], "system");
|
||||
assert_eq!(
|
||||
body["messages"][1]["content"][0]["image_url"]["url"],
|
||||
"data:image/png;base64,AA"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn response_format_none_omits_the_field() {
|
||||
let body = build_request_body("m", 100, "d", ResponseFormat::None);
|
||||
assert!(body.get("response_format").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn response_format_json_object_sets_type() {
|
||||
let body = build_request_body("m", 100, "d", ResponseFormat::JsonObject);
|
||||
assert_eq!(body["response_format"]["type"], "json_object");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn response_format_json_schema_embeds_the_strict_schema() {
|
||||
let body = build_request_body("m", 100, "d", ResponseFormat::JsonSchema);
|
||||
assert_eq!(body["response_format"]["type"], "json_schema");
|
||||
assert_eq!(body["response_format"]["json_schema"]["strict"], true);
|
||||
// The schema is the real analysis schema (top-level required keys).
|
||||
let required =
|
||||
&body["response_format"]["json_schema"]["schema"]["required"];
|
||||
assert!(required.as_array().unwrap().iter().any(|v| v == "safety_flag"));
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user