fabi/Mangalord
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(analysis): use json_schema response_format + surface vision error bodies

Browse Source 
The worker hardcoded response_format={type:json_object}, which LM Studio
rejects with 400 ('must be json_schema or text'); it also leaves
reasoning models (Gemma) with an empty `content`. Fix:

- Default to response_format=json_schema with the real analysis schema
  (prompt::output_json_schema), which LM Studio accepts and which forces
  clean JSON into message.content. Configurable via ANALYSIS_RESPONSE_FORMAT
  = json_schema (default) | json_object | none (config::ResponseFormat).
- build_request_body is now a pure, unit-tested function.
- Vision errors now include the server's response body (status + text)
  instead of a bare status, so a 400's reason is visible in logs.

Tests: response-format body shapes (none/json_object/json_schema), schema
shape + round-trip with the DTO, config parsing/defaults.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
MechaCat02
2026-06-13 20:14:38 +02:00
parent 7e675b72cc
commit 268e8cc6c2
6 changed files with 224 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
backend/Cargo.lock generated
View File

@@ -1517,7 +1517,7 @@ checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
[[package]]
name = "mangalord"
version = "0.74.0"
version = "0.74.1"
dependencies = [
"anyhow",
"argon2",

2
backend/Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "mangalord"
version = "0.74.0"
version = "0.74.1"
edition = "2021"
default-run = "mangalord"

76
backend/src/analysis/prompt.rs
View File

@@ -50,10 +50,51 @@ pub const SYSTEM_PROMPT: &str = concat!(
"content. Output must be valid minified JSON and nothing else."
);
/// JSON Schema for the analysis output, used in `response_format:
/// json_schema` mode (OpenAI structured outputs / LM Studio). Mirrors
/// [`crate::domain::page_analysis::VisionAnalysis`]; `additionalProperties:
/// false` + all-required keeps it valid under OpenAI strict mode while
/// staying compatible with LM Studio.
pub fn output_json_schema() -> serde_json::Value {
serde_json::json!({
"type": "object",
"additionalProperties": false,
"properties": {
"ocr_results": {
"type": "array",
"items": {
"type": "object",
"additionalProperties": false,
"properties": {
"text": { "type": "string" },
"kind": { "type": "string", "enum": OCR_KINDS }
},
"required": ["text", "kind"]
}
},
"tagging_results": { "type": "array", "items": { "type": "string" } },
"scene_description": { "type": "string" },
"safety_flag": {
"type": "object",
"additionalProperties": false,
"properties": {
"is_nsfw": { "type": "boolean" },
"content_type": {
"type": "array",
"items": { "type": "string", "enum": CONTENT_WARNINGS }
}
},
"required": ["is_nsfw", "content_type"]
}
},
"required": ["ocr_results", "tagging_results", "scene_description", "safety_flag"]
})
}
#[cfg(test)]
mod tests {
use super::*;
use crate::domain::page_analysis::{ContentWarning, OcrKind};
use crate::domain::page_analysis::{ContentWarning, OcrKind, VisionAnalysis};
#[test]
fn vocabularies_match_the_domain_enums() {
@@ -87,4 +128,37 @@ mod tests {
assert!(SYSTEM_PROMPT.contains(key), "prompt missing {key}");
}
}
#[test]
fn output_schema_top_level_requires_all_four_keys() {
let schema = output_json_schema();
let required = schema["required"].as_array().unwrap();
for key in ["ocr_results", "tagging_results", "scene_description", "safety_flag"] {
assert!(
required.iter().any(|v| v == key),
"schema missing required {key}"
);
}
// The kind enum carries exactly the OCR vocabulary.
let kind_enum =
&schema["properties"]["ocr_results"]["items"]["properties"]["kind"]["enum"];
assert_eq!(kind_enum, &serde_json::json!(OCR_KINDS));
}
#[test]
fn output_schema_accepts_a_real_analysis_shape() {
// A response matching the domain DTO must deserialize — the schema
// and the parser describe the same shape.
let sample = serde_json::json!({
"ocr_results": [{ "text": "Hi", "kind": "speech" }],
"tagging_results": ["action"],
"scene_description": "A street.",
"safety_flag": { "is_nsfw": false, "content_type": [] }
});
let _: VisionAnalysis = serde_json::from_value(sample).unwrap();
// Schema names the same object key the parser reads.
assert!(output_json_schema()["properties"]
.get("safety_flag")
.is_some());
}
}

112
backend/src/analysis/vision.rs
View File

@@ -15,7 +15,7 @@ use serde_json::json;
use crate::analysis::prompt::{
self, MAX_OCR_PIECES, MAX_OCR_TEXT_CHARS, MAX_SCENE_CHARS, MAX_TAGS,
};
use crate::config::AnalysisConfig;
use crate::config::{AnalysisConfig, ResponseFormat};
use crate::domain::page_analysis::{ContentWarning, VisionAnalysis};
/// Vision client built from [`AnalysisConfig`]. Cheap to clone (holds a
@@ -28,6 +28,7 @@ pub struct VisionClient {
api_key: Option<String>,
max_tokens: u32,
max_image_dim: u32,
response_format: ResponseFormat,
}
impl VisionClient {
@@ -39,6 +40,7 @@ impl VisionClient {
api_key: cfg.api_key.clone(),
max_tokens: cfg.max_tokens,
max_image_dim: cfg.max_image_dim,
response_format: cfg.response_format,
}
}
@@ -52,11 +54,48 @@ impl VisionClient {
let b64 = base64::engine::general_purpose::STANDARD.encode(&bytes);
let data_url = format!("data:{mime};base64,{b64}");
let body = json!({
"model": self.model,
let body = build_request_body(
&self.model,
self.max_tokens,
&data_url,
self.response_format,
);
let mut req = self.http.post(&self.endpoint).json(&body);
if let Some(key) = &self.api_key {
req = req.bearer_auth(key);
}
let resp = req.send().await.context("vision request failed")?;
let status = resp.status();
if !status.is_success() {
// Surface the server's error body — different OpenAI-compatible
// servers reject different fields (e.g. LM Studio rejects
// response_format=json_object), and the body says which.
let body = resp.text().await.unwrap_or_default();
anyhow::bail!(
"vision endpoint returned {status}: {}",
body.trim().chars().take(800).collect::<String>()
);
}
let value: serde_json::Value =
resp.json().await.context("vision response was not JSON")?;
parse_chat_completion(&value)
}
}
/// Build the chat/completions request body. Pure so the per-format shape is
/// unit-tested without a live server. The image is sent as an `image_url`
/// content part with a base64 data URL (OpenAI vision convention).
pub fn build_request_body(
model: &str,
max_tokens: u32,
data_url: &str,
response_format: ResponseFormat,
) -> serde_json::Value {
let mut body = json!({
"model": model,
"temperature": 0,
"max_tokens": self.max_tokens,
"response_format": { "type": "json_object" },
"max_tokens": max_tokens,
"messages": [
{ "role": "system", "content": prompt::SYSTEM_PROMPT },
{ "role": "user", "content": [
@@ -64,21 +103,22 @@ impl VisionClient {
]}
]
});
let mut req = self.http.post(&self.endpoint).json(&body);
if let Some(key) = &self.api_key {
req = req.bearer_auth(key);
let rf = match response_format {
ResponseFormat::None => None,
ResponseFormat::JsonObject => Some(json!({ "type": "json_object" })),
ResponseFormat::JsonSchema => Some(json!({
"type": "json_schema",
"json_schema": {
"name": "page_analysis",
"strict": true,
"schema": prompt::output_json_schema()
}
let resp = req
.send()
.await
.context("vision request failed")?
.error_for_status()
.context("vision endpoint returned an error status")?;
let value: serde_json::Value =
resp.json().await.context("vision response was not JSON")?;
parse_chat_completion(&value)
})),
};
if let Some(rf) = rf {
body["response_format"] = rf;
}
body
}
/// Downscale `bytes` so its longest edge is `max_dim`, re-encoding as JPEG.
@@ -277,4 +317,40 @@ mod tests {
fn downscale_passes_through_undecodable_bytes() {
assert!(downscale(b"not an image", 512).is_none());
}
#[test]
fn request_body_carries_model_image_and_prompt() {
let body = build_request_body("m", 100, "data:image/png;base64,AA", ResponseFormat::None);
assert_eq!(body["model"], "m");
assert_eq!(body["max_tokens"], 100);
// System prompt + a user image_url part.
assert_eq!(body["messages"][0]["role"], "system");
assert_eq!(
body["messages"][1]["content"][0]["image_url"]["url"],
"data:image/png;base64,AA"
);
}
#[test]
fn response_format_none_omits_the_field() {
let body = build_request_body("m", 100, "d", ResponseFormat::None);
assert!(body.get("response_format").is_none());
}
#[test]
fn response_format_json_object_sets_type() {
let body = build_request_body("m", 100, "d", ResponseFormat::JsonObject);
assert_eq!(body["response_format"]["type"], "json_object");
}
#[test]
fn response_format_json_schema_embeds_the_strict_schema() {
let body = build_request_body("m", 100, "d", ResponseFormat::JsonSchema);
assert_eq!(body["response_format"]["type"], "json_schema");
assert_eq!(body["response_format"]["json_schema"]["strict"], true);
// The schema is the real analysis schema (top-level required keys).
let required =
&body["response_format"]["json_schema"]["schema"]["required"];
assert!(required.as_array().unwrap().iter().any(|v| v == "safety_flag"));
}
}

51
backend/src/config.rs
View File

@@ -66,6 +66,32 @@ impl Default for UploadConfig {
}
}
/// How the worker asks the model to constrain its output. OpenAI-compatible
/// servers disagree here: LM Studio accepts only `json_schema` or `text`
/// (NOT `json_object`); vanilla OpenAI/vLLM accept `json_object` too. The
/// default `JsonSchema` is the most portable and the most reliable — it
/// also keeps "thinking" models (e.g. Gemma) from emitting an empty
/// `content` with the answer buried in `reasoning_content`.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum ResponseFormat {
JsonSchema,
JsonObject,
/// Send no `response_format` — rely on the prompt + the parser's
/// fence/brace extraction. Needed for servers that reject the field.
None,
}
impl ResponseFormat {
fn from_str(s: &str) -> ResponseFormat {
match s.trim().to_lowercase().as_str() {
"json_object" => ResponseFormat::JsonObject,
"none" | "text" | "off" | "" => ResponseFormat::None,
// Default (incl. "json_schema" and anything unrecognized).
_ => ResponseFormat::JsonSchema,
}
}
}
/// AI content-analysis worker configuration: the enable gate, the local
/// OpenAI-compatible vision endpoint, and the worker / request knobs.
#[derive(Clone, Debug)]
@@ -93,6 +119,9 @@ pub struct AnalysisConfig {
/// Hard cap on a page image's stored size; larger pages are skipped
/// (`ANALYSIS_MAX_IMAGE_BYTES`).
pub max_image_bytes: usize,
/// Output-constraint mode (`ANALYSIS_RESPONSE_FORMAT`):
/// `json_schema` (default) | `json_object` | `none`.
pub response_format: ResponseFormat,
}
impl Default for AnalysisConfig {
@@ -108,6 +137,7 @@ impl Default for AnalysisConfig {
max_tokens: 900,
max_image_dim: 1024,
max_image_bytes: 8 * 1024 * 1024,
response_format: ResponseFormat::JsonSchema,
}
}
}
@@ -134,6 +164,9 @@ impl AnalysisConfig {
max_tokens: env_u64("ANALYSIS_MAX_TOKENS", d.max_tokens as u64) as u32,
max_image_dim: env_u64("ANALYSIS_MAX_IMAGE_DIM", d.max_image_dim as u64) as u32,
max_image_bytes: env_usize("ANALYSIS_MAX_IMAGE_BYTES", d.max_image_bytes),
response_format: std::env::var("ANALYSIS_RESPONSE_FORMAT")
.map(|s| ResponseFormat::from_str(&s))
.unwrap_or(d.response_format),
}
}
}
@@ -542,6 +575,7 @@ mod tests {
"ANALYSIS_API_KEY",
"ANALYSIS_MAX_TOKENS",
"ANALYSIS_MAX_IMAGE_DIM",
"ANALYSIS_RESPONSE_FORMAT",
] {
std::env::remove_var(k);
}
@@ -550,6 +584,23 @@ mod tests {
assert_eq!(cfg.workers, 1);
assert_eq!(cfg.max_image_dim, 1024);
assert!(cfg.api_key.is_none());
assert_eq!(cfg.response_format, ResponseFormat::JsonSchema);
}
#[test]
fn analysis_response_format_parses_modes() {
let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner());
for (raw, want) in [
("json_object", ResponseFormat::JsonObject),
("none", ResponseFormat::None),
("text", ResponseFormat::None),
("json_schema", ResponseFormat::JsonSchema),
("anything-else", ResponseFormat::JsonSchema),
] {
std::env::set_var("ANALYSIS_RESPONSE_FORMAT", raw);
assert_eq!(AnalysisConfig::from_env().response_format, want, "raw={raw}");
}
std::env::remove_var("ANALYSIS_RESPONSE_FORMAT");
}
#[test]

2
frontend/package.json
View File

@@ -1,6 +1,6 @@
{
"name": "mangalord-frontend",
"version": "0.74.0",
"version": "0.74.1",
"private": true,
"type": "module",
"scripts": {