diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 1bb0423..1c0f189 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -1517,7 +1517,7 @@ checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" [[package]] name = "mangalord" -version = "0.74.0" +version = "0.74.1" dependencies = [ "anyhow", "argon2", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 96b86f0..e23442a 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mangalord" -version = "0.74.0" +version = "0.74.1" edition = "2021" default-run = "mangalord" diff --git a/backend/src/analysis/prompt.rs b/backend/src/analysis/prompt.rs index 35f491d..0261172 100644 --- a/backend/src/analysis/prompt.rs +++ b/backend/src/analysis/prompt.rs @@ -50,10 +50,51 @@ pub const SYSTEM_PROMPT: &str = concat!( "content. Output must be valid minified JSON and nothing else." ); +/// JSON Schema for the analysis output, used in `response_format: +/// json_schema` mode (OpenAI structured outputs / LM Studio). Mirrors +/// [`crate::domain::page_analysis::VisionAnalysis`]; `additionalProperties: +/// false` + all-required keeps it valid under OpenAI strict mode while +/// staying compatible with LM Studio. +pub fn output_json_schema() -> serde_json::Value { + serde_json::json!({ + "type": "object", + "additionalProperties": false, + "properties": { + "ocr_results": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "text": { "type": "string" }, + "kind": { "type": "string", "enum": OCR_KINDS } + }, + "required": ["text", "kind"] + } + }, + "tagging_results": { "type": "array", "items": { "type": "string" } }, + "scene_description": { "type": "string" }, + "safety_flag": { + "type": "object", + "additionalProperties": false, + "properties": { + "is_nsfw": { "type": "boolean" }, + "content_type": { + "type": "array", + "items": { "type": "string", "enum": CONTENT_WARNINGS } + } + }, + "required": ["is_nsfw", "content_type"] + } + }, + "required": ["ocr_results", "tagging_results", "scene_description", "safety_flag"] + }) +} + #[cfg(test)] mod tests { use super::*; - use crate::domain::page_analysis::{ContentWarning, OcrKind}; + use crate::domain::page_analysis::{ContentWarning, OcrKind, VisionAnalysis}; #[test] fn vocabularies_match_the_domain_enums() { @@ -87,4 +128,37 @@ mod tests { assert!(SYSTEM_PROMPT.contains(key), "prompt missing {key}"); } } + + #[test] + fn output_schema_top_level_requires_all_four_keys() { + let schema = output_json_schema(); + let required = schema["required"].as_array().unwrap(); + for key in ["ocr_results", "tagging_results", "scene_description", "safety_flag"] { + assert!( + required.iter().any(|v| v == key), + "schema missing required {key}" + ); + } + // The kind enum carries exactly the OCR vocabulary. + let kind_enum = + &schema["properties"]["ocr_results"]["items"]["properties"]["kind"]["enum"]; + assert_eq!(kind_enum, &serde_json::json!(OCR_KINDS)); + } + + #[test] + fn output_schema_accepts_a_real_analysis_shape() { + // A response matching the domain DTO must deserialize — the schema + // and the parser describe the same shape. + let sample = serde_json::json!({ + "ocr_results": [{ "text": "Hi", "kind": "speech" }], + "tagging_results": ["action"], + "scene_description": "A street.", + "safety_flag": { "is_nsfw": false, "content_type": [] } + }); + let _: VisionAnalysis = serde_json::from_value(sample).unwrap(); + // Schema names the same object key the parser reads. + assert!(output_json_schema()["properties"] + .get("safety_flag") + .is_some()); + } } diff --git a/backend/src/analysis/vision.rs b/backend/src/analysis/vision.rs index 3c8f627..8b5fe1d 100644 --- a/backend/src/analysis/vision.rs +++ b/backend/src/analysis/vision.rs @@ -15,7 +15,7 @@ use serde_json::json; use crate::analysis::prompt::{ self, MAX_OCR_PIECES, MAX_OCR_TEXT_CHARS, MAX_SCENE_CHARS, MAX_TAGS, }; -use crate::config::AnalysisConfig; +use crate::config::{AnalysisConfig, ResponseFormat}; use crate::domain::page_analysis::{ContentWarning, VisionAnalysis}; /// Vision client built from [`AnalysisConfig`]. Cheap to clone (holds a @@ -28,6 +28,7 @@ pub struct VisionClient { api_key: Option, max_tokens: u32, max_image_dim: u32, + response_format: ResponseFormat, } impl VisionClient { @@ -39,6 +40,7 @@ impl VisionClient { api_key: cfg.api_key.clone(), max_tokens: cfg.max_tokens, max_image_dim: cfg.max_image_dim, + response_format: cfg.response_format, } } @@ -52,35 +54,73 @@ impl VisionClient { let b64 = base64::engine::general_purpose::STANDARD.encode(&bytes); let data_url = format!("data:{mime};base64,{b64}"); - let body = json!({ - "model": self.model, - "temperature": 0, - "max_tokens": self.max_tokens, - "response_format": { "type": "json_object" }, - "messages": [ - { "role": "system", "content": prompt::SYSTEM_PROMPT }, - { "role": "user", "content": [ - { "type": "image_url", "image_url": { "url": data_url } } - ]} - ] - }); + let body = build_request_body( + &self.model, + self.max_tokens, + &data_url, + self.response_format, + ); let mut req = self.http.post(&self.endpoint).json(&body); if let Some(key) = &self.api_key { req = req.bearer_auth(key); } - let resp = req - .send() - .await - .context("vision request failed")? - .error_for_status() - .context("vision endpoint returned an error status")?; + let resp = req.send().await.context("vision request failed")?; + let status = resp.status(); + if !status.is_success() { + // Surface the server's error body — different OpenAI-compatible + // servers reject different fields (e.g. LM Studio rejects + // response_format=json_object), and the body says which. + let body = resp.text().await.unwrap_or_default(); + anyhow::bail!( + "vision endpoint returned {status}: {}", + body.trim().chars().take(800).collect::() + ); + } let value: serde_json::Value = resp.json().await.context("vision response was not JSON")?; parse_chat_completion(&value) } } +/// Build the chat/completions request body. Pure so the per-format shape is +/// unit-tested without a live server. The image is sent as an `image_url` +/// content part with a base64 data URL (OpenAI vision convention). +pub fn build_request_body( + model: &str, + max_tokens: u32, + data_url: &str, + response_format: ResponseFormat, +) -> serde_json::Value { + let mut body = json!({ + "model": model, + "temperature": 0, + "max_tokens": max_tokens, + "messages": [ + { "role": "system", "content": prompt::SYSTEM_PROMPT }, + { "role": "user", "content": [ + { "type": "image_url", "image_url": { "url": data_url } } + ]} + ] + }); + let rf = match response_format { + ResponseFormat::None => None, + ResponseFormat::JsonObject => Some(json!({ "type": "json_object" })), + ResponseFormat::JsonSchema => Some(json!({ + "type": "json_schema", + "json_schema": { + "name": "page_analysis", + "strict": true, + "schema": prompt::output_json_schema() + } + })), + }; + if let Some(rf) = rf { + body["response_format"] = rf; + } + body +} + /// Downscale `bytes` so its longest edge is `max_dim`, re-encoding as JPEG. /// Returns `None` (use the original bytes) when the image already fits or /// can't be decoded — a non-decodable page is passed through untouched and @@ -277,4 +317,40 @@ mod tests { fn downscale_passes_through_undecodable_bytes() { assert!(downscale(b"not an image", 512).is_none()); } + + #[test] + fn request_body_carries_model_image_and_prompt() { + let body = build_request_body("m", 100, "data:image/png;base64,AA", ResponseFormat::None); + assert_eq!(body["model"], "m"); + assert_eq!(body["max_tokens"], 100); + // System prompt + a user image_url part. + assert_eq!(body["messages"][0]["role"], "system"); + assert_eq!( + body["messages"][1]["content"][0]["image_url"]["url"], + "data:image/png;base64,AA" + ); + } + + #[test] + fn response_format_none_omits_the_field() { + let body = build_request_body("m", 100, "d", ResponseFormat::None); + assert!(body.get("response_format").is_none()); + } + + #[test] + fn response_format_json_object_sets_type() { + let body = build_request_body("m", 100, "d", ResponseFormat::JsonObject); + assert_eq!(body["response_format"]["type"], "json_object"); + } + + #[test] + fn response_format_json_schema_embeds_the_strict_schema() { + let body = build_request_body("m", 100, "d", ResponseFormat::JsonSchema); + assert_eq!(body["response_format"]["type"], "json_schema"); + assert_eq!(body["response_format"]["json_schema"]["strict"], true); + // The schema is the real analysis schema (top-level required keys). + let required = + &body["response_format"]["json_schema"]["schema"]["required"]; + assert!(required.as_array().unwrap().iter().any(|v| v == "safety_flag")); + } } diff --git a/backend/src/config.rs b/backend/src/config.rs index 78f4b7e..fe73315 100644 --- a/backend/src/config.rs +++ b/backend/src/config.rs @@ -66,6 +66,32 @@ impl Default for UploadConfig { } } +/// How the worker asks the model to constrain its output. OpenAI-compatible +/// servers disagree here: LM Studio accepts only `json_schema` or `text` +/// (NOT `json_object`); vanilla OpenAI/vLLM accept `json_object` too. The +/// default `JsonSchema` is the most portable and the most reliable — it +/// also keeps "thinking" models (e.g. Gemma) from emitting an empty +/// `content` with the answer buried in `reasoning_content`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ResponseFormat { + JsonSchema, + JsonObject, + /// Send no `response_format` — rely on the prompt + the parser's + /// fence/brace extraction. Needed for servers that reject the field. + None, +} + +impl ResponseFormat { + fn from_str(s: &str) -> ResponseFormat { + match s.trim().to_lowercase().as_str() { + "json_object" => ResponseFormat::JsonObject, + "none" | "text" | "off" | "" => ResponseFormat::None, + // Default (incl. "json_schema" and anything unrecognized). + _ => ResponseFormat::JsonSchema, + } + } +} + /// AI content-analysis worker configuration: the enable gate, the local /// OpenAI-compatible vision endpoint, and the worker / request knobs. #[derive(Clone, Debug)] @@ -93,6 +119,9 @@ pub struct AnalysisConfig { /// Hard cap on a page image's stored size; larger pages are skipped /// (`ANALYSIS_MAX_IMAGE_BYTES`). pub max_image_bytes: usize, + /// Output-constraint mode (`ANALYSIS_RESPONSE_FORMAT`): + /// `json_schema` (default) | `json_object` | `none`. + pub response_format: ResponseFormat, } impl Default for AnalysisConfig { @@ -108,6 +137,7 @@ impl Default for AnalysisConfig { max_tokens: 900, max_image_dim: 1024, max_image_bytes: 8 * 1024 * 1024, + response_format: ResponseFormat::JsonSchema, } } } @@ -134,6 +164,9 @@ impl AnalysisConfig { max_tokens: env_u64("ANALYSIS_MAX_TOKENS", d.max_tokens as u64) as u32, max_image_dim: env_u64("ANALYSIS_MAX_IMAGE_DIM", d.max_image_dim as u64) as u32, max_image_bytes: env_usize("ANALYSIS_MAX_IMAGE_BYTES", d.max_image_bytes), + response_format: std::env::var("ANALYSIS_RESPONSE_FORMAT") + .map(|s| ResponseFormat::from_str(&s)) + .unwrap_or(d.response_format), } } } @@ -542,6 +575,7 @@ mod tests { "ANALYSIS_API_KEY", "ANALYSIS_MAX_TOKENS", "ANALYSIS_MAX_IMAGE_DIM", + "ANALYSIS_RESPONSE_FORMAT", ] { std::env::remove_var(k); } @@ -550,6 +584,23 @@ mod tests { assert_eq!(cfg.workers, 1); assert_eq!(cfg.max_image_dim, 1024); assert!(cfg.api_key.is_none()); + assert_eq!(cfg.response_format, ResponseFormat::JsonSchema); + } + + #[test] + fn analysis_response_format_parses_modes() { + let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner()); + for (raw, want) in [ + ("json_object", ResponseFormat::JsonObject), + ("none", ResponseFormat::None), + ("text", ResponseFormat::None), + ("json_schema", ResponseFormat::JsonSchema), + ("anything-else", ResponseFormat::JsonSchema), + ] { + std::env::set_var("ANALYSIS_RESPONSE_FORMAT", raw); + assert_eq!(AnalysisConfig::from_env().response_format, want, "raw={raw}"); + } + std::env::remove_var("ANALYSIS_RESPONSE_FORMAT"); } #[test] diff --git a/frontend/package.json b/frontend/package.json index 6083e0c..df8c36c 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "mangalord-frontend", - "version": "0.74.0", + "version": "0.74.1", "private": true, "type": "module", "scripts": {