//! Serializable, admin-editable settings DTOs for the crawler and analysis //! subsystems, and their conversion to/from the runtime [`CrawlerConfig`] / //! [`AnalysisConfig`]. //! //! These DTOs are the boundary persisted in the `app_settings` table (one //! JSONB row per group) and exchanged over the admin API. They carry **only //! the operationally-safe, UI-editable fields** as primitive types — the //! runtime configs additionally hold env-only/structural fields (browser //! launch options, proxy, TOR control, the vision API key, the cookie //! domain) that are never persisted here. //! //! Flow: at boot, [`CrawlerSettings::from_config`] / //! [`AnalysisSettings::from_config`] derive the env-seed DTO; it is written to //! the DB only when the row is absent. Thereafter the DB row is the source of //! truth, and [`CrawlerSettings::to_config`] / [`AnalysisSettings::to_config`] //! overlay it onto the env-derived **base** (which carries the env-only //! fields) to produce the effective config the daemons are (re)spawned with. use std::time::Duration; use chrono::NaiveTime; use chrono_tz::Tz; use serde::{Deserialize, Serialize}; use crate::analysis::prompt::{ GROUNDING_PROMPT_DEFAULT, OCR_PROMPT_DEFAULT, SYSTEM_PROMPT_DEFAULT, }; use crate::config::{AnalysisBackend, AnalysisConfig, CrawlerConfig, ResponseFormat}; use crate::crawler::safety::DownloadAllowlist; /// `app_settings.key` for the crawler group. pub const KEY_CRAWLER: &str = "crawler"; /// `app_settings.key` for the analysis group. pub const KEY_ANALYSIS: &str = "analysis"; // Upper bounds on numeric settings. These are sanity caps, not tuned limits — // they keep a fat-fingered (or CSRF-injected) value from spawning thousands of // workers, demanding gigabyte buffers, or sending out-of-range sampling // params. Generous enough that no realistic deployment hits them. const MAX_WORKERS: u64 = 64; const MAX_CHAPTER_WORKERS: u64 = 64; const MAX_ANALYSIS_MAX_TOKENS: u32 = 1_000_000; const MAX_ANALYSIS_SLICES: u64 = 1024; /// 1 GiB — far above any real page, well below "exhaust the host". const MAX_ANALYSIS_IMAGE_BYTES: u64 = 1024 * 1024 * 1024; /// OpenAI-compatible sampling ranges (the analysis endpoint speaks that API). const MAX_TEMPERATURE: f64 = 2.0; const FREQUENCY_PENALTY_MIN: f64 = -2.0; const FREQUENCY_PENALTY_MAX: f64 = 2.0; /// Max manga-detail fetches per metadata pass. `0` stays special-cased as /// "unlimited"; this only bounds an explicit positive value. const MAX_MANGA_LIMIT: u64 = 1_000_000; /// Upper bound (seconds) shared by every timeout knob — 24h. A timeout /// longer than a day is almost certainly a fat-fingered value (e.g. ms /// mistaken for s) and would wedge a worker for far too long. const MAX_TIMEOUT_SECS: u64 = 86_400; /// One field-level validation failure, surfaced to the UI per input. #[derive(Debug, Clone, Serialize, PartialEq, Eq)] pub struct FieldError { pub field: String, pub message: String, } /// Collected validation failures for a settings payload. Empty == valid. #[derive(Debug, Clone, Serialize, Default, PartialEq, Eq)] pub struct FieldErrors { pub errors: Vec, } impl FieldErrors { fn push(&mut self, field: &str, message: impl Into) { self.errors.push(FieldError { field: field.to_string(), message: message.into(), }); } pub fn is_empty(&self) -> bool { self.errors.is_empty() } } // --------------------------------------------------------------------------- // Crawler // --------------------------------------------------------------------------- /// Admin-editable crawler-daemon settings. Mirrors the operational `CRAWLER_*` /// env vars; host/infra and session fields stay env-only. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(default)] pub struct CrawlerSettings { pub daemon_enabled: bool, /// Daily metadata-pass time, `"HH:MM"` (24h). pub daily_at: String, /// IANA timezone for the daily schedule. pub tz: String, pub idle_timeout_secs: u64, pub chapter_workers: u64, pub retention_days: u32, pub start_url: Option, pub rate_ms: u64, pub cdn_host: Option, pub cdn_rate_ms: u64, /// Domain the session cookie (PHPSESSID) is scoped to when injecting it /// into the browser / CDN requests. Pairs with `start_url`. pub cookie_domain: Option, pub user_agent: Option, /// Hosts the crawler may download images from (in addition to the /// auto-seeded start-url / CDN hosts). Ignored when `allow_any_host`. pub download_allowlist: Vec, pub allow_any_host: bool, pub max_image_bytes: u64, /// Max manga detail fetches per metadata pass; `0` = unlimited. pub manga_limit: u64, pub job_timeout_secs: u64, pub metadata_max_consecutive_failures: u32, pub browser_restart_threshold: u32, } impl Default for CrawlerSettings { fn default() -> Self { Self::from_config(&CrawlerConfig::default()) } } impl CrawlerSettings { /// Derive the DTO from a runtime config (used to seed the DB from env). pub fn from_config(c: &CrawlerConfig) -> Self { Self { daemon_enabled: c.daemon_enabled, daily_at: c.daily_at.format("%H:%M").to_string(), tz: c.tz.name().to_string(), idle_timeout_secs: c.idle_timeout.as_secs(), chapter_workers: c.chapter_workers as u64, retention_days: c.retention_days, start_url: c.start_url.clone(), rate_ms: c.rate_ms, cdn_host: c.cdn_host.clone(), cdn_rate_ms: c.cdn_rate_ms, cookie_domain: c.cookie_domain.clone(), user_agent: c.user_agent.clone(), download_allowlist: c.download_allowlist.hosts().to_vec(), allow_any_host: c.download_allowlist.is_allow_any(), max_image_bytes: c.max_image_bytes as u64, manga_limit: c.manga_limit as u64, job_timeout_secs: c.job_timeout.as_secs(), metadata_max_consecutive_failures: c.metadata_max_consecutive_failures, browser_restart_threshold: c.browser_restart_threshold, } } /// Overlay the DTO onto an env-derived `base` (which carries the env-only /// fields: browser, proxy, TOR, session, cookie domain) to produce the /// effective runtime config. Validates and returns all field errors at /// once on failure. pub fn to_config(&self, base: &CrawlerConfig) -> Result { let mut errs = FieldErrors::default(); let daily_at = match NaiveTime::parse_from_str(&self.daily_at, "%H:%M") { Ok(t) => t, Err(_) => { errs.push("daily_at", "must be HH:MM (24-hour)"); base.daily_at } }; let tz: Tz = match self.tz.parse() { Ok(t) => t, Err(_) => { errs.push("tz", "must be a valid IANA timezone (e.g. UTC, Europe/Berlin)"); base.tz } }; if self.chapter_workers < 1 { errs.push("chapter_workers", "must be at least 1"); } else if self.chapter_workers > MAX_CHAPTER_WORKERS { errs.push("chapter_workers", format!("must be at most {MAX_CHAPTER_WORKERS}")); } if let Some(url) = self.start_url.as_deref().map(str::trim).filter(|s| !s.is_empty()) { // SSRF defence: Url::parse alone admits http://169.254.169.254 // / http://127.0.0.1:5432 etc., and the browser would happily // navigate there with the admin's session. ensure_public_target // refuses literal-IP private ranges + localhost while still // accepting public hosts. if let Err(e) = crate::crawler::safety::ensure_public_target(url) { errs.push("start_url", url_safety_message(&e)); } } if self.job_timeout_secs < 1 { errs.push("job_timeout_secs", "must be at least 1 second"); } else if self.job_timeout_secs > MAX_TIMEOUT_SECS { errs.push("job_timeout_secs", format!("must be at most {MAX_TIMEOUT_SECS} seconds")); } if self.idle_timeout_secs > MAX_TIMEOUT_SECS { errs.push("idle_timeout_secs", format!("must be at most {MAX_TIMEOUT_SECS} seconds")); } // 0 is intentionally "unlimited"; only an explicit positive value is // capped. if self.manga_limit > MAX_MANGA_LIMIT { errs.push("manga_limit", format!("must be at most {MAX_MANGA_LIMIT}")); } if !errs.is_empty() { return Err(errs); } let start_url = self .start_url .as_deref() .map(str::trim) .filter(|s| !s.is_empty()) .map(str::to_string); let cdn_host = self .cdn_host .as_deref() .map(str::trim) .filter(|s| !s.is_empty()) .map(str::to_string); let download_allowlist = build_allowlist( self.allow_any_host, start_url.as_deref(), cdn_host.as_deref(), &self.download_allowlist, ); Ok(CrawlerConfig { daemon_enabled: self.daemon_enabled, daily_at, tz, idle_timeout: Duration::from_secs(self.idle_timeout_secs), chapter_workers: (self.chapter_workers as usize).max(1), retention_days: self.retention_days, // Not a runtime-editable setting; re-read the env default so a // settings reload keeps whatever CRAWL_METRICS_RETENTION_DAYS was // configured at boot. metrics_retention_days: crate::config::env_u64("CRAWL_METRICS_RETENTION_DAYS", 90) as u32, start_url, rate_ms: self.rate_ms, cdn_host, cdn_rate_ms: self.cdn_rate_ms, user_agent: self .user_agent .as_deref() .map(str::trim) .filter(|s| !s.is_empty()) .map(str::to_string), cookie_domain: self .cookie_domain .as_deref() .map(str::trim) .filter(|s| !s.is_empty()) .map(str::to_string), download_allowlist, max_image_bytes: self.max_image_bytes as usize, // Env-only safety cap, not a runtime-editable setting — preserve // it from the base so a settings reload keeps the boot value. max_images_per_chapter: base.max_images_per_chapter, manga_limit: self.manga_limit as usize, job_timeout: Duration::from_secs(self.job_timeout_secs.max(1)), metadata_max_consecutive_failures: self.metadata_max_consecutive_failures, browser_restart_threshold: self.browser_restart_threshold, // Env-only / structural fields preserved from the base. phpsessid: base.phpsessid.clone(), proxy: base.proxy.clone(), tor_control_url: base.tor_control_url.clone(), tor_control_password: base.tor_control_password.clone(), tor_control_cookie_path: base.tor_control_cookie_path.clone(), tor_recircuit_max_attempts: base.tor_recircuit_max_attempts, browser: base.browser.clone(), ssrf_intercept: base.ssrf_intercept, }) } } /// Render a [`crate::crawler::safety::UrlSafetyError`] as an operator-friendly /// validation message. The variant detail is preserved so a deliberate /// `10.0.0.1` value gets "private/loopback IP not allowed" rather than the /// generic "invalid URL". fn url_safety_message(e: &crate::crawler::safety::UrlSafetyError) -> &'static str { use crate::crawler::safety::UrlSafetyError as E; match e { E::Unparseable => "must be a valid absolute URL", E::BadScheme(_) => "must use http:// or https://", E::NoHost => "must include a host", E::Loopback => "must not point at localhost", E::PrivateIp(_) => "must not point at a private/loopback/link-local IP (SSRF risk)", E::HostNotAllowed(_) => "host is not on the allowlist", } } /// Rebuild a [`DownloadAllowlist`] from the DTO, always seeding the start-url /// and CDN hosts (mirrors `config::build_download_allowlist`). fn build_allowlist( allow_any: bool, start_url: Option<&str>, cdn_host: Option<&str>, extras: &[String], ) -> DownloadAllowlist { if allow_any { return DownloadAllowlist::allow_any(); } let mut allow = DownloadAllowlist::new(); if let Some(url) = start_url { if let Ok(parsed) = reqwest::Url::parse(url) { if let Some(h) = parsed.host_str() { allow = allow.allow(h); } } } if let Some(host) = cdn_host { allow = allow.allow(host); } for h in extras { let trimmed = h.trim(); if !trimmed.is_empty() { allow = allow.allow(trimmed); } } allow } // --------------------------------------------------------------------------- // Analysis // --------------------------------------------------------------------------- /// Admin-editable analysis-worker settings. The vision `api_key` is env-only /// and never carried here. Prompts are `Option`: `None` means "use /// the compiled default" (the UI's "reset to default"). #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(default)] pub struct AnalysisSettings { pub enabled: bool, pub workers: u64, pub endpoint: String, pub model: String, pub request_timeout_secs: u64, pub job_timeout_secs: u64, pub max_tokens: u32, pub max_pixels: u32, pub min_slice_height: u32, pub slice_overlap: f64, pub tall_aspect_threshold: f64, pub max_slices: u64, pub max_image_bytes: u64, /// `"json_schema" | "json_object" | "none"`. pub response_format: String, pub frequency_penalty: f64, pub temperature: f64, pub system_prompt: Option, pub ocr_prompt: Option, pub grounding_prompt: Option, } impl Default for AnalysisSettings { fn default() -> Self { Self::from_config(&AnalysisConfig::default()) } } impl AnalysisSettings { pub fn from_config(c: &AnalysisConfig) -> Self { // Capture a prompt only when it diverges from the compiled default, // so an unmodified config seeds as `None` ("use default"). let opt = |cur: &str, def: &str| (cur != def).then(|| cur.to_string()); Self { enabled: c.enabled, workers: c.workers as u64, endpoint: c.endpoint.clone(), model: c.model.clone(), request_timeout_secs: c.request_timeout.as_secs(), job_timeout_secs: c.job_timeout.as_secs(), max_tokens: c.max_tokens, max_pixels: c.max_pixels, min_slice_height: c.min_slice_height, slice_overlap: c.slice_overlap, tall_aspect_threshold: c.tall_aspect_threshold, max_slices: c.max_slices as u64, max_image_bytes: c.max_image_bytes as u64, response_format: c.response_format.as_str().to_string(), frequency_penalty: c.frequency_penalty, temperature: c.temperature, system_prompt: opt(&c.system_prompt, SYSTEM_PROMPT_DEFAULT), ocr_prompt: opt(&c.ocr_prompt, OCR_PROMPT_DEFAULT), grounding_prompt: opt(&c.grounding_prompt, GROUNDING_PROMPT_DEFAULT), } } pub fn to_config(&self, base: &AnalysisConfig) -> Result { let mut errs = FieldErrors::default(); if self.workers < 1 { errs.push("workers", "must be at least 1"); } else if self.workers > MAX_WORKERS { errs.push("workers", format!("must be at most {MAX_WORKERS}")); } // The endpoint/model are vision-only knobs — the OCR backend never // dials a URL or sends a model id. While vision is dormant // (`base.backend == Ocr`), skip the live-worker SSRF gate and the // model-required check so an OCR operator can enable the worker // without a vision endpoint/model (the OCR settings UI doesn't even // expose them). The basic malformed-URL sanity check below stays // unconditional. Re-enabling vision restores the full gate via the // `base.backend == Vision` predicate. let vision_active = base.backend == AnalysisBackend::Vision; let trimmed_endpoint = self.endpoint.trim(); if trimmed_endpoint.is_empty() { errs.push("endpoint", "must be a valid absolute URL"); } else if let Err(e) = reqwest::Url::parse(trimmed_endpoint) { // Always reject obviously-malformed URLs (matches prior behaviour). let _ = e; errs.push("endpoint", "must be a valid absolute URL"); } else if self.enabled && vision_active { // SSRF + API-key exfiltration defence — only enforced when the // worker is actually live. Worker attaches an env-managed bearer // token to every call; without this check, an admin (or one // CSRF-able request) could repoint the endpoint at // http://169.254.169.254 (cloud metadata) or http://postgres:5432 // (an internal service that would receive the bearer header). // Docker DNS names (`mangalord-vision`) still pass because // they're hostnames, not IP literals. // // A disabled config is allowed to carry the dev-default // `http://localhost:8000/...` placeholder; the worker won't dial // it, and toggling `enabled=true` later re-runs this validator. if let Err(e) = crate::crawler::safety::ensure_public_target(trimmed_endpoint) { errs.push("endpoint", url_safety_message(&e)); } } if self.enabled && vision_active && self.model.trim().is_empty() { errs.push("model", "required when analysis is enabled"); } if self.max_tokens < 1 { errs.push("max_tokens", "must be at least 1"); } else if self.max_tokens > MAX_ANALYSIS_MAX_TOKENS { errs.push("max_tokens", format!("must be at most {MAX_ANALYSIS_MAX_TOKENS}")); } if self.request_timeout_secs < 1 { errs.push("request_timeout_secs", "must be at least 1 second"); } else if self.request_timeout_secs > MAX_TIMEOUT_SECS { errs.push( "request_timeout_secs", format!("must be at most {MAX_TIMEOUT_SECS} seconds"), ); } if self.job_timeout_secs < 1 { errs.push("job_timeout_secs", "must be at least 1 second"); } else if self.job_timeout_secs > MAX_TIMEOUT_SECS { errs.push( "job_timeout_secs", format!("must be at most {MAX_TIMEOUT_SECS} seconds"), ); } if self.max_pixels < 1 { errs.push("max_pixels", "must be at least 1"); } if self.max_image_bytes < 1 { errs.push("max_image_bytes", "must be greater than 0"); } else if self.max_image_bytes > MAX_ANALYSIS_IMAGE_BYTES { errs.push( "max_image_bytes", format!("must be at most {MAX_ANALYSIS_IMAGE_BYTES} (1 GiB)"), ); } if !(0.0..=0.9).contains(&self.slice_overlap) { errs.push("slice_overlap", "must be between 0.0 and 0.9"); } if self.tall_aspect_threshold < 1.0 { errs.push("tall_aspect_threshold", "must be at least 1.0"); } if self.min_slice_height < 1 { errs.push("min_slice_height", "must be at least 1"); } if self.max_slices < 1 { errs.push("max_slices", "must be at least 1"); } else if self.max_slices > MAX_ANALYSIS_SLICES { errs.push("max_slices", format!("must be at most {MAX_ANALYSIS_SLICES}")); } if !(0.0..=MAX_TEMPERATURE).contains(&self.temperature) { errs.push("temperature", format!("must be between 0 and {MAX_TEMPERATURE}")); } if !(FREQUENCY_PENALTY_MIN..=FREQUENCY_PENALTY_MAX).contains(&self.frequency_penalty) { errs.push( "frequency_penalty", format!("must be between {FREQUENCY_PENALTY_MIN} and {FREQUENCY_PENALTY_MAX}"), ); } let response_format = match ResponseFormat::parse_strict(&self.response_format) { Some(rf) => rf, None => { errs.push( "response_format", "must be one of: json_schema, json_object, none", ); base.response_format } }; if !errs.is_empty() { return Err(errs); } let prompt = |o: &Option, def: &str| { o.as_deref() .map(str::trim) .filter(|s| !s.is_empty()) .unwrap_or(def) .to_string() }; Ok(AnalysisConfig { enabled: self.enabled, workers: (self.workers as usize).max(1), endpoint: self.endpoint.trim().to_string(), model: self.model.trim().to_string(), request_timeout: Duration::from_secs(self.request_timeout_secs), job_timeout: Duration::from_secs(self.job_timeout_secs), max_tokens: self.max_tokens, max_pixels: self.max_pixels, min_slice_height: self.min_slice_height.max(1), slice_overlap: self.slice_overlap.clamp(0.0, 0.9), tall_aspect_threshold: self.tall_aspect_threshold.max(1.0), max_slices: (self.max_slices as usize).max(1), max_image_bytes: self.max_image_bytes as usize, response_format, frequency_penalty: self.frequency_penalty, temperature: self.temperature.max(0.0), system_prompt: prompt(&self.system_prompt, SYSTEM_PROMPT_DEFAULT), ocr_prompt: prompt(&self.ocr_prompt, OCR_PROMPT_DEFAULT), grounding_prompt: prompt(&self.grounding_prompt, GROUNDING_PROMPT_DEFAULT), // Env-only secret preserved from the base. api_key: base.api_key.clone(), // Env-only readiness probe URL preserved from the base. vision_health_url: base.vision_health_url.clone(), // Deploy-time engine selection + ocrs model paths: env-only, not // admin-tunable, so carry them through from the base unchanged. backend: base.backend, ocr_detection_model: base.ocr_detection_model.clone(), ocr_recognition_model: base.ocr_recognition_model.clone(), // Env-only decompression-bomb decode cap, carried from the base. ocr_max_decode_pixels: base.ocr_max_decode_pixels, }) } } /// Prompt defaults, returned by the API so the UI can render placeholders and /// implement "reset to default". #[derive(Debug, Clone, Serialize)] pub struct PromptDefaults { pub system_prompt: &'static str, pub ocr_prompt: &'static str, pub grounding_prompt: &'static str, } impl PromptDefaults { pub fn get() -> Self { Self { system_prompt: SYSTEM_PROMPT_DEFAULT, ocr_prompt: OCR_PROMPT_DEFAULT, grounding_prompt: GROUNDING_PROMPT_DEFAULT, } } } #[cfg(test)] mod tests { use super::*; // --- crawler round-trip & validation --- #[test] fn crawler_round_trips_through_dto() { let base = CrawlerConfig { start_url: Some("https://example.com/".to_string()), tz: Tz::Europe__Berlin, chapter_workers: 3, cookie_domain: Some("example.com".to_string()), ..Default::default() }; let dto = CrawlerSettings::from_config(&base); let back = dto.to_config(&base).expect("valid"); assert_eq!(back.tz, Tz::Europe__Berlin); assert_eq!(back.chapter_workers, 3); assert_eq!(back.start_url.as_deref(), Some("https://example.com/")); // cookie_domain is now an editable DTO field, round-tripping like the rest. assert_eq!(back.cookie_domain.as_deref(), Some("example.com")); assert_eq!(dto.daily_at, "00:00"); } #[test] fn crawler_cookie_domain_is_editable() { let base = CrawlerConfig::default(); let dto = CrawlerSettings { cookie_domain: Some("new.example.org".to_string()), ..CrawlerSettings::from_config(&base) }; assert_eq!( dto.to_config(&base).unwrap().cookie_domain.as_deref(), Some("new.example.org") ); } #[test] fn crawler_overlay_preserves_env_only_fields() { let base = CrawlerConfig { proxy: Some("socks5://127.0.0.1:9050".to_string()), tor_control_password: Some("secret".to_string()), phpsessid: Some("abc123".to_string()), ..Default::default() }; // A DTO that knows nothing about the env-only fields. let dto = CrawlerSettings { rate_ms: 2000, ..CrawlerSettings::from_config(&base) }; let out = dto.to_config(&base).expect("valid"); assert_eq!(out.rate_ms, 2000); assert_eq!(out.proxy.as_deref(), Some("socks5://127.0.0.1:9050")); assert_eq!(out.tor_control_password.as_deref(), Some("secret")); assert_eq!(out.phpsessid.as_deref(), Some("abc123")); } #[test] fn crawler_rejects_bad_tz_and_time() { let base = CrawlerConfig::default(); let dto = CrawlerSettings { tz: "Mars/Phobos".to_string(), daily_at: "9am".to_string(), ..CrawlerSettings::from_config(&base) }; let errs = dto.to_config(&base).unwrap_err(); let fields: Vec<_> = errs.errors.iter().map(|e| e.field.as_str()).collect(); assert!(fields.contains(&"tz")); assert!(fields.contains(&"daily_at")); } #[test] fn crawler_allowlist_seeds_start_and_cdn_hosts() { let base = CrawlerConfig::default(); let dto = CrawlerSettings { start_url: Some("https://catalog.example.com/".to_string()), cdn_host: Some("cdn.example.com".to_string()), download_allowlist: vec!["extra.example.net".to_string()], allow_any_host: false, ..CrawlerSettings::from_config(&base) }; let out = dto.to_config(&base).expect("valid"); assert!(out.download_allowlist.contains("catalog.example.com")); assert!(out.download_allowlist.contains("cdn.example.com")); assert!(out.download_allowlist.contains("extra.example.net")); assert!(!out.download_allowlist.is_allow_any()); } #[test] fn crawler_start_url_rejects_ip_literal_attacks() { let base = CrawlerConfig::default(); for url in [ "http://169.254.169.254/", "http://127.0.0.1/catalog", "http://localhost:5432/", "http://10.0.0.1/", ] { let dto = CrawlerSettings { start_url: Some(url.to_string()), ..CrawlerSettings::from_config(&base) }; let errs = dto.to_config(&base).unwrap_err(); let fields: Vec<_> = errs.errors.iter().map(|e| e.field.as_str()).collect(); assert!(fields.contains(&"start_url"), "must reject {url}"); } } #[test] fn crawler_start_url_accepts_public_targets() { let base = CrawlerConfig::default(); for url in ["https://catalog.example.com/", "http://catalog.example.com/"] { let dto = CrawlerSettings { start_url: Some(url.to_string()), ..CrawlerSettings::from_config(&base) }; assert!(dto.to_config(&base).is_ok(), "must accept {url}"); } } #[test] fn crawler_allow_any_host_bypasses_list() { let base = CrawlerConfig::default(); let dto = CrawlerSettings { allow_any_host: true, ..CrawlerSettings::from_config(&base) }; let out = dto.to_config(&base).expect("valid"); assert!(out.download_allowlist.is_allow_any()); } // --- analysis round-trip, validation & prompt defaults --- #[test] fn analysis_round_trips_and_omits_default_prompts() { let base = AnalysisConfig::default(); let dto = AnalysisSettings::from_config(&base); // Unmodified prompts seed as None ("use default"). assert!(dto.system_prompt.is_none()); assert!(dto.ocr_prompt.is_none()); assert!(dto.grounding_prompt.is_none()); assert_eq!(dto.response_format, "json_schema"); let back = dto.to_config(&base).expect("valid"); assert_eq!(back.system_prompt, SYSTEM_PROMPT_DEFAULT); assert_eq!(back.response_format, ResponseFormat::JsonSchema); } #[test] fn analysis_captures_env_prompt_override() { let base = AnalysisConfig { system_prompt: "custom env prompt".to_string(), ..Default::default() }; let dto = AnalysisSettings::from_config(&base); assert_eq!(dto.system_prompt.as_deref(), Some("custom env prompt")); } #[test] fn analysis_prompt_override_and_reset() { let base = AnalysisConfig::default(); // Override applied. let dto = AnalysisSettings { system_prompt: Some("OVERRIDE".to_string()), ..AnalysisSettings::from_config(&base) }; assert_eq!(dto.to_config(&base).unwrap().system_prompt, "OVERRIDE"); // None / blank falls back to the compiled default. let dto = AnalysisSettings { system_prompt: Some(" ".to_string()), ..AnalysisSettings::from_config(&base) }; assert_eq!(dto.to_config(&base).unwrap().system_prompt, SYSTEM_PROMPT_DEFAULT); } #[test] fn analysis_overlay_preserves_api_key() { let base = AnalysisConfig { api_key: Some("sk-secret".to_string()), ..Default::default() }; let dto = AnalysisSettings::from_config(&base); assert_eq!(dto.to_config(&base).unwrap().api_key.as_deref(), Some("sk-secret")); } #[test] fn analysis_rejects_out_of_range() { let base = AnalysisConfig::default(); let dto = AnalysisSettings { workers: 0, slice_overlap: 1.5, tall_aspect_threshold: 0.5, response_format: "yaml".to_string(), endpoint: "not a url".to_string(), request_timeout_secs: 0, job_timeout_secs: 0, max_pixels: 0, max_image_bytes: 0, ..AnalysisSettings::from_config(&base) }; let errs = dto.to_config(&base).unwrap_err(); let fields: Vec<_> = errs.errors.iter().map(|e| e.field.as_str()).collect(); for f in [ "workers", "slice_overlap", "tall_aspect_threshold", "response_format", "endpoint", "request_timeout_secs", "job_timeout_secs", "max_pixels", "max_image_bytes", ] { assert!(fields.contains(&f), "missing error for {f}"); } } #[test] fn analysis_rejects_over_upper_bounds() { // Sanity caps: an absurdly large worker count / buffer / token budget // and out-of-range sampling params are all refused so a fat-fingered // or CSRF-injected value can't exhaust the host or break the upstream // API contract. let base = AnalysisConfig::default(); let dto = AnalysisSettings { workers: MAX_WORKERS + 1, max_tokens: MAX_ANALYSIS_MAX_TOKENS + 1, max_slices: MAX_ANALYSIS_SLICES + 1, max_image_bytes: MAX_ANALYSIS_IMAGE_BYTES + 1, temperature: MAX_TEMPERATURE + 0.5, frequency_penalty: FREQUENCY_PENALTY_MAX + 0.5, request_timeout_secs: MAX_TIMEOUT_SECS + 1, job_timeout_secs: MAX_TIMEOUT_SECS + 1, ..AnalysisSettings::from_config(&base) }; let errs = dto.to_config(&base).unwrap_err(); let fields: Vec<_> = errs.errors.iter().map(|e| e.field.as_str()).collect(); for f in [ "workers", "max_tokens", "max_slices", "max_image_bytes", "temperature", "frequency_penalty", "request_timeout_secs", "job_timeout_secs", ] { assert!(fields.contains(&f), "missing upper-bound error for {f}"); } } #[test] fn crawler_rejects_over_upper_bounds() { // manga_limit ceiling + timeout caps. manga_limit=0 stays "unlimited" // and is asserted valid by the round-trip tests above. let base = CrawlerConfig::default(); let dto = CrawlerSettings { manga_limit: MAX_MANGA_LIMIT + 1, job_timeout_secs: MAX_TIMEOUT_SECS + 1, idle_timeout_secs: MAX_TIMEOUT_SECS + 1, ..CrawlerSettings::from_config(&base) }; let errs = dto.to_config(&base).unwrap_err(); let fields: Vec<_> = errs.errors.iter().map(|e| e.field.as_str()).collect(); for f in ["manga_limit", "job_timeout_secs", "idle_timeout_secs"] { assert!(fields.contains(&f), "missing upper-bound error for {f}"); } } #[test] fn crawler_allows_unlimited_manga_limit() { // 0 means "no cap" and must stay valid. let base = CrawlerConfig::default(); let dto = CrawlerSettings { manga_limit: 0, ..CrawlerSettings::from_config(&base) }; assert!(dto.to_config(&base).is_ok()); } #[test] fn analysis_rejects_negative_frequency_penalty_below_range() { let base = AnalysisConfig::default(); let dto = AnalysisSettings { frequency_penalty: FREQUENCY_PENALTY_MIN - 0.5, ..AnalysisSettings::from_config(&base) }; let errs = dto.to_config(&base).unwrap_err(); let fields: Vec<_> = errs.errors.iter().map(|e| e.field.as_str()).collect(); assert!(fields.contains(&"frequency_penalty")); } #[test] fn analysis_accepts_in_range_sampling_params() { // A normal config with mid-range sampling values stays valid. let base = AnalysisConfig::default(); let dto = AnalysisSettings { temperature: 0.7, frequency_penalty: 0.3, ..AnalysisSettings::from_config(&base) }; assert!(dto.to_config(&base).is_ok()); } #[test] fn crawler_rejects_over_chapter_worker_cap() { let base = CrawlerConfig::default(); let dto = CrawlerSettings { chapter_workers: MAX_CHAPTER_WORKERS + 1, ..CrawlerSettings::from_config(&base) }; let errs = dto.to_config(&base).unwrap_err(); let fields: Vec<_> = errs.errors.iter().map(|e| e.field.as_str()).collect(); assert!(fields.contains(&"chapter_workers")); } #[test] fn analysis_endpoint_rejects_ip_literal_attacks_when_enabled() { // The vision worker bearer-attaches an env secret to every call; // a hostile/CSRF-able admin must NOT be able to point endpoint at // cloud metadata, loopback services, or RFC1918 hosts — when the // worker is enabled (toggling enabled=true later re-runs this gate). let base = AnalysisConfig { backend: AnalysisBackend::Vision, ..Default::default() }; for url in [ "http://169.254.169.254/v1/chat/completions", "http://127.0.0.1:5432/", "http://localhost:11434/v1/chat/completions", "http://10.0.0.5/v1/chat/completions", ] { let dto = AnalysisSettings { enabled: true, model: "test-model".to_string(), endpoint: url.to_string(), ..AnalysisSettings::from_config(&base) }; let errs = dto.to_config(&base).unwrap_err(); let fields: Vec<_> = errs.errors.iter().map(|e| e.field.as_str()).collect(); assert!(fields.contains(&"endpoint"), "must reject {url}"); } } #[test] fn analysis_endpoint_accepts_docker_dns_and_public_hosts_when_enabled() { // The documented default — docker DNS name resolving to a private IP // at runtime — must still validate, because the bearer recipient // identity is the operator-chosen hostname, not the underlying IP. let base = AnalysisConfig { backend: AnalysisBackend::Vision, ..Default::default() }; for url in [ "http://mangalord-vision:8000/v1/chat/completions", "https://api.openai.com/v1/chat/completions", ] { let dto = AnalysisSettings { enabled: true, model: "test-model".to_string(), endpoint: url.to_string(), ..AnalysisSettings::from_config(&base) }; assert!(dto.to_config(&base).is_ok(), "must accept {url}"); } } #[test] fn analysis_endpoint_skips_safety_check_when_disabled() { // Disabled-but-saved bad URL is harmless (worker won't dial it); // operators get to keep the dev-localhost placeholder until they // toggle the worker on, when the validator re-runs. let base = AnalysisConfig::default(); let dto = AnalysisSettings { enabled: false, endpoint: "http://localhost:8000/v1/chat/completions".to_string(), ..AnalysisSettings::from_config(&base) }; assert!(dto.to_config(&base).is_ok()); } #[test] fn analysis_requires_model_only_when_enabled() { // Vision base: the model id is required only when the vision worker // is actually live (see the OCR carve-out below). let base = AnalysisConfig { backend: AnalysisBackend::Vision, ..Default::default() }; let disabled = AnalysisSettings { enabled: false, model: "".to_string(), ..AnalysisSettings::from_config(&base) }; assert!(disabled.to_config(&base).is_ok()); let enabled = AnalysisSettings { enabled: true, model: "".to_string(), ..AnalysisSettings::from_config(&base) }; let errs = enabled.to_config(&base).unwrap_err(); assert!(errs.errors.iter().any(|e| e.field == "model")); } #[test] fn analysis_ocr_backend_enable_skips_vision_endpoint_and_model_validation() { // With the OCR backend active the worker never dials the vision // endpoint nor sends a model id, so enabling analysis must NOT // validate those vision-only fields. The default base carries the // dev-localhost endpoint (which the SSRF gate would otherwise reject) // and an empty model — under OCR, enabling is still valid. Without // this carve-out an OCR operator can't turn the worker on, since the // OCR settings UI doesn't expose endpoint/model to fix. let base = AnalysisConfig::default(); // backend = Ocr let dto = AnalysisSettings { enabled: true, endpoint: "http://localhost:8000/v1/chat/completions".to_string(), model: "".to_string(), ..AnalysisSettings::from_config(&base) }; assert!( dto.to_config(&base).is_ok(), "OCR-enabled config must not fail on vision endpoint/model" ); } #[test] fn dtos_serialize_to_json_and_back() { let c = CrawlerSettings::default(); let v = serde_json::to_value(&c).unwrap(); let back: CrawlerSettings = serde_json::from_value(v).unwrap(); assert_eq!(c, back); let a = AnalysisSettings::default(); let v = serde_json::to_value(&a).unwrap(); let back: AnalysisSettings = serde_json::from_value(v).unwrap(); assert_eq!(a, back); } }