Mangalord/backend/src/settings.rs

//! Serializable, admin-editable settings DTOs for the crawler and analysis
//! subsystems, and their conversion to/from the runtime [`CrawlerConfig`] /
//! [`AnalysisConfig`].
//!
//! These DTOs are the boundary persisted in the `app_settings` table (one
//! JSONB row per group) and exchanged over the admin API. They carry **only
//! the operationally-safe, UI-editable fields** as primitive types — the
//! runtime configs additionally hold env-only/structural fields (browser
//! launch options, proxy, TOR control, the vision API key, the cookie
//! domain) that are never persisted here.
//!
//! Flow: at boot, [`CrawlerSettings::from_config`] /
//! [`AnalysisSettings::from_config`] derive the env-seed DTO; it is written to
//! the DB only when the row is absent. Thereafter the DB row is the source of
//! truth, and [`CrawlerSettings::to_config`] / [`AnalysisSettings::to_config`]
//! overlay it onto the env-derived **base** (which carries the env-only
//! fields) to produce the effective config the daemons are (re)spawned with.

use std::time::Duration;

use chrono::NaiveTime;
use chrono_tz::Tz;
use serde::{Deserialize, Serialize};

use crate::analysis::prompt::{
    GROUNDING_PROMPT_DEFAULT, OCR_PROMPT_DEFAULT, SYSTEM_PROMPT_DEFAULT,
};
use crate::config::{AnalysisBackend, AnalysisConfig, CrawlerConfig, ResponseFormat};
use crate::crawler::safety::DownloadAllowlist;

/// `app_settings.key` for the crawler group.
pub const KEY_CRAWLER: &str = "crawler";
/// `app_settings.key` for the analysis group.
pub const KEY_ANALYSIS: &str = "analysis";

// Upper bounds on numeric settings. These are sanity caps, not tuned limits —
// they keep a fat-fingered (or CSRF-injected) value from spawning thousands of
// workers, demanding gigabyte buffers, or sending out-of-range sampling
// params. Generous enough that no realistic deployment hits them.
const MAX_WORKERS: u64 = 64;
const MAX_CHAPTER_WORKERS: u64 = 64;
const MAX_ANALYSIS_MAX_TOKENS: u32 = 1_000_000;
const MAX_ANALYSIS_SLICES: u64 = 1024;
/// 1 GiB — far above any real page, well below "exhaust the host".
const MAX_ANALYSIS_IMAGE_BYTES: u64 = 1024 * 1024 * 1024;
/// OpenAI-compatible sampling ranges (the analysis endpoint speaks that API).
const MAX_TEMPERATURE: f64 = 2.0;
const FREQUENCY_PENALTY_MIN: f64 = -2.0;
const FREQUENCY_PENALTY_MAX: f64 = 2.0;
/// Max manga-detail fetches per metadata pass. `0` stays special-cased as
/// "unlimited"; this only bounds an explicit positive value.
const MAX_MANGA_LIMIT: u64 = 1_000_000;
/// Upper bound (seconds) shared by every timeout knob — 24h. A timeout
/// longer than a day is almost certainly a fat-fingered value (e.g. ms
/// mistaken for s) and would wedge a worker for far too long.
const MAX_TIMEOUT_SECS: u64 = 86_400;

/// One field-level validation failure, surfaced to the UI per input.
#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
pub struct FieldError {
    pub field: String,
    pub message: String,
}

/// Collected validation failures for a settings payload. Empty == valid.
#[derive(Debug, Clone, Serialize, Default, PartialEq, Eq)]
pub struct FieldErrors {
    pub errors: Vec<FieldError>,
}

impl FieldErrors {
    fn push(&mut self, field: &str, message: impl Into<String>) {
        self.errors.push(FieldError {
            field: field.to_string(),
            message: message.into(),
        });
    }

    pub fn is_empty(&self) -> bool {
        self.errors.is_empty()
    }
}

// ---------------------------------------------------------------------------
// Crawler
// ---------------------------------------------------------------------------

/// Admin-editable crawler-daemon settings. Mirrors the operational `CRAWLER_*`
/// env vars; host/infra and session fields stay env-only.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(default)]
pub struct CrawlerSettings {
    pub daemon_enabled: bool,
    /// Daily metadata-pass time, `"HH:MM"` (24h).
    pub daily_at: String,
    /// IANA timezone for the daily schedule.
    pub tz: String,
    pub idle_timeout_secs: u64,
    pub chapter_workers: u64,
    pub retention_days: u32,
    pub start_url: Option<String>,
    pub rate_ms: u64,
    pub cdn_host: Option<String>,
    pub cdn_rate_ms: u64,
    /// Domain the session cookie (PHPSESSID) is scoped to when injecting it
    /// into the browser / CDN requests. Pairs with `start_url`.
    pub cookie_domain: Option<String>,
    pub user_agent: Option<String>,
    /// Hosts the crawler may download images from (in addition to the
    /// auto-seeded start-url / CDN hosts). Ignored when `allow_any_host`.
    pub download_allowlist: Vec<String>,
    pub allow_any_host: bool,
    pub max_image_bytes: u64,
    /// Max manga detail fetches per metadata pass; `0` = unlimited.
    pub manga_limit: u64,
    pub job_timeout_secs: u64,
    pub metadata_max_consecutive_failures: u32,
    pub browser_restart_threshold: u32,
}

impl Default for CrawlerSettings {
    fn default() -> Self {
        Self::from_config(&CrawlerConfig::default())
    }
}

impl CrawlerSettings {
    /// Derive the DTO from a runtime config (used to seed the DB from env).
    pub fn from_config(c: &CrawlerConfig) -> Self {
        Self {
            daemon_enabled: c.daemon_enabled,
            daily_at: c.daily_at.format("%H:%M").to_string(),
            tz: c.tz.name().to_string(),
            idle_timeout_secs: c.idle_timeout.as_secs(),
            chapter_workers: c.chapter_workers as u64,
            retention_days: c.retention_days,
            start_url: c.start_url.clone(),
            rate_ms: c.rate_ms,
            cdn_host: c.cdn_host.clone(),
            cdn_rate_ms: c.cdn_rate_ms,
            cookie_domain: c.cookie_domain.clone(),
            user_agent: c.user_agent.clone(),
            download_allowlist: c.download_allowlist.hosts().to_vec(),
            allow_any_host: c.download_allowlist.is_allow_any(),
            max_image_bytes: c.max_image_bytes as u64,
            manga_limit: c.manga_limit as u64,
            job_timeout_secs: c.job_timeout.as_secs(),
            metadata_max_consecutive_failures: c.metadata_max_consecutive_failures,
            browser_restart_threshold: c.browser_restart_threshold,
        }
    }

    /// Overlay the DTO onto an env-derived `base` (which carries the env-only
    /// fields: browser, proxy, TOR, session, cookie domain) to produce the
    /// effective runtime config. Validates and returns all field errors at
    /// once on failure.
    pub fn to_config(&self, base: &CrawlerConfig) -> Result<CrawlerConfig, FieldErrors> {
        let mut errs = FieldErrors::default();

        let daily_at = match NaiveTime::parse_from_str(&self.daily_at, "%H:%M") {
            Ok(t) => t,
            Err(_) => {
                errs.push("daily_at", "must be HH:MM (24-hour)");
                base.daily_at
            }
        };
        let tz: Tz = match self.tz.parse() {
            Ok(t) => t,
            Err(_) => {
                errs.push("tz", "must be a valid IANA timezone (e.g. UTC, Europe/Berlin)");
                base.tz
            }
        };
        if self.chapter_workers < 1 {
            errs.push("chapter_workers", "must be at least 1");
        } else if self.chapter_workers > MAX_CHAPTER_WORKERS {
            errs.push("chapter_workers", format!("must be at most {MAX_CHAPTER_WORKERS}"));
        }
        if let Some(url) = self.start_url.as_deref().map(str::trim).filter(|s| !s.is_empty()) {
            // SSRF defence: Url::parse alone admits http://169.254.169.254
            // / http://127.0.0.1:5432 etc., and the browser would happily
            // navigate there with the admin's session. ensure_public_target
            // refuses literal-IP private ranges + localhost while still
            // accepting public hosts.
            if let Err(e) = crate::crawler::safety::ensure_public_target(url) {
                errs.push("start_url", url_safety_message(&e));
            }
        }
        if self.job_timeout_secs < 1 {
            errs.push("job_timeout_secs", "must be at least 1 second");
        } else if self.job_timeout_secs > MAX_TIMEOUT_SECS {
            errs.push("job_timeout_secs", format!("must be at most {MAX_TIMEOUT_SECS} seconds"));
        }
        if self.idle_timeout_secs > MAX_TIMEOUT_SECS {
            errs.push("idle_timeout_secs", format!("must be at most {MAX_TIMEOUT_SECS} seconds"));
        }
        // 0 is intentionally "unlimited"; only an explicit positive value is
        // capped.
        if self.manga_limit > MAX_MANGA_LIMIT {
            errs.push("manga_limit", format!("must be at most {MAX_MANGA_LIMIT}"));
        }

        if !errs.is_empty() {
            return Err(errs);
        }

        let start_url = self
            .start_url
            .as_deref()
            .map(str::trim)
            .filter(|s| !s.is_empty())
            .map(str::to_string);
        let cdn_host = self
            .cdn_host
            .as_deref()
            .map(str::trim)
            .filter(|s| !s.is_empty())
            .map(str::to_string);

        let download_allowlist = build_allowlist(
            self.allow_any_host,
            start_url.as_deref(),
            cdn_host.as_deref(),
            &self.download_allowlist,
        );

        Ok(CrawlerConfig {
            daemon_enabled: self.daemon_enabled,
            daily_at,
            tz,
            idle_timeout: Duration::from_secs(self.idle_timeout_secs),
            chapter_workers: (self.chapter_workers as usize).max(1),
            retention_days: self.retention_days,
            // Not a runtime-editable setting; re-read the env default so a
            // settings reload keeps whatever CRAWL_METRICS_RETENTION_DAYS was
            // configured at boot.
            metrics_retention_days: crate::config::env_u64("CRAWL_METRICS_RETENTION_DAYS", 90)
                as u32,
            start_url,
            rate_ms: self.rate_ms,
            cdn_host,
            cdn_rate_ms: self.cdn_rate_ms,
            user_agent: self
                .user_agent
                .as_deref()
                .map(str::trim)
                .filter(|s| !s.is_empty())
                .map(str::to_string),
            cookie_domain: self
                .cookie_domain
                .as_deref()
                .map(str::trim)
                .filter(|s| !s.is_empty())
                .map(str::to_string),
            download_allowlist,
            max_image_bytes: self.max_image_bytes as usize,
            // Env-only safety cap, not a runtime-editable setting — preserve
            // it from the base so a settings reload keeps the boot value.
            max_images_per_chapter: base.max_images_per_chapter,
            manga_limit: self.manga_limit as usize,
            job_timeout: Duration::from_secs(self.job_timeout_secs.max(1)),
            metadata_max_consecutive_failures: self.metadata_max_consecutive_failures,
            browser_restart_threshold: self.browser_restart_threshold,
            // Env-only / structural fields preserved from the base.
            phpsessid: base.phpsessid.clone(),
            proxy: base.proxy.clone(),
            tor_control_url: base.tor_control_url.clone(),
            tor_control_password: base.tor_control_password.clone(),
            tor_control_cookie_path: base.tor_control_cookie_path.clone(),
            tor_recircuit_max_attempts: base.tor_recircuit_max_attempts,
            browser: base.browser.clone(),
            ssrf_intercept: base.ssrf_intercept,
        })
    }
}

/// Render a [`crate::crawler::safety::UrlSafetyError`] as an operator-friendly
/// validation message. The variant detail is preserved so a deliberate
/// `10.0.0.1` value gets "private/loopback IP not allowed" rather than the
/// generic "invalid URL".
fn url_safety_message(e: &crate::crawler::safety::UrlSafetyError) -> &'static str {
    use crate::crawler::safety::UrlSafetyError as E;
    match e {
        E::Unparseable => "must be a valid absolute URL",
        E::BadScheme(_) => "must use http:// or https://",
        E::NoHost => "must include a host",
        E::Loopback => "must not point at localhost",
        E::PrivateIp(_) => "must not point at a private/loopback/link-local IP (SSRF risk)",
        E::HostNotAllowed(_) => "host is not on the allowlist",
    }
}

/// Rebuild a [`DownloadAllowlist`] from the DTO, always seeding the start-url
/// and CDN hosts (mirrors `config::build_download_allowlist`).
fn build_allowlist(
    allow_any: bool,
    start_url: Option<&str>,
    cdn_host: Option<&str>,
    extras: &[String],
) -> DownloadAllowlist {
    if allow_any {
        return DownloadAllowlist::allow_any();
    }
    let mut allow = DownloadAllowlist::new();
    if let Some(url) = start_url {
        if let Ok(parsed) = reqwest::Url::parse(url) {
            if let Some(h) = parsed.host_str() {
                allow = allow.allow(h);
            }
        }
    }
    if let Some(host) = cdn_host {
        allow = allow.allow(host);
    }
    for h in extras {
        let trimmed = h.trim();
        if !trimmed.is_empty() {
            allow = allow.allow(trimmed);
        }
    }
    allow
}

// ---------------------------------------------------------------------------
// Analysis
// ---------------------------------------------------------------------------

/// Admin-editable analysis-worker settings. The vision `api_key` is env-only
/// and never carried here. Prompts are `Option<String>`: `None` means "use
/// the compiled default" (the UI's "reset to default").
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
#[serde(default)]
pub struct AnalysisSettings {
    pub enabled: bool,
    pub workers: u64,
    pub endpoint: String,
    pub model: String,
    pub request_timeout_secs: u64,
    pub job_timeout_secs: u64,
    pub max_tokens: u32,
    pub max_pixels: u32,
    pub min_slice_height: u32,
    pub slice_overlap: f64,
    pub tall_aspect_threshold: f64,
    pub max_slices: u64,
    pub max_image_bytes: u64,
    /// `"json_schema" | "json_object" | "none"`.
    pub response_format: String,
    pub frequency_penalty: f64,
    pub temperature: f64,
    pub system_prompt: Option<String>,
    pub ocr_prompt: Option<String>,
    pub grounding_prompt: Option<String>,
}

impl Default for AnalysisSettings {
    fn default() -> Self {
        Self::from_config(&AnalysisConfig::default())
    }
}

impl AnalysisSettings {
    pub fn from_config(c: &AnalysisConfig) -> Self {
        // Capture a prompt only when it diverges from the compiled default,
        // so an unmodified config seeds as `None` ("use default").
        let opt = |cur: &str, def: &str| (cur != def).then(|| cur.to_string());
        Self {
            enabled: c.enabled,
            workers: c.workers as u64,
            endpoint: c.endpoint.clone(),
            model: c.model.clone(),
            request_timeout_secs: c.request_timeout.as_secs(),
            job_timeout_secs: c.job_timeout.as_secs(),
            max_tokens: c.max_tokens,
            max_pixels: c.max_pixels,
            min_slice_height: c.min_slice_height,
            slice_overlap: c.slice_overlap,
            tall_aspect_threshold: c.tall_aspect_threshold,
            max_slices: c.max_slices as u64,
            max_image_bytes: c.max_image_bytes as u64,
            response_format: c.response_format.as_str().to_string(),
            frequency_penalty: c.frequency_penalty,
            temperature: c.temperature,
            system_prompt: opt(&c.system_prompt, SYSTEM_PROMPT_DEFAULT),
            ocr_prompt: opt(&c.ocr_prompt, OCR_PROMPT_DEFAULT),
            grounding_prompt: opt(&c.grounding_prompt, GROUNDING_PROMPT_DEFAULT),
        }
    }

    pub fn to_config(&self, base: &AnalysisConfig) -> Result<AnalysisConfig, FieldErrors> {
        let mut errs = FieldErrors::default();

        if self.workers < 1 {
            errs.push("workers", "must be at least 1");
        } else if self.workers > MAX_WORKERS {
            errs.push("workers", format!("must be at most {MAX_WORKERS}"));
        }
        // The endpoint/model are vision-only knobs — the OCR backend never
        // dials a URL or sends a model id. While vision is dormant
        // (`base.backend == Ocr`), skip the live-worker SSRF gate and the
        // model-required check so an OCR operator can enable the worker
        // without a vision endpoint/model (the OCR settings UI doesn't even
        // expose them). The basic malformed-URL sanity check below stays
        // unconditional. Re-enabling vision restores the full gate via the
        // `base.backend == Vision` predicate.
        let vision_active = base.backend == AnalysisBackend::Vision;
        let trimmed_endpoint = self.endpoint.trim();
        if trimmed_endpoint.is_empty() {
            errs.push("endpoint", "must be a valid absolute URL");
        } else if let Err(e) = reqwest::Url::parse(trimmed_endpoint) {
            // Always reject obviously-malformed URLs (matches prior behaviour).
            let _ = e;
            errs.push("endpoint", "must be a valid absolute URL");
        } else if self.enabled && vision_active {
            // SSRF + API-key exfiltration defence — only enforced when the
            // worker is actually live. Worker attaches an env-managed bearer
            // token to every call; without this check, an admin (or one
            // CSRF-able request) could repoint the endpoint at
            // http://169.254.169.254 (cloud metadata) or http://postgres:5432
            // (an internal service that would receive the bearer header).
            // Docker DNS names (`mangalord-vision`) still pass because
            // they're hostnames, not IP literals.
            //
            // A disabled config is allowed to carry the dev-default
            // `http://localhost:8000/...` placeholder; the worker won't dial
            // it, and toggling `enabled=true` later re-runs this validator.
            if let Err(e) = crate::crawler::safety::ensure_public_target(trimmed_endpoint) {
                errs.push("endpoint", url_safety_message(&e));
            }
        }
        if self.enabled && vision_active && self.model.trim().is_empty() {
            errs.push("model", "required when analysis is enabled");
        }
        if self.max_tokens < 1 {
            errs.push("max_tokens", "must be at least 1");
        } else if self.max_tokens > MAX_ANALYSIS_MAX_TOKENS {
            errs.push("max_tokens", format!("must be at most {MAX_ANALYSIS_MAX_TOKENS}"));
        }
        if self.request_timeout_secs < 1 {
            errs.push("request_timeout_secs", "must be at least 1 second");
        } else if self.request_timeout_secs > MAX_TIMEOUT_SECS {
            errs.push(
                "request_timeout_secs",
                format!("must be at most {MAX_TIMEOUT_SECS} seconds"),
            );
        }
        if self.job_timeout_secs < 1 {
            errs.push("job_timeout_secs", "must be at least 1 second");
        } else if self.job_timeout_secs > MAX_TIMEOUT_SECS {
            errs.push(
                "job_timeout_secs",
                format!("must be at most {MAX_TIMEOUT_SECS} seconds"),
            );
        }
        if self.max_pixels < 1 {
            errs.push("max_pixels", "must be at least 1");
        }
        if self.max_image_bytes < 1 {
            errs.push("max_image_bytes", "must be greater than 0");
        } else if self.max_image_bytes > MAX_ANALYSIS_IMAGE_BYTES {
            errs.push(
                "max_image_bytes",
                format!("must be at most {MAX_ANALYSIS_IMAGE_BYTES} (1 GiB)"),
            );
        }
        if !(0.0..=0.9).contains(&self.slice_overlap) {
            errs.push("slice_overlap", "must be between 0.0 and 0.9");
        }
        if self.tall_aspect_threshold < 1.0 {
            errs.push("tall_aspect_threshold", "must be at least 1.0");
        }
        if self.min_slice_height < 1 {
            errs.push("min_slice_height", "must be at least 1");
        }
        if self.max_slices < 1 {
            errs.push("max_slices", "must be at least 1");
        } else if self.max_slices > MAX_ANALYSIS_SLICES {
            errs.push("max_slices", format!("must be at most {MAX_ANALYSIS_SLICES}"));
        }
        if !(0.0..=MAX_TEMPERATURE).contains(&self.temperature) {
            errs.push("temperature", format!("must be between 0 and {MAX_TEMPERATURE}"));
        }
        if !(FREQUENCY_PENALTY_MIN..=FREQUENCY_PENALTY_MAX).contains(&self.frequency_penalty) {
            errs.push(
                "frequency_penalty",
                format!("must be between {FREQUENCY_PENALTY_MIN} and {FREQUENCY_PENALTY_MAX}"),
            );
        }
        let response_format = match ResponseFormat::parse_strict(&self.response_format) {
            Some(rf) => rf,
            None => {
                errs.push(
                    "response_format",
                    "must be one of: json_schema, json_object, none",
                );
                base.response_format
            }
        };

        if !errs.is_empty() {
            return Err(errs);
        }

        let prompt = |o: &Option<String>, def: &str| {
            o.as_deref()
                .map(str::trim)
                .filter(|s| !s.is_empty())
                .unwrap_or(def)
                .to_string()
        };

        Ok(AnalysisConfig {
            enabled: self.enabled,
            workers: (self.workers as usize).max(1),
            endpoint: self.endpoint.trim().to_string(),
            model: self.model.trim().to_string(),
            request_timeout: Duration::from_secs(self.request_timeout_secs),
            job_timeout: Duration::from_secs(self.job_timeout_secs),
            max_tokens: self.max_tokens,
            max_pixels: self.max_pixels,
            min_slice_height: self.min_slice_height.max(1),
            slice_overlap: self.slice_overlap.clamp(0.0, 0.9),
            tall_aspect_threshold: self.tall_aspect_threshold.max(1.0),
            max_slices: (self.max_slices as usize).max(1),
            max_image_bytes: self.max_image_bytes as usize,
            response_format,
            frequency_penalty: self.frequency_penalty,
            temperature: self.temperature.max(0.0),
            system_prompt: prompt(&self.system_prompt, SYSTEM_PROMPT_DEFAULT),
            ocr_prompt: prompt(&self.ocr_prompt, OCR_PROMPT_DEFAULT),
            grounding_prompt: prompt(&self.grounding_prompt, GROUNDING_PROMPT_DEFAULT),
            // Env-only secret preserved from the base.
            api_key: base.api_key.clone(),
            // Env-only readiness probe URL preserved from the base.
            vision_health_url: base.vision_health_url.clone(),
            // Deploy-time engine selection + ocrs model paths: env-only, not
            // admin-tunable, so carry them through from the base unchanged.
            backend: base.backend,
            ocr_detection_model: base.ocr_detection_model.clone(),
            ocr_recognition_model: base.ocr_recognition_model.clone(),
            // Env-only decompression-bomb decode cap, carried from the base.
            ocr_max_decode_pixels: base.ocr_max_decode_pixels,
        })
    }
}

/// Prompt defaults, returned by the API so the UI can render placeholders and
/// implement "reset to default".
#[derive(Debug, Clone, Serialize)]
pub struct PromptDefaults {
    pub system_prompt: &'static str,
    pub ocr_prompt: &'static str,
    pub grounding_prompt: &'static str,
}

impl PromptDefaults {
    pub fn get() -> Self {
        Self {
            system_prompt: SYSTEM_PROMPT_DEFAULT,
            ocr_prompt: OCR_PROMPT_DEFAULT,
            grounding_prompt: GROUNDING_PROMPT_DEFAULT,
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    // --- crawler round-trip & validation ---

    #[test]
    fn crawler_round_trips_through_dto() {
        let base = CrawlerConfig {
            start_url: Some("https://example.com/".to_string()),
            tz: Tz::Europe__Berlin,
            chapter_workers: 3,
            cookie_domain: Some("example.com".to_string()),
            ..Default::default()
        };
        let dto = CrawlerSettings::from_config(&base);
        let back = dto.to_config(&base).expect("valid");
        assert_eq!(back.tz, Tz::Europe__Berlin);
        assert_eq!(back.chapter_workers, 3);
        assert_eq!(back.start_url.as_deref(), Some("https://example.com/"));
        // cookie_domain is now an editable DTO field, round-tripping like the rest.
        assert_eq!(back.cookie_domain.as_deref(), Some("example.com"));
        assert_eq!(dto.daily_at, "00:00");
    }

    #[test]
    fn crawler_cookie_domain_is_editable() {
        let base = CrawlerConfig::default();
        let dto = CrawlerSettings {
            cookie_domain: Some("new.example.org".to_string()),
            ..CrawlerSettings::from_config(&base)
        };
        assert_eq!(
            dto.to_config(&base).unwrap().cookie_domain.as_deref(),
            Some("new.example.org")
        );
    }

    #[test]
    fn crawler_overlay_preserves_env_only_fields() {
        let base = CrawlerConfig {
            proxy: Some("socks5://127.0.0.1:9050".to_string()),
            tor_control_password: Some("secret".to_string()),
            phpsessid: Some("abc123".to_string()),
            ..Default::default()
        };
        // A DTO that knows nothing about the env-only fields.
        let dto = CrawlerSettings {
            rate_ms: 2000,
            ..CrawlerSettings::from_config(&base)
        };
        let out = dto.to_config(&base).expect("valid");
        assert_eq!(out.rate_ms, 2000);
        assert_eq!(out.proxy.as_deref(), Some("socks5://127.0.0.1:9050"));
        assert_eq!(out.tor_control_password.as_deref(), Some("secret"));
        assert_eq!(out.phpsessid.as_deref(), Some("abc123"));
    }

    #[test]
    fn crawler_rejects_bad_tz_and_time() {
        let base = CrawlerConfig::default();
        let dto = CrawlerSettings {
            tz: "Mars/Phobos".to_string(),
            daily_at: "9am".to_string(),
            ..CrawlerSettings::from_config(&base)
        };
        let errs = dto.to_config(&base).unwrap_err();
        let fields: Vec<_> = errs.errors.iter().map(|e| e.field.as_str()).collect();
        assert!(fields.contains(&"tz"));
        assert!(fields.contains(&"daily_at"));
    }

    #[test]
    fn crawler_allowlist_seeds_start_and_cdn_hosts() {
        let base = CrawlerConfig::default();
        let dto = CrawlerSettings {
            start_url: Some("https://catalog.example.com/".to_string()),
            cdn_host: Some("cdn.example.com".to_string()),
            download_allowlist: vec!["extra.example.net".to_string()],
            allow_any_host: false,
            ..CrawlerSettings::from_config(&base)
        };
        let out = dto.to_config(&base).expect("valid");
        assert!(out.download_allowlist.contains("catalog.example.com"));
        assert!(out.download_allowlist.contains("cdn.example.com"));
        assert!(out.download_allowlist.contains("extra.example.net"));
        assert!(!out.download_allowlist.is_allow_any());
    }

    #[test]
    fn crawler_start_url_rejects_ip_literal_attacks() {
        let base = CrawlerConfig::default();
        for url in [
            "http://169.254.169.254/",
            "http://127.0.0.1/catalog",
            "http://localhost:5432/",
            "http://10.0.0.1/",
        ] {
            let dto = CrawlerSettings {
                start_url: Some(url.to_string()),
                ..CrawlerSettings::from_config(&base)
            };
            let errs = dto.to_config(&base).unwrap_err();
            let fields: Vec<_> = errs.errors.iter().map(|e| e.field.as_str()).collect();
            assert!(fields.contains(&"start_url"), "must reject {url}");
        }
    }

    #[test]
    fn crawler_start_url_accepts_public_targets() {
        let base = CrawlerConfig::default();
        for url in ["https://catalog.example.com/", "http://catalog.example.com/"] {
            let dto = CrawlerSettings {
                start_url: Some(url.to_string()),
                ..CrawlerSettings::from_config(&base)
            };
            assert!(dto.to_config(&base).is_ok(), "must accept {url}");
        }
    }

    #[test]
    fn crawler_allow_any_host_bypasses_list() {
        let base = CrawlerConfig::default();
        let dto = CrawlerSettings {
            allow_any_host: true,
            ..CrawlerSettings::from_config(&base)
        };
        let out = dto.to_config(&base).expect("valid");
        assert!(out.download_allowlist.is_allow_any());
    }

    // --- analysis round-trip, validation & prompt defaults ---

    #[test]
    fn analysis_round_trips_and_omits_default_prompts() {
        let base = AnalysisConfig::default();
        let dto = AnalysisSettings::from_config(&base);
        // Unmodified prompts seed as None ("use default").
        assert!(dto.system_prompt.is_none());
        assert!(dto.ocr_prompt.is_none());
        assert!(dto.grounding_prompt.is_none());
        assert_eq!(dto.response_format, "json_schema");
        let back = dto.to_config(&base).expect("valid");
        assert_eq!(back.system_prompt, SYSTEM_PROMPT_DEFAULT);
        assert_eq!(back.response_format, ResponseFormat::JsonSchema);
    }

    #[test]
    fn analysis_captures_env_prompt_override() {
        let base = AnalysisConfig {
            system_prompt: "custom env prompt".to_string(),
            ..Default::default()
        };
        let dto = AnalysisSettings::from_config(&base);
        assert_eq!(dto.system_prompt.as_deref(), Some("custom env prompt"));
    }

    #[test]
    fn analysis_prompt_override_and_reset() {
        let base = AnalysisConfig::default();
        // Override applied.
        let dto = AnalysisSettings {
            system_prompt: Some("OVERRIDE".to_string()),
            ..AnalysisSettings::from_config(&base)
        };
        assert_eq!(dto.to_config(&base).unwrap().system_prompt, "OVERRIDE");
        // None / blank falls back to the compiled default.
        let dto = AnalysisSettings {
            system_prompt: Some("   ".to_string()),
            ..AnalysisSettings::from_config(&base)
        };
        assert_eq!(dto.to_config(&base).unwrap().system_prompt, SYSTEM_PROMPT_DEFAULT);
    }

    #[test]
    fn analysis_overlay_preserves_api_key() {
        let base = AnalysisConfig {
            api_key: Some("sk-secret".to_string()),
            ..Default::default()
        };
        let dto = AnalysisSettings::from_config(&base);
        assert_eq!(dto.to_config(&base).unwrap().api_key.as_deref(), Some("sk-secret"));
    }

    #[test]
    fn analysis_rejects_out_of_range() {
        let base = AnalysisConfig::default();
        let dto = AnalysisSettings {
            workers: 0,
            slice_overlap: 1.5,
            tall_aspect_threshold: 0.5,
            response_format: "yaml".to_string(),
            endpoint: "not a url".to_string(),
            request_timeout_secs: 0,
            job_timeout_secs: 0,
            max_pixels: 0,
            max_image_bytes: 0,
            ..AnalysisSettings::from_config(&base)
        };
        let errs = dto.to_config(&base).unwrap_err();
        let fields: Vec<_> = errs.errors.iter().map(|e| e.field.as_str()).collect();
        for f in [
            "workers",
            "slice_overlap",
            "tall_aspect_threshold",
            "response_format",
            "endpoint",
            "request_timeout_secs",
            "job_timeout_secs",
            "max_pixels",
            "max_image_bytes",
        ] {
            assert!(fields.contains(&f), "missing error for {f}");
        }
    }

    #[test]
    fn analysis_rejects_over_upper_bounds() {
        // Sanity caps: an absurdly large worker count / buffer / token budget
        // and out-of-range sampling params are all refused so a fat-fingered
        // or CSRF-injected value can't exhaust the host or break the upstream
        // API contract.
        let base = AnalysisConfig::default();
        let dto = AnalysisSettings {
            workers: MAX_WORKERS + 1,
            max_tokens: MAX_ANALYSIS_MAX_TOKENS + 1,
            max_slices: MAX_ANALYSIS_SLICES + 1,
            max_image_bytes: MAX_ANALYSIS_IMAGE_BYTES + 1,
            temperature: MAX_TEMPERATURE + 0.5,
            frequency_penalty: FREQUENCY_PENALTY_MAX + 0.5,
            request_timeout_secs: MAX_TIMEOUT_SECS + 1,
            job_timeout_secs: MAX_TIMEOUT_SECS + 1,
            ..AnalysisSettings::from_config(&base)
        };
        let errs = dto.to_config(&base).unwrap_err();
        let fields: Vec<_> = errs.errors.iter().map(|e| e.field.as_str()).collect();
        for f in [
            "workers",
            "max_tokens",
            "max_slices",
            "max_image_bytes",
            "temperature",
            "frequency_penalty",
            "request_timeout_secs",
            "job_timeout_secs",
        ] {
            assert!(fields.contains(&f), "missing upper-bound error for {f}");
        }
    }

    #[test]
    fn crawler_rejects_over_upper_bounds() {
        // manga_limit ceiling + timeout caps. manga_limit=0 stays "unlimited"
        // and is asserted valid by the round-trip tests above.
        let base = CrawlerConfig::default();
        let dto = CrawlerSettings {
            manga_limit: MAX_MANGA_LIMIT + 1,
            job_timeout_secs: MAX_TIMEOUT_SECS + 1,
            idle_timeout_secs: MAX_TIMEOUT_SECS + 1,
            ..CrawlerSettings::from_config(&base)
        };
        let errs = dto.to_config(&base).unwrap_err();
        let fields: Vec<_> = errs.errors.iter().map(|e| e.field.as_str()).collect();
        for f in ["manga_limit", "job_timeout_secs", "idle_timeout_secs"] {
            assert!(fields.contains(&f), "missing upper-bound error for {f}");
        }
    }

    #[test]
    fn crawler_allows_unlimited_manga_limit() {
        // 0 means "no cap" and must stay valid.
        let base = CrawlerConfig::default();
        let dto = CrawlerSettings {
            manga_limit: 0,
            ..CrawlerSettings::from_config(&base)
        };
        assert!(dto.to_config(&base).is_ok());
    }

    #[test]
    fn analysis_rejects_negative_frequency_penalty_below_range() {
        let base = AnalysisConfig::default();
        let dto = AnalysisSettings {
            frequency_penalty: FREQUENCY_PENALTY_MIN - 0.5,
            ..AnalysisSettings::from_config(&base)
        };
        let errs = dto.to_config(&base).unwrap_err();
        let fields: Vec<_> = errs.errors.iter().map(|e| e.field.as_str()).collect();
        assert!(fields.contains(&"frequency_penalty"));
    }

    #[test]
    fn analysis_accepts_in_range_sampling_params() {
        // A normal config with mid-range sampling values stays valid.
        let base = AnalysisConfig::default();
        let dto = AnalysisSettings {
            temperature: 0.7,
            frequency_penalty: 0.3,
            ..AnalysisSettings::from_config(&base)
        };
        assert!(dto.to_config(&base).is_ok());
    }

    #[test]
    fn crawler_rejects_over_chapter_worker_cap() {
        let base = CrawlerConfig::default();
        let dto = CrawlerSettings {
            chapter_workers: MAX_CHAPTER_WORKERS + 1,
            ..CrawlerSettings::from_config(&base)
        };
        let errs = dto.to_config(&base).unwrap_err();
        let fields: Vec<_> = errs.errors.iter().map(|e| e.field.as_str()).collect();
        assert!(fields.contains(&"chapter_workers"));
    }

    #[test]
    fn analysis_endpoint_rejects_ip_literal_attacks_when_enabled() {
        // The vision worker bearer-attaches an env secret to every call;
        // a hostile/CSRF-able admin must NOT be able to point endpoint at
        // cloud metadata, loopback services, or RFC1918 hosts — when the
        // worker is enabled (toggling enabled=true later re-runs this gate).
        let base = AnalysisConfig {
            backend: AnalysisBackend::Vision,
            ..Default::default()
        };
        for url in [
            "http://169.254.169.254/v1/chat/completions",
            "http://127.0.0.1:5432/",
            "http://localhost:11434/v1/chat/completions",
            "http://10.0.0.5/v1/chat/completions",
        ] {
            let dto = AnalysisSettings {
                enabled: true,
                model: "test-model".to_string(),
                endpoint: url.to_string(),
                ..AnalysisSettings::from_config(&base)
            };
            let errs = dto.to_config(&base).unwrap_err();
            let fields: Vec<_> = errs.errors.iter().map(|e| e.field.as_str()).collect();
            assert!(fields.contains(&"endpoint"), "must reject {url}");
        }
    }

    #[test]
    fn analysis_endpoint_accepts_docker_dns_and_public_hosts_when_enabled() {
        // The documented default — docker DNS name resolving to a private IP
        // at runtime — must still validate, because the bearer recipient
        // identity is the operator-chosen hostname, not the underlying IP.
        let base = AnalysisConfig {
            backend: AnalysisBackend::Vision,
            ..Default::default()
        };
        for url in [
            "http://mangalord-vision:8000/v1/chat/completions",
            "https://api.openai.com/v1/chat/completions",
        ] {
            let dto = AnalysisSettings {
                enabled: true,
                model: "test-model".to_string(),
                endpoint: url.to_string(),
                ..AnalysisSettings::from_config(&base)
            };
            assert!(dto.to_config(&base).is_ok(), "must accept {url}");
        }
    }

    #[test]
    fn analysis_endpoint_skips_safety_check_when_disabled() {
        // Disabled-but-saved bad URL is harmless (worker won't dial it);
        // operators get to keep the dev-localhost placeholder until they
        // toggle the worker on, when the validator re-runs.
        let base = AnalysisConfig::default();
        let dto = AnalysisSettings {
            enabled: false,
            endpoint: "http://localhost:8000/v1/chat/completions".to_string(),
            ..AnalysisSettings::from_config(&base)
        };
        assert!(dto.to_config(&base).is_ok());
    }

    #[test]
    fn analysis_requires_model_only_when_enabled() {
        // Vision base: the model id is required only when the vision worker
        // is actually live (see the OCR carve-out below).
        let base = AnalysisConfig {
            backend: AnalysisBackend::Vision,
            ..Default::default()
        };
        let disabled = AnalysisSettings {
            enabled: false,
            model: "".to_string(),
            ..AnalysisSettings::from_config(&base)
        };
        assert!(disabled.to_config(&base).is_ok());
        let enabled = AnalysisSettings {
            enabled: true,
            model: "".to_string(),
            ..AnalysisSettings::from_config(&base)
        };
        let errs = enabled.to_config(&base).unwrap_err();
        assert!(errs.errors.iter().any(|e| e.field == "model"));
    }

    #[test]
    fn analysis_ocr_backend_enable_skips_vision_endpoint_and_model_validation() {
        // With the OCR backend active the worker never dials the vision
        // endpoint nor sends a model id, so enabling analysis must NOT
        // validate those vision-only fields. The default base carries the
        // dev-localhost endpoint (which the SSRF gate would otherwise reject)
        // and an empty model — under OCR, enabling is still valid. Without
        // this carve-out an OCR operator can't turn the worker on, since the
        // OCR settings UI doesn't expose endpoint/model to fix.
        let base = AnalysisConfig::default(); // backend = Ocr
        let dto = AnalysisSettings {
            enabled: true,
            endpoint: "http://localhost:8000/v1/chat/completions".to_string(),
            model: "".to_string(),
            ..AnalysisSettings::from_config(&base)
        };
        assert!(
            dto.to_config(&base).is_ok(),
            "OCR-enabled config must not fail on vision endpoint/model"
        );
    }

    #[test]
    fn dtos_serialize_to_json_and_back() {
        let c = CrawlerSettings::default();
        let v = serde_json::to_value(&c).unwrap();
        let back: CrawlerSettings = serde_json::from_value(v).unwrap();
        assert_eq!(c, back);

        let a = AnalysisSettings::default();
        let v = serde_json::to_value(&a).unwrap();
        let back: AnalysisSettings = serde_json::from_value(v).unwrap();
        assert_eq!(a, back);
    }
}