feat(admin): runtime-editable crawler & analysis config in the dashboard
Move crawler and analysis configuration out of boot-only env vars and into
a DB-backed, admin-editable surface applied live without a restart.
- New `app_settings` table (migration 0026) holds one JSONB row per group;
env vars seed it on first boot, then the DB is the source of truth.
- `settings.rs` adds serializable Crawler/Analysis DTOs with env-base +
DB-overlay conversion and field-level validation; env-only/secret fields
(browser, proxy, TOR, vision API key, PHPSESSID) are never persisted.
- `GET/PUT /api/v1/admin/settings/{crawler,analysis}` (RequireAdmin,
cookie-only, CSRF-guarded): GET returns the editable DTO + a read-only
env-managed view + prompt defaults; PUT validates (422 + per-field
details), persists + audits in one tx, then gracefully respawns the
affected daemon via a Supervisor.
- AppState swaps the crawler control/resync handles and analysis enable
gate behind shared runtime cells so reloads take effect with no restart;
a boot-time spawn failure is logged rather than aborting startup.
- Make the three vision prompts and sampling temperature configurable
(defaults preserved); cookie_domain is admin-editable too.
- Frontend: tabbed /admin/settings page with grouped fieldsets, prompt
reset-to-default, env-managed read-only panel, save-&-apply confirm, and
per-field validation; typed client + ApiError.details.
- Bump version 0.79.1 -> 0.80.0.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
643
backend/src/settings.rs
Normal file
643
backend/src/settings.rs
Normal file
@@ -0,0 +1,643 @@
|
||||
//! Serializable, admin-editable settings DTOs for the crawler and analysis
|
||||
//! subsystems, and their conversion to/from the runtime [`CrawlerConfig`] /
|
||||
//! [`AnalysisConfig`].
|
||||
//!
|
||||
//! These DTOs are the boundary persisted in the `app_settings` table (one
|
||||
//! JSONB row per group) and exchanged over the admin API. They carry **only
|
||||
//! the operationally-safe, UI-editable fields** as primitive types — the
|
||||
//! runtime configs additionally hold env-only/structural fields (browser
|
||||
//! launch options, proxy, TOR control, the vision API key, the cookie
|
||||
//! domain) that are never persisted here.
|
||||
//!
|
||||
//! Flow: at boot, [`CrawlerSettings::from_config`] /
|
||||
//! [`AnalysisSettings::from_config`] derive the env-seed DTO; it is written to
|
||||
//! the DB only when the row is absent. Thereafter the DB row is the source of
|
||||
//! truth, and [`CrawlerSettings::to_config`] / [`AnalysisSettings::to_config`]
|
||||
//! overlay it onto the env-derived **base** (which carries the env-only
|
||||
//! fields) to produce the effective config the daemons are (re)spawned with.
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use chrono::NaiveTime;
|
||||
use chrono_tz::Tz;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::analysis::prompt::{
|
||||
GROUNDING_PROMPT_DEFAULT, OCR_PROMPT_DEFAULT, SYSTEM_PROMPT_DEFAULT,
|
||||
};
|
||||
use crate::config::{AnalysisConfig, CrawlerConfig, ResponseFormat};
|
||||
use crate::crawler::safety::DownloadAllowlist;
|
||||
|
||||
/// `app_settings.key` for the crawler group.
|
||||
pub const KEY_CRAWLER: &str = "crawler";
|
||||
/// `app_settings.key` for the analysis group.
|
||||
pub const KEY_ANALYSIS: &str = "analysis";
|
||||
|
||||
/// One field-level validation failure, surfaced to the UI per input.
|
||||
#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
|
||||
pub struct FieldError {
|
||||
pub field: String,
|
||||
pub message: String,
|
||||
}
|
||||
|
||||
/// Collected validation failures for a settings payload. Empty == valid.
|
||||
#[derive(Debug, Clone, Serialize, Default, PartialEq, Eq)]
|
||||
pub struct FieldErrors {
|
||||
pub errors: Vec<FieldError>,
|
||||
}
|
||||
|
||||
impl FieldErrors {
|
||||
fn push(&mut self, field: &str, message: impl Into<String>) {
|
||||
self.errors.push(FieldError {
|
||||
field: field.to_string(),
|
||||
message: message.into(),
|
||||
});
|
||||
}
|
||||
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.errors.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Crawler
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Admin-editable crawler-daemon settings. Mirrors the operational `CRAWLER_*`
|
||||
/// env vars; host/infra and session fields stay env-only.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(default)]
|
||||
pub struct CrawlerSettings {
|
||||
pub daemon_enabled: bool,
|
||||
/// Daily metadata-pass time, `"HH:MM"` (24h).
|
||||
pub daily_at: String,
|
||||
/// IANA timezone for the daily schedule.
|
||||
pub tz: String,
|
||||
pub idle_timeout_secs: u64,
|
||||
pub chapter_workers: u64,
|
||||
pub retention_days: u32,
|
||||
pub start_url: Option<String>,
|
||||
pub rate_ms: u64,
|
||||
pub cdn_host: Option<String>,
|
||||
pub cdn_rate_ms: u64,
|
||||
/// Domain the session cookie (PHPSESSID) is scoped to when injecting it
|
||||
/// into the browser / CDN requests. Pairs with `start_url`.
|
||||
pub cookie_domain: Option<String>,
|
||||
pub user_agent: Option<String>,
|
||||
/// Hosts the crawler may download images from (in addition to the
|
||||
/// auto-seeded start-url / CDN hosts). Ignored when `allow_any_host`.
|
||||
pub download_allowlist: Vec<String>,
|
||||
pub allow_any_host: bool,
|
||||
pub max_image_bytes: u64,
|
||||
/// Max manga detail fetches per metadata pass; `0` = unlimited.
|
||||
pub manga_limit: u64,
|
||||
pub job_timeout_secs: u64,
|
||||
pub metadata_max_consecutive_failures: u32,
|
||||
pub browser_restart_threshold: u32,
|
||||
}
|
||||
|
||||
impl Default for CrawlerSettings {
|
||||
fn default() -> Self {
|
||||
Self::from_config(&CrawlerConfig::default())
|
||||
}
|
||||
}
|
||||
|
||||
impl CrawlerSettings {
|
||||
/// Derive the DTO from a runtime config (used to seed the DB from env).
|
||||
pub fn from_config(c: &CrawlerConfig) -> Self {
|
||||
Self {
|
||||
daemon_enabled: c.daemon_enabled,
|
||||
daily_at: c.daily_at.format("%H:%M").to_string(),
|
||||
tz: c.tz.name().to_string(),
|
||||
idle_timeout_secs: c.idle_timeout.as_secs(),
|
||||
chapter_workers: c.chapter_workers as u64,
|
||||
retention_days: c.retention_days,
|
||||
start_url: c.start_url.clone(),
|
||||
rate_ms: c.rate_ms,
|
||||
cdn_host: c.cdn_host.clone(),
|
||||
cdn_rate_ms: c.cdn_rate_ms,
|
||||
cookie_domain: c.cookie_domain.clone(),
|
||||
user_agent: c.user_agent.clone(),
|
||||
download_allowlist: c.download_allowlist.hosts().to_vec(),
|
||||
allow_any_host: c.download_allowlist.is_allow_any(),
|
||||
max_image_bytes: c.max_image_bytes as u64,
|
||||
manga_limit: c.manga_limit as u64,
|
||||
job_timeout_secs: c.job_timeout.as_secs(),
|
||||
metadata_max_consecutive_failures: c.metadata_max_consecutive_failures,
|
||||
browser_restart_threshold: c.browser_restart_threshold,
|
||||
}
|
||||
}
|
||||
|
||||
/// Overlay the DTO onto an env-derived `base` (which carries the env-only
|
||||
/// fields: browser, proxy, TOR, session, cookie domain) to produce the
|
||||
/// effective runtime config. Validates and returns all field errors at
|
||||
/// once on failure.
|
||||
pub fn to_config(&self, base: &CrawlerConfig) -> Result<CrawlerConfig, FieldErrors> {
|
||||
let mut errs = FieldErrors::default();
|
||||
|
||||
let daily_at = match NaiveTime::parse_from_str(&self.daily_at, "%H:%M") {
|
||||
Ok(t) => t,
|
||||
Err(_) => {
|
||||
errs.push("daily_at", "must be HH:MM (24-hour)");
|
||||
base.daily_at
|
||||
}
|
||||
};
|
||||
let tz: Tz = match self.tz.parse() {
|
||||
Ok(t) => t,
|
||||
Err(_) => {
|
||||
errs.push("tz", "must be a valid IANA timezone (e.g. UTC, Europe/Berlin)");
|
||||
base.tz
|
||||
}
|
||||
};
|
||||
if self.chapter_workers < 1 {
|
||||
errs.push("chapter_workers", "must be at least 1");
|
||||
}
|
||||
if let Some(url) = self.start_url.as_deref().map(str::trim).filter(|s| !s.is_empty()) {
|
||||
if reqwest::Url::parse(url).is_err() {
|
||||
errs.push("start_url", "must be a valid absolute URL");
|
||||
}
|
||||
}
|
||||
if self.job_timeout_secs < 1 {
|
||||
errs.push("job_timeout_secs", "must be at least 1 second");
|
||||
}
|
||||
|
||||
if !errs.is_empty() {
|
||||
return Err(errs);
|
||||
}
|
||||
|
||||
let start_url = self
|
||||
.start_url
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|s| !s.is_empty())
|
||||
.map(str::to_string);
|
||||
let cdn_host = self
|
||||
.cdn_host
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|s| !s.is_empty())
|
||||
.map(str::to_string);
|
||||
|
||||
let download_allowlist = build_allowlist(
|
||||
self.allow_any_host,
|
||||
start_url.as_deref(),
|
||||
cdn_host.as_deref(),
|
||||
&self.download_allowlist,
|
||||
);
|
||||
|
||||
Ok(CrawlerConfig {
|
||||
daemon_enabled: self.daemon_enabled,
|
||||
daily_at,
|
||||
tz,
|
||||
idle_timeout: Duration::from_secs(self.idle_timeout_secs),
|
||||
chapter_workers: (self.chapter_workers as usize).max(1),
|
||||
retention_days: self.retention_days,
|
||||
start_url,
|
||||
rate_ms: self.rate_ms,
|
||||
cdn_host,
|
||||
cdn_rate_ms: self.cdn_rate_ms,
|
||||
user_agent: self
|
||||
.user_agent
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|s| !s.is_empty())
|
||||
.map(str::to_string),
|
||||
cookie_domain: self
|
||||
.cookie_domain
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|s| !s.is_empty())
|
||||
.map(str::to_string),
|
||||
download_allowlist,
|
||||
max_image_bytes: self.max_image_bytes as usize,
|
||||
manga_limit: self.manga_limit as usize,
|
||||
job_timeout: Duration::from_secs(self.job_timeout_secs.max(1)),
|
||||
metadata_max_consecutive_failures: self.metadata_max_consecutive_failures,
|
||||
browser_restart_threshold: self.browser_restart_threshold,
|
||||
// Env-only / structural fields preserved from the base.
|
||||
phpsessid: base.phpsessid.clone(),
|
||||
proxy: base.proxy.clone(),
|
||||
tor_control_url: base.tor_control_url.clone(),
|
||||
tor_control_password: base.tor_control_password.clone(),
|
||||
tor_control_cookie_path: base.tor_control_cookie_path.clone(),
|
||||
tor_recircuit_max_attempts: base.tor_recircuit_max_attempts,
|
||||
browser: base.browser.clone(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Rebuild a [`DownloadAllowlist`] from the DTO, always seeding the start-url
|
||||
/// and CDN hosts (mirrors `config::build_download_allowlist`).
|
||||
fn build_allowlist(
|
||||
allow_any: bool,
|
||||
start_url: Option<&str>,
|
||||
cdn_host: Option<&str>,
|
||||
extras: &[String],
|
||||
) -> DownloadAllowlist {
|
||||
if allow_any {
|
||||
return DownloadAllowlist::allow_any();
|
||||
}
|
||||
let mut allow = DownloadAllowlist::new();
|
||||
if let Some(url) = start_url {
|
||||
if let Ok(parsed) = reqwest::Url::parse(url) {
|
||||
if let Some(h) = parsed.host_str() {
|
||||
allow = allow.allow(h);
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(host) = cdn_host {
|
||||
allow = allow.allow(host);
|
||||
}
|
||||
for h in extras {
|
||||
let trimmed = h.trim();
|
||||
if !trimmed.is_empty() {
|
||||
allow = allow.allow(trimmed);
|
||||
}
|
||||
}
|
||||
allow
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Analysis
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Admin-editable analysis-worker settings. The vision `api_key` is env-only
|
||||
/// and never carried here. Prompts are `Option<String>`: `None` means "use
|
||||
/// the compiled default" (the UI's "reset to default").
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||
#[serde(default)]
|
||||
pub struct AnalysisSettings {
|
||||
pub enabled: bool,
|
||||
pub workers: u64,
|
||||
pub endpoint: String,
|
||||
pub model: String,
|
||||
pub request_timeout_secs: u64,
|
||||
pub job_timeout_secs: u64,
|
||||
pub max_tokens: u32,
|
||||
pub max_pixels: u32,
|
||||
pub min_slice_height: u32,
|
||||
pub slice_overlap: f64,
|
||||
pub tall_aspect_threshold: f64,
|
||||
pub max_slices: u64,
|
||||
pub max_image_bytes: u64,
|
||||
/// `"json_schema" | "json_object" | "none"`.
|
||||
pub response_format: String,
|
||||
pub frequency_penalty: f64,
|
||||
pub temperature: f64,
|
||||
pub system_prompt: Option<String>,
|
||||
pub ocr_prompt: Option<String>,
|
||||
pub grounding_prompt: Option<String>,
|
||||
}
|
||||
|
||||
impl Default for AnalysisSettings {
|
||||
fn default() -> Self {
|
||||
Self::from_config(&AnalysisConfig::default())
|
||||
}
|
||||
}
|
||||
|
||||
impl AnalysisSettings {
|
||||
pub fn from_config(c: &AnalysisConfig) -> Self {
|
||||
// Capture a prompt only when it diverges from the compiled default,
|
||||
// so an unmodified config seeds as `None` ("use default").
|
||||
let opt = |cur: &str, def: &str| (cur != def).then(|| cur.to_string());
|
||||
Self {
|
||||
enabled: c.enabled,
|
||||
workers: c.workers as u64,
|
||||
endpoint: c.endpoint.clone(),
|
||||
model: c.model.clone(),
|
||||
request_timeout_secs: c.request_timeout.as_secs(),
|
||||
job_timeout_secs: c.job_timeout.as_secs(),
|
||||
max_tokens: c.max_tokens,
|
||||
max_pixels: c.max_pixels,
|
||||
min_slice_height: c.min_slice_height,
|
||||
slice_overlap: c.slice_overlap,
|
||||
tall_aspect_threshold: c.tall_aspect_threshold,
|
||||
max_slices: c.max_slices as u64,
|
||||
max_image_bytes: c.max_image_bytes as u64,
|
||||
response_format: c.response_format.as_str().to_string(),
|
||||
frequency_penalty: c.frequency_penalty,
|
||||
temperature: c.temperature,
|
||||
system_prompt: opt(&c.system_prompt, SYSTEM_PROMPT_DEFAULT),
|
||||
ocr_prompt: opt(&c.ocr_prompt, OCR_PROMPT_DEFAULT),
|
||||
grounding_prompt: opt(&c.grounding_prompt, GROUNDING_PROMPT_DEFAULT),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn to_config(&self, base: &AnalysisConfig) -> Result<AnalysisConfig, FieldErrors> {
|
||||
let mut errs = FieldErrors::default();
|
||||
|
||||
if self.workers < 1 {
|
||||
errs.push("workers", "must be at least 1");
|
||||
}
|
||||
if self.endpoint.trim().is_empty() || reqwest::Url::parse(self.endpoint.trim()).is_err() {
|
||||
errs.push("endpoint", "must be a valid absolute URL");
|
||||
}
|
||||
if self.enabled && self.model.trim().is_empty() {
|
||||
errs.push("model", "required when analysis is enabled");
|
||||
}
|
||||
if self.max_tokens < 1 {
|
||||
errs.push("max_tokens", "must be at least 1");
|
||||
}
|
||||
if self.request_timeout_secs < 1 {
|
||||
errs.push("request_timeout_secs", "must be at least 1 second");
|
||||
}
|
||||
if self.job_timeout_secs < 1 {
|
||||
errs.push("job_timeout_secs", "must be at least 1 second");
|
||||
}
|
||||
if self.max_pixels < 1 {
|
||||
errs.push("max_pixels", "must be at least 1");
|
||||
}
|
||||
if self.max_image_bytes < 1 {
|
||||
errs.push("max_image_bytes", "must be greater than 0");
|
||||
}
|
||||
if !(0.0..=0.9).contains(&self.slice_overlap) {
|
||||
errs.push("slice_overlap", "must be between 0.0 and 0.9");
|
||||
}
|
||||
if self.tall_aspect_threshold < 1.0 {
|
||||
errs.push("tall_aspect_threshold", "must be at least 1.0");
|
||||
}
|
||||
if self.min_slice_height < 1 {
|
||||
errs.push("min_slice_height", "must be at least 1");
|
||||
}
|
||||
if self.max_slices < 1 {
|
||||
errs.push("max_slices", "must be at least 1");
|
||||
}
|
||||
if self.temperature < 0.0 {
|
||||
errs.push("temperature", "must be 0 or greater");
|
||||
}
|
||||
let response_format = match ResponseFormat::parse_strict(&self.response_format) {
|
||||
Some(rf) => rf,
|
||||
None => {
|
||||
errs.push(
|
||||
"response_format",
|
||||
"must be one of: json_schema, json_object, none",
|
||||
);
|
||||
base.response_format
|
||||
}
|
||||
};
|
||||
|
||||
if !errs.is_empty() {
|
||||
return Err(errs);
|
||||
}
|
||||
|
||||
let prompt = |o: &Option<String>, def: &str| {
|
||||
o.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|s| !s.is_empty())
|
||||
.unwrap_or(def)
|
||||
.to_string()
|
||||
};
|
||||
|
||||
Ok(AnalysisConfig {
|
||||
enabled: self.enabled,
|
||||
workers: (self.workers as usize).max(1),
|
||||
endpoint: self.endpoint.trim().to_string(),
|
||||
model: self.model.trim().to_string(),
|
||||
request_timeout: Duration::from_secs(self.request_timeout_secs),
|
||||
job_timeout: Duration::from_secs(self.job_timeout_secs),
|
||||
max_tokens: self.max_tokens,
|
||||
max_pixels: self.max_pixels,
|
||||
min_slice_height: self.min_slice_height.max(1),
|
||||
slice_overlap: self.slice_overlap.clamp(0.0, 0.9),
|
||||
tall_aspect_threshold: self.tall_aspect_threshold.max(1.0),
|
||||
max_slices: (self.max_slices as usize).max(1),
|
||||
max_image_bytes: self.max_image_bytes as usize,
|
||||
response_format,
|
||||
frequency_penalty: self.frequency_penalty,
|
||||
temperature: self.temperature.max(0.0),
|
||||
system_prompt: prompt(&self.system_prompt, SYSTEM_PROMPT_DEFAULT),
|
||||
ocr_prompt: prompt(&self.ocr_prompt, OCR_PROMPT_DEFAULT),
|
||||
grounding_prompt: prompt(&self.grounding_prompt, GROUNDING_PROMPT_DEFAULT),
|
||||
// Env-only secret preserved from the base.
|
||||
api_key: base.api_key.clone(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Prompt defaults, returned by the API so the UI can render placeholders and
|
||||
/// implement "reset to default".
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct PromptDefaults {
|
||||
pub system_prompt: &'static str,
|
||||
pub ocr_prompt: &'static str,
|
||||
pub grounding_prompt: &'static str,
|
||||
}
|
||||
|
||||
impl PromptDefaults {
|
||||
pub fn get() -> Self {
|
||||
Self {
|
||||
system_prompt: SYSTEM_PROMPT_DEFAULT,
|
||||
ocr_prompt: OCR_PROMPT_DEFAULT,
|
||||
grounding_prompt: GROUNDING_PROMPT_DEFAULT,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
// --- crawler round-trip & validation ---
|
||||
|
||||
#[test]
|
||||
fn crawler_round_trips_through_dto() {
|
||||
let mut base = CrawlerConfig::default();
|
||||
base.start_url = Some("https://example.com/".to_string());
|
||||
base.tz = Tz::Europe__Berlin;
|
||||
base.chapter_workers = 3;
|
||||
base.cookie_domain = Some("example.com".to_string());
|
||||
let dto = CrawlerSettings::from_config(&base);
|
||||
let back = dto.to_config(&base).expect("valid");
|
||||
assert_eq!(back.tz, Tz::Europe__Berlin);
|
||||
assert_eq!(back.chapter_workers, 3);
|
||||
assert_eq!(back.start_url.as_deref(), Some("https://example.com/"));
|
||||
// cookie_domain is now an editable DTO field, round-tripping like the rest.
|
||||
assert_eq!(back.cookie_domain.as_deref(), Some("example.com"));
|
||||
assert_eq!(dto.daily_at, "00:00");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn crawler_cookie_domain_is_editable() {
|
||||
let base = CrawlerConfig::default();
|
||||
let dto = CrawlerSettings {
|
||||
cookie_domain: Some("new.example.org".to_string()),
|
||||
..CrawlerSettings::from_config(&base)
|
||||
};
|
||||
assert_eq!(
|
||||
dto.to_config(&base).unwrap().cookie_domain.as_deref(),
|
||||
Some("new.example.org")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn crawler_overlay_preserves_env_only_fields() {
|
||||
let mut base = CrawlerConfig::default();
|
||||
base.proxy = Some("socks5://127.0.0.1:9050".to_string());
|
||||
base.tor_control_password = Some("secret".to_string());
|
||||
base.phpsessid = Some("abc123".to_string());
|
||||
// A DTO that knows nothing about the env-only fields.
|
||||
let dto = CrawlerSettings {
|
||||
rate_ms: 2000,
|
||||
..CrawlerSettings::from_config(&base)
|
||||
};
|
||||
let out = dto.to_config(&base).expect("valid");
|
||||
assert_eq!(out.rate_ms, 2000);
|
||||
assert_eq!(out.proxy.as_deref(), Some("socks5://127.0.0.1:9050"));
|
||||
assert_eq!(out.tor_control_password.as_deref(), Some("secret"));
|
||||
assert_eq!(out.phpsessid.as_deref(), Some("abc123"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn crawler_rejects_bad_tz_and_time() {
|
||||
let base = CrawlerConfig::default();
|
||||
let dto = CrawlerSettings {
|
||||
tz: "Mars/Phobos".to_string(),
|
||||
daily_at: "9am".to_string(),
|
||||
..CrawlerSettings::from_config(&base)
|
||||
};
|
||||
let errs = dto.to_config(&base).unwrap_err();
|
||||
let fields: Vec<_> = errs.errors.iter().map(|e| e.field.as_str()).collect();
|
||||
assert!(fields.contains(&"tz"));
|
||||
assert!(fields.contains(&"daily_at"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn crawler_allowlist_seeds_start_and_cdn_hosts() {
|
||||
let base = CrawlerConfig::default();
|
||||
let dto = CrawlerSettings {
|
||||
start_url: Some("https://catalog.example.com/".to_string()),
|
||||
cdn_host: Some("cdn.example.com".to_string()),
|
||||
download_allowlist: vec!["extra.example.net".to_string()],
|
||||
allow_any_host: false,
|
||||
..CrawlerSettings::from_config(&base)
|
||||
};
|
||||
let out = dto.to_config(&base).expect("valid");
|
||||
assert!(out.download_allowlist.contains("catalog.example.com"));
|
||||
assert!(out.download_allowlist.contains("cdn.example.com"));
|
||||
assert!(out.download_allowlist.contains("extra.example.net"));
|
||||
assert!(!out.download_allowlist.is_allow_any());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn crawler_allow_any_host_bypasses_list() {
|
||||
let base = CrawlerConfig::default();
|
||||
let dto = CrawlerSettings {
|
||||
allow_any_host: true,
|
||||
..CrawlerSettings::from_config(&base)
|
||||
};
|
||||
let out = dto.to_config(&base).expect("valid");
|
||||
assert!(out.download_allowlist.is_allow_any());
|
||||
}
|
||||
|
||||
// --- analysis round-trip, validation & prompt defaults ---
|
||||
|
||||
#[test]
|
||||
fn analysis_round_trips_and_omits_default_prompts() {
|
||||
let base = AnalysisConfig::default();
|
||||
let dto = AnalysisSettings::from_config(&base);
|
||||
// Unmodified prompts seed as None ("use default").
|
||||
assert!(dto.system_prompt.is_none());
|
||||
assert!(dto.ocr_prompt.is_none());
|
||||
assert!(dto.grounding_prompt.is_none());
|
||||
assert_eq!(dto.response_format, "json_schema");
|
||||
let back = dto.to_config(&base).expect("valid");
|
||||
assert_eq!(back.system_prompt, SYSTEM_PROMPT_DEFAULT);
|
||||
assert_eq!(back.response_format, ResponseFormat::JsonSchema);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn analysis_captures_env_prompt_override() {
|
||||
let mut base = AnalysisConfig::default();
|
||||
base.system_prompt = "custom env prompt".to_string();
|
||||
let dto = AnalysisSettings::from_config(&base);
|
||||
assert_eq!(dto.system_prompt.as_deref(), Some("custom env prompt"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn analysis_prompt_override_and_reset() {
|
||||
let base = AnalysisConfig::default();
|
||||
// Override applied.
|
||||
let dto = AnalysisSettings {
|
||||
system_prompt: Some("OVERRIDE".to_string()),
|
||||
..AnalysisSettings::from_config(&base)
|
||||
};
|
||||
assert_eq!(dto.to_config(&base).unwrap().system_prompt, "OVERRIDE");
|
||||
// None / blank falls back to the compiled default.
|
||||
let dto = AnalysisSettings {
|
||||
system_prompt: Some(" ".to_string()),
|
||||
..AnalysisSettings::from_config(&base)
|
||||
};
|
||||
assert_eq!(dto.to_config(&base).unwrap().system_prompt, SYSTEM_PROMPT_DEFAULT);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn analysis_overlay_preserves_api_key() {
|
||||
let mut base = AnalysisConfig::default();
|
||||
base.api_key = Some("sk-secret".to_string());
|
||||
let dto = AnalysisSettings::from_config(&base);
|
||||
assert_eq!(dto.to_config(&base).unwrap().api_key.as_deref(), Some("sk-secret"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn analysis_rejects_out_of_range() {
|
||||
let base = AnalysisConfig::default();
|
||||
let dto = AnalysisSettings {
|
||||
workers: 0,
|
||||
slice_overlap: 1.5,
|
||||
tall_aspect_threshold: 0.5,
|
||||
response_format: "yaml".to_string(),
|
||||
endpoint: "not a url".to_string(),
|
||||
request_timeout_secs: 0,
|
||||
job_timeout_secs: 0,
|
||||
max_pixels: 0,
|
||||
max_image_bytes: 0,
|
||||
..AnalysisSettings::from_config(&base)
|
||||
};
|
||||
let errs = dto.to_config(&base).unwrap_err();
|
||||
let fields: Vec<_> = errs.errors.iter().map(|e| e.field.as_str()).collect();
|
||||
for f in [
|
||||
"workers",
|
||||
"slice_overlap",
|
||||
"tall_aspect_threshold",
|
||||
"response_format",
|
||||
"endpoint",
|
||||
"request_timeout_secs",
|
||||
"job_timeout_secs",
|
||||
"max_pixels",
|
||||
"max_image_bytes",
|
||||
] {
|
||||
assert!(fields.contains(&f), "missing error for {f}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn analysis_requires_model_only_when_enabled() {
|
||||
let base = AnalysisConfig::default();
|
||||
let disabled = AnalysisSettings {
|
||||
enabled: false,
|
||||
model: "".to_string(),
|
||||
..AnalysisSettings::from_config(&base)
|
||||
};
|
||||
assert!(disabled.to_config(&base).is_ok());
|
||||
let enabled = AnalysisSettings {
|
||||
enabled: true,
|
||||
model: "".to_string(),
|
||||
..AnalysisSettings::from_config(&base)
|
||||
};
|
||||
let errs = enabled.to_config(&base).unwrap_err();
|
||||
assert!(errs.errors.iter().any(|e| e.field == "model"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dtos_serialize_to_json_and_back() {
|
||||
let c = CrawlerSettings::default();
|
||||
let v = serde_json::to_value(&c).unwrap();
|
||||
let back: CrawlerSettings = serde_json::from_value(v).unwrap();
|
||||
assert_eq!(c, back);
|
||||
|
||||
let a = AnalysisSettings::default();
|
||||
let v = serde_json::to_value(&a).unwrap();
|
||||
let back: AnalysisSettings = serde_json::from_value(v).unwrap();
|
||||
assert_eq!(a, back);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user