diff --git a/.env.example b/.env.example index 1fa1ecc..7e1069a 100644 --- a/.env.example +++ b/.env.example @@ -167,3 +167,25 @@ BACKEND_URL=http://backend:8080 # 25 Mbps; raise for users on slower upstream links or lower if a # tighter front proxy already bounds the request lifetime. BACKEND_PROXY_TIMEOUT_MS=300000 + +# ----- Runtime settings (crawler + analysis) ----- +# The crawler and analysis subsystems are configured at runtime from the +# `app_settings` table and edited live in the admin dashboard +# (Admin → Settings) — no restart needed. The CRAWLER_* / ANALYSIS_* env +# vars below act as the BOOT SEED: on first boot (when the row is absent) +# the env values populate the DB; thereafter the DB is the source of +# truth and env changes are ignored for those fields. +# +# Host/infra and secret fields stay env-ONLY (never persisted, shown +# read-only in the dashboard): the chromium binary/dir, browser mode/args, +# CRAWLER_PROXY, all CRAWLER_TOR_CONTROL_*, the cookie domain, and the +# analysis ANALYSIS_API_KEY. The important site levers (PRIVATE_MODE, +# ALLOW_SELF_REGISTER) also remain env-only by design. +# +# New analysis knobs (all optional, all seed defaults): +# ANALYSIS_TEMPERATURE Sampling temperature. Default 0 (deterministic). +# ANALYSIS_SYSTEM_PROMPT Override the single-call vision system prompt. +# ANALYSIS_OCR_PROMPT Override the tall-page OCR (pass A) prompt. +# ANALYSIS_GROUNDING_PROMPT Override the tags/scene/safety (pass B) prompt. +# Leave the prompt vars unset to use the built-in defaults (also editable, +# with a per-prompt "reset to default", in the dashboard). diff --git a/.gitignore b/.gitignore index 977232d..a22c07f 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,12 @@ .env .env.local .env.*.local +.env.dev + +# Node install / Playwright output when run from the repo root (the +# canonical copies live under /frontend, already ignored above). +/node_modules +/test-results # Claude Code (personal overrides only; .claude/settings.json is committed) .claude/settings.local.json diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 9342d3d..e50b9da 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -1517,7 +1517,7 @@ checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" [[package]] name = "mangalord" -version = "0.79.1" +version = "0.80.0" dependencies = [ "anyhow", "argon2", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 8b1bca4..25302f9 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mangalord" -version = "0.79.1" +version = "0.80.0" edition = "2021" default-run = "mangalord" diff --git a/backend/migrations/0026_app_settings.sql b/backend/migrations/0026_app_settings.sql new file mode 100644 index 0000000..2f99a26 --- /dev/null +++ b/backend/migrations/0026_app_settings.sql @@ -0,0 +1,15 @@ +-- Runtime-editable application settings, keyed by subsystem group. +-- +-- Mirrors the small key-value `crawler_state` table (0015): one row per +-- group ('crawler' | 'analysis'), the `value` JSONB holding a serialized +-- settings DTO. Env vars seed a row on first boot (when absent); after that +-- the DB row is the source of truth and the admin dashboard edits it live. +-- +-- Only operationally-safe fields are stored here — host/infra and secrets +-- (chromium paths, proxy, TOR control, vision API key, cookie domain) stay +-- env-only and are never persisted. +CREATE TABLE app_settings ( + key text PRIMARY KEY, + value jsonb NOT NULL, + updated_at timestamptz NOT NULL DEFAULT now() +); diff --git a/backend/src/analysis/prompt.rs b/backend/src/analysis/prompt.rs index 5e0941e..6c80095 100644 --- a/backend/src/analysis/prompt.rs +++ b/backend/src/analysis/prompt.rs @@ -36,10 +36,12 @@ pub const MAX_SCENE_CHARS: usize = 1000; pub const MAX_OCR_PER_CALL: usize = 50; pub const MAX_TAGS_PER_CALL: usize = 12; -/// The system prompt. Self-contained: it carries the exact JSON schema so -/// the same string drives both the model and (implicitly) the -/// [`crate::domain::page_analysis::VisionAnalysis`] parser. -pub const SYSTEM_PROMPT: &str = concat!( +/// The default system prompt. Self-contained: it carries the exact JSON +/// schema so the same string drives both the model and (implicitly) the +/// [`crate::domain::page_analysis::VisionAnalysis`] parser. An admin can +/// override it at runtime (see [`crate::config::AnalysisConfig::system_prompt`]); +/// this const is the fallback and the "reset to default" target. +pub const SYSTEM_PROMPT_DEFAULT: &str = concat!( "You are a manga/comic page analyzer. Look at the single page image and ", "return ONLY one minified JSON object — no prose, no markdown fences.\n", "Schema:\n", @@ -64,8 +66,9 @@ pub const SYSTEM_PROMPT: &str = concat!( // --- Two-pass prompts (long-page slicing) ----------------------------------- /// Pass A: OCR-only over a single vertical slice of a tall page. Kept -/// narrow so each slice call is fast and never truncates. -pub const OCR_PROMPT: &str = concat!( +/// narrow so each slice call is fast and never truncates. Runtime-overridable +/// default — see [`crate::config::AnalysisConfig::ocr_prompt`]. +pub const OCR_PROMPT_DEFAULT: &str = concat!( "You are an OCR engine for manga/comic pages. The image is one vertical ", "slice of a larger page. Transcribe EVERY visible text element verbatim ", "into ocr_results, each with its kind ", @@ -80,7 +83,9 @@ pub const OCR_PROMPT: &str = concat!( /// Pass B: tags + scene + safety for the whole page, grounded in the merged /// OCR text (supplied as a separate user message part by the client). -pub const GROUNDING_PROMPT: &str = concat!( +/// Runtime-overridable default — see +/// [`crate::config::AnalysisConfig::grounding_prompt`]. +pub const GROUNDING_PROMPT_DEFAULT: &str = concat!( "You are a manga/comic page analyzer. You are given the page image ", "(possibly downscaled) AND the OCR text already extracted from it. Using ", "both, return ONLY one minified JSON object with: tagging_results (5-10 ", @@ -230,7 +235,7 @@ mod tests { #[test] fn system_prompt_mentions_the_schema_keys() { for key in ["ocr_results", "tagging_results", "scene_description", "safety_flag"] { - assert!(SYSTEM_PROMPT.contains(key), "prompt missing {key}"); + assert!(SYSTEM_PROMPT_DEFAULT.contains(key), "prompt missing {key}"); } } diff --git a/backend/src/analysis/vision.rs b/backend/src/analysis/vision.rs index f2dbc32..cf2dd87 100644 --- a/backend/src/analysis/vision.rs +++ b/backend/src/analysis/vision.rs @@ -37,6 +37,13 @@ pub struct VisionClient { max_tokens: u32, response_format: ResponseFormat, frequency_penalty: f64, + temperature: f64, + /// System prompt for the single-call (normal-aspect) path. + system_prompt: String, + /// Pass-A OCR-only prompt (tall-page slices). + ocr_prompt: String, + /// Pass-B grounding prompt (tags/scene/safety). + grounding_prompt: String, slice: SliceParams, } @@ -60,6 +67,10 @@ impl VisionClient { max_tokens: cfg.max_tokens, response_format: cfg.response_format, frequency_penalty: cfg.frequency_penalty, + temperature: cfg.temperature, + system_prompt: cfg.system_prompt.clone(), + ocr_prompt: cfg.ocr_prompt.clone(), + grounding_prompt: cfg.grounding_prompt.clone(), slice: SliceParams { max_pixels: cfg.max_pixels, min_slice_height: cfg.min_slice_height, @@ -83,6 +94,8 @@ impl VisionClient { &url, self.response_format, self.frequency_penalty, + self.temperature, + &self.system_prompt, ); return parse_chat_completion(&self.post_chat(body).await?); }; @@ -97,6 +110,8 @@ impl VisionClient { &data_url(&jpeg), self.response_format, self.frequency_penalty, + self.temperature, + &self.system_prompt, ); parse_chat_completion(&self.post_chat(body).await?) } @@ -124,6 +139,8 @@ impl VisionClient { self.max_tokens, self.response_format, self.frequency_penalty, + self.temperature, + &self.ocr_prompt, &data_url(&jpeg), ); let parsed = parse_chat_completion(&self.post_chat(body).await?)?; @@ -148,6 +165,8 @@ impl VisionClient { self.max_tokens, self.response_format, self.frequency_penalty, + self.temperature, + &self.grounding_prompt, &data_url(&whole), &ocr_text, ); @@ -451,19 +470,23 @@ fn similar(a: &str, b: &str) -> bool { /// Combined single-call body (OCR + tags + scene + safety). Public + this /// exact signature so the existing tests keep working. +#[allow(clippy::too_many_arguments)] pub fn build_request_body( model: &str, max_tokens: u32, data_url: &str, response_format: ResponseFormat, frequency_penalty: f64, + temperature: f64, + system_prompt: &str, ) -> serde_json::Value { build_body( model, max_tokens, response_format, frequency_penalty, - prompt::SYSTEM_PROMPT, + temperature, + system_prompt, "page_analysis", prompt::output_json_schema(), data_url, @@ -472,11 +495,14 @@ pub fn build_request_body( } /// Pass-A OCR-only body. +#[allow(clippy::too_many_arguments)] fn build_ocr_body( model: &str, max_tokens: u32, response_format: ResponseFormat, frequency_penalty: f64, + temperature: f64, + ocr_prompt: &str, data_url: &str, ) -> serde_json::Value { build_body( @@ -484,7 +510,8 @@ fn build_ocr_body( max_tokens, response_format, frequency_penalty, - prompt::OCR_PROMPT, + temperature, + ocr_prompt, "page_ocr", prompt::ocr_json_schema(), data_url, @@ -493,11 +520,14 @@ fn build_ocr_body( } /// Pass-B grounding body — whole image + the merged OCR text as context. +#[allow(clippy::too_many_arguments)] fn build_grounding_body( model: &str, max_tokens: u32, response_format: ResponseFormat, frequency_penalty: f64, + temperature: f64, + grounding_prompt: &str, data_url: &str, ocr_text: &str, ) -> serde_json::Value { @@ -507,7 +537,8 @@ fn build_grounding_body( max_tokens, response_format, frequency_penalty, - prompt::GROUNDING_PROMPT, + temperature, + grounding_prompt, "page_grounding", prompt::grounding_json_schema(), data_url, @@ -524,6 +555,7 @@ fn build_body( max_tokens: u32, response_format: ResponseFormat, frequency_penalty: f64, + temperature: f64, system_prompt: &str, schema_name: &str, schema: serde_json::Value, @@ -537,7 +569,7 @@ fn build_body( } let mut body = json!({ "model": model, - "temperature": 0, + "temperature": temperature, "max_tokens": max_tokens, "messages": [ { "role": "system", "content": system_prompt }, @@ -740,8 +772,15 @@ mod tests { #[test] fn combined_body_uses_full_schema_and_image() { - let body = - build_request_body("m", 100, "data:image/png;base64,AA", ResponseFormat::JsonSchema, 0.3); + let body = build_request_body( + "m", + 100, + "data:image/png;base64,AA", + ResponseFormat::JsonSchema, + 0.3, + 0.0, + prompt::SYSTEM_PROMPT_DEFAULT, + ); assert_eq!(body["messages"][0]["role"], "system"); assert_eq!( body["messages"][1]["content"][0]["image_url"]["url"], @@ -752,9 +791,25 @@ mod tests { assert!(req.as_array().unwrap().iter().any(|v| v == "safety_flag")); } + #[test] + fn body_carries_the_configured_prompt_and_temperature() { + let body = build_request_body( + "m", + 100, + "d", + ResponseFormat::None, + 0.0, + 0.7, + "CUSTOM PROMPT", + ); + assert_eq!(body["messages"][0]["content"], "CUSTOM PROMPT"); + assert_eq!(body["temperature"], 0.7); + } + #[test] fn ocr_body_carries_only_the_ocr_schema_and_caps() { - let body = build_ocr_body("m", 100, ResponseFormat::JsonSchema, 0.3, "d"); + let body = + build_ocr_body("m", 100, ResponseFormat::JsonSchema, 0.3, 0.0, prompt::OCR_PROMPT_DEFAULT, "d"); let schema = &body["response_format"]["json_schema"]["schema"]; assert_eq!(body["response_format"]["json_schema"]["name"], "page_ocr"); assert_eq!(schema["required"], json!(["ocr_results"])); @@ -769,8 +824,16 @@ mod tests { #[test] fn grounding_body_includes_ocr_context_and_grounding_schema() { - let body = - build_grounding_body("m", 100, ResponseFormat::JsonSchema, 0.3, "d", "[speech] Hi"); + let body = build_grounding_body( + "m", + 100, + ResponseFormat::JsonSchema, + 0.3, + 0.0, + prompt::GROUNDING_PROMPT_DEFAULT, + "d", + "[speech] Hi", + ); assert_eq!(body["response_format"]["json_schema"]["name"], "page_grounding"); let parts = body["messages"][1]["content"].as_array().unwrap(); assert_eq!(parts.len(), 2, "image + text parts"); @@ -781,15 +844,25 @@ mod tests { #[test] fn response_format_none_omits_the_field() { - let body = build_request_body("m", 100, "d", ResponseFormat::None, 0.3); + let body = build_request_body( + "m", + 100, + "d", + ResponseFormat::None, + 0.3, + 0.0, + prompt::SYSTEM_PROMPT_DEFAULT, + ); assert!(body.get("response_format").is_none()); } #[test] fn frequency_penalty_included_only_when_nonzero() { - let on = build_ocr_body("m", 100, ResponseFormat::None, 0.4, "d"); + let on = + build_ocr_body("m", 100, ResponseFormat::None, 0.4, 0.0, prompt::OCR_PROMPT_DEFAULT, "d"); assert_eq!(on["frequency_penalty"], 0.4); - let off = build_ocr_body("m", 100, ResponseFormat::None, 0.0, "d"); + let off = + build_ocr_body("m", 100, ResponseFormat::None, 0.0, 0.0, prompt::OCR_PROMPT_DEFAULT, "d"); assert!(off.get("frequency_penalty").is_none()); } diff --git a/backend/src/api/admin/analysis.rs b/backend/src/api/admin/analysis.rs index 39ce75e..8ad472f 100644 --- a/backend/src/api/admin/analysis.rs +++ b/backend/src/api/admin/analysis.rs @@ -188,7 +188,7 @@ pub struct AnalyzePageResponse { /// Reject the request with 503 when the analysis worker is disabled, so /// an operator doesn't pile up jobs that nothing will ever drain. fn ensure_enabled(state: &AppState) -> AppResult<()> { - if state.analysis_enabled { + if state.analysis_enabled() { Ok(()) } else { Err(AppError::ServiceUnavailable( diff --git a/backend/src/api/admin/crawler/mod.rs b/backend/src/api/admin/crawler/mod.rs index 47dd037..08ffba9 100644 --- a/backend/src/api/admin/crawler/mod.rs +++ b/backend/src/api/admin/crawler/mod.rs @@ -37,13 +37,14 @@ pub(super) fn default_limit() -> i64 { } /// Shared 503 helper: the daemon-only control + observability endpoints -/// gate on `AppState.crawler == Some(_)`. Returns the same code and body -/// across every caller so the frontend can rely on `service_unavailable` -/// rather than each handler returning its own variant. +/// gate on a running crawler daemon. Returns the same code and body across +/// every caller so the frontend can rely on `service_unavailable` rather +/// than each handler returning its own variant. Yields an owned `Arc` (the +/// control handle is swapped on a config reload). pub(super) fn require_crawler( state: &AppState, -) -> Result<&std::sync::Arc, AppError> { - state.crawler.as_ref().ok_or_else(|| { +) -> Result, AppError> { + state.crawler().ok_or_else(|| { AppError::ServiceUnavailable("crawler daemon is disabled".into()) }) } diff --git a/backend/src/api/admin/crawler/status.rs b/backend/src/api/admin/crawler/status.rs index 8fe1255..5d1e720 100644 --- a/backend/src/api/admin/crawler/status.rs +++ b/backend/src/api/admin/crawler/status.rs @@ -140,7 +140,7 @@ async fn compose_status( dead, }; - Ok(match state.crawler.as_ref() { + Ok(match state.crawler().as_ref() { None => CrawlerStatusResponse { daemon: "disabled", phase: None, @@ -200,7 +200,7 @@ async fn stream_status( ) -> Sse>> { // Subscribe before the first emit so no change between the initial // snapshot and the first await is lost. - let rx = state.crawler.as_ref().map(|c| c.status.subscribe()); + let rx = state.crawler().as_ref().map(|c| c.status.subscribe()); let memo = QueueCountsMemo::new(); let stream = futures_util::stream::unfold( diff --git a/backend/src/api/admin/mod.rs b/backend/src/api/admin/mod.rs index 3cdb271..83bd34f 100644 --- a/backend/src/api/admin/mod.rs +++ b/backend/src/api/admin/mod.rs @@ -8,6 +8,7 @@ pub mod analysis; pub mod crawler; pub mod mangas; pub mod resync; +pub mod settings; pub mod system; pub mod users; @@ -23,4 +24,5 @@ pub fn routes() -> Router { .merge(system::routes()) .merge(crawler::routes()) .merge(analysis::routes()) + .merge(settings::routes()) } diff --git a/backend/src/api/admin/resync.rs b/backend/src/api/admin/resync.rs index c3a4c6c..a3145c9 100644 --- a/backend/src/api/admin/resync.rs +++ b/backend/src/api/admin/resync.rs @@ -58,8 +58,7 @@ async fn resync_manga( return Err(AppError::NotFound); } let resync = state - .resync - .as_ref() + .resync() .ok_or_else(|| AppError::ServiceUnavailable( "crawler daemon is disabled; force resync unavailable".into(), ))?; @@ -96,8 +95,7 @@ async fn resync_chapter( Path(chapter_id): Path, ) -> AppResult> { let resync = state - .resync - .as_ref() + .resync() .ok_or_else(|| AppError::ServiceUnavailable( "crawler daemon is disabled; force resync unavailable".into(), ))?; diff --git a/backend/src/api/admin/settings.rs b/backend/src/api/admin/settings.rs new file mode 100644 index 0000000..b2d1f6f --- /dev/null +++ b/backend/src/api/admin/settings.rs @@ -0,0 +1,239 @@ +//! Admin endpoints for runtime-editable crawler / analysis configuration. +//! +//! `GET` returns the current editable settings DTO plus a read-only view of +//! the env-managed (host/infra + secret) fields, so the dashboard can show +//! the full picture while only letting the operator edit the safe knobs. +//! +//! `PUT` validates the incoming DTO against the env-derived base (re-parsing +//! timezone/time, rebuilding the download allowlist), persists a normalized +//! DTO and an audit row in one transaction, then asks the [`DaemonReloader`] +//! to gracefully respawn the affected daemon with the new config — so the +//! change takes effect without a restart. Validation failures return 422 with +//! per-field details (the established `validation_failed` envelope). +//! +//! `PUT` is a **full replace**, not a patch: the body is deserialized with +//! field-level defaults, so any omitted field is reset to its compiled +//! default rather than retaining the stored value. The dashboard always +//! sends the complete DTO (it round-trips the one it loaded); programmatic +//! callers must do the same. + +use axum::extract::State; +use axum::routing::get; +use axum::{Json, Router}; +use serde::Serialize; + +use crate::app::AppState; +use crate::auth::extractor::RequireAdmin; +use crate::config::CrawlerConfig; +use crate::crawler::browser::BrowserMode; +use crate::error::{AppError, AppResult}; +use crate::repo; +use crate::settings::{ + AnalysisSettings, CrawlerSettings, FieldErrors, PromptDefaults, KEY_ANALYSIS, KEY_CRAWLER, +}; + +pub fn routes() -> Router { + Router::new() + .route( + "/admin/settings/crawler", + get(get_crawler).put(put_crawler), + ) + .route( + "/admin/settings/analysis", + get(get_analysis).put(put_analysis), + ) +} + +// --------------------------------------------------------------------------- +// Response shapes +// --------------------------------------------------------------------------- + +/// Read-only mirror of the crawler fields managed via environment (host/infra +/// + secrets), surfaced so the admin sees what's in effect without editing it. +#[derive(Debug, Serialize)] +struct CrawlerEnvOnly { + browser_mode: &'static str, + browser_args: Vec, + proxy: Option, + tor_control_url: Option, + /// Whether a TOR control credential (password or cookie file) is set. + tor_credentials_configured: bool, + /// Whether an initial `CRAWLER_PHPSESSID` is configured (the live session + /// is managed separately via the crawler dashboard). + session_configured: bool, +} + +impl CrawlerEnvOnly { + fn from_base(base: &CrawlerConfig) -> Self { + Self { + browser_mode: match base.browser.mode { + BrowserMode::Headed => "headed", + BrowserMode::Headless => "headless", + }, + browser_args: base.browser.extra_args.clone(), + proxy: base.proxy.clone(), + tor_control_url: base.tor_control_url.clone(), + tor_credentials_configured: base.tor_control_password.is_some() + || base.tor_control_cookie_path.is_some(), + session_configured: base.phpsessid.is_some(), + } + } +} + +#[derive(Debug, Serialize)] +struct CrawlerSettingsResponse { + editable: CrawlerSettings, + env_only: CrawlerEnvOnly, +} + +/// Read-only mirror of analysis env-managed fields (just the secret). +#[derive(Debug, Serialize)] +struct AnalysisEnvOnly { + api_key_configured: bool, +} + +#[derive(Debug, Serialize)] +struct AnalysisSettingsResponse { + editable: AnalysisSettings, + env_only: AnalysisEnvOnly, + /// The compiled prompt defaults so the UI can show placeholders and + /// implement per-prompt "reset to default". + prompt_defaults: PromptDefaults, +} + +// --------------------------------------------------------------------------- +// Handlers +// --------------------------------------------------------------------------- + +fn validation_error(errs: FieldErrors) -> AppError { + AppError::ValidationFailed { + message: "invalid settings".to_string(), + details: serde_json::json!({ "fields": errs.errors }), + } +} + +/// Current effective crawler settings: the stored DTO when present, otherwise +/// derived from the env base (the row is seeded at boot, so the stored branch +/// is the norm). +async fn load_crawler_dto(state: &AppState) -> AppResult { + Ok(match repo::app_settings::get(&state.db, KEY_CRAWLER).await? { + Some(v) => serde_json::from_value(v) + .unwrap_or_else(|_| CrawlerSettings::from_config(&state.crawler_base)), + None => CrawlerSettings::from_config(&state.crawler_base), + }) +} + +async fn load_analysis_dto(state: &AppState) -> AppResult { + Ok(match repo::app_settings::get(&state.db, KEY_ANALYSIS).await? { + Some(v) => serde_json::from_value(v) + .unwrap_or_else(|_| AnalysisSettings::from_config(&state.analysis_base)), + None => AnalysisSettings::from_config(&state.analysis_base), + }) +} + +async fn get_crawler( + State(state): State, + _admin: RequireAdmin, +) -> AppResult> { + let editable = load_crawler_dto(&state).await?; + Ok(Json(CrawlerSettingsResponse { + editable, + env_only: CrawlerEnvOnly::from_base(&state.crawler_base), + })) +} + +async fn put_crawler( + State(state): State, + admin: RequireAdmin, + Json(incoming): Json, +) -> AppResult> { + // Validate by converting against the env base; this also rebuilds the + // download allowlist and re-parses tz / time. + let cfg = incoming + .to_config(&state.crawler_base) + .map_err(validation_error)?; + // Persist a normalized DTO (so the stored/returned allowlist reflects the + // effective hosts, prompts are canonicalized, etc.). + let normalized = CrawlerSettings::from_config(&cfg); + let value = serde_json::to_value(&normalized).map_err(|e| AppError::Other(e.into()))?; + + let mut tx = state.db.begin().await?; + repo::app_settings::upsert(&mut *tx, KEY_CRAWLER, &value).await?; + repo::admin_audit::insert( + &mut *tx, + admin.0.id, + "update_crawler_settings", + "settings", + None, + value.clone(), + ) + .await?; + tx.commit().await?; + + // Apply live (graceful respawn) when a reloader is wired up. + if let Some(reloader) = &state.reloader { + reloader + .reload_crawler(cfg) + .await + .map_err(AppError::Other)?; + } + + Ok(Json(CrawlerSettingsResponse { + editable: normalized, + env_only: CrawlerEnvOnly::from_base(&state.crawler_base), + })) +} + +async fn get_analysis( + State(state): State, + _admin: RequireAdmin, +) -> AppResult> { + let editable = load_analysis_dto(&state).await?; + Ok(Json(AnalysisSettingsResponse { + editable, + env_only: AnalysisEnvOnly { + api_key_configured: state.analysis_base.api_key.is_some(), + }, + prompt_defaults: PromptDefaults::get(), + })) +} + +async fn put_analysis( + State(state): State, + admin: RequireAdmin, + Json(incoming): Json, +) -> AppResult> { + let cfg = incoming + .to_config(&state.analysis_base) + .map_err(validation_error)?; + let normalized = AnalysisSettings::from_config(&cfg); + let value = serde_json::to_value(&normalized).map_err(|e| AppError::Other(e.into()))?; + + let mut tx = state.db.begin().await?; + repo::app_settings::upsert(&mut *tx, KEY_ANALYSIS, &value).await?; + repo::admin_audit::insert( + &mut *tx, + admin.0.id, + "update_analysis_settings", + "settings", + None, + value.clone(), + ) + .await?; + tx.commit().await?; + + if let Some(reloader) = &state.reloader { + reloader + .reload_analysis(cfg) + .await + .map_err(AppError::Other)?; + } + + Ok(Json(AnalysisSettingsResponse { + editable: normalized, + env_only: AnalysisEnvOnly { + api_key_configured: state.analysis_base.api_key.is_some(), + }, + prompt_defaults: PromptDefaults::get(), + })) +} diff --git a/backend/src/api/chapters.rs b/backend/src/api/chapters.rs index fdcb67a..1df30e5 100644 --- a/backend/src/api/chapters.rs +++ b/backend/src/api/chapters.rs @@ -161,7 +161,7 @@ async fn create( // rolled-back upload never leaves jobs pointing at nonexistent pages; a // failed enqueue is logged but doesn't fail the upload (the admin // re-enqueue endpoint can backfill). - if state.analysis_enabled { + if state.analysis_enabled() { for page_id in page_ids { if let Err(e) = repo::page_analysis::enqueue_for_page(&state.db, page_id, false).await diff --git a/backend/src/app.rs b/backend/src/app.rs index 7c1fb58..3fabc32 100644 --- a/backend/src/app.rs +++ b/backend/src/app.rs @@ -1,5 +1,6 @@ use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; +use std::sync::RwLock as StdRwLock; use anyhow::Context; use async_trait::async_trait; @@ -17,7 +18,7 @@ use tower_http::trace::TraceLayer; use crate::auth::extractor::CurrentUser; use crate::auth::rate_limit::AuthRateLimiter; use crate::error::AppError; -use crate::config::{AuthConfig, Config, CrawlerConfig, UploadConfig}; +use crate::config::{AnalysisConfig, AuthConfig, Config, CrawlerConfig, UploadConfig}; use crate::crawler::browser_manager::{self, BrowserManager}; use crate::crawler::content::{self, SyncOutcome}; use crate::crawler::daemon::{self, ChapterDispatcher, DaemonConfig, MetadataPass}; @@ -40,33 +41,125 @@ pub struct AppState { /// One instance per AppState so tests stay isolated across the /// same process. pub auth_limiter: Arc, - /// Admin-triggered force resync. `None` when the crawler daemon - /// is disabled (`CRAWLER_DAEMON=false`); admin handlers gate on - /// `.is_some()` and return 503 otherwise. Set by [`build`] from the - /// same wiring that builds the daemon's chapter dispatcher, so a - /// force resync uses the daemon's BrowserManager + rate limiters. - pub resync: Option>, - /// Crawler observability + control handle (live status, coordinated - /// browser restart, runtime session, manual run). `None` when the - /// daemon is disabled; admin handlers gate on `.is_some()` → 503. - pub crawler: Option>, + /// Runtime-swappable controls (crawler resync/control handles + the + /// analysis enable gate). Shared with the [`Supervisors`] so a config + /// reload that respawns a daemon updates what handlers see, live — + /// without a restart. In tests this starts empty (no daemons). + pub runtime: Arc, + /// Applies a persisted settings change to the running daemons + /// (graceful stop + respawn). `Some` in production ([`build`]); `None` + /// in the test harness, where settings still persist but no daemon is + /// spawned. See [`DaemonReloader`]. + pub reloader: Option>, + /// Env-derived crawler config — the **base** the stored settings DTO is + /// overlaid onto (carries env-only fields: browser, proxy, TOR, session). + pub crawler_base: CrawlerConfig, + /// Env-derived analysis config base (carries the env-only vision API key). + pub analysis_base: AnalysisConfig, /// Browser origins permitted to issue mutating requests to /// `/api/v1/admin/*`. See [`crate::config::Config::admin_allowed_origins`] /// for the policy. Cloned per-request into the CSRF middleware; the /// `Arc` keeps the clone cheap. Empty list → check is skipped /// (operator opt-out documented in `.env.example`). pub admin_allowed_origins: Arc>, - /// When `true`, page-create paths (upload + crawler) enqueue - /// `analyze_page` jobs and the admin re-enqueue endpoints are active. - /// Mirrors `config.analysis.enabled`; defaults to `false` so the - /// feature is opt-in and the test harness inherits a no-op gate. - pub analysis_enabled: bool, /// Live analysis-event broadcaster. Always present (the worker and the /// admin enqueue path publish here; the SSE endpoint subscribes). When /// the worker is disabled the channel simply stays quiet. pub analysis_events: Arc, } +impl AppState { + /// Current crawler control handle, or `None` when the daemon is stopped. + pub fn crawler(&self) -> Option> { + self.runtime.crawler_control() + } + + /// Current force-resync service, or `None` when the daemon is stopped. + pub fn resync(&self) -> Option> { + self.runtime.resync() + } + + /// Whether AI page analysis is currently enabled (read live). + pub fn analysis_enabled(&self) -> bool { + self.runtime.analysis_enabled() + } +} + +/// The crawler control handles, swapped atomically when the daemon is +/// (re)spawned or stopped. Cloned cheaply on every read. +#[derive(Clone, Default)] +struct CrawlerControls { + resync: Option>, + control: Option>, +} + +/// Shared, runtime-mutable surface read by handlers and mutated by the +/// [`Supervisors`] on a config reload. The analysis gate is an `Arc` +/// so the crawler's chapter dispatcher (which enqueues `analyze_page` jobs) +/// sees toggles live without being respawned. +pub struct RuntimeControls { + crawler: StdRwLock, + analysis_enabled: Arc, +} + +impl RuntimeControls { + /// Empty controls (no crawler daemon) with the given initial analysis gate. + pub fn new(analysis_enabled: bool) -> Self { + Self { + crawler: StdRwLock::new(CrawlerControls::default()), + analysis_enabled: Arc::new(AtomicBool::new(analysis_enabled)), + } + } + + pub fn analysis_enabled(&self) -> bool { + self.analysis_enabled.load(Ordering::Relaxed) + } + + /// Flip the analysis gate. Used by the [`Supervisors`] on reload and by + /// the test harness's stub reloader. + pub fn set_analysis_enabled(&self, v: bool) { + self.analysis_enabled.store(v, Ordering::Relaxed); + } + + /// The shared gate handle, passed to the crawler chapter dispatcher. + fn analysis_gate(&self) -> Arc { + Arc::clone(&self.analysis_enabled) + } + + pub fn crawler_control(&self) -> Option> { + self.crawler.read().unwrap().control.clone() + } + + pub fn resync(&self) -> Option> { + self.crawler.read().unwrap().resync.clone() + } + + /// Test helper: install a resync service without a running daemon. + pub fn set_resync(&self, resync: Option>) { + self.crawler.write().unwrap().resync = resync; + } + + fn set_crawler( + &self, + resync: Option>, + control: Option>, + ) { + let mut g = self.crawler.write().unwrap(); + g.resync = resync; + g.control = control; + } +} + +/// Applies a validated settings change to the running daemons by gracefully +/// stopping and respawning them with the new effective config (and updating +/// the shared [`RuntimeControls`]). Implemented by [`Supervisors`] in +/// production; the settings handlers call it after persisting. +#[async_trait] +pub trait DaemonReloader: Send + Sync { + async fn reload_crawler(&self, cfg: CrawlerConfig) -> anyhow::Result<()>; + async fn reload_analysis(&self, cfg: AnalysisConfig) -> anyhow::Result<()>; +} + /// Shared handle the admin crawler endpoints use to observe and control /// the running daemon. Bundled so the handlers take one optional field on /// `AppState` rather than many. @@ -90,27 +183,96 @@ pub struct CrawlerControl { } /// Bundle returned by [`build`]. The router is what `axum::serve` consumes; -/// the daemon (when enabled) outlives the HTTP server and is awaited via -/// [`AppHandle::shutdown`] after the listener has finished gracefully. +/// the [`Supervisors`] own the background daemons (when enabled), outlive the +/// HTTP server, and are awaited via [`AppHandle::shutdown`] after the listener +/// has finished gracefully. pub struct AppHandle { pub router: Router, - pub daemon: Option, - /// AI content-analysis worker daemon; `None` when `ANALYSIS_ENABLED` - /// is off. Awaited alongside the crawler daemon on shutdown. - pub analysis_daemon: Option, + pub supervisors: Arc, } impl AppHandle { pub async fn shutdown(self) { - if let Some(d) = self.daemon { - d.shutdown().await; + self.supervisors.shutdown().await; + } +} + +/// Owns the crawler + analysis daemon handles and respawns them on a config +/// reload. Holds the build-time inputs (db, storage, env-base configs, the +/// shared event bus and [`RuntimeControls`]) needed to spawn from scratch. +pub struct Supervisors { + db: PgPool, + storage: Arc, + runtime: Arc, + analysis_events: Arc, + /// Serializes reloads + owns the live crawler daemon handle. + crawler_handle: tokio::sync::Mutex>, + /// Serializes reloads + owns the live analysis daemon handle. + analysis_handle: tokio::sync::Mutex>, +} + +impl Supervisors { + pub async fn shutdown(&self) { + if let Some(h) = self.crawler_handle.lock().await.take() { + h.shutdown().await; } - if let Some(a) = self.analysis_daemon { - a.shutdown().await; + if let Some(h) = self.analysis_handle.lock().await.take() { + h.shutdown().await; } } } +#[async_trait] +impl DaemonReloader for Supervisors { + async fn reload_crawler(&self, cfg: CrawlerConfig) -> anyhow::Result<()> { + // Serialize reloads; do the (possibly slow) graceful shutdown under + // the handle lock but outside the read-path RwLock so status reads + // never block on a draining browser. + let mut guard = self.crawler_handle.lock().await; + if let Some(h) = guard.take() { + h.shutdown().await; + } + self.runtime.set_crawler(None, None); + if cfg.daemon_enabled { + let spawned = spawn_crawler_daemon( + self.db.clone(), + Arc::clone(&self.storage), + &cfg, + self.runtime.analysis_gate(), + ) + .await?; + self.runtime + .set_crawler(Some(spawned.resync), Some(spawned.crawler)); + *guard = Some(spawned.handle); + tracing::info!("crawler daemon (re)started from settings"); + } else { + tracing::info!("crawler daemon stopped (disabled in settings)"); + } + Ok(()) + } + + async fn reload_analysis(&self, cfg: AnalysisConfig) -> anyhow::Result<()> { + let mut guard = self.analysis_handle.lock().await; + if let Some(h) = guard.take() { + h.shutdown().await; + } + self.runtime.set_analysis_enabled(cfg.enabled); + if cfg.enabled { + let handle = spawn_analysis_daemon( + self.db.clone(), + Arc::clone(&self.storage), + &cfg, + Arc::clone(&self.analysis_events), + )?; + *guard = Some(handle); + tracing::info!(model = %cfg.model, "analysis daemon (re)started from settings"); + } else { + tracing::info!("analysis daemon stopped (disabled in settings)"); + } + Ok(()) + } +} + pub async fn build(config: Config) -> anyhow::Result { let db = PgPoolOptions::new() .max_connections(10) @@ -127,60 +289,48 @@ pub async fn build(config: Config) -> anyhow::Result { let storage: Arc = Arc::new(LocalStorage::new(config.storage_dir.clone())); - let (daemon, resync, crawler) = if config.crawler.daemon_enabled { - let spawned = spawn_crawler_daemon( - db.clone(), - Arc::clone(&storage), - &config.crawler, - config.analysis.enabled, - ) - .await?; - (Some(spawned.handle), Some(spawned.resync), Some(spawned.crawler)) - } else { - tracing::info!("crawler daemon disabled (CRAWLER_DAEMON=false)"); - (None, None, None) - }; + // Seed the settings rows from env on first boot, then load the effective + // config (DB overlaid on the env base). The DB is the source of truth + // after the first boot; env only fills a missing row. + let crawler_cfg = load_effective_crawler(&db, &config.crawler).await?; + let analysis_cfg = load_effective_analysis(&db, &config.analysis).await?; // Live-event bus shared by the worker (progress) and the admin enqueue // path; the SSE endpoint subscribes. Created unconditionally so the // stream exists even with the worker disabled. let analysis_events = Arc::new(crate::analysis::events::AnalysisEvents::new()); - // AI content-analysis worker. Independent of the crawler daemon (works - // for uploads with the crawler off), gated on ANALYSIS_ENABLED. Uses a - // plain reqwest client — no cookie jar / proxy, unlike the crawler's. - let analysis_daemon = if config.analysis.enabled { - let http = reqwest::Client::builder() - .timeout(config.analysis.request_timeout) - .build() - .context("build analysis http client")?; - let vision = crate::analysis::vision::VisionClient::new(http, &config.analysis); - let dispatcher = Arc::new(crate::analysis::daemon::RealAnalyzeDispatcher { - db: db.clone(), - storage: Arc::clone(&storage), - vision, - model: config.analysis.model.clone(), - max_image_bytes: config.analysis.max_image_bytes, - }); - let handle = crate::analysis::daemon::spawn( - db.clone(), - tokio_util::sync::CancellationToken::new(), - crate::analysis::daemon::AnalysisDaemonConfig { - dispatcher, - workers: config.analysis.workers, - job_timeout: config.analysis.job_timeout, - events: Arc::clone(&analysis_events), - }, - ); - tracing::info!( - workers = config.analysis.workers, - model = %config.analysis.model, - "analysis worker daemon started" - ); - Some(handle) + let runtime = Arc::new(RuntimeControls::new(analysis_cfg.enabled)); + let supervisors = Arc::new(Supervisors { + db: db.clone(), + storage: Arc::clone(&storage), + runtime: Arc::clone(&runtime), + analysis_events: Arc::clone(&analysis_events), + crawler_handle: tokio::sync::Mutex::new(None), + analysis_handle: tokio::sync::Mutex::new(None), + }); + + // Initial spawn goes through the same reload path so there's one code + // path for "bring the daemon up with this config". A spawn failure is + // logged but does NOT abort startup: the persisted settings are the + // source of truth and a bad value (e.g. a config that won't launch the + // browser) must not wedge the whole server into a boot loop — the + // server comes up with that daemon stopped so an admin can fix it via + // the settings UI. + if crawler_cfg.daemon_enabled { + if let Err(e) = supervisors.reload_crawler(crawler_cfg).await { + tracing::error!(?e, "crawler daemon failed to start; continuing with it stopped"); + } } else { - None - }; + tracing::info!("crawler daemon disabled"); + } + if analysis_cfg.enabled { + if let Err(e) = supervisors.reload_analysis(analysis_cfg).await { + tracing::error!(?e, "analysis worker failed to start; continuing with it stopped"); + } + } else { + tracing::info!("analysis worker disabled"); + } let auth_limiter = Arc::new(AuthRateLimiter::new(config.auth.rate_limit)); let state = AppState { @@ -189,14 +339,99 @@ pub async fn build(config: Config) -> anyhow::Result { auth: config.auth.clone(), upload: config.upload.clone(), auth_limiter, - resync, - crawler, + runtime, + reloader: Some(Arc::clone(&supervisors) as Arc), + crawler_base: config.crawler.clone(), + analysis_base: config.analysis.clone(), admin_allowed_origins: Arc::new(config.admin_allowed_origins.clone()), - analysis_enabled: config.analysis.enabled, analysis_events, }; let router = router(state).layer(cors_layer(&config.cors_allowed_origins)); - Ok(AppHandle { router, daemon, analysis_daemon }) + Ok(AppHandle { router, supervisors }) +} + +/// Seed the crawler settings row from env when absent, then return the +/// effective [`CrawlerConfig`] (DB DTO overlaid on the env base). A stored +/// DTO that fails to convert (corruption / drift) falls back to the env base +/// rather than blocking startup. +async fn load_effective_crawler( + db: &PgPool, + base: &CrawlerConfig, +) -> anyhow::Result { + use crate::settings::{CrawlerSettings, KEY_CRAWLER}; + let dto = match repo::app_settings::get(db, KEY_CRAWLER).await? { + Some(v) => serde_json::from_value::(v) + .unwrap_or_else(|_| CrawlerSettings::from_config(base)), + None => { + let dto = CrawlerSettings::from_config(base); + repo::app_settings::seed_if_absent(db, KEY_CRAWLER, &serde_json::to_value(&dto)?) + .await?; + tracing::info!("seeded crawler settings from env"); + dto + } + }; + Ok(dto.to_config(base).unwrap_or_else(|e| { + tracing::warn!(?e, "stored crawler settings invalid; using env base"); + base.clone() + })) +} + +/// Analysis counterpart of [`load_effective_crawler`]. +async fn load_effective_analysis( + db: &PgPool, + base: &AnalysisConfig, +) -> anyhow::Result { + use crate::settings::{AnalysisSettings, KEY_ANALYSIS}; + let dto = match repo::app_settings::get(db, KEY_ANALYSIS).await? { + Some(v) => serde_json::from_value::(v) + .unwrap_or_else(|_| AnalysisSettings::from_config(base)), + None => { + let dto = AnalysisSettings::from_config(base); + repo::app_settings::seed_if_absent(db, KEY_ANALYSIS, &serde_json::to_value(&dto)?) + .await?; + tracing::info!("seeded analysis settings from env"); + dto + } + }; + Ok(dto.to_config(base).unwrap_or_else(|e| { + tracing::warn!(?e, "stored analysis settings invalid; using env base"); + base.clone() + })) +} + +/// Spawn the AI content-analysis worker daemon with the given config. Returns +/// its handle. Independent of the crawler daemon (works for uploads with the +/// crawler off). Uses a plain reqwest client — no cookie jar / proxy. +fn spawn_analysis_daemon( + db: PgPool, + storage: Arc, + cfg: &AnalysisConfig, + events: Arc, +) -> anyhow::Result { + let http = reqwest::Client::builder() + .timeout(cfg.request_timeout) + .build() + .context("build analysis http client")?; + let vision = crate::analysis::vision::VisionClient::new(http, cfg); + let dispatcher = Arc::new(crate::analysis::daemon::RealAnalyzeDispatcher { + db: db.clone(), + storage, + vision, + model: cfg.model.clone(), + max_image_bytes: cfg.max_image_bytes, + }); + let handle = crate::analysis::daemon::spawn( + db, + CancellationToken::new(), + crate::analysis::daemon::AnalysisDaemonConfig { + dispatcher, + workers: cfg.workers, + job_timeout: cfg.job_timeout, + events, + }, + ); + tracing::info!(workers = cfg.workers, model = %cfg.model, "analysis worker daemon started"); + Ok(handle) } /// Bundle returned by [`spawn_crawler_daemon`]. The handle owns the @@ -213,7 +448,7 @@ async fn spawn_crawler_daemon( db: PgPool, storage: Arc, cfg: &CrawlerConfig, - analysis_enabled: bool, + analysis_enabled: Arc, ) -> anyhow::Result { // Reqwest client with a shared cookie jar so CDN image fetches include // PHPSESSID. The same `Arc` is held by the SessionController, so a @@ -512,9 +747,10 @@ struct RealChapterDispatcher { rate: Arc, download_allowlist: DownloadAllowlist, max_image_bytes: usize, - /// Enqueue `analyze_page` jobs for freshly-crawled pages. Mirrors - /// `config.analysis.enabled`. - analysis_enabled: bool, + /// Enqueue `analyze_page` jobs for freshly-crawled pages. Shared gate + /// (read live) so toggling analysis at runtime takes effect without a + /// crawler respawn. Mirrors the analysis enable setting. + analysis_enabled: Arc, /// Consecutive transient chapter failures; resets on any success. /// Drives the automatic coordinated browser restart. transient_failures: Arc, @@ -570,7 +806,7 @@ impl ChapterDispatcher for RealChapterDispatcher { self.max_image_bytes, self.tor.as_deref(), Some(&self.status), - self.analysis_enabled, + self.analysis_enabled.load(Ordering::Relaxed), ) .await; drop(lease); diff --git a/backend/src/config.rs b/backend/src/config.rs index 30f0368..34eb6ed 100644 --- a/backend/src/config.rs +++ b/backend/src/config.rs @@ -90,6 +90,27 @@ impl ResponseFormat { _ => ResponseFormat::JsonSchema, } } + + /// Canonical wire string, the inverse of [`Self::from_str`] for the + /// three modes the API exposes. + pub fn as_str(self) -> &'static str { + match self { + ResponseFormat::JsonSchema => "json_schema", + ResponseFormat::JsonObject => "json_object", + ResponseFormat::None => "none", + } + } + + /// Parse a wire value, rejecting unknown modes (stricter than + /// [`Self::from_str`], which the env loader uses to stay lenient). + pub fn parse_strict(s: &str) -> Option { + match s.trim().to_lowercase().as_str() { + "json_schema" => Some(ResponseFormat::JsonSchema), + "json_object" => Some(ResponseFormat::JsonObject), + "none" => Some(ResponseFormat::None), + _ => None, + } + } } /// AI content-analysis worker configuration: the enable gate, the local @@ -146,6 +167,21 @@ pub struct AnalysisConfig { /// the repetition loops that otherwise run small models into the token /// ceiling. `0` omits the field. pub frequency_penalty: f64, + /// Sampling `temperature` sent with each request (`ANALYSIS_TEMPERATURE`). + /// Defaults to `0.0` (deterministic), which is the most reliable for + /// structured JSON output; some models behave better with a small + /// positive value. + pub temperature: f64, + /// System prompt for the single-call (normal-aspect) analysis path + /// (`ANALYSIS_SYSTEM_PROMPT`). Defaults to + /// [`crate::analysis::prompt::SYSTEM_PROMPT_DEFAULT`]. + pub system_prompt: String, + /// Pass-A OCR-only prompt for tall-page slices (`ANALYSIS_OCR_PROMPT`). + /// Defaults to [`crate::analysis::prompt::OCR_PROMPT_DEFAULT`]. + pub ocr_prompt: String, + /// Pass-B grounding prompt — tags/scene/safety (`ANALYSIS_GROUNDING_PROMPT`). + /// Defaults to [`crate::analysis::prompt::GROUNDING_PROMPT_DEFAULT`]. + pub grounding_prompt: String, } impl Default for AnalysisConfig { @@ -170,6 +206,10 @@ impl Default for AnalysisConfig { max_image_bytes: 8 * 1024 * 1024, response_format: ResponseFormat::JsonSchema, frequency_penalty: 0.3, + temperature: 0.0, + system_prompt: crate::analysis::prompt::SYSTEM_PROMPT_DEFAULT.to_string(), + ocr_prompt: crate::analysis::prompt::OCR_PROMPT_DEFAULT.to_string(), + grounding_prompt: crate::analysis::prompt::GROUNDING_PROMPT_DEFAULT.to_string(), } } } @@ -206,10 +246,23 @@ impl AnalysisConfig { .map(|s| ResponseFormat::from_str(&s)) .unwrap_or(d.response_format), frequency_penalty: env_f64("ANALYSIS_FREQUENCY_PENALTY", d.frequency_penalty), + temperature: env_f64("ANALYSIS_TEMPERATURE", d.temperature), + system_prompt: env_prompt("ANALYSIS_SYSTEM_PROMPT", d.system_prompt), + ocr_prompt: env_prompt("ANALYSIS_OCR_PROMPT", d.ocr_prompt), + grounding_prompt: env_prompt("ANALYSIS_GROUNDING_PROMPT", d.grounding_prompt), } } } +/// Read a prompt override from env, falling back to the default when unset +/// or blank (so `ANALYSIS_SYSTEM_PROMPT=` doesn't wipe the prompt). +fn env_prompt(name: &str, default: String) -> String { + std::env::var(name) + .ok() + .filter(|s| !s.trim().is_empty()) + .unwrap_or(default) +} + #[derive(Clone, Debug)] pub struct Config { pub database_url: String, diff --git a/backend/src/crawler/safety.rs b/backend/src/crawler/safety.rs index 3ce6c71..b9af92a 100644 --- a/backend/src/crawler/safety.rs +++ b/backend/src/crawler/safety.rs @@ -91,6 +91,16 @@ impl DownloadAllowlist { self.hosts.is_empty() } + /// The explicitly-allowed hosts (lowercased). Empty when `allow_any`. + pub fn hosts(&self) -> &[String] { + &self.hosts + } + + /// Whether the host check is bypassed entirely (`CRAWLER_ALLOW_ANY_HOST`). + pub fn is_allow_any(&self) -> bool { + self.allow_any + } + pub fn contains(&self, host: &str) -> bool { if self.allow_any { return true; diff --git a/backend/src/lib.rs b/backend/src/lib.rs index 37cde37..d15eb8f 100644 --- a/backend/src/lib.rs +++ b/backend/src/lib.rs @@ -7,5 +7,6 @@ pub mod crawler; pub mod domain; pub mod error; pub mod repo; +pub mod settings; pub mod storage; pub mod upload; diff --git a/backend/src/main.rs b/backend/src/main.rs index 20fc3c4..1288303 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -21,11 +21,8 @@ async fn main() -> anyhow::Result<()> { let config = mangalord::config::Config::from_env()?; let addr: SocketAddr = config.bind_address.parse()?; - let mangalord::app::AppHandle { - router, - daemon, - analysis_daemon, - } = mangalord::app::build(config).await?; + let mangalord::app::AppHandle { router, supervisors } = + mangalord::app::build(config).await?; tracing::info!(%addr, "mangalord listening"); let listener = tokio::net::TcpListener::bind(addr).await?; @@ -33,30 +30,17 @@ async fn main() -> anyhow::Result<()> { .with_graceful_shutdown(shutdown_signal()) .await?; - // Drain background tasks (crawler daemon) before exiting so Chromium - // gets a clean shutdown rather than relying on kill-on-drop. Bounded - // by a timeout so a wedged shutdown path can't trap the process. - if let Some(d) = daemon { - if tokio::time::timeout(CRAWLER_SHUTDOWN_TIMEOUT, d.shutdown()) - .await - .is_err() - { - tracing::warn!( - timeout_s = CRAWLER_SHUTDOWN_TIMEOUT.as_secs(), - "crawler daemon shutdown exceeded timeout; abandoning" - ); - } - } - if let Some(a) = analysis_daemon { - if tokio::time::timeout(CRAWLER_SHUTDOWN_TIMEOUT, a.shutdown()) - .await - .is_err() - { - tracing::warn!( - timeout_s = CRAWLER_SHUTDOWN_TIMEOUT.as_secs(), - "analysis daemon shutdown exceeded timeout; abandoning" - ); - } + // Drain background daemons (crawler + analysis) before exiting so + // Chromium gets a clean shutdown rather than relying on kill-on-drop. + // Bounded by a timeout so a wedged shutdown path can't trap the process. + if tokio::time::timeout(CRAWLER_SHUTDOWN_TIMEOUT, supervisors.shutdown()) + .await + .is_err() + { + tracing::warn!( + timeout_s = CRAWLER_SHUTDOWN_TIMEOUT.as_secs(), + "daemon shutdown exceeded timeout; abandoning" + ); } Ok(()) } diff --git a/backend/src/repo/app_settings.rs b/backend/src/repo/app_settings.rs new file mode 100644 index 0000000..5de7f60 --- /dev/null +++ b/backend/src/repo/app_settings.rs @@ -0,0 +1,54 @@ +//! Persistence for runtime-editable application settings (`app_settings`), +//! a small key-value JSONB table (one row per subsystem group). Mirrors the +//! `crawler_state` access style: plain async fns over `&PgPool`, the `value` +//! a `serde_json::Value` the caller (de)serializes into a settings DTO. + +use sqlx::{PgExecutor, PgPool}; + +/// Fetch the raw JSONB for a settings group, or `None` if unset. +pub async fn get(pool: &PgPool, key: &str) -> sqlx::Result> { + sqlx::query_scalar("SELECT value FROM app_settings WHERE key = $1") + .bind(key) + .fetch_optional(pool) + .await +} + +/// Insert-or-replace the JSONB for a settings group, stamping `updated_at`. +/// Generic over the executor so it can run inside the same transaction as +/// the matching admin-audit insert. +pub async fn upsert<'e, E: PgExecutor<'e>>( + executor: E, + key: &str, + value: &serde_json::Value, +) -> sqlx::Result<()> { + sqlx::query( + "INSERT INTO app_settings (key, value, updated_at) \ + VALUES ($1, $2, now()) \ + ON CONFLICT (key) DO UPDATE \ + SET value = EXCLUDED.value, updated_at = now()", + ) + .bind(key) + .bind(value) + .execute(executor) + .await?; + Ok(()) +} + +/// Write the value only if the row is absent (the env → DB boot seed). Returns +/// `true` when a row was inserted, `false` when one already existed. Uses +/// `ON CONFLICT DO NOTHING` so a concurrent boot can't race two seeds. +pub async fn seed_if_absent( + pool: &PgPool, + key: &str, + value: &serde_json::Value, +) -> sqlx::Result { + let result = sqlx::query( + "INSERT INTO app_settings (key, value) VALUES ($1, $2) \ + ON CONFLICT (key) DO NOTHING", + ) + .bind(key) + .bind(value) + .execute(pool) + .await?; + Ok(result.rows_affected() > 0) +} diff --git a/backend/src/repo/mod.rs b/backend/src/repo/mod.rs index 63a02fc..cee32a0 100644 --- a/backend/src/repo/mod.rs +++ b/backend/src/repo/mod.rs @@ -1,6 +1,7 @@ pub mod admin_audit; pub mod admin_view; pub mod api_token; +pub mod app_settings; pub mod author; pub mod bookmark; pub mod chapter; diff --git a/backend/src/settings.rs b/backend/src/settings.rs new file mode 100644 index 0000000..1be7d6b --- /dev/null +++ b/backend/src/settings.rs @@ -0,0 +1,643 @@ +//! Serializable, admin-editable settings DTOs for the crawler and analysis +//! subsystems, and their conversion to/from the runtime [`CrawlerConfig`] / +//! [`AnalysisConfig`]. +//! +//! These DTOs are the boundary persisted in the `app_settings` table (one +//! JSONB row per group) and exchanged over the admin API. They carry **only +//! the operationally-safe, UI-editable fields** as primitive types — the +//! runtime configs additionally hold env-only/structural fields (browser +//! launch options, proxy, TOR control, the vision API key, the cookie +//! domain) that are never persisted here. +//! +//! Flow: at boot, [`CrawlerSettings::from_config`] / +//! [`AnalysisSettings::from_config`] derive the env-seed DTO; it is written to +//! the DB only when the row is absent. Thereafter the DB row is the source of +//! truth, and [`CrawlerSettings::to_config`] / [`AnalysisSettings::to_config`] +//! overlay it onto the env-derived **base** (which carries the env-only +//! fields) to produce the effective config the daemons are (re)spawned with. + +use std::time::Duration; + +use chrono::NaiveTime; +use chrono_tz::Tz; +use serde::{Deserialize, Serialize}; + +use crate::analysis::prompt::{ + GROUNDING_PROMPT_DEFAULT, OCR_PROMPT_DEFAULT, SYSTEM_PROMPT_DEFAULT, +}; +use crate::config::{AnalysisConfig, CrawlerConfig, ResponseFormat}; +use crate::crawler::safety::DownloadAllowlist; + +/// `app_settings.key` for the crawler group. +pub const KEY_CRAWLER: &str = "crawler"; +/// `app_settings.key` for the analysis group. +pub const KEY_ANALYSIS: &str = "analysis"; + +/// One field-level validation failure, surfaced to the UI per input. +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +pub struct FieldError { + pub field: String, + pub message: String, +} + +/// Collected validation failures for a settings payload. Empty == valid. +#[derive(Debug, Clone, Serialize, Default, PartialEq, Eq)] +pub struct FieldErrors { + pub errors: Vec, +} + +impl FieldErrors { + fn push(&mut self, field: &str, message: impl Into) { + self.errors.push(FieldError { + field: field.to_string(), + message: message.into(), + }); + } + + pub fn is_empty(&self) -> bool { + self.errors.is_empty() + } +} + +// --------------------------------------------------------------------------- +// Crawler +// --------------------------------------------------------------------------- + +/// Admin-editable crawler-daemon settings. Mirrors the operational `CRAWLER_*` +/// env vars; host/infra and session fields stay env-only. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(default)] +pub struct CrawlerSettings { + pub daemon_enabled: bool, + /// Daily metadata-pass time, `"HH:MM"` (24h). + pub daily_at: String, + /// IANA timezone for the daily schedule. + pub tz: String, + pub idle_timeout_secs: u64, + pub chapter_workers: u64, + pub retention_days: u32, + pub start_url: Option, + pub rate_ms: u64, + pub cdn_host: Option, + pub cdn_rate_ms: u64, + /// Domain the session cookie (PHPSESSID) is scoped to when injecting it + /// into the browser / CDN requests. Pairs with `start_url`. + pub cookie_domain: Option, + pub user_agent: Option, + /// Hosts the crawler may download images from (in addition to the + /// auto-seeded start-url / CDN hosts). Ignored when `allow_any_host`. + pub download_allowlist: Vec, + pub allow_any_host: bool, + pub max_image_bytes: u64, + /// Max manga detail fetches per metadata pass; `0` = unlimited. + pub manga_limit: u64, + pub job_timeout_secs: u64, + pub metadata_max_consecutive_failures: u32, + pub browser_restart_threshold: u32, +} + +impl Default for CrawlerSettings { + fn default() -> Self { + Self::from_config(&CrawlerConfig::default()) + } +} + +impl CrawlerSettings { + /// Derive the DTO from a runtime config (used to seed the DB from env). + pub fn from_config(c: &CrawlerConfig) -> Self { + Self { + daemon_enabled: c.daemon_enabled, + daily_at: c.daily_at.format("%H:%M").to_string(), + tz: c.tz.name().to_string(), + idle_timeout_secs: c.idle_timeout.as_secs(), + chapter_workers: c.chapter_workers as u64, + retention_days: c.retention_days, + start_url: c.start_url.clone(), + rate_ms: c.rate_ms, + cdn_host: c.cdn_host.clone(), + cdn_rate_ms: c.cdn_rate_ms, + cookie_domain: c.cookie_domain.clone(), + user_agent: c.user_agent.clone(), + download_allowlist: c.download_allowlist.hosts().to_vec(), + allow_any_host: c.download_allowlist.is_allow_any(), + max_image_bytes: c.max_image_bytes as u64, + manga_limit: c.manga_limit as u64, + job_timeout_secs: c.job_timeout.as_secs(), + metadata_max_consecutive_failures: c.metadata_max_consecutive_failures, + browser_restart_threshold: c.browser_restart_threshold, + } + } + + /// Overlay the DTO onto an env-derived `base` (which carries the env-only + /// fields: browser, proxy, TOR, session, cookie domain) to produce the + /// effective runtime config. Validates and returns all field errors at + /// once on failure. + pub fn to_config(&self, base: &CrawlerConfig) -> Result { + let mut errs = FieldErrors::default(); + + let daily_at = match NaiveTime::parse_from_str(&self.daily_at, "%H:%M") { + Ok(t) => t, + Err(_) => { + errs.push("daily_at", "must be HH:MM (24-hour)"); + base.daily_at + } + }; + let tz: Tz = match self.tz.parse() { + Ok(t) => t, + Err(_) => { + errs.push("tz", "must be a valid IANA timezone (e.g. UTC, Europe/Berlin)"); + base.tz + } + }; + if self.chapter_workers < 1 { + errs.push("chapter_workers", "must be at least 1"); + } + if let Some(url) = self.start_url.as_deref().map(str::trim).filter(|s| !s.is_empty()) { + if reqwest::Url::parse(url).is_err() { + errs.push("start_url", "must be a valid absolute URL"); + } + } + if self.job_timeout_secs < 1 { + errs.push("job_timeout_secs", "must be at least 1 second"); + } + + if !errs.is_empty() { + return Err(errs); + } + + let start_url = self + .start_url + .as_deref() + .map(str::trim) + .filter(|s| !s.is_empty()) + .map(str::to_string); + let cdn_host = self + .cdn_host + .as_deref() + .map(str::trim) + .filter(|s| !s.is_empty()) + .map(str::to_string); + + let download_allowlist = build_allowlist( + self.allow_any_host, + start_url.as_deref(), + cdn_host.as_deref(), + &self.download_allowlist, + ); + + Ok(CrawlerConfig { + daemon_enabled: self.daemon_enabled, + daily_at, + tz, + idle_timeout: Duration::from_secs(self.idle_timeout_secs), + chapter_workers: (self.chapter_workers as usize).max(1), + retention_days: self.retention_days, + start_url, + rate_ms: self.rate_ms, + cdn_host, + cdn_rate_ms: self.cdn_rate_ms, + user_agent: self + .user_agent + .as_deref() + .map(str::trim) + .filter(|s| !s.is_empty()) + .map(str::to_string), + cookie_domain: self + .cookie_domain + .as_deref() + .map(str::trim) + .filter(|s| !s.is_empty()) + .map(str::to_string), + download_allowlist, + max_image_bytes: self.max_image_bytes as usize, + manga_limit: self.manga_limit as usize, + job_timeout: Duration::from_secs(self.job_timeout_secs.max(1)), + metadata_max_consecutive_failures: self.metadata_max_consecutive_failures, + browser_restart_threshold: self.browser_restart_threshold, + // Env-only / structural fields preserved from the base. + phpsessid: base.phpsessid.clone(), + proxy: base.proxy.clone(), + tor_control_url: base.tor_control_url.clone(), + tor_control_password: base.tor_control_password.clone(), + tor_control_cookie_path: base.tor_control_cookie_path.clone(), + tor_recircuit_max_attempts: base.tor_recircuit_max_attempts, + browser: base.browser.clone(), + }) + } +} + +/// Rebuild a [`DownloadAllowlist`] from the DTO, always seeding the start-url +/// and CDN hosts (mirrors `config::build_download_allowlist`). +fn build_allowlist( + allow_any: bool, + start_url: Option<&str>, + cdn_host: Option<&str>, + extras: &[String], +) -> DownloadAllowlist { + if allow_any { + return DownloadAllowlist::allow_any(); + } + let mut allow = DownloadAllowlist::new(); + if let Some(url) = start_url { + if let Ok(parsed) = reqwest::Url::parse(url) { + if let Some(h) = parsed.host_str() { + allow = allow.allow(h); + } + } + } + if let Some(host) = cdn_host { + allow = allow.allow(host); + } + for h in extras { + let trimmed = h.trim(); + if !trimmed.is_empty() { + allow = allow.allow(trimmed); + } + } + allow +} + +// --------------------------------------------------------------------------- +// Analysis +// --------------------------------------------------------------------------- + +/// Admin-editable analysis-worker settings. The vision `api_key` is env-only +/// and never carried here. Prompts are `Option`: `None` means "use +/// the compiled default" (the UI's "reset to default"). +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(default)] +pub struct AnalysisSettings { + pub enabled: bool, + pub workers: u64, + pub endpoint: String, + pub model: String, + pub request_timeout_secs: u64, + pub job_timeout_secs: u64, + pub max_tokens: u32, + pub max_pixels: u32, + pub min_slice_height: u32, + pub slice_overlap: f64, + pub tall_aspect_threshold: f64, + pub max_slices: u64, + pub max_image_bytes: u64, + /// `"json_schema" | "json_object" | "none"`. + pub response_format: String, + pub frequency_penalty: f64, + pub temperature: f64, + pub system_prompt: Option, + pub ocr_prompt: Option, + pub grounding_prompt: Option, +} + +impl Default for AnalysisSettings { + fn default() -> Self { + Self::from_config(&AnalysisConfig::default()) + } +} + +impl AnalysisSettings { + pub fn from_config(c: &AnalysisConfig) -> Self { + // Capture a prompt only when it diverges from the compiled default, + // so an unmodified config seeds as `None` ("use default"). + let opt = |cur: &str, def: &str| (cur != def).then(|| cur.to_string()); + Self { + enabled: c.enabled, + workers: c.workers as u64, + endpoint: c.endpoint.clone(), + model: c.model.clone(), + request_timeout_secs: c.request_timeout.as_secs(), + job_timeout_secs: c.job_timeout.as_secs(), + max_tokens: c.max_tokens, + max_pixels: c.max_pixels, + min_slice_height: c.min_slice_height, + slice_overlap: c.slice_overlap, + tall_aspect_threshold: c.tall_aspect_threshold, + max_slices: c.max_slices as u64, + max_image_bytes: c.max_image_bytes as u64, + response_format: c.response_format.as_str().to_string(), + frequency_penalty: c.frequency_penalty, + temperature: c.temperature, + system_prompt: opt(&c.system_prompt, SYSTEM_PROMPT_DEFAULT), + ocr_prompt: opt(&c.ocr_prompt, OCR_PROMPT_DEFAULT), + grounding_prompt: opt(&c.grounding_prompt, GROUNDING_PROMPT_DEFAULT), + } + } + + pub fn to_config(&self, base: &AnalysisConfig) -> Result { + let mut errs = FieldErrors::default(); + + if self.workers < 1 { + errs.push("workers", "must be at least 1"); + } + if self.endpoint.trim().is_empty() || reqwest::Url::parse(self.endpoint.trim()).is_err() { + errs.push("endpoint", "must be a valid absolute URL"); + } + if self.enabled && self.model.trim().is_empty() { + errs.push("model", "required when analysis is enabled"); + } + if self.max_tokens < 1 { + errs.push("max_tokens", "must be at least 1"); + } + if self.request_timeout_secs < 1 { + errs.push("request_timeout_secs", "must be at least 1 second"); + } + if self.job_timeout_secs < 1 { + errs.push("job_timeout_secs", "must be at least 1 second"); + } + if self.max_pixels < 1 { + errs.push("max_pixels", "must be at least 1"); + } + if self.max_image_bytes < 1 { + errs.push("max_image_bytes", "must be greater than 0"); + } + if !(0.0..=0.9).contains(&self.slice_overlap) { + errs.push("slice_overlap", "must be between 0.0 and 0.9"); + } + if self.tall_aspect_threshold < 1.0 { + errs.push("tall_aspect_threshold", "must be at least 1.0"); + } + if self.min_slice_height < 1 { + errs.push("min_slice_height", "must be at least 1"); + } + if self.max_slices < 1 { + errs.push("max_slices", "must be at least 1"); + } + if self.temperature < 0.0 { + errs.push("temperature", "must be 0 or greater"); + } + let response_format = match ResponseFormat::parse_strict(&self.response_format) { + Some(rf) => rf, + None => { + errs.push( + "response_format", + "must be one of: json_schema, json_object, none", + ); + base.response_format + } + }; + + if !errs.is_empty() { + return Err(errs); + } + + let prompt = |o: &Option, def: &str| { + o.as_deref() + .map(str::trim) + .filter(|s| !s.is_empty()) + .unwrap_or(def) + .to_string() + }; + + Ok(AnalysisConfig { + enabled: self.enabled, + workers: (self.workers as usize).max(1), + endpoint: self.endpoint.trim().to_string(), + model: self.model.trim().to_string(), + request_timeout: Duration::from_secs(self.request_timeout_secs), + job_timeout: Duration::from_secs(self.job_timeout_secs), + max_tokens: self.max_tokens, + max_pixels: self.max_pixels, + min_slice_height: self.min_slice_height.max(1), + slice_overlap: self.slice_overlap.clamp(0.0, 0.9), + tall_aspect_threshold: self.tall_aspect_threshold.max(1.0), + max_slices: (self.max_slices as usize).max(1), + max_image_bytes: self.max_image_bytes as usize, + response_format, + frequency_penalty: self.frequency_penalty, + temperature: self.temperature.max(0.0), + system_prompt: prompt(&self.system_prompt, SYSTEM_PROMPT_DEFAULT), + ocr_prompt: prompt(&self.ocr_prompt, OCR_PROMPT_DEFAULT), + grounding_prompt: prompt(&self.grounding_prompt, GROUNDING_PROMPT_DEFAULT), + // Env-only secret preserved from the base. + api_key: base.api_key.clone(), + }) + } +} + +/// Prompt defaults, returned by the API so the UI can render placeholders and +/// implement "reset to default". +#[derive(Debug, Clone, Serialize)] +pub struct PromptDefaults { + pub system_prompt: &'static str, + pub ocr_prompt: &'static str, + pub grounding_prompt: &'static str, +} + +impl PromptDefaults { + pub fn get() -> Self { + Self { + system_prompt: SYSTEM_PROMPT_DEFAULT, + ocr_prompt: OCR_PROMPT_DEFAULT, + grounding_prompt: GROUNDING_PROMPT_DEFAULT, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // --- crawler round-trip & validation --- + + #[test] + fn crawler_round_trips_through_dto() { + let mut base = CrawlerConfig::default(); + base.start_url = Some("https://example.com/".to_string()); + base.tz = Tz::Europe__Berlin; + base.chapter_workers = 3; + base.cookie_domain = Some("example.com".to_string()); + let dto = CrawlerSettings::from_config(&base); + let back = dto.to_config(&base).expect("valid"); + assert_eq!(back.tz, Tz::Europe__Berlin); + assert_eq!(back.chapter_workers, 3); + assert_eq!(back.start_url.as_deref(), Some("https://example.com/")); + // cookie_domain is now an editable DTO field, round-tripping like the rest. + assert_eq!(back.cookie_domain.as_deref(), Some("example.com")); + assert_eq!(dto.daily_at, "00:00"); + } + + #[test] + fn crawler_cookie_domain_is_editable() { + let base = CrawlerConfig::default(); + let dto = CrawlerSettings { + cookie_domain: Some("new.example.org".to_string()), + ..CrawlerSettings::from_config(&base) + }; + assert_eq!( + dto.to_config(&base).unwrap().cookie_domain.as_deref(), + Some("new.example.org") + ); + } + + #[test] + fn crawler_overlay_preserves_env_only_fields() { + let mut base = CrawlerConfig::default(); + base.proxy = Some("socks5://127.0.0.1:9050".to_string()); + base.tor_control_password = Some("secret".to_string()); + base.phpsessid = Some("abc123".to_string()); + // A DTO that knows nothing about the env-only fields. + let dto = CrawlerSettings { + rate_ms: 2000, + ..CrawlerSettings::from_config(&base) + }; + let out = dto.to_config(&base).expect("valid"); + assert_eq!(out.rate_ms, 2000); + assert_eq!(out.proxy.as_deref(), Some("socks5://127.0.0.1:9050")); + assert_eq!(out.tor_control_password.as_deref(), Some("secret")); + assert_eq!(out.phpsessid.as_deref(), Some("abc123")); + } + + #[test] + fn crawler_rejects_bad_tz_and_time() { + let base = CrawlerConfig::default(); + let dto = CrawlerSettings { + tz: "Mars/Phobos".to_string(), + daily_at: "9am".to_string(), + ..CrawlerSettings::from_config(&base) + }; + let errs = dto.to_config(&base).unwrap_err(); + let fields: Vec<_> = errs.errors.iter().map(|e| e.field.as_str()).collect(); + assert!(fields.contains(&"tz")); + assert!(fields.contains(&"daily_at")); + } + + #[test] + fn crawler_allowlist_seeds_start_and_cdn_hosts() { + let base = CrawlerConfig::default(); + let dto = CrawlerSettings { + start_url: Some("https://catalog.example.com/".to_string()), + cdn_host: Some("cdn.example.com".to_string()), + download_allowlist: vec!["extra.example.net".to_string()], + allow_any_host: false, + ..CrawlerSettings::from_config(&base) + }; + let out = dto.to_config(&base).expect("valid"); + assert!(out.download_allowlist.contains("catalog.example.com")); + assert!(out.download_allowlist.contains("cdn.example.com")); + assert!(out.download_allowlist.contains("extra.example.net")); + assert!(!out.download_allowlist.is_allow_any()); + } + + #[test] + fn crawler_allow_any_host_bypasses_list() { + let base = CrawlerConfig::default(); + let dto = CrawlerSettings { + allow_any_host: true, + ..CrawlerSettings::from_config(&base) + }; + let out = dto.to_config(&base).expect("valid"); + assert!(out.download_allowlist.is_allow_any()); + } + + // --- analysis round-trip, validation & prompt defaults --- + + #[test] + fn analysis_round_trips_and_omits_default_prompts() { + let base = AnalysisConfig::default(); + let dto = AnalysisSettings::from_config(&base); + // Unmodified prompts seed as None ("use default"). + assert!(dto.system_prompt.is_none()); + assert!(dto.ocr_prompt.is_none()); + assert!(dto.grounding_prompt.is_none()); + assert_eq!(dto.response_format, "json_schema"); + let back = dto.to_config(&base).expect("valid"); + assert_eq!(back.system_prompt, SYSTEM_PROMPT_DEFAULT); + assert_eq!(back.response_format, ResponseFormat::JsonSchema); + } + + #[test] + fn analysis_captures_env_prompt_override() { + let mut base = AnalysisConfig::default(); + base.system_prompt = "custom env prompt".to_string(); + let dto = AnalysisSettings::from_config(&base); + assert_eq!(dto.system_prompt.as_deref(), Some("custom env prompt")); + } + + #[test] + fn analysis_prompt_override_and_reset() { + let base = AnalysisConfig::default(); + // Override applied. + let dto = AnalysisSettings { + system_prompt: Some("OVERRIDE".to_string()), + ..AnalysisSettings::from_config(&base) + }; + assert_eq!(dto.to_config(&base).unwrap().system_prompt, "OVERRIDE"); + // None / blank falls back to the compiled default. + let dto = AnalysisSettings { + system_prompt: Some(" ".to_string()), + ..AnalysisSettings::from_config(&base) + }; + assert_eq!(dto.to_config(&base).unwrap().system_prompt, SYSTEM_PROMPT_DEFAULT); + } + + #[test] + fn analysis_overlay_preserves_api_key() { + let mut base = AnalysisConfig::default(); + base.api_key = Some("sk-secret".to_string()); + let dto = AnalysisSettings::from_config(&base); + assert_eq!(dto.to_config(&base).unwrap().api_key.as_deref(), Some("sk-secret")); + } + + #[test] + fn analysis_rejects_out_of_range() { + let base = AnalysisConfig::default(); + let dto = AnalysisSettings { + workers: 0, + slice_overlap: 1.5, + tall_aspect_threshold: 0.5, + response_format: "yaml".to_string(), + endpoint: "not a url".to_string(), + request_timeout_secs: 0, + job_timeout_secs: 0, + max_pixels: 0, + max_image_bytes: 0, + ..AnalysisSettings::from_config(&base) + }; + let errs = dto.to_config(&base).unwrap_err(); + let fields: Vec<_> = errs.errors.iter().map(|e| e.field.as_str()).collect(); + for f in [ + "workers", + "slice_overlap", + "tall_aspect_threshold", + "response_format", + "endpoint", + "request_timeout_secs", + "job_timeout_secs", + "max_pixels", + "max_image_bytes", + ] { + assert!(fields.contains(&f), "missing error for {f}"); + } + } + + #[test] + fn analysis_requires_model_only_when_enabled() { + let base = AnalysisConfig::default(); + let disabled = AnalysisSettings { + enabled: false, + model: "".to_string(), + ..AnalysisSettings::from_config(&base) + }; + assert!(disabled.to_config(&base).is_ok()); + let enabled = AnalysisSettings { + enabled: true, + model: "".to_string(), + ..AnalysisSettings::from_config(&base) + }; + let errs = enabled.to_config(&base).unwrap_err(); + assert!(errs.errors.iter().any(|e| e.field == "model")); + } + + #[test] + fn dtos_serialize_to_json_and_back() { + let c = CrawlerSettings::default(); + let v = serde_json::to_value(&c).unwrap(); + let back: CrawlerSettings = serde_json::from_value(v).unwrap(); + assert_eq!(c, back); + + let a = AnalysisSettings::default(); + let v = serde_json::to_value(&a).unwrap(); + let back: AnalysisSettings = serde_json::from_value(v).unwrap(); + assert_eq!(a, back); + } +} diff --git a/backend/tests/api_admin_settings.rs b/backend/tests/api_admin_settings.rs new file mode 100644 index 0000000..cf92502 --- /dev/null +++ b/backend/tests/api_admin_settings.rs @@ -0,0 +1,272 @@ +//! Integration tests for the runtime-editable settings endpoints +//! (`/api/v1/admin/settings/{crawler,analysis}`): +//! +//! * the `RequireAdmin` gate, +//! * `GET` returns the editable DTO + the read-only env-managed view (and the +//! analysis prompt defaults), never the secret, +//! * `PUT` validates (422 with per-field details on bad input), persists, and +//! writes an `admin_audit` row, +//! * `PUT` invokes the daemon reloader with the converted config and moves the +//! analysis enable gate, +//! * the repo-level env→DB seed is idempotent. + +mod common; + +use axum::http::StatusCode; +use axum::Router; +use serde_json::json; +use sqlx::PgPool; +use tower::ServiceExt; +use uuid::Uuid; + +use mangalord::repo; + +async fn seed_admin(pool: &PgPool, app: &Router) -> (String, String, Uuid) { + let (username, cookie) = common::register_user(app).await; + let u = repo::user::find_by_username(pool, &username) + .await + .unwrap() + .unwrap(); + repo::user::set_is_admin_unchecked(pool, u.id, true).await.unwrap(); + (username, cookie, u.id) +} + +// ---- RequireAdmin gate ----------------------------------------------------- + +#[sqlx::test(migrations = "./migrations")] +async fn get_settings_requires_admin(pool: PgPool) { + let h = common::harness(pool); + let (_u, cookie) = common::register_user(&h.app).await; + let resp = h + .app + .oneshot(common::get_with_cookie("/api/v1/admin/settings/crawler", &cookie)) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::FORBIDDEN); +} + +#[sqlx::test(migrations = "./migrations")] +async fn get_settings_rejects_anonymous(pool: PgPool) { + let h = common::harness(pool); + // No cookie at all → not logged in. + let resp = h + .app + .oneshot(common::get("/api/v1/admin/settings/crawler")) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::UNAUTHORIZED); +} + +#[sqlx::test(migrations = "./migrations")] +async fn put_settings_requires_admin(pool: PgPool) { + let h = common::harness(pool); + let (_u, cookie) = common::register_user(&h.app).await; + let resp = h + .app + .oneshot(common::put_json_with_cookie( + "/api/v1/admin/settings/analysis", + json!({ "enabled": false }), + &cookie, + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::FORBIDDEN); +} + +// ---- GET shape ------------------------------------------------------------- + +#[sqlx::test(migrations = "./migrations")] +async fn get_crawler_returns_editable_and_env_only(pool: PgPool) { + let h = common::harness(pool.clone()); + let (_u, cookie, _id) = seed_admin(&pool, &h.app).await; + let resp = h + .app + .oneshot(common::get_with_cookie("/api/v1/admin/settings/crawler", &cookie)) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + let body = common::body_json(resp).await; + // Editable knobs present with their defaults. + assert_eq!(body["editable"]["daily_at"], "00:00"); + assert_eq!(body["editable"]["chapter_workers"], 1); + // Env-managed view present and read-only. + assert_eq!(body["env_only"]["browser_mode"], "headless"); + assert_eq!(body["env_only"]["session_configured"], false); +} + +#[sqlx::test(migrations = "./migrations")] +async fn get_analysis_returns_defaults_and_no_secret(pool: PgPool) { + let h = common::harness(pool.clone()); + let (_u, cookie, _id) = seed_admin(&pool, &h.app).await; + let resp = h + .app + .oneshot(common::get_with_cookie("/api/v1/admin/settings/analysis", &cookie)) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + let body = common::body_json(resp).await; + // Unmodified prompts seed as null ("use default"). + assert!(body["editable"]["system_prompt"].is_null()); + // Compiled defaults exposed for the UI placeholder / reset. + assert!(body["prompt_defaults"]["system_prompt"] + .as_str() + .unwrap() + .contains("manga")); + // The secret is never echoed; only a boolean indicator. + assert!(body["editable"].get("api_key").is_none()); + assert_eq!(body["env_only"]["api_key_configured"], false); +} + +// ---- PUT validation -------------------------------------------------------- + +#[sqlx::test(migrations = "./migrations")] +async fn put_crawler_invalid_tz_returns_422(pool: PgPool) { + let h = common::harness(pool.clone()); + let (_u, cookie, _id) = seed_admin(&pool, &h.app).await; + let resp = h + .app + .oneshot(common::put_json_with_cookie( + "/api/v1/admin/settings/crawler", + json!({ "tz": "Mars/Phobos", "daily_at": "9am" }), + &cookie, + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::UNPROCESSABLE_ENTITY); + let body = common::body_json(resp).await; + let fields: Vec<&str> = body["error"]["details"]["fields"] + .as_array() + .unwrap() + .iter() + .map(|e| e["field"].as_str().unwrap()) + .collect(); + assert!(fields.contains(&"tz")); + assert!(fields.contains(&"daily_at")); +} + +#[sqlx::test(migrations = "./migrations")] +async fn put_analysis_invalid_returns_422(pool: PgPool) { + let h = common::harness(pool.clone()); + let (_u, cookie, _id) = seed_admin(&pool, &h.app).await; + let resp = h + .app + .oneshot(common::put_json_with_cookie( + "/api/v1/admin/settings/analysis", + json!({ "workers": 0, "slice_overlap": 2.0, "endpoint": "nope" }), + &cookie, + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::UNPROCESSABLE_ENTITY); + let body = common::body_json(resp).await; + let fields: Vec<&str> = body["error"]["details"]["fields"] + .as_array() + .unwrap() + .iter() + .map(|e| e["field"].as_str().unwrap()) + .collect(); + assert!(fields.contains(&"workers")); + assert!(fields.contains(&"slice_overlap")); + assert!(fields.contains(&"endpoint")); +} + +// ---- PUT persistence + audit ---------------------------------------------- + +#[sqlx::test(migrations = "./migrations")] +async fn put_crawler_persists_and_audits(pool: PgPool) { + let h = common::harness(pool.clone()); + let (_u, cookie, admin_id) = seed_admin(&pool, &h.app).await; + + let resp = h + .app + .clone() + .oneshot(common::put_json_with_cookie( + "/api/v1/admin/settings/crawler", + json!({ "rate_ms": 2500, "chapter_workers": 4, "start_url": "https://example.com/" }), + &cookie, + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + let body = common::body_json(resp).await; + assert_eq!(body["editable"]["rate_ms"], 2500); + assert_eq!(body["editable"]["chapter_workers"], 4); + // Allowlist normalized to include the start-url host. + let allow: Vec<&str> = body["editable"]["download_allowlist"] + .as_array() + .unwrap() + .iter() + .map(|v| v.as_str().unwrap()) + .collect(); + assert!(allow.contains(&"example.com")); + + // A second GET reflects the persisted change. + let resp = h + .app + .clone() + .oneshot(common::get_with_cookie("/api/v1/admin/settings/crawler", &cookie)) + .await + .unwrap(); + let body = common::body_json(resp).await; + assert_eq!(body["editable"]["rate_ms"], 2500); + + // An audit row landed. + let (action, kind): (String, String) = sqlx::query_as( + "SELECT action, target_kind FROM admin_audit WHERE actor_user_id = $1 ORDER BY at DESC LIMIT 1", + ) + .bind(admin_id) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(action, "update_crawler_settings"); + assert_eq!(kind, "settings"); +} + +// ---- PUT triggers reload --------------------------------------------------- + +#[sqlx::test(migrations = "./migrations")] +async fn put_analysis_triggers_reload_and_flips_gate(pool: PgPool) { + let (h, reloader) = common::harness_with_settings_reloader(pool.clone()); + let (_u, cookie, _id) = seed_admin(&pool, &h.app).await; + + // Gate starts closed. + assert!(!reloader.runtime.analysis_enabled()); + + let resp = h + .app + .clone() + .oneshot(common::put_json_with_cookie( + "/api/v1/admin/settings/analysis", + json!({ "enabled": true, "model": "qwen2-vl", "temperature": 0.4 }), + &cookie, + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + + // The reloader was invoked with the converted config, and the shared gate + // flipped on — both without a restart. + assert!(reloader.runtime.analysis_enabled()); + let applied = reloader.analysis.lock().unwrap().clone().expect("reload called"); + assert!(applied.enabled); + assert_eq!(applied.model, "qwen2-vl"); + assert_eq!(applied.temperature, 0.4); +} + +// ---- repo-level seed ------------------------------------------------------- + +#[sqlx::test(migrations = "./migrations")] +async fn seed_if_absent_is_idempotent(pool: PgPool) { + let v1 = json!({ "rate_ms": 1000 }); + let v2 = json!({ "rate_ms": 9999 }); + // First seed inserts. + assert!(repo::app_settings::seed_if_absent(&pool, "crawler", &v1).await.unwrap()); + // Second seed is a no-op (row already present) and does not overwrite. + assert!(!repo::app_settings::seed_if_absent(&pool, "crawler", &v2).await.unwrap()); + let got = repo::app_settings::get(&pool, "crawler").await.unwrap().unwrap(); + assert_eq!(got["rate_ms"], 1000); + // upsert does overwrite. + repo::app_settings::upsert(&pool, "crawler", &v2).await.unwrap(); + let got = repo::app_settings::get(&pool, "crawler").await.unwrap().unwrap(); + assert_eq!(got["rate_ms"], 9999); +} diff --git a/backend/tests/common/mod.rs b/backend/tests/common/mod.rs index 7e9e622..abed40e 100644 --- a/backend/tests/common/mod.rs +++ b/backend/tests/common/mod.rs @@ -14,9 +14,9 @@ use sqlx::PgPool; use tempfile::TempDir; use tower::ServiceExt; -use mangalord::app::{router, AppState}; +use mangalord::app::{router, AppState, RuntimeControls}; use mangalord::auth::rate_limit::AuthRateLimiter; -use mangalord::config::{AuthConfig, UploadConfig}; +use mangalord::config::{AnalysisConfig, AuthConfig, CrawlerConfig, UploadConfig}; use mangalord::storage::{LocalStorage, PutByteStream, Storage, StorageError, StreamingFile}; use async_trait::async_trait; @@ -76,13 +76,15 @@ fn harness_with_auth_config( auth_limiter, // Default harness has no crawler daemon wired up; admin resync // handlers return 503 in this config. Tests that need a stub - // resync service swap it in via `harness_with_resync`. - resync: None, - crawler: None, + // resync service swap it in via `harness_with_resync`. No reloader, + // so settings still persist but spawn no daemon. + runtime: Arc::new(RuntimeControls::new(false)), + reloader: None, + crawler_base: CrawlerConfig::default(), + analysis_base: AnalysisConfig::default(), // Empty allowlist = CSRF check skipped. The CSRF-specific test // harness `harness_with_admin_origins` overrides this. admin_allowed_origins: Arc::new(Vec::new()), - analysis_enabled: false, analysis_events: Arc::new(mangalord::analysis::events::AnalysisEvents::new()), }; Harness { app: router(state), _storage_dir: storage_dir } @@ -148,6 +150,8 @@ pub fn harness_with_resync( ..AuthConfig::default() }; let auth_limiter = Arc::new(AuthRateLimiter::new(auth.rate_limit)); + let runtime = Arc::new(RuntimeControls::new(false)); + runtime.set_resync(Some(resync)); let state = AppState { db: pool, storage, @@ -157,10 +161,11 @@ pub fn harness_with_resync( max_file_bytes: 256 * 1024, }, auth_limiter, - resync: Some(resync), - crawler: None, + runtime, + reloader: None, + crawler_base: CrawlerConfig::default(), + analysis_base: AnalysisConfig::default(), admin_allowed_origins: Arc::new(Vec::new()), - analysis_enabled: false, analysis_events: Arc::new(mangalord::analysis::events::AnalysisEvents::new()), }; Harness { @@ -189,10 +194,11 @@ pub fn harness_with_analysis(pool: PgPool) -> Harness { max_file_bytes: 256 * 1024, }, auth_limiter, - resync: None, - crawler: None, + runtime: Arc::new(RuntimeControls::new(true)), + reloader: None, + crawler_base: CrawlerConfig::default(), + analysis_base: AnalysisConfig::default(), admin_allowed_origins: Arc::new(Vec::new()), - analysis_enabled: true, analysis_events: Arc::new(mangalord::analysis::events::AnalysisEvents::new()), }; Harness { @@ -201,6 +207,71 @@ pub fn harness_with_analysis(pool: PgPool) -> Harness { } } +/// A [`DaemonReloader`] stub that records the configs it was asked to apply +/// and flips the shared analysis gate, without spawning any real daemon. Lets +/// settings tests assert that a `PUT` triggers a reload with the converted +/// config (and that the analysis enable gate moves). +pub struct StubReloader { + pub runtime: Arc, + pub crawler: std::sync::Mutex>, + pub analysis: std::sync::Mutex>, +} + +#[async_trait] +impl mangalord::app::DaemonReloader for StubReloader { + async fn reload_crawler(&self, cfg: CrawlerConfig) -> anyhow::Result<()> { + *self.crawler.lock().unwrap() = Some(cfg); + Ok(()) + } + async fn reload_analysis(&self, cfg: AnalysisConfig) -> anyhow::Result<()> { + self.runtime.set_analysis_enabled(cfg.enabled); + *self.analysis.lock().unwrap() = Some(cfg); + Ok(()) + } +} + +/// Like [`harness`] but wires a [`StubReloader`] so the settings `PUT` +/// endpoints exercise the reload path. Returns the harness plus the shared +/// stub so the test can inspect what was applied. +pub fn harness_with_settings_reloader(pool: PgPool) -> (Harness, Arc) { + let storage_dir = tempfile::tempdir().expect("tempdir"); + let storage = Arc::new(LocalStorage::new(storage_dir.path())); + let auth = AuthConfig { + cookie_secure: false, + ..AuthConfig::default() + }; + let auth_limiter = Arc::new(AuthRateLimiter::new(auth.rate_limit)); + let runtime = Arc::new(RuntimeControls::new(false)); + let reloader = Arc::new(StubReloader { + runtime: Arc::clone(&runtime), + crawler: std::sync::Mutex::new(None), + analysis: std::sync::Mutex::new(None), + }); + let state = AppState { + db: pool, + storage, + auth, + upload: UploadConfig { + max_request_bytes: 4 * 1024 * 1024, + max_file_bytes: 256 * 1024, + }, + auth_limiter, + runtime, + reloader: Some(Arc::clone(&reloader) as Arc), + crawler_base: CrawlerConfig::default(), + analysis_base: AnalysisConfig::default(), + admin_allowed_origins: Arc::new(Vec::new()), + analysis_events: Arc::new(mangalord::analysis::events::AnalysisEvents::new()), + }; + ( + Harness { + app: router(state), + _storage_dir: storage_dir, + }, + reloader, + ) +} + /// Like [`harness`] but configures an admin CSRF allowlist so the /// `/admin/*` mutating endpoints reject cross-origin browser POSTs. /// Used by the admin CSRF integration tests. @@ -220,10 +291,11 @@ pub fn harness_with_admin_origins(pool: PgPool, origins: Vec) -> Harness max_file_bytes: 256 * 1024, }, auth_limiter, - resync: None, - crawler: None, + runtime: Arc::new(RuntimeControls::new(false)), + reloader: None, + crawler_base: CrawlerConfig::default(), + analysis_base: AnalysisConfig::default(), admin_allowed_origins: Arc::new(origins), - analysis_enabled: false, analysis_events: Arc::new(mangalord::analysis::events::AnalysisEvents::new()), }; Harness { diff --git a/frontend/package.json b/frontend/package.json index ec8c959..9ebeba7 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "mangalord-frontend", - "version": "0.79.1", + "version": "0.80.0", "private": true, "type": "module", "scripts": { diff --git a/frontend/src/lib/api/admin.ts b/frontend/src/lib/api/admin.ts index 3080946..778c235 100644 --- a/frontend/src/lib/api/admin.ts +++ b/frontend/src/lib/api/admin.ts @@ -556,3 +556,110 @@ export type AnalysisEvent = export function analysisStatusStreamUrl(): string { return apiUrl('/v1/admin/analysis/status/stream'); } + +// ---- runtime settings (crawler + analysis) --------------------------------- + +/** Operationally-safe, admin-editable crawler settings. Host/infra and + * session/secret fields are managed via environment and surfaced read-only + * in {@link CrawlerEnvOnly}. */ +export type CrawlerSettings = { + daemon_enabled: boolean; + daily_at: string; // "HH:MM" + tz: string; // IANA + idle_timeout_secs: number; + chapter_workers: number; + retention_days: number; + start_url: string | null; + rate_ms: number; + cdn_host: string | null; + cdn_rate_ms: number; + cookie_domain: string | null; + user_agent: string | null; + download_allowlist: string[]; + allow_any_host: boolean; + max_image_bytes: number; + manga_limit: number; + job_timeout_secs: number; + metadata_max_consecutive_failures: number; + browser_restart_threshold: number; +}; + +/** Read-only view of crawler fields managed via environment. */ +export type CrawlerEnvOnly = { + browser_mode: string; + browser_args: string[]; + proxy: string | null; + tor_control_url: string | null; + tor_credentials_configured: boolean; + session_configured: boolean; +}; + +export type CrawlerSettingsResponse = { + editable: CrawlerSettings; + env_only: CrawlerEnvOnly; +}; + +/** Admin-editable analysis settings. Prompts are `null` to mean "use the + * compiled default" (see {@link PromptDefaults}). The vision API key is + * env-only and never returned. */ +export type AnalysisSettings = { + enabled: boolean; + workers: number; + endpoint: string; + model: string; + request_timeout_secs: number; + job_timeout_secs: number; + max_tokens: number; + max_pixels: number; + min_slice_height: number; + slice_overlap: number; + tall_aspect_threshold: number; + max_slices: number; + max_image_bytes: number; + response_format: 'json_schema' | 'json_object' | 'none'; + frequency_penalty: number; + temperature: number; + system_prompt: string | null; + ocr_prompt: string | null; + grounding_prompt: string | null; +}; + +export type PromptDefaults = { + system_prompt: string; + ocr_prompt: string; + grounding_prompt: string; +}; + +export type AnalysisSettingsResponse = { + editable: AnalysisSettings; + env_only: { api_key_configured: boolean }; + prompt_defaults: PromptDefaults; +}; + +export async function getCrawlerSettings(): Promise { + return request('/v1/admin/settings/crawler'); +} + +export async function updateCrawlerSettings( + settings: CrawlerSettings +): Promise { + return request('/v1/admin/settings/crawler', { + method: 'PUT', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(settings) + }); +} + +export async function getAnalysisSettings(): Promise { + return request('/v1/admin/settings/analysis'); +} + +export async function updateAnalysisSettings( + settings: AnalysisSettings +): Promise { + return request('/v1/admin/settings/analysis', { + method: 'PUT', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(settings) + }); +} diff --git a/frontend/src/lib/api/client.ts b/frontend/src/lib/api/client.ts index c96414f..179f79d 100644 --- a/frontend/src/lib/api/client.ts +++ b/frontend/src/lib/api/client.ts @@ -25,14 +25,19 @@ export class ApiError extends Error { constructor( public readonly status: number, public readonly code: string, - message: string + message: string, + /** The error envelope's `details` payload, when present (e.g. the + * per-field `{ fields: [...] }` of a `validation_failed` response). */ + public readonly details?: unknown ) { super(message); this.name = 'ApiError'; } } -type ErrorEnvelope = { error?: { code?: unknown; message?: unknown } }; +type ErrorEnvelope = { + error?: { code?: unknown; message?: unknown; details?: unknown }; +}; /** * Optional hook fired the first moment `request()` observes a 401 on @@ -59,6 +64,7 @@ export async function request(path: string, init?: RequestInit): Promise { if (!res.ok) { let code = 'http_error'; let message = `${res.status} ${res.statusText}`; + let details: unknown; const ct = res.headers.get('content-type') ?? ''; try { if (ct.includes('application/json')) { @@ -70,6 +76,9 @@ export async function request(path: string, init?: RequestInit): Promise { if (typeof body.error.message === 'string' && body.error.message) { message = body.error.message; } + if (body.error.details !== undefined) { + details = body.error.details; + } } } else { const text = await res.text(); @@ -88,7 +97,7 @@ export async function request(path: string, init?: RequestInit): Promise { console.error('on401 hook threw:', e); } } - throw new ApiError(res.status, code, message); + throw new ApiError(res.status, code, message, details); } // Any empty body (not just 204) returns undefined — the manga-add // endpoint, for instance, signals create-vs-already-present via diff --git a/frontend/src/lib/api/settings.test.ts b/frontend/src/lib/api/settings.test.ts new file mode 100644 index 0000000..68e8cba --- /dev/null +++ b/frontend/src/lib/api/settings.test.ts @@ -0,0 +1,167 @@ +import { + describe, + it, + expect, + vi, + beforeEach, + afterEach, + type MockInstance +} from 'vitest'; +import { + getCrawlerSettings, + updateCrawlerSettings, + getAnalysisSettings, + updateAnalysisSettings, + type CrawlerSettings, + type AnalysisSettings +} from './admin'; +import { ApiError } from './client'; + +function ok(body: unknown, status = 200): Response { + return new Response(JSON.stringify(body), { + status, + headers: { 'content-type': 'application/json' } + }); +} + +const crawlerFixture: CrawlerSettings = { + daemon_enabled: true, + daily_at: '00:00', + tz: 'UTC', + idle_timeout_secs: 600, + chapter_workers: 1, + retention_days: 7, + start_url: null, + rate_ms: 1000, + cdn_host: null, + cdn_rate_ms: 1000, + cookie_domain: null, + user_agent: null, + download_allowlist: [], + allow_any_host: false, + max_image_bytes: 33554432, + manga_limit: 0, + job_timeout_secs: 600, + metadata_max_consecutive_failures: 10, + browser_restart_threshold: 3 +}; + +const analysisFixture: AnalysisSettings = { + enabled: false, + workers: 1, + endpoint: 'http://localhost:8000/v1/chat/completions', + model: '', + request_timeout_secs: 120, + job_timeout_secs: 600, + max_tokens: 4096, + max_pixels: 1000000, + min_slice_height: 640, + slice_overlap: 0.12, + tall_aspect_threshold: 1.6, + max_slices: 16, + max_image_bytes: 8388608, + response_format: 'json_schema', + frequency_penalty: 0.3, + temperature: 0, + system_prompt: null, + ocr_prompt: null, + grounding_prompt: null +}; + +describe('settings api client', () => { + let fetchSpy: MockInstance; + + beforeEach(() => { + fetchSpy = vi.spyOn(globalThis, 'fetch'); + }); + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('getCrawlerSettings GETs the crawler settings endpoint', async () => { + fetchSpy.mockResolvedValueOnce( + ok({ + editable: crawlerFixture, + env_only: { + browser_mode: 'headless', + browser_args: [], + proxy: null, + tor_control_url: null, + tor_credentials_configured: false, + session_configured: false + } + }) + ); + const r = await getCrawlerSettings(); + expect(r.editable.rate_ms).toBe(1000); + expect(r.env_only.browser_mode).toBe('headless'); + const url = fetchSpy.mock.calls[0][0] as string; + expect(url).toMatch(/\/v1\/admin\/settings\/crawler$/); + }); + + it('updateCrawlerSettings PUTs the body as JSON', async () => { + fetchSpy.mockResolvedValueOnce( + ok({ + editable: { ...crawlerFixture, rate_ms: 2000 }, + env_only: { + browser_mode: 'headless', + browser_args: [], + proxy: null, + tor_control_url: null, + tor_credentials_configured: false, + session_configured: false + } + }) + ); + const r = await updateCrawlerSettings({ ...crawlerFixture, rate_ms: 2000 }); + expect(r.editable.rate_ms).toBe(2000); + const init = fetchSpy.mock.calls[0][1]!; + expect(init.method).toBe('PUT'); + expect(JSON.parse(init.body as string).rate_ms).toBe(2000); + }); + + it('getAnalysisSettings returns prompt defaults and the no-secret indicator', async () => { + fetchSpy.mockResolvedValueOnce( + ok({ + editable: analysisFixture, + env_only: { api_key_configured: true }, + prompt_defaults: { + system_prompt: 'SYS', + ocr_prompt: 'OCR', + grounding_prompt: 'GND' + } + }) + ); + const r = await getAnalysisSettings(); + expect(r.editable.system_prompt).toBeNull(); + expect(r.env_only.api_key_configured).toBe(true); + expect(r.prompt_defaults.system_prompt).toBe('SYS'); + // The secret is never present on the editable DTO. + expect('api_key' in (r.editable as object)).toBe(false); + }); + + it('updateAnalysisSettings surfaces 422 field errors via ApiError.details', async () => { + fetchSpy.mockResolvedValueOnce( + new Response( + JSON.stringify({ + error: { + code: 'validation_failed', + message: 'invalid settings', + details: { fields: [{ field: 'workers', message: 'must be at least 1' }] } + } + }), + { status: 422, headers: { 'content-type': 'application/json' } } + ) + ); + try { + await updateAnalysisSettings({ ...analysisFixture, workers: 0 }); + expect.unreachable('should have thrown'); + } catch (e) { + expect(e).toBeInstanceOf(ApiError); + expect((e as ApiError).status).toBe(422); + expect((e as ApiError).code).toBe('validation_failed'); + const details = (e as ApiError).details as { fields: { field: string }[] }; + expect(details.fields[0].field).toBe('workers'); + } + }); +}); diff --git a/frontend/src/routes/admin/+layout.svelte b/frontend/src/routes/admin/+layout.svelte index d482017..48d8aa0 100644 --- a/frontend/src/routes/admin/+layout.svelte +++ b/frontend/src/routes/admin/+layout.svelte @@ -8,6 +8,7 @@ { href: '/admin/mangas', label: 'Mangas' }, { href: '/admin/crawler', label: 'Crawler' }, { href: '/admin/analysis', label: 'Analysis' }, + { href: '/admin/settings', label: 'Settings' }, { href: '/admin/system', label: 'System' } ]; diff --git a/frontend/src/routes/admin/settings/+page.svelte b/frontend/src/routes/admin/settings/+page.svelte new file mode 100644 index 0000000..3de6cbe --- /dev/null +++ b/frontend/src/routes/admin/settings/+page.svelte @@ -0,0 +1,664 @@ + + +

Settings

+

+ Edit crawler and analysis configuration. Saving applies immediately — the affected daemon is + gracefully restarted, so in-flight jobs are retried rather than lost. Host-level and secret + values stay environment-managed and are shown read-only. +

+ +
+ + +
+ +{#if loading} +

Loading…

+{:else if loadError} + +{:else} +
+ {#if tab === 'crawler' && crawler} + + {:else if tab === 'analysis' && analysis} + + {/if} +
+ +
+ + {#if success}

{success}

{/if} + {#if saveError}{/if} + + {#if tab === 'crawler' && crawler} +
+
+ Schedule + + + +
+ +
+ Source + + + + +
+ +
+ Rate limiting + + +
+ +
+ Workers & retention + + + +
+ +
+ Download safety + + + +
+ +
+ Reliability + + + +
+ + {#if crawlerEnv} +
+ Managed via environment (read-only) +
+
Browser mode
+
{crawlerEnv.browser_mode}
+
Browser args
+
{crawlerEnv.browser_args.join(' ') || '—'}
+
Proxy
+
{crawlerEnv.proxy ?? '—'}
+
TOR control URL
+
{crawlerEnv.tor_control_url ?? '—'}
+
TOR credentials
+
{crawlerEnv.tor_credentials_configured ? 'configured' : '—'}
+
Initial session
+
{crawlerEnv.session_configured ? 'configured' : '—'}
+
+
+ {/if} +
+ {:else if tab === 'analysis' && analysis} +
+
+ Endpoint & model + + +

+ API key: {apiKeyConfigured + ? 'configured via environment' + : 'not set'} (managed via environment). +

+
+ +
+ Workers & timeouts + + + +
+ +
+ Sampling & output + + + + +
+ +
+ Image slicing + + + + + + +
+ +
+ Prompts + {#each [['system_prompt', 'System prompt'], ['ocr_prompt', 'OCR prompt (slices)'], ['grounding_prompt', 'Grounding prompt']] as [key, label] (key)} + {@const k = key as 'system_prompt' | 'ocr_prompt' | 'grounding_prompt'} +
+
+ {label} + +
+ + + {analysis[k] == null + ? 'Using compiled default (shown as placeholder).' + : `${analysis[k]?.length ?? 0} chars (override).`} + +
+ {/each} +
+
+ {/if} +{/if} + + (confirmOpen = false)} + size="sm" + closeOnBackdrop + testid="settings-confirm" +> +

+ Saving restarts the {tab} subsystem with the new settings. In-flight jobs are leased and + will be retried, but any work in progress is interrupted briefly. +

+ {#snippet footer()} + + + {/snippet} + + +