feat(admin): runtime-editable crawler & analysis config in the dashboard
Move crawler and analysis configuration out of boot-only env vars and into
a DB-backed, admin-editable surface applied live without a restart.
- New `app_settings` table (migration 0026) holds one JSONB row per group;
env vars seed it on first boot, then the DB is the source of truth.
- `settings.rs` adds serializable Crawler/Analysis DTOs with env-base +
DB-overlay conversion and field-level validation; env-only/secret fields
(browser, proxy, TOR, vision API key, PHPSESSID) are never persisted.
- `GET/PUT /api/v1/admin/settings/{crawler,analysis}` (RequireAdmin,
cookie-only, CSRF-guarded): GET returns the editable DTO + a read-only
env-managed view + prompt defaults; PUT validates (422 + per-field
details), persists + audits in one tx, then gracefully respawns the
affected daemon via a Supervisor.
- AppState swaps the crawler control/resync handles and analysis enable
gate behind shared runtime cells so reloads take effect with no restart;
a boot-time spawn failure is logged rather than aborting startup.
- Make the three vision prompts and sampling temperature configurable
(defaults preserved); cookie_domain is admin-editable too.
- Frontend: tabbed /admin/settings page with grouped fieldsets, prompt
reset-to-default, env-managed read-only panel, save-&-apply confirm, and
per-field validation; typed client + ApiError.details.
- Bump version 0.79.1 -> 0.80.0.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
MechaCat02
22
.env.example
22
.env.example
@@ -167,3 +167,25 @@ BACKEND_URL=http://backend:8080
|
||||
# 25 Mbps; raise for users on slower upstream links or lower if a
|
||||
# tighter front proxy already bounds the request lifetime.
|
||||
BACKEND_PROXY_TIMEOUT_MS=300000
|
||||
|
||||
# ----- Runtime settings (crawler + analysis) -----
|
||||
# The crawler and analysis subsystems are configured at runtime from the
|
||||
# `app_settings` table and edited live in the admin dashboard
|
||||
# (Admin → Settings) — no restart needed. The CRAWLER_* / ANALYSIS_* env
|
||||
# vars below act as the BOOT SEED: on first boot (when the row is absent)
|
||||
# the env values populate the DB; thereafter the DB is the source of
|
||||
# truth and env changes are ignored for those fields.
|
||||
#
|
||||
# Host/infra and secret fields stay env-ONLY (never persisted, shown
|
||||
# read-only in the dashboard): the chromium binary/dir, browser mode/args,
|
||||
# CRAWLER_PROXY, all CRAWLER_TOR_CONTROL_*, the cookie domain, and the
|
||||
# analysis ANALYSIS_API_KEY. The important site levers (PRIVATE_MODE,
|
||||
# ALLOW_SELF_REGISTER) also remain env-only by design.
|
||||
#
|
||||
# New analysis knobs (all optional, all seed defaults):
|
||||
# ANALYSIS_TEMPERATURE Sampling temperature. Default 0 (deterministic).
|
||||
# ANALYSIS_SYSTEM_PROMPT Override the single-call vision system prompt.
|
||||
# ANALYSIS_OCR_PROMPT Override the tall-page OCR (pass A) prompt.
|
||||
# ANALYSIS_GROUNDING_PROMPT Override the tags/scene/safety (pass B) prompt.
|
||||
# Leave the prompt vars unset to use the built-in defaults (also editable,
|
||||
# with a per-prompt "reset to default", in the dashboard).
|
||||
|
||||
6
.gitignore
vendored
6
.gitignore
vendored
@@ -20,6 +20,12 @@
|
||||
.env
|
||||
.env.local
|
||||
.env.*.local
|
||||
.env.dev
|
||||
|
||||
# Node install / Playwright output when run from the repo root (the
|
||||
# canonical copies live under /frontend, already ignored above).
|
||||
/node_modules
|
||||
/test-results
|
||||
|
||||
# Claude Code (personal overrides only; .claude/settings.json is committed)
|
||||
.claude/settings.local.json
|
||||
|
||||
2
backend/Cargo.lock
generated
2
backend/Cargo.lock
generated
@@ -1517,7 +1517,7 @@ checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
|
||||
|
||||
[[package]]
|
||||
name = "mangalord"
|
||||
version = "0.79.1"
|
||||
version = "0.80.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"argon2",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "mangalord"
|
||||
version = "0.79.1"
|
||||
version = "0.80.0"
|
||||
edition = "2021"
|
||||
default-run = "mangalord"
|
||||
|
||||
|
||||
15
backend/migrations/0026_app_settings.sql
Normal file
15
backend/migrations/0026_app_settings.sql
Normal file
@@ -0,0 +1,15 @@
|
||||
-- Runtime-editable application settings, keyed by subsystem group.
|
||||
--
|
||||
-- Mirrors the small key-value `crawler_state` table (0015): one row per
|
||||
-- group ('crawler' | 'analysis'), the `value` JSONB holding a serialized
|
||||
-- settings DTO. Env vars seed a row on first boot (when absent); after that
|
||||
-- the DB row is the source of truth and the admin dashboard edits it live.
|
||||
--
|
||||
-- Only operationally-safe fields are stored here — host/infra and secrets
|
||||
-- (chromium paths, proxy, TOR control, vision API key, cookie domain) stay
|
||||
-- env-only and are never persisted.
|
||||
CREATE TABLE app_settings (
|
||||
key text PRIMARY KEY,
|
||||
value jsonb NOT NULL,
|
||||
updated_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
@@ -36,10 +36,12 @@ pub const MAX_SCENE_CHARS: usize = 1000;
|
||||
pub const MAX_OCR_PER_CALL: usize = 50;
|
||||
pub const MAX_TAGS_PER_CALL: usize = 12;
|
||||
|
||||
/// The system prompt. Self-contained: it carries the exact JSON schema so
|
||||
/// the same string drives both the model and (implicitly) the
|
||||
/// [`crate::domain::page_analysis::VisionAnalysis`] parser.
|
||||
pub const SYSTEM_PROMPT: &str = concat!(
|
||||
/// The default system prompt. Self-contained: it carries the exact JSON
|
||||
/// schema so the same string drives both the model and (implicitly) the
|
||||
/// [`crate::domain::page_analysis::VisionAnalysis`] parser. An admin can
|
||||
/// override it at runtime (see [`crate::config::AnalysisConfig::system_prompt`]);
|
||||
/// this const is the fallback and the "reset to default" target.
|
||||
pub const SYSTEM_PROMPT_DEFAULT: &str = concat!(
|
||||
"You are a manga/comic page analyzer. Look at the single page image and ",
|
||||
"return ONLY one minified JSON object — no prose, no markdown fences.\n",
|
||||
"Schema:\n",
|
||||
@@ -64,8 +66,9 @@ pub const SYSTEM_PROMPT: &str = concat!(
|
||||
// --- Two-pass prompts (long-page slicing) -----------------------------------
|
||||
|
||||
/// Pass A: OCR-only over a single vertical slice of a tall page. Kept
|
||||
/// narrow so each slice call is fast and never truncates.
|
||||
pub const OCR_PROMPT: &str = concat!(
|
||||
/// narrow so each slice call is fast and never truncates. Runtime-overridable
|
||||
/// default — see [`crate::config::AnalysisConfig::ocr_prompt`].
|
||||
pub const OCR_PROMPT_DEFAULT: &str = concat!(
|
||||
"You are an OCR engine for manga/comic pages. The image is one vertical ",
|
||||
"slice of a larger page. Transcribe EVERY visible text element verbatim ",
|
||||
"into ocr_results, each with its kind ",
|
||||
@@ -80,7 +83,9 @@ pub const OCR_PROMPT: &str = concat!(
|
||||
|
||||
/// Pass B: tags + scene + safety for the whole page, grounded in the merged
|
||||
/// OCR text (supplied as a separate user message part by the client).
|
||||
pub const GROUNDING_PROMPT: &str = concat!(
|
||||
/// Runtime-overridable default — see
|
||||
/// [`crate::config::AnalysisConfig::grounding_prompt`].
|
||||
pub const GROUNDING_PROMPT_DEFAULT: &str = concat!(
|
||||
"You are a manga/comic page analyzer. You are given the page image ",
|
||||
"(possibly downscaled) AND the OCR text already extracted from it. Using ",
|
||||
"both, return ONLY one minified JSON object with: tagging_results (5-10 ",
|
||||
@@ -230,7 +235,7 @@ mod tests {
|
||||
#[test]
|
||||
fn system_prompt_mentions_the_schema_keys() {
|
||||
for key in ["ocr_results", "tagging_results", "scene_description", "safety_flag"] {
|
||||
assert!(SYSTEM_PROMPT.contains(key), "prompt missing {key}");
|
||||
assert!(SYSTEM_PROMPT_DEFAULT.contains(key), "prompt missing {key}");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -37,6 +37,13 @@ pub struct VisionClient {
|
||||
max_tokens: u32,
|
||||
response_format: ResponseFormat,
|
||||
frequency_penalty: f64,
|
||||
temperature: f64,
|
||||
/// System prompt for the single-call (normal-aspect) path.
|
||||
system_prompt: String,
|
||||
/// Pass-A OCR-only prompt (tall-page slices).
|
||||
ocr_prompt: String,
|
||||
/// Pass-B grounding prompt (tags/scene/safety).
|
||||
grounding_prompt: String,
|
||||
slice: SliceParams,
|
||||
}
|
||||
|
||||
@@ -60,6 +67,10 @@ impl VisionClient {
|
||||
max_tokens: cfg.max_tokens,
|
||||
response_format: cfg.response_format,
|
||||
frequency_penalty: cfg.frequency_penalty,
|
||||
temperature: cfg.temperature,
|
||||
system_prompt: cfg.system_prompt.clone(),
|
||||
ocr_prompt: cfg.ocr_prompt.clone(),
|
||||
grounding_prompt: cfg.grounding_prompt.clone(),
|
||||
slice: SliceParams {
|
||||
max_pixels: cfg.max_pixels,
|
||||
min_slice_height: cfg.min_slice_height,
|
||||
@@ -83,6 +94,8 @@ impl VisionClient {
|
||||
&url,
|
||||
self.response_format,
|
||||
self.frequency_penalty,
|
||||
self.temperature,
|
||||
&self.system_prompt,
|
||||
);
|
||||
return parse_chat_completion(&self.post_chat(body).await?);
|
||||
};
|
||||
@@ -97,6 +110,8 @@ impl VisionClient {
|
||||
&data_url(&jpeg),
|
||||
self.response_format,
|
||||
self.frequency_penalty,
|
||||
self.temperature,
|
||||
&self.system_prompt,
|
||||
);
|
||||
parse_chat_completion(&self.post_chat(body).await?)
|
||||
}
|
||||
@@ -124,6 +139,8 @@ impl VisionClient {
|
||||
self.max_tokens,
|
||||
self.response_format,
|
||||
self.frequency_penalty,
|
||||
self.temperature,
|
||||
&self.ocr_prompt,
|
||||
&data_url(&jpeg),
|
||||
);
|
||||
let parsed = parse_chat_completion(&self.post_chat(body).await?)?;
|
||||
@@ -148,6 +165,8 @@ impl VisionClient {
|
||||
self.max_tokens,
|
||||
self.response_format,
|
||||
self.frequency_penalty,
|
||||
self.temperature,
|
||||
&self.grounding_prompt,
|
||||
&data_url(&whole),
|
||||
&ocr_text,
|
||||
);
|
||||
@@ -451,19 +470,23 @@ fn similar(a: &str, b: &str) -> bool {
|
||||
|
||||
/// Combined single-call body (OCR + tags + scene + safety). Public + this
|
||||
/// exact signature so the existing tests keep working.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn build_request_body(
|
||||
model: &str,
|
||||
max_tokens: u32,
|
||||
data_url: &str,
|
||||
response_format: ResponseFormat,
|
||||
frequency_penalty: f64,
|
||||
temperature: f64,
|
||||
system_prompt: &str,
|
||||
) -> serde_json::Value {
|
||||
build_body(
|
||||
model,
|
||||
max_tokens,
|
||||
response_format,
|
||||
frequency_penalty,
|
||||
prompt::SYSTEM_PROMPT,
|
||||
temperature,
|
||||
system_prompt,
|
||||
"page_analysis",
|
||||
prompt::output_json_schema(),
|
||||
data_url,
|
||||
@@ -472,11 +495,14 @@ pub fn build_request_body(
|
||||
}
|
||||
|
||||
/// Pass-A OCR-only body.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn build_ocr_body(
|
||||
model: &str,
|
||||
max_tokens: u32,
|
||||
response_format: ResponseFormat,
|
||||
frequency_penalty: f64,
|
||||
temperature: f64,
|
||||
ocr_prompt: &str,
|
||||
data_url: &str,
|
||||
) -> serde_json::Value {
|
||||
build_body(
|
||||
@@ -484,7 +510,8 @@ fn build_ocr_body(
|
||||
max_tokens,
|
||||
response_format,
|
||||
frequency_penalty,
|
||||
prompt::OCR_PROMPT,
|
||||
temperature,
|
||||
ocr_prompt,
|
||||
"page_ocr",
|
||||
prompt::ocr_json_schema(),
|
||||
data_url,
|
||||
@@ -493,11 +520,14 @@ fn build_ocr_body(
|
||||
}
|
||||
|
||||
/// Pass-B grounding body — whole image + the merged OCR text as context.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn build_grounding_body(
|
||||
model: &str,
|
||||
max_tokens: u32,
|
||||
response_format: ResponseFormat,
|
||||
frequency_penalty: f64,
|
||||
temperature: f64,
|
||||
grounding_prompt: &str,
|
||||
data_url: &str,
|
||||
ocr_text: &str,
|
||||
) -> serde_json::Value {
|
||||
@@ -507,7 +537,8 @@ fn build_grounding_body(
|
||||
max_tokens,
|
||||
response_format,
|
||||
frequency_penalty,
|
||||
prompt::GROUNDING_PROMPT,
|
||||
temperature,
|
||||
grounding_prompt,
|
||||
"page_grounding",
|
||||
prompt::grounding_json_schema(),
|
||||
data_url,
|
||||
@@ -524,6 +555,7 @@ fn build_body(
|
||||
max_tokens: u32,
|
||||
response_format: ResponseFormat,
|
||||
frequency_penalty: f64,
|
||||
temperature: f64,
|
||||
system_prompt: &str,
|
||||
schema_name: &str,
|
||||
schema: serde_json::Value,
|
||||
@@ -537,7 +569,7 @@ fn build_body(
|
||||
}
|
||||
let mut body = json!({
|
||||
"model": model,
|
||||
"temperature": 0,
|
||||
"temperature": temperature,
|
||||
"max_tokens": max_tokens,
|
||||
"messages": [
|
||||
{ "role": "system", "content": system_prompt },
|
||||
@@ -740,8 +772,15 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn combined_body_uses_full_schema_and_image() {
|
||||
let body =
|
||||
build_request_body("m", 100, "data:image/png;base64,AA", ResponseFormat::JsonSchema, 0.3);
|
||||
let body = build_request_body(
|
||||
"m",
|
||||
100,
|
||||
"data:image/png;base64,AA",
|
||||
ResponseFormat::JsonSchema,
|
||||
0.3,
|
||||
0.0,
|
||||
prompt::SYSTEM_PROMPT_DEFAULT,
|
||||
);
|
||||
assert_eq!(body["messages"][0]["role"], "system");
|
||||
assert_eq!(
|
||||
body["messages"][1]["content"][0]["image_url"]["url"],
|
||||
@@ -752,9 +791,25 @@ mod tests {
|
||||
assert!(req.as_array().unwrap().iter().any(|v| v == "safety_flag"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn body_carries_the_configured_prompt_and_temperature() {
|
||||
let body = build_request_body(
|
||||
"m",
|
||||
100,
|
||||
"d",
|
||||
ResponseFormat::None,
|
||||
0.0,
|
||||
0.7,
|
||||
"CUSTOM PROMPT",
|
||||
);
|
||||
assert_eq!(body["messages"][0]["content"], "CUSTOM PROMPT");
|
||||
assert_eq!(body["temperature"], 0.7);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ocr_body_carries_only_the_ocr_schema_and_caps() {
|
||||
let body = build_ocr_body("m", 100, ResponseFormat::JsonSchema, 0.3, "d");
|
||||
let body =
|
||||
build_ocr_body("m", 100, ResponseFormat::JsonSchema, 0.3, 0.0, prompt::OCR_PROMPT_DEFAULT, "d");
|
||||
let schema = &body["response_format"]["json_schema"]["schema"];
|
||||
assert_eq!(body["response_format"]["json_schema"]["name"], "page_ocr");
|
||||
assert_eq!(schema["required"], json!(["ocr_results"]));
|
||||
@@ -769,8 +824,16 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn grounding_body_includes_ocr_context_and_grounding_schema() {
|
||||
let body =
|
||||
build_grounding_body("m", 100, ResponseFormat::JsonSchema, 0.3, "d", "[speech] Hi");
|
||||
let body = build_grounding_body(
|
||||
"m",
|
||||
100,
|
||||
ResponseFormat::JsonSchema,
|
||||
0.3,
|
||||
0.0,
|
||||
prompt::GROUNDING_PROMPT_DEFAULT,
|
||||
"d",
|
||||
"[speech] Hi",
|
||||
);
|
||||
assert_eq!(body["response_format"]["json_schema"]["name"], "page_grounding");
|
||||
let parts = body["messages"][1]["content"].as_array().unwrap();
|
||||
assert_eq!(parts.len(), 2, "image + text parts");
|
||||
@@ -781,15 +844,25 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn response_format_none_omits_the_field() {
|
||||
let body = build_request_body("m", 100, "d", ResponseFormat::None, 0.3);
|
||||
let body = build_request_body(
|
||||
"m",
|
||||
100,
|
||||
"d",
|
||||
ResponseFormat::None,
|
||||
0.3,
|
||||
0.0,
|
||||
prompt::SYSTEM_PROMPT_DEFAULT,
|
||||
);
|
||||
assert!(body.get("response_format").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn frequency_penalty_included_only_when_nonzero() {
|
||||
let on = build_ocr_body("m", 100, ResponseFormat::None, 0.4, "d");
|
||||
let on =
|
||||
build_ocr_body("m", 100, ResponseFormat::None, 0.4, 0.0, prompt::OCR_PROMPT_DEFAULT, "d");
|
||||
assert_eq!(on["frequency_penalty"], 0.4);
|
||||
let off = build_ocr_body("m", 100, ResponseFormat::None, 0.0, "d");
|
||||
let off =
|
||||
build_ocr_body("m", 100, ResponseFormat::None, 0.0, 0.0, prompt::OCR_PROMPT_DEFAULT, "d");
|
||||
assert!(off.get("frequency_penalty").is_none());
|
||||
}
|
||||
|
||||
|
||||
@@ -188,7 +188,7 @@ pub struct AnalyzePageResponse {
|
||||
/// Reject the request with 503 when the analysis worker is disabled, so
|
||||
/// an operator doesn't pile up jobs that nothing will ever drain.
|
||||
fn ensure_enabled(state: &AppState) -> AppResult<()> {
|
||||
if state.analysis_enabled {
|
||||
if state.analysis_enabled() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(AppError::ServiceUnavailable(
|
||||
|
||||
@@ -37,13 +37,14 @@ pub(super) fn default_limit() -> i64 {
|
||||
}
|
||||
|
||||
/// Shared 503 helper: the daemon-only control + observability endpoints
|
||||
/// gate on `AppState.crawler == Some(_)`. Returns the same code and body
|
||||
/// across every caller so the frontend can rely on `service_unavailable`
|
||||
/// rather than each handler returning its own variant.
|
||||
/// gate on a running crawler daemon. Returns the same code and body across
|
||||
/// every caller so the frontend can rely on `service_unavailable` rather
|
||||
/// than each handler returning its own variant. Yields an owned `Arc` (the
|
||||
/// control handle is swapped on a config reload).
|
||||
pub(super) fn require_crawler(
|
||||
state: &AppState,
|
||||
) -> Result<&std::sync::Arc<CrawlerControl>, AppError> {
|
||||
state.crawler.as_ref().ok_or_else(|| {
|
||||
) -> Result<std::sync::Arc<CrawlerControl>, AppError> {
|
||||
state.crawler().ok_or_else(|| {
|
||||
AppError::ServiceUnavailable("crawler daemon is disabled".into())
|
||||
})
|
||||
}
|
||||
|
||||
@@ -140,7 +140,7 @@ async fn compose_status(
|
||||
dead,
|
||||
};
|
||||
|
||||
Ok(match state.crawler.as_ref() {
|
||||
Ok(match state.crawler().as_ref() {
|
||||
None => CrawlerStatusResponse {
|
||||
daemon: "disabled",
|
||||
phase: None,
|
||||
@@ -200,7 +200,7 @@ async fn stream_status(
|
||||
) -> Sse<impl Stream<Item = Result<Event, Infallible>>> {
|
||||
// Subscribe before the first emit so no change between the initial
|
||||
// snapshot and the first await is lost.
|
||||
let rx = state.crawler.as_ref().map(|c| c.status.subscribe());
|
||||
let rx = state.crawler().as_ref().map(|c| c.status.subscribe());
|
||||
let memo = QueueCountsMemo::new();
|
||||
|
||||
let stream = futures_util::stream::unfold(
|
||||
|
||||
@@ -8,6 +8,7 @@ pub mod analysis;
|
||||
pub mod crawler;
|
||||
pub mod mangas;
|
||||
pub mod resync;
|
||||
pub mod settings;
|
||||
pub mod system;
|
||||
pub mod users;
|
||||
|
||||
@@ -23,4 +24,5 @@ pub fn routes() -> Router<AppState> {
|
||||
.merge(system::routes())
|
||||
.merge(crawler::routes())
|
||||
.merge(analysis::routes())
|
||||
.merge(settings::routes())
|
||||
}
|
||||
|
||||
@@ -58,8 +58,7 @@ async fn resync_manga(
|
||||
return Err(AppError::NotFound);
|
||||
}
|
||||
let resync = state
|
||||
.resync
|
||||
.as_ref()
|
||||
.resync()
|
||||
.ok_or_else(|| AppError::ServiceUnavailable(
|
||||
"crawler daemon is disabled; force resync unavailable".into(),
|
||||
))?;
|
||||
@@ -96,8 +95,7 @@ async fn resync_chapter(
|
||||
Path(chapter_id): Path<Uuid>,
|
||||
) -> AppResult<Json<ChapterResyncResponse>> {
|
||||
let resync = state
|
||||
.resync
|
||||
.as_ref()
|
||||
.resync()
|
||||
.ok_or_else(|| AppError::ServiceUnavailable(
|
||||
"crawler daemon is disabled; force resync unavailable".into(),
|
||||
))?;
|
||||
|
||||
239
backend/src/api/admin/settings.rs
Normal file
239
backend/src/api/admin/settings.rs
Normal file
@@ -0,0 +1,239 @@
|
||||
//! Admin endpoints for runtime-editable crawler / analysis configuration.
|
||||
//!
|
||||
//! `GET` returns the current editable settings DTO plus a read-only view of
|
||||
//! the env-managed (host/infra + secret) fields, so the dashboard can show
|
||||
//! the full picture while only letting the operator edit the safe knobs.
|
||||
//!
|
||||
//! `PUT` validates the incoming DTO against the env-derived base (re-parsing
|
||||
//! timezone/time, rebuilding the download allowlist), persists a normalized
|
||||
//! DTO and an audit row in one transaction, then asks the [`DaemonReloader`]
|
||||
//! to gracefully respawn the affected daemon with the new config — so the
|
||||
//! change takes effect without a restart. Validation failures return 422 with
|
||||
//! per-field details (the established `validation_failed` envelope).
|
||||
//!
|
||||
//! `PUT` is a **full replace**, not a patch: the body is deserialized with
|
||||
//! field-level defaults, so any omitted field is reset to its compiled
|
||||
//! default rather than retaining the stored value. The dashboard always
|
||||
//! sends the complete DTO (it round-trips the one it loaded); programmatic
|
||||
//! callers must do the same.
|
||||
|
||||
use axum::extract::State;
|
||||
use axum::routing::get;
|
||||
use axum::{Json, Router};
|
||||
use serde::Serialize;
|
||||
|
||||
use crate::app::AppState;
|
||||
use crate::auth::extractor::RequireAdmin;
|
||||
use crate::config::CrawlerConfig;
|
||||
use crate::crawler::browser::BrowserMode;
|
||||
use crate::error::{AppError, AppResult};
|
||||
use crate::repo;
|
||||
use crate::settings::{
|
||||
AnalysisSettings, CrawlerSettings, FieldErrors, PromptDefaults, KEY_ANALYSIS, KEY_CRAWLER,
|
||||
};
|
||||
|
||||
pub fn routes() -> Router<AppState> {
|
||||
Router::new()
|
||||
.route(
|
||||
"/admin/settings/crawler",
|
||||
get(get_crawler).put(put_crawler),
|
||||
)
|
||||
.route(
|
||||
"/admin/settings/analysis",
|
||||
get(get_analysis).put(put_analysis),
|
||||
)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Response shapes
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Read-only mirror of the crawler fields managed via environment (host/infra
|
||||
/// + secrets), surfaced so the admin sees what's in effect without editing it.
|
||||
#[derive(Debug, Serialize)]
|
||||
struct CrawlerEnvOnly {
|
||||
browser_mode: &'static str,
|
||||
browser_args: Vec<String>,
|
||||
proxy: Option<String>,
|
||||
tor_control_url: Option<String>,
|
||||
/// Whether a TOR control credential (password or cookie file) is set.
|
||||
tor_credentials_configured: bool,
|
||||
/// Whether an initial `CRAWLER_PHPSESSID` is configured (the live session
|
||||
/// is managed separately via the crawler dashboard).
|
||||
session_configured: bool,
|
||||
}
|
||||
|
||||
impl CrawlerEnvOnly {
|
||||
fn from_base(base: &CrawlerConfig) -> Self {
|
||||
Self {
|
||||
browser_mode: match base.browser.mode {
|
||||
BrowserMode::Headed => "headed",
|
||||
BrowserMode::Headless => "headless",
|
||||
},
|
||||
browser_args: base.browser.extra_args.clone(),
|
||||
proxy: base.proxy.clone(),
|
||||
tor_control_url: base.tor_control_url.clone(),
|
||||
tor_credentials_configured: base.tor_control_password.is_some()
|
||||
|| base.tor_control_cookie_path.is_some(),
|
||||
session_configured: base.phpsessid.is_some(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct CrawlerSettingsResponse {
|
||||
editable: CrawlerSettings,
|
||||
env_only: CrawlerEnvOnly,
|
||||
}
|
||||
|
||||
/// Read-only mirror of analysis env-managed fields (just the secret).
|
||||
#[derive(Debug, Serialize)]
|
||||
struct AnalysisEnvOnly {
|
||||
api_key_configured: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct AnalysisSettingsResponse {
|
||||
editable: AnalysisSettings,
|
||||
env_only: AnalysisEnvOnly,
|
||||
/// The compiled prompt defaults so the UI can show placeholders and
|
||||
/// implement per-prompt "reset to default".
|
||||
prompt_defaults: PromptDefaults,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Handlers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn validation_error(errs: FieldErrors) -> AppError {
|
||||
AppError::ValidationFailed {
|
||||
message: "invalid settings".to_string(),
|
||||
details: serde_json::json!({ "fields": errs.errors }),
|
||||
}
|
||||
}
|
||||
|
||||
/// Current effective crawler settings: the stored DTO when present, otherwise
|
||||
/// derived from the env base (the row is seeded at boot, so the stored branch
|
||||
/// is the norm).
|
||||
async fn load_crawler_dto(state: &AppState) -> AppResult<CrawlerSettings> {
|
||||
Ok(match repo::app_settings::get(&state.db, KEY_CRAWLER).await? {
|
||||
Some(v) => serde_json::from_value(v)
|
||||
.unwrap_or_else(|_| CrawlerSettings::from_config(&state.crawler_base)),
|
||||
None => CrawlerSettings::from_config(&state.crawler_base),
|
||||
})
|
||||
}
|
||||
|
||||
async fn load_analysis_dto(state: &AppState) -> AppResult<AnalysisSettings> {
|
||||
Ok(match repo::app_settings::get(&state.db, KEY_ANALYSIS).await? {
|
||||
Some(v) => serde_json::from_value(v)
|
||||
.unwrap_or_else(|_| AnalysisSettings::from_config(&state.analysis_base)),
|
||||
None => AnalysisSettings::from_config(&state.analysis_base),
|
||||
})
|
||||
}
|
||||
|
||||
async fn get_crawler(
|
||||
State(state): State<AppState>,
|
||||
_admin: RequireAdmin,
|
||||
) -> AppResult<Json<CrawlerSettingsResponse>> {
|
||||
let editable = load_crawler_dto(&state).await?;
|
||||
Ok(Json(CrawlerSettingsResponse {
|
||||
editable,
|
||||
env_only: CrawlerEnvOnly::from_base(&state.crawler_base),
|
||||
}))
|
||||
}
|
||||
|
||||
async fn put_crawler(
|
||||
State(state): State<AppState>,
|
||||
admin: RequireAdmin,
|
||||
Json(incoming): Json<CrawlerSettings>,
|
||||
) -> AppResult<Json<CrawlerSettingsResponse>> {
|
||||
// Validate by converting against the env base; this also rebuilds the
|
||||
// download allowlist and re-parses tz / time.
|
||||
let cfg = incoming
|
||||
.to_config(&state.crawler_base)
|
||||
.map_err(validation_error)?;
|
||||
// Persist a normalized DTO (so the stored/returned allowlist reflects the
|
||||
// effective hosts, prompts are canonicalized, etc.).
|
||||
let normalized = CrawlerSettings::from_config(&cfg);
|
||||
let value = serde_json::to_value(&normalized).map_err(|e| AppError::Other(e.into()))?;
|
||||
|
||||
let mut tx = state.db.begin().await?;
|
||||
repo::app_settings::upsert(&mut *tx, KEY_CRAWLER, &value).await?;
|
||||
repo::admin_audit::insert(
|
||||
&mut *tx,
|
||||
admin.0.id,
|
||||
"update_crawler_settings",
|
||||
"settings",
|
||||
None,
|
||||
value.clone(),
|
||||
)
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
|
||||
// Apply live (graceful respawn) when a reloader is wired up.
|
||||
if let Some(reloader) = &state.reloader {
|
||||
reloader
|
||||
.reload_crawler(cfg)
|
||||
.await
|
||||
.map_err(AppError::Other)?;
|
||||
}
|
||||
|
||||
Ok(Json(CrawlerSettingsResponse {
|
||||
editable: normalized,
|
||||
env_only: CrawlerEnvOnly::from_base(&state.crawler_base),
|
||||
}))
|
||||
}
|
||||
|
||||
async fn get_analysis(
|
||||
State(state): State<AppState>,
|
||||
_admin: RequireAdmin,
|
||||
) -> AppResult<Json<AnalysisSettingsResponse>> {
|
||||
let editable = load_analysis_dto(&state).await?;
|
||||
Ok(Json(AnalysisSettingsResponse {
|
||||
editable,
|
||||
env_only: AnalysisEnvOnly {
|
||||
api_key_configured: state.analysis_base.api_key.is_some(),
|
||||
},
|
||||
prompt_defaults: PromptDefaults::get(),
|
||||
}))
|
||||
}
|
||||
|
||||
async fn put_analysis(
|
||||
State(state): State<AppState>,
|
||||
admin: RequireAdmin,
|
||||
Json(incoming): Json<AnalysisSettings>,
|
||||
) -> AppResult<Json<AnalysisSettingsResponse>> {
|
||||
let cfg = incoming
|
||||
.to_config(&state.analysis_base)
|
||||
.map_err(validation_error)?;
|
||||
let normalized = AnalysisSettings::from_config(&cfg);
|
||||
let value = serde_json::to_value(&normalized).map_err(|e| AppError::Other(e.into()))?;
|
||||
|
||||
let mut tx = state.db.begin().await?;
|
||||
repo::app_settings::upsert(&mut *tx, KEY_ANALYSIS, &value).await?;
|
||||
repo::admin_audit::insert(
|
||||
&mut *tx,
|
||||
admin.0.id,
|
||||
"update_analysis_settings",
|
||||
"settings",
|
||||
None,
|
||||
value.clone(),
|
||||
)
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
|
||||
if let Some(reloader) = &state.reloader {
|
||||
reloader
|
||||
.reload_analysis(cfg)
|
||||
.await
|
||||
.map_err(AppError::Other)?;
|
||||
}
|
||||
|
||||
Ok(Json(AnalysisSettingsResponse {
|
||||
editable: normalized,
|
||||
env_only: AnalysisEnvOnly {
|
||||
api_key_configured: state.analysis_base.api_key.is_some(),
|
||||
},
|
||||
prompt_defaults: PromptDefaults::get(),
|
||||
}))
|
||||
}
|
||||
@@ -161,7 +161,7 @@ async fn create(
|
||||
// rolled-back upload never leaves jobs pointing at nonexistent pages; a
|
||||
// failed enqueue is logged but doesn't fail the upload (the admin
|
||||
// re-enqueue endpoint can backfill).
|
||||
if state.analysis_enabled {
|
||||
if state.analysis_enabled() {
|
||||
for page_id in page_ids {
|
||||
if let Err(e) =
|
||||
repo::page_analysis::enqueue_for_page(&state.db, page_id, false).await
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
|
||||
use std::sync::RwLock as StdRwLock;
|
||||
|
||||
use anyhow::Context;
|
||||
use async_trait::async_trait;
|
||||
@@ -17,7 +18,7 @@ use tower_http::trace::TraceLayer;
|
||||
use crate::auth::extractor::CurrentUser;
|
||||
use crate::auth::rate_limit::AuthRateLimiter;
|
||||
use crate::error::AppError;
|
||||
use crate::config::{AuthConfig, Config, CrawlerConfig, UploadConfig};
|
||||
use crate::config::{AnalysisConfig, AuthConfig, Config, CrawlerConfig, UploadConfig};
|
||||
use crate::crawler::browser_manager::{self, BrowserManager};
|
||||
use crate::crawler::content::{self, SyncOutcome};
|
||||
use crate::crawler::daemon::{self, ChapterDispatcher, DaemonConfig, MetadataPass};
|
||||
@@ -40,33 +41,125 @@ pub struct AppState {
|
||||
/// One instance per AppState so tests stay isolated across the
|
||||
/// same process.
|
||||
pub auth_limiter: Arc<AuthRateLimiter>,
|
||||
/// Admin-triggered force resync. `None` when the crawler daemon
|
||||
/// is disabled (`CRAWLER_DAEMON=false`); admin handlers gate on
|
||||
/// `.is_some()` and return 503 otherwise. Set by [`build`] from the
|
||||
/// same wiring that builds the daemon's chapter dispatcher, so a
|
||||
/// force resync uses the daemon's BrowserManager + rate limiters.
|
||||
pub resync: Option<Arc<dyn ResyncService>>,
|
||||
/// Crawler observability + control handle (live status, coordinated
|
||||
/// browser restart, runtime session, manual run). `None` when the
|
||||
/// daemon is disabled; admin handlers gate on `.is_some()` → 503.
|
||||
pub crawler: Option<Arc<CrawlerControl>>,
|
||||
/// Runtime-swappable controls (crawler resync/control handles + the
|
||||
/// analysis enable gate). Shared with the [`Supervisors`] so a config
|
||||
/// reload that respawns a daemon updates what handlers see, live —
|
||||
/// without a restart. In tests this starts empty (no daemons).
|
||||
pub runtime: Arc<RuntimeControls>,
|
||||
/// Applies a persisted settings change to the running daemons
|
||||
/// (graceful stop + respawn). `Some` in production ([`build`]); `None`
|
||||
/// in the test harness, where settings still persist but no daemon is
|
||||
/// spawned. See [`DaemonReloader`].
|
||||
pub reloader: Option<Arc<dyn DaemonReloader>>,
|
||||
/// Env-derived crawler config — the **base** the stored settings DTO is
|
||||
/// overlaid onto (carries env-only fields: browser, proxy, TOR, session).
|
||||
pub crawler_base: CrawlerConfig,
|
||||
/// Env-derived analysis config base (carries the env-only vision API key).
|
||||
pub analysis_base: AnalysisConfig,
|
||||
/// Browser origins permitted to issue mutating requests to
|
||||
/// `/api/v1/admin/*`. See [`crate::config::Config::admin_allowed_origins`]
|
||||
/// for the policy. Cloned per-request into the CSRF middleware; the
|
||||
/// `Arc` keeps the clone cheap. Empty list → check is skipped
|
||||
/// (operator opt-out documented in `.env.example`).
|
||||
pub admin_allowed_origins: Arc<Vec<String>>,
|
||||
/// When `true`, page-create paths (upload + crawler) enqueue
|
||||
/// `analyze_page` jobs and the admin re-enqueue endpoints are active.
|
||||
/// Mirrors `config.analysis.enabled`; defaults to `false` so the
|
||||
/// feature is opt-in and the test harness inherits a no-op gate.
|
||||
pub analysis_enabled: bool,
|
||||
/// Live analysis-event broadcaster. Always present (the worker and the
|
||||
/// admin enqueue path publish here; the SSE endpoint subscribes). When
|
||||
/// the worker is disabled the channel simply stays quiet.
|
||||
pub analysis_events: Arc<crate::analysis::events::AnalysisEvents>,
|
||||
}
|
||||
|
||||
impl AppState {
|
||||
/// Current crawler control handle, or `None` when the daemon is stopped.
|
||||
pub fn crawler(&self) -> Option<Arc<CrawlerControl>> {
|
||||
self.runtime.crawler_control()
|
||||
}
|
||||
|
||||
/// Current force-resync service, or `None` when the daemon is stopped.
|
||||
pub fn resync(&self) -> Option<Arc<dyn ResyncService>> {
|
||||
self.runtime.resync()
|
||||
}
|
||||
|
||||
/// Whether AI page analysis is currently enabled (read live).
|
||||
pub fn analysis_enabled(&self) -> bool {
|
||||
self.runtime.analysis_enabled()
|
||||
}
|
||||
}
|
||||
|
||||
/// The crawler control handles, swapped atomically when the daemon is
|
||||
/// (re)spawned or stopped. Cloned cheaply on every read.
|
||||
#[derive(Clone, Default)]
|
||||
struct CrawlerControls {
|
||||
resync: Option<Arc<dyn ResyncService>>,
|
||||
control: Option<Arc<CrawlerControl>>,
|
||||
}
|
||||
|
||||
/// Shared, runtime-mutable surface read by handlers and mutated by the
|
||||
/// [`Supervisors`] on a config reload. The analysis gate is an `Arc<AtomicBool>`
|
||||
/// so the crawler's chapter dispatcher (which enqueues `analyze_page` jobs)
|
||||
/// sees toggles live without being respawned.
|
||||
pub struct RuntimeControls {
|
||||
crawler: StdRwLock<CrawlerControls>,
|
||||
analysis_enabled: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
impl RuntimeControls {
|
||||
/// Empty controls (no crawler daemon) with the given initial analysis gate.
|
||||
pub fn new(analysis_enabled: bool) -> Self {
|
||||
Self {
|
||||
crawler: StdRwLock::new(CrawlerControls::default()),
|
||||
analysis_enabled: Arc::new(AtomicBool::new(analysis_enabled)),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn analysis_enabled(&self) -> bool {
|
||||
self.analysis_enabled.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Flip the analysis gate. Used by the [`Supervisors`] on reload and by
|
||||
/// the test harness's stub reloader.
|
||||
pub fn set_analysis_enabled(&self, v: bool) {
|
||||
self.analysis_enabled.store(v, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// The shared gate handle, passed to the crawler chapter dispatcher.
|
||||
fn analysis_gate(&self) -> Arc<AtomicBool> {
|
||||
Arc::clone(&self.analysis_enabled)
|
||||
}
|
||||
|
||||
pub fn crawler_control(&self) -> Option<Arc<CrawlerControl>> {
|
||||
self.crawler.read().unwrap().control.clone()
|
||||
}
|
||||
|
||||
pub fn resync(&self) -> Option<Arc<dyn ResyncService>> {
|
||||
self.crawler.read().unwrap().resync.clone()
|
||||
}
|
||||
|
||||
/// Test helper: install a resync service without a running daemon.
|
||||
pub fn set_resync(&self, resync: Option<Arc<dyn ResyncService>>) {
|
||||
self.crawler.write().unwrap().resync = resync;
|
||||
}
|
||||
|
||||
fn set_crawler(
|
||||
&self,
|
||||
resync: Option<Arc<dyn ResyncService>>,
|
||||
control: Option<Arc<CrawlerControl>>,
|
||||
) {
|
||||
let mut g = self.crawler.write().unwrap();
|
||||
g.resync = resync;
|
||||
g.control = control;
|
||||
}
|
||||
}
|
||||
|
||||
/// Applies a validated settings change to the running daemons by gracefully
|
||||
/// stopping and respawning them with the new effective config (and updating
|
||||
/// the shared [`RuntimeControls`]). Implemented by [`Supervisors`] in
|
||||
/// production; the settings handlers call it after persisting.
|
||||
#[async_trait]
|
||||
pub trait DaemonReloader: Send + Sync {
|
||||
async fn reload_crawler(&self, cfg: CrawlerConfig) -> anyhow::Result<()>;
|
||||
async fn reload_analysis(&self, cfg: AnalysisConfig) -> anyhow::Result<()>;
|
||||
}
|
||||
|
||||
/// Shared handle the admin crawler endpoints use to observe and control
|
||||
/// the running daemon. Bundled so the handlers take one optional field on
|
||||
/// `AppState` rather than many.
|
||||
@@ -90,27 +183,96 @@ pub struct CrawlerControl {
|
||||
}
|
||||
|
||||
/// Bundle returned by [`build`]. The router is what `axum::serve` consumes;
|
||||
/// the daemon (when enabled) outlives the HTTP server and is awaited via
|
||||
/// [`AppHandle::shutdown`] after the listener has finished gracefully.
|
||||
/// the [`Supervisors`] own the background daemons (when enabled), outlive the
|
||||
/// HTTP server, and are awaited via [`AppHandle::shutdown`] after the listener
|
||||
/// has finished gracefully.
|
||||
pub struct AppHandle {
|
||||
pub router: Router,
|
||||
pub daemon: Option<daemon::DaemonHandle>,
|
||||
/// AI content-analysis worker daemon; `None` when `ANALYSIS_ENABLED`
|
||||
/// is off. Awaited alongside the crawler daemon on shutdown.
|
||||
pub analysis_daemon: Option<crate::analysis::daemon::AnalysisDaemonHandle>,
|
||||
pub supervisors: Arc<Supervisors>,
|
||||
}
|
||||
|
||||
impl AppHandle {
|
||||
pub async fn shutdown(self) {
|
||||
if let Some(d) = self.daemon {
|
||||
d.shutdown().await;
|
||||
self.supervisors.shutdown().await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Owns the crawler + analysis daemon handles and respawns them on a config
|
||||
/// reload. Holds the build-time inputs (db, storage, env-base configs, the
|
||||
/// shared event bus and [`RuntimeControls`]) needed to spawn from scratch.
|
||||
pub struct Supervisors {
|
||||
db: PgPool,
|
||||
storage: Arc<dyn Storage>,
|
||||
runtime: Arc<RuntimeControls>,
|
||||
analysis_events: Arc<crate::analysis::events::AnalysisEvents>,
|
||||
/// Serializes reloads + owns the live crawler daemon handle.
|
||||
crawler_handle: tokio::sync::Mutex<Option<daemon::DaemonHandle>>,
|
||||
/// Serializes reloads + owns the live analysis daemon handle.
|
||||
analysis_handle: tokio::sync::Mutex<Option<crate::analysis::daemon::AnalysisDaemonHandle>>,
|
||||
}
|
||||
|
||||
impl Supervisors {
|
||||
pub async fn shutdown(&self) {
|
||||
if let Some(h) = self.crawler_handle.lock().await.take() {
|
||||
h.shutdown().await;
|
||||
}
|
||||
if let Some(a) = self.analysis_daemon {
|
||||
a.shutdown().await;
|
||||
if let Some(h) = self.analysis_handle.lock().await.take() {
|
||||
h.shutdown().await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl DaemonReloader for Supervisors {
|
||||
async fn reload_crawler(&self, cfg: CrawlerConfig) -> anyhow::Result<()> {
|
||||
// Serialize reloads; do the (possibly slow) graceful shutdown under
|
||||
// the handle lock but outside the read-path RwLock so status reads
|
||||
// never block on a draining browser.
|
||||
let mut guard = self.crawler_handle.lock().await;
|
||||
if let Some(h) = guard.take() {
|
||||
h.shutdown().await;
|
||||
}
|
||||
self.runtime.set_crawler(None, None);
|
||||
if cfg.daemon_enabled {
|
||||
let spawned = spawn_crawler_daemon(
|
||||
self.db.clone(),
|
||||
Arc::clone(&self.storage),
|
||||
&cfg,
|
||||
self.runtime.analysis_gate(),
|
||||
)
|
||||
.await?;
|
||||
self.runtime
|
||||
.set_crawler(Some(spawned.resync), Some(spawned.crawler));
|
||||
*guard = Some(spawned.handle);
|
||||
tracing::info!("crawler daemon (re)started from settings");
|
||||
} else {
|
||||
tracing::info!("crawler daemon stopped (disabled in settings)");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn reload_analysis(&self, cfg: AnalysisConfig) -> anyhow::Result<()> {
|
||||
let mut guard = self.analysis_handle.lock().await;
|
||||
if let Some(h) = guard.take() {
|
||||
h.shutdown().await;
|
||||
}
|
||||
self.runtime.set_analysis_enabled(cfg.enabled);
|
||||
if cfg.enabled {
|
||||
let handle = spawn_analysis_daemon(
|
||||
self.db.clone(),
|
||||
Arc::clone(&self.storage),
|
||||
&cfg,
|
||||
Arc::clone(&self.analysis_events),
|
||||
)?;
|
||||
*guard = Some(handle);
|
||||
tracing::info!(model = %cfg.model, "analysis daemon (re)started from settings");
|
||||
} else {
|
||||
tracing::info!("analysis daemon stopped (disabled in settings)");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn build(config: Config) -> anyhow::Result<AppHandle> {
|
||||
let db = PgPoolOptions::new()
|
||||
.max_connections(10)
|
||||
@@ -127,60 +289,48 @@ pub async fn build(config: Config) -> anyhow::Result<AppHandle> {
|
||||
|
||||
let storage: Arc<dyn Storage> = Arc::new(LocalStorage::new(config.storage_dir.clone()));
|
||||
|
||||
let (daemon, resync, crawler) = if config.crawler.daemon_enabled {
|
||||
let spawned = spawn_crawler_daemon(
|
||||
db.clone(),
|
||||
Arc::clone(&storage),
|
||||
&config.crawler,
|
||||
config.analysis.enabled,
|
||||
)
|
||||
.await?;
|
||||
(Some(spawned.handle), Some(spawned.resync), Some(spawned.crawler))
|
||||
} else {
|
||||
tracing::info!("crawler daemon disabled (CRAWLER_DAEMON=false)");
|
||||
(None, None, None)
|
||||
};
|
||||
// Seed the settings rows from env on first boot, then load the effective
|
||||
// config (DB overlaid on the env base). The DB is the source of truth
|
||||
// after the first boot; env only fills a missing row.
|
||||
let crawler_cfg = load_effective_crawler(&db, &config.crawler).await?;
|
||||
let analysis_cfg = load_effective_analysis(&db, &config.analysis).await?;
|
||||
|
||||
// Live-event bus shared by the worker (progress) and the admin enqueue
|
||||
// path; the SSE endpoint subscribes. Created unconditionally so the
|
||||
// stream exists even with the worker disabled.
|
||||
let analysis_events = Arc::new(crate::analysis::events::AnalysisEvents::new());
|
||||
|
||||
// AI content-analysis worker. Independent of the crawler daemon (works
|
||||
// for uploads with the crawler off), gated on ANALYSIS_ENABLED. Uses a
|
||||
// plain reqwest client — no cookie jar / proxy, unlike the crawler's.
|
||||
let analysis_daemon = if config.analysis.enabled {
|
||||
let http = reqwest::Client::builder()
|
||||
.timeout(config.analysis.request_timeout)
|
||||
.build()
|
||||
.context("build analysis http client")?;
|
||||
let vision = crate::analysis::vision::VisionClient::new(http, &config.analysis);
|
||||
let dispatcher = Arc::new(crate::analysis::daemon::RealAnalyzeDispatcher {
|
||||
db: db.clone(),
|
||||
storage: Arc::clone(&storage),
|
||||
vision,
|
||||
model: config.analysis.model.clone(),
|
||||
max_image_bytes: config.analysis.max_image_bytes,
|
||||
});
|
||||
let handle = crate::analysis::daemon::spawn(
|
||||
db.clone(),
|
||||
tokio_util::sync::CancellationToken::new(),
|
||||
crate::analysis::daemon::AnalysisDaemonConfig {
|
||||
dispatcher,
|
||||
workers: config.analysis.workers,
|
||||
job_timeout: config.analysis.job_timeout,
|
||||
events: Arc::clone(&analysis_events),
|
||||
},
|
||||
);
|
||||
tracing::info!(
|
||||
workers = config.analysis.workers,
|
||||
model = %config.analysis.model,
|
||||
"analysis worker daemon started"
|
||||
);
|
||||
Some(handle)
|
||||
let runtime = Arc::new(RuntimeControls::new(analysis_cfg.enabled));
|
||||
let supervisors = Arc::new(Supervisors {
|
||||
db: db.clone(),
|
||||
storage: Arc::clone(&storage),
|
||||
runtime: Arc::clone(&runtime),
|
||||
analysis_events: Arc::clone(&analysis_events),
|
||||
crawler_handle: tokio::sync::Mutex::new(None),
|
||||
analysis_handle: tokio::sync::Mutex::new(None),
|
||||
});
|
||||
|
||||
// Initial spawn goes through the same reload path so there's one code
|
||||
// path for "bring the daemon up with this config". A spawn failure is
|
||||
// logged but does NOT abort startup: the persisted settings are the
|
||||
// source of truth and a bad value (e.g. a config that won't launch the
|
||||
// browser) must not wedge the whole server into a boot loop — the
|
||||
// server comes up with that daemon stopped so an admin can fix it via
|
||||
// the settings UI.
|
||||
if crawler_cfg.daemon_enabled {
|
||||
if let Err(e) = supervisors.reload_crawler(crawler_cfg).await {
|
||||
tracing::error!(?e, "crawler daemon failed to start; continuing with it stopped");
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
tracing::info!("crawler daemon disabled");
|
||||
}
|
||||
if analysis_cfg.enabled {
|
||||
if let Err(e) = supervisors.reload_analysis(analysis_cfg).await {
|
||||
tracing::error!(?e, "analysis worker failed to start; continuing with it stopped");
|
||||
}
|
||||
} else {
|
||||
tracing::info!("analysis worker disabled");
|
||||
}
|
||||
|
||||
let auth_limiter = Arc::new(AuthRateLimiter::new(config.auth.rate_limit));
|
||||
let state = AppState {
|
||||
@@ -189,14 +339,99 @@ pub async fn build(config: Config) -> anyhow::Result<AppHandle> {
|
||||
auth: config.auth.clone(),
|
||||
upload: config.upload.clone(),
|
||||
auth_limiter,
|
||||
resync,
|
||||
crawler,
|
||||
runtime,
|
||||
reloader: Some(Arc::clone(&supervisors) as Arc<dyn DaemonReloader>),
|
||||
crawler_base: config.crawler.clone(),
|
||||
analysis_base: config.analysis.clone(),
|
||||
admin_allowed_origins: Arc::new(config.admin_allowed_origins.clone()),
|
||||
analysis_enabled: config.analysis.enabled,
|
||||
analysis_events,
|
||||
};
|
||||
let router = router(state).layer(cors_layer(&config.cors_allowed_origins));
|
||||
Ok(AppHandle { router, daemon, analysis_daemon })
|
||||
Ok(AppHandle { router, supervisors })
|
||||
}
|
||||
|
||||
/// Seed the crawler settings row from env when absent, then return the
|
||||
/// effective [`CrawlerConfig`] (DB DTO overlaid on the env base). A stored
|
||||
/// DTO that fails to convert (corruption / drift) falls back to the env base
|
||||
/// rather than blocking startup.
|
||||
async fn load_effective_crawler(
|
||||
db: &PgPool,
|
||||
base: &CrawlerConfig,
|
||||
) -> anyhow::Result<CrawlerConfig> {
|
||||
use crate::settings::{CrawlerSettings, KEY_CRAWLER};
|
||||
let dto = match repo::app_settings::get(db, KEY_CRAWLER).await? {
|
||||
Some(v) => serde_json::from_value::<CrawlerSettings>(v)
|
||||
.unwrap_or_else(|_| CrawlerSettings::from_config(base)),
|
||||
None => {
|
||||
let dto = CrawlerSettings::from_config(base);
|
||||
repo::app_settings::seed_if_absent(db, KEY_CRAWLER, &serde_json::to_value(&dto)?)
|
||||
.await?;
|
||||
tracing::info!("seeded crawler settings from env");
|
||||
dto
|
||||
}
|
||||
};
|
||||
Ok(dto.to_config(base).unwrap_or_else(|e| {
|
||||
tracing::warn!(?e, "stored crawler settings invalid; using env base");
|
||||
base.clone()
|
||||
}))
|
||||
}
|
||||
|
||||
/// Analysis counterpart of [`load_effective_crawler`].
|
||||
async fn load_effective_analysis(
|
||||
db: &PgPool,
|
||||
base: &AnalysisConfig,
|
||||
) -> anyhow::Result<AnalysisConfig> {
|
||||
use crate::settings::{AnalysisSettings, KEY_ANALYSIS};
|
||||
let dto = match repo::app_settings::get(db, KEY_ANALYSIS).await? {
|
||||
Some(v) => serde_json::from_value::<AnalysisSettings>(v)
|
||||
.unwrap_or_else(|_| AnalysisSettings::from_config(base)),
|
||||
None => {
|
||||
let dto = AnalysisSettings::from_config(base);
|
||||
repo::app_settings::seed_if_absent(db, KEY_ANALYSIS, &serde_json::to_value(&dto)?)
|
||||
.await?;
|
||||
tracing::info!("seeded analysis settings from env");
|
||||
dto
|
||||
}
|
||||
};
|
||||
Ok(dto.to_config(base).unwrap_or_else(|e| {
|
||||
tracing::warn!(?e, "stored analysis settings invalid; using env base");
|
||||
base.clone()
|
||||
}))
|
||||
}
|
||||
|
||||
/// Spawn the AI content-analysis worker daemon with the given config. Returns
|
||||
/// its handle. Independent of the crawler daemon (works for uploads with the
|
||||
/// crawler off). Uses a plain reqwest client — no cookie jar / proxy.
|
||||
fn spawn_analysis_daemon(
|
||||
db: PgPool,
|
||||
storage: Arc<dyn Storage>,
|
||||
cfg: &AnalysisConfig,
|
||||
events: Arc<crate::analysis::events::AnalysisEvents>,
|
||||
) -> anyhow::Result<crate::analysis::daemon::AnalysisDaemonHandle> {
|
||||
let http = reqwest::Client::builder()
|
||||
.timeout(cfg.request_timeout)
|
||||
.build()
|
||||
.context("build analysis http client")?;
|
||||
let vision = crate::analysis::vision::VisionClient::new(http, cfg);
|
||||
let dispatcher = Arc::new(crate::analysis::daemon::RealAnalyzeDispatcher {
|
||||
db: db.clone(),
|
||||
storage,
|
||||
vision,
|
||||
model: cfg.model.clone(),
|
||||
max_image_bytes: cfg.max_image_bytes,
|
||||
});
|
||||
let handle = crate::analysis::daemon::spawn(
|
||||
db,
|
||||
CancellationToken::new(),
|
||||
crate::analysis::daemon::AnalysisDaemonConfig {
|
||||
dispatcher,
|
||||
workers: cfg.workers,
|
||||
job_timeout: cfg.job_timeout,
|
||||
events,
|
||||
},
|
||||
);
|
||||
tracing::info!(workers = cfg.workers, model = %cfg.model, "analysis worker daemon started");
|
||||
Ok(handle)
|
||||
}
|
||||
|
||||
/// Bundle returned by [`spawn_crawler_daemon`]. The handle owns the
|
||||
@@ -213,7 +448,7 @@ async fn spawn_crawler_daemon(
|
||||
db: PgPool,
|
||||
storage: Arc<dyn Storage>,
|
||||
cfg: &CrawlerConfig,
|
||||
analysis_enabled: bool,
|
||||
analysis_enabled: Arc<AtomicBool>,
|
||||
) -> anyhow::Result<SpawnedDaemon> {
|
||||
// Reqwest client with a shared cookie jar so CDN image fetches include
|
||||
// PHPSESSID. The same `Arc<Jar>` is held by the SessionController, so a
|
||||
@@ -512,9 +747,10 @@ struct RealChapterDispatcher {
|
||||
rate: Arc<HostRateLimiters>,
|
||||
download_allowlist: DownloadAllowlist,
|
||||
max_image_bytes: usize,
|
||||
/// Enqueue `analyze_page` jobs for freshly-crawled pages. Mirrors
|
||||
/// `config.analysis.enabled`.
|
||||
analysis_enabled: bool,
|
||||
/// Enqueue `analyze_page` jobs for freshly-crawled pages. Shared gate
|
||||
/// (read live) so toggling analysis at runtime takes effect without a
|
||||
/// crawler respawn. Mirrors the analysis enable setting.
|
||||
analysis_enabled: Arc<AtomicBool>,
|
||||
/// Consecutive transient chapter failures; resets on any success.
|
||||
/// Drives the automatic coordinated browser restart.
|
||||
transient_failures: Arc<std::sync::atomic::AtomicU32>,
|
||||
@@ -570,7 +806,7 @@ impl ChapterDispatcher for RealChapterDispatcher {
|
||||
self.max_image_bytes,
|
||||
self.tor.as_deref(),
|
||||
Some(&self.status),
|
||||
self.analysis_enabled,
|
||||
self.analysis_enabled.load(Ordering::Relaxed),
|
||||
)
|
||||
.await;
|
||||
drop(lease);
|
||||
|
||||
@@ -90,6 +90,27 @@ impl ResponseFormat {
|
||||
_ => ResponseFormat::JsonSchema,
|
||||
}
|
||||
}
|
||||
|
||||
/// Canonical wire string, the inverse of [`Self::from_str`] for the
|
||||
/// three modes the API exposes.
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
ResponseFormat::JsonSchema => "json_schema",
|
||||
ResponseFormat::JsonObject => "json_object",
|
||||
ResponseFormat::None => "none",
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse a wire value, rejecting unknown modes (stricter than
|
||||
/// [`Self::from_str`], which the env loader uses to stay lenient).
|
||||
pub fn parse_strict(s: &str) -> Option<ResponseFormat> {
|
||||
match s.trim().to_lowercase().as_str() {
|
||||
"json_schema" => Some(ResponseFormat::JsonSchema),
|
||||
"json_object" => Some(ResponseFormat::JsonObject),
|
||||
"none" => Some(ResponseFormat::None),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// AI content-analysis worker configuration: the enable gate, the local
|
||||
@@ -146,6 +167,21 @@ pub struct AnalysisConfig {
|
||||
/// the repetition loops that otherwise run small models into the token
|
||||
/// ceiling. `0` omits the field.
|
||||
pub frequency_penalty: f64,
|
||||
/// Sampling `temperature` sent with each request (`ANALYSIS_TEMPERATURE`).
|
||||
/// Defaults to `0.0` (deterministic), which is the most reliable for
|
||||
/// structured JSON output; some models behave better with a small
|
||||
/// positive value.
|
||||
pub temperature: f64,
|
||||
/// System prompt for the single-call (normal-aspect) analysis path
|
||||
/// (`ANALYSIS_SYSTEM_PROMPT`). Defaults to
|
||||
/// [`crate::analysis::prompt::SYSTEM_PROMPT_DEFAULT`].
|
||||
pub system_prompt: String,
|
||||
/// Pass-A OCR-only prompt for tall-page slices (`ANALYSIS_OCR_PROMPT`).
|
||||
/// Defaults to [`crate::analysis::prompt::OCR_PROMPT_DEFAULT`].
|
||||
pub ocr_prompt: String,
|
||||
/// Pass-B grounding prompt — tags/scene/safety (`ANALYSIS_GROUNDING_PROMPT`).
|
||||
/// Defaults to [`crate::analysis::prompt::GROUNDING_PROMPT_DEFAULT`].
|
||||
pub grounding_prompt: String,
|
||||
}
|
||||
|
||||
impl Default for AnalysisConfig {
|
||||
@@ -170,6 +206,10 @@ impl Default for AnalysisConfig {
|
||||
max_image_bytes: 8 * 1024 * 1024,
|
||||
response_format: ResponseFormat::JsonSchema,
|
||||
frequency_penalty: 0.3,
|
||||
temperature: 0.0,
|
||||
system_prompt: crate::analysis::prompt::SYSTEM_PROMPT_DEFAULT.to_string(),
|
||||
ocr_prompt: crate::analysis::prompt::OCR_PROMPT_DEFAULT.to_string(),
|
||||
grounding_prompt: crate::analysis::prompt::GROUNDING_PROMPT_DEFAULT.to_string(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -206,10 +246,23 @@ impl AnalysisConfig {
|
||||
.map(|s| ResponseFormat::from_str(&s))
|
||||
.unwrap_or(d.response_format),
|
||||
frequency_penalty: env_f64("ANALYSIS_FREQUENCY_PENALTY", d.frequency_penalty),
|
||||
temperature: env_f64("ANALYSIS_TEMPERATURE", d.temperature),
|
||||
system_prompt: env_prompt("ANALYSIS_SYSTEM_PROMPT", d.system_prompt),
|
||||
ocr_prompt: env_prompt("ANALYSIS_OCR_PROMPT", d.ocr_prompt),
|
||||
grounding_prompt: env_prompt("ANALYSIS_GROUNDING_PROMPT", d.grounding_prompt),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Read a prompt override from env, falling back to the default when unset
|
||||
/// or blank (so `ANALYSIS_SYSTEM_PROMPT=` doesn't wipe the prompt).
|
||||
fn env_prompt(name: &str, default: String) -> String {
|
||||
std::env::var(name)
|
||||
.ok()
|
||||
.filter(|s| !s.trim().is_empty())
|
||||
.unwrap_or(default)
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Config {
|
||||
pub database_url: String,
|
||||
|
||||
@@ -91,6 +91,16 @@ impl DownloadAllowlist {
|
||||
self.hosts.is_empty()
|
||||
}
|
||||
|
||||
/// The explicitly-allowed hosts (lowercased). Empty when `allow_any`.
|
||||
pub fn hosts(&self) -> &[String] {
|
||||
&self.hosts
|
||||
}
|
||||
|
||||
/// Whether the host check is bypassed entirely (`CRAWLER_ALLOW_ANY_HOST`).
|
||||
pub fn is_allow_any(&self) -> bool {
|
||||
self.allow_any
|
||||
}
|
||||
|
||||
pub fn contains(&self, host: &str) -> bool {
|
||||
if self.allow_any {
|
||||
return true;
|
||||
|
||||
@@ -7,5 +7,6 @@ pub mod crawler;
|
||||
pub mod domain;
|
||||
pub mod error;
|
||||
pub mod repo;
|
||||
pub mod settings;
|
||||
pub mod storage;
|
||||
pub mod upload;
|
||||
|
||||
@@ -21,11 +21,8 @@ async fn main() -> anyhow::Result<()> {
|
||||
|
||||
let config = mangalord::config::Config::from_env()?;
|
||||
let addr: SocketAddr = config.bind_address.parse()?;
|
||||
let mangalord::app::AppHandle {
|
||||
router,
|
||||
daemon,
|
||||
analysis_daemon,
|
||||
} = mangalord::app::build(config).await?;
|
||||
let mangalord::app::AppHandle { router, supervisors } =
|
||||
mangalord::app::build(config).await?;
|
||||
|
||||
tracing::info!(%addr, "mangalord listening");
|
||||
let listener = tokio::net::TcpListener::bind(addr).await?;
|
||||
@@ -33,30 +30,17 @@ async fn main() -> anyhow::Result<()> {
|
||||
.with_graceful_shutdown(shutdown_signal())
|
||||
.await?;
|
||||
|
||||
// Drain background tasks (crawler daemon) before exiting so Chromium
|
||||
// gets a clean shutdown rather than relying on kill-on-drop. Bounded
|
||||
// by a timeout so a wedged shutdown path can't trap the process.
|
||||
if let Some(d) = daemon {
|
||||
if tokio::time::timeout(CRAWLER_SHUTDOWN_TIMEOUT, d.shutdown())
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
tracing::warn!(
|
||||
timeout_s = CRAWLER_SHUTDOWN_TIMEOUT.as_secs(),
|
||||
"crawler daemon shutdown exceeded timeout; abandoning"
|
||||
);
|
||||
}
|
||||
}
|
||||
if let Some(a) = analysis_daemon {
|
||||
if tokio::time::timeout(CRAWLER_SHUTDOWN_TIMEOUT, a.shutdown())
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
tracing::warn!(
|
||||
timeout_s = CRAWLER_SHUTDOWN_TIMEOUT.as_secs(),
|
||||
"analysis daemon shutdown exceeded timeout; abandoning"
|
||||
);
|
||||
}
|
||||
// Drain background daemons (crawler + analysis) before exiting so
|
||||
// Chromium gets a clean shutdown rather than relying on kill-on-drop.
|
||||
// Bounded by a timeout so a wedged shutdown path can't trap the process.
|
||||
if tokio::time::timeout(CRAWLER_SHUTDOWN_TIMEOUT, supervisors.shutdown())
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
tracing::warn!(
|
||||
timeout_s = CRAWLER_SHUTDOWN_TIMEOUT.as_secs(),
|
||||
"daemon shutdown exceeded timeout; abandoning"
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
54
backend/src/repo/app_settings.rs
Normal file
54
backend/src/repo/app_settings.rs
Normal file
@@ -0,0 +1,54 @@
|
||||
//! Persistence for runtime-editable application settings (`app_settings`),
|
||||
//! a small key-value JSONB table (one row per subsystem group). Mirrors the
|
||||
//! `crawler_state` access style: plain async fns over `&PgPool`, the `value`
|
||||
//! a `serde_json::Value` the caller (de)serializes into a settings DTO.
|
||||
|
||||
use sqlx::{PgExecutor, PgPool};
|
||||
|
||||
/// Fetch the raw JSONB for a settings group, or `None` if unset.
|
||||
pub async fn get(pool: &PgPool, key: &str) -> sqlx::Result<Option<serde_json::Value>> {
|
||||
sqlx::query_scalar("SELECT value FROM app_settings WHERE key = $1")
|
||||
.bind(key)
|
||||
.fetch_optional(pool)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Insert-or-replace the JSONB for a settings group, stamping `updated_at`.
|
||||
/// Generic over the executor so it can run inside the same transaction as
|
||||
/// the matching admin-audit insert.
|
||||
pub async fn upsert<'e, E: PgExecutor<'e>>(
|
||||
executor: E,
|
||||
key: &str,
|
||||
value: &serde_json::Value,
|
||||
) -> sqlx::Result<()> {
|
||||
sqlx::query(
|
||||
"INSERT INTO app_settings (key, value, updated_at) \
|
||||
VALUES ($1, $2, now()) \
|
||||
ON CONFLICT (key) DO UPDATE \
|
||||
SET value = EXCLUDED.value, updated_at = now()",
|
||||
)
|
||||
.bind(key)
|
||||
.bind(value)
|
||||
.execute(executor)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Write the value only if the row is absent (the env → DB boot seed). Returns
|
||||
/// `true` when a row was inserted, `false` when one already existed. Uses
|
||||
/// `ON CONFLICT DO NOTHING` so a concurrent boot can't race two seeds.
|
||||
pub async fn seed_if_absent(
|
||||
pool: &PgPool,
|
||||
key: &str,
|
||||
value: &serde_json::Value,
|
||||
) -> sqlx::Result<bool> {
|
||||
let result = sqlx::query(
|
||||
"INSERT INTO app_settings (key, value) VALUES ($1, $2) \
|
||||
ON CONFLICT (key) DO NOTHING",
|
||||
)
|
||||
.bind(key)
|
||||
.bind(value)
|
||||
.execute(pool)
|
||||
.await?;
|
||||
Ok(result.rows_affected() > 0)
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
pub mod admin_audit;
|
||||
pub mod admin_view;
|
||||
pub mod api_token;
|
||||
pub mod app_settings;
|
||||
pub mod author;
|
||||
pub mod bookmark;
|
||||
pub mod chapter;
|
||||
|
||||
643
backend/src/settings.rs
Normal file
643
backend/src/settings.rs
Normal file
@@ -0,0 +1,643 @@
|
||||
//! Serializable, admin-editable settings DTOs for the crawler and analysis
|
||||
//! subsystems, and their conversion to/from the runtime [`CrawlerConfig`] /
|
||||
//! [`AnalysisConfig`].
|
||||
//!
|
||||
//! These DTOs are the boundary persisted in the `app_settings` table (one
|
||||
//! JSONB row per group) and exchanged over the admin API. They carry **only
|
||||
//! the operationally-safe, UI-editable fields** as primitive types — the
|
||||
//! runtime configs additionally hold env-only/structural fields (browser
|
||||
//! launch options, proxy, TOR control, the vision API key, the cookie
|
||||
//! domain) that are never persisted here.
|
||||
//!
|
||||
//! Flow: at boot, [`CrawlerSettings::from_config`] /
|
||||
//! [`AnalysisSettings::from_config`] derive the env-seed DTO; it is written to
|
||||
//! the DB only when the row is absent. Thereafter the DB row is the source of
|
||||
//! truth, and [`CrawlerSettings::to_config`] / [`AnalysisSettings::to_config`]
|
||||
//! overlay it onto the env-derived **base** (which carries the env-only
|
||||
//! fields) to produce the effective config the daemons are (re)spawned with.
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use chrono::NaiveTime;
|
||||
use chrono_tz::Tz;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::analysis::prompt::{
|
||||
GROUNDING_PROMPT_DEFAULT, OCR_PROMPT_DEFAULT, SYSTEM_PROMPT_DEFAULT,
|
||||
};
|
||||
use crate::config::{AnalysisConfig, CrawlerConfig, ResponseFormat};
|
||||
use crate::crawler::safety::DownloadAllowlist;
|
||||
|
||||
/// `app_settings.key` for the crawler group.
|
||||
pub const KEY_CRAWLER: &str = "crawler";
|
||||
/// `app_settings.key` for the analysis group.
|
||||
pub const KEY_ANALYSIS: &str = "analysis";
|
||||
|
||||
/// One field-level validation failure, surfaced to the UI per input.
|
||||
#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
|
||||
pub struct FieldError {
|
||||
pub field: String,
|
||||
pub message: String,
|
||||
}
|
||||
|
||||
/// Collected validation failures for a settings payload. Empty == valid.
|
||||
#[derive(Debug, Clone, Serialize, Default, PartialEq, Eq)]
|
||||
pub struct FieldErrors {
|
||||
pub errors: Vec<FieldError>,
|
||||
}
|
||||
|
||||
impl FieldErrors {
|
||||
fn push(&mut self, field: &str, message: impl Into<String>) {
|
||||
self.errors.push(FieldError {
|
||||
field: field.to_string(),
|
||||
message: message.into(),
|
||||
});
|
||||
}
|
||||
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.errors.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Crawler
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Admin-editable crawler-daemon settings. Mirrors the operational `CRAWLER_*`
|
||||
/// env vars; host/infra and session fields stay env-only.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(default)]
|
||||
pub struct CrawlerSettings {
|
||||
pub daemon_enabled: bool,
|
||||
/// Daily metadata-pass time, `"HH:MM"` (24h).
|
||||
pub daily_at: String,
|
||||
/// IANA timezone for the daily schedule.
|
||||
pub tz: String,
|
||||
pub idle_timeout_secs: u64,
|
||||
pub chapter_workers: u64,
|
||||
pub retention_days: u32,
|
||||
pub start_url: Option<String>,
|
||||
pub rate_ms: u64,
|
||||
pub cdn_host: Option<String>,
|
||||
pub cdn_rate_ms: u64,
|
||||
/// Domain the session cookie (PHPSESSID) is scoped to when injecting it
|
||||
/// into the browser / CDN requests. Pairs with `start_url`.
|
||||
pub cookie_domain: Option<String>,
|
||||
pub user_agent: Option<String>,
|
||||
/// Hosts the crawler may download images from (in addition to the
|
||||
/// auto-seeded start-url / CDN hosts). Ignored when `allow_any_host`.
|
||||
pub download_allowlist: Vec<String>,
|
||||
pub allow_any_host: bool,
|
||||
pub max_image_bytes: u64,
|
||||
/// Max manga detail fetches per metadata pass; `0` = unlimited.
|
||||
pub manga_limit: u64,
|
||||
pub job_timeout_secs: u64,
|
||||
pub metadata_max_consecutive_failures: u32,
|
||||
pub browser_restart_threshold: u32,
|
||||
}
|
||||
|
||||
impl Default for CrawlerSettings {
|
||||
fn default() -> Self {
|
||||
Self::from_config(&CrawlerConfig::default())
|
||||
}
|
||||
}
|
||||
|
||||
impl CrawlerSettings {
|
||||
/// Derive the DTO from a runtime config (used to seed the DB from env).
|
||||
pub fn from_config(c: &CrawlerConfig) -> Self {
|
||||
Self {
|
||||
daemon_enabled: c.daemon_enabled,
|
||||
daily_at: c.daily_at.format("%H:%M").to_string(),
|
||||
tz: c.tz.name().to_string(),
|
||||
idle_timeout_secs: c.idle_timeout.as_secs(),
|
||||
chapter_workers: c.chapter_workers as u64,
|
||||
retention_days: c.retention_days,
|
||||
start_url: c.start_url.clone(),
|
||||
rate_ms: c.rate_ms,
|
||||
cdn_host: c.cdn_host.clone(),
|
||||
cdn_rate_ms: c.cdn_rate_ms,
|
||||
cookie_domain: c.cookie_domain.clone(),
|
||||
user_agent: c.user_agent.clone(),
|
||||
download_allowlist: c.download_allowlist.hosts().to_vec(),
|
||||
allow_any_host: c.download_allowlist.is_allow_any(),
|
||||
max_image_bytes: c.max_image_bytes as u64,
|
||||
manga_limit: c.manga_limit as u64,
|
||||
job_timeout_secs: c.job_timeout.as_secs(),
|
||||
metadata_max_consecutive_failures: c.metadata_max_consecutive_failures,
|
||||
browser_restart_threshold: c.browser_restart_threshold,
|
||||
}
|
||||
}
|
||||
|
||||
/// Overlay the DTO onto an env-derived `base` (which carries the env-only
|
||||
/// fields: browser, proxy, TOR, session, cookie domain) to produce the
|
||||
/// effective runtime config. Validates and returns all field errors at
|
||||
/// once on failure.
|
||||
pub fn to_config(&self, base: &CrawlerConfig) -> Result<CrawlerConfig, FieldErrors> {
|
||||
let mut errs = FieldErrors::default();
|
||||
|
||||
let daily_at = match NaiveTime::parse_from_str(&self.daily_at, "%H:%M") {
|
||||
Ok(t) => t,
|
||||
Err(_) => {
|
||||
errs.push("daily_at", "must be HH:MM (24-hour)");
|
||||
base.daily_at
|
||||
}
|
||||
};
|
||||
let tz: Tz = match self.tz.parse() {
|
||||
Ok(t) => t,
|
||||
Err(_) => {
|
||||
errs.push("tz", "must be a valid IANA timezone (e.g. UTC, Europe/Berlin)");
|
||||
base.tz
|
||||
}
|
||||
};
|
||||
if self.chapter_workers < 1 {
|
||||
errs.push("chapter_workers", "must be at least 1");
|
||||
}
|
||||
if let Some(url) = self.start_url.as_deref().map(str::trim).filter(|s| !s.is_empty()) {
|
||||
if reqwest::Url::parse(url).is_err() {
|
||||
errs.push("start_url", "must be a valid absolute URL");
|
||||
}
|
||||
}
|
||||
if self.job_timeout_secs < 1 {
|
||||
errs.push("job_timeout_secs", "must be at least 1 second");
|
||||
}
|
||||
|
||||
if !errs.is_empty() {
|
||||
return Err(errs);
|
||||
}
|
||||
|
||||
let start_url = self
|
||||
.start_url
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|s| !s.is_empty())
|
||||
.map(str::to_string);
|
||||
let cdn_host = self
|
||||
.cdn_host
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|s| !s.is_empty())
|
||||
.map(str::to_string);
|
||||
|
||||
let download_allowlist = build_allowlist(
|
||||
self.allow_any_host,
|
||||
start_url.as_deref(),
|
||||
cdn_host.as_deref(),
|
||||
&self.download_allowlist,
|
||||
);
|
||||
|
||||
Ok(CrawlerConfig {
|
||||
daemon_enabled: self.daemon_enabled,
|
||||
daily_at,
|
||||
tz,
|
||||
idle_timeout: Duration::from_secs(self.idle_timeout_secs),
|
||||
chapter_workers: (self.chapter_workers as usize).max(1),
|
||||
retention_days: self.retention_days,
|
||||
start_url,
|
||||
rate_ms: self.rate_ms,
|
||||
cdn_host,
|
||||
cdn_rate_ms: self.cdn_rate_ms,
|
||||
user_agent: self
|
||||
.user_agent
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|s| !s.is_empty())
|
||||
.map(str::to_string),
|
||||
cookie_domain: self
|
||||
.cookie_domain
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|s| !s.is_empty())
|
||||
.map(str::to_string),
|
||||
download_allowlist,
|
||||
max_image_bytes: self.max_image_bytes as usize,
|
||||
manga_limit: self.manga_limit as usize,
|
||||
job_timeout: Duration::from_secs(self.job_timeout_secs.max(1)),
|
||||
metadata_max_consecutive_failures: self.metadata_max_consecutive_failures,
|
||||
browser_restart_threshold: self.browser_restart_threshold,
|
||||
// Env-only / structural fields preserved from the base.
|
||||
phpsessid: base.phpsessid.clone(),
|
||||
proxy: base.proxy.clone(),
|
||||
tor_control_url: base.tor_control_url.clone(),
|
||||
tor_control_password: base.tor_control_password.clone(),
|
||||
tor_control_cookie_path: base.tor_control_cookie_path.clone(),
|
||||
tor_recircuit_max_attempts: base.tor_recircuit_max_attempts,
|
||||
browser: base.browser.clone(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Rebuild a [`DownloadAllowlist`] from the DTO, always seeding the start-url
|
||||
/// and CDN hosts (mirrors `config::build_download_allowlist`).
|
||||
fn build_allowlist(
|
||||
allow_any: bool,
|
||||
start_url: Option<&str>,
|
||||
cdn_host: Option<&str>,
|
||||
extras: &[String],
|
||||
) -> DownloadAllowlist {
|
||||
if allow_any {
|
||||
return DownloadAllowlist::allow_any();
|
||||
}
|
||||
let mut allow = DownloadAllowlist::new();
|
||||
if let Some(url) = start_url {
|
||||
if let Ok(parsed) = reqwest::Url::parse(url) {
|
||||
if let Some(h) = parsed.host_str() {
|
||||
allow = allow.allow(h);
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(host) = cdn_host {
|
||||
allow = allow.allow(host);
|
||||
}
|
||||
for h in extras {
|
||||
let trimmed = h.trim();
|
||||
if !trimmed.is_empty() {
|
||||
allow = allow.allow(trimmed);
|
||||
}
|
||||
}
|
||||
allow
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Analysis
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Admin-editable analysis-worker settings. The vision `api_key` is env-only
|
||||
/// and never carried here. Prompts are `Option<String>`: `None` means "use
|
||||
/// the compiled default" (the UI's "reset to default").
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||
#[serde(default)]
|
||||
pub struct AnalysisSettings {
|
||||
pub enabled: bool,
|
||||
pub workers: u64,
|
||||
pub endpoint: String,
|
||||
pub model: String,
|
||||
pub request_timeout_secs: u64,
|
||||
pub job_timeout_secs: u64,
|
||||
pub max_tokens: u32,
|
||||
pub max_pixels: u32,
|
||||
pub min_slice_height: u32,
|
||||
pub slice_overlap: f64,
|
||||
pub tall_aspect_threshold: f64,
|
||||
pub max_slices: u64,
|
||||
pub max_image_bytes: u64,
|
||||
/// `"json_schema" | "json_object" | "none"`.
|
||||
pub response_format: String,
|
||||
pub frequency_penalty: f64,
|
||||
pub temperature: f64,
|
||||
pub system_prompt: Option<String>,
|
||||
pub ocr_prompt: Option<String>,
|
||||
pub grounding_prompt: Option<String>,
|
||||
}
|
||||
|
||||
impl Default for AnalysisSettings {
|
||||
fn default() -> Self {
|
||||
Self::from_config(&AnalysisConfig::default())
|
||||
}
|
||||
}
|
||||
|
||||
impl AnalysisSettings {
|
||||
pub fn from_config(c: &AnalysisConfig) -> Self {
|
||||
// Capture a prompt only when it diverges from the compiled default,
|
||||
// so an unmodified config seeds as `None` ("use default").
|
||||
let opt = |cur: &str, def: &str| (cur != def).then(|| cur.to_string());
|
||||
Self {
|
||||
enabled: c.enabled,
|
||||
workers: c.workers as u64,
|
||||
endpoint: c.endpoint.clone(),
|
||||
model: c.model.clone(),
|
||||
request_timeout_secs: c.request_timeout.as_secs(),
|
||||
job_timeout_secs: c.job_timeout.as_secs(),
|
||||
max_tokens: c.max_tokens,
|
||||
max_pixels: c.max_pixels,
|
||||
min_slice_height: c.min_slice_height,
|
||||
slice_overlap: c.slice_overlap,
|
||||
tall_aspect_threshold: c.tall_aspect_threshold,
|
||||
max_slices: c.max_slices as u64,
|
||||
max_image_bytes: c.max_image_bytes as u64,
|
||||
response_format: c.response_format.as_str().to_string(),
|
||||
frequency_penalty: c.frequency_penalty,
|
||||
temperature: c.temperature,
|
||||
system_prompt: opt(&c.system_prompt, SYSTEM_PROMPT_DEFAULT),
|
||||
ocr_prompt: opt(&c.ocr_prompt, OCR_PROMPT_DEFAULT),
|
||||
grounding_prompt: opt(&c.grounding_prompt, GROUNDING_PROMPT_DEFAULT),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn to_config(&self, base: &AnalysisConfig) -> Result<AnalysisConfig, FieldErrors> {
|
||||
let mut errs = FieldErrors::default();
|
||||
|
||||
if self.workers < 1 {
|
||||
errs.push("workers", "must be at least 1");
|
||||
}
|
||||
if self.endpoint.trim().is_empty() || reqwest::Url::parse(self.endpoint.trim()).is_err() {
|
||||
errs.push("endpoint", "must be a valid absolute URL");
|
||||
}
|
||||
if self.enabled && self.model.trim().is_empty() {
|
||||
errs.push("model", "required when analysis is enabled");
|
||||
}
|
||||
if self.max_tokens < 1 {
|
||||
errs.push("max_tokens", "must be at least 1");
|
||||
}
|
||||
if self.request_timeout_secs < 1 {
|
||||
errs.push("request_timeout_secs", "must be at least 1 second");
|
||||
}
|
||||
if self.job_timeout_secs < 1 {
|
||||
errs.push("job_timeout_secs", "must be at least 1 second");
|
||||
}
|
||||
if self.max_pixels < 1 {
|
||||
errs.push("max_pixels", "must be at least 1");
|
||||
}
|
||||
if self.max_image_bytes < 1 {
|
||||
errs.push("max_image_bytes", "must be greater than 0");
|
||||
}
|
||||
if !(0.0..=0.9).contains(&self.slice_overlap) {
|
||||
errs.push("slice_overlap", "must be between 0.0 and 0.9");
|
||||
}
|
||||
if self.tall_aspect_threshold < 1.0 {
|
||||
errs.push("tall_aspect_threshold", "must be at least 1.0");
|
||||
}
|
||||
if self.min_slice_height < 1 {
|
||||
errs.push("min_slice_height", "must be at least 1");
|
||||
}
|
||||
if self.max_slices < 1 {
|
||||
errs.push("max_slices", "must be at least 1");
|
||||
}
|
||||
if self.temperature < 0.0 {
|
||||
errs.push("temperature", "must be 0 or greater");
|
||||
}
|
||||
let response_format = match ResponseFormat::parse_strict(&self.response_format) {
|
||||
Some(rf) => rf,
|
||||
None => {
|
||||
errs.push(
|
||||
"response_format",
|
||||
"must be one of: json_schema, json_object, none",
|
||||
);
|
||||
base.response_format
|
||||
}
|
||||
};
|
||||
|
||||
if !errs.is_empty() {
|
||||
return Err(errs);
|
||||
}
|
||||
|
||||
let prompt = |o: &Option<String>, def: &str| {
|
||||
o.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|s| !s.is_empty())
|
||||
.unwrap_or(def)
|
||||
.to_string()
|
||||
};
|
||||
|
||||
Ok(AnalysisConfig {
|
||||
enabled: self.enabled,
|
||||
workers: (self.workers as usize).max(1),
|
||||
endpoint: self.endpoint.trim().to_string(),
|
||||
model: self.model.trim().to_string(),
|
||||
request_timeout: Duration::from_secs(self.request_timeout_secs),
|
||||
job_timeout: Duration::from_secs(self.job_timeout_secs),
|
||||
max_tokens: self.max_tokens,
|
||||
max_pixels: self.max_pixels,
|
||||
min_slice_height: self.min_slice_height.max(1),
|
||||
slice_overlap: self.slice_overlap.clamp(0.0, 0.9),
|
||||
tall_aspect_threshold: self.tall_aspect_threshold.max(1.0),
|
||||
max_slices: (self.max_slices as usize).max(1),
|
||||
max_image_bytes: self.max_image_bytes as usize,
|
||||
response_format,
|
||||
frequency_penalty: self.frequency_penalty,
|
||||
temperature: self.temperature.max(0.0),
|
||||
system_prompt: prompt(&self.system_prompt, SYSTEM_PROMPT_DEFAULT),
|
||||
ocr_prompt: prompt(&self.ocr_prompt, OCR_PROMPT_DEFAULT),
|
||||
grounding_prompt: prompt(&self.grounding_prompt, GROUNDING_PROMPT_DEFAULT),
|
||||
// Env-only secret preserved from the base.
|
||||
api_key: base.api_key.clone(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Prompt defaults, returned by the API so the UI can render placeholders and
|
||||
/// implement "reset to default".
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct PromptDefaults {
|
||||
pub system_prompt: &'static str,
|
||||
pub ocr_prompt: &'static str,
|
||||
pub grounding_prompt: &'static str,
|
||||
}
|
||||
|
||||
impl PromptDefaults {
|
||||
pub fn get() -> Self {
|
||||
Self {
|
||||
system_prompt: SYSTEM_PROMPT_DEFAULT,
|
||||
ocr_prompt: OCR_PROMPT_DEFAULT,
|
||||
grounding_prompt: GROUNDING_PROMPT_DEFAULT,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
// --- crawler round-trip & validation ---
|
||||
|
||||
#[test]
|
||||
fn crawler_round_trips_through_dto() {
|
||||
let mut base = CrawlerConfig::default();
|
||||
base.start_url = Some("https://example.com/".to_string());
|
||||
base.tz = Tz::Europe__Berlin;
|
||||
base.chapter_workers = 3;
|
||||
base.cookie_domain = Some("example.com".to_string());
|
||||
let dto = CrawlerSettings::from_config(&base);
|
||||
let back = dto.to_config(&base).expect("valid");
|
||||
assert_eq!(back.tz, Tz::Europe__Berlin);
|
||||
assert_eq!(back.chapter_workers, 3);
|
||||
assert_eq!(back.start_url.as_deref(), Some("https://example.com/"));
|
||||
// cookie_domain is now an editable DTO field, round-tripping like the rest.
|
||||
assert_eq!(back.cookie_domain.as_deref(), Some("example.com"));
|
||||
assert_eq!(dto.daily_at, "00:00");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn crawler_cookie_domain_is_editable() {
|
||||
let base = CrawlerConfig::default();
|
||||
let dto = CrawlerSettings {
|
||||
cookie_domain: Some("new.example.org".to_string()),
|
||||
..CrawlerSettings::from_config(&base)
|
||||
};
|
||||
assert_eq!(
|
||||
dto.to_config(&base).unwrap().cookie_domain.as_deref(),
|
||||
Some("new.example.org")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn crawler_overlay_preserves_env_only_fields() {
|
||||
let mut base = CrawlerConfig::default();
|
||||
base.proxy = Some("socks5://127.0.0.1:9050".to_string());
|
||||
base.tor_control_password = Some("secret".to_string());
|
||||
base.phpsessid = Some("abc123".to_string());
|
||||
// A DTO that knows nothing about the env-only fields.
|
||||
let dto = CrawlerSettings {
|
||||
rate_ms: 2000,
|
||||
..CrawlerSettings::from_config(&base)
|
||||
};
|
||||
let out = dto.to_config(&base).expect("valid");
|
||||
assert_eq!(out.rate_ms, 2000);
|
||||
assert_eq!(out.proxy.as_deref(), Some("socks5://127.0.0.1:9050"));
|
||||
assert_eq!(out.tor_control_password.as_deref(), Some("secret"));
|
||||
assert_eq!(out.phpsessid.as_deref(), Some("abc123"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn crawler_rejects_bad_tz_and_time() {
|
||||
let base = CrawlerConfig::default();
|
||||
let dto = CrawlerSettings {
|
||||
tz: "Mars/Phobos".to_string(),
|
||||
daily_at: "9am".to_string(),
|
||||
..CrawlerSettings::from_config(&base)
|
||||
};
|
||||
let errs = dto.to_config(&base).unwrap_err();
|
||||
let fields: Vec<_> = errs.errors.iter().map(|e| e.field.as_str()).collect();
|
||||
assert!(fields.contains(&"tz"));
|
||||
assert!(fields.contains(&"daily_at"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn crawler_allowlist_seeds_start_and_cdn_hosts() {
|
||||
let base = CrawlerConfig::default();
|
||||
let dto = CrawlerSettings {
|
||||
start_url: Some("https://catalog.example.com/".to_string()),
|
||||
cdn_host: Some("cdn.example.com".to_string()),
|
||||
download_allowlist: vec!["extra.example.net".to_string()],
|
||||
allow_any_host: false,
|
||||
..CrawlerSettings::from_config(&base)
|
||||
};
|
||||
let out = dto.to_config(&base).expect("valid");
|
||||
assert!(out.download_allowlist.contains("catalog.example.com"));
|
||||
assert!(out.download_allowlist.contains("cdn.example.com"));
|
||||
assert!(out.download_allowlist.contains("extra.example.net"));
|
||||
assert!(!out.download_allowlist.is_allow_any());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn crawler_allow_any_host_bypasses_list() {
|
||||
let base = CrawlerConfig::default();
|
||||
let dto = CrawlerSettings {
|
||||
allow_any_host: true,
|
||||
..CrawlerSettings::from_config(&base)
|
||||
};
|
||||
let out = dto.to_config(&base).expect("valid");
|
||||
assert!(out.download_allowlist.is_allow_any());
|
||||
}
|
||||
|
||||
// --- analysis round-trip, validation & prompt defaults ---
|
||||
|
||||
#[test]
|
||||
fn analysis_round_trips_and_omits_default_prompts() {
|
||||
let base = AnalysisConfig::default();
|
||||
let dto = AnalysisSettings::from_config(&base);
|
||||
// Unmodified prompts seed as None ("use default").
|
||||
assert!(dto.system_prompt.is_none());
|
||||
assert!(dto.ocr_prompt.is_none());
|
||||
assert!(dto.grounding_prompt.is_none());
|
||||
assert_eq!(dto.response_format, "json_schema");
|
||||
let back = dto.to_config(&base).expect("valid");
|
||||
assert_eq!(back.system_prompt, SYSTEM_PROMPT_DEFAULT);
|
||||
assert_eq!(back.response_format, ResponseFormat::JsonSchema);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn analysis_captures_env_prompt_override() {
|
||||
let mut base = AnalysisConfig::default();
|
||||
base.system_prompt = "custom env prompt".to_string();
|
||||
let dto = AnalysisSettings::from_config(&base);
|
||||
assert_eq!(dto.system_prompt.as_deref(), Some("custom env prompt"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn analysis_prompt_override_and_reset() {
|
||||
let base = AnalysisConfig::default();
|
||||
// Override applied.
|
||||
let dto = AnalysisSettings {
|
||||
system_prompt: Some("OVERRIDE".to_string()),
|
||||
..AnalysisSettings::from_config(&base)
|
||||
};
|
||||
assert_eq!(dto.to_config(&base).unwrap().system_prompt, "OVERRIDE");
|
||||
// None / blank falls back to the compiled default.
|
||||
let dto = AnalysisSettings {
|
||||
system_prompt: Some(" ".to_string()),
|
||||
..AnalysisSettings::from_config(&base)
|
||||
};
|
||||
assert_eq!(dto.to_config(&base).unwrap().system_prompt, SYSTEM_PROMPT_DEFAULT);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn analysis_overlay_preserves_api_key() {
|
||||
let mut base = AnalysisConfig::default();
|
||||
base.api_key = Some("sk-secret".to_string());
|
||||
let dto = AnalysisSettings::from_config(&base);
|
||||
assert_eq!(dto.to_config(&base).unwrap().api_key.as_deref(), Some("sk-secret"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn analysis_rejects_out_of_range() {
|
||||
let base = AnalysisConfig::default();
|
||||
let dto = AnalysisSettings {
|
||||
workers: 0,
|
||||
slice_overlap: 1.5,
|
||||
tall_aspect_threshold: 0.5,
|
||||
response_format: "yaml".to_string(),
|
||||
endpoint: "not a url".to_string(),
|
||||
request_timeout_secs: 0,
|
||||
job_timeout_secs: 0,
|
||||
max_pixels: 0,
|
||||
max_image_bytes: 0,
|
||||
..AnalysisSettings::from_config(&base)
|
||||
};
|
||||
let errs = dto.to_config(&base).unwrap_err();
|
||||
let fields: Vec<_> = errs.errors.iter().map(|e| e.field.as_str()).collect();
|
||||
for f in [
|
||||
"workers",
|
||||
"slice_overlap",
|
||||
"tall_aspect_threshold",
|
||||
"response_format",
|
||||
"endpoint",
|
||||
"request_timeout_secs",
|
||||
"job_timeout_secs",
|
||||
"max_pixels",
|
||||
"max_image_bytes",
|
||||
] {
|
||||
assert!(fields.contains(&f), "missing error for {f}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn analysis_requires_model_only_when_enabled() {
|
||||
let base = AnalysisConfig::default();
|
||||
let disabled = AnalysisSettings {
|
||||
enabled: false,
|
||||
model: "".to_string(),
|
||||
..AnalysisSettings::from_config(&base)
|
||||
};
|
||||
assert!(disabled.to_config(&base).is_ok());
|
||||
let enabled = AnalysisSettings {
|
||||
enabled: true,
|
||||
model: "".to_string(),
|
||||
..AnalysisSettings::from_config(&base)
|
||||
};
|
||||
let errs = enabled.to_config(&base).unwrap_err();
|
||||
assert!(errs.errors.iter().any(|e| e.field == "model"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dtos_serialize_to_json_and_back() {
|
||||
let c = CrawlerSettings::default();
|
||||
let v = serde_json::to_value(&c).unwrap();
|
||||
let back: CrawlerSettings = serde_json::from_value(v).unwrap();
|
||||
assert_eq!(c, back);
|
||||
|
||||
let a = AnalysisSettings::default();
|
||||
let v = serde_json::to_value(&a).unwrap();
|
||||
let back: AnalysisSettings = serde_json::from_value(v).unwrap();
|
||||
assert_eq!(a, back);
|
||||
}
|
||||
}
|
||||
272
backend/tests/api_admin_settings.rs
Normal file
272
backend/tests/api_admin_settings.rs
Normal file
@@ -0,0 +1,272 @@
|
||||
//! Integration tests for the runtime-editable settings endpoints
|
||||
//! (`/api/v1/admin/settings/{crawler,analysis}`):
|
||||
//!
|
||||
//! * the `RequireAdmin` gate,
|
||||
//! * `GET` returns the editable DTO + the read-only env-managed view (and the
|
||||
//! analysis prompt defaults), never the secret,
|
||||
//! * `PUT` validates (422 with per-field details on bad input), persists, and
|
||||
//! writes an `admin_audit` row,
|
||||
//! * `PUT` invokes the daemon reloader with the converted config and moves the
|
||||
//! analysis enable gate,
|
||||
//! * the repo-level env→DB seed is idempotent.
|
||||
|
||||
mod common;
|
||||
|
||||
use axum::http::StatusCode;
|
||||
use axum::Router;
|
||||
use serde_json::json;
|
||||
use sqlx::PgPool;
|
||||
use tower::ServiceExt;
|
||||
use uuid::Uuid;
|
||||
|
||||
use mangalord::repo;
|
||||
|
||||
async fn seed_admin(pool: &PgPool, app: &Router) -> (String, String, Uuid) {
|
||||
let (username, cookie) = common::register_user(app).await;
|
||||
let u = repo::user::find_by_username(pool, &username)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
repo::user::set_is_admin_unchecked(pool, u.id, true).await.unwrap();
|
||||
(username, cookie, u.id)
|
||||
}
|
||||
|
||||
// ---- RequireAdmin gate -----------------------------------------------------
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn get_settings_requires_admin(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
let (_u, cookie) = common::register_user(&h.app).await;
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::get_with_cookie("/api/v1/admin/settings/crawler", &cookie))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn get_settings_rejects_anonymous(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
// No cookie at all → not logged in.
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::get("/api/v1/admin/settings/crawler"))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::UNAUTHORIZED);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn put_settings_requires_admin(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
let (_u, cookie) = common::register_user(&h.app).await;
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::put_json_with_cookie(
|
||||
"/api/v1/admin/settings/analysis",
|
||||
json!({ "enabled": false }),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
|
||||
}
|
||||
|
||||
// ---- GET shape -------------------------------------------------------------
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn get_crawler_returns_editable_and_env_only(pool: PgPool) {
|
||||
let h = common::harness(pool.clone());
|
||||
let (_u, cookie, _id) = seed_admin(&pool, &h.app).await;
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::get_with_cookie("/api/v1/admin/settings/crawler", &cookie))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = common::body_json(resp).await;
|
||||
// Editable knobs present with their defaults.
|
||||
assert_eq!(body["editable"]["daily_at"], "00:00");
|
||||
assert_eq!(body["editable"]["chapter_workers"], 1);
|
||||
// Env-managed view present and read-only.
|
||||
assert_eq!(body["env_only"]["browser_mode"], "headless");
|
||||
assert_eq!(body["env_only"]["session_configured"], false);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn get_analysis_returns_defaults_and_no_secret(pool: PgPool) {
|
||||
let h = common::harness(pool.clone());
|
||||
let (_u, cookie, _id) = seed_admin(&pool, &h.app).await;
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::get_with_cookie("/api/v1/admin/settings/analysis", &cookie))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = common::body_json(resp).await;
|
||||
// Unmodified prompts seed as null ("use default").
|
||||
assert!(body["editable"]["system_prompt"].is_null());
|
||||
// Compiled defaults exposed for the UI placeholder / reset.
|
||||
assert!(body["prompt_defaults"]["system_prompt"]
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.contains("manga"));
|
||||
// The secret is never echoed; only a boolean indicator.
|
||||
assert!(body["editable"].get("api_key").is_none());
|
||||
assert_eq!(body["env_only"]["api_key_configured"], false);
|
||||
}
|
||||
|
||||
// ---- PUT validation --------------------------------------------------------
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn put_crawler_invalid_tz_returns_422(pool: PgPool) {
|
||||
let h = common::harness(pool.clone());
|
||||
let (_u, cookie, _id) = seed_admin(&pool, &h.app).await;
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::put_json_with_cookie(
|
||||
"/api/v1/admin/settings/crawler",
|
||||
json!({ "tz": "Mars/Phobos", "daily_at": "9am" }),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::UNPROCESSABLE_ENTITY);
|
||||
let body = common::body_json(resp).await;
|
||||
let fields: Vec<&str> = body["error"]["details"]["fields"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|e| e["field"].as_str().unwrap())
|
||||
.collect();
|
||||
assert!(fields.contains(&"tz"));
|
||||
assert!(fields.contains(&"daily_at"));
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn put_analysis_invalid_returns_422(pool: PgPool) {
|
||||
let h = common::harness(pool.clone());
|
||||
let (_u, cookie, _id) = seed_admin(&pool, &h.app).await;
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::put_json_with_cookie(
|
||||
"/api/v1/admin/settings/analysis",
|
||||
json!({ "workers": 0, "slice_overlap": 2.0, "endpoint": "nope" }),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::UNPROCESSABLE_ENTITY);
|
||||
let body = common::body_json(resp).await;
|
||||
let fields: Vec<&str> = body["error"]["details"]["fields"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|e| e["field"].as_str().unwrap())
|
||||
.collect();
|
||||
assert!(fields.contains(&"workers"));
|
||||
assert!(fields.contains(&"slice_overlap"));
|
||||
assert!(fields.contains(&"endpoint"));
|
||||
}
|
||||
|
||||
// ---- PUT persistence + audit ----------------------------------------------
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn put_crawler_persists_and_audits(pool: PgPool) {
|
||||
let h = common::harness(pool.clone());
|
||||
let (_u, cookie, admin_id) = seed_admin(&pool, &h.app).await;
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::put_json_with_cookie(
|
||||
"/api/v1/admin/settings/crawler",
|
||||
json!({ "rate_ms": 2500, "chapter_workers": 4, "start_url": "https://example.com/" }),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = common::body_json(resp).await;
|
||||
assert_eq!(body["editable"]["rate_ms"], 2500);
|
||||
assert_eq!(body["editable"]["chapter_workers"], 4);
|
||||
// Allowlist normalized to include the start-url host.
|
||||
let allow: Vec<&str> = body["editable"]["download_allowlist"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|v| v.as_str().unwrap())
|
||||
.collect();
|
||||
assert!(allow.contains(&"example.com"));
|
||||
|
||||
// A second GET reflects the persisted change.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::get_with_cookie("/api/v1/admin/settings/crawler", &cookie))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = common::body_json(resp).await;
|
||||
assert_eq!(body["editable"]["rate_ms"], 2500);
|
||||
|
||||
// An audit row landed.
|
||||
let (action, kind): (String, String) = sqlx::query_as(
|
||||
"SELECT action, target_kind FROM admin_audit WHERE actor_user_id = $1 ORDER BY at DESC LIMIT 1",
|
||||
)
|
||||
.bind(admin_id)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(action, "update_crawler_settings");
|
||||
assert_eq!(kind, "settings");
|
||||
}
|
||||
|
||||
// ---- PUT triggers reload ---------------------------------------------------
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn put_analysis_triggers_reload_and_flips_gate(pool: PgPool) {
|
||||
let (h, reloader) = common::harness_with_settings_reloader(pool.clone());
|
||||
let (_u, cookie, _id) = seed_admin(&pool, &h.app).await;
|
||||
|
||||
// Gate starts closed.
|
||||
assert!(!reloader.runtime.analysis_enabled());
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::put_json_with_cookie(
|
||||
"/api/v1/admin/settings/analysis",
|
||||
json!({ "enabled": true, "model": "qwen2-vl", "temperature": 0.4 }),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
|
||||
// The reloader was invoked with the converted config, and the shared gate
|
||||
// flipped on — both without a restart.
|
||||
assert!(reloader.runtime.analysis_enabled());
|
||||
let applied = reloader.analysis.lock().unwrap().clone().expect("reload called");
|
||||
assert!(applied.enabled);
|
||||
assert_eq!(applied.model, "qwen2-vl");
|
||||
assert_eq!(applied.temperature, 0.4);
|
||||
}
|
||||
|
||||
// ---- repo-level seed -------------------------------------------------------
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn seed_if_absent_is_idempotent(pool: PgPool) {
|
||||
let v1 = json!({ "rate_ms": 1000 });
|
||||
let v2 = json!({ "rate_ms": 9999 });
|
||||
// First seed inserts.
|
||||
assert!(repo::app_settings::seed_if_absent(&pool, "crawler", &v1).await.unwrap());
|
||||
// Second seed is a no-op (row already present) and does not overwrite.
|
||||
assert!(!repo::app_settings::seed_if_absent(&pool, "crawler", &v2).await.unwrap());
|
||||
let got = repo::app_settings::get(&pool, "crawler").await.unwrap().unwrap();
|
||||
assert_eq!(got["rate_ms"], 1000);
|
||||
// upsert does overwrite.
|
||||
repo::app_settings::upsert(&pool, "crawler", &v2).await.unwrap();
|
||||
let got = repo::app_settings::get(&pool, "crawler").await.unwrap().unwrap();
|
||||
assert_eq!(got["rate_ms"], 9999);
|
||||
}
|
||||
@@ -14,9 +14,9 @@ use sqlx::PgPool;
|
||||
use tempfile::TempDir;
|
||||
use tower::ServiceExt;
|
||||
|
||||
use mangalord::app::{router, AppState};
|
||||
use mangalord::app::{router, AppState, RuntimeControls};
|
||||
use mangalord::auth::rate_limit::AuthRateLimiter;
|
||||
use mangalord::config::{AuthConfig, UploadConfig};
|
||||
use mangalord::config::{AnalysisConfig, AuthConfig, CrawlerConfig, UploadConfig};
|
||||
use mangalord::storage::{LocalStorage, PutByteStream, Storage, StorageError, StreamingFile};
|
||||
|
||||
use async_trait::async_trait;
|
||||
@@ -76,13 +76,15 @@ fn harness_with_auth_config(
|
||||
auth_limiter,
|
||||
// Default harness has no crawler daemon wired up; admin resync
|
||||
// handlers return 503 in this config. Tests that need a stub
|
||||
// resync service swap it in via `harness_with_resync`.
|
||||
resync: None,
|
||||
crawler: None,
|
||||
// resync service swap it in via `harness_with_resync`. No reloader,
|
||||
// so settings still persist but spawn no daemon.
|
||||
runtime: Arc::new(RuntimeControls::new(false)),
|
||||
reloader: None,
|
||||
crawler_base: CrawlerConfig::default(),
|
||||
analysis_base: AnalysisConfig::default(),
|
||||
// Empty allowlist = CSRF check skipped. The CSRF-specific test
|
||||
// harness `harness_with_admin_origins` overrides this.
|
||||
admin_allowed_origins: Arc::new(Vec::new()),
|
||||
analysis_enabled: false,
|
||||
analysis_events: Arc::new(mangalord::analysis::events::AnalysisEvents::new()),
|
||||
};
|
||||
Harness { app: router(state), _storage_dir: storage_dir }
|
||||
@@ -148,6 +150,8 @@ pub fn harness_with_resync(
|
||||
..AuthConfig::default()
|
||||
};
|
||||
let auth_limiter = Arc::new(AuthRateLimiter::new(auth.rate_limit));
|
||||
let runtime = Arc::new(RuntimeControls::new(false));
|
||||
runtime.set_resync(Some(resync));
|
||||
let state = AppState {
|
||||
db: pool,
|
||||
storage,
|
||||
@@ -157,10 +161,11 @@ pub fn harness_with_resync(
|
||||
max_file_bytes: 256 * 1024,
|
||||
},
|
||||
auth_limiter,
|
||||
resync: Some(resync),
|
||||
crawler: None,
|
||||
runtime,
|
||||
reloader: None,
|
||||
crawler_base: CrawlerConfig::default(),
|
||||
analysis_base: AnalysisConfig::default(),
|
||||
admin_allowed_origins: Arc::new(Vec::new()),
|
||||
analysis_enabled: false,
|
||||
analysis_events: Arc::new(mangalord::analysis::events::AnalysisEvents::new()),
|
||||
};
|
||||
Harness {
|
||||
@@ -189,10 +194,11 @@ pub fn harness_with_analysis(pool: PgPool) -> Harness {
|
||||
max_file_bytes: 256 * 1024,
|
||||
},
|
||||
auth_limiter,
|
||||
resync: None,
|
||||
crawler: None,
|
||||
runtime: Arc::new(RuntimeControls::new(true)),
|
||||
reloader: None,
|
||||
crawler_base: CrawlerConfig::default(),
|
||||
analysis_base: AnalysisConfig::default(),
|
||||
admin_allowed_origins: Arc::new(Vec::new()),
|
||||
analysis_enabled: true,
|
||||
analysis_events: Arc::new(mangalord::analysis::events::AnalysisEvents::new()),
|
||||
};
|
||||
Harness {
|
||||
@@ -201,6 +207,71 @@ pub fn harness_with_analysis(pool: PgPool) -> Harness {
|
||||
}
|
||||
}
|
||||
|
||||
/// A [`DaemonReloader`] stub that records the configs it was asked to apply
|
||||
/// and flips the shared analysis gate, without spawning any real daemon. Lets
|
||||
/// settings tests assert that a `PUT` triggers a reload with the converted
|
||||
/// config (and that the analysis enable gate moves).
|
||||
pub struct StubReloader {
|
||||
pub runtime: Arc<RuntimeControls>,
|
||||
pub crawler: std::sync::Mutex<Option<CrawlerConfig>>,
|
||||
pub analysis: std::sync::Mutex<Option<AnalysisConfig>>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl mangalord::app::DaemonReloader for StubReloader {
|
||||
async fn reload_crawler(&self, cfg: CrawlerConfig) -> anyhow::Result<()> {
|
||||
*self.crawler.lock().unwrap() = Some(cfg);
|
||||
Ok(())
|
||||
}
|
||||
async fn reload_analysis(&self, cfg: AnalysisConfig) -> anyhow::Result<()> {
|
||||
self.runtime.set_analysis_enabled(cfg.enabled);
|
||||
*self.analysis.lock().unwrap() = Some(cfg);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Like [`harness`] but wires a [`StubReloader`] so the settings `PUT`
|
||||
/// endpoints exercise the reload path. Returns the harness plus the shared
|
||||
/// stub so the test can inspect what was applied.
|
||||
pub fn harness_with_settings_reloader(pool: PgPool) -> (Harness, Arc<StubReloader>) {
|
||||
let storage_dir = tempfile::tempdir().expect("tempdir");
|
||||
let storage = Arc::new(LocalStorage::new(storage_dir.path()));
|
||||
let auth = AuthConfig {
|
||||
cookie_secure: false,
|
||||
..AuthConfig::default()
|
||||
};
|
||||
let auth_limiter = Arc::new(AuthRateLimiter::new(auth.rate_limit));
|
||||
let runtime = Arc::new(RuntimeControls::new(false));
|
||||
let reloader = Arc::new(StubReloader {
|
||||
runtime: Arc::clone(&runtime),
|
||||
crawler: std::sync::Mutex::new(None),
|
||||
analysis: std::sync::Mutex::new(None),
|
||||
});
|
||||
let state = AppState {
|
||||
db: pool,
|
||||
storage,
|
||||
auth,
|
||||
upload: UploadConfig {
|
||||
max_request_bytes: 4 * 1024 * 1024,
|
||||
max_file_bytes: 256 * 1024,
|
||||
},
|
||||
auth_limiter,
|
||||
runtime,
|
||||
reloader: Some(Arc::clone(&reloader) as Arc<dyn mangalord::app::DaemonReloader>),
|
||||
crawler_base: CrawlerConfig::default(),
|
||||
analysis_base: AnalysisConfig::default(),
|
||||
admin_allowed_origins: Arc::new(Vec::new()),
|
||||
analysis_events: Arc::new(mangalord::analysis::events::AnalysisEvents::new()),
|
||||
};
|
||||
(
|
||||
Harness {
|
||||
app: router(state),
|
||||
_storage_dir: storage_dir,
|
||||
},
|
||||
reloader,
|
||||
)
|
||||
}
|
||||
|
||||
/// Like [`harness`] but configures an admin CSRF allowlist so the
|
||||
/// `/admin/*` mutating endpoints reject cross-origin browser POSTs.
|
||||
/// Used by the admin CSRF integration tests.
|
||||
@@ -220,10 +291,11 @@ pub fn harness_with_admin_origins(pool: PgPool, origins: Vec<String>) -> Harness
|
||||
max_file_bytes: 256 * 1024,
|
||||
},
|
||||
auth_limiter,
|
||||
resync: None,
|
||||
crawler: None,
|
||||
runtime: Arc::new(RuntimeControls::new(false)),
|
||||
reloader: None,
|
||||
crawler_base: CrawlerConfig::default(),
|
||||
analysis_base: AnalysisConfig::default(),
|
||||
admin_allowed_origins: Arc::new(origins),
|
||||
analysis_enabled: false,
|
||||
analysis_events: Arc::new(mangalord::analysis::events::AnalysisEvents::new()),
|
||||
};
|
||||
Harness {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "mangalord-frontend",
|
||||
"version": "0.79.1",
|
||||
"version": "0.80.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
|
||||
@@ -556,3 +556,110 @@ export type AnalysisEvent =
|
||||
export function analysisStatusStreamUrl(): string {
|
||||
return apiUrl('/v1/admin/analysis/status/stream');
|
||||
}
|
||||
|
||||
// ---- runtime settings (crawler + analysis) ---------------------------------
|
||||
|
||||
/** Operationally-safe, admin-editable crawler settings. Host/infra and
|
||||
* session/secret fields are managed via environment and surfaced read-only
|
||||
* in {@link CrawlerEnvOnly}. */
|
||||
export type CrawlerSettings = {
|
||||
daemon_enabled: boolean;
|
||||
daily_at: string; // "HH:MM"
|
||||
tz: string; // IANA
|
||||
idle_timeout_secs: number;
|
||||
chapter_workers: number;
|
||||
retention_days: number;
|
||||
start_url: string | null;
|
||||
rate_ms: number;
|
||||
cdn_host: string | null;
|
||||
cdn_rate_ms: number;
|
||||
cookie_domain: string | null;
|
||||
user_agent: string | null;
|
||||
download_allowlist: string[];
|
||||
allow_any_host: boolean;
|
||||
max_image_bytes: number;
|
||||
manga_limit: number;
|
||||
job_timeout_secs: number;
|
||||
metadata_max_consecutive_failures: number;
|
||||
browser_restart_threshold: number;
|
||||
};
|
||||
|
||||
/** Read-only view of crawler fields managed via environment. */
|
||||
export type CrawlerEnvOnly = {
|
||||
browser_mode: string;
|
||||
browser_args: string[];
|
||||
proxy: string | null;
|
||||
tor_control_url: string | null;
|
||||
tor_credentials_configured: boolean;
|
||||
session_configured: boolean;
|
||||
};
|
||||
|
||||
export type CrawlerSettingsResponse = {
|
||||
editable: CrawlerSettings;
|
||||
env_only: CrawlerEnvOnly;
|
||||
};
|
||||
|
||||
/** Admin-editable analysis settings. Prompts are `null` to mean "use the
|
||||
* compiled default" (see {@link PromptDefaults}). The vision API key is
|
||||
* env-only and never returned. */
|
||||
export type AnalysisSettings = {
|
||||
enabled: boolean;
|
||||
workers: number;
|
||||
endpoint: string;
|
||||
model: string;
|
||||
request_timeout_secs: number;
|
||||
job_timeout_secs: number;
|
||||
max_tokens: number;
|
||||
max_pixels: number;
|
||||
min_slice_height: number;
|
||||
slice_overlap: number;
|
||||
tall_aspect_threshold: number;
|
||||
max_slices: number;
|
||||
max_image_bytes: number;
|
||||
response_format: 'json_schema' | 'json_object' | 'none';
|
||||
frequency_penalty: number;
|
||||
temperature: number;
|
||||
system_prompt: string | null;
|
||||
ocr_prompt: string | null;
|
||||
grounding_prompt: string | null;
|
||||
};
|
||||
|
||||
export type PromptDefaults = {
|
||||
system_prompt: string;
|
||||
ocr_prompt: string;
|
||||
grounding_prompt: string;
|
||||
};
|
||||
|
||||
export type AnalysisSettingsResponse = {
|
||||
editable: AnalysisSettings;
|
||||
env_only: { api_key_configured: boolean };
|
||||
prompt_defaults: PromptDefaults;
|
||||
};
|
||||
|
||||
export async function getCrawlerSettings(): Promise<CrawlerSettingsResponse> {
|
||||
return request<CrawlerSettingsResponse>('/v1/admin/settings/crawler');
|
||||
}
|
||||
|
||||
export async function updateCrawlerSettings(
|
||||
settings: CrawlerSettings
|
||||
): Promise<CrawlerSettingsResponse> {
|
||||
return request<CrawlerSettingsResponse>('/v1/admin/settings/crawler', {
|
||||
method: 'PUT',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify(settings)
|
||||
});
|
||||
}
|
||||
|
||||
export async function getAnalysisSettings(): Promise<AnalysisSettingsResponse> {
|
||||
return request<AnalysisSettingsResponse>('/v1/admin/settings/analysis');
|
||||
}
|
||||
|
||||
export async function updateAnalysisSettings(
|
||||
settings: AnalysisSettings
|
||||
): Promise<AnalysisSettingsResponse> {
|
||||
return request<AnalysisSettingsResponse>('/v1/admin/settings/analysis', {
|
||||
method: 'PUT',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify(settings)
|
||||
});
|
||||
}
|
||||
|
||||
@@ -25,14 +25,19 @@ export class ApiError extends Error {
|
||||
constructor(
|
||||
public readonly status: number,
|
||||
public readonly code: string,
|
||||
message: string
|
||||
message: string,
|
||||
/** The error envelope's `details` payload, when present (e.g. the
|
||||
* per-field `{ fields: [...] }` of a `validation_failed` response). */
|
||||
public readonly details?: unknown
|
||||
) {
|
||||
super(message);
|
||||
this.name = 'ApiError';
|
||||
}
|
||||
}
|
||||
|
||||
type ErrorEnvelope = { error?: { code?: unknown; message?: unknown } };
|
||||
type ErrorEnvelope = {
|
||||
error?: { code?: unknown; message?: unknown; details?: unknown };
|
||||
};
|
||||
|
||||
/**
|
||||
* Optional hook fired the first moment `request()` observes a 401 on
|
||||
@@ -59,6 +64,7 @@ export async function request<T>(path: string, init?: RequestInit): Promise<T> {
|
||||
if (!res.ok) {
|
||||
let code = 'http_error';
|
||||
let message = `${res.status} ${res.statusText}`;
|
||||
let details: unknown;
|
||||
const ct = res.headers.get('content-type') ?? '';
|
||||
try {
|
||||
if (ct.includes('application/json')) {
|
||||
@@ -70,6 +76,9 @@ export async function request<T>(path: string, init?: RequestInit): Promise<T> {
|
||||
if (typeof body.error.message === 'string' && body.error.message) {
|
||||
message = body.error.message;
|
||||
}
|
||||
if (body.error.details !== undefined) {
|
||||
details = body.error.details;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
const text = await res.text();
|
||||
@@ -88,7 +97,7 @@ export async function request<T>(path: string, init?: RequestInit): Promise<T> {
|
||||
console.error('on401 hook threw:', e);
|
||||
}
|
||||
}
|
||||
throw new ApiError(res.status, code, message);
|
||||
throw new ApiError(res.status, code, message, details);
|
||||
}
|
||||
// Any empty body (not just 204) returns undefined — the manga-add
|
||||
// endpoint, for instance, signals create-vs-already-present via
|
||||
|
||||
167
frontend/src/lib/api/settings.test.ts
Normal file
167
frontend/src/lib/api/settings.test.ts
Normal file
@@ -0,0 +1,167 @@
|
||||
import {
|
||||
describe,
|
||||
it,
|
||||
expect,
|
||||
vi,
|
||||
beforeEach,
|
||||
afterEach,
|
||||
type MockInstance
|
||||
} from 'vitest';
|
||||
import {
|
||||
getCrawlerSettings,
|
||||
updateCrawlerSettings,
|
||||
getAnalysisSettings,
|
||||
updateAnalysisSettings,
|
||||
type CrawlerSettings,
|
||||
type AnalysisSettings
|
||||
} from './admin';
|
||||
import { ApiError } from './client';
|
||||
|
||||
function ok(body: unknown, status = 200): Response {
|
||||
return new Response(JSON.stringify(body), {
|
||||
status,
|
||||
headers: { 'content-type': 'application/json' }
|
||||
});
|
||||
}
|
||||
|
||||
const crawlerFixture: CrawlerSettings = {
|
||||
daemon_enabled: true,
|
||||
daily_at: '00:00',
|
||||
tz: 'UTC',
|
||||
idle_timeout_secs: 600,
|
||||
chapter_workers: 1,
|
||||
retention_days: 7,
|
||||
start_url: null,
|
||||
rate_ms: 1000,
|
||||
cdn_host: null,
|
||||
cdn_rate_ms: 1000,
|
||||
cookie_domain: null,
|
||||
user_agent: null,
|
||||
download_allowlist: [],
|
||||
allow_any_host: false,
|
||||
max_image_bytes: 33554432,
|
||||
manga_limit: 0,
|
||||
job_timeout_secs: 600,
|
||||
metadata_max_consecutive_failures: 10,
|
||||
browser_restart_threshold: 3
|
||||
};
|
||||
|
||||
const analysisFixture: AnalysisSettings = {
|
||||
enabled: false,
|
||||
workers: 1,
|
||||
endpoint: 'http://localhost:8000/v1/chat/completions',
|
||||
model: '',
|
||||
request_timeout_secs: 120,
|
||||
job_timeout_secs: 600,
|
||||
max_tokens: 4096,
|
||||
max_pixels: 1000000,
|
||||
min_slice_height: 640,
|
||||
slice_overlap: 0.12,
|
||||
tall_aspect_threshold: 1.6,
|
||||
max_slices: 16,
|
||||
max_image_bytes: 8388608,
|
||||
response_format: 'json_schema',
|
||||
frequency_penalty: 0.3,
|
||||
temperature: 0,
|
||||
system_prompt: null,
|
||||
ocr_prompt: null,
|
||||
grounding_prompt: null
|
||||
};
|
||||
|
||||
describe('settings api client', () => {
|
||||
let fetchSpy: MockInstance<typeof globalThis.fetch>;
|
||||
|
||||
beforeEach(() => {
|
||||
fetchSpy = vi.spyOn(globalThis, 'fetch');
|
||||
});
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it('getCrawlerSettings GETs the crawler settings endpoint', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(
|
||||
ok({
|
||||
editable: crawlerFixture,
|
||||
env_only: {
|
||||
browser_mode: 'headless',
|
||||
browser_args: [],
|
||||
proxy: null,
|
||||
tor_control_url: null,
|
||||
tor_credentials_configured: false,
|
||||
session_configured: false
|
||||
}
|
||||
})
|
||||
);
|
||||
const r = await getCrawlerSettings();
|
||||
expect(r.editable.rate_ms).toBe(1000);
|
||||
expect(r.env_only.browser_mode).toBe('headless');
|
||||
const url = fetchSpy.mock.calls[0][0] as string;
|
||||
expect(url).toMatch(/\/v1\/admin\/settings\/crawler$/);
|
||||
});
|
||||
|
||||
it('updateCrawlerSettings PUTs the body as JSON', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(
|
||||
ok({
|
||||
editable: { ...crawlerFixture, rate_ms: 2000 },
|
||||
env_only: {
|
||||
browser_mode: 'headless',
|
||||
browser_args: [],
|
||||
proxy: null,
|
||||
tor_control_url: null,
|
||||
tor_credentials_configured: false,
|
||||
session_configured: false
|
||||
}
|
||||
})
|
||||
);
|
||||
const r = await updateCrawlerSettings({ ...crawlerFixture, rate_ms: 2000 });
|
||||
expect(r.editable.rate_ms).toBe(2000);
|
||||
const init = fetchSpy.mock.calls[0][1]!;
|
||||
expect(init.method).toBe('PUT');
|
||||
expect(JSON.parse(init.body as string).rate_ms).toBe(2000);
|
||||
});
|
||||
|
||||
it('getAnalysisSettings returns prompt defaults and the no-secret indicator', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(
|
||||
ok({
|
||||
editable: analysisFixture,
|
||||
env_only: { api_key_configured: true },
|
||||
prompt_defaults: {
|
||||
system_prompt: 'SYS',
|
||||
ocr_prompt: 'OCR',
|
||||
grounding_prompt: 'GND'
|
||||
}
|
||||
})
|
||||
);
|
||||
const r = await getAnalysisSettings();
|
||||
expect(r.editable.system_prompt).toBeNull();
|
||||
expect(r.env_only.api_key_configured).toBe(true);
|
||||
expect(r.prompt_defaults.system_prompt).toBe('SYS');
|
||||
// The secret is never present on the editable DTO.
|
||||
expect('api_key' in (r.editable as object)).toBe(false);
|
||||
});
|
||||
|
||||
it('updateAnalysisSettings surfaces 422 field errors via ApiError.details', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(
|
||||
new Response(
|
||||
JSON.stringify({
|
||||
error: {
|
||||
code: 'validation_failed',
|
||||
message: 'invalid settings',
|
||||
details: { fields: [{ field: 'workers', message: 'must be at least 1' }] }
|
||||
}
|
||||
}),
|
||||
{ status: 422, headers: { 'content-type': 'application/json' } }
|
||||
)
|
||||
);
|
||||
try {
|
||||
await updateAnalysisSettings({ ...analysisFixture, workers: 0 });
|
||||
expect.unreachable('should have thrown');
|
||||
} catch (e) {
|
||||
expect(e).toBeInstanceOf(ApiError);
|
||||
expect((e as ApiError).status).toBe(422);
|
||||
expect((e as ApiError).code).toBe('validation_failed');
|
||||
const details = (e as ApiError).details as { fields: { field: string }[] };
|
||||
expect(details.fields[0].field).toBe('workers');
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -8,6 +8,7 @@
|
||||
{ href: '/admin/mangas', label: 'Mangas' },
|
||||
{ href: '/admin/crawler', label: 'Crawler' },
|
||||
{ href: '/admin/analysis', label: 'Analysis' },
|
||||
{ href: '/admin/settings', label: 'Settings' },
|
||||
{ href: '/admin/system', label: 'System' }
|
||||
];
|
||||
</script>
|
||||
|
||||
664
frontend/src/routes/admin/settings/+page.svelte
Normal file
664
frontend/src/routes/admin/settings/+page.svelte
Normal file
@@ -0,0 +1,664 @@
|
||||
<script lang="ts">
|
||||
import { onMount } from 'svelte';
|
||||
import Modal from '$lib/components/Modal.svelte';
|
||||
import { ApiError } from '$lib/api/client';
|
||||
import {
|
||||
getCrawlerSettings,
|
||||
updateCrawlerSettings,
|
||||
getAnalysisSettings,
|
||||
updateAnalysisSettings,
|
||||
type CrawlerSettings,
|
||||
type CrawlerEnvOnly,
|
||||
type AnalysisSettings,
|
||||
type PromptDefaults
|
||||
} from '$lib/api/admin';
|
||||
|
||||
type Tab = 'crawler' | 'analysis';
|
||||
let tab = $state<Tab>('crawler');
|
||||
|
||||
// Crawler state
|
||||
let crawler = $state<CrawlerSettings | null>(null);
|
||||
let crawlerEnv = $state<CrawlerEnvOnly | null>(null);
|
||||
let allowlistText = $state(''); // textarea mirror of download_allowlist
|
||||
|
||||
// Analysis state
|
||||
let analysis = $state<AnalysisSettings | null>(null);
|
||||
let promptDefaults = $state<PromptDefaults | null>(null);
|
||||
let apiKeyConfigured = $state(false);
|
||||
|
||||
let loading = $state(true);
|
||||
let loadError = $state<string | null>(null);
|
||||
let saving = $state(false);
|
||||
let saveError = $state<string | null>(null);
|
||||
let success = $state<string | null>(null);
|
||||
// field -> message, parsed from a 422 validation_failed response
|
||||
let fieldErrors = $state<Record<string, string>>({});
|
||||
let confirmOpen = $state(false);
|
||||
|
||||
onMount(load);
|
||||
|
||||
async function load() {
|
||||
loading = true;
|
||||
loadError = null;
|
||||
try {
|
||||
const [c, a] = await Promise.all([getCrawlerSettings(), getAnalysisSettings()]);
|
||||
crawler = c.editable;
|
||||
crawlerEnv = c.env_only;
|
||||
allowlistText = c.editable.download_allowlist.join('\n');
|
||||
analysis = a.editable;
|
||||
promptDefaults = a.prompt_defaults;
|
||||
apiKeyConfigured = a.env_only.api_key_configured;
|
||||
} catch (e) {
|
||||
loadError = e instanceof Error ? e.message : 'Failed to load settings.';
|
||||
} finally {
|
||||
loading = false;
|
||||
}
|
||||
}
|
||||
|
||||
function parseFieldErrors(e: unknown): Record<string, string> {
|
||||
const out: Record<string, string> = {};
|
||||
if (
|
||||
e instanceof ApiError &&
|
||||
e.details &&
|
||||
typeof e.details === 'object' &&
|
||||
'fields' in e.details
|
||||
) {
|
||||
const fields = (e.details as { fields?: { field: string; message: string }[] }).fields;
|
||||
for (const f of fields ?? []) out[f.field] = f.message;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function askSave() {
|
||||
saveError = null;
|
||||
success = null;
|
||||
fieldErrors = {};
|
||||
confirmOpen = true;
|
||||
}
|
||||
|
||||
async function save() {
|
||||
confirmOpen = false;
|
||||
saving = true;
|
||||
saveError = null;
|
||||
success = null;
|
||||
fieldErrors = {};
|
||||
try {
|
||||
if (tab === 'crawler' && crawler) {
|
||||
// Sync the allowlist textarea back into the model.
|
||||
crawler.download_allowlist = allowlistText
|
||||
.split('\n')
|
||||
.map((s) => s.trim())
|
||||
.filter(Boolean);
|
||||
const r = await updateCrawlerSettings(crawler);
|
||||
crawler = r.editable;
|
||||
crawlerEnv = r.env_only;
|
||||
allowlistText = r.editable.download_allowlist.join('\n');
|
||||
} else if (tab === 'analysis' && analysis) {
|
||||
const r = await updateAnalysisSettings(analysis);
|
||||
analysis = r.editable;
|
||||
promptDefaults = r.prompt_defaults;
|
||||
apiKeyConfigured = r.env_only.api_key_configured;
|
||||
}
|
||||
success = 'Saved and applied. The subsystem was restarted with the new settings.';
|
||||
} catch (e) {
|
||||
fieldErrors = parseFieldErrors(e);
|
||||
saveError =
|
||||
Object.keys(fieldErrors).length > 0
|
||||
? 'Some fields need fixing.'
|
||||
: e instanceof Error
|
||||
? e.message
|
||||
: 'Save failed.';
|
||||
} finally {
|
||||
saving = false;
|
||||
}
|
||||
}
|
||||
|
||||
function resetPrompt(which: 'system_prompt' | 'ocr_prompt' | 'grounding_prompt') {
|
||||
if (analysis) analysis[which] = null;
|
||||
}
|
||||
</script>
|
||||
|
||||
<h1>Settings</h1>
|
||||
<p class="lead muted">
|
||||
Edit crawler and analysis configuration. Saving applies immediately — the affected daemon is
|
||||
gracefully restarted, so in-flight jobs are retried rather than lost. Host-level and secret
|
||||
values stay environment-managed and are shown read-only.
|
||||
</p>
|
||||
|
||||
<div class="tabs" role="tablist">
|
||||
<button
|
||||
role="tab"
|
||||
aria-selected={tab === 'crawler'}
|
||||
class:active={tab === 'crawler'}
|
||||
onclick={() => (tab = 'crawler')}
|
||||
data-testid="settings-tab-crawler">Crawler</button
|
||||
>
|
||||
<button
|
||||
role="tab"
|
||||
aria-selected={tab === 'analysis'}
|
||||
class:active={tab === 'analysis'}
|
||||
onclick={() => (tab = 'analysis')}
|
||||
data-testid="settings-tab-analysis">Analysis</button
|
||||
>
|
||||
</div>
|
||||
|
||||
{#if loading}
|
||||
<p class="muted">Loading…</p>
|
||||
{:else if loadError}
|
||||
<p class="error" role="alert">{loadError}</p>
|
||||
{:else}
|
||||
<div class="sticky-bar">
|
||||
{#if tab === 'crawler' && crawler}
|
||||
<label class="toggle">
|
||||
<input
|
||||
type="checkbox"
|
||||
bind:checked={crawler.daemon_enabled}
|
||||
data-testid="settings-crawler-enabled"
|
||||
/>
|
||||
<span>Crawler daemon {crawler.daemon_enabled ? 'enabled' : 'disabled'}</span>
|
||||
</label>
|
||||
{:else if tab === 'analysis' && analysis}
|
||||
<label class="toggle">
|
||||
<input
|
||||
type="checkbox"
|
||||
bind:checked={analysis.enabled}
|
||||
data-testid="settings-analysis-enabled"
|
||||
/>
|
||||
<span>Analysis worker {analysis.enabled ? 'enabled' : 'disabled'}</span>
|
||||
</label>
|
||||
{/if}
|
||||
<div class="spacer"></div>
|
||||
<button class="primary" disabled={saving} onclick={askSave} data-testid="settings-save">
|
||||
{saving ? 'Saving…' : 'Save & apply'}
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{#if success}<p class="notice" role="status">{success}</p>{/if}
|
||||
{#if saveError}<p class="error" role="alert">{saveError}</p>{/if}
|
||||
|
||||
{#if tab === 'crawler' && crawler}
|
||||
<div class="form">
|
||||
<fieldset>
|
||||
<legend>Schedule</legend>
|
||||
<label class="field">
|
||||
<span>Daily metadata pass (HH:MM)</span>
|
||||
<input type="time" bind:value={crawler.daily_at} />
|
||||
{#if fieldErrors.daily_at}<small class="fe">{fieldErrors.daily_at}</small>{/if}
|
||||
</label>
|
||||
<label class="field">
|
||||
<span>Timezone (IANA)</span>
|
||||
<input type="text" bind:value={crawler.tz} placeholder="UTC" />
|
||||
{#if fieldErrors.tz}<small class="fe">{fieldErrors.tz}</small>{/if}
|
||||
</label>
|
||||
<label class="field">
|
||||
<span>Idle timeout (seconds)</span>
|
||||
<input type="number" min="0" bind:value={crawler.idle_timeout_secs} />
|
||||
</label>
|
||||
</fieldset>
|
||||
|
||||
<fieldset>
|
||||
<legend>Source</legend>
|
||||
<label class="field">
|
||||
<span>Start URL</span>
|
||||
<input
|
||||
type="text"
|
||||
bind:value={crawler.start_url}
|
||||
placeholder="https://source.example/"
|
||||
/>
|
||||
{#if fieldErrors.start_url}<small class="fe">{fieldErrors.start_url}</small>{/if}
|
||||
</label>
|
||||
<label class="field">
|
||||
<span>CDN host</span>
|
||||
<input type="text" bind:value={crawler.cdn_host} placeholder="cdn.example" />
|
||||
</label>
|
||||
<label class="field">
|
||||
<span>Cookie domain</span>
|
||||
<input
|
||||
type="text"
|
||||
bind:value={crawler.cookie_domain}
|
||||
placeholder="source.example"
|
||||
/>
|
||||
</label>
|
||||
<label class="field">
|
||||
<span>User-Agent</span>
|
||||
<input type="text" bind:value={crawler.user_agent} />
|
||||
</label>
|
||||
</fieldset>
|
||||
|
||||
<fieldset>
|
||||
<legend>Rate limiting</legend>
|
||||
<label class="field">
|
||||
<span>Request interval (ms)</span>
|
||||
<input type="number" min="0" bind:value={crawler.rate_ms} />
|
||||
</label>
|
||||
<label class="field">
|
||||
<span>CDN request interval (ms)</span>
|
||||
<input type="number" min="0" bind:value={crawler.cdn_rate_ms} />
|
||||
</label>
|
||||
</fieldset>
|
||||
|
||||
<fieldset>
|
||||
<legend>Workers & retention</legend>
|
||||
<label class="field">
|
||||
<span>Chapter workers</span>
|
||||
<input type="number" min="1" bind:value={crawler.chapter_workers} />
|
||||
{#if fieldErrors.chapter_workers}<small class="fe"
|
||||
>{fieldErrors.chapter_workers}</small
|
||||
>{/if}
|
||||
</label>
|
||||
<label class="field">
|
||||
<span>Job retention (days)</span>
|
||||
<input type="number" min="0" bind:value={crawler.retention_days} />
|
||||
</label>
|
||||
<label class="field">
|
||||
<span>Manga limit per pass (0 = unlimited)</span>
|
||||
<input type="number" min="0" bind:value={crawler.manga_limit} />
|
||||
</label>
|
||||
</fieldset>
|
||||
|
||||
<fieldset>
|
||||
<legend>Download safety</legend>
|
||||
<label class="field-inline">
|
||||
<input type="checkbox" bind:checked={crawler.allow_any_host} />
|
||||
<span>Allow any host (bypass allowlist; still blocks private IPs)</span>
|
||||
</label>
|
||||
<label class="field">
|
||||
<span>Download allowlist (one host per line)</span>
|
||||
<textarea
|
||||
rows="4"
|
||||
bind:value={allowlistText}
|
||||
disabled={crawler.allow_any_host}
|
||||
></textarea>
|
||||
</label>
|
||||
<label class="field">
|
||||
<span>Max image bytes</span>
|
||||
<input type="number" min="0" bind:value={crawler.max_image_bytes} />
|
||||
</label>
|
||||
</fieldset>
|
||||
|
||||
<fieldset>
|
||||
<legend>Reliability</legend>
|
||||
<label class="field">
|
||||
<span>Chapter job timeout (seconds)</span>
|
||||
<input type="number" min="1" bind:value={crawler.job_timeout_secs} />
|
||||
{#if fieldErrors.job_timeout_secs}<small class="fe"
|
||||
>{fieldErrors.job_timeout_secs}</small
|
||||
>{/if}
|
||||
</label>
|
||||
<label class="field">
|
||||
<span>Metadata max consecutive failures</span>
|
||||
<input
|
||||
type="number"
|
||||
min="0"
|
||||
bind:value={crawler.metadata_max_consecutive_failures}
|
||||
/>
|
||||
</label>
|
||||
<label class="field">
|
||||
<span>Browser restart threshold</span>
|
||||
<input type="number" min="0" bind:value={crawler.browser_restart_threshold} />
|
||||
</label>
|
||||
</fieldset>
|
||||
|
||||
{#if crawlerEnv}
|
||||
<fieldset class="env">
|
||||
<legend>Managed via environment (read-only)</legend>
|
||||
<dl>
|
||||
<dt>Browser mode</dt>
|
||||
<dd>{crawlerEnv.browser_mode}</dd>
|
||||
<dt>Browser args</dt>
|
||||
<dd>{crawlerEnv.browser_args.join(' ') || '—'}</dd>
|
||||
<dt>Proxy</dt>
|
||||
<dd>{crawlerEnv.proxy ?? '—'}</dd>
|
||||
<dt>TOR control URL</dt>
|
||||
<dd>{crawlerEnv.tor_control_url ?? '—'}</dd>
|
||||
<dt>TOR credentials</dt>
|
||||
<dd>{crawlerEnv.tor_credentials_configured ? 'configured' : '—'}</dd>
|
||||
<dt>Initial session</dt>
|
||||
<dd>{crawlerEnv.session_configured ? 'configured' : '—'}</dd>
|
||||
</dl>
|
||||
</fieldset>
|
||||
{/if}
|
||||
</div>
|
||||
{:else if tab === 'analysis' && analysis}
|
||||
<div class="form">
|
||||
<fieldset>
|
||||
<legend>Endpoint & model</legend>
|
||||
<label class="field">
|
||||
<span>Vision endpoint URL</span>
|
||||
<input type="text" bind:value={analysis.endpoint} />
|
||||
{#if fieldErrors.endpoint}<small class="fe">{fieldErrors.endpoint}</small>{/if}
|
||||
</label>
|
||||
<label class="field">
|
||||
<span>Model</span>
|
||||
<input type="text" bind:value={analysis.model} />
|
||||
{#if fieldErrors.model}<small class="fe">{fieldErrors.model}</small>{/if}
|
||||
</label>
|
||||
<p class="env-note muted">
|
||||
API key: {apiKeyConfigured
|
||||
? 'configured via environment'
|
||||
: 'not set'} (managed via environment).
|
||||
</p>
|
||||
</fieldset>
|
||||
|
||||
<fieldset>
|
||||
<legend>Workers & timeouts</legend>
|
||||
<label class="field">
|
||||
<span>Workers</span>
|
||||
<input type="number" min="1" bind:value={analysis.workers} />
|
||||
{#if fieldErrors.workers}<small class="fe">{fieldErrors.workers}</small>{/if}
|
||||
</label>
|
||||
<label class="field">
|
||||
<span>Request timeout (seconds)</span>
|
||||
<input type="number" min="1" bind:value={analysis.request_timeout_secs} />
|
||||
</label>
|
||||
<label class="field">
|
||||
<span>Job timeout (seconds)</span>
|
||||
<input type="number" min="1" bind:value={analysis.job_timeout_secs} />
|
||||
</label>
|
||||
</fieldset>
|
||||
|
||||
<fieldset>
|
||||
<legend>Sampling & output</legend>
|
||||
<label class="field">
|
||||
<span>Temperature</span>
|
||||
<input type="number" min="0" step="0.05" bind:value={analysis.temperature} />
|
||||
{#if fieldErrors.temperature}<small class="fe"
|
||||
>{fieldErrors.temperature}</small
|
||||
>{/if}
|
||||
</label>
|
||||
<label class="field">
|
||||
<span>Frequency penalty</span>
|
||||
<input type="number" step="0.05" bind:value={analysis.frequency_penalty} />
|
||||
</label>
|
||||
<label class="field">
|
||||
<span>Max output tokens</span>
|
||||
<input type="number" min="1" bind:value={analysis.max_tokens} />
|
||||
</label>
|
||||
<label class="field">
|
||||
<span>Response format</span>
|
||||
<select bind:value={analysis.response_format}>
|
||||
<option value="json_schema">json_schema</option>
|
||||
<option value="json_object">json_object</option>
|
||||
<option value="none">none</option>
|
||||
</select>
|
||||
</label>
|
||||
</fieldset>
|
||||
|
||||
<fieldset>
|
||||
<legend>Image slicing</legend>
|
||||
<label class="field">
|
||||
<span>Max pixels per slice</span>
|
||||
<input type="number" min="1" bind:value={analysis.max_pixels} />
|
||||
</label>
|
||||
<label class="field">
|
||||
<span>Min slice height (px)</span>
|
||||
<input type="number" min="1" bind:value={analysis.min_slice_height} />
|
||||
</label>
|
||||
<label class="field">
|
||||
<span>Slice overlap (0–0.9)</span>
|
||||
<input
|
||||
type="number"
|
||||
min="0"
|
||||
max="0.9"
|
||||
step="0.01"
|
||||
bind:value={analysis.slice_overlap}
|
||||
/>
|
||||
{#if fieldErrors.slice_overlap}<small class="fe"
|
||||
>{fieldErrors.slice_overlap}</small
|
||||
>{/if}
|
||||
</label>
|
||||
<label class="field">
|
||||
<span>Tall aspect threshold (≥1.0)</span>
|
||||
<input
|
||||
type="number"
|
||||
min="1"
|
||||
step="0.1"
|
||||
bind:value={analysis.tall_aspect_threshold}
|
||||
/>
|
||||
{#if fieldErrors.tall_aspect_threshold}<small class="fe"
|
||||
>{fieldErrors.tall_aspect_threshold}</small
|
||||
>{/if}
|
||||
</label>
|
||||
<label class="field">
|
||||
<span>Max slices</span>
|
||||
<input type="number" min="1" bind:value={analysis.max_slices} />
|
||||
</label>
|
||||
<label class="field">
|
||||
<span>Max image bytes</span>
|
||||
<input type="number" min="0" bind:value={analysis.max_image_bytes} />
|
||||
</label>
|
||||
</fieldset>
|
||||
|
||||
<fieldset>
|
||||
<legend>Prompts</legend>
|
||||
{#each [['system_prompt', 'System prompt'], ['ocr_prompt', 'OCR prompt (slices)'], ['grounding_prompt', 'Grounding prompt']] as [key, label] (key)}
|
||||
{@const k = key as 'system_prompt' | 'ocr_prompt' | 'grounding_prompt'}
|
||||
<div class="prompt">
|
||||
<div class="prompt-head">
|
||||
<span>{label}</span>
|
||||
<button
|
||||
type="button"
|
||||
class="link"
|
||||
disabled={analysis[k] == null}
|
||||
onclick={() => resetPrompt(k)}>Reset to default</button
|
||||
>
|
||||
</div>
|
||||
<textarea
|
||||
rows="6"
|
||||
class="mono"
|
||||
value={analysis[k] ?? ''}
|
||||
placeholder={promptDefaults?.[k] ?? ''}
|
||||
oninput={(e) => {
|
||||
const v = (e.currentTarget as HTMLTextAreaElement).value;
|
||||
if (analysis) analysis[k] = v.trim() === '' ? null : v;
|
||||
}}
|
||||
></textarea>
|
||||
<small class="muted">
|
||||
{analysis[k] == null
|
||||
? 'Using compiled default (shown as placeholder).'
|
||||
: `${analysis[k]?.length ?? 0} chars (override).`}
|
||||
</small>
|
||||
</div>
|
||||
{/each}
|
||||
</fieldset>
|
||||
</div>
|
||||
{/if}
|
||||
{/if}
|
||||
|
||||
<Modal
|
||||
open={confirmOpen}
|
||||
title="Apply settings?"
|
||||
onClose={() => (confirmOpen = false)}
|
||||
size="sm"
|
||||
closeOnBackdrop
|
||||
testid="settings-confirm"
|
||||
>
|
||||
<p>
|
||||
Saving restarts the {tab} subsystem with the new settings. In-flight jobs are leased and
|
||||
will be retried, but any work in progress is interrupted briefly.
|
||||
</p>
|
||||
{#snippet footer()}
|
||||
<button type="button" onclick={() => (confirmOpen = false)}>Cancel</button>
|
||||
<button type="button" class="primary" onclick={save} data-testid="settings-confirm-apply">
|
||||
Save & apply
|
||||
</button>
|
||||
{/snippet}
|
||||
</Modal>
|
||||
|
||||
<style>
|
||||
h1 {
|
||||
margin: 0 0 var(--space-2) 0;
|
||||
}
|
||||
.lead {
|
||||
margin: 0 0 var(--space-4) 0;
|
||||
font-size: var(--font-sm);
|
||||
max-width: 60ch;
|
||||
}
|
||||
.muted {
|
||||
color: var(--text-muted);
|
||||
}
|
||||
.tabs {
|
||||
display: flex;
|
||||
gap: var(--space-2);
|
||||
margin-bottom: var(--space-4);
|
||||
border-bottom: 1px solid var(--border);
|
||||
}
|
||||
.tabs button {
|
||||
padding: var(--space-2) var(--space-3);
|
||||
background: none;
|
||||
border: none;
|
||||
border-bottom: 2px solid transparent;
|
||||
color: var(--text-muted);
|
||||
cursor: pointer;
|
||||
font-size: var(--font-sm);
|
||||
}
|
||||
.tabs button.active {
|
||||
color: var(--text);
|
||||
border-bottom-color: var(--primary);
|
||||
font-weight: var(--weight-semibold);
|
||||
}
|
||||
.sticky-bar {
|
||||
position: sticky;
|
||||
top: calc(var(--app-header-h) + var(--space-2));
|
||||
z-index: 1;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: var(--space-3);
|
||||
padding: var(--space-3);
|
||||
margin-bottom: var(--space-3);
|
||||
background: var(--surface-elevated);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius-md);
|
||||
}
|
||||
.sticky-bar .spacer {
|
||||
flex: 1;
|
||||
}
|
||||
.toggle {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: var(--space-2);
|
||||
font-size: var(--font-sm);
|
||||
}
|
||||
.form {
|
||||
display: grid;
|
||||
gap: var(--space-4);
|
||||
}
|
||||
fieldset {
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius-md);
|
||||
padding: var(--space-3);
|
||||
display: grid;
|
||||
gap: var(--space-3);
|
||||
}
|
||||
legend {
|
||||
padding: 0 var(--space-2);
|
||||
font-weight: var(--weight-semibold);
|
||||
font-size: var(--font-sm);
|
||||
}
|
||||
.field {
|
||||
display: grid;
|
||||
gap: var(--space-1);
|
||||
}
|
||||
.field > span {
|
||||
font-size: var(--font-sm);
|
||||
color: var(--text-muted);
|
||||
}
|
||||
.field input,
|
||||
.field select,
|
||||
.field textarea {
|
||||
padding: var(--space-2) var(--space-3);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius-md);
|
||||
background: var(--surface);
|
||||
color: var(--text);
|
||||
width: 100%;
|
||||
box-sizing: border-box;
|
||||
}
|
||||
.field-inline {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: var(--space-2);
|
||||
font-size: var(--font-sm);
|
||||
}
|
||||
.fe {
|
||||
color: var(--danger, #dc2626);
|
||||
font-size: var(--font-xs);
|
||||
}
|
||||
.mono {
|
||||
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
|
||||
font-size: var(--font-xs);
|
||||
width: 100%;
|
||||
box-sizing: border-box;
|
||||
padding: var(--space-2);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius-md);
|
||||
background: var(--surface);
|
||||
color: var(--text);
|
||||
}
|
||||
.prompt {
|
||||
display: grid;
|
||||
gap: var(--space-1);
|
||||
}
|
||||
.prompt-head {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
font-size: var(--font-sm);
|
||||
}
|
||||
button.link {
|
||||
background: none;
|
||||
border: none;
|
||||
color: var(--primary);
|
||||
cursor: pointer;
|
||||
font-size: var(--font-xs);
|
||||
padding: 0;
|
||||
}
|
||||
button.link:disabled {
|
||||
color: var(--text-muted);
|
||||
cursor: not-allowed;
|
||||
}
|
||||
.env dl {
|
||||
display: grid;
|
||||
grid-template-columns: max-content 1fr;
|
||||
gap: var(--space-1) var(--space-3);
|
||||
margin: 0;
|
||||
font-size: var(--font-sm);
|
||||
}
|
||||
.env dt {
|
||||
color: var(--text-muted);
|
||||
}
|
||||
.env dd {
|
||||
margin: 0;
|
||||
word-break: break-all;
|
||||
}
|
||||
.env-note {
|
||||
font-size: var(--font-xs);
|
||||
margin: 0;
|
||||
}
|
||||
.error {
|
||||
color: var(--danger, #dc2626);
|
||||
padding: var(--space-2) var(--space-3);
|
||||
border: 1px solid var(--danger, #dc2626);
|
||||
border-radius: var(--radius-md);
|
||||
margin-bottom: var(--space-3);
|
||||
}
|
||||
.notice {
|
||||
color: var(--success, #16a34a);
|
||||
padding: var(--space-2) var(--space-3);
|
||||
border: 1px solid var(--success, #16a34a);
|
||||
border-radius: var(--radius-md);
|
||||
margin-bottom: var(--space-3);
|
||||
}
|
||||
button.primary {
|
||||
padding: var(--space-2) var(--space-3);
|
||||
border-radius: var(--radius-md);
|
||||
border: 1px solid var(--primary);
|
||||
background: var(--primary);
|
||||
color: white;
|
||||
cursor: pointer;
|
||||
}
|
||||
button.primary:disabled {
|
||||
opacity: 0.5;
|
||||
cursor: not-allowed;
|
||||
}
|
||||
</style>
|
||||
Reference in New Issue
Block a user