feat(admin): runtime-editable crawler & analysis config in the dashboard
Move crawler and analysis configuration out of boot-only env vars and into
a DB-backed, admin-editable surface applied live without a restart.
- New `app_settings` table (migration 0026) holds one JSONB row per group;
env vars seed it on first boot, then the DB is the source of truth.
- `settings.rs` adds serializable Crawler/Analysis DTOs with env-base +
DB-overlay conversion and field-level validation; env-only/secret fields
(browser, proxy, TOR, vision API key, PHPSESSID) are never persisted.
- `GET/PUT /api/v1/admin/settings/{crawler,analysis}` (RequireAdmin,
cookie-only, CSRF-guarded): GET returns the editable DTO + a read-only
env-managed view + prompt defaults; PUT validates (422 + per-field
details), persists + audits in one tx, then gracefully respawns the
affected daemon via a Supervisor.
- AppState swaps the crawler control/resync handles and analysis enable
gate behind shared runtime cells so reloads take effect with no restart;
a boot-time spawn failure is logged rather than aborting startup.
- Make the three vision prompts and sampling temperature configurable
(defaults preserved); cookie_domain is admin-editable too.
- Frontend: tabbed /admin/settings page with grouped fieldsets, prompt
reset-to-default, env-managed read-only panel, save-&-apply confirm, and
per-field validation; typed client + ApiError.details.
- Bump version 0.79.1 -> 0.80.0.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
22
.env.example
22
.env.example
@@ -167,3 +167,25 @@ BACKEND_URL=http://backend:8080
|
|||||||
# 25 Mbps; raise for users on slower upstream links or lower if a
|
# 25 Mbps; raise for users on slower upstream links or lower if a
|
||||||
# tighter front proxy already bounds the request lifetime.
|
# tighter front proxy already bounds the request lifetime.
|
||||||
BACKEND_PROXY_TIMEOUT_MS=300000
|
BACKEND_PROXY_TIMEOUT_MS=300000
|
||||||
|
|
||||||
|
# ----- Runtime settings (crawler + analysis) -----
|
||||||
|
# The crawler and analysis subsystems are configured at runtime from the
|
||||||
|
# `app_settings` table and edited live in the admin dashboard
|
||||||
|
# (Admin → Settings) — no restart needed. The CRAWLER_* / ANALYSIS_* env
|
||||||
|
# vars below act as the BOOT SEED: on first boot (when the row is absent)
|
||||||
|
# the env values populate the DB; thereafter the DB is the source of
|
||||||
|
# truth and env changes are ignored for those fields.
|
||||||
|
#
|
||||||
|
# Host/infra and secret fields stay env-ONLY (never persisted, shown
|
||||||
|
# read-only in the dashboard): the chromium binary/dir, browser mode/args,
|
||||||
|
# CRAWLER_PROXY, all CRAWLER_TOR_CONTROL_*, the cookie domain, and the
|
||||||
|
# analysis ANALYSIS_API_KEY. The important site levers (PRIVATE_MODE,
|
||||||
|
# ALLOW_SELF_REGISTER) also remain env-only by design.
|
||||||
|
#
|
||||||
|
# New analysis knobs (all optional, all seed defaults):
|
||||||
|
# ANALYSIS_TEMPERATURE Sampling temperature. Default 0 (deterministic).
|
||||||
|
# ANALYSIS_SYSTEM_PROMPT Override the single-call vision system prompt.
|
||||||
|
# ANALYSIS_OCR_PROMPT Override the tall-page OCR (pass A) prompt.
|
||||||
|
# ANALYSIS_GROUNDING_PROMPT Override the tags/scene/safety (pass B) prompt.
|
||||||
|
# Leave the prompt vars unset to use the built-in defaults (also editable,
|
||||||
|
# with a per-prompt "reset to default", in the dashboard).
|
||||||
|
|||||||
6
.gitignore
vendored
6
.gitignore
vendored
@@ -20,6 +20,12 @@
|
|||||||
.env
|
.env
|
||||||
.env.local
|
.env.local
|
||||||
.env.*.local
|
.env.*.local
|
||||||
|
.env.dev
|
||||||
|
|
||||||
|
# Node install / Playwright output when run from the repo root (the
|
||||||
|
# canonical copies live under /frontend, already ignored above).
|
||||||
|
/node_modules
|
||||||
|
/test-results
|
||||||
|
|
||||||
# Claude Code (personal overrides only; .claude/settings.json is committed)
|
# Claude Code (personal overrides only; .claude/settings.json is committed)
|
||||||
.claude/settings.local.json
|
.claude/settings.local.json
|
||||||
|
|||||||
2
backend/Cargo.lock
generated
2
backend/Cargo.lock
generated
@@ -1517,7 +1517,7 @@ checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "mangalord"
|
name = "mangalord"
|
||||||
version = "0.79.1"
|
version = "0.80.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"argon2",
|
"argon2",
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "mangalord"
|
name = "mangalord"
|
||||||
version = "0.79.1"
|
version = "0.80.0"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
default-run = "mangalord"
|
default-run = "mangalord"
|
||||||
|
|
||||||
|
|||||||
15
backend/migrations/0026_app_settings.sql
Normal file
15
backend/migrations/0026_app_settings.sql
Normal file
@@ -0,0 +1,15 @@
|
|||||||
|
-- Runtime-editable application settings, keyed by subsystem group.
|
||||||
|
--
|
||||||
|
-- Mirrors the small key-value `crawler_state` table (0015): one row per
|
||||||
|
-- group ('crawler' | 'analysis'), the `value` JSONB holding a serialized
|
||||||
|
-- settings DTO. Env vars seed a row on first boot (when absent); after that
|
||||||
|
-- the DB row is the source of truth and the admin dashboard edits it live.
|
||||||
|
--
|
||||||
|
-- Only operationally-safe fields are stored here — host/infra and secrets
|
||||||
|
-- (chromium paths, proxy, TOR control, vision API key, cookie domain) stay
|
||||||
|
-- env-only and are never persisted.
|
||||||
|
CREATE TABLE app_settings (
|
||||||
|
key text PRIMARY KEY,
|
||||||
|
value jsonb NOT NULL,
|
||||||
|
updated_at timestamptz NOT NULL DEFAULT now()
|
||||||
|
);
|
||||||
@@ -36,10 +36,12 @@ pub const MAX_SCENE_CHARS: usize = 1000;
|
|||||||
pub const MAX_OCR_PER_CALL: usize = 50;
|
pub const MAX_OCR_PER_CALL: usize = 50;
|
||||||
pub const MAX_TAGS_PER_CALL: usize = 12;
|
pub const MAX_TAGS_PER_CALL: usize = 12;
|
||||||
|
|
||||||
/// The system prompt. Self-contained: it carries the exact JSON schema so
|
/// The default system prompt. Self-contained: it carries the exact JSON
|
||||||
/// the same string drives both the model and (implicitly) the
|
/// schema so the same string drives both the model and (implicitly) the
|
||||||
/// [`crate::domain::page_analysis::VisionAnalysis`] parser.
|
/// [`crate::domain::page_analysis::VisionAnalysis`] parser. An admin can
|
||||||
pub const SYSTEM_PROMPT: &str = concat!(
|
/// override it at runtime (see [`crate::config::AnalysisConfig::system_prompt`]);
|
||||||
|
/// this const is the fallback and the "reset to default" target.
|
||||||
|
pub const SYSTEM_PROMPT_DEFAULT: &str = concat!(
|
||||||
"You are a manga/comic page analyzer. Look at the single page image and ",
|
"You are a manga/comic page analyzer. Look at the single page image and ",
|
||||||
"return ONLY one minified JSON object — no prose, no markdown fences.\n",
|
"return ONLY one minified JSON object — no prose, no markdown fences.\n",
|
||||||
"Schema:\n",
|
"Schema:\n",
|
||||||
@@ -64,8 +66,9 @@ pub const SYSTEM_PROMPT: &str = concat!(
|
|||||||
// --- Two-pass prompts (long-page slicing) -----------------------------------
|
// --- Two-pass prompts (long-page slicing) -----------------------------------
|
||||||
|
|
||||||
/// Pass A: OCR-only over a single vertical slice of a tall page. Kept
|
/// Pass A: OCR-only over a single vertical slice of a tall page. Kept
|
||||||
/// narrow so each slice call is fast and never truncates.
|
/// narrow so each slice call is fast and never truncates. Runtime-overridable
|
||||||
pub const OCR_PROMPT: &str = concat!(
|
/// default — see [`crate::config::AnalysisConfig::ocr_prompt`].
|
||||||
|
pub const OCR_PROMPT_DEFAULT: &str = concat!(
|
||||||
"You are an OCR engine for manga/comic pages. The image is one vertical ",
|
"You are an OCR engine for manga/comic pages. The image is one vertical ",
|
||||||
"slice of a larger page. Transcribe EVERY visible text element verbatim ",
|
"slice of a larger page. Transcribe EVERY visible text element verbatim ",
|
||||||
"into ocr_results, each with its kind ",
|
"into ocr_results, each with its kind ",
|
||||||
@@ -80,7 +83,9 @@ pub const OCR_PROMPT: &str = concat!(
|
|||||||
|
|
||||||
/// Pass B: tags + scene + safety for the whole page, grounded in the merged
|
/// Pass B: tags + scene + safety for the whole page, grounded in the merged
|
||||||
/// OCR text (supplied as a separate user message part by the client).
|
/// OCR text (supplied as a separate user message part by the client).
|
||||||
pub const GROUNDING_PROMPT: &str = concat!(
|
/// Runtime-overridable default — see
|
||||||
|
/// [`crate::config::AnalysisConfig::grounding_prompt`].
|
||||||
|
pub const GROUNDING_PROMPT_DEFAULT: &str = concat!(
|
||||||
"You are a manga/comic page analyzer. You are given the page image ",
|
"You are a manga/comic page analyzer. You are given the page image ",
|
||||||
"(possibly downscaled) AND the OCR text already extracted from it. Using ",
|
"(possibly downscaled) AND the OCR text already extracted from it. Using ",
|
||||||
"both, return ONLY one minified JSON object with: tagging_results (5-10 ",
|
"both, return ONLY one minified JSON object with: tagging_results (5-10 ",
|
||||||
@@ -230,7 +235,7 @@ mod tests {
|
|||||||
#[test]
|
#[test]
|
||||||
fn system_prompt_mentions_the_schema_keys() {
|
fn system_prompt_mentions_the_schema_keys() {
|
||||||
for key in ["ocr_results", "tagging_results", "scene_description", "safety_flag"] {
|
for key in ["ocr_results", "tagging_results", "scene_description", "safety_flag"] {
|
||||||
assert!(SYSTEM_PROMPT.contains(key), "prompt missing {key}");
|
assert!(SYSTEM_PROMPT_DEFAULT.contains(key), "prompt missing {key}");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -37,6 +37,13 @@ pub struct VisionClient {
|
|||||||
max_tokens: u32,
|
max_tokens: u32,
|
||||||
response_format: ResponseFormat,
|
response_format: ResponseFormat,
|
||||||
frequency_penalty: f64,
|
frequency_penalty: f64,
|
||||||
|
temperature: f64,
|
||||||
|
/// System prompt for the single-call (normal-aspect) path.
|
||||||
|
system_prompt: String,
|
||||||
|
/// Pass-A OCR-only prompt (tall-page slices).
|
||||||
|
ocr_prompt: String,
|
||||||
|
/// Pass-B grounding prompt (tags/scene/safety).
|
||||||
|
grounding_prompt: String,
|
||||||
slice: SliceParams,
|
slice: SliceParams,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -60,6 +67,10 @@ impl VisionClient {
|
|||||||
max_tokens: cfg.max_tokens,
|
max_tokens: cfg.max_tokens,
|
||||||
response_format: cfg.response_format,
|
response_format: cfg.response_format,
|
||||||
frequency_penalty: cfg.frequency_penalty,
|
frequency_penalty: cfg.frequency_penalty,
|
||||||
|
temperature: cfg.temperature,
|
||||||
|
system_prompt: cfg.system_prompt.clone(),
|
||||||
|
ocr_prompt: cfg.ocr_prompt.clone(),
|
||||||
|
grounding_prompt: cfg.grounding_prompt.clone(),
|
||||||
slice: SliceParams {
|
slice: SliceParams {
|
||||||
max_pixels: cfg.max_pixels,
|
max_pixels: cfg.max_pixels,
|
||||||
min_slice_height: cfg.min_slice_height,
|
min_slice_height: cfg.min_slice_height,
|
||||||
@@ -83,6 +94,8 @@ impl VisionClient {
|
|||||||
&url,
|
&url,
|
||||||
self.response_format,
|
self.response_format,
|
||||||
self.frequency_penalty,
|
self.frequency_penalty,
|
||||||
|
self.temperature,
|
||||||
|
&self.system_prompt,
|
||||||
);
|
);
|
||||||
return parse_chat_completion(&self.post_chat(body).await?);
|
return parse_chat_completion(&self.post_chat(body).await?);
|
||||||
};
|
};
|
||||||
@@ -97,6 +110,8 @@ impl VisionClient {
|
|||||||
&data_url(&jpeg),
|
&data_url(&jpeg),
|
||||||
self.response_format,
|
self.response_format,
|
||||||
self.frequency_penalty,
|
self.frequency_penalty,
|
||||||
|
self.temperature,
|
||||||
|
&self.system_prompt,
|
||||||
);
|
);
|
||||||
parse_chat_completion(&self.post_chat(body).await?)
|
parse_chat_completion(&self.post_chat(body).await?)
|
||||||
}
|
}
|
||||||
@@ -124,6 +139,8 @@ impl VisionClient {
|
|||||||
self.max_tokens,
|
self.max_tokens,
|
||||||
self.response_format,
|
self.response_format,
|
||||||
self.frequency_penalty,
|
self.frequency_penalty,
|
||||||
|
self.temperature,
|
||||||
|
&self.ocr_prompt,
|
||||||
&data_url(&jpeg),
|
&data_url(&jpeg),
|
||||||
);
|
);
|
||||||
let parsed = parse_chat_completion(&self.post_chat(body).await?)?;
|
let parsed = parse_chat_completion(&self.post_chat(body).await?)?;
|
||||||
@@ -148,6 +165,8 @@ impl VisionClient {
|
|||||||
self.max_tokens,
|
self.max_tokens,
|
||||||
self.response_format,
|
self.response_format,
|
||||||
self.frequency_penalty,
|
self.frequency_penalty,
|
||||||
|
self.temperature,
|
||||||
|
&self.grounding_prompt,
|
||||||
&data_url(&whole),
|
&data_url(&whole),
|
||||||
&ocr_text,
|
&ocr_text,
|
||||||
);
|
);
|
||||||
@@ -451,19 +470,23 @@ fn similar(a: &str, b: &str) -> bool {
|
|||||||
|
|
||||||
/// Combined single-call body (OCR + tags + scene + safety). Public + this
|
/// Combined single-call body (OCR + tags + scene + safety). Public + this
|
||||||
/// exact signature so the existing tests keep working.
|
/// exact signature so the existing tests keep working.
|
||||||
|
#[allow(clippy::too_many_arguments)]
|
||||||
pub fn build_request_body(
|
pub fn build_request_body(
|
||||||
model: &str,
|
model: &str,
|
||||||
max_tokens: u32,
|
max_tokens: u32,
|
||||||
data_url: &str,
|
data_url: &str,
|
||||||
response_format: ResponseFormat,
|
response_format: ResponseFormat,
|
||||||
frequency_penalty: f64,
|
frequency_penalty: f64,
|
||||||
|
temperature: f64,
|
||||||
|
system_prompt: &str,
|
||||||
) -> serde_json::Value {
|
) -> serde_json::Value {
|
||||||
build_body(
|
build_body(
|
||||||
model,
|
model,
|
||||||
max_tokens,
|
max_tokens,
|
||||||
response_format,
|
response_format,
|
||||||
frequency_penalty,
|
frequency_penalty,
|
||||||
prompt::SYSTEM_PROMPT,
|
temperature,
|
||||||
|
system_prompt,
|
||||||
"page_analysis",
|
"page_analysis",
|
||||||
prompt::output_json_schema(),
|
prompt::output_json_schema(),
|
||||||
data_url,
|
data_url,
|
||||||
@@ -472,11 +495,14 @@ pub fn build_request_body(
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Pass-A OCR-only body.
|
/// Pass-A OCR-only body.
|
||||||
|
#[allow(clippy::too_many_arguments)]
|
||||||
fn build_ocr_body(
|
fn build_ocr_body(
|
||||||
model: &str,
|
model: &str,
|
||||||
max_tokens: u32,
|
max_tokens: u32,
|
||||||
response_format: ResponseFormat,
|
response_format: ResponseFormat,
|
||||||
frequency_penalty: f64,
|
frequency_penalty: f64,
|
||||||
|
temperature: f64,
|
||||||
|
ocr_prompt: &str,
|
||||||
data_url: &str,
|
data_url: &str,
|
||||||
) -> serde_json::Value {
|
) -> serde_json::Value {
|
||||||
build_body(
|
build_body(
|
||||||
@@ -484,7 +510,8 @@ fn build_ocr_body(
|
|||||||
max_tokens,
|
max_tokens,
|
||||||
response_format,
|
response_format,
|
||||||
frequency_penalty,
|
frequency_penalty,
|
||||||
prompt::OCR_PROMPT,
|
temperature,
|
||||||
|
ocr_prompt,
|
||||||
"page_ocr",
|
"page_ocr",
|
||||||
prompt::ocr_json_schema(),
|
prompt::ocr_json_schema(),
|
||||||
data_url,
|
data_url,
|
||||||
@@ -493,11 +520,14 @@ fn build_ocr_body(
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Pass-B grounding body — whole image + the merged OCR text as context.
|
/// Pass-B grounding body — whole image + the merged OCR text as context.
|
||||||
|
#[allow(clippy::too_many_arguments)]
|
||||||
fn build_grounding_body(
|
fn build_grounding_body(
|
||||||
model: &str,
|
model: &str,
|
||||||
max_tokens: u32,
|
max_tokens: u32,
|
||||||
response_format: ResponseFormat,
|
response_format: ResponseFormat,
|
||||||
frequency_penalty: f64,
|
frequency_penalty: f64,
|
||||||
|
temperature: f64,
|
||||||
|
grounding_prompt: &str,
|
||||||
data_url: &str,
|
data_url: &str,
|
||||||
ocr_text: &str,
|
ocr_text: &str,
|
||||||
) -> serde_json::Value {
|
) -> serde_json::Value {
|
||||||
@@ -507,7 +537,8 @@ fn build_grounding_body(
|
|||||||
max_tokens,
|
max_tokens,
|
||||||
response_format,
|
response_format,
|
||||||
frequency_penalty,
|
frequency_penalty,
|
||||||
prompt::GROUNDING_PROMPT,
|
temperature,
|
||||||
|
grounding_prompt,
|
||||||
"page_grounding",
|
"page_grounding",
|
||||||
prompt::grounding_json_schema(),
|
prompt::grounding_json_schema(),
|
||||||
data_url,
|
data_url,
|
||||||
@@ -524,6 +555,7 @@ fn build_body(
|
|||||||
max_tokens: u32,
|
max_tokens: u32,
|
||||||
response_format: ResponseFormat,
|
response_format: ResponseFormat,
|
||||||
frequency_penalty: f64,
|
frequency_penalty: f64,
|
||||||
|
temperature: f64,
|
||||||
system_prompt: &str,
|
system_prompt: &str,
|
||||||
schema_name: &str,
|
schema_name: &str,
|
||||||
schema: serde_json::Value,
|
schema: serde_json::Value,
|
||||||
@@ -537,7 +569,7 @@ fn build_body(
|
|||||||
}
|
}
|
||||||
let mut body = json!({
|
let mut body = json!({
|
||||||
"model": model,
|
"model": model,
|
||||||
"temperature": 0,
|
"temperature": temperature,
|
||||||
"max_tokens": max_tokens,
|
"max_tokens": max_tokens,
|
||||||
"messages": [
|
"messages": [
|
||||||
{ "role": "system", "content": system_prompt },
|
{ "role": "system", "content": system_prompt },
|
||||||
@@ -740,8 +772,15 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn combined_body_uses_full_schema_and_image() {
|
fn combined_body_uses_full_schema_and_image() {
|
||||||
let body =
|
let body = build_request_body(
|
||||||
build_request_body("m", 100, "data:image/png;base64,AA", ResponseFormat::JsonSchema, 0.3);
|
"m",
|
||||||
|
100,
|
||||||
|
"data:image/png;base64,AA",
|
||||||
|
ResponseFormat::JsonSchema,
|
||||||
|
0.3,
|
||||||
|
0.0,
|
||||||
|
prompt::SYSTEM_PROMPT_DEFAULT,
|
||||||
|
);
|
||||||
assert_eq!(body["messages"][0]["role"], "system");
|
assert_eq!(body["messages"][0]["role"], "system");
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
body["messages"][1]["content"][0]["image_url"]["url"],
|
body["messages"][1]["content"][0]["image_url"]["url"],
|
||||||
@@ -752,9 +791,25 @@ mod tests {
|
|||||||
assert!(req.as_array().unwrap().iter().any(|v| v == "safety_flag"));
|
assert!(req.as_array().unwrap().iter().any(|v| v == "safety_flag"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn body_carries_the_configured_prompt_and_temperature() {
|
||||||
|
let body = build_request_body(
|
||||||
|
"m",
|
||||||
|
100,
|
||||||
|
"d",
|
||||||
|
ResponseFormat::None,
|
||||||
|
0.0,
|
||||||
|
0.7,
|
||||||
|
"CUSTOM PROMPT",
|
||||||
|
);
|
||||||
|
assert_eq!(body["messages"][0]["content"], "CUSTOM PROMPT");
|
||||||
|
assert_eq!(body["temperature"], 0.7);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn ocr_body_carries_only_the_ocr_schema_and_caps() {
|
fn ocr_body_carries_only_the_ocr_schema_and_caps() {
|
||||||
let body = build_ocr_body("m", 100, ResponseFormat::JsonSchema, 0.3, "d");
|
let body =
|
||||||
|
build_ocr_body("m", 100, ResponseFormat::JsonSchema, 0.3, 0.0, prompt::OCR_PROMPT_DEFAULT, "d");
|
||||||
let schema = &body["response_format"]["json_schema"]["schema"];
|
let schema = &body["response_format"]["json_schema"]["schema"];
|
||||||
assert_eq!(body["response_format"]["json_schema"]["name"], "page_ocr");
|
assert_eq!(body["response_format"]["json_schema"]["name"], "page_ocr");
|
||||||
assert_eq!(schema["required"], json!(["ocr_results"]));
|
assert_eq!(schema["required"], json!(["ocr_results"]));
|
||||||
@@ -769,8 +824,16 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn grounding_body_includes_ocr_context_and_grounding_schema() {
|
fn grounding_body_includes_ocr_context_and_grounding_schema() {
|
||||||
let body =
|
let body = build_grounding_body(
|
||||||
build_grounding_body("m", 100, ResponseFormat::JsonSchema, 0.3, "d", "[speech] Hi");
|
"m",
|
||||||
|
100,
|
||||||
|
ResponseFormat::JsonSchema,
|
||||||
|
0.3,
|
||||||
|
0.0,
|
||||||
|
prompt::GROUNDING_PROMPT_DEFAULT,
|
||||||
|
"d",
|
||||||
|
"[speech] Hi",
|
||||||
|
);
|
||||||
assert_eq!(body["response_format"]["json_schema"]["name"], "page_grounding");
|
assert_eq!(body["response_format"]["json_schema"]["name"], "page_grounding");
|
||||||
let parts = body["messages"][1]["content"].as_array().unwrap();
|
let parts = body["messages"][1]["content"].as_array().unwrap();
|
||||||
assert_eq!(parts.len(), 2, "image + text parts");
|
assert_eq!(parts.len(), 2, "image + text parts");
|
||||||
@@ -781,15 +844,25 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn response_format_none_omits_the_field() {
|
fn response_format_none_omits_the_field() {
|
||||||
let body = build_request_body("m", 100, "d", ResponseFormat::None, 0.3);
|
let body = build_request_body(
|
||||||
|
"m",
|
||||||
|
100,
|
||||||
|
"d",
|
||||||
|
ResponseFormat::None,
|
||||||
|
0.3,
|
||||||
|
0.0,
|
||||||
|
prompt::SYSTEM_PROMPT_DEFAULT,
|
||||||
|
);
|
||||||
assert!(body.get("response_format").is_none());
|
assert!(body.get("response_format").is_none());
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn frequency_penalty_included_only_when_nonzero() {
|
fn frequency_penalty_included_only_when_nonzero() {
|
||||||
let on = build_ocr_body("m", 100, ResponseFormat::None, 0.4, "d");
|
let on =
|
||||||
|
build_ocr_body("m", 100, ResponseFormat::None, 0.4, 0.0, prompt::OCR_PROMPT_DEFAULT, "d");
|
||||||
assert_eq!(on["frequency_penalty"], 0.4);
|
assert_eq!(on["frequency_penalty"], 0.4);
|
||||||
let off = build_ocr_body("m", 100, ResponseFormat::None, 0.0, "d");
|
let off =
|
||||||
|
build_ocr_body("m", 100, ResponseFormat::None, 0.0, 0.0, prompt::OCR_PROMPT_DEFAULT, "d");
|
||||||
assert!(off.get("frequency_penalty").is_none());
|
assert!(off.get("frequency_penalty").is_none());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -188,7 +188,7 @@ pub struct AnalyzePageResponse {
|
|||||||
/// Reject the request with 503 when the analysis worker is disabled, so
|
/// Reject the request with 503 when the analysis worker is disabled, so
|
||||||
/// an operator doesn't pile up jobs that nothing will ever drain.
|
/// an operator doesn't pile up jobs that nothing will ever drain.
|
||||||
fn ensure_enabled(state: &AppState) -> AppResult<()> {
|
fn ensure_enabled(state: &AppState) -> AppResult<()> {
|
||||||
if state.analysis_enabled {
|
if state.analysis_enabled() {
|
||||||
Ok(())
|
Ok(())
|
||||||
} else {
|
} else {
|
||||||
Err(AppError::ServiceUnavailable(
|
Err(AppError::ServiceUnavailable(
|
||||||
|
|||||||
@@ -37,13 +37,14 @@ pub(super) fn default_limit() -> i64 {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Shared 503 helper: the daemon-only control + observability endpoints
|
/// Shared 503 helper: the daemon-only control + observability endpoints
|
||||||
/// gate on `AppState.crawler == Some(_)`. Returns the same code and body
|
/// gate on a running crawler daemon. Returns the same code and body across
|
||||||
/// across every caller so the frontend can rely on `service_unavailable`
|
/// every caller so the frontend can rely on `service_unavailable` rather
|
||||||
/// rather than each handler returning its own variant.
|
/// than each handler returning its own variant. Yields an owned `Arc` (the
|
||||||
|
/// control handle is swapped on a config reload).
|
||||||
pub(super) fn require_crawler(
|
pub(super) fn require_crawler(
|
||||||
state: &AppState,
|
state: &AppState,
|
||||||
) -> Result<&std::sync::Arc<CrawlerControl>, AppError> {
|
) -> Result<std::sync::Arc<CrawlerControl>, AppError> {
|
||||||
state.crawler.as_ref().ok_or_else(|| {
|
state.crawler().ok_or_else(|| {
|
||||||
AppError::ServiceUnavailable("crawler daemon is disabled".into())
|
AppError::ServiceUnavailable("crawler daemon is disabled".into())
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -140,7 +140,7 @@ async fn compose_status(
|
|||||||
dead,
|
dead,
|
||||||
};
|
};
|
||||||
|
|
||||||
Ok(match state.crawler.as_ref() {
|
Ok(match state.crawler().as_ref() {
|
||||||
None => CrawlerStatusResponse {
|
None => CrawlerStatusResponse {
|
||||||
daemon: "disabled",
|
daemon: "disabled",
|
||||||
phase: None,
|
phase: None,
|
||||||
@@ -200,7 +200,7 @@ async fn stream_status(
|
|||||||
) -> Sse<impl Stream<Item = Result<Event, Infallible>>> {
|
) -> Sse<impl Stream<Item = Result<Event, Infallible>>> {
|
||||||
// Subscribe before the first emit so no change between the initial
|
// Subscribe before the first emit so no change between the initial
|
||||||
// snapshot and the first await is lost.
|
// snapshot and the first await is lost.
|
||||||
let rx = state.crawler.as_ref().map(|c| c.status.subscribe());
|
let rx = state.crawler().as_ref().map(|c| c.status.subscribe());
|
||||||
let memo = QueueCountsMemo::new();
|
let memo = QueueCountsMemo::new();
|
||||||
|
|
||||||
let stream = futures_util::stream::unfold(
|
let stream = futures_util::stream::unfold(
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ pub mod analysis;
|
|||||||
pub mod crawler;
|
pub mod crawler;
|
||||||
pub mod mangas;
|
pub mod mangas;
|
||||||
pub mod resync;
|
pub mod resync;
|
||||||
|
pub mod settings;
|
||||||
pub mod system;
|
pub mod system;
|
||||||
pub mod users;
|
pub mod users;
|
||||||
|
|
||||||
@@ -23,4 +24,5 @@ pub fn routes() -> Router<AppState> {
|
|||||||
.merge(system::routes())
|
.merge(system::routes())
|
||||||
.merge(crawler::routes())
|
.merge(crawler::routes())
|
||||||
.merge(analysis::routes())
|
.merge(analysis::routes())
|
||||||
|
.merge(settings::routes())
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -58,8 +58,7 @@ async fn resync_manga(
|
|||||||
return Err(AppError::NotFound);
|
return Err(AppError::NotFound);
|
||||||
}
|
}
|
||||||
let resync = state
|
let resync = state
|
||||||
.resync
|
.resync()
|
||||||
.as_ref()
|
|
||||||
.ok_or_else(|| AppError::ServiceUnavailable(
|
.ok_or_else(|| AppError::ServiceUnavailable(
|
||||||
"crawler daemon is disabled; force resync unavailable".into(),
|
"crawler daemon is disabled; force resync unavailable".into(),
|
||||||
))?;
|
))?;
|
||||||
@@ -96,8 +95,7 @@ async fn resync_chapter(
|
|||||||
Path(chapter_id): Path<Uuid>,
|
Path(chapter_id): Path<Uuid>,
|
||||||
) -> AppResult<Json<ChapterResyncResponse>> {
|
) -> AppResult<Json<ChapterResyncResponse>> {
|
||||||
let resync = state
|
let resync = state
|
||||||
.resync
|
.resync()
|
||||||
.as_ref()
|
|
||||||
.ok_or_else(|| AppError::ServiceUnavailable(
|
.ok_or_else(|| AppError::ServiceUnavailable(
|
||||||
"crawler daemon is disabled; force resync unavailable".into(),
|
"crawler daemon is disabled; force resync unavailable".into(),
|
||||||
))?;
|
))?;
|
||||||
|
|||||||
239
backend/src/api/admin/settings.rs
Normal file
239
backend/src/api/admin/settings.rs
Normal file
@@ -0,0 +1,239 @@
|
|||||||
|
//! Admin endpoints for runtime-editable crawler / analysis configuration.
|
||||||
|
//!
|
||||||
|
//! `GET` returns the current editable settings DTO plus a read-only view of
|
||||||
|
//! the env-managed (host/infra + secret) fields, so the dashboard can show
|
||||||
|
//! the full picture while only letting the operator edit the safe knobs.
|
||||||
|
//!
|
||||||
|
//! `PUT` validates the incoming DTO against the env-derived base (re-parsing
|
||||||
|
//! timezone/time, rebuilding the download allowlist), persists a normalized
|
||||||
|
//! DTO and an audit row in one transaction, then asks the [`DaemonReloader`]
|
||||||
|
//! to gracefully respawn the affected daemon with the new config — so the
|
||||||
|
//! change takes effect without a restart. Validation failures return 422 with
|
||||||
|
//! per-field details (the established `validation_failed` envelope).
|
||||||
|
//!
|
||||||
|
//! `PUT` is a **full replace**, not a patch: the body is deserialized with
|
||||||
|
//! field-level defaults, so any omitted field is reset to its compiled
|
||||||
|
//! default rather than retaining the stored value. The dashboard always
|
||||||
|
//! sends the complete DTO (it round-trips the one it loaded); programmatic
|
||||||
|
//! callers must do the same.
|
||||||
|
|
||||||
|
use axum::extract::State;
|
||||||
|
use axum::routing::get;
|
||||||
|
use axum::{Json, Router};
|
||||||
|
use serde::Serialize;
|
||||||
|
|
||||||
|
use crate::app::AppState;
|
||||||
|
use crate::auth::extractor::RequireAdmin;
|
||||||
|
use crate::config::CrawlerConfig;
|
||||||
|
use crate::crawler::browser::BrowserMode;
|
||||||
|
use crate::error::{AppError, AppResult};
|
||||||
|
use crate::repo;
|
||||||
|
use crate::settings::{
|
||||||
|
AnalysisSettings, CrawlerSettings, FieldErrors, PromptDefaults, KEY_ANALYSIS, KEY_CRAWLER,
|
||||||
|
};
|
||||||
|
|
||||||
|
pub fn routes() -> Router<AppState> {
|
||||||
|
Router::new()
|
||||||
|
.route(
|
||||||
|
"/admin/settings/crawler",
|
||||||
|
get(get_crawler).put(put_crawler),
|
||||||
|
)
|
||||||
|
.route(
|
||||||
|
"/admin/settings/analysis",
|
||||||
|
get(get_analysis).put(put_analysis),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Response shapes
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/// Read-only mirror of the crawler fields managed via environment (host/infra
|
||||||
|
/// + secrets), surfaced so the admin sees what's in effect without editing it.
|
||||||
|
#[derive(Debug, Serialize)]
|
||||||
|
struct CrawlerEnvOnly {
|
||||||
|
browser_mode: &'static str,
|
||||||
|
browser_args: Vec<String>,
|
||||||
|
proxy: Option<String>,
|
||||||
|
tor_control_url: Option<String>,
|
||||||
|
/// Whether a TOR control credential (password or cookie file) is set.
|
||||||
|
tor_credentials_configured: bool,
|
||||||
|
/// Whether an initial `CRAWLER_PHPSESSID` is configured (the live session
|
||||||
|
/// is managed separately via the crawler dashboard).
|
||||||
|
session_configured: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl CrawlerEnvOnly {
|
||||||
|
fn from_base(base: &CrawlerConfig) -> Self {
|
||||||
|
Self {
|
||||||
|
browser_mode: match base.browser.mode {
|
||||||
|
BrowserMode::Headed => "headed",
|
||||||
|
BrowserMode::Headless => "headless",
|
||||||
|
},
|
||||||
|
browser_args: base.browser.extra_args.clone(),
|
||||||
|
proxy: base.proxy.clone(),
|
||||||
|
tor_control_url: base.tor_control_url.clone(),
|
||||||
|
tor_credentials_configured: base.tor_control_password.is_some()
|
||||||
|
|| base.tor_control_cookie_path.is_some(),
|
||||||
|
session_configured: base.phpsessid.is_some(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize)]
|
||||||
|
struct CrawlerSettingsResponse {
|
||||||
|
editable: CrawlerSettings,
|
||||||
|
env_only: CrawlerEnvOnly,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Read-only mirror of analysis env-managed fields (just the secret).
|
||||||
|
#[derive(Debug, Serialize)]
|
||||||
|
struct AnalysisEnvOnly {
|
||||||
|
api_key_configured: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize)]
|
||||||
|
struct AnalysisSettingsResponse {
|
||||||
|
editable: AnalysisSettings,
|
||||||
|
env_only: AnalysisEnvOnly,
|
||||||
|
/// The compiled prompt defaults so the UI can show placeholders and
|
||||||
|
/// implement per-prompt "reset to default".
|
||||||
|
prompt_defaults: PromptDefaults,
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Handlers
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
fn validation_error(errs: FieldErrors) -> AppError {
|
||||||
|
AppError::ValidationFailed {
|
||||||
|
message: "invalid settings".to_string(),
|
||||||
|
details: serde_json::json!({ "fields": errs.errors }),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Current effective crawler settings: the stored DTO when present, otherwise
|
||||||
|
/// derived from the env base (the row is seeded at boot, so the stored branch
|
||||||
|
/// is the norm).
|
||||||
|
async fn load_crawler_dto(state: &AppState) -> AppResult<CrawlerSettings> {
|
||||||
|
Ok(match repo::app_settings::get(&state.db, KEY_CRAWLER).await? {
|
||||||
|
Some(v) => serde_json::from_value(v)
|
||||||
|
.unwrap_or_else(|_| CrawlerSettings::from_config(&state.crawler_base)),
|
||||||
|
None => CrawlerSettings::from_config(&state.crawler_base),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn load_analysis_dto(state: &AppState) -> AppResult<AnalysisSettings> {
|
||||||
|
Ok(match repo::app_settings::get(&state.db, KEY_ANALYSIS).await? {
|
||||||
|
Some(v) => serde_json::from_value(v)
|
||||||
|
.unwrap_or_else(|_| AnalysisSettings::from_config(&state.analysis_base)),
|
||||||
|
None => AnalysisSettings::from_config(&state.analysis_base),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn get_crawler(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
_admin: RequireAdmin,
|
||||||
|
) -> AppResult<Json<CrawlerSettingsResponse>> {
|
||||||
|
let editable = load_crawler_dto(&state).await?;
|
||||||
|
Ok(Json(CrawlerSettingsResponse {
|
||||||
|
editable,
|
||||||
|
env_only: CrawlerEnvOnly::from_base(&state.crawler_base),
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn put_crawler(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
admin: RequireAdmin,
|
||||||
|
Json(incoming): Json<CrawlerSettings>,
|
||||||
|
) -> AppResult<Json<CrawlerSettingsResponse>> {
|
||||||
|
// Validate by converting against the env base; this also rebuilds the
|
||||||
|
// download allowlist and re-parses tz / time.
|
||||||
|
let cfg = incoming
|
||||||
|
.to_config(&state.crawler_base)
|
||||||
|
.map_err(validation_error)?;
|
||||||
|
// Persist a normalized DTO (so the stored/returned allowlist reflects the
|
||||||
|
// effective hosts, prompts are canonicalized, etc.).
|
||||||
|
let normalized = CrawlerSettings::from_config(&cfg);
|
||||||
|
let value = serde_json::to_value(&normalized).map_err(|e| AppError::Other(e.into()))?;
|
||||||
|
|
||||||
|
let mut tx = state.db.begin().await?;
|
||||||
|
repo::app_settings::upsert(&mut *tx, KEY_CRAWLER, &value).await?;
|
||||||
|
repo::admin_audit::insert(
|
||||||
|
&mut *tx,
|
||||||
|
admin.0.id,
|
||||||
|
"update_crawler_settings",
|
||||||
|
"settings",
|
||||||
|
None,
|
||||||
|
value.clone(),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
tx.commit().await?;
|
||||||
|
|
||||||
|
// Apply live (graceful respawn) when a reloader is wired up.
|
||||||
|
if let Some(reloader) = &state.reloader {
|
||||||
|
reloader
|
||||||
|
.reload_crawler(cfg)
|
||||||
|
.await
|
||||||
|
.map_err(AppError::Other)?;
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(Json(CrawlerSettingsResponse {
|
||||||
|
editable: normalized,
|
||||||
|
env_only: CrawlerEnvOnly::from_base(&state.crawler_base),
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn get_analysis(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
_admin: RequireAdmin,
|
||||||
|
) -> AppResult<Json<AnalysisSettingsResponse>> {
|
||||||
|
let editable = load_analysis_dto(&state).await?;
|
||||||
|
Ok(Json(AnalysisSettingsResponse {
|
||||||
|
editable,
|
||||||
|
env_only: AnalysisEnvOnly {
|
||||||
|
api_key_configured: state.analysis_base.api_key.is_some(),
|
||||||
|
},
|
||||||
|
prompt_defaults: PromptDefaults::get(),
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn put_analysis(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
admin: RequireAdmin,
|
||||||
|
Json(incoming): Json<AnalysisSettings>,
|
||||||
|
) -> AppResult<Json<AnalysisSettingsResponse>> {
|
||||||
|
let cfg = incoming
|
||||||
|
.to_config(&state.analysis_base)
|
||||||
|
.map_err(validation_error)?;
|
||||||
|
let normalized = AnalysisSettings::from_config(&cfg);
|
||||||
|
let value = serde_json::to_value(&normalized).map_err(|e| AppError::Other(e.into()))?;
|
||||||
|
|
||||||
|
let mut tx = state.db.begin().await?;
|
||||||
|
repo::app_settings::upsert(&mut *tx, KEY_ANALYSIS, &value).await?;
|
||||||
|
repo::admin_audit::insert(
|
||||||
|
&mut *tx,
|
||||||
|
admin.0.id,
|
||||||
|
"update_analysis_settings",
|
||||||
|
"settings",
|
||||||
|
None,
|
||||||
|
value.clone(),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
tx.commit().await?;
|
||||||
|
|
||||||
|
if let Some(reloader) = &state.reloader {
|
||||||
|
reloader
|
||||||
|
.reload_analysis(cfg)
|
||||||
|
.await
|
||||||
|
.map_err(AppError::Other)?;
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(Json(AnalysisSettingsResponse {
|
||||||
|
editable: normalized,
|
||||||
|
env_only: AnalysisEnvOnly {
|
||||||
|
api_key_configured: state.analysis_base.api_key.is_some(),
|
||||||
|
},
|
||||||
|
prompt_defaults: PromptDefaults::get(),
|
||||||
|
}))
|
||||||
|
}
|
||||||
@@ -161,7 +161,7 @@ async fn create(
|
|||||||
// rolled-back upload never leaves jobs pointing at nonexistent pages; a
|
// rolled-back upload never leaves jobs pointing at nonexistent pages; a
|
||||||
// failed enqueue is logged but doesn't fail the upload (the admin
|
// failed enqueue is logged but doesn't fail the upload (the admin
|
||||||
// re-enqueue endpoint can backfill).
|
// re-enqueue endpoint can backfill).
|
||||||
if state.analysis_enabled {
|
if state.analysis_enabled() {
|
||||||
for page_id in page_ids {
|
for page_id in page_ids {
|
||||||
if let Err(e) =
|
if let Err(e) =
|
||||||
repo::page_analysis::enqueue_for_page(&state.db, page_id, false).await
|
repo::page_analysis::enqueue_for_page(&state.db, page_id, false).await
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
|
use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
|
||||||
|
use std::sync::RwLock as StdRwLock;
|
||||||
|
|
||||||
use anyhow::Context;
|
use anyhow::Context;
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
@@ -17,7 +18,7 @@ use tower_http::trace::TraceLayer;
|
|||||||
use crate::auth::extractor::CurrentUser;
|
use crate::auth::extractor::CurrentUser;
|
||||||
use crate::auth::rate_limit::AuthRateLimiter;
|
use crate::auth::rate_limit::AuthRateLimiter;
|
||||||
use crate::error::AppError;
|
use crate::error::AppError;
|
||||||
use crate::config::{AuthConfig, Config, CrawlerConfig, UploadConfig};
|
use crate::config::{AnalysisConfig, AuthConfig, Config, CrawlerConfig, UploadConfig};
|
||||||
use crate::crawler::browser_manager::{self, BrowserManager};
|
use crate::crawler::browser_manager::{self, BrowserManager};
|
||||||
use crate::crawler::content::{self, SyncOutcome};
|
use crate::crawler::content::{self, SyncOutcome};
|
||||||
use crate::crawler::daemon::{self, ChapterDispatcher, DaemonConfig, MetadataPass};
|
use crate::crawler::daemon::{self, ChapterDispatcher, DaemonConfig, MetadataPass};
|
||||||
@@ -40,33 +41,125 @@ pub struct AppState {
|
|||||||
/// One instance per AppState so tests stay isolated across the
|
/// One instance per AppState so tests stay isolated across the
|
||||||
/// same process.
|
/// same process.
|
||||||
pub auth_limiter: Arc<AuthRateLimiter>,
|
pub auth_limiter: Arc<AuthRateLimiter>,
|
||||||
/// Admin-triggered force resync. `None` when the crawler daemon
|
/// Runtime-swappable controls (crawler resync/control handles + the
|
||||||
/// is disabled (`CRAWLER_DAEMON=false`); admin handlers gate on
|
/// analysis enable gate). Shared with the [`Supervisors`] so a config
|
||||||
/// `.is_some()` and return 503 otherwise. Set by [`build`] from the
|
/// reload that respawns a daemon updates what handlers see, live —
|
||||||
/// same wiring that builds the daemon's chapter dispatcher, so a
|
/// without a restart. In tests this starts empty (no daemons).
|
||||||
/// force resync uses the daemon's BrowserManager + rate limiters.
|
pub runtime: Arc<RuntimeControls>,
|
||||||
pub resync: Option<Arc<dyn ResyncService>>,
|
/// Applies a persisted settings change to the running daemons
|
||||||
/// Crawler observability + control handle (live status, coordinated
|
/// (graceful stop + respawn). `Some` in production ([`build`]); `None`
|
||||||
/// browser restart, runtime session, manual run). `None` when the
|
/// in the test harness, where settings still persist but no daemon is
|
||||||
/// daemon is disabled; admin handlers gate on `.is_some()` → 503.
|
/// spawned. See [`DaemonReloader`].
|
||||||
pub crawler: Option<Arc<CrawlerControl>>,
|
pub reloader: Option<Arc<dyn DaemonReloader>>,
|
||||||
|
/// Env-derived crawler config — the **base** the stored settings DTO is
|
||||||
|
/// overlaid onto (carries env-only fields: browser, proxy, TOR, session).
|
||||||
|
pub crawler_base: CrawlerConfig,
|
||||||
|
/// Env-derived analysis config base (carries the env-only vision API key).
|
||||||
|
pub analysis_base: AnalysisConfig,
|
||||||
/// Browser origins permitted to issue mutating requests to
|
/// Browser origins permitted to issue mutating requests to
|
||||||
/// `/api/v1/admin/*`. See [`crate::config::Config::admin_allowed_origins`]
|
/// `/api/v1/admin/*`. See [`crate::config::Config::admin_allowed_origins`]
|
||||||
/// for the policy. Cloned per-request into the CSRF middleware; the
|
/// for the policy. Cloned per-request into the CSRF middleware; the
|
||||||
/// `Arc` keeps the clone cheap. Empty list → check is skipped
|
/// `Arc` keeps the clone cheap. Empty list → check is skipped
|
||||||
/// (operator opt-out documented in `.env.example`).
|
/// (operator opt-out documented in `.env.example`).
|
||||||
pub admin_allowed_origins: Arc<Vec<String>>,
|
pub admin_allowed_origins: Arc<Vec<String>>,
|
||||||
/// When `true`, page-create paths (upload + crawler) enqueue
|
|
||||||
/// `analyze_page` jobs and the admin re-enqueue endpoints are active.
|
|
||||||
/// Mirrors `config.analysis.enabled`; defaults to `false` so the
|
|
||||||
/// feature is opt-in and the test harness inherits a no-op gate.
|
|
||||||
pub analysis_enabled: bool,
|
|
||||||
/// Live analysis-event broadcaster. Always present (the worker and the
|
/// Live analysis-event broadcaster. Always present (the worker and the
|
||||||
/// admin enqueue path publish here; the SSE endpoint subscribes). When
|
/// admin enqueue path publish here; the SSE endpoint subscribes). When
|
||||||
/// the worker is disabled the channel simply stays quiet.
|
/// the worker is disabled the channel simply stays quiet.
|
||||||
pub analysis_events: Arc<crate::analysis::events::AnalysisEvents>,
|
pub analysis_events: Arc<crate::analysis::events::AnalysisEvents>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl AppState {
|
||||||
|
/// Current crawler control handle, or `None` when the daemon is stopped.
|
||||||
|
pub fn crawler(&self) -> Option<Arc<CrawlerControl>> {
|
||||||
|
self.runtime.crawler_control()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Current force-resync service, or `None` when the daemon is stopped.
|
||||||
|
pub fn resync(&self) -> Option<Arc<dyn ResyncService>> {
|
||||||
|
self.runtime.resync()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Whether AI page analysis is currently enabled (read live).
|
||||||
|
pub fn analysis_enabled(&self) -> bool {
|
||||||
|
self.runtime.analysis_enabled()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The crawler control handles, swapped atomically when the daemon is
|
||||||
|
/// (re)spawned or stopped. Cloned cheaply on every read.
|
||||||
|
#[derive(Clone, Default)]
|
||||||
|
struct CrawlerControls {
|
||||||
|
resync: Option<Arc<dyn ResyncService>>,
|
||||||
|
control: Option<Arc<CrawlerControl>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Shared, runtime-mutable surface read by handlers and mutated by the
|
||||||
|
/// [`Supervisors`] on a config reload. The analysis gate is an `Arc<AtomicBool>`
|
||||||
|
/// so the crawler's chapter dispatcher (which enqueues `analyze_page` jobs)
|
||||||
|
/// sees toggles live without being respawned.
|
||||||
|
pub struct RuntimeControls {
|
||||||
|
crawler: StdRwLock<CrawlerControls>,
|
||||||
|
analysis_enabled: Arc<AtomicBool>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl RuntimeControls {
|
||||||
|
/// Empty controls (no crawler daemon) with the given initial analysis gate.
|
||||||
|
pub fn new(analysis_enabled: bool) -> Self {
|
||||||
|
Self {
|
||||||
|
crawler: StdRwLock::new(CrawlerControls::default()),
|
||||||
|
analysis_enabled: Arc::new(AtomicBool::new(analysis_enabled)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn analysis_enabled(&self) -> bool {
|
||||||
|
self.analysis_enabled.load(Ordering::Relaxed)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Flip the analysis gate. Used by the [`Supervisors`] on reload and by
|
||||||
|
/// the test harness's stub reloader.
|
||||||
|
pub fn set_analysis_enabled(&self, v: bool) {
|
||||||
|
self.analysis_enabled.store(v, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The shared gate handle, passed to the crawler chapter dispatcher.
|
||||||
|
fn analysis_gate(&self) -> Arc<AtomicBool> {
|
||||||
|
Arc::clone(&self.analysis_enabled)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn crawler_control(&self) -> Option<Arc<CrawlerControl>> {
|
||||||
|
self.crawler.read().unwrap().control.clone()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn resync(&self) -> Option<Arc<dyn ResyncService>> {
|
||||||
|
self.crawler.read().unwrap().resync.clone()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Test helper: install a resync service without a running daemon.
|
||||||
|
pub fn set_resync(&self, resync: Option<Arc<dyn ResyncService>>) {
|
||||||
|
self.crawler.write().unwrap().resync = resync;
|
||||||
|
}
|
||||||
|
|
||||||
|
fn set_crawler(
|
||||||
|
&self,
|
||||||
|
resync: Option<Arc<dyn ResyncService>>,
|
||||||
|
control: Option<Arc<CrawlerControl>>,
|
||||||
|
) {
|
||||||
|
let mut g = self.crawler.write().unwrap();
|
||||||
|
g.resync = resync;
|
||||||
|
g.control = control;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Applies a validated settings change to the running daemons by gracefully
|
||||||
|
/// stopping and respawning them with the new effective config (and updating
|
||||||
|
/// the shared [`RuntimeControls`]). Implemented by [`Supervisors`] in
|
||||||
|
/// production; the settings handlers call it after persisting.
|
||||||
|
#[async_trait]
|
||||||
|
pub trait DaemonReloader: Send + Sync {
|
||||||
|
async fn reload_crawler(&self, cfg: CrawlerConfig) -> anyhow::Result<()>;
|
||||||
|
async fn reload_analysis(&self, cfg: AnalysisConfig) -> anyhow::Result<()>;
|
||||||
|
}
|
||||||
|
|
||||||
/// Shared handle the admin crawler endpoints use to observe and control
|
/// Shared handle the admin crawler endpoints use to observe and control
|
||||||
/// the running daemon. Bundled so the handlers take one optional field on
|
/// the running daemon. Bundled so the handlers take one optional field on
|
||||||
/// `AppState` rather than many.
|
/// `AppState` rather than many.
|
||||||
@@ -90,27 +183,96 @@ pub struct CrawlerControl {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Bundle returned by [`build`]. The router is what `axum::serve` consumes;
|
/// Bundle returned by [`build`]. The router is what `axum::serve` consumes;
|
||||||
/// the daemon (when enabled) outlives the HTTP server and is awaited via
|
/// the [`Supervisors`] own the background daemons (when enabled), outlive the
|
||||||
/// [`AppHandle::shutdown`] after the listener has finished gracefully.
|
/// HTTP server, and are awaited via [`AppHandle::shutdown`] after the listener
|
||||||
|
/// has finished gracefully.
|
||||||
pub struct AppHandle {
|
pub struct AppHandle {
|
||||||
pub router: Router,
|
pub router: Router,
|
||||||
pub daemon: Option<daemon::DaemonHandle>,
|
pub supervisors: Arc<Supervisors>,
|
||||||
/// AI content-analysis worker daemon; `None` when `ANALYSIS_ENABLED`
|
|
||||||
/// is off. Awaited alongside the crawler daemon on shutdown.
|
|
||||||
pub analysis_daemon: Option<crate::analysis::daemon::AnalysisDaemonHandle>,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl AppHandle {
|
impl AppHandle {
|
||||||
pub async fn shutdown(self) {
|
pub async fn shutdown(self) {
|
||||||
if let Some(d) = self.daemon {
|
self.supervisors.shutdown().await;
|
||||||
d.shutdown().await;
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Owns the crawler + analysis daemon handles and respawns them on a config
|
||||||
|
/// reload. Holds the build-time inputs (db, storage, env-base configs, the
|
||||||
|
/// shared event bus and [`RuntimeControls`]) needed to spawn from scratch.
|
||||||
|
pub struct Supervisors {
|
||||||
|
db: PgPool,
|
||||||
|
storage: Arc<dyn Storage>,
|
||||||
|
runtime: Arc<RuntimeControls>,
|
||||||
|
analysis_events: Arc<crate::analysis::events::AnalysisEvents>,
|
||||||
|
/// Serializes reloads + owns the live crawler daemon handle.
|
||||||
|
crawler_handle: tokio::sync::Mutex<Option<daemon::DaemonHandle>>,
|
||||||
|
/// Serializes reloads + owns the live analysis daemon handle.
|
||||||
|
analysis_handle: tokio::sync::Mutex<Option<crate::analysis::daemon::AnalysisDaemonHandle>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Supervisors {
|
||||||
|
pub async fn shutdown(&self) {
|
||||||
|
if let Some(h) = self.crawler_handle.lock().await.take() {
|
||||||
|
h.shutdown().await;
|
||||||
}
|
}
|
||||||
if let Some(a) = self.analysis_daemon {
|
if let Some(h) = self.analysis_handle.lock().await.take() {
|
||||||
a.shutdown().await;
|
h.shutdown().await;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
impl DaemonReloader for Supervisors {
|
||||||
|
async fn reload_crawler(&self, cfg: CrawlerConfig) -> anyhow::Result<()> {
|
||||||
|
// Serialize reloads; do the (possibly slow) graceful shutdown under
|
||||||
|
// the handle lock but outside the read-path RwLock so status reads
|
||||||
|
// never block on a draining browser.
|
||||||
|
let mut guard = self.crawler_handle.lock().await;
|
||||||
|
if let Some(h) = guard.take() {
|
||||||
|
h.shutdown().await;
|
||||||
|
}
|
||||||
|
self.runtime.set_crawler(None, None);
|
||||||
|
if cfg.daemon_enabled {
|
||||||
|
let spawned = spawn_crawler_daemon(
|
||||||
|
self.db.clone(),
|
||||||
|
Arc::clone(&self.storage),
|
||||||
|
&cfg,
|
||||||
|
self.runtime.analysis_gate(),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
self.runtime
|
||||||
|
.set_crawler(Some(spawned.resync), Some(spawned.crawler));
|
||||||
|
*guard = Some(spawned.handle);
|
||||||
|
tracing::info!("crawler daemon (re)started from settings");
|
||||||
|
} else {
|
||||||
|
tracing::info!("crawler daemon stopped (disabled in settings)");
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn reload_analysis(&self, cfg: AnalysisConfig) -> anyhow::Result<()> {
|
||||||
|
let mut guard = self.analysis_handle.lock().await;
|
||||||
|
if let Some(h) = guard.take() {
|
||||||
|
h.shutdown().await;
|
||||||
|
}
|
||||||
|
self.runtime.set_analysis_enabled(cfg.enabled);
|
||||||
|
if cfg.enabled {
|
||||||
|
let handle = spawn_analysis_daemon(
|
||||||
|
self.db.clone(),
|
||||||
|
Arc::clone(&self.storage),
|
||||||
|
&cfg,
|
||||||
|
Arc::clone(&self.analysis_events),
|
||||||
|
)?;
|
||||||
|
*guard = Some(handle);
|
||||||
|
tracing::info!(model = %cfg.model, "analysis daemon (re)started from settings");
|
||||||
|
} else {
|
||||||
|
tracing::info!("analysis daemon stopped (disabled in settings)");
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn build(config: Config) -> anyhow::Result<AppHandle> {
|
pub async fn build(config: Config) -> anyhow::Result<AppHandle> {
|
||||||
let db = PgPoolOptions::new()
|
let db = PgPoolOptions::new()
|
||||||
.max_connections(10)
|
.max_connections(10)
|
||||||
@@ -127,60 +289,48 @@ pub async fn build(config: Config) -> anyhow::Result<AppHandle> {
|
|||||||
|
|
||||||
let storage: Arc<dyn Storage> = Arc::new(LocalStorage::new(config.storage_dir.clone()));
|
let storage: Arc<dyn Storage> = Arc::new(LocalStorage::new(config.storage_dir.clone()));
|
||||||
|
|
||||||
let (daemon, resync, crawler) = if config.crawler.daemon_enabled {
|
// Seed the settings rows from env on first boot, then load the effective
|
||||||
let spawned = spawn_crawler_daemon(
|
// config (DB overlaid on the env base). The DB is the source of truth
|
||||||
db.clone(),
|
// after the first boot; env only fills a missing row.
|
||||||
Arc::clone(&storage),
|
let crawler_cfg = load_effective_crawler(&db, &config.crawler).await?;
|
||||||
&config.crawler,
|
let analysis_cfg = load_effective_analysis(&db, &config.analysis).await?;
|
||||||
config.analysis.enabled,
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
(Some(spawned.handle), Some(spawned.resync), Some(spawned.crawler))
|
|
||||||
} else {
|
|
||||||
tracing::info!("crawler daemon disabled (CRAWLER_DAEMON=false)");
|
|
||||||
(None, None, None)
|
|
||||||
};
|
|
||||||
|
|
||||||
// Live-event bus shared by the worker (progress) and the admin enqueue
|
// Live-event bus shared by the worker (progress) and the admin enqueue
|
||||||
// path; the SSE endpoint subscribes. Created unconditionally so the
|
// path; the SSE endpoint subscribes. Created unconditionally so the
|
||||||
// stream exists even with the worker disabled.
|
// stream exists even with the worker disabled.
|
||||||
let analysis_events = Arc::new(crate::analysis::events::AnalysisEvents::new());
|
let analysis_events = Arc::new(crate::analysis::events::AnalysisEvents::new());
|
||||||
|
|
||||||
// AI content-analysis worker. Independent of the crawler daemon (works
|
let runtime = Arc::new(RuntimeControls::new(analysis_cfg.enabled));
|
||||||
// for uploads with the crawler off), gated on ANALYSIS_ENABLED. Uses a
|
let supervisors = Arc::new(Supervisors {
|
||||||
// plain reqwest client — no cookie jar / proxy, unlike the crawler's.
|
db: db.clone(),
|
||||||
let analysis_daemon = if config.analysis.enabled {
|
storage: Arc::clone(&storage),
|
||||||
let http = reqwest::Client::builder()
|
runtime: Arc::clone(&runtime),
|
||||||
.timeout(config.analysis.request_timeout)
|
analysis_events: Arc::clone(&analysis_events),
|
||||||
.build()
|
crawler_handle: tokio::sync::Mutex::new(None),
|
||||||
.context("build analysis http client")?;
|
analysis_handle: tokio::sync::Mutex::new(None),
|
||||||
let vision = crate::analysis::vision::VisionClient::new(http, &config.analysis);
|
});
|
||||||
let dispatcher = Arc::new(crate::analysis::daemon::RealAnalyzeDispatcher {
|
|
||||||
db: db.clone(),
|
// Initial spawn goes through the same reload path so there's one code
|
||||||
storage: Arc::clone(&storage),
|
// path for "bring the daemon up with this config". A spawn failure is
|
||||||
vision,
|
// logged but does NOT abort startup: the persisted settings are the
|
||||||
model: config.analysis.model.clone(),
|
// source of truth and a bad value (e.g. a config that won't launch the
|
||||||
max_image_bytes: config.analysis.max_image_bytes,
|
// browser) must not wedge the whole server into a boot loop — the
|
||||||
});
|
// server comes up with that daemon stopped so an admin can fix it via
|
||||||
let handle = crate::analysis::daemon::spawn(
|
// the settings UI.
|
||||||
db.clone(),
|
if crawler_cfg.daemon_enabled {
|
||||||
tokio_util::sync::CancellationToken::new(),
|
if let Err(e) = supervisors.reload_crawler(crawler_cfg).await {
|
||||||
crate::analysis::daemon::AnalysisDaemonConfig {
|
tracing::error!(?e, "crawler daemon failed to start; continuing with it stopped");
|
||||||
dispatcher,
|
}
|
||||||
workers: config.analysis.workers,
|
|
||||||
job_timeout: config.analysis.job_timeout,
|
|
||||||
events: Arc::clone(&analysis_events),
|
|
||||||
},
|
|
||||||
);
|
|
||||||
tracing::info!(
|
|
||||||
workers = config.analysis.workers,
|
|
||||||
model = %config.analysis.model,
|
|
||||||
"analysis worker daemon started"
|
|
||||||
);
|
|
||||||
Some(handle)
|
|
||||||
} else {
|
} else {
|
||||||
None
|
tracing::info!("crawler daemon disabled");
|
||||||
};
|
}
|
||||||
|
if analysis_cfg.enabled {
|
||||||
|
if let Err(e) = supervisors.reload_analysis(analysis_cfg).await {
|
||||||
|
tracing::error!(?e, "analysis worker failed to start; continuing with it stopped");
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
tracing::info!("analysis worker disabled");
|
||||||
|
}
|
||||||
|
|
||||||
let auth_limiter = Arc::new(AuthRateLimiter::new(config.auth.rate_limit));
|
let auth_limiter = Arc::new(AuthRateLimiter::new(config.auth.rate_limit));
|
||||||
let state = AppState {
|
let state = AppState {
|
||||||
@@ -189,14 +339,99 @@ pub async fn build(config: Config) -> anyhow::Result<AppHandle> {
|
|||||||
auth: config.auth.clone(),
|
auth: config.auth.clone(),
|
||||||
upload: config.upload.clone(),
|
upload: config.upload.clone(),
|
||||||
auth_limiter,
|
auth_limiter,
|
||||||
resync,
|
runtime,
|
||||||
crawler,
|
reloader: Some(Arc::clone(&supervisors) as Arc<dyn DaemonReloader>),
|
||||||
|
crawler_base: config.crawler.clone(),
|
||||||
|
analysis_base: config.analysis.clone(),
|
||||||
admin_allowed_origins: Arc::new(config.admin_allowed_origins.clone()),
|
admin_allowed_origins: Arc::new(config.admin_allowed_origins.clone()),
|
||||||
analysis_enabled: config.analysis.enabled,
|
|
||||||
analysis_events,
|
analysis_events,
|
||||||
};
|
};
|
||||||
let router = router(state).layer(cors_layer(&config.cors_allowed_origins));
|
let router = router(state).layer(cors_layer(&config.cors_allowed_origins));
|
||||||
Ok(AppHandle { router, daemon, analysis_daemon })
|
Ok(AppHandle { router, supervisors })
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Seed the crawler settings row from env when absent, then return the
|
||||||
|
/// effective [`CrawlerConfig`] (DB DTO overlaid on the env base). A stored
|
||||||
|
/// DTO that fails to convert (corruption / drift) falls back to the env base
|
||||||
|
/// rather than blocking startup.
|
||||||
|
async fn load_effective_crawler(
|
||||||
|
db: &PgPool,
|
||||||
|
base: &CrawlerConfig,
|
||||||
|
) -> anyhow::Result<CrawlerConfig> {
|
||||||
|
use crate::settings::{CrawlerSettings, KEY_CRAWLER};
|
||||||
|
let dto = match repo::app_settings::get(db, KEY_CRAWLER).await? {
|
||||||
|
Some(v) => serde_json::from_value::<CrawlerSettings>(v)
|
||||||
|
.unwrap_or_else(|_| CrawlerSettings::from_config(base)),
|
||||||
|
None => {
|
||||||
|
let dto = CrawlerSettings::from_config(base);
|
||||||
|
repo::app_settings::seed_if_absent(db, KEY_CRAWLER, &serde_json::to_value(&dto)?)
|
||||||
|
.await?;
|
||||||
|
tracing::info!("seeded crawler settings from env");
|
||||||
|
dto
|
||||||
|
}
|
||||||
|
};
|
||||||
|
Ok(dto.to_config(base).unwrap_or_else(|e| {
|
||||||
|
tracing::warn!(?e, "stored crawler settings invalid; using env base");
|
||||||
|
base.clone()
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Analysis counterpart of [`load_effective_crawler`].
|
||||||
|
async fn load_effective_analysis(
|
||||||
|
db: &PgPool,
|
||||||
|
base: &AnalysisConfig,
|
||||||
|
) -> anyhow::Result<AnalysisConfig> {
|
||||||
|
use crate::settings::{AnalysisSettings, KEY_ANALYSIS};
|
||||||
|
let dto = match repo::app_settings::get(db, KEY_ANALYSIS).await? {
|
||||||
|
Some(v) => serde_json::from_value::<AnalysisSettings>(v)
|
||||||
|
.unwrap_or_else(|_| AnalysisSettings::from_config(base)),
|
||||||
|
None => {
|
||||||
|
let dto = AnalysisSettings::from_config(base);
|
||||||
|
repo::app_settings::seed_if_absent(db, KEY_ANALYSIS, &serde_json::to_value(&dto)?)
|
||||||
|
.await?;
|
||||||
|
tracing::info!("seeded analysis settings from env");
|
||||||
|
dto
|
||||||
|
}
|
||||||
|
};
|
||||||
|
Ok(dto.to_config(base).unwrap_or_else(|e| {
|
||||||
|
tracing::warn!(?e, "stored analysis settings invalid; using env base");
|
||||||
|
base.clone()
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Spawn the AI content-analysis worker daemon with the given config. Returns
|
||||||
|
/// its handle. Independent of the crawler daemon (works for uploads with the
|
||||||
|
/// crawler off). Uses a plain reqwest client — no cookie jar / proxy.
|
||||||
|
fn spawn_analysis_daemon(
|
||||||
|
db: PgPool,
|
||||||
|
storage: Arc<dyn Storage>,
|
||||||
|
cfg: &AnalysisConfig,
|
||||||
|
events: Arc<crate::analysis::events::AnalysisEvents>,
|
||||||
|
) -> anyhow::Result<crate::analysis::daemon::AnalysisDaemonHandle> {
|
||||||
|
let http = reqwest::Client::builder()
|
||||||
|
.timeout(cfg.request_timeout)
|
||||||
|
.build()
|
||||||
|
.context("build analysis http client")?;
|
||||||
|
let vision = crate::analysis::vision::VisionClient::new(http, cfg);
|
||||||
|
let dispatcher = Arc::new(crate::analysis::daemon::RealAnalyzeDispatcher {
|
||||||
|
db: db.clone(),
|
||||||
|
storage,
|
||||||
|
vision,
|
||||||
|
model: cfg.model.clone(),
|
||||||
|
max_image_bytes: cfg.max_image_bytes,
|
||||||
|
});
|
||||||
|
let handle = crate::analysis::daemon::spawn(
|
||||||
|
db,
|
||||||
|
CancellationToken::new(),
|
||||||
|
crate::analysis::daemon::AnalysisDaemonConfig {
|
||||||
|
dispatcher,
|
||||||
|
workers: cfg.workers,
|
||||||
|
job_timeout: cfg.job_timeout,
|
||||||
|
events,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
tracing::info!(workers = cfg.workers, model = %cfg.model, "analysis worker daemon started");
|
||||||
|
Ok(handle)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Bundle returned by [`spawn_crawler_daemon`]. The handle owns the
|
/// Bundle returned by [`spawn_crawler_daemon`]. The handle owns the
|
||||||
@@ -213,7 +448,7 @@ async fn spawn_crawler_daemon(
|
|||||||
db: PgPool,
|
db: PgPool,
|
||||||
storage: Arc<dyn Storage>,
|
storage: Arc<dyn Storage>,
|
||||||
cfg: &CrawlerConfig,
|
cfg: &CrawlerConfig,
|
||||||
analysis_enabled: bool,
|
analysis_enabled: Arc<AtomicBool>,
|
||||||
) -> anyhow::Result<SpawnedDaemon> {
|
) -> anyhow::Result<SpawnedDaemon> {
|
||||||
// Reqwest client with a shared cookie jar so CDN image fetches include
|
// Reqwest client with a shared cookie jar so CDN image fetches include
|
||||||
// PHPSESSID. The same `Arc<Jar>` is held by the SessionController, so a
|
// PHPSESSID. The same `Arc<Jar>` is held by the SessionController, so a
|
||||||
@@ -512,9 +747,10 @@ struct RealChapterDispatcher {
|
|||||||
rate: Arc<HostRateLimiters>,
|
rate: Arc<HostRateLimiters>,
|
||||||
download_allowlist: DownloadAllowlist,
|
download_allowlist: DownloadAllowlist,
|
||||||
max_image_bytes: usize,
|
max_image_bytes: usize,
|
||||||
/// Enqueue `analyze_page` jobs for freshly-crawled pages. Mirrors
|
/// Enqueue `analyze_page` jobs for freshly-crawled pages. Shared gate
|
||||||
/// `config.analysis.enabled`.
|
/// (read live) so toggling analysis at runtime takes effect without a
|
||||||
analysis_enabled: bool,
|
/// crawler respawn. Mirrors the analysis enable setting.
|
||||||
|
analysis_enabled: Arc<AtomicBool>,
|
||||||
/// Consecutive transient chapter failures; resets on any success.
|
/// Consecutive transient chapter failures; resets on any success.
|
||||||
/// Drives the automatic coordinated browser restart.
|
/// Drives the automatic coordinated browser restart.
|
||||||
transient_failures: Arc<std::sync::atomic::AtomicU32>,
|
transient_failures: Arc<std::sync::atomic::AtomicU32>,
|
||||||
@@ -570,7 +806,7 @@ impl ChapterDispatcher for RealChapterDispatcher {
|
|||||||
self.max_image_bytes,
|
self.max_image_bytes,
|
||||||
self.tor.as_deref(),
|
self.tor.as_deref(),
|
||||||
Some(&self.status),
|
Some(&self.status),
|
||||||
self.analysis_enabled,
|
self.analysis_enabled.load(Ordering::Relaxed),
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
drop(lease);
|
drop(lease);
|
||||||
|
|||||||
@@ -90,6 +90,27 @@ impl ResponseFormat {
|
|||||||
_ => ResponseFormat::JsonSchema,
|
_ => ResponseFormat::JsonSchema,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Canonical wire string, the inverse of [`Self::from_str`] for the
|
||||||
|
/// three modes the API exposes.
|
||||||
|
pub fn as_str(self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
ResponseFormat::JsonSchema => "json_schema",
|
||||||
|
ResponseFormat::JsonObject => "json_object",
|
||||||
|
ResponseFormat::None => "none",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Parse a wire value, rejecting unknown modes (stricter than
|
||||||
|
/// [`Self::from_str`], which the env loader uses to stay lenient).
|
||||||
|
pub fn parse_strict(s: &str) -> Option<ResponseFormat> {
|
||||||
|
match s.trim().to_lowercase().as_str() {
|
||||||
|
"json_schema" => Some(ResponseFormat::JsonSchema),
|
||||||
|
"json_object" => Some(ResponseFormat::JsonObject),
|
||||||
|
"none" => Some(ResponseFormat::None),
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// AI content-analysis worker configuration: the enable gate, the local
|
/// AI content-analysis worker configuration: the enable gate, the local
|
||||||
@@ -146,6 +167,21 @@ pub struct AnalysisConfig {
|
|||||||
/// the repetition loops that otherwise run small models into the token
|
/// the repetition loops that otherwise run small models into the token
|
||||||
/// ceiling. `0` omits the field.
|
/// ceiling. `0` omits the field.
|
||||||
pub frequency_penalty: f64,
|
pub frequency_penalty: f64,
|
||||||
|
/// Sampling `temperature` sent with each request (`ANALYSIS_TEMPERATURE`).
|
||||||
|
/// Defaults to `0.0` (deterministic), which is the most reliable for
|
||||||
|
/// structured JSON output; some models behave better with a small
|
||||||
|
/// positive value.
|
||||||
|
pub temperature: f64,
|
||||||
|
/// System prompt for the single-call (normal-aspect) analysis path
|
||||||
|
/// (`ANALYSIS_SYSTEM_PROMPT`). Defaults to
|
||||||
|
/// [`crate::analysis::prompt::SYSTEM_PROMPT_DEFAULT`].
|
||||||
|
pub system_prompt: String,
|
||||||
|
/// Pass-A OCR-only prompt for tall-page slices (`ANALYSIS_OCR_PROMPT`).
|
||||||
|
/// Defaults to [`crate::analysis::prompt::OCR_PROMPT_DEFAULT`].
|
||||||
|
pub ocr_prompt: String,
|
||||||
|
/// Pass-B grounding prompt — tags/scene/safety (`ANALYSIS_GROUNDING_PROMPT`).
|
||||||
|
/// Defaults to [`crate::analysis::prompt::GROUNDING_PROMPT_DEFAULT`].
|
||||||
|
pub grounding_prompt: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Default for AnalysisConfig {
|
impl Default for AnalysisConfig {
|
||||||
@@ -170,6 +206,10 @@ impl Default for AnalysisConfig {
|
|||||||
max_image_bytes: 8 * 1024 * 1024,
|
max_image_bytes: 8 * 1024 * 1024,
|
||||||
response_format: ResponseFormat::JsonSchema,
|
response_format: ResponseFormat::JsonSchema,
|
||||||
frequency_penalty: 0.3,
|
frequency_penalty: 0.3,
|
||||||
|
temperature: 0.0,
|
||||||
|
system_prompt: crate::analysis::prompt::SYSTEM_PROMPT_DEFAULT.to_string(),
|
||||||
|
ocr_prompt: crate::analysis::prompt::OCR_PROMPT_DEFAULT.to_string(),
|
||||||
|
grounding_prompt: crate::analysis::prompt::GROUNDING_PROMPT_DEFAULT.to_string(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -206,10 +246,23 @@ impl AnalysisConfig {
|
|||||||
.map(|s| ResponseFormat::from_str(&s))
|
.map(|s| ResponseFormat::from_str(&s))
|
||||||
.unwrap_or(d.response_format),
|
.unwrap_or(d.response_format),
|
||||||
frequency_penalty: env_f64("ANALYSIS_FREQUENCY_PENALTY", d.frequency_penalty),
|
frequency_penalty: env_f64("ANALYSIS_FREQUENCY_PENALTY", d.frequency_penalty),
|
||||||
|
temperature: env_f64("ANALYSIS_TEMPERATURE", d.temperature),
|
||||||
|
system_prompt: env_prompt("ANALYSIS_SYSTEM_PROMPT", d.system_prompt),
|
||||||
|
ocr_prompt: env_prompt("ANALYSIS_OCR_PROMPT", d.ocr_prompt),
|
||||||
|
grounding_prompt: env_prompt("ANALYSIS_GROUNDING_PROMPT", d.grounding_prompt),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Read a prompt override from env, falling back to the default when unset
|
||||||
|
/// or blank (so `ANALYSIS_SYSTEM_PROMPT=` doesn't wipe the prompt).
|
||||||
|
fn env_prompt(name: &str, default: String) -> String {
|
||||||
|
std::env::var(name)
|
||||||
|
.ok()
|
||||||
|
.filter(|s| !s.trim().is_empty())
|
||||||
|
.unwrap_or(default)
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug)]
|
#[derive(Clone, Debug)]
|
||||||
pub struct Config {
|
pub struct Config {
|
||||||
pub database_url: String,
|
pub database_url: String,
|
||||||
|
|||||||
@@ -91,6 +91,16 @@ impl DownloadAllowlist {
|
|||||||
self.hosts.is_empty()
|
self.hosts.is_empty()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The explicitly-allowed hosts (lowercased). Empty when `allow_any`.
|
||||||
|
pub fn hosts(&self) -> &[String] {
|
||||||
|
&self.hosts
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Whether the host check is bypassed entirely (`CRAWLER_ALLOW_ANY_HOST`).
|
||||||
|
pub fn is_allow_any(&self) -> bool {
|
||||||
|
self.allow_any
|
||||||
|
}
|
||||||
|
|
||||||
pub fn contains(&self, host: &str) -> bool {
|
pub fn contains(&self, host: &str) -> bool {
|
||||||
if self.allow_any {
|
if self.allow_any {
|
||||||
return true;
|
return true;
|
||||||
|
|||||||
@@ -7,5 +7,6 @@ pub mod crawler;
|
|||||||
pub mod domain;
|
pub mod domain;
|
||||||
pub mod error;
|
pub mod error;
|
||||||
pub mod repo;
|
pub mod repo;
|
||||||
|
pub mod settings;
|
||||||
pub mod storage;
|
pub mod storage;
|
||||||
pub mod upload;
|
pub mod upload;
|
||||||
|
|||||||
@@ -21,11 +21,8 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
|
|
||||||
let config = mangalord::config::Config::from_env()?;
|
let config = mangalord::config::Config::from_env()?;
|
||||||
let addr: SocketAddr = config.bind_address.parse()?;
|
let addr: SocketAddr = config.bind_address.parse()?;
|
||||||
let mangalord::app::AppHandle {
|
let mangalord::app::AppHandle { router, supervisors } =
|
||||||
router,
|
mangalord::app::build(config).await?;
|
||||||
daemon,
|
|
||||||
analysis_daemon,
|
|
||||||
} = mangalord::app::build(config).await?;
|
|
||||||
|
|
||||||
tracing::info!(%addr, "mangalord listening");
|
tracing::info!(%addr, "mangalord listening");
|
||||||
let listener = tokio::net::TcpListener::bind(addr).await?;
|
let listener = tokio::net::TcpListener::bind(addr).await?;
|
||||||
@@ -33,30 +30,17 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
.with_graceful_shutdown(shutdown_signal())
|
.with_graceful_shutdown(shutdown_signal())
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
// Drain background tasks (crawler daemon) before exiting so Chromium
|
// Drain background daemons (crawler + analysis) before exiting so
|
||||||
// gets a clean shutdown rather than relying on kill-on-drop. Bounded
|
// Chromium gets a clean shutdown rather than relying on kill-on-drop.
|
||||||
// by a timeout so a wedged shutdown path can't trap the process.
|
// Bounded by a timeout so a wedged shutdown path can't trap the process.
|
||||||
if let Some(d) = daemon {
|
if tokio::time::timeout(CRAWLER_SHUTDOWN_TIMEOUT, supervisors.shutdown())
|
||||||
if tokio::time::timeout(CRAWLER_SHUTDOWN_TIMEOUT, d.shutdown())
|
.await
|
||||||
.await
|
.is_err()
|
||||||
.is_err()
|
{
|
||||||
{
|
tracing::warn!(
|
||||||
tracing::warn!(
|
timeout_s = CRAWLER_SHUTDOWN_TIMEOUT.as_secs(),
|
||||||
timeout_s = CRAWLER_SHUTDOWN_TIMEOUT.as_secs(),
|
"daemon shutdown exceeded timeout; abandoning"
|
||||||
"crawler daemon shutdown exceeded timeout; abandoning"
|
);
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if let Some(a) = analysis_daemon {
|
|
||||||
if tokio::time::timeout(CRAWLER_SHUTDOWN_TIMEOUT, a.shutdown())
|
|
||||||
.await
|
|
||||||
.is_err()
|
|
||||||
{
|
|
||||||
tracing::warn!(
|
|
||||||
timeout_s = CRAWLER_SHUTDOWN_TIMEOUT.as_secs(),
|
|
||||||
"analysis daemon shutdown exceeded timeout; abandoning"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|||||||
54
backend/src/repo/app_settings.rs
Normal file
54
backend/src/repo/app_settings.rs
Normal file
@@ -0,0 +1,54 @@
|
|||||||
|
//! Persistence for runtime-editable application settings (`app_settings`),
|
||||||
|
//! a small key-value JSONB table (one row per subsystem group). Mirrors the
|
||||||
|
//! `crawler_state` access style: plain async fns over `&PgPool`, the `value`
|
||||||
|
//! a `serde_json::Value` the caller (de)serializes into a settings DTO.
|
||||||
|
|
||||||
|
use sqlx::{PgExecutor, PgPool};
|
||||||
|
|
||||||
|
/// Fetch the raw JSONB for a settings group, or `None` if unset.
|
||||||
|
pub async fn get(pool: &PgPool, key: &str) -> sqlx::Result<Option<serde_json::Value>> {
|
||||||
|
sqlx::query_scalar("SELECT value FROM app_settings WHERE key = $1")
|
||||||
|
.bind(key)
|
||||||
|
.fetch_optional(pool)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Insert-or-replace the JSONB for a settings group, stamping `updated_at`.
|
||||||
|
/// Generic over the executor so it can run inside the same transaction as
|
||||||
|
/// the matching admin-audit insert.
|
||||||
|
pub async fn upsert<'e, E: PgExecutor<'e>>(
|
||||||
|
executor: E,
|
||||||
|
key: &str,
|
||||||
|
value: &serde_json::Value,
|
||||||
|
) -> sqlx::Result<()> {
|
||||||
|
sqlx::query(
|
||||||
|
"INSERT INTO app_settings (key, value, updated_at) \
|
||||||
|
VALUES ($1, $2, now()) \
|
||||||
|
ON CONFLICT (key) DO UPDATE \
|
||||||
|
SET value = EXCLUDED.value, updated_at = now()",
|
||||||
|
)
|
||||||
|
.bind(key)
|
||||||
|
.bind(value)
|
||||||
|
.execute(executor)
|
||||||
|
.await?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Write the value only if the row is absent (the env → DB boot seed). Returns
|
||||||
|
/// `true` when a row was inserted, `false` when one already existed. Uses
|
||||||
|
/// `ON CONFLICT DO NOTHING` so a concurrent boot can't race two seeds.
|
||||||
|
pub async fn seed_if_absent(
|
||||||
|
pool: &PgPool,
|
||||||
|
key: &str,
|
||||||
|
value: &serde_json::Value,
|
||||||
|
) -> sqlx::Result<bool> {
|
||||||
|
let result = sqlx::query(
|
||||||
|
"INSERT INTO app_settings (key, value) VALUES ($1, $2) \
|
||||||
|
ON CONFLICT (key) DO NOTHING",
|
||||||
|
)
|
||||||
|
.bind(key)
|
||||||
|
.bind(value)
|
||||||
|
.execute(pool)
|
||||||
|
.await?;
|
||||||
|
Ok(result.rows_affected() > 0)
|
||||||
|
}
|
||||||
@@ -1,6 +1,7 @@
|
|||||||
pub mod admin_audit;
|
pub mod admin_audit;
|
||||||
pub mod admin_view;
|
pub mod admin_view;
|
||||||
pub mod api_token;
|
pub mod api_token;
|
||||||
|
pub mod app_settings;
|
||||||
pub mod author;
|
pub mod author;
|
||||||
pub mod bookmark;
|
pub mod bookmark;
|
||||||
pub mod chapter;
|
pub mod chapter;
|
||||||
|
|||||||
643
backend/src/settings.rs
Normal file
643
backend/src/settings.rs
Normal file
@@ -0,0 +1,643 @@
|
|||||||
|
//! Serializable, admin-editable settings DTOs for the crawler and analysis
|
||||||
|
//! subsystems, and their conversion to/from the runtime [`CrawlerConfig`] /
|
||||||
|
//! [`AnalysisConfig`].
|
||||||
|
//!
|
||||||
|
//! These DTOs are the boundary persisted in the `app_settings` table (one
|
||||||
|
//! JSONB row per group) and exchanged over the admin API. They carry **only
|
||||||
|
//! the operationally-safe, UI-editable fields** as primitive types — the
|
||||||
|
//! runtime configs additionally hold env-only/structural fields (browser
|
||||||
|
//! launch options, proxy, TOR control, the vision API key, the cookie
|
||||||
|
//! domain) that are never persisted here.
|
||||||
|
//!
|
||||||
|
//! Flow: at boot, [`CrawlerSettings::from_config`] /
|
||||||
|
//! [`AnalysisSettings::from_config`] derive the env-seed DTO; it is written to
|
||||||
|
//! the DB only when the row is absent. Thereafter the DB row is the source of
|
||||||
|
//! truth, and [`CrawlerSettings::to_config`] / [`AnalysisSettings::to_config`]
|
||||||
|
//! overlay it onto the env-derived **base** (which carries the env-only
|
||||||
|
//! fields) to produce the effective config the daemons are (re)spawned with.
|
||||||
|
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use chrono::NaiveTime;
|
||||||
|
use chrono_tz::Tz;
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
use crate::analysis::prompt::{
|
||||||
|
GROUNDING_PROMPT_DEFAULT, OCR_PROMPT_DEFAULT, SYSTEM_PROMPT_DEFAULT,
|
||||||
|
};
|
||||||
|
use crate::config::{AnalysisConfig, CrawlerConfig, ResponseFormat};
|
||||||
|
use crate::crawler::safety::DownloadAllowlist;
|
||||||
|
|
||||||
|
/// `app_settings.key` for the crawler group.
|
||||||
|
pub const KEY_CRAWLER: &str = "crawler";
|
||||||
|
/// `app_settings.key` for the analysis group.
|
||||||
|
pub const KEY_ANALYSIS: &str = "analysis";
|
||||||
|
|
||||||
|
/// One field-level validation failure, surfaced to the UI per input.
|
||||||
|
#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
|
||||||
|
pub struct FieldError {
|
||||||
|
pub field: String,
|
||||||
|
pub message: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Collected validation failures for a settings payload. Empty == valid.
|
||||||
|
#[derive(Debug, Clone, Serialize, Default, PartialEq, Eq)]
|
||||||
|
pub struct FieldErrors {
|
||||||
|
pub errors: Vec<FieldError>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl FieldErrors {
|
||||||
|
fn push(&mut self, field: &str, message: impl Into<String>) {
|
||||||
|
self.errors.push(FieldError {
|
||||||
|
field: field.to_string(),
|
||||||
|
message: message.into(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn is_empty(&self) -> bool {
|
||||||
|
self.errors.is_empty()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Crawler
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/// Admin-editable crawler-daemon settings. Mirrors the operational `CRAWLER_*`
|
||||||
|
/// env vars; host/infra and session fields stay env-only.
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||||
|
#[serde(default)]
|
||||||
|
pub struct CrawlerSettings {
|
||||||
|
pub daemon_enabled: bool,
|
||||||
|
/// Daily metadata-pass time, `"HH:MM"` (24h).
|
||||||
|
pub daily_at: String,
|
||||||
|
/// IANA timezone for the daily schedule.
|
||||||
|
pub tz: String,
|
||||||
|
pub idle_timeout_secs: u64,
|
||||||
|
pub chapter_workers: u64,
|
||||||
|
pub retention_days: u32,
|
||||||
|
pub start_url: Option<String>,
|
||||||
|
pub rate_ms: u64,
|
||||||
|
pub cdn_host: Option<String>,
|
||||||
|
pub cdn_rate_ms: u64,
|
||||||
|
/// Domain the session cookie (PHPSESSID) is scoped to when injecting it
|
||||||
|
/// into the browser / CDN requests. Pairs with `start_url`.
|
||||||
|
pub cookie_domain: Option<String>,
|
||||||
|
pub user_agent: Option<String>,
|
||||||
|
/// Hosts the crawler may download images from (in addition to the
|
||||||
|
/// auto-seeded start-url / CDN hosts). Ignored when `allow_any_host`.
|
||||||
|
pub download_allowlist: Vec<String>,
|
||||||
|
pub allow_any_host: bool,
|
||||||
|
pub max_image_bytes: u64,
|
||||||
|
/// Max manga detail fetches per metadata pass; `0` = unlimited.
|
||||||
|
pub manga_limit: u64,
|
||||||
|
pub job_timeout_secs: u64,
|
||||||
|
pub metadata_max_consecutive_failures: u32,
|
||||||
|
pub browser_restart_threshold: u32,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for CrawlerSettings {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self::from_config(&CrawlerConfig::default())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl CrawlerSettings {
|
||||||
|
/// Derive the DTO from a runtime config (used to seed the DB from env).
|
||||||
|
pub fn from_config(c: &CrawlerConfig) -> Self {
|
||||||
|
Self {
|
||||||
|
daemon_enabled: c.daemon_enabled,
|
||||||
|
daily_at: c.daily_at.format("%H:%M").to_string(),
|
||||||
|
tz: c.tz.name().to_string(),
|
||||||
|
idle_timeout_secs: c.idle_timeout.as_secs(),
|
||||||
|
chapter_workers: c.chapter_workers as u64,
|
||||||
|
retention_days: c.retention_days,
|
||||||
|
start_url: c.start_url.clone(),
|
||||||
|
rate_ms: c.rate_ms,
|
||||||
|
cdn_host: c.cdn_host.clone(),
|
||||||
|
cdn_rate_ms: c.cdn_rate_ms,
|
||||||
|
cookie_domain: c.cookie_domain.clone(),
|
||||||
|
user_agent: c.user_agent.clone(),
|
||||||
|
download_allowlist: c.download_allowlist.hosts().to_vec(),
|
||||||
|
allow_any_host: c.download_allowlist.is_allow_any(),
|
||||||
|
max_image_bytes: c.max_image_bytes as u64,
|
||||||
|
manga_limit: c.manga_limit as u64,
|
||||||
|
job_timeout_secs: c.job_timeout.as_secs(),
|
||||||
|
metadata_max_consecutive_failures: c.metadata_max_consecutive_failures,
|
||||||
|
browser_restart_threshold: c.browser_restart_threshold,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Overlay the DTO onto an env-derived `base` (which carries the env-only
|
||||||
|
/// fields: browser, proxy, TOR, session, cookie domain) to produce the
|
||||||
|
/// effective runtime config. Validates and returns all field errors at
|
||||||
|
/// once on failure.
|
||||||
|
pub fn to_config(&self, base: &CrawlerConfig) -> Result<CrawlerConfig, FieldErrors> {
|
||||||
|
let mut errs = FieldErrors::default();
|
||||||
|
|
||||||
|
let daily_at = match NaiveTime::parse_from_str(&self.daily_at, "%H:%M") {
|
||||||
|
Ok(t) => t,
|
||||||
|
Err(_) => {
|
||||||
|
errs.push("daily_at", "must be HH:MM (24-hour)");
|
||||||
|
base.daily_at
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let tz: Tz = match self.tz.parse() {
|
||||||
|
Ok(t) => t,
|
||||||
|
Err(_) => {
|
||||||
|
errs.push("tz", "must be a valid IANA timezone (e.g. UTC, Europe/Berlin)");
|
||||||
|
base.tz
|
||||||
|
}
|
||||||
|
};
|
||||||
|
if self.chapter_workers < 1 {
|
||||||
|
errs.push("chapter_workers", "must be at least 1");
|
||||||
|
}
|
||||||
|
if let Some(url) = self.start_url.as_deref().map(str::trim).filter(|s| !s.is_empty()) {
|
||||||
|
if reqwest::Url::parse(url).is_err() {
|
||||||
|
errs.push("start_url", "must be a valid absolute URL");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if self.job_timeout_secs < 1 {
|
||||||
|
errs.push("job_timeout_secs", "must be at least 1 second");
|
||||||
|
}
|
||||||
|
|
||||||
|
if !errs.is_empty() {
|
||||||
|
return Err(errs);
|
||||||
|
}
|
||||||
|
|
||||||
|
let start_url = self
|
||||||
|
.start_url
|
||||||
|
.as_deref()
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|s| !s.is_empty())
|
||||||
|
.map(str::to_string);
|
||||||
|
let cdn_host = self
|
||||||
|
.cdn_host
|
||||||
|
.as_deref()
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|s| !s.is_empty())
|
||||||
|
.map(str::to_string);
|
||||||
|
|
||||||
|
let download_allowlist = build_allowlist(
|
||||||
|
self.allow_any_host,
|
||||||
|
start_url.as_deref(),
|
||||||
|
cdn_host.as_deref(),
|
||||||
|
&self.download_allowlist,
|
||||||
|
);
|
||||||
|
|
||||||
|
Ok(CrawlerConfig {
|
||||||
|
daemon_enabled: self.daemon_enabled,
|
||||||
|
daily_at,
|
||||||
|
tz,
|
||||||
|
idle_timeout: Duration::from_secs(self.idle_timeout_secs),
|
||||||
|
chapter_workers: (self.chapter_workers as usize).max(1),
|
||||||
|
retention_days: self.retention_days,
|
||||||
|
start_url,
|
||||||
|
rate_ms: self.rate_ms,
|
||||||
|
cdn_host,
|
||||||
|
cdn_rate_ms: self.cdn_rate_ms,
|
||||||
|
user_agent: self
|
||||||
|
.user_agent
|
||||||
|
.as_deref()
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|s| !s.is_empty())
|
||||||
|
.map(str::to_string),
|
||||||
|
cookie_domain: self
|
||||||
|
.cookie_domain
|
||||||
|
.as_deref()
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|s| !s.is_empty())
|
||||||
|
.map(str::to_string),
|
||||||
|
download_allowlist,
|
||||||
|
max_image_bytes: self.max_image_bytes as usize,
|
||||||
|
manga_limit: self.manga_limit as usize,
|
||||||
|
job_timeout: Duration::from_secs(self.job_timeout_secs.max(1)),
|
||||||
|
metadata_max_consecutive_failures: self.metadata_max_consecutive_failures,
|
||||||
|
browser_restart_threshold: self.browser_restart_threshold,
|
||||||
|
// Env-only / structural fields preserved from the base.
|
||||||
|
phpsessid: base.phpsessid.clone(),
|
||||||
|
proxy: base.proxy.clone(),
|
||||||
|
tor_control_url: base.tor_control_url.clone(),
|
||||||
|
tor_control_password: base.tor_control_password.clone(),
|
||||||
|
tor_control_cookie_path: base.tor_control_cookie_path.clone(),
|
||||||
|
tor_recircuit_max_attempts: base.tor_recircuit_max_attempts,
|
||||||
|
browser: base.browser.clone(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Rebuild a [`DownloadAllowlist`] from the DTO, always seeding the start-url
|
||||||
|
/// and CDN hosts (mirrors `config::build_download_allowlist`).
|
||||||
|
fn build_allowlist(
|
||||||
|
allow_any: bool,
|
||||||
|
start_url: Option<&str>,
|
||||||
|
cdn_host: Option<&str>,
|
||||||
|
extras: &[String],
|
||||||
|
) -> DownloadAllowlist {
|
||||||
|
if allow_any {
|
||||||
|
return DownloadAllowlist::allow_any();
|
||||||
|
}
|
||||||
|
let mut allow = DownloadAllowlist::new();
|
||||||
|
if let Some(url) = start_url {
|
||||||
|
if let Ok(parsed) = reqwest::Url::parse(url) {
|
||||||
|
if let Some(h) = parsed.host_str() {
|
||||||
|
allow = allow.allow(h);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if let Some(host) = cdn_host {
|
||||||
|
allow = allow.allow(host);
|
||||||
|
}
|
||||||
|
for h in extras {
|
||||||
|
let trimmed = h.trim();
|
||||||
|
if !trimmed.is_empty() {
|
||||||
|
allow = allow.allow(trimmed);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
allow
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Analysis
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/// Admin-editable analysis-worker settings. The vision `api_key` is env-only
|
||||||
|
/// and never carried here. Prompts are `Option<String>`: `None` means "use
|
||||||
|
/// the compiled default" (the UI's "reset to default").
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||||
|
#[serde(default)]
|
||||||
|
pub struct AnalysisSettings {
|
||||||
|
pub enabled: bool,
|
||||||
|
pub workers: u64,
|
||||||
|
pub endpoint: String,
|
||||||
|
pub model: String,
|
||||||
|
pub request_timeout_secs: u64,
|
||||||
|
pub job_timeout_secs: u64,
|
||||||
|
pub max_tokens: u32,
|
||||||
|
pub max_pixels: u32,
|
||||||
|
pub min_slice_height: u32,
|
||||||
|
pub slice_overlap: f64,
|
||||||
|
pub tall_aspect_threshold: f64,
|
||||||
|
pub max_slices: u64,
|
||||||
|
pub max_image_bytes: u64,
|
||||||
|
/// `"json_schema" | "json_object" | "none"`.
|
||||||
|
pub response_format: String,
|
||||||
|
pub frequency_penalty: f64,
|
||||||
|
pub temperature: f64,
|
||||||
|
pub system_prompt: Option<String>,
|
||||||
|
pub ocr_prompt: Option<String>,
|
||||||
|
pub grounding_prompt: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for AnalysisSettings {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self::from_config(&AnalysisConfig::default())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl AnalysisSettings {
|
||||||
|
pub fn from_config(c: &AnalysisConfig) -> Self {
|
||||||
|
// Capture a prompt only when it diverges from the compiled default,
|
||||||
|
// so an unmodified config seeds as `None` ("use default").
|
||||||
|
let opt = |cur: &str, def: &str| (cur != def).then(|| cur.to_string());
|
||||||
|
Self {
|
||||||
|
enabled: c.enabled,
|
||||||
|
workers: c.workers as u64,
|
||||||
|
endpoint: c.endpoint.clone(),
|
||||||
|
model: c.model.clone(),
|
||||||
|
request_timeout_secs: c.request_timeout.as_secs(),
|
||||||
|
job_timeout_secs: c.job_timeout.as_secs(),
|
||||||
|
max_tokens: c.max_tokens,
|
||||||
|
max_pixels: c.max_pixels,
|
||||||
|
min_slice_height: c.min_slice_height,
|
||||||
|
slice_overlap: c.slice_overlap,
|
||||||
|
tall_aspect_threshold: c.tall_aspect_threshold,
|
||||||
|
max_slices: c.max_slices as u64,
|
||||||
|
max_image_bytes: c.max_image_bytes as u64,
|
||||||
|
response_format: c.response_format.as_str().to_string(),
|
||||||
|
frequency_penalty: c.frequency_penalty,
|
||||||
|
temperature: c.temperature,
|
||||||
|
system_prompt: opt(&c.system_prompt, SYSTEM_PROMPT_DEFAULT),
|
||||||
|
ocr_prompt: opt(&c.ocr_prompt, OCR_PROMPT_DEFAULT),
|
||||||
|
grounding_prompt: opt(&c.grounding_prompt, GROUNDING_PROMPT_DEFAULT),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn to_config(&self, base: &AnalysisConfig) -> Result<AnalysisConfig, FieldErrors> {
|
||||||
|
let mut errs = FieldErrors::default();
|
||||||
|
|
||||||
|
if self.workers < 1 {
|
||||||
|
errs.push("workers", "must be at least 1");
|
||||||
|
}
|
||||||
|
if self.endpoint.trim().is_empty() || reqwest::Url::parse(self.endpoint.trim()).is_err() {
|
||||||
|
errs.push("endpoint", "must be a valid absolute URL");
|
||||||
|
}
|
||||||
|
if self.enabled && self.model.trim().is_empty() {
|
||||||
|
errs.push("model", "required when analysis is enabled");
|
||||||
|
}
|
||||||
|
if self.max_tokens < 1 {
|
||||||
|
errs.push("max_tokens", "must be at least 1");
|
||||||
|
}
|
||||||
|
if self.request_timeout_secs < 1 {
|
||||||
|
errs.push("request_timeout_secs", "must be at least 1 second");
|
||||||
|
}
|
||||||
|
if self.job_timeout_secs < 1 {
|
||||||
|
errs.push("job_timeout_secs", "must be at least 1 second");
|
||||||
|
}
|
||||||
|
if self.max_pixels < 1 {
|
||||||
|
errs.push("max_pixels", "must be at least 1");
|
||||||
|
}
|
||||||
|
if self.max_image_bytes < 1 {
|
||||||
|
errs.push("max_image_bytes", "must be greater than 0");
|
||||||
|
}
|
||||||
|
if !(0.0..=0.9).contains(&self.slice_overlap) {
|
||||||
|
errs.push("slice_overlap", "must be between 0.0 and 0.9");
|
||||||
|
}
|
||||||
|
if self.tall_aspect_threshold < 1.0 {
|
||||||
|
errs.push("tall_aspect_threshold", "must be at least 1.0");
|
||||||
|
}
|
||||||
|
if self.min_slice_height < 1 {
|
||||||
|
errs.push("min_slice_height", "must be at least 1");
|
||||||
|
}
|
||||||
|
if self.max_slices < 1 {
|
||||||
|
errs.push("max_slices", "must be at least 1");
|
||||||
|
}
|
||||||
|
if self.temperature < 0.0 {
|
||||||
|
errs.push("temperature", "must be 0 or greater");
|
||||||
|
}
|
||||||
|
let response_format = match ResponseFormat::parse_strict(&self.response_format) {
|
||||||
|
Some(rf) => rf,
|
||||||
|
None => {
|
||||||
|
errs.push(
|
||||||
|
"response_format",
|
||||||
|
"must be one of: json_schema, json_object, none",
|
||||||
|
);
|
||||||
|
base.response_format
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
if !errs.is_empty() {
|
||||||
|
return Err(errs);
|
||||||
|
}
|
||||||
|
|
||||||
|
let prompt = |o: &Option<String>, def: &str| {
|
||||||
|
o.as_deref()
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|s| !s.is_empty())
|
||||||
|
.unwrap_or(def)
|
||||||
|
.to_string()
|
||||||
|
};
|
||||||
|
|
||||||
|
Ok(AnalysisConfig {
|
||||||
|
enabled: self.enabled,
|
||||||
|
workers: (self.workers as usize).max(1),
|
||||||
|
endpoint: self.endpoint.trim().to_string(),
|
||||||
|
model: self.model.trim().to_string(),
|
||||||
|
request_timeout: Duration::from_secs(self.request_timeout_secs),
|
||||||
|
job_timeout: Duration::from_secs(self.job_timeout_secs),
|
||||||
|
max_tokens: self.max_tokens,
|
||||||
|
max_pixels: self.max_pixels,
|
||||||
|
min_slice_height: self.min_slice_height.max(1),
|
||||||
|
slice_overlap: self.slice_overlap.clamp(0.0, 0.9),
|
||||||
|
tall_aspect_threshold: self.tall_aspect_threshold.max(1.0),
|
||||||
|
max_slices: (self.max_slices as usize).max(1),
|
||||||
|
max_image_bytes: self.max_image_bytes as usize,
|
||||||
|
response_format,
|
||||||
|
frequency_penalty: self.frequency_penalty,
|
||||||
|
temperature: self.temperature.max(0.0),
|
||||||
|
system_prompt: prompt(&self.system_prompt, SYSTEM_PROMPT_DEFAULT),
|
||||||
|
ocr_prompt: prompt(&self.ocr_prompt, OCR_PROMPT_DEFAULT),
|
||||||
|
grounding_prompt: prompt(&self.grounding_prompt, GROUNDING_PROMPT_DEFAULT),
|
||||||
|
// Env-only secret preserved from the base.
|
||||||
|
api_key: base.api_key.clone(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Prompt defaults, returned by the API so the UI can render placeholders and
|
||||||
|
/// implement "reset to default".
|
||||||
|
#[derive(Debug, Clone, Serialize)]
|
||||||
|
pub struct PromptDefaults {
|
||||||
|
pub system_prompt: &'static str,
|
||||||
|
pub ocr_prompt: &'static str,
|
||||||
|
pub grounding_prompt: &'static str,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl PromptDefaults {
|
||||||
|
pub fn get() -> Self {
|
||||||
|
Self {
|
||||||
|
system_prompt: SYSTEM_PROMPT_DEFAULT,
|
||||||
|
ocr_prompt: OCR_PROMPT_DEFAULT,
|
||||||
|
grounding_prompt: GROUNDING_PROMPT_DEFAULT,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
// --- crawler round-trip & validation ---
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn crawler_round_trips_through_dto() {
|
||||||
|
let mut base = CrawlerConfig::default();
|
||||||
|
base.start_url = Some("https://example.com/".to_string());
|
||||||
|
base.tz = Tz::Europe__Berlin;
|
||||||
|
base.chapter_workers = 3;
|
||||||
|
base.cookie_domain = Some("example.com".to_string());
|
||||||
|
let dto = CrawlerSettings::from_config(&base);
|
||||||
|
let back = dto.to_config(&base).expect("valid");
|
||||||
|
assert_eq!(back.tz, Tz::Europe__Berlin);
|
||||||
|
assert_eq!(back.chapter_workers, 3);
|
||||||
|
assert_eq!(back.start_url.as_deref(), Some("https://example.com/"));
|
||||||
|
// cookie_domain is now an editable DTO field, round-tripping like the rest.
|
||||||
|
assert_eq!(back.cookie_domain.as_deref(), Some("example.com"));
|
||||||
|
assert_eq!(dto.daily_at, "00:00");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn crawler_cookie_domain_is_editable() {
|
||||||
|
let base = CrawlerConfig::default();
|
||||||
|
let dto = CrawlerSettings {
|
||||||
|
cookie_domain: Some("new.example.org".to_string()),
|
||||||
|
..CrawlerSettings::from_config(&base)
|
||||||
|
};
|
||||||
|
assert_eq!(
|
||||||
|
dto.to_config(&base).unwrap().cookie_domain.as_deref(),
|
||||||
|
Some("new.example.org")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn crawler_overlay_preserves_env_only_fields() {
|
||||||
|
let mut base = CrawlerConfig::default();
|
||||||
|
base.proxy = Some("socks5://127.0.0.1:9050".to_string());
|
||||||
|
base.tor_control_password = Some("secret".to_string());
|
||||||
|
base.phpsessid = Some("abc123".to_string());
|
||||||
|
// A DTO that knows nothing about the env-only fields.
|
||||||
|
let dto = CrawlerSettings {
|
||||||
|
rate_ms: 2000,
|
||||||
|
..CrawlerSettings::from_config(&base)
|
||||||
|
};
|
||||||
|
let out = dto.to_config(&base).expect("valid");
|
||||||
|
assert_eq!(out.rate_ms, 2000);
|
||||||
|
assert_eq!(out.proxy.as_deref(), Some("socks5://127.0.0.1:9050"));
|
||||||
|
assert_eq!(out.tor_control_password.as_deref(), Some("secret"));
|
||||||
|
assert_eq!(out.phpsessid.as_deref(), Some("abc123"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn crawler_rejects_bad_tz_and_time() {
|
||||||
|
let base = CrawlerConfig::default();
|
||||||
|
let dto = CrawlerSettings {
|
||||||
|
tz: "Mars/Phobos".to_string(),
|
||||||
|
daily_at: "9am".to_string(),
|
||||||
|
..CrawlerSettings::from_config(&base)
|
||||||
|
};
|
||||||
|
let errs = dto.to_config(&base).unwrap_err();
|
||||||
|
let fields: Vec<_> = errs.errors.iter().map(|e| e.field.as_str()).collect();
|
||||||
|
assert!(fields.contains(&"tz"));
|
||||||
|
assert!(fields.contains(&"daily_at"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn crawler_allowlist_seeds_start_and_cdn_hosts() {
|
||||||
|
let base = CrawlerConfig::default();
|
||||||
|
let dto = CrawlerSettings {
|
||||||
|
start_url: Some("https://catalog.example.com/".to_string()),
|
||||||
|
cdn_host: Some("cdn.example.com".to_string()),
|
||||||
|
download_allowlist: vec!["extra.example.net".to_string()],
|
||||||
|
allow_any_host: false,
|
||||||
|
..CrawlerSettings::from_config(&base)
|
||||||
|
};
|
||||||
|
let out = dto.to_config(&base).expect("valid");
|
||||||
|
assert!(out.download_allowlist.contains("catalog.example.com"));
|
||||||
|
assert!(out.download_allowlist.contains("cdn.example.com"));
|
||||||
|
assert!(out.download_allowlist.contains("extra.example.net"));
|
||||||
|
assert!(!out.download_allowlist.is_allow_any());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn crawler_allow_any_host_bypasses_list() {
|
||||||
|
let base = CrawlerConfig::default();
|
||||||
|
let dto = CrawlerSettings {
|
||||||
|
allow_any_host: true,
|
||||||
|
..CrawlerSettings::from_config(&base)
|
||||||
|
};
|
||||||
|
let out = dto.to_config(&base).expect("valid");
|
||||||
|
assert!(out.download_allowlist.is_allow_any());
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- analysis round-trip, validation & prompt defaults ---
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn analysis_round_trips_and_omits_default_prompts() {
|
||||||
|
let base = AnalysisConfig::default();
|
||||||
|
let dto = AnalysisSettings::from_config(&base);
|
||||||
|
// Unmodified prompts seed as None ("use default").
|
||||||
|
assert!(dto.system_prompt.is_none());
|
||||||
|
assert!(dto.ocr_prompt.is_none());
|
||||||
|
assert!(dto.grounding_prompt.is_none());
|
||||||
|
assert_eq!(dto.response_format, "json_schema");
|
||||||
|
let back = dto.to_config(&base).expect("valid");
|
||||||
|
assert_eq!(back.system_prompt, SYSTEM_PROMPT_DEFAULT);
|
||||||
|
assert_eq!(back.response_format, ResponseFormat::JsonSchema);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn analysis_captures_env_prompt_override() {
|
||||||
|
let mut base = AnalysisConfig::default();
|
||||||
|
base.system_prompt = "custom env prompt".to_string();
|
||||||
|
let dto = AnalysisSettings::from_config(&base);
|
||||||
|
assert_eq!(dto.system_prompt.as_deref(), Some("custom env prompt"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn analysis_prompt_override_and_reset() {
|
||||||
|
let base = AnalysisConfig::default();
|
||||||
|
// Override applied.
|
||||||
|
let dto = AnalysisSettings {
|
||||||
|
system_prompt: Some("OVERRIDE".to_string()),
|
||||||
|
..AnalysisSettings::from_config(&base)
|
||||||
|
};
|
||||||
|
assert_eq!(dto.to_config(&base).unwrap().system_prompt, "OVERRIDE");
|
||||||
|
// None / blank falls back to the compiled default.
|
||||||
|
let dto = AnalysisSettings {
|
||||||
|
system_prompt: Some(" ".to_string()),
|
||||||
|
..AnalysisSettings::from_config(&base)
|
||||||
|
};
|
||||||
|
assert_eq!(dto.to_config(&base).unwrap().system_prompt, SYSTEM_PROMPT_DEFAULT);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn analysis_overlay_preserves_api_key() {
|
||||||
|
let mut base = AnalysisConfig::default();
|
||||||
|
base.api_key = Some("sk-secret".to_string());
|
||||||
|
let dto = AnalysisSettings::from_config(&base);
|
||||||
|
assert_eq!(dto.to_config(&base).unwrap().api_key.as_deref(), Some("sk-secret"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn analysis_rejects_out_of_range() {
|
||||||
|
let base = AnalysisConfig::default();
|
||||||
|
let dto = AnalysisSettings {
|
||||||
|
workers: 0,
|
||||||
|
slice_overlap: 1.5,
|
||||||
|
tall_aspect_threshold: 0.5,
|
||||||
|
response_format: "yaml".to_string(),
|
||||||
|
endpoint: "not a url".to_string(),
|
||||||
|
request_timeout_secs: 0,
|
||||||
|
job_timeout_secs: 0,
|
||||||
|
max_pixels: 0,
|
||||||
|
max_image_bytes: 0,
|
||||||
|
..AnalysisSettings::from_config(&base)
|
||||||
|
};
|
||||||
|
let errs = dto.to_config(&base).unwrap_err();
|
||||||
|
let fields: Vec<_> = errs.errors.iter().map(|e| e.field.as_str()).collect();
|
||||||
|
for f in [
|
||||||
|
"workers",
|
||||||
|
"slice_overlap",
|
||||||
|
"tall_aspect_threshold",
|
||||||
|
"response_format",
|
||||||
|
"endpoint",
|
||||||
|
"request_timeout_secs",
|
||||||
|
"job_timeout_secs",
|
||||||
|
"max_pixels",
|
||||||
|
"max_image_bytes",
|
||||||
|
] {
|
||||||
|
assert!(fields.contains(&f), "missing error for {f}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn analysis_requires_model_only_when_enabled() {
|
||||||
|
let base = AnalysisConfig::default();
|
||||||
|
let disabled = AnalysisSettings {
|
||||||
|
enabled: false,
|
||||||
|
model: "".to_string(),
|
||||||
|
..AnalysisSettings::from_config(&base)
|
||||||
|
};
|
||||||
|
assert!(disabled.to_config(&base).is_ok());
|
||||||
|
let enabled = AnalysisSettings {
|
||||||
|
enabled: true,
|
||||||
|
model: "".to_string(),
|
||||||
|
..AnalysisSettings::from_config(&base)
|
||||||
|
};
|
||||||
|
let errs = enabled.to_config(&base).unwrap_err();
|
||||||
|
assert!(errs.errors.iter().any(|e| e.field == "model"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn dtos_serialize_to_json_and_back() {
|
||||||
|
let c = CrawlerSettings::default();
|
||||||
|
let v = serde_json::to_value(&c).unwrap();
|
||||||
|
let back: CrawlerSettings = serde_json::from_value(v).unwrap();
|
||||||
|
assert_eq!(c, back);
|
||||||
|
|
||||||
|
let a = AnalysisSettings::default();
|
||||||
|
let v = serde_json::to_value(&a).unwrap();
|
||||||
|
let back: AnalysisSettings = serde_json::from_value(v).unwrap();
|
||||||
|
assert_eq!(a, back);
|
||||||
|
}
|
||||||
|
}
|
||||||
272
backend/tests/api_admin_settings.rs
Normal file
272
backend/tests/api_admin_settings.rs
Normal file
@@ -0,0 +1,272 @@
|
|||||||
|
//! Integration tests for the runtime-editable settings endpoints
|
||||||
|
//! (`/api/v1/admin/settings/{crawler,analysis}`):
|
||||||
|
//!
|
||||||
|
//! * the `RequireAdmin` gate,
|
||||||
|
//! * `GET` returns the editable DTO + the read-only env-managed view (and the
|
||||||
|
//! analysis prompt defaults), never the secret,
|
||||||
|
//! * `PUT` validates (422 with per-field details on bad input), persists, and
|
||||||
|
//! writes an `admin_audit` row,
|
||||||
|
//! * `PUT` invokes the daemon reloader with the converted config and moves the
|
||||||
|
//! analysis enable gate,
|
||||||
|
//! * the repo-level env→DB seed is idempotent.
|
||||||
|
|
||||||
|
mod common;
|
||||||
|
|
||||||
|
use axum::http::StatusCode;
|
||||||
|
use axum::Router;
|
||||||
|
use serde_json::json;
|
||||||
|
use sqlx::PgPool;
|
||||||
|
use tower::ServiceExt;
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
use mangalord::repo;
|
||||||
|
|
||||||
|
async fn seed_admin(pool: &PgPool, app: &Router) -> (String, String, Uuid) {
|
||||||
|
let (username, cookie) = common::register_user(app).await;
|
||||||
|
let u = repo::user::find_by_username(pool, &username)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
repo::user::set_is_admin_unchecked(pool, u.id, true).await.unwrap();
|
||||||
|
(username, cookie, u.id)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- RequireAdmin gate -----------------------------------------------------
|
||||||
|
|
||||||
|
#[sqlx::test(migrations = "./migrations")]
|
||||||
|
async fn get_settings_requires_admin(pool: PgPool) {
|
||||||
|
let h = common::harness(pool);
|
||||||
|
let (_u, cookie) = common::register_user(&h.app).await;
|
||||||
|
let resp = h
|
||||||
|
.app
|
||||||
|
.oneshot(common::get_with_cookie("/api/v1/admin/settings/crawler", &cookie))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[sqlx::test(migrations = "./migrations")]
|
||||||
|
async fn get_settings_rejects_anonymous(pool: PgPool) {
|
||||||
|
let h = common::harness(pool);
|
||||||
|
// No cookie at all → not logged in.
|
||||||
|
let resp = h
|
||||||
|
.app
|
||||||
|
.oneshot(common::get("/api/v1/admin/settings/crawler"))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(resp.status(), StatusCode::UNAUTHORIZED);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[sqlx::test(migrations = "./migrations")]
|
||||||
|
async fn put_settings_requires_admin(pool: PgPool) {
|
||||||
|
let h = common::harness(pool);
|
||||||
|
let (_u, cookie) = common::register_user(&h.app).await;
|
||||||
|
let resp = h
|
||||||
|
.app
|
||||||
|
.oneshot(common::put_json_with_cookie(
|
||||||
|
"/api/v1/admin/settings/analysis",
|
||||||
|
json!({ "enabled": false }),
|
||||||
|
&cookie,
|
||||||
|
))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- GET shape -------------------------------------------------------------
|
||||||
|
|
||||||
|
#[sqlx::test(migrations = "./migrations")]
|
||||||
|
async fn get_crawler_returns_editable_and_env_only(pool: PgPool) {
|
||||||
|
let h = common::harness(pool.clone());
|
||||||
|
let (_u, cookie, _id) = seed_admin(&pool, &h.app).await;
|
||||||
|
let resp = h
|
||||||
|
.app
|
||||||
|
.oneshot(common::get_with_cookie("/api/v1/admin/settings/crawler", &cookie))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(resp.status(), StatusCode::OK);
|
||||||
|
let body = common::body_json(resp).await;
|
||||||
|
// Editable knobs present with their defaults.
|
||||||
|
assert_eq!(body["editable"]["daily_at"], "00:00");
|
||||||
|
assert_eq!(body["editable"]["chapter_workers"], 1);
|
||||||
|
// Env-managed view present and read-only.
|
||||||
|
assert_eq!(body["env_only"]["browser_mode"], "headless");
|
||||||
|
assert_eq!(body["env_only"]["session_configured"], false);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[sqlx::test(migrations = "./migrations")]
|
||||||
|
async fn get_analysis_returns_defaults_and_no_secret(pool: PgPool) {
|
||||||
|
let h = common::harness(pool.clone());
|
||||||
|
let (_u, cookie, _id) = seed_admin(&pool, &h.app).await;
|
||||||
|
let resp = h
|
||||||
|
.app
|
||||||
|
.oneshot(common::get_with_cookie("/api/v1/admin/settings/analysis", &cookie))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(resp.status(), StatusCode::OK);
|
||||||
|
let body = common::body_json(resp).await;
|
||||||
|
// Unmodified prompts seed as null ("use default").
|
||||||
|
assert!(body["editable"]["system_prompt"].is_null());
|
||||||
|
// Compiled defaults exposed for the UI placeholder / reset.
|
||||||
|
assert!(body["prompt_defaults"]["system_prompt"]
|
||||||
|
.as_str()
|
||||||
|
.unwrap()
|
||||||
|
.contains("manga"));
|
||||||
|
// The secret is never echoed; only a boolean indicator.
|
||||||
|
assert!(body["editable"].get("api_key").is_none());
|
||||||
|
assert_eq!(body["env_only"]["api_key_configured"], false);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- PUT validation --------------------------------------------------------
|
||||||
|
|
||||||
|
#[sqlx::test(migrations = "./migrations")]
|
||||||
|
async fn put_crawler_invalid_tz_returns_422(pool: PgPool) {
|
||||||
|
let h = common::harness(pool.clone());
|
||||||
|
let (_u, cookie, _id) = seed_admin(&pool, &h.app).await;
|
||||||
|
let resp = h
|
||||||
|
.app
|
||||||
|
.oneshot(common::put_json_with_cookie(
|
||||||
|
"/api/v1/admin/settings/crawler",
|
||||||
|
json!({ "tz": "Mars/Phobos", "daily_at": "9am" }),
|
||||||
|
&cookie,
|
||||||
|
))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(resp.status(), StatusCode::UNPROCESSABLE_ENTITY);
|
||||||
|
let body = common::body_json(resp).await;
|
||||||
|
let fields: Vec<&str> = body["error"]["details"]["fields"]
|
||||||
|
.as_array()
|
||||||
|
.unwrap()
|
||||||
|
.iter()
|
||||||
|
.map(|e| e["field"].as_str().unwrap())
|
||||||
|
.collect();
|
||||||
|
assert!(fields.contains(&"tz"));
|
||||||
|
assert!(fields.contains(&"daily_at"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[sqlx::test(migrations = "./migrations")]
|
||||||
|
async fn put_analysis_invalid_returns_422(pool: PgPool) {
|
||||||
|
let h = common::harness(pool.clone());
|
||||||
|
let (_u, cookie, _id) = seed_admin(&pool, &h.app).await;
|
||||||
|
let resp = h
|
||||||
|
.app
|
||||||
|
.oneshot(common::put_json_with_cookie(
|
||||||
|
"/api/v1/admin/settings/analysis",
|
||||||
|
json!({ "workers": 0, "slice_overlap": 2.0, "endpoint": "nope" }),
|
||||||
|
&cookie,
|
||||||
|
))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(resp.status(), StatusCode::UNPROCESSABLE_ENTITY);
|
||||||
|
let body = common::body_json(resp).await;
|
||||||
|
let fields: Vec<&str> = body["error"]["details"]["fields"]
|
||||||
|
.as_array()
|
||||||
|
.unwrap()
|
||||||
|
.iter()
|
||||||
|
.map(|e| e["field"].as_str().unwrap())
|
||||||
|
.collect();
|
||||||
|
assert!(fields.contains(&"workers"));
|
||||||
|
assert!(fields.contains(&"slice_overlap"));
|
||||||
|
assert!(fields.contains(&"endpoint"));
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- PUT persistence + audit ----------------------------------------------
|
||||||
|
|
||||||
|
#[sqlx::test(migrations = "./migrations")]
|
||||||
|
async fn put_crawler_persists_and_audits(pool: PgPool) {
|
||||||
|
let h = common::harness(pool.clone());
|
||||||
|
let (_u, cookie, admin_id) = seed_admin(&pool, &h.app).await;
|
||||||
|
|
||||||
|
let resp = h
|
||||||
|
.app
|
||||||
|
.clone()
|
||||||
|
.oneshot(common::put_json_with_cookie(
|
||||||
|
"/api/v1/admin/settings/crawler",
|
||||||
|
json!({ "rate_ms": 2500, "chapter_workers": 4, "start_url": "https://example.com/" }),
|
||||||
|
&cookie,
|
||||||
|
))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(resp.status(), StatusCode::OK);
|
||||||
|
let body = common::body_json(resp).await;
|
||||||
|
assert_eq!(body["editable"]["rate_ms"], 2500);
|
||||||
|
assert_eq!(body["editable"]["chapter_workers"], 4);
|
||||||
|
// Allowlist normalized to include the start-url host.
|
||||||
|
let allow: Vec<&str> = body["editable"]["download_allowlist"]
|
||||||
|
.as_array()
|
||||||
|
.unwrap()
|
||||||
|
.iter()
|
||||||
|
.map(|v| v.as_str().unwrap())
|
||||||
|
.collect();
|
||||||
|
assert!(allow.contains(&"example.com"));
|
||||||
|
|
||||||
|
// A second GET reflects the persisted change.
|
||||||
|
let resp = h
|
||||||
|
.app
|
||||||
|
.clone()
|
||||||
|
.oneshot(common::get_with_cookie("/api/v1/admin/settings/crawler", &cookie))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let body = common::body_json(resp).await;
|
||||||
|
assert_eq!(body["editable"]["rate_ms"], 2500);
|
||||||
|
|
||||||
|
// An audit row landed.
|
||||||
|
let (action, kind): (String, String) = sqlx::query_as(
|
||||||
|
"SELECT action, target_kind FROM admin_audit WHERE actor_user_id = $1 ORDER BY at DESC LIMIT 1",
|
||||||
|
)
|
||||||
|
.bind(admin_id)
|
||||||
|
.fetch_one(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(action, "update_crawler_settings");
|
||||||
|
assert_eq!(kind, "settings");
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- PUT triggers reload ---------------------------------------------------
|
||||||
|
|
||||||
|
#[sqlx::test(migrations = "./migrations")]
|
||||||
|
async fn put_analysis_triggers_reload_and_flips_gate(pool: PgPool) {
|
||||||
|
let (h, reloader) = common::harness_with_settings_reloader(pool.clone());
|
||||||
|
let (_u, cookie, _id) = seed_admin(&pool, &h.app).await;
|
||||||
|
|
||||||
|
// Gate starts closed.
|
||||||
|
assert!(!reloader.runtime.analysis_enabled());
|
||||||
|
|
||||||
|
let resp = h
|
||||||
|
.app
|
||||||
|
.clone()
|
||||||
|
.oneshot(common::put_json_with_cookie(
|
||||||
|
"/api/v1/admin/settings/analysis",
|
||||||
|
json!({ "enabled": true, "model": "qwen2-vl", "temperature": 0.4 }),
|
||||||
|
&cookie,
|
||||||
|
))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(resp.status(), StatusCode::OK);
|
||||||
|
|
||||||
|
// The reloader was invoked with the converted config, and the shared gate
|
||||||
|
// flipped on — both without a restart.
|
||||||
|
assert!(reloader.runtime.analysis_enabled());
|
||||||
|
let applied = reloader.analysis.lock().unwrap().clone().expect("reload called");
|
||||||
|
assert!(applied.enabled);
|
||||||
|
assert_eq!(applied.model, "qwen2-vl");
|
||||||
|
assert_eq!(applied.temperature, 0.4);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- repo-level seed -------------------------------------------------------
|
||||||
|
|
||||||
|
#[sqlx::test(migrations = "./migrations")]
|
||||||
|
async fn seed_if_absent_is_idempotent(pool: PgPool) {
|
||||||
|
let v1 = json!({ "rate_ms": 1000 });
|
||||||
|
let v2 = json!({ "rate_ms": 9999 });
|
||||||
|
// First seed inserts.
|
||||||
|
assert!(repo::app_settings::seed_if_absent(&pool, "crawler", &v1).await.unwrap());
|
||||||
|
// Second seed is a no-op (row already present) and does not overwrite.
|
||||||
|
assert!(!repo::app_settings::seed_if_absent(&pool, "crawler", &v2).await.unwrap());
|
||||||
|
let got = repo::app_settings::get(&pool, "crawler").await.unwrap().unwrap();
|
||||||
|
assert_eq!(got["rate_ms"], 1000);
|
||||||
|
// upsert does overwrite.
|
||||||
|
repo::app_settings::upsert(&pool, "crawler", &v2).await.unwrap();
|
||||||
|
let got = repo::app_settings::get(&pool, "crawler").await.unwrap().unwrap();
|
||||||
|
assert_eq!(got["rate_ms"], 9999);
|
||||||
|
}
|
||||||
@@ -14,9 +14,9 @@ use sqlx::PgPool;
|
|||||||
use tempfile::TempDir;
|
use tempfile::TempDir;
|
||||||
use tower::ServiceExt;
|
use tower::ServiceExt;
|
||||||
|
|
||||||
use mangalord::app::{router, AppState};
|
use mangalord::app::{router, AppState, RuntimeControls};
|
||||||
use mangalord::auth::rate_limit::AuthRateLimiter;
|
use mangalord::auth::rate_limit::AuthRateLimiter;
|
||||||
use mangalord::config::{AuthConfig, UploadConfig};
|
use mangalord::config::{AnalysisConfig, AuthConfig, CrawlerConfig, UploadConfig};
|
||||||
use mangalord::storage::{LocalStorage, PutByteStream, Storage, StorageError, StreamingFile};
|
use mangalord::storage::{LocalStorage, PutByteStream, Storage, StorageError, StreamingFile};
|
||||||
|
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
@@ -76,13 +76,15 @@ fn harness_with_auth_config(
|
|||||||
auth_limiter,
|
auth_limiter,
|
||||||
// Default harness has no crawler daemon wired up; admin resync
|
// Default harness has no crawler daemon wired up; admin resync
|
||||||
// handlers return 503 in this config. Tests that need a stub
|
// handlers return 503 in this config. Tests that need a stub
|
||||||
// resync service swap it in via `harness_with_resync`.
|
// resync service swap it in via `harness_with_resync`. No reloader,
|
||||||
resync: None,
|
// so settings still persist but spawn no daemon.
|
||||||
crawler: None,
|
runtime: Arc::new(RuntimeControls::new(false)),
|
||||||
|
reloader: None,
|
||||||
|
crawler_base: CrawlerConfig::default(),
|
||||||
|
analysis_base: AnalysisConfig::default(),
|
||||||
// Empty allowlist = CSRF check skipped. The CSRF-specific test
|
// Empty allowlist = CSRF check skipped. The CSRF-specific test
|
||||||
// harness `harness_with_admin_origins` overrides this.
|
// harness `harness_with_admin_origins` overrides this.
|
||||||
admin_allowed_origins: Arc::new(Vec::new()),
|
admin_allowed_origins: Arc::new(Vec::new()),
|
||||||
analysis_enabled: false,
|
|
||||||
analysis_events: Arc::new(mangalord::analysis::events::AnalysisEvents::new()),
|
analysis_events: Arc::new(mangalord::analysis::events::AnalysisEvents::new()),
|
||||||
};
|
};
|
||||||
Harness { app: router(state), _storage_dir: storage_dir }
|
Harness { app: router(state), _storage_dir: storage_dir }
|
||||||
@@ -148,6 +150,8 @@ pub fn harness_with_resync(
|
|||||||
..AuthConfig::default()
|
..AuthConfig::default()
|
||||||
};
|
};
|
||||||
let auth_limiter = Arc::new(AuthRateLimiter::new(auth.rate_limit));
|
let auth_limiter = Arc::new(AuthRateLimiter::new(auth.rate_limit));
|
||||||
|
let runtime = Arc::new(RuntimeControls::new(false));
|
||||||
|
runtime.set_resync(Some(resync));
|
||||||
let state = AppState {
|
let state = AppState {
|
||||||
db: pool,
|
db: pool,
|
||||||
storage,
|
storage,
|
||||||
@@ -157,10 +161,11 @@ pub fn harness_with_resync(
|
|||||||
max_file_bytes: 256 * 1024,
|
max_file_bytes: 256 * 1024,
|
||||||
},
|
},
|
||||||
auth_limiter,
|
auth_limiter,
|
||||||
resync: Some(resync),
|
runtime,
|
||||||
crawler: None,
|
reloader: None,
|
||||||
|
crawler_base: CrawlerConfig::default(),
|
||||||
|
analysis_base: AnalysisConfig::default(),
|
||||||
admin_allowed_origins: Arc::new(Vec::new()),
|
admin_allowed_origins: Arc::new(Vec::new()),
|
||||||
analysis_enabled: false,
|
|
||||||
analysis_events: Arc::new(mangalord::analysis::events::AnalysisEvents::new()),
|
analysis_events: Arc::new(mangalord::analysis::events::AnalysisEvents::new()),
|
||||||
};
|
};
|
||||||
Harness {
|
Harness {
|
||||||
@@ -189,10 +194,11 @@ pub fn harness_with_analysis(pool: PgPool) -> Harness {
|
|||||||
max_file_bytes: 256 * 1024,
|
max_file_bytes: 256 * 1024,
|
||||||
},
|
},
|
||||||
auth_limiter,
|
auth_limiter,
|
||||||
resync: None,
|
runtime: Arc::new(RuntimeControls::new(true)),
|
||||||
crawler: None,
|
reloader: None,
|
||||||
|
crawler_base: CrawlerConfig::default(),
|
||||||
|
analysis_base: AnalysisConfig::default(),
|
||||||
admin_allowed_origins: Arc::new(Vec::new()),
|
admin_allowed_origins: Arc::new(Vec::new()),
|
||||||
analysis_enabled: true,
|
|
||||||
analysis_events: Arc::new(mangalord::analysis::events::AnalysisEvents::new()),
|
analysis_events: Arc::new(mangalord::analysis::events::AnalysisEvents::new()),
|
||||||
};
|
};
|
||||||
Harness {
|
Harness {
|
||||||
@@ -201,6 +207,71 @@ pub fn harness_with_analysis(pool: PgPool) -> Harness {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A [`DaemonReloader`] stub that records the configs it was asked to apply
|
||||||
|
/// and flips the shared analysis gate, without spawning any real daemon. Lets
|
||||||
|
/// settings tests assert that a `PUT` triggers a reload with the converted
|
||||||
|
/// config (and that the analysis enable gate moves).
|
||||||
|
pub struct StubReloader {
|
||||||
|
pub runtime: Arc<RuntimeControls>,
|
||||||
|
pub crawler: std::sync::Mutex<Option<CrawlerConfig>>,
|
||||||
|
pub analysis: std::sync::Mutex<Option<AnalysisConfig>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
impl mangalord::app::DaemonReloader for StubReloader {
|
||||||
|
async fn reload_crawler(&self, cfg: CrawlerConfig) -> anyhow::Result<()> {
|
||||||
|
*self.crawler.lock().unwrap() = Some(cfg);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
async fn reload_analysis(&self, cfg: AnalysisConfig) -> anyhow::Result<()> {
|
||||||
|
self.runtime.set_analysis_enabled(cfg.enabled);
|
||||||
|
*self.analysis.lock().unwrap() = Some(cfg);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Like [`harness`] but wires a [`StubReloader`] so the settings `PUT`
|
||||||
|
/// endpoints exercise the reload path. Returns the harness plus the shared
|
||||||
|
/// stub so the test can inspect what was applied.
|
||||||
|
pub fn harness_with_settings_reloader(pool: PgPool) -> (Harness, Arc<StubReloader>) {
|
||||||
|
let storage_dir = tempfile::tempdir().expect("tempdir");
|
||||||
|
let storage = Arc::new(LocalStorage::new(storage_dir.path()));
|
||||||
|
let auth = AuthConfig {
|
||||||
|
cookie_secure: false,
|
||||||
|
..AuthConfig::default()
|
||||||
|
};
|
||||||
|
let auth_limiter = Arc::new(AuthRateLimiter::new(auth.rate_limit));
|
||||||
|
let runtime = Arc::new(RuntimeControls::new(false));
|
||||||
|
let reloader = Arc::new(StubReloader {
|
||||||
|
runtime: Arc::clone(&runtime),
|
||||||
|
crawler: std::sync::Mutex::new(None),
|
||||||
|
analysis: std::sync::Mutex::new(None),
|
||||||
|
});
|
||||||
|
let state = AppState {
|
||||||
|
db: pool,
|
||||||
|
storage,
|
||||||
|
auth,
|
||||||
|
upload: UploadConfig {
|
||||||
|
max_request_bytes: 4 * 1024 * 1024,
|
||||||
|
max_file_bytes: 256 * 1024,
|
||||||
|
},
|
||||||
|
auth_limiter,
|
||||||
|
runtime,
|
||||||
|
reloader: Some(Arc::clone(&reloader) as Arc<dyn mangalord::app::DaemonReloader>),
|
||||||
|
crawler_base: CrawlerConfig::default(),
|
||||||
|
analysis_base: AnalysisConfig::default(),
|
||||||
|
admin_allowed_origins: Arc::new(Vec::new()),
|
||||||
|
analysis_events: Arc::new(mangalord::analysis::events::AnalysisEvents::new()),
|
||||||
|
};
|
||||||
|
(
|
||||||
|
Harness {
|
||||||
|
app: router(state),
|
||||||
|
_storage_dir: storage_dir,
|
||||||
|
},
|
||||||
|
reloader,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
/// Like [`harness`] but configures an admin CSRF allowlist so the
|
/// Like [`harness`] but configures an admin CSRF allowlist so the
|
||||||
/// `/admin/*` mutating endpoints reject cross-origin browser POSTs.
|
/// `/admin/*` mutating endpoints reject cross-origin browser POSTs.
|
||||||
/// Used by the admin CSRF integration tests.
|
/// Used by the admin CSRF integration tests.
|
||||||
@@ -220,10 +291,11 @@ pub fn harness_with_admin_origins(pool: PgPool, origins: Vec<String>) -> Harness
|
|||||||
max_file_bytes: 256 * 1024,
|
max_file_bytes: 256 * 1024,
|
||||||
},
|
},
|
||||||
auth_limiter,
|
auth_limiter,
|
||||||
resync: None,
|
runtime: Arc::new(RuntimeControls::new(false)),
|
||||||
crawler: None,
|
reloader: None,
|
||||||
|
crawler_base: CrawlerConfig::default(),
|
||||||
|
analysis_base: AnalysisConfig::default(),
|
||||||
admin_allowed_origins: Arc::new(origins),
|
admin_allowed_origins: Arc::new(origins),
|
||||||
analysis_enabled: false,
|
|
||||||
analysis_events: Arc::new(mangalord::analysis::events::AnalysisEvents::new()),
|
analysis_events: Arc::new(mangalord::analysis::events::AnalysisEvents::new()),
|
||||||
};
|
};
|
||||||
Harness {
|
Harness {
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "mangalord-frontend",
|
"name": "mangalord-frontend",
|
||||||
"version": "0.79.1",
|
"version": "0.80.0",
|
||||||
"private": true,
|
"private": true,
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
|
|||||||
@@ -556,3 +556,110 @@ export type AnalysisEvent =
|
|||||||
export function analysisStatusStreamUrl(): string {
|
export function analysisStatusStreamUrl(): string {
|
||||||
return apiUrl('/v1/admin/analysis/status/stream');
|
return apiUrl('/v1/admin/analysis/status/stream');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ---- runtime settings (crawler + analysis) ---------------------------------
|
||||||
|
|
||||||
|
/** Operationally-safe, admin-editable crawler settings. Host/infra and
|
||||||
|
* session/secret fields are managed via environment and surfaced read-only
|
||||||
|
* in {@link CrawlerEnvOnly}. */
|
||||||
|
export type CrawlerSettings = {
|
||||||
|
daemon_enabled: boolean;
|
||||||
|
daily_at: string; // "HH:MM"
|
||||||
|
tz: string; // IANA
|
||||||
|
idle_timeout_secs: number;
|
||||||
|
chapter_workers: number;
|
||||||
|
retention_days: number;
|
||||||
|
start_url: string | null;
|
||||||
|
rate_ms: number;
|
||||||
|
cdn_host: string | null;
|
||||||
|
cdn_rate_ms: number;
|
||||||
|
cookie_domain: string | null;
|
||||||
|
user_agent: string | null;
|
||||||
|
download_allowlist: string[];
|
||||||
|
allow_any_host: boolean;
|
||||||
|
max_image_bytes: number;
|
||||||
|
manga_limit: number;
|
||||||
|
job_timeout_secs: number;
|
||||||
|
metadata_max_consecutive_failures: number;
|
||||||
|
browser_restart_threshold: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
/** Read-only view of crawler fields managed via environment. */
|
||||||
|
export type CrawlerEnvOnly = {
|
||||||
|
browser_mode: string;
|
||||||
|
browser_args: string[];
|
||||||
|
proxy: string | null;
|
||||||
|
tor_control_url: string | null;
|
||||||
|
tor_credentials_configured: boolean;
|
||||||
|
session_configured: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type CrawlerSettingsResponse = {
|
||||||
|
editable: CrawlerSettings;
|
||||||
|
env_only: CrawlerEnvOnly;
|
||||||
|
};
|
||||||
|
|
||||||
|
/** Admin-editable analysis settings. Prompts are `null` to mean "use the
|
||||||
|
* compiled default" (see {@link PromptDefaults}). The vision API key is
|
||||||
|
* env-only and never returned. */
|
||||||
|
export type AnalysisSettings = {
|
||||||
|
enabled: boolean;
|
||||||
|
workers: number;
|
||||||
|
endpoint: string;
|
||||||
|
model: string;
|
||||||
|
request_timeout_secs: number;
|
||||||
|
job_timeout_secs: number;
|
||||||
|
max_tokens: number;
|
||||||
|
max_pixels: number;
|
||||||
|
min_slice_height: number;
|
||||||
|
slice_overlap: number;
|
||||||
|
tall_aspect_threshold: number;
|
||||||
|
max_slices: number;
|
||||||
|
max_image_bytes: number;
|
||||||
|
response_format: 'json_schema' | 'json_object' | 'none';
|
||||||
|
frequency_penalty: number;
|
||||||
|
temperature: number;
|
||||||
|
system_prompt: string | null;
|
||||||
|
ocr_prompt: string | null;
|
||||||
|
grounding_prompt: string | null;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type PromptDefaults = {
|
||||||
|
system_prompt: string;
|
||||||
|
ocr_prompt: string;
|
||||||
|
grounding_prompt: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type AnalysisSettingsResponse = {
|
||||||
|
editable: AnalysisSettings;
|
||||||
|
env_only: { api_key_configured: boolean };
|
||||||
|
prompt_defaults: PromptDefaults;
|
||||||
|
};
|
||||||
|
|
||||||
|
export async function getCrawlerSettings(): Promise<CrawlerSettingsResponse> {
|
||||||
|
return request<CrawlerSettingsResponse>('/v1/admin/settings/crawler');
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function updateCrawlerSettings(
|
||||||
|
settings: CrawlerSettings
|
||||||
|
): Promise<CrawlerSettingsResponse> {
|
||||||
|
return request<CrawlerSettingsResponse>('/v1/admin/settings/crawler', {
|
||||||
|
method: 'PUT',
|
||||||
|
headers: { 'content-type': 'application/json' },
|
||||||
|
body: JSON.stringify(settings)
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function getAnalysisSettings(): Promise<AnalysisSettingsResponse> {
|
||||||
|
return request<AnalysisSettingsResponse>('/v1/admin/settings/analysis');
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function updateAnalysisSettings(
|
||||||
|
settings: AnalysisSettings
|
||||||
|
): Promise<AnalysisSettingsResponse> {
|
||||||
|
return request<AnalysisSettingsResponse>('/v1/admin/settings/analysis', {
|
||||||
|
method: 'PUT',
|
||||||
|
headers: { 'content-type': 'application/json' },
|
||||||
|
body: JSON.stringify(settings)
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|||||||
@@ -25,14 +25,19 @@ export class ApiError extends Error {
|
|||||||
constructor(
|
constructor(
|
||||||
public readonly status: number,
|
public readonly status: number,
|
||||||
public readonly code: string,
|
public readonly code: string,
|
||||||
message: string
|
message: string,
|
||||||
|
/** The error envelope's `details` payload, when present (e.g. the
|
||||||
|
* per-field `{ fields: [...] }` of a `validation_failed` response). */
|
||||||
|
public readonly details?: unknown
|
||||||
) {
|
) {
|
||||||
super(message);
|
super(message);
|
||||||
this.name = 'ApiError';
|
this.name = 'ApiError';
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
type ErrorEnvelope = { error?: { code?: unknown; message?: unknown } };
|
type ErrorEnvelope = {
|
||||||
|
error?: { code?: unknown; message?: unknown; details?: unknown };
|
||||||
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Optional hook fired the first moment `request()` observes a 401 on
|
* Optional hook fired the first moment `request()` observes a 401 on
|
||||||
@@ -59,6 +64,7 @@ export async function request<T>(path: string, init?: RequestInit): Promise<T> {
|
|||||||
if (!res.ok) {
|
if (!res.ok) {
|
||||||
let code = 'http_error';
|
let code = 'http_error';
|
||||||
let message = `${res.status} ${res.statusText}`;
|
let message = `${res.status} ${res.statusText}`;
|
||||||
|
let details: unknown;
|
||||||
const ct = res.headers.get('content-type') ?? '';
|
const ct = res.headers.get('content-type') ?? '';
|
||||||
try {
|
try {
|
||||||
if (ct.includes('application/json')) {
|
if (ct.includes('application/json')) {
|
||||||
@@ -70,6 +76,9 @@ export async function request<T>(path: string, init?: RequestInit): Promise<T> {
|
|||||||
if (typeof body.error.message === 'string' && body.error.message) {
|
if (typeof body.error.message === 'string' && body.error.message) {
|
||||||
message = body.error.message;
|
message = body.error.message;
|
||||||
}
|
}
|
||||||
|
if (body.error.details !== undefined) {
|
||||||
|
details = body.error.details;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
const text = await res.text();
|
const text = await res.text();
|
||||||
@@ -88,7 +97,7 @@ export async function request<T>(path: string, init?: RequestInit): Promise<T> {
|
|||||||
console.error('on401 hook threw:', e);
|
console.error('on401 hook threw:', e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
throw new ApiError(res.status, code, message);
|
throw new ApiError(res.status, code, message, details);
|
||||||
}
|
}
|
||||||
// Any empty body (not just 204) returns undefined — the manga-add
|
// Any empty body (not just 204) returns undefined — the manga-add
|
||||||
// endpoint, for instance, signals create-vs-already-present via
|
// endpoint, for instance, signals create-vs-already-present via
|
||||||
|
|||||||
167
frontend/src/lib/api/settings.test.ts
Normal file
167
frontend/src/lib/api/settings.test.ts
Normal file
@@ -0,0 +1,167 @@
|
|||||||
|
import {
|
||||||
|
describe,
|
||||||
|
it,
|
||||||
|
expect,
|
||||||
|
vi,
|
||||||
|
beforeEach,
|
||||||
|
afterEach,
|
||||||
|
type MockInstance
|
||||||
|
} from 'vitest';
|
||||||
|
import {
|
||||||
|
getCrawlerSettings,
|
||||||
|
updateCrawlerSettings,
|
||||||
|
getAnalysisSettings,
|
||||||
|
updateAnalysisSettings,
|
||||||
|
type CrawlerSettings,
|
||||||
|
type AnalysisSettings
|
||||||
|
} from './admin';
|
||||||
|
import { ApiError } from './client';
|
||||||
|
|
||||||
|
function ok(body: unknown, status = 200): Response {
|
||||||
|
return new Response(JSON.stringify(body), {
|
||||||
|
status,
|
||||||
|
headers: { 'content-type': 'application/json' }
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
const crawlerFixture: CrawlerSettings = {
|
||||||
|
daemon_enabled: true,
|
||||||
|
daily_at: '00:00',
|
||||||
|
tz: 'UTC',
|
||||||
|
idle_timeout_secs: 600,
|
||||||
|
chapter_workers: 1,
|
||||||
|
retention_days: 7,
|
||||||
|
start_url: null,
|
||||||
|
rate_ms: 1000,
|
||||||
|
cdn_host: null,
|
||||||
|
cdn_rate_ms: 1000,
|
||||||
|
cookie_domain: null,
|
||||||
|
user_agent: null,
|
||||||
|
download_allowlist: [],
|
||||||
|
allow_any_host: false,
|
||||||
|
max_image_bytes: 33554432,
|
||||||
|
manga_limit: 0,
|
||||||
|
job_timeout_secs: 600,
|
||||||
|
metadata_max_consecutive_failures: 10,
|
||||||
|
browser_restart_threshold: 3
|
||||||
|
};
|
||||||
|
|
||||||
|
const analysisFixture: AnalysisSettings = {
|
||||||
|
enabled: false,
|
||||||
|
workers: 1,
|
||||||
|
endpoint: 'http://localhost:8000/v1/chat/completions',
|
||||||
|
model: '',
|
||||||
|
request_timeout_secs: 120,
|
||||||
|
job_timeout_secs: 600,
|
||||||
|
max_tokens: 4096,
|
||||||
|
max_pixels: 1000000,
|
||||||
|
min_slice_height: 640,
|
||||||
|
slice_overlap: 0.12,
|
||||||
|
tall_aspect_threshold: 1.6,
|
||||||
|
max_slices: 16,
|
||||||
|
max_image_bytes: 8388608,
|
||||||
|
response_format: 'json_schema',
|
||||||
|
frequency_penalty: 0.3,
|
||||||
|
temperature: 0,
|
||||||
|
system_prompt: null,
|
||||||
|
ocr_prompt: null,
|
||||||
|
grounding_prompt: null
|
||||||
|
};
|
||||||
|
|
||||||
|
describe('settings api client', () => {
|
||||||
|
let fetchSpy: MockInstance<typeof globalThis.fetch>;
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
fetchSpy = vi.spyOn(globalThis, 'fetch');
|
||||||
|
});
|
||||||
|
afterEach(() => {
|
||||||
|
vi.restoreAllMocks();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('getCrawlerSettings GETs the crawler settings endpoint', async () => {
|
||||||
|
fetchSpy.mockResolvedValueOnce(
|
||||||
|
ok({
|
||||||
|
editable: crawlerFixture,
|
||||||
|
env_only: {
|
||||||
|
browser_mode: 'headless',
|
||||||
|
browser_args: [],
|
||||||
|
proxy: null,
|
||||||
|
tor_control_url: null,
|
||||||
|
tor_credentials_configured: false,
|
||||||
|
session_configured: false
|
||||||
|
}
|
||||||
|
})
|
||||||
|
);
|
||||||
|
const r = await getCrawlerSettings();
|
||||||
|
expect(r.editable.rate_ms).toBe(1000);
|
||||||
|
expect(r.env_only.browser_mode).toBe('headless');
|
||||||
|
const url = fetchSpy.mock.calls[0][0] as string;
|
||||||
|
expect(url).toMatch(/\/v1\/admin\/settings\/crawler$/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('updateCrawlerSettings PUTs the body as JSON', async () => {
|
||||||
|
fetchSpy.mockResolvedValueOnce(
|
||||||
|
ok({
|
||||||
|
editable: { ...crawlerFixture, rate_ms: 2000 },
|
||||||
|
env_only: {
|
||||||
|
browser_mode: 'headless',
|
||||||
|
browser_args: [],
|
||||||
|
proxy: null,
|
||||||
|
tor_control_url: null,
|
||||||
|
tor_credentials_configured: false,
|
||||||
|
session_configured: false
|
||||||
|
}
|
||||||
|
})
|
||||||
|
);
|
||||||
|
const r = await updateCrawlerSettings({ ...crawlerFixture, rate_ms: 2000 });
|
||||||
|
expect(r.editable.rate_ms).toBe(2000);
|
||||||
|
const init = fetchSpy.mock.calls[0][1]!;
|
||||||
|
expect(init.method).toBe('PUT');
|
||||||
|
expect(JSON.parse(init.body as string).rate_ms).toBe(2000);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('getAnalysisSettings returns prompt defaults and the no-secret indicator', async () => {
|
||||||
|
fetchSpy.mockResolvedValueOnce(
|
||||||
|
ok({
|
||||||
|
editable: analysisFixture,
|
||||||
|
env_only: { api_key_configured: true },
|
||||||
|
prompt_defaults: {
|
||||||
|
system_prompt: 'SYS',
|
||||||
|
ocr_prompt: 'OCR',
|
||||||
|
grounding_prompt: 'GND'
|
||||||
|
}
|
||||||
|
})
|
||||||
|
);
|
||||||
|
const r = await getAnalysisSettings();
|
||||||
|
expect(r.editable.system_prompt).toBeNull();
|
||||||
|
expect(r.env_only.api_key_configured).toBe(true);
|
||||||
|
expect(r.prompt_defaults.system_prompt).toBe('SYS');
|
||||||
|
// The secret is never present on the editable DTO.
|
||||||
|
expect('api_key' in (r.editable as object)).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('updateAnalysisSettings surfaces 422 field errors via ApiError.details', async () => {
|
||||||
|
fetchSpy.mockResolvedValueOnce(
|
||||||
|
new Response(
|
||||||
|
JSON.stringify({
|
||||||
|
error: {
|
||||||
|
code: 'validation_failed',
|
||||||
|
message: 'invalid settings',
|
||||||
|
details: { fields: [{ field: 'workers', message: 'must be at least 1' }] }
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
{ status: 422, headers: { 'content-type': 'application/json' } }
|
||||||
|
)
|
||||||
|
);
|
||||||
|
try {
|
||||||
|
await updateAnalysisSettings({ ...analysisFixture, workers: 0 });
|
||||||
|
expect.unreachable('should have thrown');
|
||||||
|
} catch (e) {
|
||||||
|
expect(e).toBeInstanceOf(ApiError);
|
||||||
|
expect((e as ApiError).status).toBe(422);
|
||||||
|
expect((e as ApiError).code).toBe('validation_failed');
|
||||||
|
const details = (e as ApiError).details as { fields: { field: string }[] };
|
||||||
|
expect(details.fields[0].field).toBe('workers');
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -8,6 +8,7 @@
|
|||||||
{ href: '/admin/mangas', label: 'Mangas' },
|
{ href: '/admin/mangas', label: 'Mangas' },
|
||||||
{ href: '/admin/crawler', label: 'Crawler' },
|
{ href: '/admin/crawler', label: 'Crawler' },
|
||||||
{ href: '/admin/analysis', label: 'Analysis' },
|
{ href: '/admin/analysis', label: 'Analysis' },
|
||||||
|
{ href: '/admin/settings', label: 'Settings' },
|
||||||
{ href: '/admin/system', label: 'System' }
|
{ href: '/admin/system', label: 'System' }
|
||||||
];
|
];
|
||||||
</script>
|
</script>
|
||||||
|
|||||||
664
frontend/src/routes/admin/settings/+page.svelte
Normal file
664
frontend/src/routes/admin/settings/+page.svelte
Normal file
@@ -0,0 +1,664 @@
|
|||||||
|
<script lang="ts">
|
||||||
|
import { onMount } from 'svelte';
|
||||||
|
import Modal from '$lib/components/Modal.svelte';
|
||||||
|
import { ApiError } from '$lib/api/client';
|
||||||
|
import {
|
||||||
|
getCrawlerSettings,
|
||||||
|
updateCrawlerSettings,
|
||||||
|
getAnalysisSettings,
|
||||||
|
updateAnalysisSettings,
|
||||||
|
type CrawlerSettings,
|
||||||
|
type CrawlerEnvOnly,
|
||||||
|
type AnalysisSettings,
|
||||||
|
type PromptDefaults
|
||||||
|
} from '$lib/api/admin';
|
||||||
|
|
||||||
|
type Tab = 'crawler' | 'analysis';
|
||||||
|
let tab = $state<Tab>('crawler');
|
||||||
|
|
||||||
|
// Crawler state
|
||||||
|
let crawler = $state<CrawlerSettings | null>(null);
|
||||||
|
let crawlerEnv = $state<CrawlerEnvOnly | null>(null);
|
||||||
|
let allowlistText = $state(''); // textarea mirror of download_allowlist
|
||||||
|
|
||||||
|
// Analysis state
|
||||||
|
let analysis = $state<AnalysisSettings | null>(null);
|
||||||
|
let promptDefaults = $state<PromptDefaults | null>(null);
|
||||||
|
let apiKeyConfigured = $state(false);
|
||||||
|
|
||||||
|
let loading = $state(true);
|
||||||
|
let loadError = $state<string | null>(null);
|
||||||
|
let saving = $state(false);
|
||||||
|
let saveError = $state<string | null>(null);
|
||||||
|
let success = $state<string | null>(null);
|
||||||
|
// field -> message, parsed from a 422 validation_failed response
|
||||||
|
let fieldErrors = $state<Record<string, string>>({});
|
||||||
|
let confirmOpen = $state(false);
|
||||||
|
|
||||||
|
onMount(load);
|
||||||
|
|
||||||
|
async function load() {
|
||||||
|
loading = true;
|
||||||
|
loadError = null;
|
||||||
|
try {
|
||||||
|
const [c, a] = await Promise.all([getCrawlerSettings(), getAnalysisSettings()]);
|
||||||
|
crawler = c.editable;
|
||||||
|
crawlerEnv = c.env_only;
|
||||||
|
allowlistText = c.editable.download_allowlist.join('\n');
|
||||||
|
analysis = a.editable;
|
||||||
|
promptDefaults = a.prompt_defaults;
|
||||||
|
apiKeyConfigured = a.env_only.api_key_configured;
|
||||||
|
} catch (e) {
|
||||||
|
loadError = e instanceof Error ? e.message : 'Failed to load settings.';
|
||||||
|
} finally {
|
||||||
|
loading = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseFieldErrors(e: unknown): Record<string, string> {
|
||||||
|
const out: Record<string, string> = {};
|
||||||
|
if (
|
||||||
|
e instanceof ApiError &&
|
||||||
|
e.details &&
|
||||||
|
typeof e.details === 'object' &&
|
||||||
|
'fields' in e.details
|
||||||
|
) {
|
||||||
|
const fields = (e.details as { fields?: { field: string; message: string }[] }).fields;
|
||||||
|
for (const f of fields ?? []) out[f.field] = f.message;
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
function askSave() {
|
||||||
|
saveError = null;
|
||||||
|
success = null;
|
||||||
|
fieldErrors = {};
|
||||||
|
confirmOpen = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function save() {
|
||||||
|
confirmOpen = false;
|
||||||
|
saving = true;
|
||||||
|
saveError = null;
|
||||||
|
success = null;
|
||||||
|
fieldErrors = {};
|
||||||
|
try {
|
||||||
|
if (tab === 'crawler' && crawler) {
|
||||||
|
// Sync the allowlist textarea back into the model.
|
||||||
|
crawler.download_allowlist = allowlistText
|
||||||
|
.split('\n')
|
||||||
|
.map((s) => s.trim())
|
||||||
|
.filter(Boolean);
|
||||||
|
const r = await updateCrawlerSettings(crawler);
|
||||||
|
crawler = r.editable;
|
||||||
|
crawlerEnv = r.env_only;
|
||||||
|
allowlistText = r.editable.download_allowlist.join('\n');
|
||||||
|
} else if (tab === 'analysis' && analysis) {
|
||||||
|
const r = await updateAnalysisSettings(analysis);
|
||||||
|
analysis = r.editable;
|
||||||
|
promptDefaults = r.prompt_defaults;
|
||||||
|
apiKeyConfigured = r.env_only.api_key_configured;
|
||||||
|
}
|
||||||
|
success = 'Saved and applied. The subsystem was restarted with the new settings.';
|
||||||
|
} catch (e) {
|
||||||
|
fieldErrors = parseFieldErrors(e);
|
||||||
|
saveError =
|
||||||
|
Object.keys(fieldErrors).length > 0
|
||||||
|
? 'Some fields need fixing.'
|
||||||
|
: e instanceof Error
|
||||||
|
? e.message
|
||||||
|
: 'Save failed.';
|
||||||
|
} finally {
|
||||||
|
saving = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function resetPrompt(which: 'system_prompt' | 'ocr_prompt' | 'grounding_prompt') {
|
||||||
|
if (analysis) analysis[which] = null;
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<h1>Settings</h1>
|
||||||
|
<p class="lead muted">
|
||||||
|
Edit crawler and analysis configuration. Saving applies immediately — the affected daemon is
|
||||||
|
gracefully restarted, so in-flight jobs are retried rather than lost. Host-level and secret
|
||||||
|
values stay environment-managed and are shown read-only.
|
||||||
|
</p>
|
||||||
|
|
||||||
|
<div class="tabs" role="tablist">
|
||||||
|
<button
|
||||||
|
role="tab"
|
||||||
|
aria-selected={tab === 'crawler'}
|
||||||
|
class:active={tab === 'crawler'}
|
||||||
|
onclick={() => (tab = 'crawler')}
|
||||||
|
data-testid="settings-tab-crawler">Crawler</button
|
||||||
|
>
|
||||||
|
<button
|
||||||
|
role="tab"
|
||||||
|
aria-selected={tab === 'analysis'}
|
||||||
|
class:active={tab === 'analysis'}
|
||||||
|
onclick={() => (tab = 'analysis')}
|
||||||
|
data-testid="settings-tab-analysis">Analysis</button
|
||||||
|
>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{#if loading}
|
||||||
|
<p class="muted">Loading…</p>
|
||||||
|
{:else if loadError}
|
||||||
|
<p class="error" role="alert">{loadError}</p>
|
||||||
|
{:else}
|
||||||
|
<div class="sticky-bar">
|
||||||
|
{#if tab === 'crawler' && crawler}
|
||||||
|
<label class="toggle">
|
||||||
|
<input
|
||||||
|
type="checkbox"
|
||||||
|
bind:checked={crawler.daemon_enabled}
|
||||||
|
data-testid="settings-crawler-enabled"
|
||||||
|
/>
|
||||||
|
<span>Crawler daemon {crawler.daemon_enabled ? 'enabled' : 'disabled'}</span>
|
||||||
|
</label>
|
||||||
|
{:else if tab === 'analysis' && analysis}
|
||||||
|
<label class="toggle">
|
||||||
|
<input
|
||||||
|
type="checkbox"
|
||||||
|
bind:checked={analysis.enabled}
|
||||||
|
data-testid="settings-analysis-enabled"
|
||||||
|
/>
|
||||||
|
<span>Analysis worker {analysis.enabled ? 'enabled' : 'disabled'}</span>
|
||||||
|
</label>
|
||||||
|
{/if}
|
||||||
|
<div class="spacer"></div>
|
||||||
|
<button class="primary" disabled={saving} onclick={askSave} data-testid="settings-save">
|
||||||
|
{saving ? 'Saving…' : 'Save & apply'}
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{#if success}<p class="notice" role="status">{success}</p>{/if}
|
||||||
|
{#if saveError}<p class="error" role="alert">{saveError}</p>{/if}
|
||||||
|
|
||||||
|
{#if tab === 'crawler' && crawler}
|
||||||
|
<div class="form">
|
||||||
|
<fieldset>
|
||||||
|
<legend>Schedule</legend>
|
||||||
|
<label class="field">
|
||||||
|
<span>Daily metadata pass (HH:MM)</span>
|
||||||
|
<input type="time" bind:value={crawler.daily_at} />
|
||||||
|
{#if fieldErrors.daily_at}<small class="fe">{fieldErrors.daily_at}</small>{/if}
|
||||||
|
</label>
|
||||||
|
<label class="field">
|
||||||
|
<span>Timezone (IANA)</span>
|
||||||
|
<input type="text" bind:value={crawler.tz} placeholder="UTC" />
|
||||||
|
{#if fieldErrors.tz}<small class="fe">{fieldErrors.tz}</small>{/if}
|
||||||
|
</label>
|
||||||
|
<label class="field">
|
||||||
|
<span>Idle timeout (seconds)</span>
|
||||||
|
<input type="number" min="0" bind:value={crawler.idle_timeout_secs} />
|
||||||
|
</label>
|
||||||
|
</fieldset>
|
||||||
|
|
||||||
|
<fieldset>
|
||||||
|
<legend>Source</legend>
|
||||||
|
<label class="field">
|
||||||
|
<span>Start URL</span>
|
||||||
|
<input
|
||||||
|
type="text"
|
||||||
|
bind:value={crawler.start_url}
|
||||||
|
placeholder="https://source.example/"
|
||||||
|
/>
|
||||||
|
{#if fieldErrors.start_url}<small class="fe">{fieldErrors.start_url}</small>{/if}
|
||||||
|
</label>
|
||||||
|
<label class="field">
|
||||||
|
<span>CDN host</span>
|
||||||
|
<input type="text" bind:value={crawler.cdn_host} placeholder="cdn.example" />
|
||||||
|
</label>
|
||||||
|
<label class="field">
|
||||||
|
<span>Cookie domain</span>
|
||||||
|
<input
|
||||||
|
type="text"
|
||||||
|
bind:value={crawler.cookie_domain}
|
||||||
|
placeholder="source.example"
|
||||||
|
/>
|
||||||
|
</label>
|
||||||
|
<label class="field">
|
||||||
|
<span>User-Agent</span>
|
||||||
|
<input type="text" bind:value={crawler.user_agent} />
|
||||||
|
</label>
|
||||||
|
</fieldset>
|
||||||
|
|
||||||
|
<fieldset>
|
||||||
|
<legend>Rate limiting</legend>
|
||||||
|
<label class="field">
|
||||||
|
<span>Request interval (ms)</span>
|
||||||
|
<input type="number" min="0" bind:value={crawler.rate_ms} />
|
||||||
|
</label>
|
||||||
|
<label class="field">
|
||||||
|
<span>CDN request interval (ms)</span>
|
||||||
|
<input type="number" min="0" bind:value={crawler.cdn_rate_ms} />
|
||||||
|
</label>
|
||||||
|
</fieldset>
|
||||||
|
|
||||||
|
<fieldset>
|
||||||
|
<legend>Workers & retention</legend>
|
||||||
|
<label class="field">
|
||||||
|
<span>Chapter workers</span>
|
||||||
|
<input type="number" min="1" bind:value={crawler.chapter_workers} />
|
||||||
|
{#if fieldErrors.chapter_workers}<small class="fe"
|
||||||
|
>{fieldErrors.chapter_workers}</small
|
||||||
|
>{/if}
|
||||||
|
</label>
|
||||||
|
<label class="field">
|
||||||
|
<span>Job retention (days)</span>
|
||||||
|
<input type="number" min="0" bind:value={crawler.retention_days} />
|
||||||
|
</label>
|
||||||
|
<label class="field">
|
||||||
|
<span>Manga limit per pass (0 = unlimited)</span>
|
||||||
|
<input type="number" min="0" bind:value={crawler.manga_limit} />
|
||||||
|
</label>
|
||||||
|
</fieldset>
|
||||||
|
|
||||||
|
<fieldset>
|
||||||
|
<legend>Download safety</legend>
|
||||||
|
<label class="field-inline">
|
||||||
|
<input type="checkbox" bind:checked={crawler.allow_any_host} />
|
||||||
|
<span>Allow any host (bypass allowlist; still blocks private IPs)</span>
|
||||||
|
</label>
|
||||||
|
<label class="field">
|
||||||
|
<span>Download allowlist (one host per line)</span>
|
||||||
|
<textarea
|
||||||
|
rows="4"
|
||||||
|
bind:value={allowlistText}
|
||||||
|
disabled={crawler.allow_any_host}
|
||||||
|
></textarea>
|
||||||
|
</label>
|
||||||
|
<label class="field">
|
||||||
|
<span>Max image bytes</span>
|
||||||
|
<input type="number" min="0" bind:value={crawler.max_image_bytes} />
|
||||||
|
</label>
|
||||||
|
</fieldset>
|
||||||
|
|
||||||
|
<fieldset>
|
||||||
|
<legend>Reliability</legend>
|
||||||
|
<label class="field">
|
||||||
|
<span>Chapter job timeout (seconds)</span>
|
||||||
|
<input type="number" min="1" bind:value={crawler.job_timeout_secs} />
|
||||||
|
{#if fieldErrors.job_timeout_secs}<small class="fe"
|
||||||
|
>{fieldErrors.job_timeout_secs}</small
|
||||||
|
>{/if}
|
||||||
|
</label>
|
||||||
|
<label class="field">
|
||||||
|
<span>Metadata max consecutive failures</span>
|
||||||
|
<input
|
||||||
|
type="number"
|
||||||
|
min="0"
|
||||||
|
bind:value={crawler.metadata_max_consecutive_failures}
|
||||||
|
/>
|
||||||
|
</label>
|
||||||
|
<label class="field">
|
||||||
|
<span>Browser restart threshold</span>
|
||||||
|
<input type="number" min="0" bind:value={crawler.browser_restart_threshold} />
|
||||||
|
</label>
|
||||||
|
</fieldset>
|
||||||
|
|
||||||
|
{#if crawlerEnv}
|
||||||
|
<fieldset class="env">
|
||||||
|
<legend>Managed via environment (read-only)</legend>
|
||||||
|
<dl>
|
||||||
|
<dt>Browser mode</dt>
|
||||||
|
<dd>{crawlerEnv.browser_mode}</dd>
|
||||||
|
<dt>Browser args</dt>
|
||||||
|
<dd>{crawlerEnv.browser_args.join(' ') || '—'}</dd>
|
||||||
|
<dt>Proxy</dt>
|
||||||
|
<dd>{crawlerEnv.proxy ?? '—'}</dd>
|
||||||
|
<dt>TOR control URL</dt>
|
||||||
|
<dd>{crawlerEnv.tor_control_url ?? '—'}</dd>
|
||||||
|
<dt>TOR credentials</dt>
|
||||||
|
<dd>{crawlerEnv.tor_credentials_configured ? 'configured' : '—'}</dd>
|
||||||
|
<dt>Initial session</dt>
|
||||||
|
<dd>{crawlerEnv.session_configured ? 'configured' : '—'}</dd>
|
||||||
|
</dl>
|
||||||
|
</fieldset>
|
||||||
|
{/if}
|
||||||
|
</div>
|
||||||
|
{:else if tab === 'analysis' && analysis}
|
||||||
|
<div class="form">
|
||||||
|
<fieldset>
|
||||||
|
<legend>Endpoint & model</legend>
|
||||||
|
<label class="field">
|
||||||
|
<span>Vision endpoint URL</span>
|
||||||
|
<input type="text" bind:value={analysis.endpoint} />
|
||||||
|
{#if fieldErrors.endpoint}<small class="fe">{fieldErrors.endpoint}</small>{/if}
|
||||||
|
</label>
|
||||||
|
<label class="field">
|
||||||
|
<span>Model</span>
|
||||||
|
<input type="text" bind:value={analysis.model} />
|
||||||
|
{#if fieldErrors.model}<small class="fe">{fieldErrors.model}</small>{/if}
|
||||||
|
</label>
|
||||||
|
<p class="env-note muted">
|
||||||
|
API key: {apiKeyConfigured
|
||||||
|
? 'configured via environment'
|
||||||
|
: 'not set'} (managed via environment).
|
||||||
|
</p>
|
||||||
|
</fieldset>
|
||||||
|
|
||||||
|
<fieldset>
|
||||||
|
<legend>Workers & timeouts</legend>
|
||||||
|
<label class="field">
|
||||||
|
<span>Workers</span>
|
||||||
|
<input type="number" min="1" bind:value={analysis.workers} />
|
||||||
|
{#if fieldErrors.workers}<small class="fe">{fieldErrors.workers}</small>{/if}
|
||||||
|
</label>
|
||||||
|
<label class="field">
|
||||||
|
<span>Request timeout (seconds)</span>
|
||||||
|
<input type="number" min="1" bind:value={analysis.request_timeout_secs} />
|
||||||
|
</label>
|
||||||
|
<label class="field">
|
||||||
|
<span>Job timeout (seconds)</span>
|
||||||
|
<input type="number" min="1" bind:value={analysis.job_timeout_secs} />
|
||||||
|
</label>
|
||||||
|
</fieldset>
|
||||||
|
|
||||||
|
<fieldset>
|
||||||
|
<legend>Sampling & output</legend>
|
||||||
|
<label class="field">
|
||||||
|
<span>Temperature</span>
|
||||||
|
<input type="number" min="0" step="0.05" bind:value={analysis.temperature} />
|
||||||
|
{#if fieldErrors.temperature}<small class="fe"
|
||||||
|
>{fieldErrors.temperature}</small
|
||||||
|
>{/if}
|
||||||
|
</label>
|
||||||
|
<label class="field">
|
||||||
|
<span>Frequency penalty</span>
|
||||||
|
<input type="number" step="0.05" bind:value={analysis.frequency_penalty} />
|
||||||
|
</label>
|
||||||
|
<label class="field">
|
||||||
|
<span>Max output tokens</span>
|
||||||
|
<input type="number" min="1" bind:value={analysis.max_tokens} />
|
||||||
|
</label>
|
||||||
|
<label class="field">
|
||||||
|
<span>Response format</span>
|
||||||
|
<select bind:value={analysis.response_format}>
|
||||||
|
<option value="json_schema">json_schema</option>
|
||||||
|
<option value="json_object">json_object</option>
|
||||||
|
<option value="none">none</option>
|
||||||
|
</select>
|
||||||
|
</label>
|
||||||
|
</fieldset>
|
||||||
|
|
||||||
|
<fieldset>
|
||||||
|
<legend>Image slicing</legend>
|
||||||
|
<label class="field">
|
||||||
|
<span>Max pixels per slice</span>
|
||||||
|
<input type="number" min="1" bind:value={analysis.max_pixels} />
|
||||||
|
</label>
|
||||||
|
<label class="field">
|
||||||
|
<span>Min slice height (px)</span>
|
||||||
|
<input type="number" min="1" bind:value={analysis.min_slice_height} />
|
||||||
|
</label>
|
||||||
|
<label class="field">
|
||||||
|
<span>Slice overlap (0–0.9)</span>
|
||||||
|
<input
|
||||||
|
type="number"
|
||||||
|
min="0"
|
||||||
|
max="0.9"
|
||||||
|
step="0.01"
|
||||||
|
bind:value={analysis.slice_overlap}
|
||||||
|
/>
|
||||||
|
{#if fieldErrors.slice_overlap}<small class="fe"
|
||||||
|
>{fieldErrors.slice_overlap}</small
|
||||||
|
>{/if}
|
||||||
|
</label>
|
||||||
|
<label class="field">
|
||||||
|
<span>Tall aspect threshold (≥1.0)</span>
|
||||||
|
<input
|
||||||
|
type="number"
|
||||||
|
min="1"
|
||||||
|
step="0.1"
|
||||||
|
bind:value={analysis.tall_aspect_threshold}
|
||||||
|
/>
|
||||||
|
{#if fieldErrors.tall_aspect_threshold}<small class="fe"
|
||||||
|
>{fieldErrors.tall_aspect_threshold}</small
|
||||||
|
>{/if}
|
||||||
|
</label>
|
||||||
|
<label class="field">
|
||||||
|
<span>Max slices</span>
|
||||||
|
<input type="number" min="1" bind:value={analysis.max_slices} />
|
||||||
|
</label>
|
||||||
|
<label class="field">
|
||||||
|
<span>Max image bytes</span>
|
||||||
|
<input type="number" min="0" bind:value={analysis.max_image_bytes} />
|
||||||
|
</label>
|
||||||
|
</fieldset>
|
||||||
|
|
||||||
|
<fieldset>
|
||||||
|
<legend>Prompts</legend>
|
||||||
|
{#each [['system_prompt', 'System prompt'], ['ocr_prompt', 'OCR prompt (slices)'], ['grounding_prompt', 'Grounding prompt']] as [key, label] (key)}
|
||||||
|
{@const k = key as 'system_prompt' | 'ocr_prompt' | 'grounding_prompt'}
|
||||||
|
<div class="prompt">
|
||||||
|
<div class="prompt-head">
|
||||||
|
<span>{label}</span>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
class="link"
|
||||||
|
disabled={analysis[k] == null}
|
||||||
|
onclick={() => resetPrompt(k)}>Reset to default</button
|
||||||
|
>
|
||||||
|
</div>
|
||||||
|
<textarea
|
||||||
|
rows="6"
|
||||||
|
class="mono"
|
||||||
|
value={analysis[k] ?? ''}
|
||||||
|
placeholder={promptDefaults?.[k] ?? ''}
|
||||||
|
oninput={(e) => {
|
||||||
|
const v = (e.currentTarget as HTMLTextAreaElement).value;
|
||||||
|
if (analysis) analysis[k] = v.trim() === '' ? null : v;
|
||||||
|
}}
|
||||||
|
></textarea>
|
||||||
|
<small class="muted">
|
||||||
|
{analysis[k] == null
|
||||||
|
? 'Using compiled default (shown as placeholder).'
|
||||||
|
: `${analysis[k]?.length ?? 0} chars (override).`}
|
||||||
|
</small>
|
||||||
|
</div>
|
||||||
|
{/each}
|
||||||
|
</fieldset>
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
|
{/if}
|
||||||
|
|
||||||
|
<Modal
|
||||||
|
open={confirmOpen}
|
||||||
|
title="Apply settings?"
|
||||||
|
onClose={() => (confirmOpen = false)}
|
||||||
|
size="sm"
|
||||||
|
closeOnBackdrop
|
||||||
|
testid="settings-confirm"
|
||||||
|
>
|
||||||
|
<p>
|
||||||
|
Saving restarts the {tab} subsystem with the new settings. In-flight jobs are leased and
|
||||||
|
will be retried, but any work in progress is interrupted briefly.
|
||||||
|
</p>
|
||||||
|
{#snippet footer()}
|
||||||
|
<button type="button" onclick={() => (confirmOpen = false)}>Cancel</button>
|
||||||
|
<button type="button" class="primary" onclick={save} data-testid="settings-confirm-apply">
|
||||||
|
Save & apply
|
||||||
|
</button>
|
||||||
|
{/snippet}
|
||||||
|
</Modal>
|
||||||
|
|
||||||
|
<style>
|
||||||
|
h1 {
|
||||||
|
margin: 0 0 var(--space-2) 0;
|
||||||
|
}
|
||||||
|
.lead {
|
||||||
|
margin: 0 0 var(--space-4) 0;
|
||||||
|
font-size: var(--font-sm);
|
||||||
|
max-width: 60ch;
|
||||||
|
}
|
||||||
|
.muted {
|
||||||
|
color: var(--text-muted);
|
||||||
|
}
|
||||||
|
.tabs {
|
||||||
|
display: flex;
|
||||||
|
gap: var(--space-2);
|
||||||
|
margin-bottom: var(--space-4);
|
||||||
|
border-bottom: 1px solid var(--border);
|
||||||
|
}
|
||||||
|
.tabs button {
|
||||||
|
padding: var(--space-2) var(--space-3);
|
||||||
|
background: none;
|
||||||
|
border: none;
|
||||||
|
border-bottom: 2px solid transparent;
|
||||||
|
color: var(--text-muted);
|
||||||
|
cursor: pointer;
|
||||||
|
font-size: var(--font-sm);
|
||||||
|
}
|
||||||
|
.tabs button.active {
|
||||||
|
color: var(--text);
|
||||||
|
border-bottom-color: var(--primary);
|
||||||
|
font-weight: var(--weight-semibold);
|
||||||
|
}
|
||||||
|
.sticky-bar {
|
||||||
|
position: sticky;
|
||||||
|
top: calc(var(--app-header-h) + var(--space-2));
|
||||||
|
z-index: 1;
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: var(--space-3);
|
||||||
|
padding: var(--space-3);
|
||||||
|
margin-bottom: var(--space-3);
|
||||||
|
background: var(--surface-elevated);
|
||||||
|
border: 1px solid var(--border);
|
||||||
|
border-radius: var(--radius-md);
|
||||||
|
}
|
||||||
|
.sticky-bar .spacer {
|
||||||
|
flex: 1;
|
||||||
|
}
|
||||||
|
.toggle {
|
||||||
|
display: inline-flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: var(--space-2);
|
||||||
|
font-size: var(--font-sm);
|
||||||
|
}
|
||||||
|
.form {
|
||||||
|
display: grid;
|
||||||
|
gap: var(--space-4);
|
||||||
|
}
|
||||||
|
fieldset {
|
||||||
|
border: 1px solid var(--border);
|
||||||
|
border-radius: var(--radius-md);
|
||||||
|
padding: var(--space-3);
|
||||||
|
display: grid;
|
||||||
|
gap: var(--space-3);
|
||||||
|
}
|
||||||
|
legend {
|
||||||
|
padding: 0 var(--space-2);
|
||||||
|
font-weight: var(--weight-semibold);
|
||||||
|
font-size: var(--font-sm);
|
||||||
|
}
|
||||||
|
.field {
|
||||||
|
display: grid;
|
||||||
|
gap: var(--space-1);
|
||||||
|
}
|
||||||
|
.field > span {
|
||||||
|
font-size: var(--font-sm);
|
||||||
|
color: var(--text-muted);
|
||||||
|
}
|
||||||
|
.field input,
|
||||||
|
.field select,
|
||||||
|
.field textarea {
|
||||||
|
padding: var(--space-2) var(--space-3);
|
||||||
|
border: 1px solid var(--border);
|
||||||
|
border-radius: var(--radius-md);
|
||||||
|
background: var(--surface);
|
||||||
|
color: var(--text);
|
||||||
|
width: 100%;
|
||||||
|
box-sizing: border-box;
|
||||||
|
}
|
||||||
|
.field-inline {
|
||||||
|
display: inline-flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: var(--space-2);
|
||||||
|
font-size: var(--font-sm);
|
||||||
|
}
|
||||||
|
.fe {
|
||||||
|
color: var(--danger, #dc2626);
|
||||||
|
font-size: var(--font-xs);
|
||||||
|
}
|
||||||
|
.mono {
|
||||||
|
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
|
||||||
|
font-size: var(--font-xs);
|
||||||
|
width: 100%;
|
||||||
|
box-sizing: border-box;
|
||||||
|
padding: var(--space-2);
|
||||||
|
border: 1px solid var(--border);
|
||||||
|
border-radius: var(--radius-md);
|
||||||
|
background: var(--surface);
|
||||||
|
color: var(--text);
|
||||||
|
}
|
||||||
|
.prompt {
|
||||||
|
display: grid;
|
||||||
|
gap: var(--space-1);
|
||||||
|
}
|
||||||
|
.prompt-head {
|
||||||
|
display: flex;
|
||||||
|
justify-content: space-between;
|
||||||
|
align-items: center;
|
||||||
|
font-size: var(--font-sm);
|
||||||
|
}
|
||||||
|
button.link {
|
||||||
|
background: none;
|
||||||
|
border: none;
|
||||||
|
color: var(--primary);
|
||||||
|
cursor: pointer;
|
||||||
|
font-size: var(--font-xs);
|
||||||
|
padding: 0;
|
||||||
|
}
|
||||||
|
button.link:disabled {
|
||||||
|
color: var(--text-muted);
|
||||||
|
cursor: not-allowed;
|
||||||
|
}
|
||||||
|
.env dl {
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: max-content 1fr;
|
||||||
|
gap: var(--space-1) var(--space-3);
|
||||||
|
margin: 0;
|
||||||
|
font-size: var(--font-sm);
|
||||||
|
}
|
||||||
|
.env dt {
|
||||||
|
color: var(--text-muted);
|
||||||
|
}
|
||||||
|
.env dd {
|
||||||
|
margin: 0;
|
||||||
|
word-break: break-all;
|
||||||
|
}
|
||||||
|
.env-note {
|
||||||
|
font-size: var(--font-xs);
|
||||||
|
margin: 0;
|
||||||
|
}
|
||||||
|
.error {
|
||||||
|
color: var(--danger, #dc2626);
|
||||||
|
padding: var(--space-2) var(--space-3);
|
||||||
|
border: 1px solid var(--danger, #dc2626);
|
||||||
|
border-radius: var(--radius-md);
|
||||||
|
margin-bottom: var(--space-3);
|
||||||
|
}
|
||||||
|
.notice {
|
||||||
|
color: var(--success, #16a34a);
|
||||||
|
padding: var(--space-2) var(--space-3);
|
||||||
|
border: 1px solid var(--success, #16a34a);
|
||||||
|
border-radius: var(--radius-md);
|
||||||
|
margin-bottom: var(--space-3);
|
||||||
|
}
|
||||||
|
button.primary {
|
||||||
|
padding: var(--space-2) var(--space-3);
|
||||||
|
border-radius: var(--radius-md);
|
||||||
|
border: 1px solid var(--primary);
|
||||||
|
background: var(--primary);
|
||||||
|
color: white;
|
||||||
|
cursor: pointer;
|
||||||
|
}
|
||||||
|
button.primary:disabled {
|
||||||
|
opacity: 0.5;
|
||||||
|
cursor: not-allowed;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
Reference in New Issue
Block a user