fabi/Mangalord
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(admin): runtime-editable crawler & analysis config in the dashboard
Some checks failed
deploy / test-backend (push) Waiting to run
Details
deploy / build-and-push (push) Has been cancelled
Details
deploy / deploy (push) Has been cancelled
Details
deploy / test-frontend (push) Has been cancelled
Details

Browse Source 
Move crawler and analysis configuration out of boot-only env vars and into
a DB-backed, admin-editable surface applied live without a restart.

- New `app_settings` table (migration 0026) holds one JSONB row per group;
  env vars seed it on first boot, then the DB is the source of truth.
- `settings.rs` adds serializable Crawler/Analysis DTOs with env-base +
  DB-overlay conversion and field-level validation; env-only/secret fields
  (browser, proxy, TOR, vision API key, PHPSESSID) are never persisted.
- `GET/PUT /api/v1/admin/settings/{crawler,analysis}` (RequireAdmin,
  cookie-only, CSRF-guarded): GET returns the editable DTO + a read-only
  env-managed view + prompt defaults; PUT validates (422 + per-field
  details), persists + audits in one tx, then gracefully respawns the
  affected daemon via a Supervisor.
- AppState swaps the crawler control/resync handles and analysis enable
  gate behind shared runtime cells so reloads take effect with no restart;
  a boot-time spawn failure is logged rather than aborting startup.
- Make the three vision prompts and sampling temperature configurable
  (defaults preserved); cookie_domain is admin-editable too.
- Frontend: tabbed /admin/settings page with grouped fieldsets, prompt
  reset-to-default, env-managed read-only panel, save-&-apply confirm, and
  per-field validation; typed client + ApiError.details.
- Bump version 0.79.1 -> 0.80.0.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
MechaCat02
2026-06-14 13:32:20 +02:00
parent d3b827421f
commit 2b7a11b480
30 changed files with 2800 additions and 165 deletions
Download Patch File Download Diff File Expand all files Collapse all files

97
backend/src/analysis/vision.rs
View File

@@ -37,6 +37,13 @@ pub struct VisionClient {
max_tokens: u32,
response_format: ResponseFormat,
frequency_penalty: f64,
temperature: f64,
/// System prompt for the single-call (normal-aspect) path.
system_prompt: String,
/// Pass-A OCR-only prompt (tall-page slices).
ocr_prompt: String,
/// Pass-B grounding prompt (tags/scene/safety).
grounding_prompt: String,
slice: SliceParams,
}
@@ -60,6 +67,10 @@ impl VisionClient {
max_tokens: cfg.max_tokens,
response_format: cfg.response_format,
frequency_penalty: cfg.frequency_penalty,
temperature: cfg.temperature,
system_prompt: cfg.system_prompt.clone(),
ocr_prompt: cfg.ocr_prompt.clone(),
grounding_prompt: cfg.grounding_prompt.clone(),
slice: SliceParams {
max_pixels: cfg.max_pixels,
min_slice_height: cfg.min_slice_height,
@@ -83,6 +94,8 @@ impl VisionClient {
&url,
self.response_format,
self.frequency_penalty,
self.temperature,
&self.system_prompt,
);
return parse_chat_completion(&self.post_chat(body).await?);
};
@@ -97,6 +110,8 @@ impl VisionClient {
&data_url(&jpeg),
self.response_format,
self.frequency_penalty,
self.temperature,
&self.system_prompt,
);
parse_chat_completion(&self.post_chat(body).await?)
}
@@ -124,6 +139,8 @@ impl VisionClient {
self.max_tokens,
self.response_format,
self.frequency_penalty,
self.temperature,
&self.ocr_prompt,
&data_url(&jpeg),
);
let parsed = parse_chat_completion(&self.post_chat(body).await?)?;
@@ -148,6 +165,8 @@ impl VisionClient {
self.max_tokens,
self.response_format,
self.frequency_penalty,
self.temperature,
&self.grounding_prompt,
&data_url(&whole),
&ocr_text,
);
@@ -451,19 +470,23 @@ fn similar(a: &str, b: &str) -> bool {
/// Combined single-call body (OCR + tags + scene + safety). Public + this
/// exact signature so the existing tests keep working.
#[allow(clippy::too_many_arguments)]
pub fn build_request_body(
model: &str,
max_tokens: u32,
data_url: &str,
response_format: ResponseFormat,
frequency_penalty: f64,
temperature: f64,
system_prompt: &str,
) -> serde_json::Value {
build_body(
model,
max_tokens,
response_format,
frequency_penalty,
prompt::SYSTEM_PROMPT,
temperature,
system_prompt,
"page_analysis",
prompt::output_json_schema(),
data_url,
@@ -472,11 +495,14 @@ pub fn build_request_body(
}
/// Pass-A OCR-only body.
#[allow(clippy::too_many_arguments)]
fn build_ocr_body(
model: &str,
max_tokens: u32,
response_format: ResponseFormat,
frequency_penalty: f64,
temperature: f64,
ocr_prompt: &str,
data_url: &str,
) -> serde_json::Value {
build_body(
@@ -484,7 +510,8 @@ fn build_ocr_body(
max_tokens,
response_format,
frequency_penalty,
prompt::OCR_PROMPT,
temperature,
ocr_prompt,
"page_ocr",
prompt::ocr_json_schema(),
data_url,
@@ -493,11 +520,14 @@ fn build_ocr_body(
}
/// Pass-B grounding body — whole image + the merged OCR text as context.
#[allow(clippy::too_many_arguments)]
fn build_grounding_body(
model: &str,
max_tokens: u32,
response_format: ResponseFormat,
frequency_penalty: f64,
temperature: f64,
grounding_prompt: &str,
data_url: &str,
ocr_text: &str,
) -> serde_json::Value {
@@ -507,7 +537,8 @@ fn build_grounding_body(
max_tokens,
response_format,
frequency_penalty,
prompt::GROUNDING_PROMPT,
temperature,
grounding_prompt,
"page_grounding",
prompt::grounding_json_schema(),
data_url,
@@ -524,6 +555,7 @@ fn build_body(
max_tokens: u32,
response_format: ResponseFormat,
frequency_penalty: f64,
temperature: f64,
system_prompt: &str,
schema_name: &str,
schema: serde_json::Value,
@@ -537,7 +569,7 @@ fn build_body(
}
let mut body = json!({
"model": model,
"temperature": 0,
"temperature": temperature,
"max_tokens": max_tokens,
"messages": [
{ "role": "system", "content": system_prompt },
@@ -740,8 +772,15 @@ mod tests {
#[test]
fn combined_body_uses_full_schema_and_image() {
let body =
build_request_body("m", 100, "data:image/png;base64,AA", ResponseFormat::JsonSchema, 0.3);
let body = build_request_body(
"m",
100,
"data:image/png;base64,AA",
ResponseFormat::JsonSchema,
0.3,
0.0,
prompt::SYSTEM_PROMPT_DEFAULT,
);
assert_eq!(body["messages"][0]["role"], "system");
assert_eq!(
body["messages"][1]["content"][0]["image_url"]["url"],
@@ -752,9 +791,25 @@ mod tests {
assert!(req.as_array().unwrap().iter().any(|v| v == "safety_flag"));
}
#[test]
fn body_carries_the_configured_prompt_and_temperature() {
let body = build_request_body(
"m",
100,
"d",
ResponseFormat::None,
0.0,
0.7,
"CUSTOM PROMPT",
);
assert_eq!(body["messages"][0]["content"], "CUSTOM PROMPT");
assert_eq!(body["temperature"], 0.7);
}
#[test]
fn ocr_body_carries_only_the_ocr_schema_and_caps() {
let body = build_ocr_body("m", 100, ResponseFormat::JsonSchema, 0.3, "d");
let body =
build_ocr_body("m", 100, ResponseFormat::JsonSchema, 0.3, 0.0, prompt::OCR_PROMPT_DEFAULT, "d");
let schema = &body["response_format"]["json_schema"]["schema"];
assert_eq!(body["response_format"]["json_schema"]["name"], "page_ocr");
assert_eq!(schema["required"], json!(["ocr_results"]));
@@ -769,8 +824,16 @@ mod tests {
#[test]
fn grounding_body_includes_ocr_context_and_grounding_schema() {
let body =
build_grounding_body("m", 100, ResponseFormat::JsonSchema, 0.3, "d", "[speech] Hi");
let body = build_grounding_body(
"m",
100,
ResponseFormat::JsonSchema,
0.3,
0.0,
prompt::GROUNDING_PROMPT_DEFAULT,
"d",
"[speech] Hi",
);
assert_eq!(body["response_format"]["json_schema"]["name"], "page_grounding");
let parts = body["messages"][1]["content"].as_array().unwrap();
assert_eq!(parts.len(), 2, "image + text parts");
@@ -781,15 +844,25 @@ mod tests {
#[test]
fn response_format_none_omits_the_field() {
let body = build_request_body("m", 100, "d", ResponseFormat::None, 0.3);
let body = build_request_body(
"m",
100,
"d",
ResponseFormat::None,
0.3,
0.0,
prompt::SYSTEM_PROMPT_DEFAULT,
);
assert!(body.get("response_format").is_none());
}
#[test]
fn frequency_penalty_included_only_when_nonzero() {
let on = build_ocr_body("m", 100, ResponseFormat::None, 0.4, "d");
let on =
build_ocr_body("m", 100, ResponseFormat::None, 0.4, 0.0, prompt::OCR_PROMPT_DEFAULT, "d");
assert_eq!(on["frequency_penalty"], 0.4);
let off = build_ocr_body("m", 100, ResponseFormat::None, 0.0, "d");
let off =
build_ocr_body("m", 100, ResponseFormat::None, 0.0, 0.0, prompt::OCR_PROMPT_DEFAULT, "d");
assert!(off.get("frequency_penalty").is_none());
}