Files
Mangalord/backend/src/api/admin/settings.rs
MechaCat02 2b7a11b480
Some checks failed
deploy / test-backend (push) Waiting to run
deploy / build-and-push (push) Has been cancelled
deploy / deploy (push) Has been cancelled
deploy / test-frontend (push) Has been cancelled
feat(admin): runtime-editable crawler & analysis config in the dashboard
Move crawler and analysis configuration out of boot-only env vars and into
a DB-backed, admin-editable surface applied live without a restart.

- New `app_settings` table (migration 0026) holds one JSONB row per group;
  env vars seed it on first boot, then the DB is the source of truth.
- `settings.rs` adds serializable Crawler/Analysis DTOs with env-base +
  DB-overlay conversion and field-level validation; env-only/secret fields
  (browser, proxy, TOR, vision API key, PHPSESSID) are never persisted.
- `GET/PUT /api/v1/admin/settings/{crawler,analysis}` (RequireAdmin,
  cookie-only, CSRF-guarded): GET returns the editable DTO + a read-only
  env-managed view + prompt defaults; PUT validates (422 + per-field
  details), persists + audits in one tx, then gracefully respawns the
  affected daemon via a Supervisor.
- AppState swaps the crawler control/resync handles and analysis enable
  gate behind shared runtime cells so reloads take effect with no restart;
  a boot-time spawn failure is logged rather than aborting startup.
- Make the three vision prompts and sampling temperature configurable
  (defaults preserved); cookie_domain is admin-editable too.
- Frontend: tabbed /admin/settings page with grouped fieldsets, prompt
  reset-to-default, env-managed read-only panel, save-&-apply confirm, and
  per-field validation; typed client + ApiError.details.
- Bump version 0.79.1 -> 0.80.0.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-14 13:32:20 +02:00

240 lines
8.1 KiB
Rust

//! Admin endpoints for runtime-editable crawler / analysis configuration.
//!
//! `GET` returns the current editable settings DTO plus a read-only view of
//! the env-managed (host/infra + secret) fields, so the dashboard can show
//! the full picture while only letting the operator edit the safe knobs.
//!
//! `PUT` validates the incoming DTO against the env-derived base (re-parsing
//! timezone/time, rebuilding the download allowlist), persists a normalized
//! DTO and an audit row in one transaction, then asks the [`DaemonReloader`]
//! to gracefully respawn the affected daemon with the new config — so the
//! change takes effect without a restart. Validation failures return 422 with
//! per-field details (the established `validation_failed` envelope).
//!
//! `PUT` is a **full replace**, not a patch: the body is deserialized with
//! field-level defaults, so any omitted field is reset to its compiled
//! default rather than retaining the stored value. The dashboard always
//! sends the complete DTO (it round-trips the one it loaded); programmatic
//! callers must do the same.
use axum::extract::State;
use axum::routing::get;
use axum::{Json, Router};
use serde::Serialize;
use crate::app::AppState;
use crate::auth::extractor::RequireAdmin;
use crate::config::CrawlerConfig;
use crate::crawler::browser::BrowserMode;
use crate::error::{AppError, AppResult};
use crate::repo;
use crate::settings::{
AnalysisSettings, CrawlerSettings, FieldErrors, PromptDefaults, KEY_ANALYSIS, KEY_CRAWLER,
};
pub fn routes() -> Router<AppState> {
Router::new()
.route(
"/admin/settings/crawler",
get(get_crawler).put(put_crawler),
)
.route(
"/admin/settings/analysis",
get(get_analysis).put(put_analysis),
)
}
// ---------------------------------------------------------------------------
// Response shapes
// ---------------------------------------------------------------------------
/// Read-only mirror of the crawler fields managed via environment (host/infra
/// + secrets), surfaced so the admin sees what's in effect without editing it.
#[derive(Debug, Serialize)]
struct CrawlerEnvOnly {
browser_mode: &'static str,
browser_args: Vec<String>,
proxy: Option<String>,
tor_control_url: Option<String>,
/// Whether a TOR control credential (password or cookie file) is set.
tor_credentials_configured: bool,
/// Whether an initial `CRAWLER_PHPSESSID` is configured (the live session
/// is managed separately via the crawler dashboard).
session_configured: bool,
}
impl CrawlerEnvOnly {
fn from_base(base: &CrawlerConfig) -> Self {
Self {
browser_mode: match base.browser.mode {
BrowserMode::Headed => "headed",
BrowserMode::Headless => "headless",
},
browser_args: base.browser.extra_args.clone(),
proxy: base.proxy.clone(),
tor_control_url: base.tor_control_url.clone(),
tor_credentials_configured: base.tor_control_password.is_some()
|| base.tor_control_cookie_path.is_some(),
session_configured: base.phpsessid.is_some(),
}
}
}
#[derive(Debug, Serialize)]
struct CrawlerSettingsResponse {
editable: CrawlerSettings,
env_only: CrawlerEnvOnly,
}
/// Read-only mirror of analysis env-managed fields (just the secret).
#[derive(Debug, Serialize)]
struct AnalysisEnvOnly {
api_key_configured: bool,
}
#[derive(Debug, Serialize)]
struct AnalysisSettingsResponse {
editable: AnalysisSettings,
env_only: AnalysisEnvOnly,
/// The compiled prompt defaults so the UI can show placeholders and
/// implement per-prompt "reset to default".
prompt_defaults: PromptDefaults,
}
// ---------------------------------------------------------------------------
// Handlers
// ---------------------------------------------------------------------------
fn validation_error(errs: FieldErrors) -> AppError {
AppError::ValidationFailed {
message: "invalid settings".to_string(),
details: serde_json::json!({ "fields": errs.errors }),
}
}
/// Current effective crawler settings: the stored DTO when present, otherwise
/// derived from the env base (the row is seeded at boot, so the stored branch
/// is the norm).
async fn load_crawler_dto(state: &AppState) -> AppResult<CrawlerSettings> {
Ok(match repo::app_settings::get(&state.db, KEY_CRAWLER).await? {
Some(v) => serde_json::from_value(v)
.unwrap_or_else(|_| CrawlerSettings::from_config(&state.crawler_base)),
None => CrawlerSettings::from_config(&state.crawler_base),
})
}
async fn load_analysis_dto(state: &AppState) -> AppResult<AnalysisSettings> {
Ok(match repo::app_settings::get(&state.db, KEY_ANALYSIS).await? {
Some(v) => serde_json::from_value(v)
.unwrap_or_else(|_| AnalysisSettings::from_config(&state.analysis_base)),
None => AnalysisSettings::from_config(&state.analysis_base),
})
}
async fn get_crawler(
State(state): State<AppState>,
_admin: RequireAdmin,
) -> AppResult<Json<CrawlerSettingsResponse>> {
let editable = load_crawler_dto(&state).await?;
Ok(Json(CrawlerSettingsResponse {
editable,
env_only: CrawlerEnvOnly::from_base(&state.crawler_base),
}))
}
async fn put_crawler(
State(state): State<AppState>,
admin: RequireAdmin,
Json(incoming): Json<CrawlerSettings>,
) -> AppResult<Json<CrawlerSettingsResponse>> {
// Validate by converting against the env base; this also rebuilds the
// download allowlist and re-parses tz / time.
let cfg = incoming
.to_config(&state.crawler_base)
.map_err(validation_error)?;
// Persist a normalized DTO (so the stored/returned allowlist reflects the
// effective hosts, prompts are canonicalized, etc.).
let normalized = CrawlerSettings::from_config(&cfg);
let value = serde_json::to_value(&normalized).map_err(|e| AppError::Other(e.into()))?;
let mut tx = state.db.begin().await?;
repo::app_settings::upsert(&mut *tx, KEY_CRAWLER, &value).await?;
repo::admin_audit::insert(
&mut *tx,
admin.0.id,
"update_crawler_settings",
"settings",
None,
value.clone(),
)
.await?;
tx.commit().await?;
// Apply live (graceful respawn) when a reloader is wired up.
if let Some(reloader) = &state.reloader {
reloader
.reload_crawler(cfg)
.await
.map_err(AppError::Other)?;
}
Ok(Json(CrawlerSettingsResponse {
editable: normalized,
env_only: CrawlerEnvOnly::from_base(&state.crawler_base),
}))
}
async fn get_analysis(
State(state): State<AppState>,
_admin: RequireAdmin,
) -> AppResult<Json<AnalysisSettingsResponse>> {
let editable = load_analysis_dto(&state).await?;
Ok(Json(AnalysisSettingsResponse {
editable,
env_only: AnalysisEnvOnly {
api_key_configured: state.analysis_base.api_key.is_some(),
},
prompt_defaults: PromptDefaults::get(),
}))
}
async fn put_analysis(
State(state): State<AppState>,
admin: RequireAdmin,
Json(incoming): Json<AnalysisSettings>,
) -> AppResult<Json<AnalysisSettingsResponse>> {
let cfg = incoming
.to_config(&state.analysis_base)
.map_err(validation_error)?;
let normalized = AnalysisSettings::from_config(&cfg);
let value = serde_json::to_value(&normalized).map_err(|e| AppError::Other(e.into()))?;
let mut tx = state.db.begin().await?;
repo::app_settings::upsert(&mut *tx, KEY_ANALYSIS, &value).await?;
repo::admin_audit::insert(
&mut *tx,
admin.0.id,
"update_analysis_settings",
"settings",
None,
value.clone(),
)
.await?;
tx.commit().await?;
if let Some(reloader) = &state.reloader {
reloader
.reload_analysis(cfg)
.await
.map_err(AppError::Other)?;
}
Ok(Json(AnalysisSettingsResponse {
editable: normalized,
env_only: AnalysisEnvOnly {
api_key_configured: state.analysis_base.api_key.is_some(),
},
prompt_defaults: PromptDefaults::get(),
}))
}