Move crawler and analysis configuration out of boot-only env vars and into
a DB-backed, admin-editable surface applied live without a restart.
- New `app_settings` table (migration 0026) holds one JSONB row per group;
env vars seed it on first boot, then the DB is the source of truth.
- `settings.rs` adds serializable Crawler/Analysis DTOs with env-base +
DB-overlay conversion and field-level validation; env-only/secret fields
(browser, proxy, TOR, vision API key, PHPSESSID) are never persisted.
- `GET/PUT /api/v1/admin/settings/{crawler,analysis}` (RequireAdmin,
cookie-only, CSRF-guarded): GET returns the editable DTO + a read-only
env-managed view + prompt defaults; PUT validates (422 + per-field
details), persists + audits in one tx, then gracefully respawns the
affected daemon via a Supervisor.
- AppState swaps the crawler control/resync handles and analysis enable
gate behind shared runtime cells so reloads take effect with no restart;
a boot-time spawn failure is logged rather than aborting startup.
- Make the three vision prompts and sampling temperature configurable
(defaults preserved); cookie_domain is admin-editable too.
- Frontend: tabbed /admin/settings page with grouped fieldsets, prompt
reset-to-default, env-managed read-only panel, save-&-apply confirm, and
per-field validation; typed client + ApiError.details.
- Bump version 0.79.1 -> 0.80.0.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
240 lines
8.1 KiB
Rust
240 lines
8.1 KiB
Rust
//! Admin endpoints for runtime-editable crawler / analysis configuration.
|
|
//!
|
|
//! `GET` returns the current editable settings DTO plus a read-only view of
|
|
//! the env-managed (host/infra + secret) fields, so the dashboard can show
|
|
//! the full picture while only letting the operator edit the safe knobs.
|
|
//!
|
|
//! `PUT` validates the incoming DTO against the env-derived base (re-parsing
|
|
//! timezone/time, rebuilding the download allowlist), persists a normalized
|
|
//! DTO and an audit row in one transaction, then asks the [`DaemonReloader`]
|
|
//! to gracefully respawn the affected daemon with the new config — so the
|
|
//! change takes effect without a restart. Validation failures return 422 with
|
|
//! per-field details (the established `validation_failed` envelope).
|
|
//!
|
|
//! `PUT` is a **full replace**, not a patch: the body is deserialized with
|
|
//! field-level defaults, so any omitted field is reset to its compiled
|
|
//! default rather than retaining the stored value. The dashboard always
|
|
//! sends the complete DTO (it round-trips the one it loaded); programmatic
|
|
//! callers must do the same.
|
|
|
|
use axum::extract::State;
|
|
use axum::routing::get;
|
|
use axum::{Json, Router};
|
|
use serde::Serialize;
|
|
|
|
use crate::app::AppState;
|
|
use crate::auth::extractor::RequireAdmin;
|
|
use crate::config::CrawlerConfig;
|
|
use crate::crawler::browser::BrowserMode;
|
|
use crate::error::{AppError, AppResult};
|
|
use crate::repo;
|
|
use crate::settings::{
|
|
AnalysisSettings, CrawlerSettings, FieldErrors, PromptDefaults, KEY_ANALYSIS, KEY_CRAWLER,
|
|
};
|
|
|
|
pub fn routes() -> Router<AppState> {
|
|
Router::new()
|
|
.route(
|
|
"/admin/settings/crawler",
|
|
get(get_crawler).put(put_crawler),
|
|
)
|
|
.route(
|
|
"/admin/settings/analysis",
|
|
get(get_analysis).put(put_analysis),
|
|
)
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Response shapes
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/// Read-only mirror of the crawler fields managed via environment (host/infra
|
|
/// + secrets), surfaced so the admin sees what's in effect without editing it.
|
|
#[derive(Debug, Serialize)]
|
|
struct CrawlerEnvOnly {
|
|
browser_mode: &'static str,
|
|
browser_args: Vec<String>,
|
|
proxy: Option<String>,
|
|
tor_control_url: Option<String>,
|
|
/// Whether a TOR control credential (password or cookie file) is set.
|
|
tor_credentials_configured: bool,
|
|
/// Whether an initial `CRAWLER_PHPSESSID` is configured (the live session
|
|
/// is managed separately via the crawler dashboard).
|
|
session_configured: bool,
|
|
}
|
|
|
|
impl CrawlerEnvOnly {
|
|
fn from_base(base: &CrawlerConfig) -> Self {
|
|
Self {
|
|
browser_mode: match base.browser.mode {
|
|
BrowserMode::Headed => "headed",
|
|
BrowserMode::Headless => "headless",
|
|
},
|
|
browser_args: base.browser.extra_args.clone(),
|
|
proxy: base.proxy.clone(),
|
|
tor_control_url: base.tor_control_url.clone(),
|
|
tor_credentials_configured: base.tor_control_password.is_some()
|
|
|| base.tor_control_cookie_path.is_some(),
|
|
session_configured: base.phpsessid.is_some(),
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Serialize)]
|
|
struct CrawlerSettingsResponse {
|
|
editable: CrawlerSettings,
|
|
env_only: CrawlerEnvOnly,
|
|
}
|
|
|
|
/// Read-only mirror of analysis env-managed fields (just the secret).
|
|
#[derive(Debug, Serialize)]
|
|
struct AnalysisEnvOnly {
|
|
api_key_configured: bool,
|
|
}
|
|
|
|
#[derive(Debug, Serialize)]
|
|
struct AnalysisSettingsResponse {
|
|
editable: AnalysisSettings,
|
|
env_only: AnalysisEnvOnly,
|
|
/// The compiled prompt defaults so the UI can show placeholders and
|
|
/// implement per-prompt "reset to default".
|
|
prompt_defaults: PromptDefaults,
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Handlers
|
|
// ---------------------------------------------------------------------------
|
|
|
|
fn validation_error(errs: FieldErrors) -> AppError {
|
|
AppError::ValidationFailed {
|
|
message: "invalid settings".to_string(),
|
|
details: serde_json::json!({ "fields": errs.errors }),
|
|
}
|
|
}
|
|
|
|
/// Current effective crawler settings: the stored DTO when present, otherwise
|
|
/// derived from the env base (the row is seeded at boot, so the stored branch
|
|
/// is the norm).
|
|
async fn load_crawler_dto(state: &AppState) -> AppResult<CrawlerSettings> {
|
|
Ok(match repo::app_settings::get(&state.db, KEY_CRAWLER).await? {
|
|
Some(v) => serde_json::from_value(v)
|
|
.unwrap_or_else(|_| CrawlerSettings::from_config(&state.crawler_base)),
|
|
None => CrawlerSettings::from_config(&state.crawler_base),
|
|
})
|
|
}
|
|
|
|
async fn load_analysis_dto(state: &AppState) -> AppResult<AnalysisSettings> {
|
|
Ok(match repo::app_settings::get(&state.db, KEY_ANALYSIS).await? {
|
|
Some(v) => serde_json::from_value(v)
|
|
.unwrap_or_else(|_| AnalysisSettings::from_config(&state.analysis_base)),
|
|
None => AnalysisSettings::from_config(&state.analysis_base),
|
|
})
|
|
}
|
|
|
|
async fn get_crawler(
|
|
State(state): State<AppState>,
|
|
_admin: RequireAdmin,
|
|
) -> AppResult<Json<CrawlerSettingsResponse>> {
|
|
let editable = load_crawler_dto(&state).await?;
|
|
Ok(Json(CrawlerSettingsResponse {
|
|
editable,
|
|
env_only: CrawlerEnvOnly::from_base(&state.crawler_base),
|
|
}))
|
|
}
|
|
|
|
async fn put_crawler(
|
|
State(state): State<AppState>,
|
|
admin: RequireAdmin,
|
|
Json(incoming): Json<CrawlerSettings>,
|
|
) -> AppResult<Json<CrawlerSettingsResponse>> {
|
|
// Validate by converting against the env base; this also rebuilds the
|
|
// download allowlist and re-parses tz / time.
|
|
let cfg = incoming
|
|
.to_config(&state.crawler_base)
|
|
.map_err(validation_error)?;
|
|
// Persist a normalized DTO (so the stored/returned allowlist reflects the
|
|
// effective hosts, prompts are canonicalized, etc.).
|
|
let normalized = CrawlerSettings::from_config(&cfg);
|
|
let value = serde_json::to_value(&normalized).map_err(|e| AppError::Other(e.into()))?;
|
|
|
|
let mut tx = state.db.begin().await?;
|
|
repo::app_settings::upsert(&mut *tx, KEY_CRAWLER, &value).await?;
|
|
repo::admin_audit::insert(
|
|
&mut *tx,
|
|
admin.0.id,
|
|
"update_crawler_settings",
|
|
"settings",
|
|
None,
|
|
value.clone(),
|
|
)
|
|
.await?;
|
|
tx.commit().await?;
|
|
|
|
// Apply live (graceful respawn) when a reloader is wired up.
|
|
if let Some(reloader) = &state.reloader {
|
|
reloader
|
|
.reload_crawler(cfg)
|
|
.await
|
|
.map_err(AppError::Other)?;
|
|
}
|
|
|
|
Ok(Json(CrawlerSettingsResponse {
|
|
editable: normalized,
|
|
env_only: CrawlerEnvOnly::from_base(&state.crawler_base),
|
|
}))
|
|
}
|
|
|
|
async fn get_analysis(
|
|
State(state): State<AppState>,
|
|
_admin: RequireAdmin,
|
|
) -> AppResult<Json<AnalysisSettingsResponse>> {
|
|
let editable = load_analysis_dto(&state).await?;
|
|
Ok(Json(AnalysisSettingsResponse {
|
|
editable,
|
|
env_only: AnalysisEnvOnly {
|
|
api_key_configured: state.analysis_base.api_key.is_some(),
|
|
},
|
|
prompt_defaults: PromptDefaults::get(),
|
|
}))
|
|
}
|
|
|
|
async fn put_analysis(
|
|
State(state): State<AppState>,
|
|
admin: RequireAdmin,
|
|
Json(incoming): Json<AnalysisSettings>,
|
|
) -> AppResult<Json<AnalysisSettingsResponse>> {
|
|
let cfg = incoming
|
|
.to_config(&state.analysis_base)
|
|
.map_err(validation_error)?;
|
|
let normalized = AnalysisSettings::from_config(&cfg);
|
|
let value = serde_json::to_value(&normalized).map_err(|e| AppError::Other(e.into()))?;
|
|
|
|
let mut tx = state.db.begin().await?;
|
|
repo::app_settings::upsert(&mut *tx, KEY_ANALYSIS, &value).await?;
|
|
repo::admin_audit::insert(
|
|
&mut *tx,
|
|
admin.0.id,
|
|
"update_analysis_settings",
|
|
"settings",
|
|
None,
|
|
value.clone(),
|
|
)
|
|
.await?;
|
|
tx.commit().await?;
|
|
|
|
if let Some(reloader) = &state.reloader {
|
|
reloader
|
|
.reload_analysis(cfg)
|
|
.await
|
|
.map_err(AppError::Other)?;
|
|
}
|
|
|
|
Ok(Json(AnalysisSettingsResponse {
|
|
editable: normalized,
|
|
env_only: AnalysisEnvOnly {
|
|
api_key_configured: state.analysis_base.api_key.is_some(),
|
|
},
|
|
prompt_defaults: PromptDefaults::get(),
|
|
}))
|
|
}
|