feat(admin): runtime-editable crawler & analysis config in the dashboard
Some checks failed
deploy / test-backend (push) Waiting to run
deploy / build-and-push (push) Has been cancelled
deploy / deploy (push) Has been cancelled
deploy / test-frontend (push) Has been cancelled

Move crawler and analysis configuration out of boot-only env vars and into
a DB-backed, admin-editable surface applied live without a restart.

- New `app_settings` table (migration 0026) holds one JSONB row per group;
  env vars seed it on first boot, then the DB is the source of truth.
- `settings.rs` adds serializable Crawler/Analysis DTOs with env-base +
  DB-overlay conversion and field-level validation; env-only/secret fields
  (browser, proxy, TOR, vision API key, PHPSESSID) are never persisted.
- `GET/PUT /api/v1/admin/settings/{crawler,analysis}` (RequireAdmin,
  cookie-only, CSRF-guarded): GET returns the editable DTO + a read-only
  env-managed view + prompt defaults; PUT validates (422 + per-field
  details), persists + audits in one tx, then gracefully respawns the
  affected daemon via a Supervisor.
- AppState swaps the crawler control/resync handles and analysis enable
  gate behind shared runtime cells so reloads take effect with no restart;
  a boot-time spawn failure is logged rather than aborting startup.
- Make the three vision prompts and sampling temperature configurable
  (defaults preserved); cookie_domain is admin-editable too.
- Frontend: tabbed /admin/settings page with grouped fieldsets, prompt
  reset-to-default, env-managed read-only panel, save-&-apply confirm, and
  per-field validation; typed client + ApiError.details.
- Bump version 0.79.1 -> 0.80.0.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
MechaCat02
2026-06-14 13:32:20 +02:00
parent d3b827421f
commit 2b7a11b480
30 changed files with 2800 additions and 165 deletions

View File

@@ -188,7 +188,7 @@ pub struct AnalyzePageResponse {
/// Reject the request with 503 when the analysis worker is disabled, so
/// an operator doesn't pile up jobs that nothing will ever drain.
fn ensure_enabled(state: &AppState) -> AppResult<()> {
if state.analysis_enabled {
if state.analysis_enabled() {
Ok(())
} else {
Err(AppError::ServiceUnavailable(

View File

@@ -37,13 +37,14 @@ pub(super) fn default_limit() -> i64 {
}
/// Shared 503 helper: the daemon-only control + observability endpoints
/// gate on `AppState.crawler == Some(_)`. Returns the same code and body
/// across every caller so the frontend can rely on `service_unavailable`
/// rather than each handler returning its own variant.
/// gate on a running crawler daemon. Returns the same code and body across
/// every caller so the frontend can rely on `service_unavailable` rather
/// than each handler returning its own variant. Yields an owned `Arc` (the
/// control handle is swapped on a config reload).
pub(super) fn require_crawler(
state: &AppState,
) -> Result<&std::sync::Arc<CrawlerControl>, AppError> {
state.crawler.as_ref().ok_or_else(|| {
) -> Result<std::sync::Arc<CrawlerControl>, AppError> {
state.crawler().ok_or_else(|| {
AppError::ServiceUnavailable("crawler daemon is disabled".into())
})
}

View File

@@ -140,7 +140,7 @@ async fn compose_status(
dead,
};
Ok(match state.crawler.as_ref() {
Ok(match state.crawler().as_ref() {
None => CrawlerStatusResponse {
daemon: "disabled",
phase: None,
@@ -200,7 +200,7 @@ async fn stream_status(
) -> Sse<impl Stream<Item = Result<Event, Infallible>>> {
// Subscribe before the first emit so no change between the initial
// snapshot and the first await is lost.
let rx = state.crawler.as_ref().map(|c| c.status.subscribe());
let rx = state.crawler().as_ref().map(|c| c.status.subscribe());
let memo = QueueCountsMemo::new();
let stream = futures_util::stream::unfold(

View File

@@ -8,6 +8,7 @@ pub mod analysis;
pub mod crawler;
pub mod mangas;
pub mod resync;
pub mod settings;
pub mod system;
pub mod users;
@@ -23,4 +24,5 @@ pub fn routes() -> Router<AppState> {
.merge(system::routes())
.merge(crawler::routes())
.merge(analysis::routes())
.merge(settings::routes())
}

View File

@@ -58,8 +58,7 @@ async fn resync_manga(
return Err(AppError::NotFound);
}
let resync = state
.resync
.as_ref()
.resync()
.ok_or_else(|| AppError::ServiceUnavailable(
"crawler daemon is disabled; force resync unavailable".into(),
))?;
@@ -96,8 +95,7 @@ async fn resync_chapter(
Path(chapter_id): Path<Uuid>,
) -> AppResult<Json<ChapterResyncResponse>> {
let resync = state
.resync
.as_ref()
.resync()
.ok_or_else(|| AppError::ServiceUnavailable(
"crawler daemon is disabled; force resync unavailable".into(),
))?;

View File

@@ -0,0 +1,239 @@
//! Admin endpoints for runtime-editable crawler / analysis configuration.
//!
//! `GET` returns the current editable settings DTO plus a read-only view of
//! the env-managed (host/infra + secret) fields, so the dashboard can show
//! the full picture while only letting the operator edit the safe knobs.
//!
//! `PUT` validates the incoming DTO against the env-derived base (re-parsing
//! timezone/time, rebuilding the download allowlist), persists a normalized
//! DTO and an audit row in one transaction, then asks the [`DaemonReloader`]
//! to gracefully respawn the affected daemon with the new config — so the
//! change takes effect without a restart. Validation failures return 422 with
//! per-field details (the established `validation_failed` envelope).
//!
//! `PUT` is a **full replace**, not a patch: the body is deserialized with
//! field-level defaults, so any omitted field is reset to its compiled
//! default rather than retaining the stored value. The dashboard always
//! sends the complete DTO (it round-trips the one it loaded); programmatic
//! callers must do the same.
use axum::extract::State;
use axum::routing::get;
use axum::{Json, Router};
use serde::Serialize;
use crate::app::AppState;
use crate::auth::extractor::RequireAdmin;
use crate::config::CrawlerConfig;
use crate::crawler::browser::BrowserMode;
use crate::error::{AppError, AppResult};
use crate::repo;
use crate::settings::{
AnalysisSettings, CrawlerSettings, FieldErrors, PromptDefaults, KEY_ANALYSIS, KEY_CRAWLER,
};
pub fn routes() -> Router<AppState> {
Router::new()
.route(
"/admin/settings/crawler",
get(get_crawler).put(put_crawler),
)
.route(
"/admin/settings/analysis",
get(get_analysis).put(put_analysis),
)
}
// ---------------------------------------------------------------------------
// Response shapes
// ---------------------------------------------------------------------------
/// Read-only mirror of the crawler fields managed via environment (host/infra
/// + secrets), surfaced so the admin sees what's in effect without editing it.
#[derive(Debug, Serialize)]
struct CrawlerEnvOnly {
browser_mode: &'static str,
browser_args: Vec<String>,
proxy: Option<String>,
tor_control_url: Option<String>,
/// Whether a TOR control credential (password or cookie file) is set.
tor_credentials_configured: bool,
/// Whether an initial `CRAWLER_PHPSESSID` is configured (the live session
/// is managed separately via the crawler dashboard).
session_configured: bool,
}
impl CrawlerEnvOnly {
fn from_base(base: &CrawlerConfig) -> Self {
Self {
browser_mode: match base.browser.mode {
BrowserMode::Headed => "headed",
BrowserMode::Headless => "headless",
},
browser_args: base.browser.extra_args.clone(),
proxy: base.proxy.clone(),
tor_control_url: base.tor_control_url.clone(),
tor_credentials_configured: base.tor_control_password.is_some()
|| base.tor_control_cookie_path.is_some(),
session_configured: base.phpsessid.is_some(),
}
}
}
#[derive(Debug, Serialize)]
struct CrawlerSettingsResponse {
editable: CrawlerSettings,
env_only: CrawlerEnvOnly,
}
/// Read-only mirror of analysis env-managed fields (just the secret).
#[derive(Debug, Serialize)]
struct AnalysisEnvOnly {
api_key_configured: bool,
}
#[derive(Debug, Serialize)]
struct AnalysisSettingsResponse {
editable: AnalysisSettings,
env_only: AnalysisEnvOnly,
/// The compiled prompt defaults so the UI can show placeholders and
/// implement per-prompt "reset to default".
prompt_defaults: PromptDefaults,
}
// ---------------------------------------------------------------------------
// Handlers
// ---------------------------------------------------------------------------
fn validation_error(errs: FieldErrors) -> AppError {
AppError::ValidationFailed {
message: "invalid settings".to_string(),
details: serde_json::json!({ "fields": errs.errors }),
}
}
/// Current effective crawler settings: the stored DTO when present, otherwise
/// derived from the env base (the row is seeded at boot, so the stored branch
/// is the norm).
async fn load_crawler_dto(state: &AppState) -> AppResult<CrawlerSettings> {
Ok(match repo::app_settings::get(&state.db, KEY_CRAWLER).await? {
Some(v) => serde_json::from_value(v)
.unwrap_or_else(|_| CrawlerSettings::from_config(&state.crawler_base)),
None => CrawlerSettings::from_config(&state.crawler_base),
})
}
async fn load_analysis_dto(state: &AppState) -> AppResult<AnalysisSettings> {
Ok(match repo::app_settings::get(&state.db, KEY_ANALYSIS).await? {
Some(v) => serde_json::from_value(v)
.unwrap_or_else(|_| AnalysisSettings::from_config(&state.analysis_base)),
None => AnalysisSettings::from_config(&state.analysis_base),
})
}
async fn get_crawler(
State(state): State<AppState>,
_admin: RequireAdmin,
) -> AppResult<Json<CrawlerSettingsResponse>> {
let editable = load_crawler_dto(&state).await?;
Ok(Json(CrawlerSettingsResponse {
editable,
env_only: CrawlerEnvOnly::from_base(&state.crawler_base),
}))
}
async fn put_crawler(
State(state): State<AppState>,
admin: RequireAdmin,
Json(incoming): Json<CrawlerSettings>,
) -> AppResult<Json<CrawlerSettingsResponse>> {
// Validate by converting against the env base; this also rebuilds the
// download allowlist and re-parses tz / time.
let cfg = incoming
.to_config(&state.crawler_base)
.map_err(validation_error)?;
// Persist a normalized DTO (so the stored/returned allowlist reflects the
// effective hosts, prompts are canonicalized, etc.).
let normalized = CrawlerSettings::from_config(&cfg);
let value = serde_json::to_value(&normalized).map_err(|e| AppError::Other(e.into()))?;
let mut tx = state.db.begin().await?;
repo::app_settings::upsert(&mut *tx, KEY_CRAWLER, &value).await?;
repo::admin_audit::insert(
&mut *tx,
admin.0.id,
"update_crawler_settings",
"settings",
None,
value.clone(),
)
.await?;
tx.commit().await?;
// Apply live (graceful respawn) when a reloader is wired up.
if let Some(reloader) = &state.reloader {
reloader
.reload_crawler(cfg)
.await
.map_err(AppError::Other)?;
}
Ok(Json(CrawlerSettingsResponse {
editable: normalized,
env_only: CrawlerEnvOnly::from_base(&state.crawler_base),
}))
}
async fn get_analysis(
State(state): State<AppState>,
_admin: RequireAdmin,
) -> AppResult<Json<AnalysisSettingsResponse>> {
let editable = load_analysis_dto(&state).await?;
Ok(Json(AnalysisSettingsResponse {
editable,
env_only: AnalysisEnvOnly {
api_key_configured: state.analysis_base.api_key.is_some(),
},
prompt_defaults: PromptDefaults::get(),
}))
}
async fn put_analysis(
State(state): State<AppState>,
admin: RequireAdmin,
Json(incoming): Json<AnalysisSettings>,
) -> AppResult<Json<AnalysisSettingsResponse>> {
let cfg = incoming
.to_config(&state.analysis_base)
.map_err(validation_error)?;
let normalized = AnalysisSettings::from_config(&cfg);
let value = serde_json::to_value(&normalized).map_err(|e| AppError::Other(e.into()))?;
let mut tx = state.db.begin().await?;
repo::app_settings::upsert(&mut *tx, KEY_ANALYSIS, &value).await?;
repo::admin_audit::insert(
&mut *tx,
admin.0.id,
"update_analysis_settings",
"settings",
None,
value.clone(),
)
.await?;
tx.commit().await?;
if let Some(reloader) = &state.reloader {
reloader
.reload_analysis(cfg)
.await
.map_err(AppError::Other)?;
}
Ok(Json(AnalysisSettingsResponse {
editable: normalized,
env_only: AnalysisEnvOnly {
api_key_configured: state.analysis_base.api_key.is_some(),
},
prompt_defaults: PromptDefaults::get(),
}))
}

View File

@@ -161,7 +161,7 @@ async fn create(
// rolled-back upload never leaves jobs pointing at nonexistent pages; a
// failed enqueue is logged but doesn't fail the upload (the admin
// re-enqueue endpoint can backfill).
if state.analysis_enabled {
if state.analysis_enabled() {
for page_id in page_ids {
if let Err(e) =
repo::page_analysis::enqueue_for_page(&state.db, page_id, false).await