feat(admin): runtime-editable crawler & analysis config in the dashboard
Move crawler and analysis configuration out of boot-only env vars and into
a DB-backed, admin-editable surface applied live without a restart.
- New `app_settings` table (migration 0026) holds one JSONB row per group;
env vars seed it on first boot, then the DB is the source of truth.
- `settings.rs` adds serializable Crawler/Analysis DTOs with env-base +
DB-overlay conversion and field-level validation; env-only/secret fields
(browser, proxy, TOR, vision API key, PHPSESSID) are never persisted.
- `GET/PUT /api/v1/admin/settings/{crawler,analysis}` (RequireAdmin,
cookie-only, CSRF-guarded): GET returns the editable DTO + a read-only
env-managed view + prompt defaults; PUT validates (422 + per-field
details), persists + audits in one tx, then gracefully respawns the
affected daemon via a Supervisor.
- AppState swaps the crawler control/resync handles and analysis enable
gate behind shared runtime cells so reloads take effect with no restart;
a boot-time spawn failure is logged rather than aborting startup.
- Make the three vision prompts and sampling temperature configurable
(defaults preserved); cookie_domain is admin-editable too.
- Frontend: tabbed /admin/settings page with grouped fieldsets, prompt
reset-to-default, env-managed read-only panel, save-&-apply confirm, and
per-field validation; typed client + ApiError.details.
- Bump version 0.79.1 -> 0.80.0.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -188,7 +188,7 @@ pub struct AnalyzePageResponse {
|
||||
/// Reject the request with 503 when the analysis worker is disabled, so
|
||||
/// an operator doesn't pile up jobs that nothing will ever drain.
|
||||
fn ensure_enabled(state: &AppState) -> AppResult<()> {
|
||||
if state.analysis_enabled {
|
||||
if state.analysis_enabled() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(AppError::ServiceUnavailable(
|
||||
|
||||
@@ -37,13 +37,14 @@ pub(super) fn default_limit() -> i64 {
|
||||
}
|
||||
|
||||
/// Shared 503 helper: the daemon-only control + observability endpoints
|
||||
/// gate on `AppState.crawler == Some(_)`. Returns the same code and body
|
||||
/// across every caller so the frontend can rely on `service_unavailable`
|
||||
/// rather than each handler returning its own variant.
|
||||
/// gate on a running crawler daemon. Returns the same code and body across
|
||||
/// every caller so the frontend can rely on `service_unavailable` rather
|
||||
/// than each handler returning its own variant. Yields an owned `Arc` (the
|
||||
/// control handle is swapped on a config reload).
|
||||
pub(super) fn require_crawler(
|
||||
state: &AppState,
|
||||
) -> Result<&std::sync::Arc<CrawlerControl>, AppError> {
|
||||
state.crawler.as_ref().ok_or_else(|| {
|
||||
) -> Result<std::sync::Arc<CrawlerControl>, AppError> {
|
||||
state.crawler().ok_or_else(|| {
|
||||
AppError::ServiceUnavailable("crawler daemon is disabled".into())
|
||||
})
|
||||
}
|
||||
|
||||
@@ -140,7 +140,7 @@ async fn compose_status(
|
||||
dead,
|
||||
};
|
||||
|
||||
Ok(match state.crawler.as_ref() {
|
||||
Ok(match state.crawler().as_ref() {
|
||||
None => CrawlerStatusResponse {
|
||||
daemon: "disabled",
|
||||
phase: None,
|
||||
@@ -200,7 +200,7 @@ async fn stream_status(
|
||||
) -> Sse<impl Stream<Item = Result<Event, Infallible>>> {
|
||||
// Subscribe before the first emit so no change between the initial
|
||||
// snapshot and the first await is lost.
|
||||
let rx = state.crawler.as_ref().map(|c| c.status.subscribe());
|
||||
let rx = state.crawler().as_ref().map(|c| c.status.subscribe());
|
||||
let memo = QueueCountsMemo::new();
|
||||
|
||||
let stream = futures_util::stream::unfold(
|
||||
|
||||
@@ -8,6 +8,7 @@ pub mod analysis;
|
||||
pub mod crawler;
|
||||
pub mod mangas;
|
||||
pub mod resync;
|
||||
pub mod settings;
|
||||
pub mod system;
|
||||
pub mod users;
|
||||
|
||||
@@ -23,4 +24,5 @@ pub fn routes() -> Router<AppState> {
|
||||
.merge(system::routes())
|
||||
.merge(crawler::routes())
|
||||
.merge(analysis::routes())
|
||||
.merge(settings::routes())
|
||||
}
|
||||
|
||||
@@ -58,8 +58,7 @@ async fn resync_manga(
|
||||
return Err(AppError::NotFound);
|
||||
}
|
||||
let resync = state
|
||||
.resync
|
||||
.as_ref()
|
||||
.resync()
|
||||
.ok_or_else(|| AppError::ServiceUnavailable(
|
||||
"crawler daemon is disabled; force resync unavailable".into(),
|
||||
))?;
|
||||
@@ -96,8 +95,7 @@ async fn resync_chapter(
|
||||
Path(chapter_id): Path<Uuid>,
|
||||
) -> AppResult<Json<ChapterResyncResponse>> {
|
||||
let resync = state
|
||||
.resync
|
||||
.as_ref()
|
||||
.resync()
|
||||
.ok_or_else(|| AppError::ServiceUnavailable(
|
||||
"crawler daemon is disabled; force resync unavailable".into(),
|
||||
))?;
|
||||
|
||||
239
backend/src/api/admin/settings.rs
Normal file
239
backend/src/api/admin/settings.rs
Normal file
@@ -0,0 +1,239 @@
|
||||
//! Admin endpoints for runtime-editable crawler / analysis configuration.
|
||||
//!
|
||||
//! `GET` returns the current editable settings DTO plus a read-only view of
|
||||
//! the env-managed (host/infra + secret) fields, so the dashboard can show
|
||||
//! the full picture while only letting the operator edit the safe knobs.
|
||||
//!
|
||||
//! `PUT` validates the incoming DTO against the env-derived base (re-parsing
|
||||
//! timezone/time, rebuilding the download allowlist), persists a normalized
|
||||
//! DTO and an audit row in one transaction, then asks the [`DaemonReloader`]
|
||||
//! to gracefully respawn the affected daemon with the new config — so the
|
||||
//! change takes effect without a restart. Validation failures return 422 with
|
||||
//! per-field details (the established `validation_failed` envelope).
|
||||
//!
|
||||
//! `PUT` is a **full replace**, not a patch: the body is deserialized with
|
||||
//! field-level defaults, so any omitted field is reset to its compiled
|
||||
//! default rather than retaining the stored value. The dashboard always
|
||||
//! sends the complete DTO (it round-trips the one it loaded); programmatic
|
||||
//! callers must do the same.
|
||||
|
||||
use axum::extract::State;
|
||||
use axum::routing::get;
|
||||
use axum::{Json, Router};
|
||||
use serde::Serialize;
|
||||
|
||||
use crate::app::AppState;
|
||||
use crate::auth::extractor::RequireAdmin;
|
||||
use crate::config::CrawlerConfig;
|
||||
use crate::crawler::browser::BrowserMode;
|
||||
use crate::error::{AppError, AppResult};
|
||||
use crate::repo;
|
||||
use crate::settings::{
|
||||
AnalysisSettings, CrawlerSettings, FieldErrors, PromptDefaults, KEY_ANALYSIS, KEY_CRAWLER,
|
||||
};
|
||||
|
||||
pub fn routes() -> Router<AppState> {
|
||||
Router::new()
|
||||
.route(
|
||||
"/admin/settings/crawler",
|
||||
get(get_crawler).put(put_crawler),
|
||||
)
|
||||
.route(
|
||||
"/admin/settings/analysis",
|
||||
get(get_analysis).put(put_analysis),
|
||||
)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Response shapes
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Read-only mirror of the crawler fields managed via environment (host/infra
|
||||
/// + secrets), surfaced so the admin sees what's in effect without editing it.
|
||||
#[derive(Debug, Serialize)]
|
||||
struct CrawlerEnvOnly {
|
||||
browser_mode: &'static str,
|
||||
browser_args: Vec<String>,
|
||||
proxy: Option<String>,
|
||||
tor_control_url: Option<String>,
|
||||
/// Whether a TOR control credential (password or cookie file) is set.
|
||||
tor_credentials_configured: bool,
|
||||
/// Whether an initial `CRAWLER_PHPSESSID` is configured (the live session
|
||||
/// is managed separately via the crawler dashboard).
|
||||
session_configured: bool,
|
||||
}
|
||||
|
||||
impl CrawlerEnvOnly {
|
||||
fn from_base(base: &CrawlerConfig) -> Self {
|
||||
Self {
|
||||
browser_mode: match base.browser.mode {
|
||||
BrowserMode::Headed => "headed",
|
||||
BrowserMode::Headless => "headless",
|
||||
},
|
||||
browser_args: base.browser.extra_args.clone(),
|
||||
proxy: base.proxy.clone(),
|
||||
tor_control_url: base.tor_control_url.clone(),
|
||||
tor_credentials_configured: base.tor_control_password.is_some()
|
||||
|| base.tor_control_cookie_path.is_some(),
|
||||
session_configured: base.phpsessid.is_some(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct CrawlerSettingsResponse {
|
||||
editable: CrawlerSettings,
|
||||
env_only: CrawlerEnvOnly,
|
||||
}
|
||||
|
||||
/// Read-only mirror of analysis env-managed fields (just the secret).
|
||||
#[derive(Debug, Serialize)]
|
||||
struct AnalysisEnvOnly {
|
||||
api_key_configured: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct AnalysisSettingsResponse {
|
||||
editable: AnalysisSettings,
|
||||
env_only: AnalysisEnvOnly,
|
||||
/// The compiled prompt defaults so the UI can show placeholders and
|
||||
/// implement per-prompt "reset to default".
|
||||
prompt_defaults: PromptDefaults,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Handlers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn validation_error(errs: FieldErrors) -> AppError {
|
||||
AppError::ValidationFailed {
|
||||
message: "invalid settings".to_string(),
|
||||
details: serde_json::json!({ "fields": errs.errors }),
|
||||
}
|
||||
}
|
||||
|
||||
/// Current effective crawler settings: the stored DTO when present, otherwise
|
||||
/// derived from the env base (the row is seeded at boot, so the stored branch
|
||||
/// is the norm).
|
||||
async fn load_crawler_dto(state: &AppState) -> AppResult<CrawlerSettings> {
|
||||
Ok(match repo::app_settings::get(&state.db, KEY_CRAWLER).await? {
|
||||
Some(v) => serde_json::from_value(v)
|
||||
.unwrap_or_else(|_| CrawlerSettings::from_config(&state.crawler_base)),
|
||||
None => CrawlerSettings::from_config(&state.crawler_base),
|
||||
})
|
||||
}
|
||||
|
||||
async fn load_analysis_dto(state: &AppState) -> AppResult<AnalysisSettings> {
|
||||
Ok(match repo::app_settings::get(&state.db, KEY_ANALYSIS).await? {
|
||||
Some(v) => serde_json::from_value(v)
|
||||
.unwrap_or_else(|_| AnalysisSettings::from_config(&state.analysis_base)),
|
||||
None => AnalysisSettings::from_config(&state.analysis_base),
|
||||
})
|
||||
}
|
||||
|
||||
async fn get_crawler(
|
||||
State(state): State<AppState>,
|
||||
_admin: RequireAdmin,
|
||||
) -> AppResult<Json<CrawlerSettingsResponse>> {
|
||||
let editable = load_crawler_dto(&state).await?;
|
||||
Ok(Json(CrawlerSettingsResponse {
|
||||
editable,
|
||||
env_only: CrawlerEnvOnly::from_base(&state.crawler_base),
|
||||
}))
|
||||
}
|
||||
|
||||
async fn put_crawler(
|
||||
State(state): State<AppState>,
|
||||
admin: RequireAdmin,
|
||||
Json(incoming): Json<CrawlerSettings>,
|
||||
) -> AppResult<Json<CrawlerSettingsResponse>> {
|
||||
// Validate by converting against the env base; this also rebuilds the
|
||||
// download allowlist and re-parses tz / time.
|
||||
let cfg = incoming
|
||||
.to_config(&state.crawler_base)
|
||||
.map_err(validation_error)?;
|
||||
// Persist a normalized DTO (so the stored/returned allowlist reflects the
|
||||
// effective hosts, prompts are canonicalized, etc.).
|
||||
let normalized = CrawlerSettings::from_config(&cfg);
|
||||
let value = serde_json::to_value(&normalized).map_err(|e| AppError::Other(e.into()))?;
|
||||
|
||||
let mut tx = state.db.begin().await?;
|
||||
repo::app_settings::upsert(&mut *tx, KEY_CRAWLER, &value).await?;
|
||||
repo::admin_audit::insert(
|
||||
&mut *tx,
|
||||
admin.0.id,
|
||||
"update_crawler_settings",
|
||||
"settings",
|
||||
None,
|
||||
value.clone(),
|
||||
)
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
|
||||
// Apply live (graceful respawn) when a reloader is wired up.
|
||||
if let Some(reloader) = &state.reloader {
|
||||
reloader
|
||||
.reload_crawler(cfg)
|
||||
.await
|
||||
.map_err(AppError::Other)?;
|
||||
}
|
||||
|
||||
Ok(Json(CrawlerSettingsResponse {
|
||||
editable: normalized,
|
||||
env_only: CrawlerEnvOnly::from_base(&state.crawler_base),
|
||||
}))
|
||||
}
|
||||
|
||||
async fn get_analysis(
|
||||
State(state): State<AppState>,
|
||||
_admin: RequireAdmin,
|
||||
) -> AppResult<Json<AnalysisSettingsResponse>> {
|
||||
let editable = load_analysis_dto(&state).await?;
|
||||
Ok(Json(AnalysisSettingsResponse {
|
||||
editable,
|
||||
env_only: AnalysisEnvOnly {
|
||||
api_key_configured: state.analysis_base.api_key.is_some(),
|
||||
},
|
||||
prompt_defaults: PromptDefaults::get(),
|
||||
}))
|
||||
}
|
||||
|
||||
async fn put_analysis(
|
||||
State(state): State<AppState>,
|
||||
admin: RequireAdmin,
|
||||
Json(incoming): Json<AnalysisSettings>,
|
||||
) -> AppResult<Json<AnalysisSettingsResponse>> {
|
||||
let cfg = incoming
|
||||
.to_config(&state.analysis_base)
|
||||
.map_err(validation_error)?;
|
||||
let normalized = AnalysisSettings::from_config(&cfg);
|
||||
let value = serde_json::to_value(&normalized).map_err(|e| AppError::Other(e.into()))?;
|
||||
|
||||
let mut tx = state.db.begin().await?;
|
||||
repo::app_settings::upsert(&mut *tx, KEY_ANALYSIS, &value).await?;
|
||||
repo::admin_audit::insert(
|
||||
&mut *tx,
|
||||
admin.0.id,
|
||||
"update_analysis_settings",
|
||||
"settings",
|
||||
None,
|
||||
value.clone(),
|
||||
)
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
|
||||
if let Some(reloader) = &state.reloader {
|
||||
reloader
|
||||
.reload_analysis(cfg)
|
||||
.await
|
||||
.map_err(AppError::Other)?;
|
||||
}
|
||||
|
||||
Ok(Json(AnalysisSettingsResponse {
|
||||
editable: normalized,
|
||||
env_only: AnalysisEnvOnly {
|
||||
api_key_configured: state.analysis_base.api_key.is_some(),
|
||||
},
|
||||
prompt_defaults: PromptDefaults::get(),
|
||||
}))
|
||||
}
|
||||
@@ -161,7 +161,7 @@ async fn create(
|
||||
// rolled-back upload never leaves jobs pointing at nonexistent pages; a
|
||||
// failed enqueue is logged but doesn't fail the upload (the admin
|
||||
// re-enqueue endpoint can backfill).
|
||||
if state.analysis_enabled {
|
||||
if state.analysis_enabled() {
|
||||
for page_id in page_ids {
|
||||
if let Err(e) =
|
||||
repo::page_analysis::enqueue_for_page(&state.db, page_id, false).await
|
||||
|
||||
Reference in New Issue
Block a user