feat(admin): runtime-editable crawler & analysis config in the dashboard
Move crawler and analysis configuration out of boot-only env vars and into
a DB-backed, admin-editable surface applied live without a restart.
- New `app_settings` table (migration 0026) holds one JSONB row per group;
env vars seed it on first boot, then the DB is the source of truth.
- `settings.rs` adds serializable Crawler/Analysis DTOs with env-base +
DB-overlay conversion and field-level validation; env-only/secret fields
(browser, proxy, TOR, vision API key, PHPSESSID) are never persisted.
- `GET/PUT /api/v1/admin/settings/{crawler,analysis}` (RequireAdmin,
cookie-only, CSRF-guarded): GET returns the editable DTO + a read-only
env-managed view + prompt defaults; PUT validates (422 + per-field
details), persists + audits in one tx, then gracefully respawns the
affected daemon via a Supervisor.
- AppState swaps the crawler control/resync handles and analysis enable
gate behind shared runtime cells so reloads take effect with no restart;
a boot-time spawn failure is logged rather than aborting startup.
- Make the three vision prompts and sampling temperature configurable
(defaults preserved); cookie_domain is admin-editable too.
- Frontend: tabbed /admin/settings page with grouped fieldsets, prompt
reset-to-default, env-managed read-only panel, save-&-apply confirm, and
per-field validation; typed client + ApiError.details.
- Bump version 0.79.1 -> 0.80.0.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -14,9 +14,9 @@ use sqlx::PgPool;
|
||||
use tempfile::TempDir;
|
||||
use tower::ServiceExt;
|
||||
|
||||
use mangalord::app::{router, AppState};
|
||||
use mangalord::app::{router, AppState, RuntimeControls};
|
||||
use mangalord::auth::rate_limit::AuthRateLimiter;
|
||||
use mangalord::config::{AuthConfig, UploadConfig};
|
||||
use mangalord::config::{AnalysisConfig, AuthConfig, CrawlerConfig, UploadConfig};
|
||||
use mangalord::storage::{LocalStorage, PutByteStream, Storage, StorageError, StreamingFile};
|
||||
|
||||
use async_trait::async_trait;
|
||||
@@ -76,13 +76,15 @@ fn harness_with_auth_config(
|
||||
auth_limiter,
|
||||
// Default harness has no crawler daemon wired up; admin resync
|
||||
// handlers return 503 in this config. Tests that need a stub
|
||||
// resync service swap it in via `harness_with_resync`.
|
||||
resync: None,
|
||||
crawler: None,
|
||||
// resync service swap it in via `harness_with_resync`. No reloader,
|
||||
// so settings still persist but spawn no daemon.
|
||||
runtime: Arc::new(RuntimeControls::new(false)),
|
||||
reloader: None,
|
||||
crawler_base: CrawlerConfig::default(),
|
||||
analysis_base: AnalysisConfig::default(),
|
||||
// Empty allowlist = CSRF check skipped. The CSRF-specific test
|
||||
// harness `harness_with_admin_origins` overrides this.
|
||||
admin_allowed_origins: Arc::new(Vec::new()),
|
||||
analysis_enabled: false,
|
||||
analysis_events: Arc::new(mangalord::analysis::events::AnalysisEvents::new()),
|
||||
};
|
||||
Harness { app: router(state), _storage_dir: storage_dir }
|
||||
@@ -148,6 +150,8 @@ pub fn harness_with_resync(
|
||||
..AuthConfig::default()
|
||||
};
|
||||
let auth_limiter = Arc::new(AuthRateLimiter::new(auth.rate_limit));
|
||||
let runtime = Arc::new(RuntimeControls::new(false));
|
||||
runtime.set_resync(Some(resync));
|
||||
let state = AppState {
|
||||
db: pool,
|
||||
storage,
|
||||
@@ -157,10 +161,11 @@ pub fn harness_with_resync(
|
||||
max_file_bytes: 256 * 1024,
|
||||
},
|
||||
auth_limiter,
|
||||
resync: Some(resync),
|
||||
crawler: None,
|
||||
runtime,
|
||||
reloader: None,
|
||||
crawler_base: CrawlerConfig::default(),
|
||||
analysis_base: AnalysisConfig::default(),
|
||||
admin_allowed_origins: Arc::new(Vec::new()),
|
||||
analysis_enabled: false,
|
||||
analysis_events: Arc::new(mangalord::analysis::events::AnalysisEvents::new()),
|
||||
};
|
||||
Harness {
|
||||
@@ -189,10 +194,11 @@ pub fn harness_with_analysis(pool: PgPool) -> Harness {
|
||||
max_file_bytes: 256 * 1024,
|
||||
},
|
||||
auth_limiter,
|
||||
resync: None,
|
||||
crawler: None,
|
||||
runtime: Arc::new(RuntimeControls::new(true)),
|
||||
reloader: None,
|
||||
crawler_base: CrawlerConfig::default(),
|
||||
analysis_base: AnalysisConfig::default(),
|
||||
admin_allowed_origins: Arc::new(Vec::new()),
|
||||
analysis_enabled: true,
|
||||
analysis_events: Arc::new(mangalord::analysis::events::AnalysisEvents::new()),
|
||||
};
|
||||
Harness {
|
||||
@@ -201,6 +207,71 @@ pub fn harness_with_analysis(pool: PgPool) -> Harness {
|
||||
}
|
||||
}
|
||||
|
||||
/// A [`DaemonReloader`] stub that records the configs it was asked to apply
|
||||
/// and flips the shared analysis gate, without spawning any real daemon. Lets
|
||||
/// settings tests assert that a `PUT` triggers a reload with the converted
|
||||
/// config (and that the analysis enable gate moves).
|
||||
pub struct StubReloader {
|
||||
pub runtime: Arc<RuntimeControls>,
|
||||
pub crawler: std::sync::Mutex<Option<CrawlerConfig>>,
|
||||
pub analysis: std::sync::Mutex<Option<AnalysisConfig>>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl mangalord::app::DaemonReloader for StubReloader {
|
||||
async fn reload_crawler(&self, cfg: CrawlerConfig) -> anyhow::Result<()> {
|
||||
*self.crawler.lock().unwrap() = Some(cfg);
|
||||
Ok(())
|
||||
}
|
||||
async fn reload_analysis(&self, cfg: AnalysisConfig) -> anyhow::Result<()> {
|
||||
self.runtime.set_analysis_enabled(cfg.enabled);
|
||||
*self.analysis.lock().unwrap() = Some(cfg);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Like [`harness`] but wires a [`StubReloader`] so the settings `PUT`
|
||||
/// endpoints exercise the reload path. Returns the harness plus the shared
|
||||
/// stub so the test can inspect what was applied.
|
||||
pub fn harness_with_settings_reloader(pool: PgPool) -> (Harness, Arc<StubReloader>) {
|
||||
let storage_dir = tempfile::tempdir().expect("tempdir");
|
||||
let storage = Arc::new(LocalStorage::new(storage_dir.path()));
|
||||
let auth = AuthConfig {
|
||||
cookie_secure: false,
|
||||
..AuthConfig::default()
|
||||
};
|
||||
let auth_limiter = Arc::new(AuthRateLimiter::new(auth.rate_limit));
|
||||
let runtime = Arc::new(RuntimeControls::new(false));
|
||||
let reloader = Arc::new(StubReloader {
|
||||
runtime: Arc::clone(&runtime),
|
||||
crawler: std::sync::Mutex::new(None),
|
||||
analysis: std::sync::Mutex::new(None),
|
||||
});
|
||||
let state = AppState {
|
||||
db: pool,
|
||||
storage,
|
||||
auth,
|
||||
upload: UploadConfig {
|
||||
max_request_bytes: 4 * 1024 * 1024,
|
||||
max_file_bytes: 256 * 1024,
|
||||
},
|
||||
auth_limiter,
|
||||
runtime,
|
||||
reloader: Some(Arc::clone(&reloader) as Arc<dyn mangalord::app::DaemonReloader>),
|
||||
crawler_base: CrawlerConfig::default(),
|
||||
analysis_base: AnalysisConfig::default(),
|
||||
admin_allowed_origins: Arc::new(Vec::new()),
|
||||
analysis_events: Arc::new(mangalord::analysis::events::AnalysisEvents::new()),
|
||||
};
|
||||
(
|
||||
Harness {
|
||||
app: router(state),
|
||||
_storage_dir: storage_dir,
|
||||
},
|
||||
reloader,
|
||||
)
|
||||
}
|
||||
|
||||
/// Like [`harness`] but configures an admin CSRF allowlist so the
|
||||
/// `/admin/*` mutating endpoints reject cross-origin browser POSTs.
|
||||
/// Used by the admin CSRF integration tests.
|
||||
@@ -220,10 +291,11 @@ pub fn harness_with_admin_origins(pool: PgPool, origins: Vec<String>) -> Harness
|
||||
max_file_bytes: 256 * 1024,
|
||||
},
|
||||
auth_limiter,
|
||||
resync: None,
|
||||
crawler: None,
|
||||
runtime: Arc::new(RuntimeControls::new(false)),
|
||||
reloader: None,
|
||||
crawler_base: CrawlerConfig::default(),
|
||||
analysis_base: AnalysisConfig::default(),
|
||||
admin_allowed_origins: Arc::new(origins),
|
||||
analysis_enabled: false,
|
||||
analysis_events: Arc::new(mangalord::analysis::events::AnalysisEvents::new()),
|
||||
};
|
||||
Harness {
|
||||
|
||||
Reference in New Issue
Block a user