feat(admin): runtime-editable crawler & analysis config in the dashboard
Some checks failed
deploy / test-backend (push) Waiting to run
deploy / build-and-push (push) Has been cancelled
deploy / deploy (push) Has been cancelled
deploy / test-frontend (push) Has been cancelled

Move crawler and analysis configuration out of boot-only env vars and into
a DB-backed, admin-editable surface applied live without a restart.

- New `app_settings` table (migration 0026) holds one JSONB row per group;
  env vars seed it on first boot, then the DB is the source of truth.
- `settings.rs` adds serializable Crawler/Analysis DTOs with env-base +
  DB-overlay conversion and field-level validation; env-only/secret fields
  (browser, proxy, TOR, vision API key, PHPSESSID) are never persisted.
- `GET/PUT /api/v1/admin/settings/{crawler,analysis}` (RequireAdmin,
  cookie-only, CSRF-guarded): GET returns the editable DTO + a read-only
  env-managed view + prompt defaults; PUT validates (422 + per-field
  details), persists + audits in one tx, then gracefully respawns the
  affected daemon via a Supervisor.
- AppState swaps the crawler control/resync handles and analysis enable
  gate behind shared runtime cells so reloads take effect with no restart;
  a boot-time spawn failure is logged rather than aborting startup.
- Make the three vision prompts and sampling temperature configurable
  (defaults preserved); cookie_domain is admin-editable too.
- Frontend: tabbed /admin/settings page with grouped fieldsets, prompt
  reset-to-default, env-managed read-only panel, save-&-apply confirm, and
  per-field validation; typed client + ApiError.details.
- Bump version 0.79.1 -> 0.80.0.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
MechaCat02
2026-06-14 13:32:20 +02:00
parent d3b827421f
commit 2b7a11b480
30 changed files with 2800 additions and 165 deletions

View File

@@ -14,9 +14,9 @@ use sqlx::PgPool;
use tempfile::TempDir;
use tower::ServiceExt;
use mangalord::app::{router, AppState};
use mangalord::app::{router, AppState, RuntimeControls};
use mangalord::auth::rate_limit::AuthRateLimiter;
use mangalord::config::{AuthConfig, UploadConfig};
use mangalord::config::{AnalysisConfig, AuthConfig, CrawlerConfig, UploadConfig};
use mangalord::storage::{LocalStorage, PutByteStream, Storage, StorageError, StreamingFile};
use async_trait::async_trait;
@@ -76,13 +76,15 @@ fn harness_with_auth_config(
auth_limiter,
// Default harness has no crawler daemon wired up; admin resync
// handlers return 503 in this config. Tests that need a stub
// resync service swap it in via `harness_with_resync`.
resync: None,
crawler: None,
// resync service swap it in via `harness_with_resync`. No reloader,
// so settings still persist but spawn no daemon.
runtime: Arc::new(RuntimeControls::new(false)),
reloader: None,
crawler_base: CrawlerConfig::default(),
analysis_base: AnalysisConfig::default(),
// Empty allowlist = CSRF check skipped. The CSRF-specific test
// harness `harness_with_admin_origins` overrides this.
admin_allowed_origins: Arc::new(Vec::new()),
analysis_enabled: false,
analysis_events: Arc::new(mangalord::analysis::events::AnalysisEvents::new()),
};
Harness { app: router(state), _storage_dir: storage_dir }
@@ -148,6 +150,8 @@ pub fn harness_with_resync(
..AuthConfig::default()
};
let auth_limiter = Arc::new(AuthRateLimiter::new(auth.rate_limit));
let runtime = Arc::new(RuntimeControls::new(false));
runtime.set_resync(Some(resync));
let state = AppState {
db: pool,
storage,
@@ -157,10 +161,11 @@ pub fn harness_with_resync(
max_file_bytes: 256 * 1024,
},
auth_limiter,
resync: Some(resync),
crawler: None,
runtime,
reloader: None,
crawler_base: CrawlerConfig::default(),
analysis_base: AnalysisConfig::default(),
admin_allowed_origins: Arc::new(Vec::new()),
analysis_enabled: false,
analysis_events: Arc::new(mangalord::analysis::events::AnalysisEvents::new()),
};
Harness {
@@ -189,10 +194,11 @@ pub fn harness_with_analysis(pool: PgPool) -> Harness {
max_file_bytes: 256 * 1024,
},
auth_limiter,
resync: None,
crawler: None,
runtime: Arc::new(RuntimeControls::new(true)),
reloader: None,
crawler_base: CrawlerConfig::default(),
analysis_base: AnalysisConfig::default(),
admin_allowed_origins: Arc::new(Vec::new()),
analysis_enabled: true,
analysis_events: Arc::new(mangalord::analysis::events::AnalysisEvents::new()),
};
Harness {
@@ -201,6 +207,71 @@ pub fn harness_with_analysis(pool: PgPool) -> Harness {
}
}
/// A [`DaemonReloader`] stub that records the configs it was asked to apply
/// and flips the shared analysis gate, without spawning any real daemon. Lets
/// settings tests assert that a `PUT` triggers a reload with the converted
/// config (and that the analysis enable gate moves).
pub struct StubReloader {
pub runtime: Arc<RuntimeControls>,
pub crawler: std::sync::Mutex<Option<CrawlerConfig>>,
pub analysis: std::sync::Mutex<Option<AnalysisConfig>>,
}
#[async_trait]
impl mangalord::app::DaemonReloader for StubReloader {
async fn reload_crawler(&self, cfg: CrawlerConfig) -> anyhow::Result<()> {
*self.crawler.lock().unwrap() = Some(cfg);
Ok(())
}
async fn reload_analysis(&self, cfg: AnalysisConfig) -> anyhow::Result<()> {
self.runtime.set_analysis_enabled(cfg.enabled);
*self.analysis.lock().unwrap() = Some(cfg);
Ok(())
}
}
/// Like [`harness`] but wires a [`StubReloader`] so the settings `PUT`
/// endpoints exercise the reload path. Returns the harness plus the shared
/// stub so the test can inspect what was applied.
pub fn harness_with_settings_reloader(pool: PgPool) -> (Harness, Arc<StubReloader>) {
let storage_dir = tempfile::tempdir().expect("tempdir");
let storage = Arc::new(LocalStorage::new(storage_dir.path()));
let auth = AuthConfig {
cookie_secure: false,
..AuthConfig::default()
};
let auth_limiter = Arc::new(AuthRateLimiter::new(auth.rate_limit));
let runtime = Arc::new(RuntimeControls::new(false));
let reloader = Arc::new(StubReloader {
runtime: Arc::clone(&runtime),
crawler: std::sync::Mutex::new(None),
analysis: std::sync::Mutex::new(None),
});
let state = AppState {
db: pool,
storage,
auth,
upload: UploadConfig {
max_request_bytes: 4 * 1024 * 1024,
max_file_bytes: 256 * 1024,
},
auth_limiter,
runtime,
reloader: Some(Arc::clone(&reloader) as Arc<dyn mangalord::app::DaemonReloader>),
crawler_base: CrawlerConfig::default(),
analysis_base: AnalysisConfig::default(),
admin_allowed_origins: Arc::new(Vec::new()),
analysis_events: Arc::new(mangalord::analysis::events::AnalysisEvents::new()),
};
(
Harness {
app: router(state),
_storage_dir: storage_dir,
},
reloader,
)
}
/// Like [`harness`] but configures an admin CSRF allowlist so the
/// `/admin/*` mutating endpoints reject cross-origin browser POSTs.
/// Used by the admin CSRF integration tests.
@@ -220,10 +291,11 @@ pub fn harness_with_admin_origins(pool: PgPool, origins: Vec<String>) -> Harness
max_file_bytes: 256 * 1024,
},
auth_limiter,
resync: None,
crawler: None,
runtime: Arc::new(RuntimeControls::new(false)),
reloader: None,
crawler_base: CrawlerConfig::default(),
analysis_base: AnalysisConfig::default(),
admin_allowed_origins: Arc::new(origins),
analysis_enabled: false,
analysis_events: Arc::new(mangalord::analysis::events::AnalysisEvents::new()),
};
Harness {