Move crawler and analysis configuration out of boot-only env vars and into
a DB-backed, admin-editable surface applied live without a restart.
- New `app_settings` table (migration 0026) holds one JSONB row per group;
env vars seed it on first boot, then the DB is the source of truth.
- `settings.rs` adds serializable Crawler/Analysis DTOs with env-base +
DB-overlay conversion and field-level validation; env-only/secret fields
(browser, proxy, TOR, vision API key, PHPSESSID) are never persisted.
- `GET/PUT /api/v1/admin/settings/{crawler,analysis}` (RequireAdmin,
cookie-only, CSRF-guarded): GET returns the editable DTO + a read-only
env-managed view + prompt defaults; PUT validates (422 + per-field
details), persists + audits in one tx, then gracefully respawns the
affected daemon via a Supervisor.
- AppState swaps the crawler control/resync handles and analysis enable
gate behind shared runtime cells so reloads take effect with no restart;
a boot-time spawn failure is logged rather than aborting startup.
- Make the three vision prompts and sampling temperature configurable
(defaults preserved); cookie_domain is admin-editable too.
- Frontend: tabbed /admin/settings page with grouped fieldsets, prompt
reset-to-default, env-managed read-only panel, save-&-apply confirm, and
per-field validation; typed client + ApiError.details.
- Bump version 0.79.1 -> 0.80.0.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
273 lines
9.6 KiB
Rust
273 lines
9.6 KiB
Rust
//! Integration tests for the runtime-editable settings endpoints
|
|
//! (`/api/v1/admin/settings/{crawler,analysis}`):
|
|
//!
|
|
//! * the `RequireAdmin` gate,
|
|
//! * `GET` returns the editable DTO + the read-only env-managed view (and the
|
|
//! analysis prompt defaults), never the secret,
|
|
//! * `PUT` validates (422 with per-field details on bad input), persists, and
|
|
//! writes an `admin_audit` row,
|
|
//! * `PUT` invokes the daemon reloader with the converted config and moves the
|
|
//! analysis enable gate,
|
|
//! * the repo-level env→DB seed is idempotent.
|
|
|
|
mod common;
|
|
|
|
use axum::http::StatusCode;
|
|
use axum::Router;
|
|
use serde_json::json;
|
|
use sqlx::PgPool;
|
|
use tower::ServiceExt;
|
|
use uuid::Uuid;
|
|
|
|
use mangalord::repo;
|
|
|
|
async fn seed_admin(pool: &PgPool, app: &Router) -> (String, String, Uuid) {
|
|
let (username, cookie) = common::register_user(app).await;
|
|
let u = repo::user::find_by_username(pool, &username)
|
|
.await
|
|
.unwrap()
|
|
.unwrap();
|
|
repo::user::set_is_admin_unchecked(pool, u.id, true).await.unwrap();
|
|
(username, cookie, u.id)
|
|
}
|
|
|
|
// ---- RequireAdmin gate -----------------------------------------------------
|
|
|
|
#[sqlx::test(migrations = "./migrations")]
|
|
async fn get_settings_requires_admin(pool: PgPool) {
|
|
let h = common::harness(pool);
|
|
let (_u, cookie) = common::register_user(&h.app).await;
|
|
let resp = h
|
|
.app
|
|
.oneshot(common::get_with_cookie("/api/v1/admin/settings/crawler", &cookie))
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
|
|
}
|
|
|
|
#[sqlx::test(migrations = "./migrations")]
|
|
async fn get_settings_rejects_anonymous(pool: PgPool) {
|
|
let h = common::harness(pool);
|
|
// No cookie at all → not logged in.
|
|
let resp = h
|
|
.app
|
|
.oneshot(common::get("/api/v1/admin/settings/crawler"))
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(resp.status(), StatusCode::UNAUTHORIZED);
|
|
}
|
|
|
|
#[sqlx::test(migrations = "./migrations")]
|
|
async fn put_settings_requires_admin(pool: PgPool) {
|
|
let h = common::harness(pool);
|
|
let (_u, cookie) = common::register_user(&h.app).await;
|
|
let resp = h
|
|
.app
|
|
.oneshot(common::put_json_with_cookie(
|
|
"/api/v1/admin/settings/analysis",
|
|
json!({ "enabled": false }),
|
|
&cookie,
|
|
))
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
|
|
}
|
|
|
|
// ---- GET shape -------------------------------------------------------------
|
|
|
|
#[sqlx::test(migrations = "./migrations")]
|
|
async fn get_crawler_returns_editable_and_env_only(pool: PgPool) {
|
|
let h = common::harness(pool.clone());
|
|
let (_u, cookie, _id) = seed_admin(&pool, &h.app).await;
|
|
let resp = h
|
|
.app
|
|
.oneshot(common::get_with_cookie("/api/v1/admin/settings/crawler", &cookie))
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(resp.status(), StatusCode::OK);
|
|
let body = common::body_json(resp).await;
|
|
// Editable knobs present with their defaults.
|
|
assert_eq!(body["editable"]["daily_at"], "00:00");
|
|
assert_eq!(body["editable"]["chapter_workers"], 1);
|
|
// Env-managed view present and read-only.
|
|
assert_eq!(body["env_only"]["browser_mode"], "headless");
|
|
assert_eq!(body["env_only"]["session_configured"], false);
|
|
}
|
|
|
|
#[sqlx::test(migrations = "./migrations")]
|
|
async fn get_analysis_returns_defaults_and_no_secret(pool: PgPool) {
|
|
let h = common::harness(pool.clone());
|
|
let (_u, cookie, _id) = seed_admin(&pool, &h.app).await;
|
|
let resp = h
|
|
.app
|
|
.oneshot(common::get_with_cookie("/api/v1/admin/settings/analysis", &cookie))
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(resp.status(), StatusCode::OK);
|
|
let body = common::body_json(resp).await;
|
|
// Unmodified prompts seed as null ("use default").
|
|
assert!(body["editable"]["system_prompt"].is_null());
|
|
// Compiled defaults exposed for the UI placeholder / reset.
|
|
assert!(body["prompt_defaults"]["system_prompt"]
|
|
.as_str()
|
|
.unwrap()
|
|
.contains("manga"));
|
|
// The secret is never echoed; only a boolean indicator.
|
|
assert!(body["editable"].get("api_key").is_none());
|
|
assert_eq!(body["env_only"]["api_key_configured"], false);
|
|
}
|
|
|
|
// ---- PUT validation --------------------------------------------------------
|
|
|
|
#[sqlx::test(migrations = "./migrations")]
|
|
async fn put_crawler_invalid_tz_returns_422(pool: PgPool) {
|
|
let h = common::harness(pool.clone());
|
|
let (_u, cookie, _id) = seed_admin(&pool, &h.app).await;
|
|
let resp = h
|
|
.app
|
|
.oneshot(common::put_json_with_cookie(
|
|
"/api/v1/admin/settings/crawler",
|
|
json!({ "tz": "Mars/Phobos", "daily_at": "9am" }),
|
|
&cookie,
|
|
))
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(resp.status(), StatusCode::UNPROCESSABLE_ENTITY);
|
|
let body = common::body_json(resp).await;
|
|
let fields: Vec<&str> = body["error"]["details"]["fields"]
|
|
.as_array()
|
|
.unwrap()
|
|
.iter()
|
|
.map(|e| e["field"].as_str().unwrap())
|
|
.collect();
|
|
assert!(fields.contains(&"tz"));
|
|
assert!(fields.contains(&"daily_at"));
|
|
}
|
|
|
|
#[sqlx::test(migrations = "./migrations")]
|
|
async fn put_analysis_invalid_returns_422(pool: PgPool) {
|
|
let h = common::harness(pool.clone());
|
|
let (_u, cookie, _id) = seed_admin(&pool, &h.app).await;
|
|
let resp = h
|
|
.app
|
|
.oneshot(common::put_json_with_cookie(
|
|
"/api/v1/admin/settings/analysis",
|
|
json!({ "workers": 0, "slice_overlap": 2.0, "endpoint": "nope" }),
|
|
&cookie,
|
|
))
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(resp.status(), StatusCode::UNPROCESSABLE_ENTITY);
|
|
let body = common::body_json(resp).await;
|
|
let fields: Vec<&str> = body["error"]["details"]["fields"]
|
|
.as_array()
|
|
.unwrap()
|
|
.iter()
|
|
.map(|e| e["field"].as_str().unwrap())
|
|
.collect();
|
|
assert!(fields.contains(&"workers"));
|
|
assert!(fields.contains(&"slice_overlap"));
|
|
assert!(fields.contains(&"endpoint"));
|
|
}
|
|
|
|
// ---- PUT persistence + audit ----------------------------------------------
|
|
|
|
#[sqlx::test(migrations = "./migrations")]
|
|
async fn put_crawler_persists_and_audits(pool: PgPool) {
|
|
let h = common::harness(pool.clone());
|
|
let (_u, cookie, admin_id) = seed_admin(&pool, &h.app).await;
|
|
|
|
let resp = h
|
|
.app
|
|
.clone()
|
|
.oneshot(common::put_json_with_cookie(
|
|
"/api/v1/admin/settings/crawler",
|
|
json!({ "rate_ms": 2500, "chapter_workers": 4, "start_url": "https://example.com/" }),
|
|
&cookie,
|
|
))
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(resp.status(), StatusCode::OK);
|
|
let body = common::body_json(resp).await;
|
|
assert_eq!(body["editable"]["rate_ms"], 2500);
|
|
assert_eq!(body["editable"]["chapter_workers"], 4);
|
|
// Allowlist normalized to include the start-url host.
|
|
let allow: Vec<&str> = body["editable"]["download_allowlist"]
|
|
.as_array()
|
|
.unwrap()
|
|
.iter()
|
|
.map(|v| v.as_str().unwrap())
|
|
.collect();
|
|
assert!(allow.contains(&"example.com"));
|
|
|
|
// A second GET reflects the persisted change.
|
|
let resp = h
|
|
.app
|
|
.clone()
|
|
.oneshot(common::get_with_cookie("/api/v1/admin/settings/crawler", &cookie))
|
|
.await
|
|
.unwrap();
|
|
let body = common::body_json(resp).await;
|
|
assert_eq!(body["editable"]["rate_ms"], 2500);
|
|
|
|
// An audit row landed.
|
|
let (action, kind): (String, String) = sqlx::query_as(
|
|
"SELECT action, target_kind FROM admin_audit WHERE actor_user_id = $1 ORDER BY at DESC LIMIT 1",
|
|
)
|
|
.bind(admin_id)
|
|
.fetch_one(&pool)
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(action, "update_crawler_settings");
|
|
assert_eq!(kind, "settings");
|
|
}
|
|
|
|
// ---- PUT triggers reload ---------------------------------------------------
|
|
|
|
#[sqlx::test(migrations = "./migrations")]
|
|
async fn put_analysis_triggers_reload_and_flips_gate(pool: PgPool) {
|
|
let (h, reloader) = common::harness_with_settings_reloader(pool.clone());
|
|
let (_u, cookie, _id) = seed_admin(&pool, &h.app).await;
|
|
|
|
// Gate starts closed.
|
|
assert!(!reloader.runtime.analysis_enabled());
|
|
|
|
let resp = h
|
|
.app
|
|
.clone()
|
|
.oneshot(common::put_json_with_cookie(
|
|
"/api/v1/admin/settings/analysis",
|
|
json!({ "enabled": true, "model": "qwen2-vl", "temperature": 0.4 }),
|
|
&cookie,
|
|
))
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(resp.status(), StatusCode::OK);
|
|
|
|
// The reloader was invoked with the converted config, and the shared gate
|
|
// flipped on — both without a restart.
|
|
assert!(reloader.runtime.analysis_enabled());
|
|
let applied = reloader.analysis.lock().unwrap().clone().expect("reload called");
|
|
assert!(applied.enabled);
|
|
assert_eq!(applied.model, "qwen2-vl");
|
|
assert_eq!(applied.temperature, 0.4);
|
|
}
|
|
|
|
// ---- repo-level seed -------------------------------------------------------
|
|
|
|
#[sqlx::test(migrations = "./migrations")]
|
|
async fn seed_if_absent_is_idempotent(pool: PgPool) {
|
|
let v1 = json!({ "rate_ms": 1000 });
|
|
let v2 = json!({ "rate_ms": 9999 });
|
|
// First seed inserts.
|
|
assert!(repo::app_settings::seed_if_absent(&pool, "crawler", &v1).await.unwrap());
|
|
// Second seed is a no-op (row already present) and does not overwrite.
|
|
assert!(!repo::app_settings::seed_if_absent(&pool, "crawler", &v2).await.unwrap());
|
|
let got = repo::app_settings::get(&pool, "crawler").await.unwrap().unwrap();
|
|
assert_eq!(got["rate_ms"], 1000);
|
|
// upsert does overwrite.
|
|
repo::app_settings::upsert(&pool, "crawler", &v2).await.unwrap();
|
|
let got = repo::app_settings::get(&pool, "crawler").await.unwrap().unwrap();
|
|
assert_eq!(got["rate_ms"], 9999);
|
|
}
|