feat(admin): runtime-editable crawler & analysis config in the dashboard
Move crawler and analysis configuration out of boot-only env vars and into
a DB-backed, admin-editable surface applied live without a restart.
- New `app_settings` table (migration 0026) holds one JSONB row per group;
env vars seed it on first boot, then the DB is the source of truth.
- `settings.rs` adds serializable Crawler/Analysis DTOs with env-base +
DB-overlay conversion and field-level validation; env-only/secret fields
(browser, proxy, TOR, vision API key, PHPSESSID) are never persisted.
- `GET/PUT /api/v1/admin/settings/{crawler,analysis}` (RequireAdmin,
cookie-only, CSRF-guarded): GET returns the editable DTO + a read-only
env-managed view + prompt defaults; PUT validates (422 + per-field
details), persists + audits in one tx, then gracefully respawns the
affected daemon via a Supervisor.
- AppState swaps the crawler control/resync handles and analysis enable
gate behind shared runtime cells so reloads take effect with no restart;
a boot-time spawn failure is logged rather than aborting startup.
- Make the three vision prompts and sampling temperature configurable
(defaults preserved); cookie_domain is admin-editable too.
- Frontend: tabbed /admin/settings page with grouped fieldsets, prompt
reset-to-default, env-managed read-only panel, save-&-apply confirm, and
per-field validation; typed client + ApiError.details.
- Bump version 0.79.1 -> 0.80.0.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -556,3 +556,110 @@ export type AnalysisEvent =
|
||||
export function analysisStatusStreamUrl(): string {
|
||||
return apiUrl('/v1/admin/analysis/status/stream');
|
||||
}
|
||||
|
||||
// ---- runtime settings (crawler + analysis) ---------------------------------
|
||||
|
||||
/** Operationally-safe, admin-editable crawler settings. Host/infra and
|
||||
* session/secret fields are managed via environment and surfaced read-only
|
||||
* in {@link CrawlerEnvOnly}. */
|
||||
export type CrawlerSettings = {
|
||||
daemon_enabled: boolean;
|
||||
daily_at: string; // "HH:MM"
|
||||
tz: string; // IANA
|
||||
idle_timeout_secs: number;
|
||||
chapter_workers: number;
|
||||
retention_days: number;
|
||||
start_url: string | null;
|
||||
rate_ms: number;
|
||||
cdn_host: string | null;
|
||||
cdn_rate_ms: number;
|
||||
cookie_domain: string | null;
|
||||
user_agent: string | null;
|
||||
download_allowlist: string[];
|
||||
allow_any_host: boolean;
|
||||
max_image_bytes: number;
|
||||
manga_limit: number;
|
||||
job_timeout_secs: number;
|
||||
metadata_max_consecutive_failures: number;
|
||||
browser_restart_threshold: number;
|
||||
};
|
||||
|
||||
/** Read-only view of crawler fields managed via environment. */
|
||||
export type CrawlerEnvOnly = {
|
||||
browser_mode: string;
|
||||
browser_args: string[];
|
||||
proxy: string | null;
|
||||
tor_control_url: string | null;
|
||||
tor_credentials_configured: boolean;
|
||||
session_configured: boolean;
|
||||
};
|
||||
|
||||
export type CrawlerSettingsResponse = {
|
||||
editable: CrawlerSettings;
|
||||
env_only: CrawlerEnvOnly;
|
||||
};
|
||||
|
||||
/** Admin-editable analysis settings. Prompts are `null` to mean "use the
|
||||
* compiled default" (see {@link PromptDefaults}). The vision API key is
|
||||
* env-only and never returned. */
|
||||
export type AnalysisSettings = {
|
||||
enabled: boolean;
|
||||
workers: number;
|
||||
endpoint: string;
|
||||
model: string;
|
||||
request_timeout_secs: number;
|
||||
job_timeout_secs: number;
|
||||
max_tokens: number;
|
||||
max_pixels: number;
|
||||
min_slice_height: number;
|
||||
slice_overlap: number;
|
||||
tall_aspect_threshold: number;
|
||||
max_slices: number;
|
||||
max_image_bytes: number;
|
||||
response_format: 'json_schema' | 'json_object' | 'none';
|
||||
frequency_penalty: number;
|
||||
temperature: number;
|
||||
system_prompt: string | null;
|
||||
ocr_prompt: string | null;
|
||||
grounding_prompt: string | null;
|
||||
};
|
||||
|
||||
export type PromptDefaults = {
|
||||
system_prompt: string;
|
||||
ocr_prompt: string;
|
||||
grounding_prompt: string;
|
||||
};
|
||||
|
||||
export type AnalysisSettingsResponse = {
|
||||
editable: AnalysisSettings;
|
||||
env_only: { api_key_configured: boolean };
|
||||
prompt_defaults: PromptDefaults;
|
||||
};
|
||||
|
||||
export async function getCrawlerSettings(): Promise<CrawlerSettingsResponse> {
|
||||
return request<CrawlerSettingsResponse>('/v1/admin/settings/crawler');
|
||||
}
|
||||
|
||||
export async function updateCrawlerSettings(
|
||||
settings: CrawlerSettings
|
||||
): Promise<CrawlerSettingsResponse> {
|
||||
return request<CrawlerSettingsResponse>('/v1/admin/settings/crawler', {
|
||||
method: 'PUT',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify(settings)
|
||||
});
|
||||
}
|
||||
|
||||
export async function getAnalysisSettings(): Promise<AnalysisSettingsResponse> {
|
||||
return request<AnalysisSettingsResponse>('/v1/admin/settings/analysis');
|
||||
}
|
||||
|
||||
export async function updateAnalysisSettings(
|
||||
settings: AnalysisSettings
|
||||
): Promise<AnalysisSettingsResponse> {
|
||||
return request<AnalysisSettingsResponse>('/v1/admin/settings/analysis', {
|
||||
method: 'PUT',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify(settings)
|
||||
});
|
||||
}
|
||||
|
||||
@@ -25,14 +25,19 @@ export class ApiError extends Error {
|
||||
constructor(
|
||||
public readonly status: number,
|
||||
public readonly code: string,
|
||||
message: string
|
||||
message: string,
|
||||
/** The error envelope's `details` payload, when present (e.g. the
|
||||
* per-field `{ fields: [...] }` of a `validation_failed` response). */
|
||||
public readonly details?: unknown
|
||||
) {
|
||||
super(message);
|
||||
this.name = 'ApiError';
|
||||
}
|
||||
}
|
||||
|
||||
type ErrorEnvelope = { error?: { code?: unknown; message?: unknown } };
|
||||
type ErrorEnvelope = {
|
||||
error?: { code?: unknown; message?: unknown; details?: unknown };
|
||||
};
|
||||
|
||||
/**
|
||||
* Optional hook fired the first moment `request()` observes a 401 on
|
||||
@@ -59,6 +64,7 @@ export async function request<T>(path: string, init?: RequestInit): Promise<T> {
|
||||
if (!res.ok) {
|
||||
let code = 'http_error';
|
||||
let message = `${res.status} ${res.statusText}`;
|
||||
let details: unknown;
|
||||
const ct = res.headers.get('content-type') ?? '';
|
||||
try {
|
||||
if (ct.includes('application/json')) {
|
||||
@@ -70,6 +76,9 @@ export async function request<T>(path: string, init?: RequestInit): Promise<T> {
|
||||
if (typeof body.error.message === 'string' && body.error.message) {
|
||||
message = body.error.message;
|
||||
}
|
||||
if (body.error.details !== undefined) {
|
||||
details = body.error.details;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
const text = await res.text();
|
||||
@@ -88,7 +97,7 @@ export async function request<T>(path: string, init?: RequestInit): Promise<T> {
|
||||
console.error('on401 hook threw:', e);
|
||||
}
|
||||
}
|
||||
throw new ApiError(res.status, code, message);
|
||||
throw new ApiError(res.status, code, message, details);
|
||||
}
|
||||
// Any empty body (not just 204) returns undefined — the manga-add
|
||||
// endpoint, for instance, signals create-vs-already-present via
|
||||
|
||||
167
frontend/src/lib/api/settings.test.ts
Normal file
167
frontend/src/lib/api/settings.test.ts
Normal file
@@ -0,0 +1,167 @@
|
||||
import {
|
||||
describe,
|
||||
it,
|
||||
expect,
|
||||
vi,
|
||||
beforeEach,
|
||||
afterEach,
|
||||
type MockInstance
|
||||
} from 'vitest';
|
||||
import {
|
||||
getCrawlerSettings,
|
||||
updateCrawlerSettings,
|
||||
getAnalysisSettings,
|
||||
updateAnalysisSettings,
|
||||
type CrawlerSettings,
|
||||
type AnalysisSettings
|
||||
} from './admin';
|
||||
import { ApiError } from './client';
|
||||
|
||||
function ok(body: unknown, status = 200): Response {
|
||||
return new Response(JSON.stringify(body), {
|
||||
status,
|
||||
headers: { 'content-type': 'application/json' }
|
||||
});
|
||||
}
|
||||
|
||||
const crawlerFixture: CrawlerSettings = {
|
||||
daemon_enabled: true,
|
||||
daily_at: '00:00',
|
||||
tz: 'UTC',
|
||||
idle_timeout_secs: 600,
|
||||
chapter_workers: 1,
|
||||
retention_days: 7,
|
||||
start_url: null,
|
||||
rate_ms: 1000,
|
||||
cdn_host: null,
|
||||
cdn_rate_ms: 1000,
|
||||
cookie_domain: null,
|
||||
user_agent: null,
|
||||
download_allowlist: [],
|
||||
allow_any_host: false,
|
||||
max_image_bytes: 33554432,
|
||||
manga_limit: 0,
|
||||
job_timeout_secs: 600,
|
||||
metadata_max_consecutive_failures: 10,
|
||||
browser_restart_threshold: 3
|
||||
};
|
||||
|
||||
const analysisFixture: AnalysisSettings = {
|
||||
enabled: false,
|
||||
workers: 1,
|
||||
endpoint: 'http://localhost:8000/v1/chat/completions',
|
||||
model: '',
|
||||
request_timeout_secs: 120,
|
||||
job_timeout_secs: 600,
|
||||
max_tokens: 4096,
|
||||
max_pixels: 1000000,
|
||||
min_slice_height: 640,
|
||||
slice_overlap: 0.12,
|
||||
tall_aspect_threshold: 1.6,
|
||||
max_slices: 16,
|
||||
max_image_bytes: 8388608,
|
||||
response_format: 'json_schema',
|
||||
frequency_penalty: 0.3,
|
||||
temperature: 0,
|
||||
system_prompt: null,
|
||||
ocr_prompt: null,
|
||||
grounding_prompt: null
|
||||
};
|
||||
|
||||
describe('settings api client', () => {
|
||||
let fetchSpy: MockInstance<typeof globalThis.fetch>;
|
||||
|
||||
beforeEach(() => {
|
||||
fetchSpy = vi.spyOn(globalThis, 'fetch');
|
||||
});
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it('getCrawlerSettings GETs the crawler settings endpoint', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(
|
||||
ok({
|
||||
editable: crawlerFixture,
|
||||
env_only: {
|
||||
browser_mode: 'headless',
|
||||
browser_args: [],
|
||||
proxy: null,
|
||||
tor_control_url: null,
|
||||
tor_credentials_configured: false,
|
||||
session_configured: false
|
||||
}
|
||||
})
|
||||
);
|
||||
const r = await getCrawlerSettings();
|
||||
expect(r.editable.rate_ms).toBe(1000);
|
||||
expect(r.env_only.browser_mode).toBe('headless');
|
||||
const url = fetchSpy.mock.calls[0][0] as string;
|
||||
expect(url).toMatch(/\/v1\/admin\/settings\/crawler$/);
|
||||
});
|
||||
|
||||
it('updateCrawlerSettings PUTs the body as JSON', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(
|
||||
ok({
|
||||
editable: { ...crawlerFixture, rate_ms: 2000 },
|
||||
env_only: {
|
||||
browser_mode: 'headless',
|
||||
browser_args: [],
|
||||
proxy: null,
|
||||
tor_control_url: null,
|
||||
tor_credentials_configured: false,
|
||||
session_configured: false
|
||||
}
|
||||
})
|
||||
);
|
||||
const r = await updateCrawlerSettings({ ...crawlerFixture, rate_ms: 2000 });
|
||||
expect(r.editable.rate_ms).toBe(2000);
|
||||
const init = fetchSpy.mock.calls[0][1]!;
|
||||
expect(init.method).toBe('PUT');
|
||||
expect(JSON.parse(init.body as string).rate_ms).toBe(2000);
|
||||
});
|
||||
|
||||
it('getAnalysisSettings returns prompt defaults and the no-secret indicator', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(
|
||||
ok({
|
||||
editable: analysisFixture,
|
||||
env_only: { api_key_configured: true },
|
||||
prompt_defaults: {
|
||||
system_prompt: 'SYS',
|
||||
ocr_prompt: 'OCR',
|
||||
grounding_prompt: 'GND'
|
||||
}
|
||||
})
|
||||
);
|
||||
const r = await getAnalysisSettings();
|
||||
expect(r.editable.system_prompt).toBeNull();
|
||||
expect(r.env_only.api_key_configured).toBe(true);
|
||||
expect(r.prompt_defaults.system_prompt).toBe('SYS');
|
||||
// The secret is never present on the editable DTO.
|
||||
expect('api_key' in (r.editable as object)).toBe(false);
|
||||
});
|
||||
|
||||
it('updateAnalysisSettings surfaces 422 field errors via ApiError.details', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(
|
||||
new Response(
|
||||
JSON.stringify({
|
||||
error: {
|
||||
code: 'validation_failed',
|
||||
message: 'invalid settings',
|
||||
details: { fields: [{ field: 'workers', message: 'must be at least 1' }] }
|
||||
}
|
||||
}),
|
||||
{ status: 422, headers: { 'content-type': 'application/json' } }
|
||||
)
|
||||
);
|
||||
try {
|
||||
await updateAnalysisSettings({ ...analysisFixture, workers: 0 });
|
||||
expect.unreachable('should have thrown');
|
||||
} catch (e) {
|
||||
expect(e).toBeInstanceOf(ApiError);
|
||||
expect((e as ApiError).status).toBe(422);
|
||||
expect((e as ApiError).code).toBe('validation_failed');
|
||||
const details = (e as ApiError).details as { fields: { field: string }[] };
|
||||
expect(details.fields[0].field).toBe('workers');
|
||||
}
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user