Move crawler and analysis configuration out of boot-only env vars and into
a DB-backed, admin-editable surface applied live without a restart.
- New `app_settings` table (migration 0026) holds one JSONB row per group;
env vars seed it on first boot, then the DB is the source of truth.
- `settings.rs` adds serializable Crawler/Analysis DTOs with env-base +
DB-overlay conversion and field-level validation; env-only/secret fields
(browser, proxy, TOR, vision API key, PHPSESSID) are never persisted.
- `GET/PUT /api/v1/admin/settings/{crawler,analysis}` (RequireAdmin,
cookie-only, CSRF-guarded): GET returns the editable DTO + a read-only
env-managed view + prompt defaults; PUT validates (422 + per-field
details), persists + audits in one tx, then gracefully respawns the
affected daemon via a Supervisor.
- AppState swaps the crawler control/resync handles and analysis enable
gate behind shared runtime cells so reloads take effect with no restart;
a boot-time spawn failure is logged rather than aborting startup.
- Make the three vision prompts and sampling temperature configurable
(defaults preserved); cookie_domain is admin-editable too.
- Frontend: tabbed /admin/settings page with grouped fieldsets, prompt
reset-to-default, env-managed read-only panel, save-&-apply confirm, and
per-field validation; typed client + ApiError.details.
- Bump version 0.79.1 -> 0.80.0.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
168 lines
5.2 KiB
TypeScript
168 lines
5.2 KiB
TypeScript
import {
|
|
describe,
|
|
it,
|
|
expect,
|
|
vi,
|
|
beforeEach,
|
|
afterEach,
|
|
type MockInstance
|
|
} from 'vitest';
|
|
import {
|
|
getCrawlerSettings,
|
|
updateCrawlerSettings,
|
|
getAnalysisSettings,
|
|
updateAnalysisSettings,
|
|
type CrawlerSettings,
|
|
type AnalysisSettings
|
|
} from './admin';
|
|
import { ApiError } from './client';
|
|
|
|
function ok(body: unknown, status = 200): Response {
|
|
return new Response(JSON.stringify(body), {
|
|
status,
|
|
headers: { 'content-type': 'application/json' }
|
|
});
|
|
}
|
|
|
|
const crawlerFixture: CrawlerSettings = {
|
|
daemon_enabled: true,
|
|
daily_at: '00:00',
|
|
tz: 'UTC',
|
|
idle_timeout_secs: 600,
|
|
chapter_workers: 1,
|
|
retention_days: 7,
|
|
start_url: null,
|
|
rate_ms: 1000,
|
|
cdn_host: null,
|
|
cdn_rate_ms: 1000,
|
|
cookie_domain: null,
|
|
user_agent: null,
|
|
download_allowlist: [],
|
|
allow_any_host: false,
|
|
max_image_bytes: 33554432,
|
|
manga_limit: 0,
|
|
job_timeout_secs: 600,
|
|
metadata_max_consecutive_failures: 10,
|
|
browser_restart_threshold: 3
|
|
};
|
|
|
|
const analysisFixture: AnalysisSettings = {
|
|
enabled: false,
|
|
workers: 1,
|
|
endpoint: 'http://localhost:8000/v1/chat/completions',
|
|
model: '',
|
|
request_timeout_secs: 120,
|
|
job_timeout_secs: 600,
|
|
max_tokens: 4096,
|
|
max_pixels: 1000000,
|
|
min_slice_height: 640,
|
|
slice_overlap: 0.12,
|
|
tall_aspect_threshold: 1.6,
|
|
max_slices: 16,
|
|
max_image_bytes: 8388608,
|
|
response_format: 'json_schema',
|
|
frequency_penalty: 0.3,
|
|
temperature: 0,
|
|
system_prompt: null,
|
|
ocr_prompt: null,
|
|
grounding_prompt: null
|
|
};
|
|
|
|
describe('settings api client', () => {
|
|
let fetchSpy: MockInstance<typeof globalThis.fetch>;
|
|
|
|
beforeEach(() => {
|
|
fetchSpy = vi.spyOn(globalThis, 'fetch');
|
|
});
|
|
afterEach(() => {
|
|
vi.restoreAllMocks();
|
|
});
|
|
|
|
it('getCrawlerSettings GETs the crawler settings endpoint', async () => {
|
|
fetchSpy.mockResolvedValueOnce(
|
|
ok({
|
|
editable: crawlerFixture,
|
|
env_only: {
|
|
browser_mode: 'headless',
|
|
browser_args: [],
|
|
proxy: null,
|
|
tor_control_url: null,
|
|
tor_credentials_configured: false,
|
|
session_configured: false
|
|
}
|
|
})
|
|
);
|
|
const r = await getCrawlerSettings();
|
|
expect(r.editable.rate_ms).toBe(1000);
|
|
expect(r.env_only.browser_mode).toBe('headless');
|
|
const url = fetchSpy.mock.calls[0][0] as string;
|
|
expect(url).toMatch(/\/v1\/admin\/settings\/crawler$/);
|
|
});
|
|
|
|
it('updateCrawlerSettings PUTs the body as JSON', async () => {
|
|
fetchSpy.mockResolvedValueOnce(
|
|
ok({
|
|
editable: { ...crawlerFixture, rate_ms: 2000 },
|
|
env_only: {
|
|
browser_mode: 'headless',
|
|
browser_args: [],
|
|
proxy: null,
|
|
tor_control_url: null,
|
|
tor_credentials_configured: false,
|
|
session_configured: false
|
|
}
|
|
})
|
|
);
|
|
const r = await updateCrawlerSettings({ ...crawlerFixture, rate_ms: 2000 });
|
|
expect(r.editable.rate_ms).toBe(2000);
|
|
const init = fetchSpy.mock.calls[0][1]!;
|
|
expect(init.method).toBe('PUT');
|
|
expect(JSON.parse(init.body as string).rate_ms).toBe(2000);
|
|
});
|
|
|
|
it('getAnalysisSettings returns prompt defaults and the no-secret indicator', async () => {
|
|
fetchSpy.mockResolvedValueOnce(
|
|
ok({
|
|
editable: analysisFixture,
|
|
env_only: { api_key_configured: true },
|
|
prompt_defaults: {
|
|
system_prompt: 'SYS',
|
|
ocr_prompt: 'OCR',
|
|
grounding_prompt: 'GND'
|
|
}
|
|
})
|
|
);
|
|
const r = await getAnalysisSettings();
|
|
expect(r.editable.system_prompt).toBeNull();
|
|
expect(r.env_only.api_key_configured).toBe(true);
|
|
expect(r.prompt_defaults.system_prompt).toBe('SYS');
|
|
// The secret is never present on the editable DTO.
|
|
expect('api_key' in (r.editable as object)).toBe(false);
|
|
});
|
|
|
|
it('updateAnalysisSettings surfaces 422 field errors via ApiError.details', async () => {
|
|
fetchSpy.mockResolvedValueOnce(
|
|
new Response(
|
|
JSON.stringify({
|
|
error: {
|
|
code: 'validation_failed',
|
|
message: 'invalid settings',
|
|
details: { fields: [{ field: 'workers', message: 'must be at least 1' }] }
|
|
}
|
|
}),
|
|
{ status: 422, headers: { 'content-type': 'application/json' } }
|
|
)
|
|
);
|
|
try {
|
|
await updateAnalysisSettings({ ...analysisFixture, workers: 0 });
|
|
expect.unreachable('should have thrown');
|
|
} catch (e) {
|
|
expect(e).toBeInstanceOf(ApiError);
|
|
expect((e as ApiError).status).toBe(422);
|
|
expect((e as ApiError).code).toBe('validation_failed');
|
|
const details = (e as ApiError).details as { fields: { field: string }[] };
|
|
expect(details.fields[0].field).toBe('workers');
|
|
}
|
|
});
|
|
});
|