feat(admin): runtime-editable crawler & analysis config in the dashboard
Some checks failed
deploy / test-backend (push) Waiting to run
deploy / build-and-push (push) Has been cancelled
deploy / deploy (push) Has been cancelled
deploy / test-frontend (push) Has been cancelled

Move crawler and analysis configuration out of boot-only env vars and into
a DB-backed, admin-editable surface applied live without a restart.

- New `app_settings` table (migration 0026) holds one JSONB row per group;
  env vars seed it on first boot, then the DB is the source of truth.
- `settings.rs` adds serializable Crawler/Analysis DTOs with env-base +
  DB-overlay conversion and field-level validation; env-only/secret fields
  (browser, proxy, TOR, vision API key, PHPSESSID) are never persisted.
- `GET/PUT /api/v1/admin/settings/{crawler,analysis}` (RequireAdmin,
  cookie-only, CSRF-guarded): GET returns the editable DTO + a read-only
  env-managed view + prompt defaults; PUT validates (422 + per-field
  details), persists + audits in one tx, then gracefully respawns the
  affected daemon via a Supervisor.
- AppState swaps the crawler control/resync handles and analysis enable
  gate behind shared runtime cells so reloads take effect with no restart;
  a boot-time spawn failure is logged rather than aborting startup.
- Make the three vision prompts and sampling temperature configurable
  (defaults preserved); cookie_domain is admin-editable too.
- Frontend: tabbed /admin/settings page with grouped fieldsets, prompt
  reset-to-default, env-managed read-only panel, save-&-apply confirm, and
  per-field validation; typed client + ApiError.details.
- Bump version 0.79.1 -> 0.80.0.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
MechaCat02
2026-06-14 13:32:20 +02:00
parent d3b827421f
commit 2b7a11b480
30 changed files with 2800 additions and 165 deletions

View File

@@ -8,6 +8,7 @@
{ href: '/admin/mangas', label: 'Mangas' },
{ href: '/admin/crawler', label: 'Crawler' },
{ href: '/admin/analysis', label: 'Analysis' },
{ href: '/admin/settings', label: 'Settings' },
{ href: '/admin/system', label: 'System' }
];
</script>

View File

@@ -0,0 +1,664 @@
<script lang="ts">
import { onMount } from 'svelte';
import Modal from '$lib/components/Modal.svelte';
import { ApiError } from '$lib/api/client';
import {
getCrawlerSettings,
updateCrawlerSettings,
getAnalysisSettings,
updateAnalysisSettings,
type CrawlerSettings,
type CrawlerEnvOnly,
type AnalysisSettings,
type PromptDefaults
} from '$lib/api/admin';
type Tab = 'crawler' | 'analysis';
let tab = $state<Tab>('crawler');
// Crawler state
let crawler = $state<CrawlerSettings | null>(null);
let crawlerEnv = $state<CrawlerEnvOnly | null>(null);
let allowlistText = $state(''); // textarea mirror of download_allowlist
// Analysis state
let analysis = $state<AnalysisSettings | null>(null);
let promptDefaults = $state<PromptDefaults | null>(null);
let apiKeyConfigured = $state(false);
let loading = $state(true);
let loadError = $state<string | null>(null);
let saving = $state(false);
let saveError = $state<string | null>(null);
let success = $state<string | null>(null);
// field -> message, parsed from a 422 validation_failed response
let fieldErrors = $state<Record<string, string>>({});
let confirmOpen = $state(false);
onMount(load);
async function load() {
loading = true;
loadError = null;
try {
const [c, a] = await Promise.all([getCrawlerSettings(), getAnalysisSettings()]);
crawler = c.editable;
crawlerEnv = c.env_only;
allowlistText = c.editable.download_allowlist.join('\n');
analysis = a.editable;
promptDefaults = a.prompt_defaults;
apiKeyConfigured = a.env_only.api_key_configured;
} catch (e) {
loadError = e instanceof Error ? e.message : 'Failed to load settings.';
} finally {
loading = false;
}
}
function parseFieldErrors(e: unknown): Record<string, string> {
const out: Record<string, string> = {};
if (
e instanceof ApiError &&
e.details &&
typeof e.details === 'object' &&
'fields' in e.details
) {
const fields = (e.details as { fields?: { field: string; message: string }[] }).fields;
for (const f of fields ?? []) out[f.field] = f.message;
}
return out;
}
function askSave() {
saveError = null;
success = null;
fieldErrors = {};
confirmOpen = true;
}
async function save() {
confirmOpen = false;
saving = true;
saveError = null;
success = null;
fieldErrors = {};
try {
if (tab === 'crawler' && crawler) {
// Sync the allowlist textarea back into the model.
crawler.download_allowlist = allowlistText
.split('\n')
.map((s) => s.trim())
.filter(Boolean);
const r = await updateCrawlerSettings(crawler);
crawler = r.editable;
crawlerEnv = r.env_only;
allowlistText = r.editable.download_allowlist.join('\n');
} else if (tab === 'analysis' && analysis) {
const r = await updateAnalysisSettings(analysis);
analysis = r.editable;
promptDefaults = r.prompt_defaults;
apiKeyConfigured = r.env_only.api_key_configured;
}
success = 'Saved and applied. The subsystem was restarted with the new settings.';
} catch (e) {
fieldErrors = parseFieldErrors(e);
saveError =
Object.keys(fieldErrors).length > 0
? 'Some fields need fixing.'
: e instanceof Error
? e.message
: 'Save failed.';
} finally {
saving = false;
}
}
function resetPrompt(which: 'system_prompt' | 'ocr_prompt' | 'grounding_prompt') {
if (analysis) analysis[which] = null;
}
</script>
<h1>Settings</h1>
<p class="lead muted">
Edit crawler and analysis configuration. Saving applies immediately — the affected daemon is
gracefully restarted, so in-flight jobs are retried rather than lost. Host-level and secret
values stay environment-managed and are shown read-only.
</p>
<div class="tabs" role="tablist">
<button
role="tab"
aria-selected={tab === 'crawler'}
class:active={tab === 'crawler'}
onclick={() => (tab = 'crawler')}
data-testid="settings-tab-crawler">Crawler</button
>
<button
role="tab"
aria-selected={tab === 'analysis'}
class:active={tab === 'analysis'}
onclick={() => (tab = 'analysis')}
data-testid="settings-tab-analysis">Analysis</button
>
</div>
{#if loading}
<p class="muted">Loading…</p>
{:else if loadError}
<p class="error" role="alert">{loadError}</p>
{:else}
<div class="sticky-bar">
{#if tab === 'crawler' && crawler}
<label class="toggle">
<input
type="checkbox"
bind:checked={crawler.daemon_enabled}
data-testid="settings-crawler-enabled"
/>
<span>Crawler daemon {crawler.daemon_enabled ? 'enabled' : 'disabled'}</span>
</label>
{:else if tab === 'analysis' && analysis}
<label class="toggle">
<input
type="checkbox"
bind:checked={analysis.enabled}
data-testid="settings-analysis-enabled"
/>
<span>Analysis worker {analysis.enabled ? 'enabled' : 'disabled'}</span>
</label>
{/if}
<div class="spacer"></div>
<button class="primary" disabled={saving} onclick={askSave} data-testid="settings-save">
{saving ? 'Saving…' : 'Save & apply'}
</button>
</div>
{#if success}<p class="notice" role="status">{success}</p>{/if}
{#if saveError}<p class="error" role="alert">{saveError}</p>{/if}
{#if tab === 'crawler' && crawler}
<div class="form">
<fieldset>
<legend>Schedule</legend>
<label class="field">
<span>Daily metadata pass (HH:MM)</span>
<input type="time" bind:value={crawler.daily_at} />
{#if fieldErrors.daily_at}<small class="fe">{fieldErrors.daily_at}</small>{/if}
</label>
<label class="field">
<span>Timezone (IANA)</span>
<input type="text" bind:value={crawler.tz} placeholder="UTC" />
{#if fieldErrors.tz}<small class="fe">{fieldErrors.tz}</small>{/if}
</label>
<label class="field">
<span>Idle timeout (seconds)</span>
<input type="number" min="0" bind:value={crawler.idle_timeout_secs} />
</label>
</fieldset>
<fieldset>
<legend>Source</legend>
<label class="field">
<span>Start URL</span>
<input
type="text"
bind:value={crawler.start_url}
placeholder="https://source.example/"
/>
{#if fieldErrors.start_url}<small class="fe">{fieldErrors.start_url}</small>{/if}
</label>
<label class="field">
<span>CDN host</span>
<input type="text" bind:value={crawler.cdn_host} placeholder="cdn.example" />
</label>
<label class="field">
<span>Cookie domain</span>
<input
type="text"
bind:value={crawler.cookie_domain}
placeholder="source.example"
/>
</label>
<label class="field">
<span>User-Agent</span>
<input type="text" bind:value={crawler.user_agent} />
</label>
</fieldset>
<fieldset>
<legend>Rate limiting</legend>
<label class="field">
<span>Request interval (ms)</span>
<input type="number" min="0" bind:value={crawler.rate_ms} />
</label>
<label class="field">
<span>CDN request interval (ms)</span>
<input type="number" min="0" bind:value={crawler.cdn_rate_ms} />
</label>
</fieldset>
<fieldset>
<legend>Workers &amp; retention</legend>
<label class="field">
<span>Chapter workers</span>
<input type="number" min="1" bind:value={crawler.chapter_workers} />
{#if fieldErrors.chapter_workers}<small class="fe"
>{fieldErrors.chapter_workers}</small
>{/if}
</label>
<label class="field">
<span>Job retention (days)</span>
<input type="number" min="0" bind:value={crawler.retention_days} />
</label>
<label class="field">
<span>Manga limit per pass (0 = unlimited)</span>
<input type="number" min="0" bind:value={crawler.manga_limit} />
</label>
</fieldset>
<fieldset>
<legend>Download safety</legend>
<label class="field-inline">
<input type="checkbox" bind:checked={crawler.allow_any_host} />
<span>Allow any host (bypass allowlist; still blocks private IPs)</span>
</label>
<label class="field">
<span>Download allowlist (one host per line)</span>
<textarea
rows="4"
bind:value={allowlistText}
disabled={crawler.allow_any_host}
></textarea>
</label>
<label class="field">
<span>Max image bytes</span>
<input type="number" min="0" bind:value={crawler.max_image_bytes} />
</label>
</fieldset>
<fieldset>
<legend>Reliability</legend>
<label class="field">
<span>Chapter job timeout (seconds)</span>
<input type="number" min="1" bind:value={crawler.job_timeout_secs} />
{#if fieldErrors.job_timeout_secs}<small class="fe"
>{fieldErrors.job_timeout_secs}</small
>{/if}
</label>
<label class="field">
<span>Metadata max consecutive failures</span>
<input
type="number"
min="0"
bind:value={crawler.metadata_max_consecutive_failures}
/>
</label>
<label class="field">
<span>Browser restart threshold</span>
<input type="number" min="0" bind:value={crawler.browser_restart_threshold} />
</label>
</fieldset>
{#if crawlerEnv}
<fieldset class="env">
<legend>Managed via environment (read-only)</legend>
<dl>
<dt>Browser mode</dt>
<dd>{crawlerEnv.browser_mode}</dd>
<dt>Browser args</dt>
<dd>{crawlerEnv.browser_args.join(' ') || '—'}</dd>
<dt>Proxy</dt>
<dd>{crawlerEnv.proxy ?? '—'}</dd>
<dt>TOR control URL</dt>
<dd>{crawlerEnv.tor_control_url ?? '—'}</dd>
<dt>TOR credentials</dt>
<dd>{crawlerEnv.tor_credentials_configured ? 'configured' : '—'}</dd>
<dt>Initial session</dt>
<dd>{crawlerEnv.session_configured ? 'configured' : '—'}</dd>
</dl>
</fieldset>
{/if}
</div>
{:else if tab === 'analysis' && analysis}
<div class="form">
<fieldset>
<legend>Endpoint &amp; model</legend>
<label class="field">
<span>Vision endpoint URL</span>
<input type="text" bind:value={analysis.endpoint} />
{#if fieldErrors.endpoint}<small class="fe">{fieldErrors.endpoint}</small>{/if}
</label>
<label class="field">
<span>Model</span>
<input type="text" bind:value={analysis.model} />
{#if fieldErrors.model}<small class="fe">{fieldErrors.model}</small>{/if}
</label>
<p class="env-note muted">
API key: {apiKeyConfigured
? 'configured via environment'
: 'not set'} (managed via environment).
</p>
</fieldset>
<fieldset>
<legend>Workers &amp; timeouts</legend>
<label class="field">
<span>Workers</span>
<input type="number" min="1" bind:value={analysis.workers} />
{#if fieldErrors.workers}<small class="fe">{fieldErrors.workers}</small>{/if}
</label>
<label class="field">
<span>Request timeout (seconds)</span>
<input type="number" min="1" bind:value={analysis.request_timeout_secs} />
</label>
<label class="field">
<span>Job timeout (seconds)</span>
<input type="number" min="1" bind:value={analysis.job_timeout_secs} />
</label>
</fieldset>
<fieldset>
<legend>Sampling &amp; output</legend>
<label class="field">
<span>Temperature</span>
<input type="number" min="0" step="0.05" bind:value={analysis.temperature} />
{#if fieldErrors.temperature}<small class="fe"
>{fieldErrors.temperature}</small
>{/if}
</label>
<label class="field">
<span>Frequency penalty</span>
<input type="number" step="0.05" bind:value={analysis.frequency_penalty} />
</label>
<label class="field">
<span>Max output tokens</span>
<input type="number" min="1" bind:value={analysis.max_tokens} />
</label>
<label class="field">
<span>Response format</span>
<select bind:value={analysis.response_format}>
<option value="json_schema">json_schema</option>
<option value="json_object">json_object</option>
<option value="none">none</option>
</select>
</label>
</fieldset>
<fieldset>
<legend>Image slicing</legend>
<label class="field">
<span>Max pixels per slice</span>
<input type="number" min="1" bind:value={analysis.max_pixels} />
</label>
<label class="field">
<span>Min slice height (px)</span>
<input type="number" min="1" bind:value={analysis.min_slice_height} />
</label>
<label class="field">
<span>Slice overlap (00.9)</span>
<input
type="number"
min="0"
max="0.9"
step="0.01"
bind:value={analysis.slice_overlap}
/>
{#if fieldErrors.slice_overlap}<small class="fe"
>{fieldErrors.slice_overlap}</small
>{/if}
</label>
<label class="field">
<span>Tall aspect threshold (≥1.0)</span>
<input
type="number"
min="1"
step="0.1"
bind:value={analysis.tall_aspect_threshold}
/>
{#if fieldErrors.tall_aspect_threshold}<small class="fe"
>{fieldErrors.tall_aspect_threshold}</small
>{/if}
</label>
<label class="field">
<span>Max slices</span>
<input type="number" min="1" bind:value={analysis.max_slices} />
</label>
<label class="field">
<span>Max image bytes</span>
<input type="number" min="0" bind:value={analysis.max_image_bytes} />
</label>
</fieldset>
<fieldset>
<legend>Prompts</legend>
{#each [['system_prompt', 'System prompt'], ['ocr_prompt', 'OCR prompt (slices)'], ['grounding_prompt', 'Grounding prompt']] as [key, label] (key)}
{@const k = key as 'system_prompt' | 'ocr_prompt' | 'grounding_prompt'}
<div class="prompt">
<div class="prompt-head">
<span>{label}</span>
<button
type="button"
class="link"
disabled={analysis[k] == null}
onclick={() => resetPrompt(k)}>Reset to default</button
>
</div>
<textarea
rows="6"
class="mono"
value={analysis[k] ?? ''}
placeholder={promptDefaults?.[k] ?? ''}
oninput={(e) => {
const v = (e.currentTarget as HTMLTextAreaElement).value;
if (analysis) analysis[k] = v.trim() === '' ? null : v;
}}
></textarea>
<small class="muted">
{analysis[k] == null
? 'Using compiled default (shown as placeholder).'
: `${analysis[k]?.length ?? 0} chars (override).`}
</small>
</div>
{/each}
</fieldset>
</div>
{/if}
{/if}
<Modal
open={confirmOpen}
title="Apply settings?"
onClose={() => (confirmOpen = false)}
size="sm"
closeOnBackdrop
testid="settings-confirm"
>
<p>
Saving restarts the {tab} subsystem with the new settings. In-flight jobs are leased and
will be retried, but any work in progress is interrupted briefly.
</p>
{#snippet footer()}
<button type="button" onclick={() => (confirmOpen = false)}>Cancel</button>
<button type="button" class="primary" onclick={save} data-testid="settings-confirm-apply">
Save &amp; apply
</button>
{/snippet}
</Modal>
<style>
h1 {
margin: 0 0 var(--space-2) 0;
}
.lead {
margin: 0 0 var(--space-4) 0;
font-size: var(--font-sm);
max-width: 60ch;
}
.muted {
color: var(--text-muted);
}
.tabs {
display: flex;
gap: var(--space-2);
margin-bottom: var(--space-4);
border-bottom: 1px solid var(--border);
}
.tabs button {
padding: var(--space-2) var(--space-3);
background: none;
border: none;
border-bottom: 2px solid transparent;
color: var(--text-muted);
cursor: pointer;
font-size: var(--font-sm);
}
.tabs button.active {
color: var(--text);
border-bottom-color: var(--primary);
font-weight: var(--weight-semibold);
}
.sticky-bar {
position: sticky;
top: calc(var(--app-header-h) + var(--space-2));
z-index: 1;
display: flex;
align-items: center;
gap: var(--space-3);
padding: var(--space-3);
margin-bottom: var(--space-3);
background: var(--surface-elevated);
border: 1px solid var(--border);
border-radius: var(--radius-md);
}
.sticky-bar .spacer {
flex: 1;
}
.toggle {
display: inline-flex;
align-items: center;
gap: var(--space-2);
font-size: var(--font-sm);
}
.form {
display: grid;
gap: var(--space-4);
}
fieldset {
border: 1px solid var(--border);
border-radius: var(--radius-md);
padding: var(--space-3);
display: grid;
gap: var(--space-3);
}
legend {
padding: 0 var(--space-2);
font-weight: var(--weight-semibold);
font-size: var(--font-sm);
}
.field {
display: grid;
gap: var(--space-1);
}
.field > span {
font-size: var(--font-sm);
color: var(--text-muted);
}
.field input,
.field select,
.field textarea {
padding: var(--space-2) var(--space-3);
border: 1px solid var(--border);
border-radius: var(--radius-md);
background: var(--surface);
color: var(--text);
width: 100%;
box-sizing: border-box;
}
.field-inline {
display: inline-flex;
align-items: center;
gap: var(--space-2);
font-size: var(--font-sm);
}
.fe {
color: var(--danger, #dc2626);
font-size: var(--font-xs);
}
.mono {
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
font-size: var(--font-xs);
width: 100%;
box-sizing: border-box;
padding: var(--space-2);
border: 1px solid var(--border);
border-radius: var(--radius-md);
background: var(--surface);
color: var(--text);
}
.prompt {
display: grid;
gap: var(--space-1);
}
.prompt-head {
display: flex;
justify-content: space-between;
align-items: center;
font-size: var(--font-sm);
}
button.link {
background: none;
border: none;
color: var(--primary);
cursor: pointer;
font-size: var(--font-xs);
padding: 0;
}
button.link:disabled {
color: var(--text-muted);
cursor: not-allowed;
}
.env dl {
display: grid;
grid-template-columns: max-content 1fr;
gap: var(--space-1) var(--space-3);
margin: 0;
font-size: var(--font-sm);
}
.env dt {
color: var(--text-muted);
}
.env dd {
margin: 0;
word-break: break-all;
}
.env-note {
font-size: var(--font-xs);
margin: 0;
}
.error {
color: var(--danger, #dc2626);
padding: var(--space-2) var(--space-3);
border: 1px solid var(--danger, #dc2626);
border-radius: var(--radius-md);
margin-bottom: var(--space-3);
}
.notice {
color: var(--success, #16a34a);
padding: var(--space-2) var(--space-3);
border: 1px solid var(--success, #16a34a);
border-radius: var(--radius-md);
margin-bottom: var(--space-3);
}
button.primary {
padding: var(--space-2) var(--space-3);
border-radius: var(--radius-md);
border: 1px solid var(--primary);
background: var(--primary);
color: white;
cursor: pointer;
}
button.primary:disabled {
opacity: 0.5;
cursor: not-allowed;
}
</style>