83c2899373
Adds an in-process ocrs OCR backend as the active analysis engine (vision left dormant), enables OCR text search on the page-tag aggregation endpoints, and reshapes the admin analysis/settings UI to present the OCR-only surface. Bumps version 0.89.0 -> 0.90.0. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
508 lines
18 KiB
Svelte
508 lines
18 KiB
Svelte
<script lang="ts">
|
|
import { onMount } from 'svelte';
|
|
import Modal from '$lib/components/Modal.svelte';
|
|
import { ApiError } from '$lib/api/client';
|
|
import {
|
|
getCrawlerSettings,
|
|
updateCrawlerSettings,
|
|
getAnalysisSettings,
|
|
updateAnalysisSettings,
|
|
type CrawlerSettings,
|
|
type CrawlerEnvOnly,
|
|
type AnalysisSettings
|
|
} from '$lib/api/admin';
|
|
|
|
type Tab = 'crawler' | 'analysis';
|
|
let tab = $state<Tab>('crawler');
|
|
|
|
// Crawler state
|
|
let crawler = $state<CrawlerSettings | null>(null);
|
|
let crawlerEnv = $state<CrawlerEnvOnly | null>(null);
|
|
let allowlistText = $state(''); // textarea mirror of download_allowlist
|
|
|
|
// Analysis state. The full settings object is loaded and saved as-is;
|
|
// vision-only fields (endpoint, model, prompts, slicing, sampling) ride
|
|
// along unchanged while the OCR backend is active — only the OCR-relevant
|
|
// knobs are surfaced below.
|
|
let analysis = $state<AnalysisSettings | null>(null);
|
|
|
|
let loading = $state(true);
|
|
let loadError = $state<string | null>(null);
|
|
let saving = $state(false);
|
|
let saveError = $state<string | null>(null);
|
|
let success = $state<string | null>(null);
|
|
// field -> message, parsed from a 422 validation_failed response
|
|
let fieldErrors = $state<Record<string, string>>({});
|
|
let confirmOpen = $state(false);
|
|
|
|
onMount(load);
|
|
|
|
async function load() {
|
|
loading = true;
|
|
loadError = null;
|
|
try {
|
|
const [c, a] = await Promise.all([getCrawlerSettings(), getAnalysisSettings()]);
|
|
crawler = c.editable;
|
|
crawlerEnv = c.env_only;
|
|
allowlistText = c.editable.download_allowlist.join('\n');
|
|
analysis = a.editable;
|
|
} catch (e) {
|
|
loadError = e instanceof Error ? e.message : 'Failed to load settings.';
|
|
} finally {
|
|
loading = false;
|
|
}
|
|
}
|
|
|
|
function parseFieldErrors(e: unknown): Record<string, string> {
|
|
const out: Record<string, string> = {};
|
|
if (
|
|
e instanceof ApiError &&
|
|
e.details &&
|
|
typeof e.details === 'object' &&
|
|
'fields' in e.details
|
|
) {
|
|
const fields = (e.details as { fields?: { field: string; message: string }[] }).fields;
|
|
for (const f of fields ?? []) out[f.field] = f.message;
|
|
}
|
|
return out;
|
|
}
|
|
|
|
function askSave() {
|
|
saveError = null;
|
|
success = null;
|
|
fieldErrors = {};
|
|
confirmOpen = true;
|
|
}
|
|
|
|
async function save() {
|
|
confirmOpen = false;
|
|
saving = true;
|
|
saveError = null;
|
|
success = null;
|
|
fieldErrors = {};
|
|
try {
|
|
if (tab === 'crawler' && crawler) {
|
|
// Sync the allowlist textarea back into the model.
|
|
crawler.download_allowlist = allowlistText
|
|
.split('\n')
|
|
.map((s) => s.trim())
|
|
.filter(Boolean);
|
|
const r = await updateCrawlerSettings(crawler);
|
|
crawler = r.editable;
|
|
crawlerEnv = r.env_only;
|
|
allowlistText = r.editable.download_allowlist.join('\n');
|
|
} else if (tab === 'analysis' && analysis) {
|
|
const r = await updateAnalysisSettings(analysis);
|
|
analysis = r.editable;
|
|
}
|
|
success = 'Saved and applied. The subsystem was restarted with the new settings.';
|
|
} catch (e) {
|
|
fieldErrors = parseFieldErrors(e);
|
|
saveError =
|
|
Object.keys(fieldErrors).length > 0
|
|
? 'Some fields need fixing.'
|
|
: e instanceof Error
|
|
? e.message
|
|
: 'Save failed.';
|
|
} finally {
|
|
saving = false;
|
|
}
|
|
}
|
|
|
|
</script>
|
|
|
|
<h1>Settings</h1>
|
|
<p class="lead muted">
|
|
Edit crawler and analysis configuration. Saving applies immediately — the affected daemon is
|
|
gracefully restarted, so in-flight jobs are retried rather than lost. Host-level and secret
|
|
values stay environment-managed and are shown read-only.
|
|
</p>
|
|
|
|
<div class="tabs" role="tablist">
|
|
<button
|
|
role="tab"
|
|
aria-selected={tab === 'crawler'}
|
|
class:active={tab === 'crawler'}
|
|
onclick={() => (tab = 'crawler')}
|
|
data-testid="settings-tab-crawler">Crawler</button
|
|
>
|
|
<button
|
|
role="tab"
|
|
aria-selected={tab === 'analysis'}
|
|
class:active={tab === 'analysis'}
|
|
onclick={() => (tab = 'analysis')}
|
|
data-testid="settings-tab-analysis">Analysis</button
|
|
>
|
|
</div>
|
|
|
|
{#if loading}
|
|
<p class="muted">Loading…</p>
|
|
{:else if loadError}
|
|
<p class="error" role="alert">{loadError}</p>
|
|
{:else}
|
|
<div class="sticky-bar">
|
|
{#if tab === 'crawler' && crawler}
|
|
<label class="toggle">
|
|
<input
|
|
type="checkbox"
|
|
bind:checked={crawler.daemon_enabled}
|
|
data-testid="settings-crawler-enabled"
|
|
/>
|
|
<span>Crawler daemon {crawler.daemon_enabled ? 'enabled' : 'disabled'}</span>
|
|
</label>
|
|
{:else if tab === 'analysis' && analysis}
|
|
<label class="toggle">
|
|
<input
|
|
type="checkbox"
|
|
bind:checked={analysis.enabled}
|
|
data-testid="settings-analysis-enabled"
|
|
/>
|
|
<span>Analysis worker {analysis.enabled ? 'enabled' : 'disabled'}</span>
|
|
</label>
|
|
{/if}
|
|
<div class="spacer"></div>
|
|
<button class="primary" disabled={saving} onclick={askSave} data-testid="settings-save">
|
|
{saving ? 'Saving…' : 'Save & apply'}
|
|
</button>
|
|
</div>
|
|
|
|
{#if success}<p class="notice" role="status">{success}</p>{/if}
|
|
{#if saveError}<p class="error" role="alert">{saveError}</p>{/if}
|
|
|
|
{#if tab === 'crawler' && crawler}
|
|
<div class="form">
|
|
<fieldset>
|
|
<legend>Schedule</legend>
|
|
<label class="field">
|
|
<span>Daily metadata pass (HH:MM)</span>
|
|
<input type="time" bind:value={crawler.daily_at} />
|
|
{#if fieldErrors.daily_at}<small class="fe">{fieldErrors.daily_at}</small>{/if}
|
|
</label>
|
|
<label class="field">
|
|
<span>Timezone (IANA)</span>
|
|
<input type="text" bind:value={crawler.tz} placeholder="UTC" />
|
|
{#if fieldErrors.tz}<small class="fe">{fieldErrors.tz}</small>{/if}
|
|
</label>
|
|
<label class="field">
|
|
<span>Idle timeout (seconds)</span>
|
|
<input type="number" min="0" bind:value={crawler.idle_timeout_secs} />
|
|
</label>
|
|
</fieldset>
|
|
|
|
<fieldset>
|
|
<legend>Source</legend>
|
|
<label class="field">
|
|
<span>Start URL</span>
|
|
<input
|
|
type="text"
|
|
bind:value={crawler.start_url}
|
|
placeholder="https://source.example/"
|
|
/>
|
|
{#if fieldErrors.start_url}<small class="fe">{fieldErrors.start_url}</small>{/if}
|
|
</label>
|
|
<label class="field">
|
|
<span>CDN host</span>
|
|
<input type="text" bind:value={crawler.cdn_host} placeholder="cdn.example" />
|
|
</label>
|
|
<label class="field">
|
|
<span>Cookie domain</span>
|
|
<input
|
|
type="text"
|
|
bind:value={crawler.cookie_domain}
|
|
placeholder="source.example"
|
|
/>
|
|
</label>
|
|
<label class="field">
|
|
<span>User-Agent</span>
|
|
<input type="text" bind:value={crawler.user_agent} />
|
|
</label>
|
|
</fieldset>
|
|
|
|
<fieldset>
|
|
<legend>Rate limiting</legend>
|
|
<label class="field">
|
|
<span>Request interval (ms)</span>
|
|
<input type="number" min="0" bind:value={crawler.rate_ms} />
|
|
</label>
|
|
<label class="field">
|
|
<span>CDN request interval (ms)</span>
|
|
<input type="number" min="0" bind:value={crawler.cdn_rate_ms} />
|
|
</label>
|
|
</fieldset>
|
|
|
|
<fieldset>
|
|
<legend>Workers & retention</legend>
|
|
<label class="field">
|
|
<span>Chapter workers</span>
|
|
<input type="number" min="1" bind:value={crawler.chapter_workers} />
|
|
{#if fieldErrors.chapter_workers}<small class="fe"
|
|
>{fieldErrors.chapter_workers}</small
|
|
>{/if}
|
|
</label>
|
|
<label class="field">
|
|
<span>Job retention (days)</span>
|
|
<input type="number" min="0" bind:value={crawler.retention_days} />
|
|
</label>
|
|
<label class="field">
|
|
<span>Manga limit per pass (0 = unlimited)</span>
|
|
<input type="number" min="0" bind:value={crawler.manga_limit} />
|
|
</label>
|
|
</fieldset>
|
|
|
|
<fieldset>
|
|
<legend>Download safety</legend>
|
|
<label class="field-inline">
|
|
<input type="checkbox" bind:checked={crawler.allow_any_host} />
|
|
<span>Allow any host (bypass allowlist; still blocks private IPs)</span>
|
|
</label>
|
|
<label class="field">
|
|
<span>Download allowlist (one host per line)</span>
|
|
<textarea
|
|
rows="4"
|
|
bind:value={allowlistText}
|
|
disabled={crawler.allow_any_host}
|
|
></textarea>
|
|
</label>
|
|
<label class="field">
|
|
<span>Max image bytes</span>
|
|
<input type="number" min="0" bind:value={crawler.max_image_bytes} />
|
|
</label>
|
|
</fieldset>
|
|
|
|
<fieldset>
|
|
<legend>Reliability</legend>
|
|
<label class="field">
|
|
<span>Chapter job timeout (seconds)</span>
|
|
<input type="number" min="1" bind:value={crawler.job_timeout_secs} />
|
|
{#if fieldErrors.job_timeout_secs}<small class="fe"
|
|
>{fieldErrors.job_timeout_secs}</small
|
|
>{/if}
|
|
</label>
|
|
<label class="field">
|
|
<span>Metadata max consecutive failures</span>
|
|
<input
|
|
type="number"
|
|
min="0"
|
|
bind:value={crawler.metadata_max_consecutive_failures}
|
|
/>
|
|
</label>
|
|
<label class="field">
|
|
<span>Browser restart threshold</span>
|
|
<input type="number" min="0" bind:value={crawler.browser_restart_threshold} />
|
|
</label>
|
|
</fieldset>
|
|
|
|
{#if crawlerEnv}
|
|
<fieldset class="env">
|
|
<legend>Managed via environment (read-only)</legend>
|
|
<dl>
|
|
<dt>Browser mode</dt>
|
|
<dd>{crawlerEnv.browser_mode}</dd>
|
|
<dt>Browser args</dt>
|
|
<dd>{crawlerEnv.browser_args.join(' ') || '—'}</dd>
|
|
<dt>Proxy</dt>
|
|
<dd>{crawlerEnv.proxy ?? '—'}</dd>
|
|
<dt>TOR control URL</dt>
|
|
<dd>{crawlerEnv.tor_control_url ?? '—'}</dd>
|
|
<dt>TOR credentials</dt>
|
|
<dd>{crawlerEnv.tor_credentials_configured ? 'configured' : '—'}</dd>
|
|
<dt>Initial session</dt>
|
|
<dd>{crawlerEnv.session_configured ? 'configured' : '—'}</dd>
|
|
</dl>
|
|
</fieldset>
|
|
{/if}
|
|
</div>
|
|
{:else if tab === 'analysis' && analysis}
|
|
<div class="form">
|
|
<fieldset>
|
|
<legend>Engine</legend>
|
|
<p class="env-note muted">
|
|
Backend: OCR (ocrs) — in-process text extraction. Selected via
|
|
environment; vision is temporarily disabled.
|
|
</p>
|
|
</fieldset>
|
|
|
|
<fieldset>
|
|
<legend>Worker</legend>
|
|
<label class="field">
|
|
<span>Workers</span>
|
|
<input type="number" min="1" bind:value={analysis.workers} />
|
|
{#if fieldErrors.workers}<small class="fe">{fieldErrors.workers}</small>{/if}
|
|
</label>
|
|
<label class="field">
|
|
<span>Job timeout (seconds)</span>
|
|
<input type="number" min="1" bind:value={analysis.job_timeout_secs} />
|
|
</label>
|
|
<label class="field">
|
|
<span>Max image bytes</span>
|
|
<input type="number" min="0" bind:value={analysis.max_image_bytes} />
|
|
</label>
|
|
</fieldset>
|
|
</div>
|
|
{/if}
|
|
{/if}
|
|
|
|
<Modal
|
|
open={confirmOpen}
|
|
title="Apply settings?"
|
|
onClose={() => (confirmOpen = false)}
|
|
size="sm"
|
|
closeOnBackdrop
|
|
testid="settings-confirm"
|
|
>
|
|
<p>
|
|
Saving restarts the {tab} subsystem with the new settings. In-flight jobs are leased and
|
|
will be retried, but any work in progress is interrupted briefly.
|
|
</p>
|
|
{#snippet footer()}
|
|
<button type="button" onclick={() => (confirmOpen = false)}>Cancel</button>
|
|
<button type="button" class="primary" onclick={save} data-testid="settings-confirm-apply">
|
|
Save & apply
|
|
</button>
|
|
{/snippet}
|
|
</Modal>
|
|
|
|
<style>
|
|
h1 {
|
|
margin: 0 0 var(--space-2) 0;
|
|
}
|
|
.lead {
|
|
margin: 0 0 var(--space-4) 0;
|
|
font-size: var(--font-sm);
|
|
max-width: 60ch;
|
|
}
|
|
.muted {
|
|
color: var(--text-muted);
|
|
}
|
|
.tabs {
|
|
display: flex;
|
|
gap: var(--space-2);
|
|
margin-bottom: var(--space-4);
|
|
border-bottom: 1px solid var(--border);
|
|
}
|
|
.tabs button {
|
|
padding: var(--space-2) var(--space-3);
|
|
background: none;
|
|
border: none;
|
|
border-bottom: 2px solid transparent;
|
|
color: var(--text-muted);
|
|
cursor: pointer;
|
|
font-size: var(--font-sm);
|
|
}
|
|
.tabs button.active {
|
|
color: var(--text);
|
|
border-bottom-color: var(--primary);
|
|
font-weight: var(--weight-semibold);
|
|
}
|
|
.sticky-bar {
|
|
position: sticky;
|
|
top: calc(var(--app-header-h) + var(--space-2));
|
|
z-index: 1;
|
|
display: flex;
|
|
align-items: center;
|
|
gap: var(--space-3);
|
|
padding: var(--space-3);
|
|
margin-bottom: var(--space-3);
|
|
background: var(--surface-elevated);
|
|
border: 1px solid var(--border);
|
|
border-radius: var(--radius-md);
|
|
}
|
|
.sticky-bar .spacer {
|
|
flex: 1;
|
|
}
|
|
.toggle {
|
|
display: inline-flex;
|
|
align-items: center;
|
|
gap: var(--space-2);
|
|
font-size: var(--font-sm);
|
|
}
|
|
.form {
|
|
display: grid;
|
|
gap: var(--space-4);
|
|
}
|
|
fieldset {
|
|
border: 1px solid var(--border);
|
|
border-radius: var(--radius-md);
|
|
padding: var(--space-3);
|
|
display: grid;
|
|
gap: var(--space-3);
|
|
}
|
|
legend {
|
|
padding: 0 var(--space-2);
|
|
font-weight: var(--weight-semibold);
|
|
font-size: var(--font-sm);
|
|
}
|
|
.field {
|
|
display: grid;
|
|
gap: var(--space-1);
|
|
}
|
|
.field > span {
|
|
font-size: var(--font-sm);
|
|
color: var(--text-muted);
|
|
}
|
|
.field input,
|
|
.field textarea {
|
|
padding: var(--space-2) var(--space-3);
|
|
border: 1px solid var(--border);
|
|
border-radius: var(--radius-md);
|
|
background: var(--surface);
|
|
color: var(--text);
|
|
width: 100%;
|
|
box-sizing: border-box;
|
|
}
|
|
.field-inline {
|
|
display: inline-flex;
|
|
align-items: center;
|
|
gap: var(--space-2);
|
|
font-size: var(--font-sm);
|
|
}
|
|
.fe {
|
|
color: var(--danger, #dc2626);
|
|
font-size: var(--font-xs);
|
|
}
|
|
.env dl {
|
|
display: grid;
|
|
grid-template-columns: max-content 1fr;
|
|
gap: var(--space-1) var(--space-3);
|
|
margin: 0;
|
|
font-size: var(--font-sm);
|
|
}
|
|
.env dt {
|
|
color: var(--text-muted);
|
|
}
|
|
.env dd {
|
|
margin: 0;
|
|
word-break: break-all;
|
|
}
|
|
.env-note {
|
|
font-size: var(--font-xs);
|
|
margin: 0;
|
|
}
|
|
.error {
|
|
color: var(--danger, #dc2626);
|
|
padding: var(--space-2) var(--space-3);
|
|
border: 1px solid var(--danger, #dc2626);
|
|
border-radius: var(--radius-md);
|
|
margin-bottom: var(--space-3);
|
|
}
|
|
.notice {
|
|
color: var(--success, #16a34a);
|
|
padding: var(--space-2) var(--space-3);
|
|
border: 1px solid var(--success, #16a34a);
|
|
border-radius: var(--radius-md);
|
|
margin-bottom: var(--space-3);
|
|
}
|
|
button.primary {
|
|
padding: var(--space-2) var(--space-3);
|
|
border-radius: var(--radius-md);
|
|
border: 1px solid var(--primary);
|
|
background: var(--primary);
|
|
color: white;
|
|
cursor: pointer;
|
|
}
|
|
button.primary:disabled {
|
|
opacity: 0.5;
|
|
cursor: not-allowed;
|
|
}
|
|
</style>
|