From 83c28993737e5c1f2b5dab72a7d1c9a6e6ad2446 Mon Sep 17 00:00:00 2001 From: MechaCat02 Date: Tue, 30 Jun 2026 19:52:43 +0200 Subject: [PATCH] feat(analysis): add ocrs OCR backend and OCR-driven page text search Adds an in-process ocrs OCR backend as the active analysis engine (vision left dormant), enables OCR text search on the page-tag aggregation endpoints, and reshapes the admin analysis/settings UI to present the OCR-only surface. Bumps version 0.89.0 -> 0.90.0. Co-Authored-By: Claude Opus 4.8 --- .env.example | 31 ++- HYBRID-OCR-VISION.md | 136 +++++++++ backend/.gitignore | 4 + backend/Cargo.lock | 258 ++++++++++++++++-- backend/Cargo.toml | 4 +- backend/Dockerfile | 14 + backend/src/analysis/mod.rs | 1 + backend/src/analysis/ocr.rs | 201 ++++++++++++++ backend/src/api/page_tags.rs | 27 +- backend/src/app.rs | 151 ++++++---- backend/src/config.rs | 118 ++++++++ backend/src/error.rs | 8 +- backend/src/repo/page_tag.rs | 121 ++++++-- backend/src/settings.rs | 53 +++- backend/tests/analysis_ocr.rs | 109 ++++++++ backend/tests/api_page_tags.rs | 115 +++++++- docker-compose.yml | 9 +- frontend/package.json | 2 +- .../analysis/AnalysisHistoryTable.svelte | 24 -- .../AnalysisHistoryTable.svelte.test.ts | 64 +++++ .../analysis/AnalysisMetricsPanel.svelte | 47 ---- .../AnalysisMetricsPanel.svelte.test.ts | 40 +++ .../src/routes/admin/analysis/+page.svelte | 66 +---- .../src/routes/admin/settings/+page.svelte | 175 +----------- 24 files changed, 1335 insertions(+), 443 deletions(-) create mode 100644 HYBRID-OCR-VISION.md create mode 100644 backend/src/analysis/ocr.rs create mode 100644 backend/tests/analysis_ocr.rs create mode 100644 frontend/src/lib/components/analysis/AnalysisHistoryTable.svelte.test.ts create mode 100644 frontend/src/lib/components/analysis/AnalysisMetricsPanel.svelte.test.ts diff --git a/.env.example b/.env.example index 33d7c57..7ffb215 100644 --- a/.env.example +++ b/.env.example @@ -62,11 +62,16 @@ CORS_ALLOWED_ORIGINS= # neither Origin nor Referer (curl, server-to-server callers) are # always allowed. # -# Default is empty: CSRF check disabled (operator opt-out). For a -# browser-exposed deployment this should be set to the SvelteKit -# origin, e.g. https://app.example.com. For a same-origin -# docker-compose deploy where only one origin exists, set the same -# value the browser uses. +# Empty does NOT mean "off" for browsers: cookie-authenticated admin +# mutations FAIL CLOSED (403) when this is empty, since there's no +# allowlist to check the Origin against. Non-cookie callers (curl, +# bots with a Bearer token, no Origin/Referer) are still allowed. +# So set this to the SvelteKit origin for any browser-exposed deploy, +# e.g. https://app.example.com. For a same-origin docker-compose deploy +# set the same value the browser uses. +# Local dev (native `npm run dev`): the Vite origin is +# http://localhost:5173 — set ADMIN_ALLOWED_ORIGINS=http://localhost:5173 +# or admin toggles (e.g. enabling the analysis worker) return 403. ADMIN_ALLOWED_ORIGINS= # ----- Admin bootstrap ----- @@ -277,6 +282,22 @@ CRAWLER_TZ=UTC # ANALYSIS_ENABLED Turn the worker on at first boot. Toggleable live in # the dashboard. Default `false`. ANALYSIS_ENABLED=false +# ANALYSIS_BACKEND Which engine the worker runs. env-ONLY (deploy-time). +# `ocr` (default) = the in-process ocrs engine: fast, +# CPU-only, text-only, ideal for a Pi. +# NOTE: the `vision` backend (local LLM at ANALYSIS_VISION_URL, +# full OCR + tags + scene + safety) is TEMPORARILY DISABLED — +# its code is kept intact but the worker forces OCR regardless, +# logging a warning if `vision` is requested. So setting +# `vision` here currently has no effect; the ANALYSIS_VISION_* / +# ANALYSIS_API_KEY knobs below are dormant until it's re-enabled. +ANALYSIS_BACKEND=ocr +# OCRS_DETECTION_MODEL / OCRS_RECOGNITION_MODEL Paths to the ocrs `.rten` +# text-detection / -recognition models (only read when ANALYSIS_BACKEND=ocr). +# The backend image bakes both into /models, so the defaults work unchanged; +# override only to point at custom-trained models. +OCRS_DETECTION_MODEL=/models/text-detection.rten +OCRS_RECOGNITION_MODEL=/models/text-recognition.rten # ANALYSIS_VISION_URL /v1/chat/completions endpoint. Required when enabled. # For the bundled vision container, use # http://mangalord-vision:8000/v1/chat/completions. diff --git a/HYBRID-OCR-VISION.md b/HYBRID-OCR-VISION.md new file mode 100644 index 0000000..80ac5d0 --- /dev/null +++ b/HYBRID-OCR-VISION.md @@ -0,0 +1,136 @@ +# Hybrid OCR→Vision analysis pipeline — design brief for the dev agent + +**Status: design only (not implemented).** Forward-looking model for a future +`ANALYSIS_BACKEND=hybrid`, on top of the shipped `ocr` (default) and `vision` +backends. Constraint that shapes it: `ocrs` and the vision LLM must **not** be +resident at the same time (Pi RAM ceiling). OCR has priority; vision is a +preemptible, only-when-OCR-is-drained background phase. + +## The key realization: this is the crawl mutex, again + +The repo already runs a **`vision-manager`** sidecar +([vision-manager/manager.sh](vision-manager/manager.sh), +[VISION-AUTOSCALE.md](VISION-AUTOSCALE.md)) that owns the `mangalord-vision` +(llama.cpp, ~4 GiB) container lifecycle via a scoped docker-socket-proxy — the +backend gets no Docker access. It already: + +- **Starts** vision when `pending_analysis()` > 0 (counts `analyze_page` jobs). +- **Idle-stops** it after `STOP_DEBOUNCE` (anti-thrash) once the queue drains. +- **Defers starting** vision while `crawl_running()` > 0 — the **"RAM mutex"**: + the crawler's browser and the LLM can't both fit, so vision waits for crawl. +- **Memory-yield**: SIGTERMs a running vision at a HIGH host-RAM watermark, + blocks starting at a LOW watermark (see [VISION-MEMORY-YIELD.md](VISION-MEMORY-YIELD.md)). + +The OCR↔vision relationship is the relationship that already exists between +crawl and vision. OCR is just a third RAM-competing, higher-priority workload +that vision must defer to. Extend the existing arbiter rather than build a new one. + +## Job model: split one queue into two + +Today there is one job kind, `analyze_page`. Split it: + +- **`ocr_page`** — the in-process ocrs pass (cheap, fast, no container). +- **`ground_page`** — the vision grounding pass (needs the LLM container). + +**Pipeline (hybrid mode):** upload → enqueue `ocr_page`. The OCR daemon runs +ocrs, writes `page_ocr_text` + `search_doc` (text searchable *immediately*), then +enqueues `ground_page` for that page. The grounding daemon later adds +tags/scene/safety. Each kind gets its own dedup index (mirror migration 0031). + +Two daemons, two queues. + +## Arbitration: OCR priority + memory exclusivity + +Responsibilities split cleanly between the in-backend daemons and the manager: + +**Backend — OCR daemon** (already built): leases `ocr_page`, always allowed to +run (ocrs is small/local). This is the priority phase. + +**Backend — grounding daemon:** leases `ground_page`, but gates leasing on +**both**: +1. `VisionReadiness` (the existing `/health` gate — vision container up), AND +2. **`ocr_backlog == 0`** (a cheap `count(ocr_page WHERE state IN pending,running)`). + +Gate #2 is the whole priority mechanism, in-process, no aborts: the instant OCR +work appears, the grounding daemon **finishes its current page** (the in-flight +lease completes normally) and then **stops leasing** new pages and parks — "it +finishes, then yields." It resumes only when OCR has fully drained. + +**vision-manager:** two one-line changes to the existing logic: +1. `pending_analysis()` counts **`ground_page`** (not `ocr_page`) — vision only + starts when there is *grounding* work. +2. Add an **OCR start-block mutex** identical to `RESPECT_CRAWL_MUTEX`: defer + starting vision while `ocr_page` backlog > 0. (`RESPECT_OCR_MUTEX=1`.) + +**Why this yields memory exclusivity:** while OCR backlog > 0, the grounding +daemon won't lease → vision goes idle → the manager's idle-debounce stops the +container (and never restarts it under the OCR mutex). Only ocrs (small) is +resident. When OCR drains, grounding resumes, `pending_analysis()` > 0 again, the +manager starts vision. Only the *big* consumer (the LLM container) is ever +mutually exclusive with active OCR — which is the actual RAM constraint. + +**The one overlap window:** if OCR work arrives mid-grounding-page, ocrs (small, +in-process) briefly coexists with the one in-flight vision page before the +grounding daemon parks. ocrs's footprint makes this a non-issue in practice. If a +deployment needs *hard* exclusivity even there, the OCR daemon can additionally +wait for `vision_running == false` before its first dispatch. No deadlock: OCR's +wait is transient (one grounding page, bounded by `job_timeout`), while vision's +deferral on OCR backlog is the persistent, priority-respecting side. + +**Anti-thrash:** the manager's `STOP_DEBOUNCE` already prevents a stray OCR page +from cold-cycling the 4 GiB model. Chapter uploads arrive as bursts, so OCR +drains a whole batch in one phase before vision resumes — the natural good case. + +## Data-model split (the real refactor) + +`persist_analysis` currently writes OCR + tags + scene + safety in one +transaction and derives `search_doc` from the OCR rows (+ scene weight C). Two +passes means splitting it, sharing the tsvector builder: + +- **`persist_ocr(page, lines)`** — writes `page_ocr_text`, sets + `search_doc` from OCR (A/B/D buckets), marks the page text-searchable. Status + reflects OCR completion. +- **`persist_grounding(page, tags, scene, safety)`** — writes auto-tags, + warnings, scene; **recomputes** `search_doc` to fold in the scene (weight C) on + top of the existing OCR buckets. Records grounding completion (e.g. a + `grounded_at` column or tags-present sentinel). + +This keeps text search live after the cheap OCR phase, with semantic search/tags +arriving after the (deferred) grounding phase. + +## Reusable pieces + +- OCR side: the shipped `OcrsEngine` + `OcrAnalyzeDispatcher` + ([backend/src/analysis/ocr.rs](backend/src/analysis/ocr.rs)) — retarget to + `ocr_page`, and on success enqueue `ground_page`. +- Vision side: factor `VisionClient::ground(image, mime, ocr_text)` out of + `analyze()`'s Pass-B block + ([backend/src/analysis/vision.rs](backend/src/analysis/vision.rs) ~240–261) — + pure refactor, existing tests cover it. The grounding daemon is the existing + analysis daemon retargeted to `ground_page` with the extra `ocr_backlog==0` + lease gate. +- Manager: `pending_analysis()` kind swap + `RESPECT_OCR_MUTEX` block, mirroring + the crawl-mutex branch already at [vision-manager/manager.sh](vision-manager/manager.sh). + +## Scope / sequencing + +Larger than an inline hybrid dispatcher: job-kind split + dedup migrations, +`persist_*` split + search_doc-builder extraction, pipeline enqueue (OCR→ground), +grounding daemon lease gate, and the manager mutex. No new privileged surface +(reuses the docker-socket-proxy). Suggested order: +1. Split `persist_analysis` → `persist_ocr` / `persist_grounding` (+ tests). +2. Split job kinds + dedup migrations; retarget the OCR daemon to `ocr_page`. +3. `VisionClient::ground()` refactor + grounding daemon (`ground_page`, gated on + `ocr_backlog==0`). +4. OCR→ground enqueue on OCR success (hybrid mode only). +5. vision-manager: `ground_page` counting + `RESPECT_OCR_MUTEX`. + +Pure-`ocr` (shipped default) and `vision` (legacy) backends are unaffected; this +is the `hybrid` backend's runtime model. + +## Open decision + +Hard exclusivity in the one overlap window (OCR daemon waits for `vision_running +== false` before its first dispatch) — needed only if ocrs's few-hundred-MB +footprint can't coexist with a single in-flight grounding page on the target +Pi's RAM. Default: don't gate (rely on the grounding daemon parking fast). diff --git a/backend/.gitignore b/backend/.gitignore index b691d12..68a188c 100644 --- a/backend/.gitignore +++ b/backend/.gitignore @@ -1,3 +1,7 @@ /target /.sqlx .env + +# Local OCR models for native dev (downloaded, not source) +models/ +*.rten diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 77e9e38..34c09f0 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -214,6 +214,12 @@ version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + [[package]] name = "bitflags" version = "2.11.1" @@ -514,6 +520,25 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-queue" version = "0.3.12" @@ -638,7 +663,7 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" dependencies = [ - "bitflags", + "bitflags 2.11.1", "objc2", ] @@ -772,6 +797,16 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "flatbuffers" +version = "24.12.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f1baf0dbf96932ec9a3038d57900329c015b0bfb7b63d904f3bc27e2b02a096" +dependencies = [ + "bitflags 1.3.2", + "rustc_version", +] + [[package]] name = "flate2" version = "1.1.9" @@ -1059,6 +1094,12 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + [[package]] name = "hex" version = "0.4.3" @@ -1448,7 +1489,7 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e02f3bb43d335493c96bf3fd3a321600bf6bd07ed34bc64118e9293bdffea46c" dependencies = [ - "bitflags", + "bitflags 2.11.1", "libc", "plain", "redox_syscall 0.7.5", @@ -1517,7 +1558,7 @@ checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" [[package]] name = "mangalord" -version = "0.89.0" +version = "0.90.0" dependencies = [ "anyhow", "argon2", @@ -1537,8 +1578,10 @@ dependencies = [ "infer", "mime", "nix 0.29.0", + "ocrs", "rand 0.8.6", "reqwest", + "rten", "scraper", "serde", "serde_json", @@ -1669,7 +1712,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ - "bitflags", + "bitflags 2.11.1", "cfg-if", "cfg_aliases", "libc", @@ -1681,7 +1724,7 @@ version = "0.31.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" dependencies = [ - "bitflags", + "bitflags 2.11.1", "cfg-if", "cfg_aliases", "libc", @@ -1757,6 +1800,16 @@ dependencies = [ "libm", ] +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + [[package]] name = "objc2" version = "0.6.4" @@ -1772,7 +1825,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73ad74d880bb43877038da939b7427bba67e9dd42004a18b809ba7d87cee241c" dependencies = [ - "bitflags", + "bitflags 2.11.1", "objc2", "objc2-foundation", ] @@ -1793,7 +1846,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" dependencies = [ - "bitflags", + "bitflags 2.11.1", "dispatch2", "objc2", ] @@ -1804,7 +1857,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e022c9d066895efa1345f8e33e584b9f958da2fd4cd116792e15e07e4720a807" dependencies = [ - "bitflags", + "bitflags 2.11.1", "dispatch2", "objc2", "objc2-core-foundation", @@ -1837,7 +1890,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cde0dfb48d25d2b4862161a4d5fcc0e3c24367869ad306b0c9ec0073bfed92d" dependencies = [ - "bitflags", + "bitflags 2.11.1", "objc2", "objc2-core-foundation", "objc2-core-graphics", @@ -1855,7 +1908,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" dependencies = [ - "bitflags", + "bitflags 2.11.1", "block2", "libc", "objc2", @@ -1868,7 +1921,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "180788110936d59bab6bd83b6060ffdfffb3b922ba1396b312ae795e1de9d81d" dependencies = [ - "bitflags", + "bitflags 2.11.1", "objc2", "objc2-core-foundation", ] @@ -1879,7 +1932,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96c1358452b371bf9f104e21ec536d37a650eb10f7ee379fff67d2e08d537f1f" dependencies = [ - "bitflags", + "bitflags 2.11.1", "objc2", "objc2-core-foundation", "objc2-foundation", @@ -1891,7 +1944,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d87d638e33c06f577498cbcc50491496a3ed4246998a7fbba7ccb98b1e7eab22" dependencies = [ - "bitflags", + "bitflags 2.11.1", "block2", "objc2", "objc2-cloud-kit", @@ -1916,6 +1969,21 @@ dependencies = [ "objc2-foundation", ] +[[package]] +name = "ocrs" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5379fdd3f11522b5a2ff53017a189463dabf5d0a9c915cb3eb97fabec4ea11c" +dependencies = [ + "anyhow", + "rayon", + "rten", + "rten-imageproc", + "rten-tensor", + "thiserror 2.0.18", + "wasm-bindgen", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -2142,7 +2210,7 @@ version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61" dependencies = [ - "bitflags", + "bitflags 2.11.1", "crc32fast", "fdeflate", "flate2", @@ -2361,13 +2429,33 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "redox_syscall" version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags", + "bitflags 2.11.1", ] [[package]] @@ -2376,7 +2464,7 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4666a1a60d8412eab19d94f6d13dcc9cea0a5ef4fdf6a5db306537413c661b1b" dependencies = [ - "bitflags", + "bitflags 2.11.1", ] [[package]] @@ -2496,19 +2584,135 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rten" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43c230fa4ade87c913f61dbd911b7eb0d49460ceff3f1e4fabc837fac191137c" +dependencies = [ + "flatbuffers", + "num_cpus", + "rayon", + "rten-base", + "rten-gemm", + "rten-model-file", + "rten-onnx", + "rten-shape-inference", + "rten-simd", + "rten-tensor", + "rten-vecmath", + "rustc-hash", + "smallvec", + "typeid", + "wasm-bindgen", +] + +[[package]] +name = "rten-base" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2738cf8bb4c27f828ac788d01ccf4e367e8e773cfec6851f81851b5211de6a79" +dependencies = [ + "rayon", +] + +[[package]] +name = "rten-gemm" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a81a0ca209fb5ce21bd17efa0bd287d5881c6cebfbff0b21c4294a1a14a9e" +dependencies = [ + "rayon", + "rten-base", + "rten-simd", + "rten-tensor", +] + +[[package]] +name = "rten-imageproc" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5f148e7e941fb5727b9046a5fa1b45525543d5105f14b384fd9261df0ee49bc" +dependencies = [ + "rten-tensor", +] + +[[package]] +name = "rten-model-file" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2f8d270f07ab1bbfff47250c6039f6caa5da59d6da7d74f66aa48559aa6fea" +dependencies = [ + "flatbuffers", + "rten-base", +] + +[[package]] +name = "rten-onnx" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23086eef75bfb55278cb0b45cf9f5a877d466d914914aafebee4ffca9b24d20c" + +[[package]] +name = "rten-shape-inference" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e8a913c7ca40e2bfbb2a0cd447cce56b33ab19435f56693271a2ef37cf58984" +dependencies = [ + "rten-tensor", + "smallvec", +] + +[[package]] +name = "rten-simd" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b19a0032dfcb70dd20960c1c51a37674b237586cbc1ce586f45b46605d108e82" + +[[package]] +name = "rten-tensor" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05dc744a270aa32d154f1a3df8e48740ccc1be9dfbcf23295ada66d83aa98de6" +dependencies = [ + "rayon", + "rten-base", + "smallvec", + "typeid", +] + +[[package]] +name = "rten-vecmath" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9574ddebf5671bc08ceb76e2e1638fadc57fdeff318634eab2c29e9a803cff64" +dependencies = [ + "rten-base", + "rten-simd", +] + [[package]] name = "rustc-hash" version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + [[package]] name = "rustix" version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags", + "bitflags 2.11.1", "errno", "libc", "linux-raw-sys 0.4.15", @@ -2521,7 +2725,7 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags", + "bitflags 2.11.1", "errno", "libc", "linux-raw-sys 0.12.1", @@ -2603,7 +2807,7 @@ version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4eb30575f3638fc8f6815f448d50cb1a2e255b0897985c8c59f4d37b72a07b06" dependencies = [ - "bitflags", + "bitflags 2.11.1", "cssparser", "derive_more", "fxhash", @@ -2911,7 +3115,7 @@ checksum = "aa003f0038df784eb8fecbbac13affe3da23b45194bd57dba231c8f48199c526" dependencies = [ "atoi", "base64", - "bitflags", + "bitflags 2.11.1", "byteorder", "bytes", "chrono", @@ -2955,7 +3159,7 @@ checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46" dependencies = [ "atoi", "base64", - "bitflags", + "bitflags 2.11.1", "byteorder", "chrono", "crc", @@ -3317,7 +3521,7 @@ version = "0.6.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68d6fdd9f81c2819c9a8b0e0cd91660e7746a8e6ea2ba7c6b2b057985f6bcb51" dependencies = [ - "bitflags", + "bitflags 2.11.1", "bytes", "futures-util", "http", @@ -3428,6 +3632,12 @@ dependencies = [ "utf-8", ] +[[package]] +name = "typeid" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" + [[package]] name = "typenum" version = "1.20.0" @@ -3668,7 +3878,7 @@ version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" dependencies = [ - "bitflags", + "bitflags 2.11.1", "hashbrown 0.15.5", "indexmap", "semver", @@ -4096,7 +4306,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" dependencies = [ "anyhow", - "bitflags", + "bitflags 2.11.1", "indexmap", "log", "serde", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 765d388..81ba63c 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mangalord" -version = "0.89.0" +version = "0.90.0" edition = "2021" default-run = "mangalord" @@ -52,6 +52,8 @@ sysinfo = { version = "0.32", default-features = false, features = ["system", "c nix = { version = "0.29", features = ["fs"] } scraper = "0.20" reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "socks", "cookies", "stream", "json"] } +ocrs = "0.12" +rten = "0.24" [dev-dependencies] tempfile = "3" diff --git a/backend/Dockerfile b/backend/Dockerfile index 065269e..7bb0e19 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -58,6 +58,20 @@ WORKDIR /app COPY --from=builder /app/target/release/mangalord /usr/local/bin/mangalord COPY --from=builder /app/migrations /app/migrations +# OCR models for the default `ANALYSIS_BACKEND=ocr` (ocrs) engine. The two +# `.rten` files are pulled at build time into /models, where the runtime's +# `OCRS_DETECTION_MODEL` / `OCRS_RECOGNITION_MODEL` defaults point. They're a +# few MB each and pure data (no native code), so they bake cleanly into the +# image and need no manual setup on the Pi. Set INSTALL_OCR_MODELS=false to +# skip (e.g. for a vision-only deploy that never runs ocrs). +ARG INSTALL_OCR_MODELS=true +ARG OCRS_MODEL_BASE_URL=https://ocrs-models.s3-accelerate.amazonaws.com +RUN if [ "$INSTALL_OCR_MODELS" = "true" ]; then \ + mkdir -p /models \ + && curl -fsSL "${OCRS_MODEL_BASE_URL}/text-detection.rten" -o /models/text-detection.rten \ + && curl -fsSL "${OCRS_MODEL_BASE_URL}/text-recognition.rten" -o /models/text-recognition.rten; \ + fi + ENV STORAGE_DIR=/var/lib/mangalord/storage # Pre-create the storage dir so the entrypoint doesn't need to # mkdir-as-root and so the named volume mount inherits the right diff --git a/backend/src/analysis/mod.rs b/backend/src/analysis/mod.rs index 5ea2d7c..2c909ee 100644 --- a/backend/src/analysis/mod.rs +++ b/backend/src/analysis/mod.rs @@ -9,5 +9,6 @@ pub mod daemon; pub mod events; +pub mod ocr; pub mod prompt; pub mod vision; diff --git a/backend/src/analysis/ocr.rs b/backend/src/analysis/ocr.rs new file mode 100644 index 0000000..e60556b --- /dev/null +++ b/backend/src/analysis/ocr.rs @@ -0,0 +1,201 @@ +//! The in-process OCR analysis backend (the `ocrs` engine). +//! +//! A lightweight alternative to [`crate::analysis::vision`]: instead of a slow +//! local LLM, each page is run through `ocrs` — a pure-Rust detect→recognize +//! OCR pipeline on the `rten` runtime. It extracts **text only** (no tags, +//! scene description or NSFW flags — those stay the vision backend's job), then +//! reuses [`repo::page_analysis::persist_analysis`] so the OCR lines land in +//! `page_ocr_text` and the weighted `search_doc` tsvector exactly as the vision +//! path produces them. That makes the existing text-search surfaces +//! (`/v1/me/page-search` and the tag aggregations) work with no further wiring. +//! +//! The engine is split behind the [`OcrEngine`] trait so the dispatcher is +//! unit-testable without shipping the (multi-MB) `.rten` model files: tests use +//! [`test_support::StubOcrEngine`], production uses [`OcrsEngine`]. + +use std::sync::Arc; + +use async_trait::async_trait; +use sqlx::PgPool; +use uuid::Uuid; + +use crate::analysis::daemon::AnalyzeDispatcher; +use crate::domain::page_analysis::{OcrResult, SafetyFlag, VisionAnalysis}; +use crate::repo; +use crate::storage::Storage; + +/// The `model` label stamped onto `page_analysis` rows written by this backend. +pub const OCR_MODEL_LABEL: &str = "ocrs"; + +/// Extracts text lines from a decoded-or-encoded page image. The production +/// impl ([`OcrsEngine`]) decodes the bytes itself; the trait takes the raw +/// stored image bytes so the dispatcher stays engine-agnostic. +pub trait OcrEngine: Send + Sync { + /// Run OCR over one page image (the bytes as stored, e.g. PNG/JPEG/WebP). + /// Returns the recognized text lines in reading order (top→bottom). + fn recognize(&self, image: &[u8]) -> anyhow::Result>; +} + +/// Turn the OCR engine's ordered text lines into the [`VisionAnalysis`] shape +/// that [`repo::page_analysis::persist_analysis`] consumes. OCR-only: the tag, +/// scene and safety fields are left empty/default. The line `kind` is left +/// blank — `persist_analysis` maps an empty kind to the neutral mid-weight +/// `OcrKind::Narration` bucket (classifying speech/sfx/… is the deferred +/// vision backend's job). +pub fn lines_to_analysis(lines: Vec) -> VisionAnalysis { + let ocr_results = lines + .into_iter() + .map(|text| OcrResult { text, kind: String::new(), y: None }) + .collect(); + VisionAnalysis { + ocr_results, + tagging_results: Vec::new(), + scene_description: String::new(), + safety_flag: SafetyFlag::default(), + } +} + +/// Production OCR engine: an `ocrs` detect+recognize pipeline with the two +/// `.rten` models loaded once at startup. Cheap to share across workers — the +/// recognize path borrows `&self`. +pub struct OcrsEngine { + engine: ocrs::OcrEngine, +} + +impl OcrsEngine { + /// Load the detection + recognition models from disk and build the engine. + /// Fails (at startup) if either model file is missing or unreadable, so a + /// misconfigured path is a loud boot error rather than a per-page failure. + pub fn from_model_paths(detection: &str, recognition: &str) -> anyhow::Result { + use anyhow::Context; + let detection_model = rten::Model::load_file(detection) + .with_context(|| format!("load ocrs detection model {detection}"))?; + let recognition_model = rten::Model::load_file(recognition) + .with_context(|| format!("load ocrs recognition model {recognition}"))?; + let engine = ocrs::OcrEngine::new(ocrs::OcrEngineParams { + detection_model: Some(detection_model), + recognition_model: Some(recognition_model), + ..Default::default() + }) + .context("construct ocrs engine")?; + Ok(Self { engine }) + } +} + +impl OcrEngine for OcrsEngine { + fn recognize(&self, image: &[u8]) -> anyhow::Result> { + use anyhow::Context; + // Decode to RGB8 so `ImageSource` gets a known channel layout. + let rgb = image::load_from_memory(image) + .context("decode page image for OCR")? + .into_rgb8(); + let source = ocrs::ImageSource::from_bytes(rgb.as_raw(), rgb.dimensions()) + .map_err(|e| anyhow::anyhow!("build OCR image source: {e}"))?; + let input = self.engine.prepare_input(source)?; + // detect words → group into lines → recognize each line. Mirrors + // `OcrEngine::get_text`, but keeps the lines as a Vec instead of + // joining them, so each becomes its own `page_ocr_text` row. + let words = self.engine.detect_words(&input)?; + let line_rects = self.engine.find_text_lines(&input, &words); + let lines = self + .engine + .recognize_text(&input, &line_rects)? + .into_iter() + .filter_map(|line| line.map(|l| l.to_string())) + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + .collect(); + Ok(lines) + } +} + +/// Production dispatcher for the OCR backend: load the page, read its image +/// from storage, run OCR on the blocking pool, and persist the lines. Mirrors +/// [`crate::analysis::daemon::RealAnalyzeDispatcher`] but with no network I/O. +pub struct OcrAnalyzeDispatcher { + pub db: PgPool, + pub storage: Arc, + pub engine: Arc, + pub max_image_bytes: usize, +} + +#[async_trait] +impl AnalyzeDispatcher for OcrAnalyzeDispatcher { + async fn dispatch(&self, page_id: Uuid) -> anyhow::Result<()> { + let Some(page) = repo::page::find_by_id(&self.db, page_id).await? else { + // Page was deleted between enqueue and dispatch — nothing to do. + return Ok(()); + }; + let bytes = self + .storage + .get(&page.storage_key) + .await + .map_err(|e| anyhow::anyhow!("read page image {}: {e}", page.storage_key))?; + if bytes.len() > self.max_image_bytes { + anyhow::bail!( + "page image {} is {} bytes, over the {} cap", + page.storage_key, + bytes.len(), + self.max_image_bytes + ); + } + // OCR inference is CPU-bound and synchronous — keep it off the async + // worker's runtime thread. + let engine = Arc::clone(&self.engine); + let lines = tokio::task::spawn_blocking(move || engine.recognize(&bytes)) + .await + .map_err(|e| anyhow::anyhow!("OCR task join error: {e}"))??; + let analysis = lines_to_analysis(lines); + repo::page_analysis::persist_analysis(&self.db, page_id, &analysis, OCR_MODEL_LABEL).await?; + Ok(()) + } +} + +/// Stubs for the OCR dispatcher's integration tests. Public because the tests +/// live in the `tests/` dir (a separate crate). +pub mod test_support { + use super::*; + + /// An [`OcrEngine`] that returns a fixed set of lines regardless of input, + /// so the dispatcher's storage→persist path can be tested without models. + pub struct StubOcrEngine { + pub lines: Vec, + } + + impl StubOcrEngine { + pub fn new(lines: &[&str]) -> Arc { + Arc::new(Self { lines: lines.iter().map(|s| s.to_string()).collect() }) + } + } + + impl OcrEngine for StubOcrEngine { + fn recognize(&self, _image: &[u8]) -> anyhow::Result> { + Ok(self.lines.clone()) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn lines_to_analysis_maps_lines_in_order_and_leaves_rest_empty() { + let v = lines_to_analysis(vec!["Hello".to_string(), "world!".to_string()]); + assert_eq!(v.ocr_results.len(), 2); + assert_eq!(v.ocr_results[0].text, "Hello"); + assert_eq!(v.ocr_results[1].text, "world!"); + // OCR-only: kind blank (→ Narration at persist), no tags/scene/safety. + assert!(v.ocr_results.iter().all(|r| r.kind.is_empty())); + assert!(v.tagging_results.is_empty()); + assert_eq!(v.scene_description, ""); + assert!(!v.safety_flag.is_nsfw); + assert!(v.safety_flag.content_type.is_empty()); + } + + #[test] + fn lines_to_analysis_handles_no_text() { + let v = lines_to_analysis(Vec::new()); + assert!(v.ocr_results.is_empty()); + } +} diff --git a/backend/src/api/page_tags.rs b/backend/src/api/page_tags.rs index 8beabe6..b27e2a0 100644 --- a/backend/src/api/page_tags.rs +++ b/backend/src/api/page_tags.rs @@ -392,10 +392,9 @@ pub struct AggregateParams { pub limit: i64, #[serde(default)] pub offset: i64, - /// Reserved for the planned OCR text-search input. Accepted on - /// the wire so adding OCR later won't break the API shape, but - /// rejected with 501 `text_search_not_yet_supported` if non-empty - /// until the backend supports it. + /// OCR text filter. When non-empty, only pages whose analysis + /// `search_doc` matches the query (`plainto_tsquery`) are aggregated. + /// Blank/absent ⇒ tag-only aggregation. #[serde(default)] pub text: Option, } @@ -411,32 +410,17 @@ fn parse_order(raw: Option<&str>) -> AppResult { } } -fn ensure_text_unsupported(text: Option<&str>) -> AppResult<()> { - // Future OCR search will plug in here. Until then, return a - // distinct code (`text_search_not_yet_supported`) so clients can - // detect "feature pending" vs. a generic 4xx — the code is the - // wire contract, not the message. - if text.is_some_and(|s| !s.trim().is_empty()) { - return Err(AppError::NotImplemented { - code: "text_search_not_yet_supported", - message: "text search is reserved for the planned OCR input but not yet supported", - }); - } - Ok(()) -} - async fn list_chapters_for_tag( State(state): State, CurrentUser(user): CurrentUser, Query(params): Query, ) -> AppResult>> { - ensure_text_unsupported(params.text.as_deref())?; let tag = normalize_tag(¶ms.tag)?; let order = parse_order(params.order.as_deref())?; let limit = params.limit.clamp(1, 200); let offset = params.offset.max(0); let (items, total) = repo::page_tag::aggregate_chapters_for_tag( - &state.db, user.id, &tag, order, limit, offset, + &state.db, user.id, &tag, order, limit, offset, params.text.as_deref(), ) .await?; Ok(Json(PagedResponse::with_total(items, limit, offset, total))) @@ -447,13 +431,12 @@ async fn list_mangas_for_tag( CurrentUser(user): CurrentUser, Query(params): Query, ) -> AppResult>> { - ensure_text_unsupported(params.text.as_deref())?; let tag = normalize_tag(¶ms.tag)?; let order = parse_order(params.order.as_deref())?; let limit = params.limit.clamp(1, 200); let offset = params.offset.max(0); let (items, total) = repo::page_tag::aggregate_mangas_for_tag( - &state.db, user.id, &tag, order, limit, offset, + &state.db, user.id, &tag, order, limit, offset, params.text.as_deref(), ) .await?; Ok(Json(PagedResponse::with_total(items, limit, offset, total))) diff --git a/backend/src/app.rs b/backend/src/app.rs index 54ba854..aeeeb7e 100644 --- a/backend/src/app.rs +++ b/backend/src/app.rs @@ -429,46 +429,74 @@ async fn spawn_analysis_daemon( ), Err(e) => tracing::warn!(?e, "analysis: reclaim_orphaned at startup failed"), } - let http = reqwest::Client::builder() - .timeout(cfg.request_timeout) - // Refuse to honour ambient HTTP_PROXY / HTTPS_PROXY container env. - // The vision call carries an env-managed bearer token + page image - // bytes; a stray upstream proxy would exfiltrate both. Mirrors the - // crawler client's `.no_proxy()` (see `spawn_crawler_daemon`). - .no_proxy() - .build() - .context("build analysis http client")?; - let vision = crate::analysis::vision::VisionClient::new(http, cfg); - let dispatcher = Arc::new(crate::analysis::daemon::RealAnalyzeDispatcher { - db: db.clone(), - storage, - vision, - model: cfg.model.clone(), - max_image_bytes: cfg.max_image_bytes, - }); - // When a readiness URL is configured, gate leasing on it so an - // autoscaler that idle-stops the vision container never lets a job burn - // its retries. A dedicated short-timeout client keeps the probe snappy - // and independent of the (long) per-request analysis timeout. - let readiness: Option> = - match &cfg.vision_health_url { - Some(url) if !url.is_empty() => { - let probe = reqwest::Client::builder() - .timeout(std::time::Duration::from_secs(5)) - // Same reasoning as the main analysis client: do not - // honour ambient HTTP_PROXY env. The readiness probe is - // unauthenticated but a hostile upstream still gets a - // useful side-channel on backend uptime + vision health. - .no_proxy() - .build() - .context("build vision readiness http client")?; - Some(Arc::new(crate::analysis::daemon::HttpVisionReadiness { - http: probe, - health_url: url.clone(), - })) - } - _ => None, - }; + // Pick the engine. The OCR backend runs in-process (no network, no + // readiness gate); the vision backend talks to a local LLM server. + let (dispatcher, readiness): ( + Arc, + Option>, + ) = match cfg.effective_backend() { + crate::config::AnalysisBackend::Ocr => { + // Load the `.rten` models once; a bad path is a loud boot error. + let engine = crate::analysis::ocr::OcrsEngine::from_model_paths( + &cfg.ocr_detection_model, + &cfg.ocr_recognition_model, + ) + .context("build ocrs engine")?; + let dispatcher = Arc::new(crate::analysis::ocr::OcrAnalyzeDispatcher { + db: db.clone(), + storage, + engine: Arc::new(engine), + max_image_bytes: cfg.max_image_bytes, + }); + // In-process engine is always ready — no gate. + (dispatcher, None) + } + crate::config::AnalysisBackend::Vision => { + let http = reqwest::Client::builder() + .timeout(cfg.request_timeout) + // Refuse to honour ambient HTTP_PROXY / HTTPS_PROXY container + // env. The vision call carries an env-managed bearer token + + // page image bytes; a stray upstream proxy would exfiltrate + // both. Mirrors the crawler client's `.no_proxy()` (see + // `spawn_crawler_daemon`). + .no_proxy() + .build() + .context("build analysis http client")?; + let vision = crate::analysis::vision::VisionClient::new(http, cfg); + let dispatcher = Arc::new(crate::analysis::daemon::RealAnalyzeDispatcher { + db: db.clone(), + storage, + vision, + model: cfg.model.clone(), + max_image_bytes: cfg.max_image_bytes, + }); + // When a readiness URL is configured, gate leasing on it so an + // autoscaler that idle-stops the vision container never lets a job + // burn its retries. A dedicated short-timeout client keeps the + // probe snappy and independent of the (long) per-request timeout. + let readiness: Option> = + match &cfg.vision_health_url { + Some(url) if !url.is_empty() => { + let probe = reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(5)) + // Same reasoning as the main analysis client: do + // not honour ambient HTTP_PROXY env. The readiness + // probe is unauthenticated but a hostile upstream + // still gets a useful side-channel on backend + // uptime + vision health. + .no_proxy() + .build() + .context("build vision readiness http client")?; + Some(Arc::new(crate::analysis::daemon::HttpVisionReadiness { + http: probe, + health_url: url.clone(), + })) + } + _ => None, + }; + (dispatcher, readiness) + } + }; let handle = crate::analysis::daemon::spawn( db, CancellationToken::new(), @@ -480,7 +508,21 @@ async fn spawn_analysis_daemon( readiness, }, ); - tracing::info!(workers = cfg.workers, model = %cfg.model, "analysis worker daemon started"); + // Log the *effective* backend (what the worker actually dispatches + // through), not the raw `cfg.backend` — with vision dormant they diverge + // when an operator requests `vision`, and a misleading line here was a + // real observability footgun. The model label tracks the effective engine. + let effective_backend = cfg.effective_backend(); + let effective_model: &str = match effective_backend { + crate::config::AnalysisBackend::Ocr => crate::analysis::ocr::OCR_MODEL_LABEL, + crate::config::AnalysisBackend::Vision => &cfg.model, + }; + tracing::info!( + workers = cfg.workers, + backend = ?effective_backend, + model = %effective_model, + "analysis worker daemon started" + ); Ok(handle) } @@ -1440,25 +1482,30 @@ mod tests { // Bind to a local so the TempDir lives for the rest of the test. // `tempfile::tempdir().unwrap().path()` would drop the TempDir // at end-of-expression and `LocalStorage` would hold a path to - // a deleted directory. (Today this is fine because the dispatch - // fails before storage is touched, but it makes the test fragile - // to any future code rearrangement.) + // a deleted directory. let storage_dir = tempfile::tempdir().unwrap(); let storage: Arc = Arc::new(LocalStorage::new(storage_dir.path())); let mut cfg = crate::config::AnalysisConfig::default(); + // The worker always runs OCR now (vision is dormant — see + // `effective_backend`), and the `.rten` models aren't shipped to unit + // CI. Point the engine at a path that can't exist so the *engine build* + // fails deterministically. Reclaim runs at the very top of + // `spawn_analysis_daemon`, before — and independently of — engine + // readiness, so the row must still be reclaimed even though spawn + // returns Err. That's exactly the regression this test guards (an + // analysis-only deploy must reclaim orphaned leases at startup). + cfg.ocr_detection_model = "/nonexistent/text-detection.rten".to_string(); + cfg.ocr_recognition_model = "/nonexistent/text-recognition.rten".to_string(); cfg.workers = 1; cfg.job_timeout = Duration::from_secs(1); let events = Arc::new(crate::analysis::events::AnalysisEvents::new()); - let handle = spawn_analysis_daemon(pool.clone(), storage, &cfg, events) - .await - .expect("spawn"); - // Immediately shut down — we're only here to prove reclaim ran. - // The workers may briefly pick up the now-pending row; that's - // fine, but we cancel before any real dispatch (the LocalStorage - // key doesn't exist, so a dispatch would fail anyway). - handle.shutdown().await; + let spawned = spawn_analysis_daemon(pool.clone(), storage, &cfg, events).await; + assert!( + spawned.is_err(), + "engine build must fail with a missing model path" + ); // The reclaim must have moved the row back to pending with the // attempt refunded (attempts goes from 1 → 0). Reaching it on the diff --git a/backend/src/config.rs b/backend/src/config.rs index b9cc936..750f0bc 100644 --- a/backend/src/config.rs +++ b/backend/src/config.rs @@ -113,6 +113,32 @@ impl ResponseFormat { } } +/// Which engine the analysis worker dispatches each page through. A +/// deploy-time choice (the engine is either installed or not), so it lives in +/// env only and is not part of the admin-editable `AnalysisSettings`. +/// +/// * `Ocr` — in-process [`crate::analysis::ocr`] (the `ocrs` engine): fast, +/// CPU-only, English text. Writes OCR text only (no tags/scene/safety). +/// * `Vision` — the local OpenAI-compatible LLM in [`crate::analysis::vision`]: +/// full OCR + tags + scene + safety, but heavy. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum AnalysisBackend { + Ocr, + Vision, +} + +impl AnalysisBackend { + /// Lenient env parse. Defaults to `Ocr` (the Pi-friendly path) for any + /// unset or unrecognized value; `vision` opts back into the LLM engine. + fn from_str(s: &str) -> AnalysisBackend { + match s.trim().to_lowercase().as_str() { + "vision" | "llm" => AnalysisBackend::Vision, + // Default (incl. "ocr", "ocrs", and anything unrecognized). + _ => AnalysisBackend::Ocr, + } + } +} + /// AI content-analysis worker configuration: the enable gate, the local /// OpenAI-compatible vision endpoint, and the worker / request knobs. #[derive(Clone, Debug)] @@ -120,6 +146,16 @@ pub struct AnalysisConfig { /// Master switch (`ANALYSIS_ENABLED`). When `false`, no analysis jobs /// are enqueued and no worker runs. Defaults to `false`. pub enabled: bool, + /// Which engine the worker dispatches through (`ANALYSIS_BACKEND`): + /// `ocr` (default, the in-process `ocrs` engine) or `vision` (the local + /// LLM). Deploy-time, env-only — see [`AnalysisBackend`]. + pub backend: AnalysisBackend, + /// Path to the ocrs text-*detection* `.rten` model + /// (`OCRS_DETECTION_MODEL`). Only read when `backend == Ocr`. + pub ocr_detection_model: String, + /// Path to the ocrs text-*recognition* `.rten` model + /// (`OCRS_RECOGNITION_MODEL`). Only read when `backend == Ocr`. + pub ocr_recognition_model: String, /// Number of concurrent analysis workers (`ANALYSIS_WORKERS`). pub workers: usize, /// OpenAI-compatible chat/completions URL (`ANALYSIS_VISION_URL`). @@ -195,6 +231,9 @@ impl Default for AnalysisConfig { fn default() -> Self { Self { enabled: false, + backend: AnalysisBackend::Ocr, + ocr_detection_model: "/models/text-detection.rten".to_string(), + ocr_recognition_model: "/models/text-recognition.rten".to_string(), workers: 1, endpoint: "http://localhost:8000/v1/chat/completions".to_string(), vision_health_url: None, @@ -223,10 +262,39 @@ impl Default for AnalysisConfig { } impl AnalysisConfig { + /// The backend the worker actually dispatches through. + /// + /// Vision is **temporarily disabled**: the engine code (`analysis::vision`, + /// `RealAnalyzeDispatcher`, the readiness probe) is kept intact but never + /// selected. Until it's re-enabled, this returns [`AnalysisBackend::Ocr`] + /// regardless of the parsed `backend`, logging a warning if `vision` was + /// requested so an env override isn't silently ignored. Re-enabling vision + /// is then a one-line change here (return `self.backend`). + pub fn effective_backend(&self) -> AnalysisBackend { + if self.backend == AnalysisBackend::Vision { + tracing::warn!( + "ANALYSIS_BACKEND=vision requested but the vision backend is temporarily \ + disabled; running OCR instead" + ); + } + AnalysisBackend::Ocr + } + pub fn from_env() -> Self { let d = AnalysisConfig::default(); Self { enabled: env_bool("ANALYSIS_ENABLED", d.enabled), + backend: std::env::var("ANALYSIS_BACKEND") + .map(|s| AnalysisBackend::from_str(&s)) + .unwrap_or(d.backend), + ocr_detection_model: std::env::var("OCRS_DETECTION_MODEL") + .ok() + .filter(|s| !s.is_empty()) + .unwrap_or(d.ocr_detection_model), + ocr_recognition_model: std::env::var("OCRS_RECOGNITION_MODEL") + .ok() + .filter(|s| !s.is_empty()) + .unwrap_or(d.ocr_recognition_model), workers: env_usize("ANALYSIS_WORKERS", d.workers).max(1), endpoint: std::env::var("ANALYSIS_VISION_URL").unwrap_or(d.endpoint), vision_health_url: std::env::var("ANALYSIS_VISION_HEALTH_URL") @@ -734,11 +802,18 @@ mod tests { "ANALYSIS_MAX_SLICES", "ANALYSIS_RESPONSE_FORMAT", "ANALYSIS_FREQUENCY_PENALTY", + "ANALYSIS_BACKEND", + "OCRS_DETECTION_MODEL", + "OCRS_RECOGNITION_MODEL", ] { std::env::remove_var(k); } let cfg = AnalysisConfig::from_env(); assert!(!cfg.enabled); + // OCR is the default engine (the Pi-friendly path). + assert_eq!(cfg.backend, AnalysisBackend::Ocr); + assert_eq!(cfg.ocr_detection_model, "/models/text-detection.rten"); + assert_eq!(cfg.ocr_recognition_model, "/models/text-recognition.rten"); assert_eq!(cfg.workers, 1); assert_eq!(cfg.max_pixels, 1_000_000); assert_eq!(cfg.min_slice_height, 640); @@ -765,6 +840,49 @@ mod tests { std::env::remove_var("ANALYSIS_RESPONSE_FORMAT"); } + #[test] + fn analysis_backend_parses_and_defaults_to_ocr() { + let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner()); + for (raw, want) in [ + ("ocr", AnalysisBackend::Ocr), + ("ocrs", AnalysisBackend::Ocr), + ("vision", AnalysisBackend::Vision), + ("llm", AnalysisBackend::Vision), + ("anything-else", AnalysisBackend::Ocr), + ] { + std::env::set_var("ANALYSIS_BACKEND", raw); + assert_eq!(AnalysisConfig::from_env().backend, want, "raw={raw}"); + } + // Unset → OCR. + std::env::remove_var("ANALYSIS_BACKEND"); + assert_eq!(AnalysisConfig::from_env().backend, AnalysisBackend::Ocr); + } + + #[test] + fn effective_backend_is_ocr_even_when_vision_requested() { + // Vision is temporarily disabled: the worker always runs OCR no matter + // what `ANALYSIS_BACKEND` parsed to. `backend` still reflects the raw + // request (so the override is visible/loggable), but `effective_backend` + // is the value the daemon actually dispatches through. + let mut cfg = AnalysisConfig::default(); + cfg.backend = AnalysisBackend::Vision; + assert_eq!(cfg.effective_backend(), AnalysisBackend::Ocr); + cfg.backend = AnalysisBackend::Ocr; + assert_eq!(cfg.effective_backend(), AnalysisBackend::Ocr); + } + + #[test] + fn ocr_model_paths_parse_from_env() { + let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner()); + std::env::set_var("OCRS_DETECTION_MODEL", "/opt/det.rten"); + std::env::set_var("OCRS_RECOGNITION_MODEL", "/opt/rec.rten"); + let cfg = AnalysisConfig::from_env(); + std::env::remove_var("OCRS_DETECTION_MODEL"); + std::env::remove_var("OCRS_RECOGNITION_MODEL"); + assert_eq!(cfg.ocr_detection_model, "/opt/det.rten"); + assert_eq!(cfg.ocr_recognition_model, "/opt/rec.rten"); + } + #[test] fn analysis_config_parses_from_env() { let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner()); diff --git a/backend/src/error.rs b/backend/src/error.rs index 630a8cf..6253893 100644 --- a/backend/src/error.rs +++ b/backend/src/error.rs @@ -39,10 +39,10 @@ pub enum AppError { details: serde_json::Value, }, /// 501 — the wire shape is accepted but the feature isn't built yet. - /// Carries a `&'static str` snake_case code so clients can detect - /// the specific pending feature (`text_search_not_yet_supported`, - /// etc.) without parsing the message. Used today by the `?text=` - /// reservation on the page-tag aggregation endpoints. + /// Carries a `&'static str` snake_case code so clients can detect the + /// specific pending feature without parsing the message. Generic; kept + /// for future reservations (the `?text=` page-tag reservation that first + /// used it now performs real OCR search). #[error("not implemented: {code}")] NotImplemented { code: &'static str, diff --git a/backend/src/repo/page_tag.rs b/backend/src/repo/page_tag.rs index 755fd47..0981712 100644 --- a/backend/src/repo/page_tag.rs +++ b/backend/src/repo/page_tag.rs @@ -247,6 +247,11 @@ pub async fn distinct_tags_for_user( /// storage keys (page-number ascending) so the row can render a /// thumbnail strip without a follow-up fetch. /// +/// When `text` is non-blank, results are further restricted to pages whose +/// analysis `search_doc` matches the query (OCR text search), and both +/// `match_count` and the sample thumbnails reflect that filtered set — +/// i.e. `match_count` counts tagged pages whose OCR also matches `text`. +/// /// `order` is inlined via `format!()` — the enum value space is /// closed (`ASC` / `DESC`) so this is not a SQL-injection vector. pub async fn aggregate_chapters_for_tag( @@ -256,7 +261,30 @@ pub async fn aggregate_chapters_for_tag( order: Order, limit: i64, offset: i64, + text: Option<&str>, ) -> AppResult<(Vec, i64)> { + // OCR text filter: when present, additionally require the page's analysis + // `search_doc` to match the query. Reuses the precomputed tsvector exactly + // like `repo::page_analysis::page_search`. `None`/blank ⇒ tag-only. + let text = text.map(str::trim).filter(|s| !s.is_empty()); + let (text_join, text_where) = match text { + // `$5` in the main query, `$3` in the count query (see binds below). + Some(_) => ( + "JOIN page_analysis pa ON pa.page_id = p.id", + "AND pa.search_doc @@ plainto_tsquery('simple', {n})", + ), + None => ("", ""), + }; + // Same filter inside the correlated sample-thumbnail subquery (its page is + // aliased `p`), so the thumbnails match what the text search matched. The + // subquery lives in the main query, so it reuses the `$5` text bind. + let (sample_join, sample_where) = match text { + Some(_) => ( + "JOIN page_analysis pa2 ON pa2.page_id = p.id", + "AND pa2.search_doc @@ plainto_tsquery('simple', $5)", + ), + None => ("", ""), + }; let sql = format!( r#" SELECT @@ -274,9 +302,11 @@ pub async fn aggregate_chapters_for_tag( FROM pages p JOIN page_tags pt2 ON pt2.page_id = p.id JOIN tags t2 ON t2.id = pt2.tag_id + {sample_join} WHERE p.chapter_id = ch.id AND pt2.user_id = $1 AND lower(t2.name) = $2 + {sample_where} ORDER BY p.page_number ASC LIMIT 3 ) p2 @@ -288,43 +318,60 @@ pub async fn aggregate_chapters_for_tag( JOIN pages p ON p.id = pt.page_id JOIN chapters ch ON ch.id = p.chapter_id JOIN mangas m ON m.id = ch.manga_id + {text_join} WHERE pt.user_id = $1 AND lower(t.name) = $2 + {text_where} GROUP BY ch.id, ch.manga_id, m.title, ch.number, ch.title ORDER BY match_count {dir}, ch.id LIMIT $3 OFFSET $4 "#, dir = order.as_sql(), + text_join = text_join, + text_where = text_where.replace("{n}", "$5"), + sample_join = sample_join, + sample_where = sample_where, ); - let rows = sqlx::query_as::<_, TaggedChapterAggregate>(&sql) + let mut q = sqlx::query_as::<_, TaggedChapterAggregate>(&sql) .bind(user_id) .bind(tag) .bind(limit) - .bind(offset) - .fetch_all(pool) - .await?; + .bind(offset); + if let Some(text) = text { + q = q.bind(text); + } + let rows = q.fetch_all(pool).await?; - let (total,): (i64,) = sqlx::query_as( + let count_sql = format!( r#" SELECT count(*) FROM ( SELECT 1 FROM page_tags pt JOIN tags t ON t.id = pt.tag_id JOIN pages p ON p.id = pt.page_id + {text_join} WHERE pt.user_id = $1 AND lower(t.name) = $2 + {text_where} GROUP BY p.chapter_id ) c "#, - ) - .bind(user_id) - .bind(tag) - .fetch_one(pool) - .await?; + text_join = text_join, + text_where = text_where.replace("{n}", "$3"), + ); + let mut cq = sqlx::query_as::<_, (i64,)>(&count_sql).bind(user_id).bind(tag); + if let Some(text) = text { + cq = cq.bind(text); + } + let (total,) = cq.fetch_one(pool).await?; Ok((rows, total)) } /// Paged list of mangas containing pages tagged `tag` for `user_id`, /// ranked by `match_count` summed across all their chapters. +/// +/// `text` behaves as in [`aggregate_chapters_for_tag`]: non-blank restricts to +/// pages whose OCR `search_doc` matches, and both `match_count` and the sample +/// thumbnails reflect that filtered set. pub async fn aggregate_mangas_for_tag( pool: &PgPool, user_id: Uuid, @@ -332,7 +379,26 @@ pub async fn aggregate_mangas_for_tag( order: Order, limit: i64, offset: i64, + text: Option<&str>, ) -> AppResult<(Vec, i64)> { + // OCR text filter — see `aggregate_chapters_for_tag` for the rationale. + let text = text.map(str::trim).filter(|s| !s.is_empty()); + let (text_join, text_where) = match text { + Some(_) => ( + "JOIN page_analysis pa ON pa.page_id = p.id", + "AND pa.search_doc @@ plainto_tsquery('simple', {n})", + ), + None => ("", ""), + }; + // Same filter inside the sample-thumbnail subquery (page aliased `p`), + // reusing the main query's `$5` text bind. + let (sample_join, sample_where) = match text { + Some(_) => ( + "JOIN page_analysis pa2 ON pa2.page_id = p.id", + "AND pa2.search_doc @@ plainto_tsquery('simple', $5)", + ), + None => ("", ""), + }; let sql = format!( r#" SELECT @@ -349,9 +415,11 @@ pub async fn aggregate_mangas_for_tag( JOIN chapters ch2 ON ch2.id = p.chapter_id JOIN page_tags pt2 ON pt2.page_id = p.id JOIN tags t2 ON t2.id = pt2.tag_id + {sample_join} WHERE ch2.manga_id = m.id AND pt2.user_id = $1 AND lower(t2.name) = $2 + {sample_where} ORDER BY p.page_number ASC LIMIT 3 ) p2 @@ -363,23 +431,31 @@ pub async fn aggregate_mangas_for_tag( JOIN pages p ON p.id = pt.page_id JOIN chapters ch ON ch.id = p.chapter_id JOIN mangas m ON m.id = ch.manga_id + {text_join} WHERE pt.user_id = $1 AND lower(t.name) = $2 + {text_where} GROUP BY m.id, m.title, m.cover_image_path ORDER BY match_count {dir}, m.id LIMIT $3 OFFSET $4 "#, dir = order.as_sql(), + text_join = text_join, + text_where = text_where.replace("{n}", "$5"), + sample_join = sample_join, + sample_where = sample_where, ); - let rows = sqlx::query_as::<_, TaggedMangaAggregate>(&sql) + let mut q = sqlx::query_as::<_, TaggedMangaAggregate>(&sql) .bind(user_id) .bind(tag) .bind(limit) - .bind(offset) - .fetch_all(pool) - .await?; + .bind(offset); + if let Some(text) = text { + q = q.bind(text); + } + let rows = q.fetch_all(pool).await?; - let (total,): (i64,) = sqlx::query_as( + let count_sql = format!( r#" SELECT count(*) FROM ( SELECT 1 @@ -387,15 +463,20 @@ pub async fn aggregate_mangas_for_tag( JOIN tags t ON t.id = pt.tag_id JOIN pages p ON p.id = pt.page_id JOIN chapters ch ON ch.id = p.chapter_id + {text_join} WHERE pt.user_id = $1 AND lower(t.name) = $2 + {text_where} GROUP BY ch.manga_id ) m "#, - ) - .bind(user_id) - .bind(tag) - .fetch_one(pool) - .await?; + text_join = text_join, + text_where = text_where.replace("{n}", "$3"), + ); + let mut cq = sqlx::query_as::<_, (i64,)>(&count_sql).bind(user_id).bind(tag); + if let Some(text) = text { + cq = cq.bind(text); + } + let (total,) = cq.fetch_one(pool).await?; Ok((rows, total)) } diff --git a/backend/src/settings.rs b/backend/src/settings.rs index 925b4ed..b79c576 100644 --- a/backend/src/settings.rs +++ b/backend/src/settings.rs @@ -25,7 +25,7 @@ use serde::{Deserialize, Serialize}; use crate::analysis::prompt::{ GROUNDING_PROMPT_DEFAULT, OCR_PROMPT_DEFAULT, SYSTEM_PROMPT_DEFAULT, }; -use crate::config::{AnalysisConfig, CrawlerConfig, ResponseFormat}; +use crate::config::{AnalysisBackend, AnalysisConfig, CrawlerConfig, ResponseFormat}; use crate::crawler::safety::DownloadAllowlist; /// `app_settings.key` for the crawler group. @@ -355,6 +355,15 @@ impl AnalysisSettings { if self.workers < 1 { errs.push("workers", "must be at least 1"); } + // The endpoint/model are vision-only knobs — the OCR backend never + // dials a URL or sends a model id. While vision is dormant + // (`base.backend == Ocr`), skip the live-worker SSRF gate and the + // model-required check so an OCR operator can enable the worker + // without a vision endpoint/model (the OCR settings UI doesn't even + // expose them). The basic malformed-URL sanity check below stays + // unconditional. Re-enabling vision restores the full gate via the + // `base.backend == Vision` predicate. + let vision_active = base.backend == AnalysisBackend::Vision; let trimmed_endpoint = self.endpoint.trim(); if trimmed_endpoint.is_empty() { errs.push("endpoint", "must be a valid absolute URL"); @@ -362,7 +371,7 @@ impl AnalysisSettings { // Always reject obviously-malformed URLs (matches prior behaviour). let _ = e; errs.push("endpoint", "must be a valid absolute URL"); - } else if self.enabled { + } else if self.enabled && vision_active { // SSRF + API-key exfiltration defence — only enforced when the // worker is actually live. Worker attaches an env-managed bearer // token to every call; without this check, an admin (or one @@ -379,7 +388,7 @@ impl AnalysisSettings { errs.push("endpoint", url_safety_message(&e)); } } - if self.enabled && self.model.trim().is_empty() { + if self.enabled && vision_active && self.model.trim().is_empty() { errs.push("model", "required when analysis is enabled"); } if self.max_tokens < 1 { @@ -459,6 +468,11 @@ impl AnalysisSettings { api_key: base.api_key.clone(), // Env-only readiness probe URL preserved from the base. vision_health_url: base.vision_health_url.clone(), + // Deploy-time engine selection + ocrs model paths: env-only, not + // admin-tunable, so carry them through from the base unchanged. + backend: base.backend, + ocr_detection_model: base.ocr_detection_model.clone(), + ocr_recognition_model: base.ocr_recognition_model.clone(), }) } } @@ -696,7 +710,8 @@ mod tests { // a hostile/CSRF-able admin must NOT be able to point endpoint at // cloud metadata, loopback services, or RFC1918 hosts — when the // worker is enabled (toggling enabled=true later re-runs this gate). - let base = AnalysisConfig::default(); + let mut base = AnalysisConfig::default(); + base.backend = AnalysisBackend::Vision; for url in [ "http://169.254.169.254/v1/chat/completions", "http://127.0.0.1:5432/", @@ -720,7 +735,8 @@ mod tests { // The documented default — docker DNS name resolving to a private IP // at runtime — must still validate, because the bearer recipient // identity is the operator-chosen hostname, not the underlying IP. - let base = AnalysisConfig::default(); + let mut base = AnalysisConfig::default(); + base.backend = AnalysisBackend::Vision; for url in [ "http://mangalord-vision:8000/v1/chat/completions", "https://api.openai.com/v1/chat/completions", @@ -751,7 +767,10 @@ mod tests { #[test] fn analysis_requires_model_only_when_enabled() { - let base = AnalysisConfig::default(); + // Vision base: the model id is required only when the vision worker + // is actually live (see the OCR carve-out below). + let mut base = AnalysisConfig::default(); + base.backend = AnalysisBackend::Vision; let disabled = AnalysisSettings { enabled: false, model: "".to_string(), @@ -767,6 +786,28 @@ mod tests { assert!(errs.errors.iter().any(|e| e.field == "model")); } + #[test] + fn analysis_ocr_backend_enable_skips_vision_endpoint_and_model_validation() { + // With the OCR backend active the worker never dials the vision + // endpoint nor sends a model id, so enabling analysis must NOT + // validate those vision-only fields. The default base carries the + // dev-localhost endpoint (which the SSRF gate would otherwise reject) + // and an empty model — under OCR, enabling is still valid. Without + // this carve-out an OCR operator can't turn the worker on, since the + // OCR settings UI doesn't expose endpoint/model to fix. + let base = AnalysisConfig::default(); // backend = Ocr + let dto = AnalysisSettings { + enabled: true, + endpoint: "http://localhost:8000/v1/chat/completions".to_string(), + model: "".to_string(), + ..AnalysisSettings::from_config(&base) + }; + assert!( + dto.to_config(&base).is_ok(), + "OCR-enabled config must not fail on vision endpoint/model" + ); + } + #[test] fn dtos_serialize_to_json_and_back() { let c = CrawlerSettings::default(); diff --git a/backend/tests/analysis_ocr.rs b/backend/tests/analysis_ocr.rs new file mode 100644 index 0000000..e3c40bc --- /dev/null +++ b/backend/tests/analysis_ocr.rs @@ -0,0 +1,109 @@ +//! Integration tests for the OCR analysis backend +//! (`analysis::ocr::OcrAnalyzeDispatcher`). A stub OCR engine stands in for +//! `ocrs` (whose `.rten` models aren't shipped to CI), so these pin the +//! storage→OCR→persist wiring: the dispatcher reads the page image, runs the +//! engine, and persists the lines via the shared `persist_analysis` path — +//! landing `page_ocr_text` rows and a populated `search_doc` exactly like the +//! vision backend. Each `#[sqlx::test]` gets a fresh migrated DB. + +mod common; + +use std::sync::Arc; + +use mangalord::analysis::daemon::AnalyzeDispatcher; +use mangalord::analysis::ocr::test_support::StubOcrEngine; +use mangalord::analysis::ocr::OcrAnalyzeDispatcher; +use mangalord::domain::page_analysis::AnalysisStatus; +use mangalord::repo; +use mangalord::storage::{LocalStorage, Storage}; +use sqlx::PgPool; +use tempfile::TempDir; +use uuid::Uuid; + +/// Seed a manga → chapter → page chain whose page points at `storage_key`, +/// and return the page id. +async fn seed_page(pool: &PgPool, storage_key: &str) -> Uuid { + let manga_id: Uuid = + sqlx::query_scalar("INSERT INTO mangas (title) VALUES ('M') RETURNING id") + .fetch_one(pool) + .await + .unwrap(); + let chapter_id: Uuid = sqlx::query_scalar( + "INSERT INTO chapters (manga_id, number) VALUES ($1, 1) RETURNING id", + ) + .bind(manga_id) + .fetch_one(pool) + .await + .unwrap(); + sqlx::query_scalar( + "INSERT INTO pages (chapter_id, page_number, storage_key, content_type) \ + VALUES ($1, 1, $2, 'image/png') RETURNING id", + ) + .bind(chapter_id) + .bind(storage_key) + .fetch_one(pool) + .await + .unwrap() +} + +fn ocr_dispatcher( + pool: &PgPool, + storage: Arc, + lines: &[&str], +) -> OcrAnalyzeDispatcher { + OcrAnalyzeDispatcher { + db: pool.clone(), + storage, + engine: StubOcrEngine::new(lines), + max_image_bytes: 8 * 1024 * 1024, + } +} + +#[sqlx::test(migrations = "./migrations")] +async fn dispatch_persists_ocr_lines_and_search_doc(pool: PgPool) { + let dir = TempDir::new().unwrap(); + let storage: Arc = Arc::new(LocalStorage::new(dir.path())); + let key = "mangas/x/p1.png"; + storage.put(key, &common::fake_png_bytes()).await.unwrap(); + let page_id = seed_page(&pool, key).await; + + let dispatcher = ocr_dispatcher(&pool, Arc::clone(&storage), &["Hello there", "general"]); + dispatcher.dispatch(page_id).await.unwrap(); + + // Two OCR rows, in order, with the recognized text. + let rows: Vec<(String, i32)> = sqlx::query_as( + "SELECT text, ord FROM page_ocr_text WHERE page_id = $1 ORDER BY ord", + ) + .bind(page_id) + .fetch_all(&pool) + .await + .unwrap(); + assert_eq!(rows.len(), 2); + assert_eq!(rows[0].0, "Hello there"); + assert_eq!(rows[1].0, "general"); + + // The analysis row is `done`, stamped with the ocrs model label, and has a + // non-empty tsvector so text search works. + let row = repo::page_analysis::load(&pool, page_id).await.unwrap().unwrap(); + assert_eq!(row.status, AnalysisStatus::Done); + assert_eq!(row.model.as_deref(), Some("ocrs")); + let has_doc: bool = sqlx::query_scalar( + "SELECT search_doc IS NOT NULL AND search_doc != ''::tsvector \ + FROM page_analysis WHERE page_id = $1", + ) + .bind(page_id) + .fetch_one(&pool) + .await + .unwrap(); + assert!(has_doc, "search_doc must be populated from OCR text"); +} + +#[sqlx::test(migrations = "./migrations")] +async fn dispatch_missing_page_is_noop(pool: PgPool) { + let dir = TempDir::new().unwrap(); + let storage: Arc = Arc::new(LocalStorage::new(dir.path())); + // A page id that was never inserted — the dispatcher must treat it as a + // deleted page and succeed without writing anything. + let dispatcher = ocr_dispatcher(&pool, storage, &["whatever"]); + dispatcher.dispatch(Uuid::new_v4()).await.unwrap(); +} diff --git a/backend/tests/api_page_tags.rs b/backend/tests/api_page_tags.rs index c855254..6bb19a2 100644 --- a/backend/tests/api_page_tags.rs +++ b/backend/tests/api_page_tags.rs @@ -680,24 +680,123 @@ async fn aggregate_rejects_invalid_order(pool: PgPool) { } #[sqlx::test(migrations = "./migrations")] -async fn aggregate_with_text_param_is_501_with_stable_code(pool: PgPool) { - // OCR text search isn't built yet; the param is accepted so adding - // OCR won't break the wire shape, but rejected with a distinct - // status + code. The code is the wire contract — clients pin on - // `text_search_not_yet_supported`, not the message. - let h = common::harness(pool); +async fn aggregate_with_text_param_filters_by_ocr(pool: PgPool) { + // OCR text search: a page tagged `funny` whose OCR contains "guts" is + // returned for `&text=guts` on both aggregation endpoints, and excluded + // for a query its OCR doesn't contain. The filter runs against the same + // precomputed `search_doc` the OCR worker writes via `persist_analysis`. + let h = common::harness(pool.clone()); let (_, cookie) = common::register_user(&h.app).await; + let manga_id = common::seed_manga_via_api(&h.app, &cookie, "M").await; + let (_, page_id) = seed_chapter_with_page(&h.app, &cookie, manga_id, 1).await; + assert_eq!(add_tag(&h.app, &cookie, &page_id, "funny").await, StatusCode::CREATED); + + // Persist OCR text exactly as the ocrs backend would (via the same mapper). + let page_uuid = Uuid::parse_str(&page_id).unwrap(); + let analysis = + mangalord::analysis::ocr::lines_to_analysis(vec!["spilling the guts here".to_string()]); + mangalord::repo::page_analysis::persist_analysis(&pool, page_uuid, &analysis, "ocrs") + .await + .unwrap(); + + for endpoint in ["chapters", "mangas"] { + // Matching text → the tagged+OCR'd row is returned. + let resp = h + .app + .clone() + .oneshot(common::get_with_cookie( + &format!("/api/v1/me/page-tags/{endpoint}?tag=funny&text=guts"), + &cookie, + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK, "{endpoint} matching"); + let body = common::body_json(resp).await; + assert_eq!(body["items"].as_array().unwrap().len(), 1, "{endpoint} matching items"); + assert_eq!(body["page"]["total"], 1, "{endpoint} matching total"); + + // Non-matching text → excluded. + let resp = h + .app + .clone() + .oneshot(common::get_with_cookie( + &format!("/api/v1/me/page-tags/{endpoint}?tag=funny&text=zzzznomatch"), + &cookie, + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK, "{endpoint} non-matching"); + let body = common::body_json(resp).await; + assert_eq!(body["items"].as_array().unwrap().len(), 0, "{endpoint} non-matching items"); + assert_eq!(body["page"]["total"], 0, "{endpoint} non-matching total"); + } +} + +#[sqlx::test(migrations = "./migrations")] +async fn aggregate_text_param_counts_only_matching_pages(pool: PgPool) { + // A chapter with TWO tagged pages where only ONE page's OCR matches the + // text. `match_count` / `total` must reflect the filtered count (1), not + // the tag-only count (2), and the sample thumbnails must contain only the + // matching page. This is what a single-page test can't distinguish — it + // pins the JOIN/placeholder wiring and the sample-subquery filter. + let h = common::harness(pool.clone()); + let (_, cookie) = common::register_user(&h.app).await; + let manga_id = common::seed_manga_via_api(&h.app, &cookie, "M").await; + let (_, page_ids) = seed_chapter_with_n_pages(&h.app, &cookie, manga_id, 1, 2).await; + for pid in &page_ids { + assert_eq!(add_tag(&h.app, &cookie, pid, "funny").await, StatusCode::CREATED); + } + + // Page 0 OCR contains "guts"; page 1 OCR contains only "filler". + let p0 = Uuid::parse_str(&page_ids[0]).unwrap(); + let p1 = Uuid::parse_str(&page_ids[1]).unwrap(); + let a0 = mangalord::analysis::ocr::lines_to_analysis(vec!["the guts spill out".to_string()]); + let a1 = mangalord::analysis::ocr::lines_to_analysis(vec!["just filler text".to_string()]); + mangalord::repo::page_analysis::persist_analysis(&pool, p0, &a0, "ocrs").await.unwrap(); + mangalord::repo::page_analysis::persist_analysis(&pool, p1, &a1, "ocrs").await.unwrap(); + let resp = h .app + .clone() .oneshot(common::get_with_cookie( "/api/v1/me/page-tags/chapters?tag=funny&text=guts", &cookie, )) .await .unwrap(); - assert_eq!(resp.status(), StatusCode::NOT_IMPLEMENTED); + assert_eq!(resp.status(), StatusCode::OK); let body = common::body_json(resp).await; - assert_eq!(body["error"]["code"], "text_search_not_yet_supported"); + let items = body["items"].as_array().unwrap(); + assert_eq!(items.len(), 1, "one chapter row"); + assert_eq!(body["page"]["total"], 1); + // Filtered match_count is the matching-page count, not the tag count. + assert_eq!(items[0]["match_count"], 1, "only one page's OCR matches"); + // Sample thumbnails reflect the filter: only the matching page's key. + let samples = items[0]["sample_storage_keys"].as_array().unwrap(); + assert_eq!(samples.len(), 1, "thumbnails restricted to matching pages"); +} + +#[sqlx::test(migrations = "./migrations")] +async fn aggregate_blank_text_param_is_tag_only(pool: PgPool) { + // A blank `text=` must not filter — it falls back to tag-only aggregation + // (the page has a tag but no analysis row at all). + let h = common::harness(pool); + let (_, cookie) = common::register_user(&h.app).await; + let manga_id = common::seed_manga_via_api(&h.app, &cookie, "M").await; + let (_, page_id) = seed_chapter_with_page(&h.app, &cookie, manga_id, 1).await; + assert_eq!(add_tag(&h.app, &cookie, &page_id, "funny").await, StatusCode::CREATED); + + let resp = h + .app + .oneshot(common::get_with_cookie( + "/api/v1/me/page-tags/chapters?tag=funny&text=", + &cookie, + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + let body = common::body_json(resp).await; + assert_eq!(body["items"].as_array().unwrap().len(), 1); } #[sqlx::test(migrations = "./migrations")] diff --git a/docker-compose.yml b/docker-compose.yml index 64ebdc9..6dc0fb9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -143,8 +143,15 @@ services: # when both are present. Leave unset for the bundled # HashedControlPassword path above. CRAWLER_TOR_CONTROL_COOKIE_PATH: ${CRAWLER_TOR_CONTROL_COOKIE_PATH:-} - # Analysis worker (vision) — boot seeds + env-only API key. + # Analysis worker — boot seeds + env-only API key. ANALYSIS_ENABLED: ${ANALYSIS_ENABLED:-false} + # Engine selection (deploy-time): `ocr` (in-process ocrs, default). + # The `vision` (local LLM) backend is temporarily disabled — the worker + # forces OCR regardless, so a `vision` override here is currently inert. + # Model paths default to the image-baked /models. + ANALYSIS_BACKEND: ${ANALYSIS_BACKEND:-ocr} + OCRS_DETECTION_MODEL: ${OCRS_DETECTION_MODEL:-/models/text-detection.rten} + OCRS_RECOGNITION_MODEL: ${OCRS_RECOGNITION_MODEL:-/models/text-recognition.rten} ANALYSIS_VISION_URL: ${ANALYSIS_VISION_URL:-} ANALYSIS_VISION_MODEL: ${ANALYSIS_VISION_MODEL:-} ANALYSIS_WORKERS: ${ANALYSIS_WORKERS:-1} diff --git a/frontend/package.json b/frontend/package.json index 50639c3..5478056 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "mangalord-frontend", - "version": "0.89.0", + "version": "0.90.0", "private": true, "type": "module", "scripts": { diff --git a/frontend/src/lib/components/analysis/AnalysisHistoryTable.svelte b/frontend/src/lib/components/analysis/AnalysisHistoryTable.svelte index 9a9bb91..ef88637 100644 --- a/frontend/src/lib/components/analysis/AnalysisHistoryTable.svelte +++ b/frontend/src/lib/components/analysis/AnalysisHistoryTable.svelte @@ -21,7 +21,6 @@ let total = $state(0); let page = $state(1); let statusFilter = $state<'' | 'done' | 'failed'>(''); - let nsfwOnly = $state(false); let search = $state(''); let loading = $state(true); let error = $state(null); @@ -34,7 +33,6 @@ try { const resp = await listAnalysisHistory({ status: statusFilter || undefined, - nsfw: nsfwOnly, search: search.trim() || undefined, limit: LIMIT, offset: (page - 1) * LIMIT @@ -86,15 +84,6 @@ - {r.status} {r.manga_title} · Ch {r.chapter_number} · p{r.page_number} - {#if r.is_nsfw}⚠ NSFW{/if} {r.model ?? '—'} {fmtDuration(r.duration_ms)} @@ -169,13 +157,6 @@ color: var(--text); font-size: var(--font-sm); } - .nsfw { - display: inline-flex; - align-items: center; - gap: var(--space-1); - font-size: var(--font-sm); - color: var(--text-muted); - } .muted { color: var(--text-muted); } @@ -218,11 +199,6 @@ background: color-mix(in srgb, var(--danger) 16%, transparent); color: var(--danger); } - .nsfw-tag { - font-size: var(--font-xs); - color: #b85e1a; - margin-left: var(--space-1); - } .error { color: var(--danger); } diff --git a/frontend/src/lib/components/analysis/AnalysisHistoryTable.svelte.test.ts b/frontend/src/lib/components/analysis/AnalysisHistoryTable.svelte.test.ts new file mode 100644 index 0000000..a9d7595 --- /dev/null +++ b/frontend/src/lib/components/analysis/AnalysisHistoryTable.svelte.test.ts @@ -0,0 +1,64 @@ +import { describe, it, expect, vi, afterEach } from 'vitest'; +import { render, screen, cleanup, waitFor } from '@testing-library/svelte'; + +// Mock the admin API so the component's onMount load() is deterministic. +const listAnalysisHistory = vi.fn(); +vi.mock('$lib/api/admin', () => ({ + listAnalysisHistory: (...args: unknown[]) => listAnalysisHistory(...args) +})); + +import AnalysisHistoryTable from './AnalysisHistoryTable.svelte'; + +afterEach(() => { + cleanup(); + listAnalysisHistory.mockReset(); +}); + +function row(over: Record = {}) { + return { + page_id: 'p1', + page_number: 1, + chapter_id: 'c1', + chapter_number: 5, + manga_id: 'm1', + manga_title: 'Berserk', + status: 'done', + is_nsfw: true, // even when the backend reports nsfw, the OCR UI must not surface it + model: 'ocrs', + error: null, + analyzed_at: '2026-01-02T00:00:00Z', + duration_ms: 1234, + ...over + }; +} + +function resolveOnce(rows: ReturnType[]) { + listAnalysisHistory.mockResolvedValue({ + items: rows, + page: { total: rows.length } + }); +} + +describe('AnalysisHistoryTable (OCR)', () => { + it('does not render the NSFW-only filter', async () => { + resolveOnce([row()]); + render(AnalysisHistoryTable, { props: { onOpenDetail: () => {} } }); + await waitFor(() => expect(listAnalysisHistory).toHaveBeenCalled()); + expect(screen.queryByTestId('analysis-history-nsfw')).toBeNull(); + }); + + it('does not pass an nsfw filter to the history query', async () => { + resolveOnce([row()]); + render(AnalysisHistoryTable, { props: { onOpenDetail: () => {} } }); + await waitFor(() => expect(listAnalysisHistory).toHaveBeenCalled()); + const arg = listAnalysisHistory.mock.calls[0][0] as Record; + expect('nsfw' in arg).toBe(false); + }); + + it('never shows an NSFW tag on a row, even if the row is flagged', async () => { + resolveOnce([row({ is_nsfw: true })]); + render(AnalysisHistoryTable, { props: { onOpenDetail: () => {} } }); + await waitFor(() => expect(screen.getByText(/Berserk/)).toBeTruthy()); + expect(screen.queryByText(/NSFW/i)).toBeNull(); + }); +}); diff --git a/frontend/src/lib/components/analysis/AnalysisMetricsPanel.svelte b/frontend/src/lib/components/analysis/AnalysisMetricsPanel.svelte index b76a64c..e4ae37a 100644 --- a/frontend/src/lib/components/analysis/AnalysisMetricsPanel.svelte +++ b/frontend/src/lib/components/analysis/AnalysisMetricsPanel.svelte @@ -105,30 +105,6 @@ {metrics.failed} - -

By model

- {#if metrics.by_model.length === 0} -

No completed analyses in this window.

- {:else} - - - - - - - - - - {#each metrics.by_model as m (m.model)} - - - - - - {/each} - -
ModelAvgN
{m.model ?? '(unknown)'}{fmtDuration(m.avg_ms)}{m.n}
- {/if} {/if} @@ -186,29 +162,6 @@ font-weight: var(--weight-semibold); font-variant-numeric: tabular-nums; } - h2 { - margin: 0 0 var(--space-2); - font-size: var(--font-sm); - color: var(--text-muted); - text-transform: uppercase; - letter-spacing: 0.04em; - } - table { - width: 100%; - border-collapse: collapse; - max-width: 32rem; - } - th, - td { - padding: var(--space-2); - text-align: left; - border-bottom: 1px solid var(--border); - font-size: var(--font-sm); - } - .num { - text-align: right; - font-variant-numeric: tabular-nums; - } .muted { color: var(--text-muted); } diff --git a/frontend/src/lib/components/analysis/AnalysisMetricsPanel.svelte.test.ts b/frontend/src/lib/components/analysis/AnalysisMetricsPanel.svelte.test.ts new file mode 100644 index 0000000..0c86782 --- /dev/null +++ b/frontend/src/lib/components/analysis/AnalysisMetricsPanel.svelte.test.ts @@ -0,0 +1,40 @@ +import { describe, it, expect, vi, afterEach } from 'vitest'; +import { render, screen, cleanup, waitFor } from '@testing-library/svelte'; + +const getAnalysisMetrics = vi.fn(); +const getAnalysisMetricsSeries = vi.fn(); +vi.mock('$lib/api/admin', () => ({ + getAnalysisMetrics: (...a: unknown[]) => getAnalysisMetrics(...a), + getAnalysisMetricsSeries: (...a: unknown[]) => getAnalysisMetricsSeries(...a) +})); + +import AnalysisMetricsPanel from './AnalysisMetricsPanel.svelte'; + +afterEach(() => { + cleanup(); + getAnalysisMetrics.mockReset(); + getAnalysisMetricsSeries.mockReset(); +}); + +describe('AnalysisMetricsPanel (OCR)', () => { + it('renders the aggregate tiles but not the by-model table', async () => { + getAnalysisMetrics.mockResolvedValue({ + n: 10, + ok: 9, + failed: 1, + avg_ms: 1200, + // Even when the API returns a by-model breakdown, OCR has a single + // engine so the panel must not surface the table. + by_model: [{ model: 'ocrs', avg_ms: 1200, n: 10 }] + }); + getAnalysisMetricsSeries.mockResolvedValue({ buckets: [] }); + + render(AnalysisMetricsPanel); + + await waitFor(() => + expect(screen.getByTestId('analysis-metrics-n').textContent).toContain('10') + ); + expect(screen.queryByText(/By model/i)).toBeNull(); + expect(screen.queryByRole('table')).toBeNull(); + }); +}); diff --git a/frontend/src/routes/admin/analysis/+page.svelte b/frontend/src/routes/admin/analysis/+page.svelte index cdad181..99ead41 100644 --- a/frontend/src/routes/admin/analysis/+page.svelte +++ b/frontend/src/routes/admin/analysis/+page.svelte @@ -533,9 +533,9 @@

- Coverage of the AI analysis worker (OCR, auto-tags, scene description, - NSFW moderation). Browse what's analyzed, queue more, and inspect any - page's result. Updates stream live as pages are queued and processed. + Coverage of the OCR text-extraction worker. Browse which pages have had + their text extracted, queue more, and inspect any page's OCR result. + Updates stream live as pages are queued and processed.

@@ -818,37 +818,12 @@

{detail.error ?? 'Analysis failed.'}

{/if} - {#if detail.is_nsfw || detail.content_warnings.length > 0} -
- ⚠ NSFW - {#each detail.content_warnings as w (w)} - {w} - {/each} -
- {/if} - - {#if detail.scene_description} -
-

Scene

-

{detail.scene_description}

-
- {/if} - - {#if detail.tags.length > 0} -
-

Tags

-
- {#each detail.tags as t (t)}{t}{/each} -
-
- {/if} - {#if detail.ocr.length > 0}

OCR text

    {#each detail.ocr as line, i (i)} -
  • {line.kind} {line.text}
    • +
  • {#if line.kind}{line.kind} {/if}{line.text}
    • {/each}
@@ -1186,28 +1161,6 @@ background: color-mix(in srgb, var(--danger) 16%, transparent); color: var(--danger); } - .warnings { - display: flex; - flex-wrap: wrap; - align-items: center; - gap: var(--space-2); - padding: var(--space-2) var(--space-3); - border-radius: var(--radius-md); - background: color-mix(in srgb, #d4762a 12%, transparent); - margin-bottom: var(--space-3); - } - .cw-label { - font-weight: 600; - color: #b85e1a; - font-size: var(--font-sm); - } - .cw-chip { - font-size: var(--font-sm); - padding: 1px var(--space-2); - border-radius: var(--radius-sm); - background: color-mix(in srgb, #d4762a 22%, transparent); - text-transform: capitalize; - } .block { margin-bottom: var(--space-3); } @@ -1218,17 +1171,6 @@ letter-spacing: 0.04em; margin: 0 0 var(--space-1); } - .tag-row { - display: flex; - flex-wrap: wrap; - gap: var(--space-1); - } - .tag { - font-size: var(--font-sm); - padding: 1px var(--space-2); - border-radius: var(--radius-pill); - background: var(--surface-elevated); - } .ocr { list-style: none; margin: 0; diff --git a/frontend/src/routes/admin/settings/+page.svelte b/frontend/src/routes/admin/settings/+page.svelte index 3de6cbe..07b2661 100644 --- a/frontend/src/routes/admin/settings/+page.svelte +++ b/frontend/src/routes/admin/settings/+page.svelte @@ -9,8 +9,7 @@ updateAnalysisSettings, type CrawlerSettings, type CrawlerEnvOnly, - type AnalysisSettings, - type PromptDefaults + type AnalysisSettings } from '$lib/api/admin'; type Tab = 'crawler' | 'analysis'; @@ -21,10 +20,11 @@ let crawlerEnv = $state(null); let allowlistText = $state(''); // textarea mirror of download_allowlist - // Analysis state + // Analysis state. The full settings object is loaded and saved as-is; + // vision-only fields (endpoint, model, prompts, slicing, sampling) ride + // along unchanged while the OCR backend is active — only the OCR-relevant + // knobs are surfaced below. let analysis = $state(null); - let promptDefaults = $state(null); - let apiKeyConfigured = $state(false); let loading = $state(true); let loadError = $state(null); @@ -46,8 +46,6 @@ crawlerEnv = c.env_only; allowlistText = c.editable.download_allowlist.join('\n'); analysis = a.editable; - promptDefaults = a.prompt_defaults; - apiKeyConfigured = a.env_only.api_key_configured; } catch (e) { loadError = e instanceof Error ? e.message : 'Failed to load settings.'; } finally { @@ -96,8 +94,6 @@ } else if (tab === 'analysis' && analysis) { const r = await updateAnalysisSettings(analysis); analysis = r.editable; - promptDefaults = r.prompt_defaults; - apiKeyConfigured = r.env_only.api_key_configured; } success = 'Saved and applied. The subsystem was restarted with the new settings.'; } catch (e) { @@ -113,9 +109,6 @@ } } - function resetPrompt(which: 'system_prompt' | 'ocr_prompt' | 'grounding_prompt') { - if (analysis) analysis[which] = null; - }

Settings

@@ -322,145 +315,29 @@ {:else if tab === 'analysis' && analysis}
- Endpoint & model - - + Engine

- API key: {apiKeyConfigured - ? 'configured via environment' - : 'not set'} (managed via environment). + Backend: OCR (ocrs) — in-process text extraction. Selected via + environment; vision is temporarily disabled.

- Workers & timeouts + Worker - -
- -
- Sampling & output - - - - -
- -
- Image slicing - - - - -
- -
- Prompts - {#each [['system_prompt', 'System prompt'], ['ocr_prompt', 'OCR prompt (slices)'], ['grounding_prompt', 'Grounding prompt']] as [key, label] (key)} - {@const k = key as 'system_prompt' | 'ocr_prompt' | 'grounding_prompt'} -
-
- {label} - -
- - - {analysis[k] == null - ? 'Using compiled default (shown as placeholder).' - : `${analysis[k]?.length ?? 0} chars (override).`} - -
- {/each} -
{/if} {/if} @@ -564,7 +441,6 @@ color: var(--text-muted); } .field input, - .field select, .field textarea { padding: var(--space-2) var(--space-3); border: 1px solid var(--border); @@ -584,39 +460,6 @@ color: var(--danger, #dc2626); font-size: var(--font-xs); } - .mono { - font-family: ui-monospace, SFMono-Regular, Menlo, monospace; - font-size: var(--font-xs); - width: 100%; - box-sizing: border-box; - padding: var(--space-2); - border: 1px solid var(--border); - border-radius: var(--radius-md); - background: var(--surface); - color: var(--text); - } - .prompt { - display: grid; - gap: var(--space-1); - } - .prompt-head { - display: flex; - justify-content: space-between; - align-items: center; - font-size: var(--font-sm); - } - button.link { - background: none; - border: none; - color: var(--primary); - cursor: pointer; - font-size: var(--font-xs); - padding: 0; - } - button.link:disabled { - color: var(--text-muted); - cursor: not-allowed; - } .env dl { display: grid; grid-template-columns: max-content 1fr;