Compare commits
31 Commits
679abae736
...
2b7a11b480
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2b7a11b480 | ||
|
|
d3b827421f | ||
|
|
25aba3ac58 | ||
|
|
85d65f5eda | ||
|
|
ab9b8fb172 | ||
|
|
8b5bd99446 | ||
|
|
d0cd31c9a7 | ||
|
|
6bb72b8775 | ||
|
|
824f5acf22 | ||
|
|
268e8cc6c2 | ||
|
|
7e675b72cc | ||
|
|
8b7ea2e1b2 | ||
|
|
9607488278 | ||
|
|
d36f24e9af | ||
|
|
bd39476ac7 | ||
|
|
af870bd157 | ||
|
|
7d80a437bf | ||
|
|
dbde6e02c4 | ||
|
|
2bbf1595ff | ||
|
|
63e1aa5484 | ||
|
|
f30600162e | ||
|
|
33f684887d | ||
|
|
6c901e64c9 | ||
|
|
9910a0a995 | ||
|
|
00577071fd | ||
|
|
1e3fd27308 | ||
|
|
f5842510b7 | ||
|
|
780632bee3 | ||
|
|
be6b974150 | ||
|
|
6dcb720a0f | ||
|
|
d6ac648ac9 |
53
.env.example
53
.env.example
@@ -52,6 +52,23 @@ AUTH_RATE_BURST=10
|
||||
# on different hosts. Example: https://app.example.com,https://app.example.de
|
||||
CORS_ALLOWED_ORIGINS=
|
||||
|
||||
# ----- Admin CSRF allowlist -----
|
||||
# Browser origins (scheme + host[:port]) permitted to POST to
|
||||
# /api/v1/admin/* mutating endpoints. Defends the session-cookie-
|
||||
# authenticated admin surface against SameSite=Lax form-POST CSRF.
|
||||
# Same shape as CORS_ALLOWED_ORIGINS (comma-separated). Compare against
|
||||
# the request's Origin header (falling back to Referer when absent);
|
||||
# safe methods (GET/HEAD/OPTIONS) are not checked, and requests with
|
||||
# neither Origin nor Referer (curl, server-to-server callers) are
|
||||
# always allowed.
|
||||
#
|
||||
# Default is empty: CSRF check disabled (operator opt-out). For a
|
||||
# browser-exposed deployment this should be set to the SvelteKit
|
||||
# origin, e.g. https://app.example.com. For a same-origin
|
||||
# docker-compose deploy where only one origin exists, set the same
|
||||
# value the browser uses.
|
||||
ADMIN_ALLOWED_ORIGINS=
|
||||
|
||||
# ----- Upload limits -----
|
||||
# Per-request body cap. axum rejects oversized requests with 413 before
|
||||
# our handlers run. Default 200 MiB.
|
||||
@@ -78,6 +95,20 @@ CRAWLER_MAX_IMAGE_BYTES=33554432
|
||||
# and the `bin/crawler` CLI). 0 means no cap — let the source walker run
|
||||
# to completion. Useful for capped test runs against a new source.
|
||||
CRAWLER_LIMIT=0
|
||||
|
||||
# ----- Crawler reliability knobs -----
|
||||
# Hard upper bound on a single chapter-content job dispatch (seconds).
|
||||
# A job that exceeds the budget is acked failed (with exponential
|
||||
# backoff) instead of wedging a worker. Default 600s.
|
||||
CRAWLER_JOB_TIMEOUT_SECS=600
|
||||
# Consecutive metadata-pass `fetch_manga` failures that abort the pass
|
||||
# (circuit breaker for a source outage). The pass does NOT mark a clean
|
||||
# exit, so the next tick does a recovery sweep. Default 10.
|
||||
CRAWLER_METADATA_MAX_CONSECUTIVE_FAILURES=10
|
||||
# Consecutive transient chapter failures (after TOR recircuit is
|
||||
# exhausted) that trigger an automatic coordinated browser restart.
|
||||
# Default 3.
|
||||
CRAWLER_BROWSER_RESTART_THRESHOLD=3
|
||||
# Path to a system Chromium binary. When set, the crawler skips the
|
||||
# bundled-fetcher download. Required on platforms without a usable
|
||||
# upstream Chromium build (notably Linux_arm64 / Raspberry Pi). On
|
||||
@@ -136,3 +167,25 @@ BACKEND_URL=http://backend:8080
|
||||
# 25 Mbps; raise for users on slower upstream links or lower if a
|
||||
# tighter front proxy already bounds the request lifetime.
|
||||
BACKEND_PROXY_TIMEOUT_MS=300000
|
||||
|
||||
# ----- Runtime settings (crawler + analysis) -----
|
||||
# The crawler and analysis subsystems are configured at runtime from the
|
||||
# `app_settings` table and edited live in the admin dashboard
|
||||
# (Admin → Settings) — no restart needed. The CRAWLER_* / ANALYSIS_* env
|
||||
# vars below act as the BOOT SEED: on first boot (when the row is absent)
|
||||
# the env values populate the DB; thereafter the DB is the source of
|
||||
# truth and env changes are ignored for those fields.
|
||||
#
|
||||
# Host/infra and secret fields stay env-ONLY (never persisted, shown
|
||||
# read-only in the dashboard): the chromium binary/dir, browser mode/args,
|
||||
# CRAWLER_PROXY, all CRAWLER_TOR_CONTROL_*, the cookie domain, and the
|
||||
# analysis ANALYSIS_API_KEY. The important site levers (PRIVATE_MODE,
|
||||
# ALLOW_SELF_REGISTER) also remain env-only by design.
|
||||
#
|
||||
# New analysis knobs (all optional, all seed defaults):
|
||||
# ANALYSIS_TEMPERATURE Sampling temperature. Default 0 (deterministic).
|
||||
# ANALYSIS_SYSTEM_PROMPT Override the single-call vision system prompt.
|
||||
# ANALYSIS_OCR_PROMPT Override the tall-page OCR (pass A) prompt.
|
||||
# ANALYSIS_GROUNDING_PROMPT Override the tags/scene/safety (pass B) prompt.
|
||||
# Leave the prompt vars unset to use the built-in defaults (also editable,
|
||||
# with a per-prompt "reset to default", in the dashboard).
|
||||
|
||||
6
.gitignore
vendored
6
.gitignore
vendored
@@ -20,6 +20,12 @@
|
||||
.env
|
||||
.env.local
|
||||
.env.*.local
|
||||
.env.dev
|
||||
|
||||
# Node install / Playwright output when run from the repo root (the
|
||||
# canonical copies live under /frontend, already ignored above).
|
||||
/node_modules
|
||||
/test-results
|
||||
|
||||
# Claude Code (personal overrides only; .claude/settings.json is committed)
|
||||
.claude/settings.local.json
|
||||
|
||||
@@ -136,6 +136,8 @@ docker compose -f docker-compose.dev.yml up -d
|
||||
These are first-class slots in the architecture. When adding any of them, plug into the existing seam rather than building parallel infrastructure.
|
||||
|
||||
- **Tags / lists**: new tables joined to `mangas`. New `domain`, `repo`, and `api` modules; the existing manga endpoints do not need to change.
|
||||
- **Per-page collections / tags**: `collections` is heterogeneous — `collection_mangas` holds whole mangas, `collection_pages` holds individual pages (FK to `pages.id`). Per-user page tags live in `page_tags`, which references the **shared** `tags` table by `tag_id` (migration 0024) — the same lookup table `manga_tags` uses, so manga tags and page tags share one global vocabulary. The HTTP contract still speaks tag *names*; `repo::page_tag` resolves name↔id via `repo::tag::upsert_by_name` and applies the stricter page-tag `normalize_tag` (lowercase, collapse whitespace, reject wildcards/control/invisible chars) at the API layer. Both `collection_pages` and `page_tags` cascade-delete with `pages` and `chapters`, so re-uploading a chapter drops saved-page references by design.
|
||||
- **Tag-based content search (`/search`)**: the user-facing search surface lives at [frontend/src/routes/search/+page.svelte](frontend/src/routes/search/+page.svelte). Three result views (Pages / Chapters / Mangas) consume the matching `/v1/me/page-tags`, `/v1/me/page-tags/chapters`, and `/v1/me/page-tags/mangas` endpoints. Note the two distinct query-param spaces: `?q=` on `/v1/me/page-tags` is a tag-name prefix (for autocomplete in the "Add tag" sheet); `?text=` on the aggregation endpoints is **reserved** for the planned OCR text-search input. Both aggregation handlers accept `text=` on the wire but reject non-empty values with 501 `text_search_not_yet_supported` so adding OCR later doesn't break the API shape. Adding OCR is then: a background worker writes `page_ocr_text` rows, a JOIN on the existing aggregation queries adds the new filter, the `text=` param starts validating instead of rejecting.
|
||||
- **Full-text / fuzzy search**: enable `pg_trgm` in a migration and add a GIN index on `mangas.title`; swap the `WHERE` in `repo::manga::list` to use `%` operator or `tsvector`. The API shape (`?search=...`) does not change.
|
||||
- **OCR / autotagging**: a background worker (a separate binary or a tokio task spawned in `app::build`) that reads pages from `storage::Storage` and writes tag rows. Do not couple OCR to upload handlers — it runs asynchronously.
|
||||
- **S3 storage**: add `storage::S3Storage` implementing `Storage`. Branch in `app::build` based on a config field (e.g., `STORAGE_BACKEND=s3`). Handlers do not change.
|
||||
|
||||
100
backend/Cargo.lock
generated
100
backend/Cargo.lock
generated
@@ -256,12 +256,24 @@ version = "3.20.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb"
|
||||
|
||||
[[package]]
|
||||
name = "bytemuck"
|
||||
version = "1.25.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec"
|
||||
|
||||
[[package]]
|
||||
name = "byteorder"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
|
||||
|
||||
[[package]]
|
||||
name = "byteorder-lite"
|
||||
version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495"
|
||||
|
||||
[[package]]
|
||||
name = "bytes"
|
||||
version = "1.11.1"
|
||||
@@ -745,6 +757,15 @@ version = "2.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6"
|
||||
|
||||
[[package]]
|
||||
name = "fdeflate"
|
||||
version = "0.3.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c"
|
||||
dependencies = [
|
||||
"simd-adler32",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "find-msvc-tools"
|
||||
version = "0.1.9"
|
||||
@@ -1323,6 +1344,32 @@ dependencies = [
|
||||
"icu_properties",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "image"
|
||||
version = "0.25.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104"
|
||||
dependencies = [
|
||||
"bytemuck",
|
||||
"byteorder-lite",
|
||||
"image-webp",
|
||||
"moxcms",
|
||||
"num-traits",
|
||||
"png",
|
||||
"zune-core",
|
||||
"zune-jpeg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "image-webp"
|
||||
version = "0.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "525e9ff3e1a4be2fbea1fdf0e98686a6d98b4d8f937e1bf7402245af1909e8c3"
|
||||
dependencies = [
|
||||
"byteorder-lite",
|
||||
"quick-error",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "indexmap"
|
||||
version = "2.14.0"
|
||||
@@ -1470,7 +1517,7 @@ checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
|
||||
|
||||
[[package]]
|
||||
name = "mangalord"
|
||||
version = "0.52.0"
|
||||
version = "0.80.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"argon2",
|
||||
@@ -1486,6 +1533,7 @@ dependencies = [
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"http-body-util",
|
||||
"image",
|
||||
"infer",
|
||||
"mime",
|
||||
"nix 0.29.0",
|
||||
@@ -1582,6 +1630,16 @@ dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "moxcms"
|
||||
version = "0.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b"
|
||||
dependencies = [
|
||||
"num-traits",
|
||||
"pxfm",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "multer"
|
||||
version = "3.1.0"
|
||||
@@ -2078,6 +2136,19 @@ version = "0.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6"
|
||||
|
||||
[[package]]
|
||||
name = "png"
|
||||
version = "0.18.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"crc32fast",
|
||||
"fdeflate",
|
||||
"flate2",
|
||||
"miniz_oxide",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "potential_utf"
|
||||
version = "0.1.5"
|
||||
@@ -2143,6 +2214,18 @@ dependencies = [
|
||||
"psl-types",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pxfm"
|
||||
version = "0.1.29"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e0c5ccf5294c6ccd63a74f1565028353830a9c2f5eb0c682c355c471726a6e3f"
|
||||
|
||||
[[package]]
|
||||
name = "quick-error"
|
||||
version = "2.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3"
|
||||
|
||||
[[package]]
|
||||
name = "quinn"
|
||||
version = "0.11.9"
|
||||
@@ -4169,3 +4252,18 @@ name = "zmij"
|
||||
version = "1.0.21"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa"
|
||||
|
||||
[[package]]
|
||||
name = "zune-core"
|
||||
version = "0.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cb8a0807f7c01457d0379ba880ba6322660448ddebc890ce29bb64da71fb40f9"
|
||||
|
||||
[[package]]
|
||||
name = "zune-jpeg"
|
||||
version = "0.5.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "27bc9d5b815bc103f142aa054f561d9187d191692ec7c2d1e2b4737f8dbd7296"
|
||||
dependencies = [
|
||||
"zune-core",
|
||||
]
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "mangalord"
|
||||
version = "0.52.0"
|
||||
version = "0.80.0"
|
||||
edition = "2021"
|
||||
default-run = "mangalord"
|
||||
|
||||
@@ -37,6 +37,9 @@ rand = "0.8"
|
||||
sha2 = "0.10"
|
||||
subtle = "2"
|
||||
base64 = "0.22"
|
||||
# Image decode + downscale for the analysis worker (keep the page image
|
||||
# under the local vision model's token budget). Only the manga page formats.
|
||||
image = { version = "0.25", default-features = false, features = ["jpeg", "png", "webp"] }
|
||||
axum-extra = { version = "0.9", features = ["cookie", "typed-header"] }
|
||||
time = "0.3"
|
||||
infer = "0.16"
|
||||
@@ -48,7 +51,7 @@ chromiumoxide = { version = "0.7", features = ["tokio-runtime", "_fetcher-rusttl
|
||||
sysinfo = { version = "0.32", default-features = false, features = ["system"] }
|
||||
nix = { version = "0.29", features = ["fs"] }
|
||||
scraper = "0.20"
|
||||
reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "socks", "cookies", "stream"] }
|
||||
reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "socks", "cookies", "stream", "json"] }
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3"
|
||||
|
||||
19
backend/migrations/0022_crawler_observability_indexes.sql
Normal file
19
backend/migrations/0022_crawler_observability_indexes.sql
Normal file
@@ -0,0 +1,19 @@
|
||||
-- Partial indexes that back the admin crawler dashboard hot reads:
|
||||
-- * mangas with no cover — drives count_missing_covers /
|
||||
-- list_missing_cover_mangas (the cover backlog the metadata pass drains).
|
||||
-- * dead jobs — drives list_dead_jobs / requeue_dead_jobs.
|
||||
-- Both filters are highly selective in steady state (the working sets are a
|
||||
-- tiny fraction of the full tables), so the partials stay small and hot.
|
||||
-- ORDER BY updated_at DESC matches the LIMIT/OFFSET page reads.
|
||||
--
|
||||
-- Not CONCURRENTLY: sqlx::migrate! wraps each migration in a transaction;
|
||||
-- CREATE INDEX CONCURRENTLY can't run inside one. Tables are small at our
|
||||
-- scale (online deploy still safe).
|
||||
|
||||
CREATE INDEX IF NOT EXISTS mangas_missing_cover_idx
|
||||
ON mangas (updated_at DESC)
|
||||
WHERE cover_image_path IS NULL;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS crawler_jobs_dead_idx
|
||||
ON crawler_jobs (updated_at DESC)
|
||||
WHERE state = 'dead';
|
||||
48
backend/migrations/0023_page_collections_and_tags.sql
Normal file
48
backend/migrations/0023_page_collections_and_tags.sql
Normal file
@@ -0,0 +1,48 @@
|
||||
-- Per-page collections and tags. Collections become heterogeneous: the
|
||||
-- existing `collection_mangas` join holds whole mangas, this new
|
||||
-- `collection_pages` join holds individual pages. A page tagged or
|
||||
-- collected references the stable `pages.id` UUID, so re-uploading a
|
||||
-- chapter (which deletes and recreates the page rows) silently drops
|
||||
-- any saves attached to it — cascade is intentional, matching the
|
||||
-- semantics of the cover-image references in existing collections.
|
||||
|
||||
CREATE TABLE collection_pages (
|
||||
collection_id uuid NOT NULL REFERENCES collections(id) ON DELETE CASCADE,
|
||||
page_id uuid NOT NULL REFERENCES pages(id) ON DELETE CASCADE,
|
||||
added_at timestamptz NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (collection_id, page_id)
|
||||
);
|
||||
|
||||
-- Reverse lookup: "which of my collections contain this page?" — the
|
||||
-- reader's context menu pre-checks the matching collection rows on
|
||||
-- open.
|
||||
CREATE INDEX collection_pages_page_idx ON collection_pages (page_id);
|
||||
|
||||
-- Per-user, per-page free-form tags. Distinct from the manga-level
|
||||
-- shared `tags` taxonomy — these are personal annotations. Length cap
|
||||
-- 64 leaves room for `namespace:value` conventions (e.g.
|
||||
-- `character:askeladd`) without making the schema enforce them.
|
||||
CREATE TABLE page_tags (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
page_id uuid NOT NULL REFERENCES pages(id) ON DELETE CASCADE,
|
||||
tag text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
CONSTRAINT page_tags_tag_nonempty CHECK (length(tag) > 0 AND length(tag) <= 64)
|
||||
);
|
||||
|
||||
-- One row per (user, page, tag). The repo upserts via ON CONFLICT DO
|
||||
-- NOTHING so re-tagging is a no-op (handler returns 200 instead of 201).
|
||||
CREATE UNIQUE INDEX page_tags_user_page_tag_uniq
|
||||
ON page_tags (user_id, page_id, tag);
|
||||
|
||||
-- "Show me everything I've tagged" — newest-first. Drives the library
|
||||
-- Page-tags tab.
|
||||
CREATE INDEX page_tags_user_idx ON page_tags (user_id, created_at DESC);
|
||||
|
||||
-- "Show me what I've tagged X" — same tab, when a chip filter is
|
||||
-- active.
|
||||
CREATE INDEX page_tags_user_tag_idx ON page_tags (user_id, tag, created_at DESC);
|
||||
|
||||
-- Per-page lookup for the context menu's contextual line.
|
||||
CREATE INDEX page_tags_page_idx ON page_tags (page_id);
|
||||
48
backend/migrations/0024_normalize_page_tags.sql
Normal file
48
backend/migrations/0024_normalize_page_tags.sql
Normal file
@@ -0,0 +1,48 @@
|
||||
-- Normalize per-page tags. Originally (0023) `page_tags.tag` stored the
|
||||
-- tag inline as free-form text, the lone un-normalized tag concept while
|
||||
-- authors/genres/manga-tags all went through a lookup table + FK. This
|
||||
-- migration folds page tags into the SAME shared `tags` table that
|
||||
-- `manga_tags` uses (0009): `page_tags.tag` becomes `page_tags.tag_id`
|
||||
-- referencing `tags(id)`. After this there is one global tag vocabulary
|
||||
-- shared by manga tags and page tags.
|
||||
--
|
||||
-- The HTTP contract is unchanged — the API still speaks tag *names*; the
|
||||
-- repo resolves name<->id via `repo::tag::upsert_by_name`, exactly as the
|
||||
-- manga-tag path does.
|
||||
|
||||
-- 1. New FK column, nullable until backfilled.
|
||||
ALTER TABLE page_tags
|
||||
ADD COLUMN tag_id uuid REFERENCES tags(id) ON DELETE CASCADE;
|
||||
|
||||
-- 2. Seed the lookup table from the existing inline values, deduping
|
||||
-- case-insensitively via the tags (lower(name)) unique index. Mirrors
|
||||
-- the author backfill in 0009.
|
||||
INSERT INTO tags (name)
|
||||
SELECT DISTINCT tag FROM page_tags
|
||||
ON CONFLICT (lower(name)) DO NOTHING;
|
||||
|
||||
-- 3. Point every page_tags row at its canonical tag row.
|
||||
UPDATE page_tags pt
|
||||
SET tag_id = t.id
|
||||
FROM tags t
|
||||
WHERE lower(t.name) = lower(pt.tag);
|
||||
|
||||
-- 4. The FK is now mandatory.
|
||||
ALTER TABLE page_tags ALTER COLUMN tag_id SET NOT NULL;
|
||||
|
||||
-- 5. Swap the inline-tag indexes/constraints for tag_id equivalents.
|
||||
DROP INDEX IF EXISTS page_tags_user_page_tag_uniq;
|
||||
CREATE UNIQUE INDEX page_tags_user_page_tag_uniq
|
||||
ON page_tags (user_id, page_id, tag_id);
|
||||
|
||||
DROP INDEX IF EXISTS page_tags_user_tag_idx;
|
||||
CREATE INDEX page_tags_user_tag_idx
|
||||
ON page_tags (user_id, tag_id, created_at DESC);
|
||||
|
||||
-- page_tags_user_idx (user_id, created_at DESC) and page_tags_page_idx
|
||||
-- (page_id) are unaffected and stay as-is.
|
||||
|
||||
-- 6. Drop the now-dead inline column. This also drops the
|
||||
-- page_tags_tag_nonempty CHECK; the 1..=64 length bound is now
|
||||
-- enforced by `repo::tag::upsert_by_name` before a row is created.
|
||||
ALTER TABLE page_tags DROP COLUMN tag;
|
||||
81
backend/migrations/0025_page_analysis.sql
Normal file
81
backend/migrations/0025_page_analysis.sql
Normal file
@@ -0,0 +1,81 @@
|
||||
-- AI content-analysis / enrichment / moderation results, one analysis
|
||||
-- pass per page image. A background worker calls a local vision model
|
||||
-- and writes four kinds of output here: OCR text (weighted by kind),
|
||||
-- global auto-tags, a scene description, and content-warning flags.
|
||||
--
|
||||
-- Everything cascades with `pages` (like page_tags / collection_pages in
|
||||
-- 0023): re-uploading a chapter recreates its page rows, intentionally
|
||||
-- dropping the stale analysis so it gets re-enqueued and re-derived.
|
||||
--
|
||||
-- The auto-tags live in their OWN table (`page_auto_tags`), NOT in the
|
||||
-- per-user `page_tags`, so re-analysis is a clean delete+reinsert that
|
||||
-- never touches a user's personal tags. They reference the SAME shared
|
||||
-- `tags` vocabulary (0009 / 0024) that manga tags and page tags use, so
|
||||
-- model-proposed tags and human tags share one global namespace.
|
||||
|
||||
-- One row per analyzed page. `search_doc` is the worker-computed,
|
||||
-- kind-weighted tsvector that the page text-search ranks against; it is
|
||||
-- filled in the same transaction that writes the OCR rows (a generated
|
||||
-- column can't aggregate the child `page_ocr_text` rows, and a trigger
|
||||
-- would re-fire on every child insert during the delete+reinsert).
|
||||
CREATE TABLE page_analysis (
|
||||
page_id uuid PRIMARY KEY REFERENCES pages(id) ON DELETE CASCADE,
|
||||
status text NOT NULL DEFAULT 'pending'
|
||||
CHECK (status IN ('pending', 'done', 'failed')),
|
||||
scene_description text,
|
||||
is_nsfw boolean NOT NULL DEFAULT false,
|
||||
model text,
|
||||
error text,
|
||||
analyzed_at timestamptz,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz NOT NULL DEFAULT now(),
|
||||
search_doc tsvector
|
||||
);
|
||||
|
||||
-- Full-text ranking over the weighted document.
|
||||
CREATE INDEX page_analysis_search_idx ON page_analysis USING gin (search_doc);
|
||||
-- Partial index for "is this page flagged?" lookups (content-warning
|
||||
-- joins and NSFW filters touch only the flagged minority).
|
||||
CREATE INDEX page_analysis_nsfw_idx ON page_analysis (page_id) WHERE is_nsfw;
|
||||
|
||||
-- Extracted text pieces, each tagged with its kind. Kind drives the
|
||||
-- tsvector weight (speech/title=A, narration/thought/caption=B,
|
||||
-- sfx=D; scene_description is weighted C from page_analysis). `ord`
|
||||
-- preserves reading order within the page.
|
||||
CREATE TABLE page_ocr_text (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
page_id uuid NOT NULL REFERENCES pages(id) ON DELETE CASCADE,
|
||||
kind text NOT NULL
|
||||
CHECK (kind IN ('speech', 'thought', 'narration', 'sfx', 'title', 'caption')),
|
||||
text text NOT NULL,
|
||||
ord integer NOT NULL DEFAULT 0
|
||||
);
|
||||
|
||||
CREATE INDEX page_ocr_text_page_idx ON page_ocr_text (page_id);
|
||||
|
||||
-- Global, model-derived page tags — visible to every user and searchable
|
||||
-- alongside personal page tags. Keyed to the shared `tags` table by id.
|
||||
CREATE TABLE page_auto_tags (
|
||||
page_id uuid NOT NULL REFERENCES pages(id) ON DELETE CASCADE,
|
||||
tag_id uuid NOT NULL REFERENCES tags(id) ON DELETE CASCADE,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (page_id, tag_id)
|
||||
);
|
||||
|
||||
-- tag -> pages (search by auto-tag) and page -> tags (render a page's
|
||||
-- auto-tags); the PK already covers (page_id, tag_id) but a standalone
|
||||
-- tag_id index serves the reverse direction.
|
||||
CREATE INDEX page_auto_tags_tag_idx ON page_auto_tags (tag_id);
|
||||
|
||||
-- Per-page content warnings from the closed moderation vocabulary. The
|
||||
-- manga detail page shows the deduplicated union across all its pages;
|
||||
-- search filters include/exclude by these.
|
||||
CREATE TABLE page_content_warnings (
|
||||
page_id uuid NOT NULL REFERENCES pages(id) ON DELETE CASCADE,
|
||||
warning text NOT NULL
|
||||
CHECK (warning IN ('sexual', 'nudity', 'gore', 'violence', 'disturbing')),
|
||||
PRIMARY KEY (page_id, warning)
|
||||
);
|
||||
|
||||
-- warning -> pages, for the manga/page content-warning filters.
|
||||
CREATE INDEX page_content_warnings_warning_idx ON page_content_warnings (warning);
|
||||
15
backend/migrations/0026_app_settings.sql
Normal file
15
backend/migrations/0026_app_settings.sql
Normal file
@@ -0,0 +1,15 @@
|
||||
-- Runtime-editable application settings, keyed by subsystem group.
|
||||
--
|
||||
-- Mirrors the small key-value `crawler_state` table (0015): one row per
|
||||
-- group ('crawler' | 'analysis'), the `value` JSONB holding a serialized
|
||||
-- settings DTO. Env vars seed a row on first boot (when absent); after that
|
||||
-- the DB row is the source of truth and the admin dashboard edits it live.
|
||||
--
|
||||
-- Only operationally-safe fields are stored here — host/infra and secrets
|
||||
-- (chromium paths, proxy, TOR control, vision API key, cookie domain) stay
|
||||
-- env-only and are never persisted.
|
||||
CREATE TABLE app_settings (
|
||||
key text PRIMARY KEY,
|
||||
value jsonb NOT NULL,
|
||||
updated_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
309
backend/src/analysis/daemon.rs
Normal file
309
backend/src/analysis/daemon.rs
Normal file
@@ -0,0 +1,309 @@
|
||||
//! The AI page-analysis worker daemon.
|
||||
//!
|
||||
//! A lean sibling of [`crate::crawler::daemon`]: it leases `analyze_page`
|
||||
//! jobs from the SAME `crawler_jobs` queue (filtered by kind, so it never
|
||||
//! contends with crawl jobs), dispatches each through an [`AnalyzeDispatcher`]
|
||||
//! seam, and acks done/failed. No cron, no browser, no session state — it
|
||||
//! runs whenever `ANALYSIS_ENABLED` is set, independent of the crawler.
|
||||
//!
|
||||
//! Per job: skip if already `done` (unless the payload sets `force`),
|
||||
//! heartbeat the lease while the (slow) vision call runs, isolate panics
|
||||
//! and an outer timeout, and on terminal failure write a `failed`
|
||||
//! `page_analysis` row so the page's state is observable.
|
||||
|
||||
use std::panic::AssertUnwindSafe;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use futures_util::FutureExt;
|
||||
use sqlx::PgPool;
|
||||
use tokio::task::JoinSet;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::analysis::events::{AnalysisEvent, AnalysisEvents};
|
||||
use crate::analysis::vision::VisionClient;
|
||||
use crate::crawler::jobs::{self, JobPayload, Lease, KIND_ANALYZE_PAGE};
|
||||
use crate::repo;
|
||||
use crate::storage::Storage;
|
||||
|
||||
/// Lease window; continuously extended by the per-job heartbeat.
|
||||
const LEASE_DURATION: Duration = Duration::from_secs(60);
|
||||
/// Heartbeat cadence — a third of the lease window.
|
||||
const LEASE_HEARTBEAT: Duration = Duration::from_secs(20);
|
||||
|
||||
/// The unit of work: analyze one page. Implemented by
|
||||
/// [`RealAnalyzeDispatcher`] in production and stubbed in tests.
|
||||
#[async_trait]
|
||||
pub trait AnalyzeDispatcher: Send + Sync {
|
||||
async fn dispatch(&self, page_id: Uuid) -> anyhow::Result<()>;
|
||||
}
|
||||
|
||||
pub struct AnalysisDaemonConfig {
|
||||
pub dispatcher: Arc<dyn AnalyzeDispatcher>,
|
||||
pub workers: usize,
|
||||
pub job_timeout: Duration,
|
||||
/// Live-event sink (Started/Completed/Failed) for admin SSE.
|
||||
pub events: Arc<AnalysisEvents>,
|
||||
}
|
||||
|
||||
pub struct AnalysisDaemonHandle {
|
||||
cancel: CancellationToken,
|
||||
join: JoinSet<()>,
|
||||
}
|
||||
|
||||
impl AnalysisDaemonHandle {
|
||||
pub async fn shutdown(mut self) {
|
||||
self.cancel.cancel();
|
||||
while self.join.join_next().await.is_some() {}
|
||||
}
|
||||
}
|
||||
|
||||
/// Spawn the analysis workers. Returns immediately; tasks run in the
|
||||
/// background until the handle is shut down (or `cancel` fires).
|
||||
pub fn spawn(
|
||||
pool: PgPool,
|
||||
cancel: CancellationToken,
|
||||
cfg: AnalysisDaemonConfig,
|
||||
) -> AnalysisDaemonHandle {
|
||||
let mut join = JoinSet::new();
|
||||
for id in 0..cfg.workers.max(1) {
|
||||
let ctx = WorkerContext {
|
||||
pool: pool.clone(),
|
||||
cancel: cancel.clone(),
|
||||
dispatcher: Arc::clone(&cfg.dispatcher),
|
||||
job_timeout: cfg.job_timeout,
|
||||
events: Arc::clone(&cfg.events),
|
||||
id,
|
||||
};
|
||||
join.spawn(async move { ctx.run().await });
|
||||
}
|
||||
AnalysisDaemonHandle { cancel, join }
|
||||
}
|
||||
|
||||
struct WorkerContext {
|
||||
pool: PgPool,
|
||||
cancel: CancellationToken,
|
||||
dispatcher: Arc<dyn AnalyzeDispatcher>,
|
||||
job_timeout: Duration,
|
||||
events: Arc<AnalysisEvents>,
|
||||
id: usize,
|
||||
}
|
||||
|
||||
impl WorkerContext {
|
||||
async fn run(self) {
|
||||
loop {
|
||||
if self.cancel.is_cancelled() {
|
||||
tracing::info!(worker = self.id, "analysis worker: shutdown");
|
||||
return;
|
||||
}
|
||||
let leases =
|
||||
match jobs::lease(&self.pool, Some(KIND_ANALYZE_PAGE), 1, LEASE_DURATION).await {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
tracing::warn!(worker = self.id, ?e, "analysis worker: lease failed");
|
||||
if self.sleep_or_cancel(Duration::from_secs(5)).await {
|
||||
return;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let Some(lease) = leases.into_iter().next() else {
|
||||
if self.sleep_or_cancel(Duration::from_secs(1)).await {
|
||||
return;
|
||||
}
|
||||
continue;
|
||||
};
|
||||
self.process_lease(lease).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Sleep `dur` or return `true` if cancelled while waiting.
|
||||
async fn sleep_or_cancel(&self, dur: Duration) -> bool {
|
||||
tokio::select! {
|
||||
_ = tokio::time::sleep(dur) => false,
|
||||
_ = self.cancel.cancelled() => true,
|
||||
}
|
||||
}
|
||||
|
||||
async fn process_lease(&self, lease: Lease) {
|
||||
let JobPayload::AnalyzePage { page_id, force } = lease.payload else {
|
||||
// Shouldn't happen — we lease only analyze_page — but ack done
|
||||
// so a misrouted job doesn't loop forever.
|
||||
tracing::warn!(worker = self.id, "analysis worker: non-analyze payload leased");
|
||||
let _ = jobs::ack_done(&self.pool, lease.id).await;
|
||||
return;
|
||||
};
|
||||
|
||||
// Skip-if-done net: a non-forced job for an already-analyzed page is
|
||||
// a no-op (e.g. a duplicate enqueue). Re-analysis goes through
|
||||
// `force` or a fresh page row.
|
||||
if !force {
|
||||
if let Ok(Some(row)) = repo::page_analysis::load(&self.pool, page_id).await {
|
||||
if row.status == crate::domain::page_analysis::AnalysisStatus::Done {
|
||||
let _ = jobs::ack_done(&self.pool, lease.id).await;
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Resolve the page breadcrumb once so live events carry the
|
||||
// manga/chapter/number the dashboard keys on. A missing breadcrumb
|
||||
// (page deleted) just suppresses events — the dispatch still runs
|
||||
// and acks normally.
|
||||
let breadcrumb = repo::page::locate(&self.pool, page_id).await.ok().flatten();
|
||||
if let Some((manga_id, chapter_id, page_number)) = breadcrumb {
|
||||
self.events.publish(AnalysisEvent::Started {
|
||||
page_id,
|
||||
manga_id,
|
||||
chapter_id,
|
||||
page_number,
|
||||
});
|
||||
}
|
||||
|
||||
// Heartbeat the lease while the vision call runs.
|
||||
let heartbeat = {
|
||||
let hb_pool = self.pool.clone();
|
||||
let hb_id = lease.id;
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
tokio::time::sleep(LEASE_HEARTBEAT).await;
|
||||
match jobs::renew(&hb_pool, hb_id, LEASE_DURATION).await {
|
||||
Ok(true) => {}
|
||||
Ok(false) => break,
|
||||
Err(e) => tracing::warn!(lease_id = %hb_id, ?e, "heartbeat renew failed"),
|
||||
}
|
||||
}
|
||||
})
|
||||
};
|
||||
|
||||
let dispatch = AssertUnwindSafe(self.dispatcher.dispatch(page_id)).catch_unwind();
|
||||
let outcome = tokio::time::timeout(self.job_timeout, dispatch).await;
|
||||
heartbeat.abort();
|
||||
|
||||
// Flatten timeout / panic / dispatch error into one Result + message.
|
||||
let result: Result<(), String> = match outcome {
|
||||
Err(_elapsed) => Err("analysis dispatch timed out".to_string()),
|
||||
Ok(Err(_panic)) => Err("analysis dispatcher panicked".to_string()),
|
||||
Ok(Ok(Err(e))) => Err(format!("{e:#}")),
|
||||
Ok(Ok(Ok(()))) => Ok(()),
|
||||
};
|
||||
|
||||
if let Some((manga_id, chapter_id, page_number)) = breadcrumb {
|
||||
let event = if result.is_ok() {
|
||||
AnalysisEvent::Completed { page_id, manga_id, chapter_id, page_number }
|
||||
} else {
|
||||
AnalysisEvent::Failed { page_id, manga_id, chapter_id, page_number }
|
||||
};
|
||||
self.events.publish(event);
|
||||
}
|
||||
|
||||
match result {
|
||||
Ok(()) => {
|
||||
let _ = jobs::ack_done(&self.pool, lease.id).await;
|
||||
}
|
||||
Err(msg) => {
|
||||
tracing::warn!(
|
||||
worker = self.id,
|
||||
%page_id,
|
||||
error = %msg,
|
||||
"analysis worker: dispatch failed — ack failed"
|
||||
);
|
||||
let _ = jobs::ack_failed(
|
||||
&self.pool,
|
||||
lease.id,
|
||||
&msg,
|
||||
lease.attempts,
|
||||
lease.max_attempts,
|
||||
)
|
||||
.await;
|
||||
// Terminal failure (retries exhausted) → record a `failed`
|
||||
// row so the page's analysis state is observable rather than
|
||||
// silently absent.
|
||||
if lease.attempts >= lease.max_attempts {
|
||||
let _ = repo::page_analysis::mark_failed(&self.pool, page_id, &msg).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Production dispatcher: load the page, read its image from storage, call
|
||||
/// the vision model, and persist the analysis.
|
||||
pub struct RealAnalyzeDispatcher {
|
||||
pub db: PgPool,
|
||||
pub storage: Arc<dyn Storage>,
|
||||
pub vision: VisionClient,
|
||||
pub model: String,
|
||||
pub max_image_bytes: usize,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl AnalyzeDispatcher for RealAnalyzeDispatcher {
|
||||
async fn dispatch(&self, page_id: Uuid) -> anyhow::Result<()> {
|
||||
let Some(page) = repo::page::find_by_id(&self.db, page_id).await? else {
|
||||
// Page was deleted between enqueue and dispatch — nothing to do.
|
||||
return Ok(());
|
||||
};
|
||||
let bytes = self
|
||||
.storage
|
||||
.get(&page.storage_key)
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("read page image {}: {e}", page.storage_key))?;
|
||||
if bytes.len() > self.max_image_bytes {
|
||||
anyhow::bail!(
|
||||
"page image {} is {} bytes, over the {} cap",
|
||||
page.storage_key,
|
||||
bytes.len(),
|
||||
self.max_image_bytes
|
||||
);
|
||||
}
|
||||
let analysis = self.vision.analyze(&bytes, &page.content_type).await?;
|
||||
repo::page_analysis::persist_analysis(&self.db, page_id, &analysis, &self.model).await?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Stubs for the daemon's integration tests. Public because the tests live
|
||||
/// in the `tests/` dir (a separate crate).
|
||||
pub mod test_support {
|
||||
use super::*;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
/// Counts dispatch calls and returns a configurable result.
|
||||
pub struct CountingDispatcher {
|
||||
pub calls: AtomicUsize,
|
||||
pub fail: bool,
|
||||
pub panic: bool,
|
||||
}
|
||||
|
||||
impl CountingDispatcher {
|
||||
pub fn ok() -> Arc<Self> {
|
||||
Arc::new(Self { calls: AtomicUsize::new(0), fail: false, panic: false })
|
||||
}
|
||||
pub fn failing() -> Arc<Self> {
|
||||
Arc::new(Self { calls: AtomicUsize::new(0), fail: true, panic: false })
|
||||
}
|
||||
pub fn panicking() -> Arc<Self> {
|
||||
Arc::new(Self { calls: AtomicUsize::new(0), fail: false, panic: true })
|
||||
}
|
||||
pub fn call_count(&self) -> usize {
|
||||
self.calls.load(Ordering::Acquire)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl AnalyzeDispatcher for CountingDispatcher {
|
||||
async fn dispatch(&self, _page_id: Uuid) -> anyhow::Result<()> {
|
||||
self.calls.fetch_add(1, Ordering::AcqRel);
|
||||
if self.panic {
|
||||
panic!("intentional analysis dispatcher panic");
|
||||
}
|
||||
if self.fail {
|
||||
anyhow::bail!("intentional analysis dispatch failure");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
78
backend/src/analysis/events.rs
Normal file
78
backend/src/analysis/events.rs
Normal file
@@ -0,0 +1,78 @@
|
||||
//! Live analysis events broadcast to admin SSE subscribers.
|
||||
//!
|
||||
//! A thin wrapper over a `tokio::sync::broadcast` channel. The worker
|
||||
//! publishes per-page progress (`Started`/`Completed`/`Failed`) and the
|
||||
//! admin enqueue path publishes `Enqueued`; the
|
||||
//! `GET /v1/admin/analysis/status/stream` SSE endpoint forwards each event.
|
||||
//! Discrete events (rather than the crawler's snapshot-on-change) map
|
||||
//! directly onto "this page/chapter/manga just changed state", which the
|
||||
//! dashboard applies incrementally.
|
||||
|
||||
use serde::Serialize;
|
||||
use tokio::sync::broadcast;
|
||||
use uuid::Uuid;
|
||||
|
||||
/// One live analysis event. Serialized with a `kind` discriminator so the
|
||||
/// frontend can switch on it.
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
#[serde(tag = "kind", rename_all = "snake_case")]
|
||||
pub enum AnalysisEvent {
|
||||
/// A bulk re-enqueue happened. `manga_id` / `chapter_id` scope it (both
|
||||
/// `None` = whole library). The UI optimistically marks in-scope pages
|
||||
/// as queued.
|
||||
Enqueued {
|
||||
count: u64,
|
||||
manga_id: Option<Uuid>,
|
||||
chapter_id: Option<Uuid>,
|
||||
},
|
||||
/// The worker began analyzing a page.
|
||||
Started {
|
||||
page_id: Uuid,
|
||||
manga_id: Uuid,
|
||||
chapter_id: Uuid,
|
||||
page_number: i32,
|
||||
},
|
||||
/// The worker finished a page successfully.
|
||||
Completed {
|
||||
page_id: Uuid,
|
||||
manga_id: Uuid,
|
||||
chapter_id: Uuid,
|
||||
page_number: i32,
|
||||
},
|
||||
/// The worker failed a page (this attempt).
|
||||
Failed {
|
||||
page_id: Uuid,
|
||||
manga_id: Uuid,
|
||||
chapter_id: Uuid,
|
||||
page_number: i32,
|
||||
},
|
||||
}
|
||||
|
||||
/// Broadcaster shared on `AppState`. Cheap to clone via `Arc`. Publishing
|
||||
/// when there are no subscribers is a no-op (the send error is ignored).
|
||||
pub struct AnalysisEvents {
|
||||
tx: broadcast::Sender<AnalysisEvent>,
|
||||
}
|
||||
|
||||
impl AnalysisEvents {
|
||||
pub fn new() -> Self {
|
||||
// Buffer is generous; a slow subscriber that lags is dropped frames
|
||||
// (the SSE handler treats `Lagged` as "skip and continue").
|
||||
let (tx, _rx) = broadcast::channel(512);
|
||||
Self { tx }
|
||||
}
|
||||
|
||||
pub fn publish(&self, event: AnalysisEvent) {
|
||||
let _ = self.tx.send(event);
|
||||
}
|
||||
|
||||
pub fn subscribe(&self) -> broadcast::Receiver<AnalysisEvent> {
|
||||
self.tx.subscribe()
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for AnalysisEvents {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
13
backend/src/analysis/mod.rs
Normal file
13
backend/src/analysis/mod.rs
Normal file
@@ -0,0 +1,13 @@
|
||||
//! AI content-analysis worker: calls a local OpenAI-compatible vision
|
||||
//! model on each page image and turns the result into OCR text, global
|
||||
//! auto-tags, a scene description, and NSFW content warnings.
|
||||
//!
|
||||
//! * [`prompt`] — the system prompt + the bounded output vocabulary and
|
||||
//! sanitization helpers (pure, unit-tested).
|
||||
//! * [`vision`] — the HTTP client: downscale → request → parse → sanitize.
|
||||
//! * [`daemon`] — the job-leasing worker loop (added in the worker phase).
|
||||
|
||||
pub mod daemon;
|
||||
pub mod events;
|
||||
pub mod prompt;
|
||||
pub mod vision;
|
||||
274
backend/src/analysis/prompt.rs
Normal file
274
backend/src/analysis/prompt.rs
Normal file
@@ -0,0 +1,274 @@
|
||||
//! The vision model's system prompt and the bounds applied to its output.
|
||||
//!
|
||||
//! The system prompt is deliberately terse: with a context budget as low
|
||||
//! as ~8192 tokens, the page image dominates, so the instructions + schema
|
||||
//! must stay small and the output is capped via `max_tokens` plus the
|
||||
//! length bounds enforced in [`crate::analysis::vision::sanitize`].
|
||||
|
||||
/// OCR text kinds the model may emit. Kept in sync with the
|
||||
/// `page_ocr_text.kind` CHECK and [`crate::domain::page_analysis::OcrKind`].
|
||||
pub const OCR_KINDS: [&str; 6] =
|
||||
["speech", "thought", "narration", "sfx", "title", "caption"];
|
||||
|
||||
/// Closed content-warning vocabulary. Kept in sync with the
|
||||
/// `page_content_warnings.warning` CHECK and
|
||||
/// [`crate::domain::page_analysis::ContentWarning`].
|
||||
pub const CONTENT_WARNINGS: [&str; 5] =
|
||||
["sexual", "nudity", "gore", "violence", "disturbing"];
|
||||
|
||||
/// Target tag count we ask the model for (a hint in the prompt; not
|
||||
/// hard-enforced beyond the [`MAX_TAGS`] cap).
|
||||
pub const MIN_TAGS: usize = 5;
|
||||
pub const MAX_TAGS: usize = 10;
|
||||
|
||||
/// Output bounds enforced at sanitize time so one verbose response can't
|
||||
/// bloat the row or the search document. The OCR cap is generous because a
|
||||
/// long sliced page legitimately accumulates many text pieces (the tsvector
|
||||
/// handles it).
|
||||
pub const MAX_OCR_PIECES: usize = 200;
|
||||
pub const MAX_OCR_TEXT_CHARS: usize = 500;
|
||||
pub const MAX_SCENE_CHARS: usize = 1000;
|
||||
|
||||
/// Hard per-call caps baked into the JSON schemas (`maxItems`/`maxLength`),
|
||||
/// so a repetition-prone model is grammar-bounded and can't loop a single
|
||||
/// call into the token ceiling. Kept per-call/per-slice; the cross-slice
|
||||
/// total is bounded separately by [`MAX_OCR_PIECES`].
|
||||
pub const MAX_OCR_PER_CALL: usize = 50;
|
||||
pub const MAX_TAGS_PER_CALL: usize = 12;
|
||||
|
||||
/// The default system prompt. Self-contained: it carries the exact JSON
|
||||
/// schema so the same string drives both the model and (implicitly) the
|
||||
/// [`crate::domain::page_analysis::VisionAnalysis`] parser. An admin can
|
||||
/// override it at runtime (see [`crate::config::AnalysisConfig::system_prompt`]);
|
||||
/// this const is the fallback and the "reset to default" target.
|
||||
pub const SYSTEM_PROMPT_DEFAULT: &str = concat!(
|
||||
"You are a manga/comic page analyzer. Look at the single page image and ",
|
||||
"return ONLY one minified JSON object — no prose, no markdown fences.\n",
|
||||
"Schema:\n",
|
||||
"{\"ocr_results\":[{\"text\":string,\"kind\":",
|
||||
"\"speech|thought|narration|sfx|title|caption\"}],",
|
||||
"\"tagging_results\":[string],",
|
||||
"\"scene_description\":string,",
|
||||
"\"safety_flag\":{\"is_nsfw\":boolean,\"content_type\":[",
|
||||
"\"sexual|nudity|gore|violence|disturbing\"]}}\n",
|
||||
"Rules: transcribe every visible text element verbatim into ocr_results ",
|
||||
"with its kind, each ONCE (never repeat or loop), at most 50; if there is ",
|
||||
"no text use []. tagging_results: 5-10 short, lowercase, distinct content ",
|
||||
"tags (characters, actions, setting, mood, genre); include explicit/",
|
||||
"sexual tags when present. scene_description: one or two sentences ",
|
||||
"describing setting, characters, and action. safety_flag.content_type: ",
|
||||
"only values from the listed set, [] if none; set is_nsfw true if the ",
|
||||
"page contains sexual, nudity, gore, violence, or disturbing content. ",
|
||||
"CRITICAL: do not repeat elements, tags or words; STOP when done. Output ",
|
||||
"must be valid minified JSON and nothing else."
|
||||
);
|
||||
|
||||
// --- Two-pass prompts (long-page slicing) -----------------------------------
|
||||
|
||||
/// Pass A: OCR-only over a single vertical slice of a tall page. Kept
|
||||
/// narrow so each slice call is fast and never truncates. Runtime-overridable
|
||||
/// default — see [`crate::config::AnalysisConfig::ocr_prompt`].
|
||||
pub const OCR_PROMPT_DEFAULT: &str = concat!(
|
||||
"You are an OCR engine for manga/comic pages. The image is one vertical ",
|
||||
"slice of a larger page. Transcribe EVERY visible text element verbatim ",
|
||||
"into ocr_results, each with its kind ",
|
||||
"(speech|thought|narration|sfx|title|caption) and y — the vertical center ",
|
||||
"of the text as a fraction from 0.0 (top) to 1.0 (bottom) of THIS image. ",
|
||||
"List them ONCE each, in top-to-bottom order. CRITICAL: transcribe each ",
|
||||
"distinct text element exactly once — never repeat an element, never loop ",
|
||||
"or pad the output. At most 50 elements. The moment you have transcribed ",
|
||||
"all visible text, STOP. If there is no text, return {\"ocr_results\":[]}. ",
|
||||
"Output ONLY one minified JSON object, no prose, no markdown."
|
||||
);
|
||||
|
||||
/// Pass B: tags + scene + safety for the whole page, grounded in the merged
|
||||
/// OCR text (supplied as a separate user message part by the client).
|
||||
/// Runtime-overridable default — see
|
||||
/// [`crate::config::AnalysisConfig::grounding_prompt`].
|
||||
pub const GROUNDING_PROMPT_DEFAULT: &str = concat!(
|
||||
"You are a manga/comic page analyzer. You are given the page image ",
|
||||
"(possibly downscaled) AND the OCR text already extracted from it. Using ",
|
||||
"both, return ONLY one minified JSON object with: tagging_results (5-10 ",
|
||||
"short lowercase content tags — characters, actions, setting, mood, ",
|
||||
"genre; include explicit/sexual tags when present; each tag distinct, no ",
|
||||
"duplicates); scene_description (one or two sentences on setting, ",
|
||||
"characters, and action, referencing the dialogue where relevant); ",
|
||||
"safety_flag (is_nsfw boolean + content_type from ",
|
||||
"sexual|nudity|gore|violence|disturbing, [] if none). CRITICAL: do not ",
|
||||
"repeat tags, words or sentences; keep each field concise and STOP when ",
|
||||
"done. No prose, no markdown."
|
||||
);
|
||||
|
||||
/// Pass-A schema: `{ ocr_results }` only.
|
||||
pub fn ocr_json_schema() -> serde_json::Value {
|
||||
serde_json::json!({
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"ocr_results": {
|
||||
"type": "array",
|
||||
"maxItems": MAX_OCR_PER_CALL,
|
||||
"items": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"text": { "type": "string", "maxLength": MAX_OCR_TEXT_CHARS },
|
||||
"kind": { "type": "string", "enum": OCR_KINDS },
|
||||
"y": { "type": "number", "minimum": 0, "maximum": 1 }
|
||||
},
|
||||
"required": ["text", "kind", "y"]
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": ["ocr_results"]
|
||||
})
|
||||
}
|
||||
|
||||
/// Pass-B schema: `{ tagging_results, scene_description, safety_flag }`.
|
||||
pub fn grounding_json_schema() -> serde_json::Value {
|
||||
serde_json::json!({
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"tagging_results": {
|
||||
"type": "array",
|
||||
"maxItems": MAX_TAGS_PER_CALL,
|
||||
"items": { "type": "string", "maxLength": 64 }
|
||||
},
|
||||
"scene_description": { "type": "string", "maxLength": MAX_SCENE_CHARS },
|
||||
"safety_flag": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"is_nsfw": { "type": "boolean" },
|
||||
"content_type": {
|
||||
"type": "array",
|
||||
"maxItems": CONTENT_WARNINGS.len(),
|
||||
"items": { "type": "string", "enum": CONTENT_WARNINGS }
|
||||
}
|
||||
},
|
||||
"required": ["is_nsfw", "content_type"]
|
||||
}
|
||||
},
|
||||
"required": ["tagging_results", "scene_description", "safety_flag"]
|
||||
})
|
||||
}
|
||||
|
||||
/// JSON Schema for the analysis output, used in `response_format:
|
||||
/// json_schema` mode (OpenAI structured outputs / LM Studio). Mirrors
|
||||
/// [`crate::domain::page_analysis::VisionAnalysis`]; `additionalProperties:
|
||||
/// false` + all-required keeps it valid under OpenAI strict mode while
|
||||
/// staying compatible with LM Studio.
|
||||
pub fn output_json_schema() -> serde_json::Value {
|
||||
serde_json::json!({
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"ocr_results": {
|
||||
"type": "array",
|
||||
"maxItems": MAX_OCR_PER_CALL,
|
||||
"items": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"text": { "type": "string", "maxLength": MAX_OCR_TEXT_CHARS },
|
||||
"kind": { "type": "string", "enum": OCR_KINDS }
|
||||
},
|
||||
"required": ["text", "kind"]
|
||||
}
|
||||
},
|
||||
"tagging_results": {
|
||||
"type": "array",
|
||||
"maxItems": MAX_TAGS_PER_CALL,
|
||||
"items": { "type": "string", "maxLength": 64 }
|
||||
},
|
||||
"scene_description": { "type": "string", "maxLength": MAX_SCENE_CHARS },
|
||||
"safety_flag": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"is_nsfw": { "type": "boolean" },
|
||||
"content_type": {
|
||||
"type": "array",
|
||||
"maxItems": CONTENT_WARNINGS.len(),
|
||||
"items": { "type": "string", "enum": CONTENT_WARNINGS }
|
||||
}
|
||||
},
|
||||
"required": ["is_nsfw", "content_type"]
|
||||
}
|
||||
},
|
||||
"required": ["ocr_results", "tagging_results", "scene_description", "safety_flag"]
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::domain::page_analysis::{ContentWarning, OcrKind, VisionAnalysis};
|
||||
|
||||
#[test]
|
||||
fn vocabularies_match_the_domain_enums() {
|
||||
// Every prompt OCR kind must parse back to a real OcrKind (no
|
||||
// fallback to narration via a typo).
|
||||
for k in OCR_KINDS {
|
||||
assert_eq!(
|
||||
OcrKind::from_model_str(k),
|
||||
match k {
|
||||
"speech" => OcrKind::Speech,
|
||||
"thought" => OcrKind::Thought,
|
||||
"narration" => OcrKind::Narration,
|
||||
"sfx" => OcrKind::Sfx,
|
||||
"title" => OcrKind::Title,
|
||||
"caption" => OcrKind::Caption,
|
||||
_ => unreachable!(),
|
||||
}
|
||||
);
|
||||
}
|
||||
for w in CONTENT_WARNINGS {
|
||||
assert!(
|
||||
ContentWarning::from_model_str(w).is_some(),
|
||||
"prompt warning {w} not in the domain vocabulary"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn system_prompt_mentions_the_schema_keys() {
|
||||
for key in ["ocr_results", "tagging_results", "scene_description", "safety_flag"] {
|
||||
assert!(SYSTEM_PROMPT_DEFAULT.contains(key), "prompt missing {key}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn output_schema_top_level_requires_all_four_keys() {
|
||||
let schema = output_json_schema();
|
||||
let required = schema["required"].as_array().unwrap();
|
||||
for key in ["ocr_results", "tagging_results", "scene_description", "safety_flag"] {
|
||||
assert!(
|
||||
required.iter().any(|v| v == key),
|
||||
"schema missing required {key}"
|
||||
);
|
||||
}
|
||||
// The kind enum carries exactly the OCR vocabulary.
|
||||
let kind_enum =
|
||||
&schema["properties"]["ocr_results"]["items"]["properties"]["kind"]["enum"];
|
||||
assert_eq!(kind_enum, &serde_json::json!(OCR_KINDS));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn output_schema_accepts_a_real_analysis_shape() {
|
||||
// A response matching the domain DTO must deserialize — the schema
|
||||
// and the parser describe the same shape.
|
||||
let sample = serde_json::json!({
|
||||
"ocr_results": [{ "text": "Hi", "kind": "speech" }],
|
||||
"tagging_results": ["action"],
|
||||
"scene_description": "A street.",
|
||||
"safety_flag": { "is_nsfw": false, "content_type": [] }
|
||||
});
|
||||
let _: VisionAnalysis = serde_json::from_value(sample).unwrap();
|
||||
// Schema names the same object key the parser reads.
|
||||
assert!(output_json_schema()["properties"]
|
||||
.get("safety_flag")
|
||||
.is_some());
|
||||
}
|
||||
}
|
||||
1046
backend/src/analysis/vision.rs
Normal file
1046
backend/src/analysis/vision.rs
Normal file
File diff suppressed because it is too large
Load Diff
303
backend/src/api/admin/analysis.rs
Normal file
303
backend/src/api/admin/analysis.rs
Normal file
@@ -0,0 +1,303 @@
|
||||
//! Admin controls for the AI content-analysis worker.
|
||||
//!
|
||||
//! Two endpoints, both admin-only (`RequireAdmin`, cookie-only) and gated
|
||||
//! on `AppState.analysis_enabled` (503 when the feature is off):
|
||||
//!
|
||||
//! * `POST /admin/analysis/reenqueue` — bulk backfill: enqueue
|
||||
//! `analyze_page` jobs for existing pages. Body `{ only_unanalyzed }`
|
||||
//! (default true) skips pages that already have a completed analysis.
|
||||
//! * `POST /admin/pages/:id/analyze` — force re-analysis of a single page
|
||||
//! (re-runs even if already `done`).
|
||||
|
||||
use std::convert::Infallible;
|
||||
|
||||
use axum::extract::{Path, Query, State};
|
||||
use axum::response::sse::{Event, KeepAlive, Sse};
|
||||
use axum::routing::{get, post};
|
||||
use axum::{Json, Router};
|
||||
use futures_util::Stream;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::json;
|
||||
use tokio::sync::broadcast::error::RecvError;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::analysis::events::AnalysisEvent;
|
||||
use crate::api::pagination::PagedResponse;
|
||||
use crate::app::AppState;
|
||||
use crate::auth::extractor::RequireAdmin;
|
||||
use crate::domain::page_analysis::{
|
||||
ChapterCoverage, MangaCoverage, PageAnalysisDetail, PageStatusItem,
|
||||
};
|
||||
use crate::error::{AppError, AppResult};
|
||||
use crate::repo;
|
||||
|
||||
pub fn routes() -> Router<AppState> {
|
||||
Router::new()
|
||||
.route("/admin/analysis/reenqueue", post(reenqueue))
|
||||
.route("/admin/pages/:id/analyze", post(analyze_page))
|
||||
// Coverage / inspection (admin-gated, but NOT analysis-enabled-gated
|
||||
// — an operator can browse coverage with the worker off).
|
||||
.route("/admin/analysis/mangas", get(coverage_mangas))
|
||||
.route("/admin/analysis/mangas/:id/chapters", get(coverage_chapters))
|
||||
.route("/admin/analysis/chapters/:id/pages", get(chapter_pages))
|
||||
.route("/admin/analysis/pages/:id", get(page_detail))
|
||||
.route("/admin/analysis/status/stream", get(stream_status))
|
||||
}
|
||||
|
||||
/// Live analysis events (SSE). Forwards each `AnalysisEvent` as a named
|
||||
/// `analysis` event; on broadcast lag (a slow client) emits a `lagged`
|
||||
/// event so the dashboard can refresh. EventSource sends the session
|
||||
/// cookie, so `RequireAdmin` gates the stream like any other admin route.
|
||||
async fn stream_status(
|
||||
State(state): State<AppState>,
|
||||
_admin: RequireAdmin,
|
||||
) -> Sse<impl Stream<Item = Result<Event, Infallible>>> {
|
||||
let rx = state.analysis_events.subscribe();
|
||||
let stream = futures_util::stream::unfold(rx, |mut rx| async move {
|
||||
loop {
|
||||
match rx.recv().await {
|
||||
Ok(ev) => {
|
||||
let event = Event::default()
|
||||
.event("analysis")
|
||||
.json_data(&ev)
|
||||
.unwrap_or_else(|_| Event::default().comment("serialize error"));
|
||||
return Some((Ok(event), rx));
|
||||
}
|
||||
Err(RecvError::Lagged(_)) => {
|
||||
return Some((Ok(Event::default().event("lagged").data("")), rx));
|
||||
}
|
||||
Err(RecvError::Closed) => return None,
|
||||
}
|
||||
}
|
||||
});
|
||||
Sse::new(stream).keep_alive(KeepAlive::default())
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct CoverageParams {
|
||||
#[serde(default)]
|
||||
pub search: Option<String>,
|
||||
#[serde(default = "default_coverage_limit")]
|
||||
pub limit: i64,
|
||||
#[serde(default)]
|
||||
pub offset: i64,
|
||||
}
|
||||
|
||||
fn default_coverage_limit() -> i64 {
|
||||
25
|
||||
}
|
||||
|
||||
async fn coverage_mangas(
|
||||
State(state): State<AppState>,
|
||||
_admin: RequireAdmin,
|
||||
Query(params): Query<CoverageParams>,
|
||||
) -> AppResult<Json<PagedResponse<MangaCoverage>>> {
|
||||
let limit = params.limit.clamp(1, 100);
|
||||
let offset = params.offset.max(0);
|
||||
let search = params
|
||||
.search
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|s| !s.is_empty());
|
||||
let (items, total) =
|
||||
repo::page_analysis::manga_coverage(&state.db, search, limit, offset).await?;
|
||||
Ok(Json(PagedResponse::with_total(items, limit, offset, total)))
|
||||
}
|
||||
|
||||
async fn coverage_chapters(
|
||||
State(state): State<AppState>,
|
||||
_admin: RequireAdmin,
|
||||
Path(manga_id): Path<Uuid>,
|
||||
) -> AppResult<Json<ItemsResponse<ChapterCoverage>>> {
|
||||
if !repo::manga::exists(&state.db, manga_id).await? {
|
||||
return Err(AppError::NotFound);
|
||||
}
|
||||
let items = repo::page_analysis::chapter_coverage(&state.db, manga_id).await?;
|
||||
Ok(Json(ItemsResponse { items }))
|
||||
}
|
||||
|
||||
async fn chapter_pages(
|
||||
State(state): State<AppState>,
|
||||
_admin: RequireAdmin,
|
||||
Path(chapter_id): Path<Uuid>,
|
||||
) -> AppResult<Json<ItemsResponse<PageStatusItem>>> {
|
||||
let exists: bool =
|
||||
sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM chapters WHERE id = $1)")
|
||||
.bind(chapter_id)
|
||||
.fetch_one(&state.db)
|
||||
.await?;
|
||||
if !exists {
|
||||
return Err(AppError::NotFound);
|
||||
}
|
||||
let items = repo::page_analysis::chapter_page_status(&state.db, chapter_id).await?;
|
||||
Ok(Json(ItemsResponse { items }))
|
||||
}
|
||||
|
||||
async fn page_detail(
|
||||
State(state): State<AppState>,
|
||||
_admin: RequireAdmin,
|
||||
Path(page_id): Path<Uuid>,
|
||||
) -> AppResult<Json<PageAnalysisDetail>> {
|
||||
let detail = repo::page_analysis::page_detail(&state.db, page_id)
|
||||
.await?
|
||||
.ok_or(AppError::NotFound)?;
|
||||
Ok(Json(detail))
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct ItemsResponse<T> {
|
||||
items: Vec<T>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct ReenqueueBody {
|
||||
/// Skip pages that already have a `done` analysis row. Defaults to
|
||||
/// true so a routine backfill only touches the gaps; `false` forces a
|
||||
/// full re-analysis of in-scope pages.
|
||||
#[serde(default = "default_true")]
|
||||
pub only_unanalyzed: bool,
|
||||
/// Limit the re-enqueue to one manga (all its chapters' pages).
|
||||
#[serde(default)]
|
||||
pub manga_id: Option<Uuid>,
|
||||
/// Limit the re-enqueue to one chapter's pages. Mutually exclusive
|
||||
/// with `manga_id`.
|
||||
#[serde(default)]
|
||||
pub chapter_id: Option<Uuid>,
|
||||
}
|
||||
|
||||
impl Default for ReenqueueBody {
|
||||
fn default() -> Self {
|
||||
Self { only_unanalyzed: true, manga_id: None, chapter_id: None }
|
||||
}
|
||||
}
|
||||
|
||||
fn default_true() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct ReenqueueResponse {
|
||||
pub enqueued: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct AnalyzePageResponse {
|
||||
pub enqueued: bool,
|
||||
}
|
||||
|
||||
/// Reject the request with 503 when the analysis worker is disabled, so
|
||||
/// an operator doesn't pile up jobs that nothing will ever drain.
|
||||
fn ensure_enabled(state: &AppState) -> AppResult<()> {
|
||||
if state.analysis_enabled() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(AppError::ServiceUnavailable(
|
||||
"content-analysis worker is disabled (ANALYSIS_ENABLED=false)".into(),
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
async fn reenqueue(
|
||||
State(state): State<AppState>,
|
||||
admin: RequireAdmin,
|
||||
body: Option<Json<ReenqueueBody>>,
|
||||
) -> AppResult<Json<ReenqueueResponse>> {
|
||||
ensure_enabled(&state)?;
|
||||
let body = body.map(|b| b.0).unwrap_or_default();
|
||||
|
||||
// Resolve the scope. manga_id and chapter_id are mutually exclusive;
|
||||
// an unknown target is a 404 rather than a silent zero-enqueue.
|
||||
if body.manga_id.is_some() && body.chapter_id.is_some() {
|
||||
return Err(AppError::ValidationFailed {
|
||||
message: "manga_id and chapter_id are mutually exclusive".into(),
|
||||
details: json!({ "scope": "pick at most one" }),
|
||||
});
|
||||
}
|
||||
let (scope, target_type, target_id) = if let Some(chapter_id) = body.chapter_id {
|
||||
let exists: bool =
|
||||
sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM chapters WHERE id = $1)")
|
||||
.bind(chapter_id)
|
||||
.fetch_one(&state.db)
|
||||
.await?;
|
||||
if !exists {
|
||||
return Err(AppError::NotFound);
|
||||
}
|
||||
(
|
||||
repo::page_analysis::ReenqueueScope::Chapter(chapter_id),
|
||||
"chapter",
|
||||
Some(chapter_id),
|
||||
)
|
||||
} else if let Some(manga_id) = body.manga_id {
|
||||
if !repo::manga::exists(&state.db, manga_id).await? {
|
||||
return Err(AppError::NotFound);
|
||||
}
|
||||
(
|
||||
repo::page_analysis::ReenqueueScope::Manga(manga_id),
|
||||
"manga",
|
||||
Some(manga_id),
|
||||
)
|
||||
} else {
|
||||
(repo::page_analysis::ReenqueueScope::All, "analysis", None)
|
||||
};
|
||||
|
||||
let enqueued =
|
||||
repo::page_analysis::enqueue_pages(&state.db, scope, body.only_unanalyzed).await?;
|
||||
|
||||
// Push a live event so connected dashboards mark the in-scope pages as
|
||||
// queued. Skip the no-op (nothing actually enqueued).
|
||||
if enqueued > 0 {
|
||||
state.analysis_events.publish(AnalysisEvent::Enqueued {
|
||||
count: enqueued,
|
||||
manga_id: body.manga_id,
|
||||
chapter_id: body.chapter_id,
|
||||
});
|
||||
}
|
||||
|
||||
repo::admin_audit::insert(
|
||||
&state.db,
|
||||
admin.0.id,
|
||||
"analysis_reenqueue",
|
||||
target_type,
|
||||
target_id,
|
||||
json!({
|
||||
"only_unanalyzed": body.only_unanalyzed,
|
||||
"manga_id": body.manga_id,
|
||||
"chapter_id": body.chapter_id,
|
||||
"enqueued": enqueued,
|
||||
}),
|
||||
)
|
||||
.await?;
|
||||
|
||||
Ok(Json(ReenqueueResponse { enqueued }))
|
||||
}
|
||||
|
||||
async fn analyze_page(
|
||||
State(state): State<AppState>,
|
||||
admin: RequireAdmin,
|
||||
Path(page_id): Path<Uuid>,
|
||||
) -> AppResult<Json<AnalyzePageResponse>> {
|
||||
ensure_enabled(&state)?;
|
||||
|
||||
let exists: bool =
|
||||
sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM pages WHERE id = $1)")
|
||||
.bind(page_id)
|
||||
.fetch_one(&state.db)
|
||||
.await?;
|
||||
if !exists {
|
||||
return Err(AppError::NotFound);
|
||||
}
|
||||
|
||||
repo::page_analysis::enqueue_for_page(&state.db, page_id, true).await?;
|
||||
|
||||
repo::admin_audit::insert(
|
||||
&state.db,
|
||||
admin.0.id,
|
||||
"analysis_force_page",
|
||||
"page",
|
||||
Some(page_id),
|
||||
json!({ "force": true }),
|
||||
)
|
||||
.await?;
|
||||
|
||||
Ok(Json(AnalyzePageResponse { enqueued: true }))
|
||||
}
|
||||
67
backend/src/api/admin/crawler/backlog.rs
Normal file
67
backend/src/api/admin/crawler/backlog.rs
Normal file
@@ -0,0 +1,67 @@
|
||||
//! GET /admin/crawler/active-jobs — paginated `pending|running` chapters
|
||||
//! GET /admin/crawler/covers — paginated mangas missing a cover
|
||||
//!
|
||||
//! These are pure DB-derived reads that drive two of the three
|
||||
//! backlog tables on the dashboard. The third (dead jobs) lives in
|
||||
//! the [`super::dead_jobs`] module because it also exposes a write.
|
||||
|
||||
use axum::extract::{Query, State};
|
||||
use axum::routing::get;
|
||||
use axum::{Json, Router};
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::app::AppState;
|
||||
use crate::auth::extractor::RequireAdmin;
|
||||
use crate::error::AppResult;
|
||||
use crate::repo;
|
||||
use crate::repo::crawler::{ActiveJob, MissingCoverRow};
|
||||
|
||||
use super::default_limit;
|
||||
|
||||
pub(super) fn routes() -> Router<AppState> {
|
||||
Router::new()
|
||||
.route("/admin/crawler/active-jobs", get(list_active_jobs))
|
||||
.route("/admin/crawler/covers", get(list_covers))
|
||||
}
|
||||
|
||||
/// Pagination + title-search params shared by the backlog list endpoints.
|
||||
#[derive(Debug, Deserialize, Default)]
|
||||
struct ListParams {
|
||||
#[serde(default)]
|
||||
search: Option<String>,
|
||||
#[serde(default = "default_limit")]
|
||||
limit: i64,
|
||||
#[serde(default)]
|
||||
offset: i64,
|
||||
}
|
||||
|
||||
async fn list_active_jobs(
|
||||
State(state): State<AppState>,
|
||||
_admin: RequireAdmin,
|
||||
Query(params): Query<ListParams>,
|
||||
) -> AppResult<Json<crate::api::pagination::PagedResponse<ActiveJob>>> {
|
||||
let limit = params.limit.clamp(1, 200);
|
||||
let offset = params.offset.max(0);
|
||||
let search = params.search.filter(|s| !s.trim().is_empty());
|
||||
let (items, total) =
|
||||
repo::crawler::list_active_jobs(&state.db, search.as_deref(), limit, offset).await?;
|
||||
Ok(Json(crate::api::pagination::PagedResponse::with_total(
|
||||
items, limit, offset, total,
|
||||
)))
|
||||
}
|
||||
|
||||
async fn list_covers(
|
||||
State(state): State<AppState>,
|
||||
_admin: RequireAdmin,
|
||||
Query(params): Query<ListParams>,
|
||||
) -> AppResult<Json<crate::api::pagination::PagedResponse<MissingCoverRow>>> {
|
||||
let limit = params.limit.clamp(1, 200);
|
||||
let offset = params.offset.max(0);
|
||||
let search = params.search.filter(|s| !s.trim().is_empty());
|
||||
let (items, total) =
|
||||
repo::crawler::list_missing_cover_mangas(&state.db, search.as_deref(), limit, offset)
|
||||
.await?;
|
||||
Ok(Json(crate::api::pagination::PagedResponse::with_total(
|
||||
items, limit, offset, total,
|
||||
)))
|
||||
}
|
||||
206
backend/src/api/admin/crawler/control.rs
Normal file
206
backend/src/api/admin/crawler/control.rs
Normal file
@@ -0,0 +1,206 @@
|
||||
//! POST /admin/crawler/run — trigger an out-of-cycle metadata pass
|
||||
//! POST /admin/crawler/browser/restart — coordinated restart
|
||||
//! POST /admin/crawler/session — refresh PHPSESSID
|
||||
//! POST /admin/crawler/session/clear-expired — clear sticky expired flag
|
||||
//!
|
||||
//! All four mutate live in-process state on `CrawlerControl` (browser
|
||||
//! manager, session controller, manual-pass mutex). They each emit an
|
||||
//! `admin_audit` row and `status.poke()` so SSE subscribers see the
|
||||
//! change instantly without polling.
|
||||
|
||||
use axum::extract::State;
|
||||
use axum::routing::post;
|
||||
use axum::{Json, Router};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::json;
|
||||
|
||||
use crate::app::AppState;
|
||||
use crate::auth::extractor::RequireAdmin;
|
||||
use crate::error::{AppError, AppResult};
|
||||
use crate::repo;
|
||||
|
||||
use super::require_crawler;
|
||||
|
||||
pub(super) fn routes() -> Router<AppState> {
|
||||
Router::new()
|
||||
.route("/admin/crawler/run", post(run_now))
|
||||
.route("/admin/crawler/browser/restart", post(restart_browser))
|
||||
.route("/admin/crawler/session", post(update_session))
|
||||
.route(
|
||||
"/admin/crawler/session/clear-expired",
|
||||
post(clear_session_expired),
|
||||
)
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct RunResponse {
|
||||
started: bool,
|
||||
}
|
||||
|
||||
async fn run_now(
|
||||
State(state): State<AppState>,
|
||||
admin: RequireAdmin,
|
||||
) -> AppResult<Json<RunResponse>> {
|
||||
let c = require_crawler(&state)?;
|
||||
let mp = c.metadata_pass.as_ref().ok_or_else(|| {
|
||||
AppError::ServiceUnavailable("no source configured (CRAWLER_START_URL unset)".into())
|
||||
})?;
|
||||
// Operator-click dedup. A pass holds the lock for its entire run
|
||||
// (minutes); a second click while it's in flight returns 409 instead
|
||||
// of fanning a second pass onto the single browser lease. The daily
|
||||
// cron does NOT take this lock — cron is single-fire by definition,
|
||||
// and its own contention with a manual pass already serialises
|
||||
// through the browser lease + advisory lock at a lower layer.
|
||||
let pass_guard = c
|
||||
.manual_pass_lock
|
||||
.clone()
|
||||
.try_lock_owned()
|
||||
.map_err(|_| AppError::Conflict("manual metadata pass already running".into()))?;
|
||||
let mp = std::sync::Arc::clone(mp);
|
||||
// Fire-and-forget: the pass can run for minutes; the dashboard
|
||||
// streams progress over SSE. The guard moves into the task so the
|
||||
// lock is released only when the pass finishes (or the task panics).
|
||||
tokio::spawn(async move {
|
||||
let _pass_guard = pass_guard;
|
||||
if let Err(e) = mp.run().await {
|
||||
tracing::warn!(error = ?e, "manual metadata pass failed");
|
||||
}
|
||||
});
|
||||
repo::admin_audit::insert(&state.db, admin.0.id, "crawler_run", "crawler", None, json!({}))
|
||||
.await?;
|
||||
Ok(Json(RunResponse { started: true }))
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct RestartResponse {
|
||||
ok: bool,
|
||||
error: Option<String>,
|
||||
}
|
||||
|
||||
async fn restart_browser(
|
||||
State(state): State<AppState>,
|
||||
admin: RequireAdmin,
|
||||
) -> AppResult<Json<RestartResponse>> {
|
||||
let c = require_crawler(&state)?;
|
||||
let result = c.browser_manager.coordinated_restart(c.drain_deadline).await;
|
||||
// A successful coordinated_restart re-runs on_launch, which re-injects
|
||||
// PHPSESSID and re-probes — i.e. the session is live. Drop the sticky
|
||||
// `session_expired` flag so chapter workers stop idling without
|
||||
// requiring a second click on "Clear expired".
|
||||
if result.is_ok() {
|
||||
c.session.clear_expired();
|
||||
}
|
||||
// Push the post-restart browser phase to live subscribers immediately.
|
||||
c.status.poke();
|
||||
repo::admin_audit::insert(
|
||||
&state.db,
|
||||
admin.0.id,
|
||||
"crawler_browser_restart",
|
||||
"crawler",
|
||||
None,
|
||||
json!({ "ok": result.is_ok() }),
|
||||
)
|
||||
.await?;
|
||||
Ok(Json(match result {
|
||||
Ok(()) => RestartResponse {
|
||||
ok: true,
|
||||
error: None,
|
||||
},
|
||||
Err(e) => RestartResponse {
|
||||
ok: false,
|
||||
error: Some(format!("{e:#}")),
|
||||
},
|
||||
}))
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct UpdateSessionRequest {
|
||||
phpsessid: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct UpdateSessionResponse {
|
||||
/// Whether the post-update browser relaunch + session probe succeeded.
|
||||
valid: bool,
|
||||
error: Option<String>,
|
||||
}
|
||||
|
||||
async fn update_session(
|
||||
State(state): State<AppState>,
|
||||
admin: RequireAdmin,
|
||||
Json(body): Json<UpdateSessionRequest>,
|
||||
) -> AppResult<Json<UpdateSessionResponse>> {
|
||||
let c = require_crawler(&state)?;
|
||||
// Fingerprint BEFORE move so the raw value never reaches tracing or
|
||||
// the audit row. SHA256-prefix is opaque to anyone reading the audit
|
||||
// log but deterministic enough to correlate two updates of the same
|
||||
// session value.
|
||||
let fingerprint = phpsessid_fingerprint(&body.phpsessid);
|
||||
c.session
|
||||
.update(&body.phpsessid)
|
||||
.await
|
||||
.map_err(|e| AppError::InvalidInput(format!("{e:#}")))?;
|
||||
// Relaunch the browser so on_launch re-injects the new cookie and
|
||||
// re-probes — the restart's success IS the session-validity signal.
|
||||
let probe = c.browser_manager.coordinated_restart(c.drain_deadline).await;
|
||||
// Session + browser state changed — push to live subscribers.
|
||||
c.status.poke();
|
||||
repo::admin_audit::insert(
|
||||
&state.db,
|
||||
admin.0.id,
|
||||
"crawler_session_update",
|
||||
"crawler",
|
||||
None,
|
||||
json!({ "valid": probe.is_ok(), "phpsessid_fingerprint": fingerprint }),
|
||||
)
|
||||
.await?;
|
||||
Ok(Json(match probe {
|
||||
Ok(()) => UpdateSessionResponse {
|
||||
valid: true,
|
||||
error: None,
|
||||
},
|
||||
Err(e) => UpdateSessionResponse {
|
||||
valid: false,
|
||||
error: Some(format!("{e:#}")),
|
||||
},
|
||||
}))
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct ClearExpiredResponse {
|
||||
cleared: bool,
|
||||
}
|
||||
|
||||
async fn clear_session_expired(
|
||||
State(state): State<AppState>,
|
||||
admin: RequireAdmin,
|
||||
) -> AppResult<Json<ClearExpiredResponse>> {
|
||||
let c = require_crawler(&state)?;
|
||||
c.session.clear_expired();
|
||||
// session.expired flipped — push to live subscribers.
|
||||
c.status.poke();
|
||||
repo::admin_audit::insert(
|
||||
&state.db,
|
||||
admin.0.id,
|
||||
"crawler_session_clear_expired",
|
||||
"crawler",
|
||||
None,
|
||||
json!({}),
|
||||
)
|
||||
.await?;
|
||||
Ok(Json(ClearExpiredResponse { cleared: true }))
|
||||
}
|
||||
|
||||
/// Opaque, short fingerprint of a PHPSESSID for the admin audit log.
|
||||
/// The first 8 hex chars of SHA-256 — enough to correlate two updates
|
||||
/// of the same value without revealing the raw cookie. Reading the audit
|
||||
/// row does not give an operator anything that can re-construct the
|
||||
/// session.
|
||||
fn phpsessid_fingerprint(sid: &str) -> String {
|
||||
use sha2::{Digest, Sha256};
|
||||
let mut h = Sha256::new();
|
||||
h.update(sid.as_bytes());
|
||||
let digest = h.finalize();
|
||||
let hex: String = digest.iter().take(4).map(|b| format!("{b:02x}")).collect();
|
||||
hex
|
||||
}
|
||||
121
backend/src/api/admin/crawler/dead_jobs.rs
Normal file
121
backend/src/api/admin/crawler/dead_jobs.rs
Normal file
@@ -0,0 +1,121 @@
|
||||
//! GET /admin/crawler/dead-jobs — paginated dead-letter list
|
||||
//! POST /admin/crawler/dead-jobs/requeue — flip dead jobs back to pending
|
||||
//!
|
||||
//! Requeue scopes:
|
||||
//! - `all` (requires `confirm: true` so a careless click / CSRF bait
|
||||
//! can't flip the whole pile in one shot)
|
||||
//! - `manga` (all dead jobs whose chapter belongs to a manga)
|
||||
//! - `chapter` (all dead jobs for a single chapter)
|
||||
//! - `job` (a single dead row by id)
|
||||
//!
|
||||
//! Each requeue emits an `admin_audit` row with the relevant
|
||||
//! `target_id` populated, so an operator review post-incident can pin
|
||||
//! exactly which scope was acted on.
|
||||
|
||||
use axum::extract::{Query, State};
|
||||
use axum::routing::{get, post};
|
||||
use axum::{Json, Router};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::json;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::app::AppState;
|
||||
use crate::auth::extractor::RequireAdmin;
|
||||
use crate::error::{AppError, AppResult};
|
||||
use crate::repo;
|
||||
use crate::repo::crawler::{DeadJob, RequeueScope};
|
||||
|
||||
use super::default_limit;
|
||||
|
||||
pub(super) fn routes() -> Router<AppState> {
|
||||
Router::new()
|
||||
.route("/admin/crawler/dead-jobs", get(list_dead_jobs))
|
||||
.route("/admin/crawler/dead-jobs/requeue", post(requeue_dead_jobs))
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Default)]
|
||||
struct DeadJobsParams {
|
||||
#[serde(default)]
|
||||
search: Option<String>,
|
||||
#[serde(default = "default_limit")]
|
||||
limit: i64,
|
||||
#[serde(default)]
|
||||
offset: i64,
|
||||
}
|
||||
|
||||
async fn list_dead_jobs(
|
||||
State(state): State<AppState>,
|
||||
_admin: RequireAdmin,
|
||||
Query(params): Query<DeadJobsParams>,
|
||||
) -> AppResult<Json<crate::api::pagination::PagedResponse<DeadJob>>> {
|
||||
let limit = params.limit.clamp(1, 200);
|
||||
let offset = params.offset.max(0);
|
||||
let search = params.search.filter(|s| !s.trim().is_empty());
|
||||
let (items, total) =
|
||||
repo::crawler::list_dead_jobs(&state.db, search.as_deref(), limit, offset).await?;
|
||||
Ok(Json(crate::api::pagination::PagedResponse::with_total(
|
||||
items, limit, offset, total,
|
||||
)))
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(tag = "scope", rename_all = "snake_case")]
|
||||
enum RequeueRequest {
|
||||
/// `confirm: true` is required so a careless click / CSRF bait can't
|
||||
/// flip the entire dead pile in one shot. Narrow scopes don't need it.
|
||||
All {
|
||||
#[serde(default)]
|
||||
confirm: bool,
|
||||
},
|
||||
Manga { manga_id: Uuid },
|
||||
Chapter { chapter_id: Uuid },
|
||||
Job { job_id: Uuid },
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct RequeueResponse {
|
||||
requeued: u64,
|
||||
}
|
||||
|
||||
async fn requeue_dead_jobs(
|
||||
State(state): State<AppState>,
|
||||
admin: RequireAdmin,
|
||||
Json(body): Json<RequeueRequest>,
|
||||
) -> AppResult<Json<RequeueResponse>> {
|
||||
// Reject scope=all without an explicit confirm so a single click or
|
||||
// CSRF bait can't flip the whole dead pile. Narrow scopes don't need
|
||||
// the confirmation — the operator already named a specific target.
|
||||
if let RequeueRequest::All { confirm: false } = &body {
|
||||
return Err(AppError::InvalidInput(
|
||||
"confirm: true is required for scope=all".into(),
|
||||
));
|
||||
}
|
||||
let (scope, target_kind, target_id) = match &body {
|
||||
RequeueRequest::All { .. } => (RequeueScope::All, "crawler", None),
|
||||
RequeueRequest::Manga { manga_id } => (RequeueScope::Manga(*manga_id), "manga", Some(*manga_id)),
|
||||
RequeueRequest::Chapter { chapter_id } => {
|
||||
(RequeueScope::Chapter(*chapter_id), "chapter", Some(*chapter_id))
|
||||
}
|
||||
RequeueRequest::Job { job_id } => (RequeueScope::Job(*job_id), "crawler_job", Some(*job_id)),
|
||||
};
|
||||
let requeued = repo::crawler::requeue_dead_jobs(&state.db, scope).await?;
|
||||
repo::admin_audit::insert(
|
||||
&state.db,
|
||||
admin.0.id,
|
||||
"crawler_dead_jobs_requeue",
|
||||
target_kind,
|
||||
target_id,
|
||||
json!({ "requeued": requeued, "scope": scope_label(&body) }),
|
||||
)
|
||||
.await?;
|
||||
Ok(Json(RequeueResponse { requeued }))
|
||||
}
|
||||
|
||||
fn scope_label(r: &RequeueRequest) -> &'static str {
|
||||
match r {
|
||||
RequeueRequest::All { .. } => "all",
|
||||
RequeueRequest::Manga { .. } => "manga",
|
||||
RequeueRequest::Chapter { .. } => "chapter",
|
||||
RequeueRequest::Job { .. } => "job",
|
||||
}
|
||||
}
|
||||
50
backend/src/api/admin/crawler/mod.rs
Normal file
50
backend/src/api/admin/crawler/mod.rs
Normal file
@@ -0,0 +1,50 @@
|
||||
//! Admin-only crawler observability + control endpoints.
|
||||
//!
|
||||
//! Mounted under `/api/v1/admin/crawler*`, cookie-only via `RequireAdmin`.
|
||||
//! All control endpoints return 503 when the crawler daemon is disabled
|
||||
//! (`AppState.crawler == None`). Reads compose the live in-process status
|
||||
//! ([`crate::crawler::status`]) with DB-derived queue counts and the
|
||||
//! session/browser flags.
|
||||
//!
|
||||
//! Split into four siblings for navigability — the surface area grew
|
||||
//! past the point where keeping it in one file made review harder:
|
||||
//! - [`status`] — SSE stream + composed status snapshot
|
||||
//! - [`control`] — run / restart / session endpoints
|
||||
//! - [`dead_jobs`] — dead-letter list + requeue
|
||||
//! - [`backlog`] — pending-chapters and missing-covers backlog reads
|
||||
|
||||
mod backlog;
|
||||
mod control;
|
||||
mod dead_jobs;
|
||||
mod status;
|
||||
|
||||
use axum::Router;
|
||||
|
||||
use crate::app::{AppState, CrawlerControl};
|
||||
use crate::error::AppError;
|
||||
|
||||
pub fn routes() -> Router<AppState> {
|
||||
Router::new()
|
||||
.merge(status::routes())
|
||||
.merge(control::routes())
|
||||
.merge(dead_jobs::routes())
|
||||
.merge(backlog::routes())
|
||||
}
|
||||
|
||||
/// Default page size for the backlog list endpoints.
|
||||
pub(super) fn default_limit() -> i64 {
|
||||
50
|
||||
}
|
||||
|
||||
/// Shared 503 helper: the daemon-only control + observability endpoints
|
||||
/// gate on a running crawler daemon. Returns the same code and body across
|
||||
/// every caller so the frontend can rely on `service_unavailable` rather
|
||||
/// than each handler returning its own variant. Yields an owned `Arc` (the
|
||||
/// control handle is swapped on a config reload).
|
||||
pub(super) fn require_crawler(
|
||||
state: &AppState,
|
||||
) -> Result<std::sync::Arc<CrawlerControl>, AppError> {
|
||||
state.crawler().ok_or_else(|| {
|
||||
AppError::ServiceUnavailable("crawler daemon is disabled".into())
|
||||
})
|
||||
}
|
||||
245
backend/src/api/admin/crawler/status.rs
Normal file
245
backend/src/api/admin/crawler/status.rs
Normal file
@@ -0,0 +1,245 @@
|
||||
//! GET /admin/crawler — composed status snapshot
|
||||
//! GET /admin/crawler/stream — Server-Sent Events live status
|
||||
//!
|
||||
//! The composed response merges the in-process status surface
|
||||
//! ([`crate::crawler::status`]) with two DB-derived counts
|
||||
//! (job-state breakdown and missing-cover backlog) so the dashboard
|
||||
//! reads the same shape from one one-shot fetch and from each SSE
|
||||
//! frame. The streaming path debounces a burst of pokes into a single
|
||||
//! frame and memoizes the DB counts for the [`QUEUE_MEMO_TTL`] window
|
||||
//! to avoid hammering Postgres.
|
||||
|
||||
use std::convert::Infallible;
|
||||
use std::time::Duration;
|
||||
|
||||
use axum::extract::State;
|
||||
use axum::response::sse::{Event, KeepAlive, Sse};
|
||||
use axum::routing::get;
|
||||
use axum::{Json, Router};
|
||||
use futures_util::stream::Stream;
|
||||
use serde::Serialize;
|
||||
|
||||
use crate::app::AppState;
|
||||
use crate::auth::extractor::RequireAdmin;
|
||||
use crate::crawler::browser_manager::RestartPhase;
|
||||
use crate::crawler::status::{ActiveChapter, CoverTarget, LastPass, Phase};
|
||||
use crate::error::AppResult;
|
||||
use crate::repo;
|
||||
|
||||
/// Backstop recompose interval for the SSE stream. Phase/worker/session
|
||||
/// changes push instantly via the status `watch`; this only bounds the
|
||||
/// staleness of DB-derived queue counts and the browser phase when those
|
||||
/// change without an accompanying status poke.
|
||||
const SSE_BACKSTOP: Duration = Duration::from_secs(5);
|
||||
|
||||
/// Coalesce a burst of status pokes (e.g. one per stored page during a
|
||||
/// chapter download) into a single SSE frame. After the first change
|
||||
/// fires we sleep this long and absorb any further changes that arrive
|
||||
/// in the window before emitting. Tighter than human-perceivable jitter
|
||||
/// (~16-100 ms is the rule of thumb for "instant").
|
||||
const SSE_DEBOUNCE: Duration = Duration::from_millis(250);
|
||||
|
||||
/// Per-connection memo TTL for DB-derived queue counts. With a busy
|
||||
/// chapter pass bumping the watch up to several times per second, the
|
||||
/// memo collapses N stream wakeups into one round-trip per second
|
||||
/// — typically a ~10x reduction in steady-state DB QPS per subscriber.
|
||||
const QUEUE_MEMO_TTL: Duration = Duration::from_millis(1000);
|
||||
|
||||
pub(super) fn routes() -> Router<AppState> {
|
||||
Router::new()
|
||||
.route("/admin/crawler", get(get_status))
|
||||
.route("/admin/crawler/stream", get(stream_status))
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct QueueCounts {
|
||||
pending: i64,
|
||||
running: i64,
|
||||
dead: i64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct SessionStatus {
|
||||
/// Whether the sticky session-expired flag is set (chapter workers idle).
|
||||
expired: bool,
|
||||
/// Whether a PHPSESSID is currently configured at all.
|
||||
configured: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct CrawlerStatusResponse {
|
||||
/// `"running"` | `"disabled"`.
|
||||
daemon: &'static str,
|
||||
phase: Option<Phase>,
|
||||
/// Configured chapter-worker count (for "N busy / M workers").
|
||||
worker_count: usize,
|
||||
/// Chapters being crawled right now, with live page counts.
|
||||
active_chapters: Vec<ActiveChapter>,
|
||||
/// The cover being fetched right now, if any.
|
||||
current_cover: Option<CoverTarget>,
|
||||
/// Mangas still queued for a cover fetch.
|
||||
covers_queued: i64,
|
||||
last_pass: LastPass,
|
||||
session: SessionStatus,
|
||||
/// `"healthy"` | `"draining"` | `"restarting"` | `"down"`.
|
||||
browser: &'static str,
|
||||
queue: QueueCounts,
|
||||
}
|
||||
|
||||
/// Per-stream memo of the two DB-derived counts shared by every SSE
|
||||
/// frame: crawler-job state breakdown and missing-cover backlog. Held
|
||||
/// across iterations of the unfold loop so a burst of status pokes
|
||||
/// emits one frame from cached counts. A fresh memo (`new`) always
|
||||
/// misses on first call, so `get_status` (one-shot) sees no stale data.
|
||||
struct QueueCountsMemo {
|
||||
cached: Option<(std::time::Instant, (i64, i64, i64), i64)>,
|
||||
}
|
||||
|
||||
impl QueueCountsMemo {
|
||||
fn new() -> Self {
|
||||
Self { cached: None }
|
||||
}
|
||||
|
||||
async fn get(
|
||||
&mut self,
|
||||
db: &sqlx::PgPool,
|
||||
) -> sqlx::Result<((i64, i64, i64), i64)> {
|
||||
if let Some((at, qc, cv)) = self.cached.as_ref() {
|
||||
if at.elapsed() < QUEUE_MEMO_TTL {
|
||||
return Ok((*qc, *cv));
|
||||
}
|
||||
}
|
||||
let qc = repo::crawler::job_state_counts(db).await?;
|
||||
let cv = repo::crawler::count_missing_covers(db).await?;
|
||||
self.cached = Some((std::time::Instant::now(), qc, cv));
|
||||
Ok((qc, cv))
|
||||
}
|
||||
}
|
||||
|
||||
fn browser_phase_str(p: RestartPhase) -> &'static str {
|
||||
match p {
|
||||
RestartPhase::Healthy => "healthy",
|
||||
RestartPhase::Draining => "draining",
|
||||
RestartPhase::Restarting => "restarting",
|
||||
}
|
||||
}
|
||||
|
||||
/// Compose a full status snapshot from the in-memory status, the
|
||||
/// browser/session flags, and DB queue-count queries (routed through
|
||||
/// a memo so a burst of pokes doesn't hammer Postgres). Shared by
|
||||
/// `get_status` (with a fresh per-call memo) and `stream_status`
|
||||
/// (with a per-stream memo held across iterations).
|
||||
async fn compose_status(
|
||||
state: &AppState,
|
||||
memo: &mut QueueCountsMemo,
|
||||
) -> AppResult<CrawlerStatusResponse> {
|
||||
let ((pending, running, dead), covers_queued) = memo.get(&state.db).await?;
|
||||
let queue = QueueCounts {
|
||||
pending,
|
||||
running,
|
||||
dead,
|
||||
};
|
||||
|
||||
Ok(match state.crawler().as_ref() {
|
||||
None => CrawlerStatusResponse {
|
||||
daemon: "disabled",
|
||||
phase: None,
|
||||
worker_count: 0,
|
||||
active_chapters: Vec::new(),
|
||||
current_cover: None,
|
||||
covers_queued,
|
||||
last_pass: LastPass::default(),
|
||||
session: SessionStatus {
|
||||
expired: false,
|
||||
configured: false,
|
||||
},
|
||||
browser: "down",
|
||||
queue,
|
||||
},
|
||||
Some(c) => {
|
||||
let snap = c.status.snapshot().await;
|
||||
CrawlerStatusResponse {
|
||||
daemon: "running",
|
||||
phase: Some(snap.phase),
|
||||
worker_count: snap.worker_count,
|
||||
active_chapters: snap.active_chapters,
|
||||
current_cover: snap.current_cover,
|
||||
covers_queued,
|
||||
last_pass: snap.last_pass,
|
||||
session: SessionStatus {
|
||||
expired: c.session.is_expired(),
|
||||
configured: c.session.current().await.is_some(),
|
||||
},
|
||||
browser: browser_phase_str(c.browser_manager.phase()),
|
||||
queue,
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
async fn get_status(
|
||||
State(state): State<AppState>,
|
||||
_admin: RequireAdmin,
|
||||
) -> AppResult<Json<CrawlerStatusResponse>> {
|
||||
// Fresh memo — one-shot calls always query the DB. The wrapper exists
|
||||
// only so compose_status can share its signature with the streaming
|
||||
// path; there is no caching across one-shot calls.
|
||||
let mut memo = QueueCountsMemo::new();
|
||||
Ok(Json(compose_status(&state, &mut memo).await?))
|
||||
}
|
||||
|
||||
/// Push live status to the dashboard instead of polling. Emits a snapshot
|
||||
/// immediately on connect, then on every status change (instant, via the
|
||||
/// `watch` notifier) and on a [`SSE_BACKSTOP`] tick (to refresh DB queue
|
||||
/// counts / browser phase that change without a status poke). The browser
|
||||
/// opens this only while the crawler page is mounted and closes it on
|
||||
/// navigate-away, so the subscription is scoped to the active page.
|
||||
async fn stream_status(
|
||||
State(state): State<AppState>,
|
||||
_admin: RequireAdmin,
|
||||
) -> Sse<impl Stream<Item = Result<Event, Infallible>>> {
|
||||
// Subscribe before the first emit so no change between the initial
|
||||
// snapshot and the first await is lost.
|
||||
let rx = state.crawler().as_ref().map(|c| c.status.subscribe());
|
||||
let memo = QueueCountsMemo::new();
|
||||
|
||||
let stream = futures_util::stream::unfold(
|
||||
(state, rx, memo, true),
|
||||
|(state, mut rx, mut memo, first)| async move {
|
||||
// After the first immediate emit, wait for a change or the
|
||||
// backstop tick before recomposing. On a change, debounce a
|
||||
// short window so a burst of pokes (one per stored page
|
||||
// during a chapter download, etc.) coalesces into a single
|
||||
// frame instead of hammering subscribers.
|
||||
if !first {
|
||||
match rx.as_mut() {
|
||||
Some(rx) => {
|
||||
tokio::select! {
|
||||
_ = rx.changed() => {
|
||||
tokio::time::sleep(SSE_DEBOUNCE).await;
|
||||
// Mark any pokes that arrived during the
|
||||
// debounce window as observed so the next
|
||||
// iteration only fires on NEW changes.
|
||||
rx.borrow_and_update();
|
||||
}
|
||||
_ = tokio::time::sleep(SSE_BACKSTOP) => {}
|
||||
}
|
||||
}
|
||||
None => tokio::time::sleep(SSE_BACKSTOP).await,
|
||||
}
|
||||
}
|
||||
// Compose; on a transient DB error, emit a keep-alive comment
|
||||
// rather than tearing down the stream.
|
||||
let event = match compose_status(&state, &mut memo).await {
|
||||
Ok(resp) => Event::default()
|
||||
.event("status")
|
||||
.json_data(&resp)
|
||||
.unwrap_or_else(|_| Event::default().comment("serialize error")),
|
||||
Err(_) => Event::default().comment("status unavailable"),
|
||||
};
|
||||
Some((Ok(event), (state, rx, memo, false)))
|
||||
},
|
||||
);
|
||||
|
||||
Sse::new(stream).keep_alive(KeepAlive::default())
|
||||
}
|
||||
@@ -4,8 +4,11 @@
|
||||
//! bot/API tokens cannot reach admin routes (see
|
||||
//! `crate::auth::extractor::RequireAdmin`).
|
||||
|
||||
pub mod analysis;
|
||||
pub mod crawler;
|
||||
pub mod mangas;
|
||||
pub mod resync;
|
||||
pub mod settings;
|
||||
pub mod system;
|
||||
pub mod users;
|
||||
|
||||
@@ -19,4 +22,7 @@ pub fn routes() -> Router<AppState> {
|
||||
.merge(mangas::routes())
|
||||
.merge(resync::routes())
|
||||
.merge(system::routes())
|
||||
.merge(crawler::routes())
|
||||
.merge(analysis::routes())
|
||||
.merge(settings::routes())
|
||||
}
|
||||
|
||||
@@ -58,8 +58,7 @@ async fn resync_manga(
|
||||
return Err(AppError::NotFound);
|
||||
}
|
||||
let resync = state
|
||||
.resync
|
||||
.as_ref()
|
||||
.resync()
|
||||
.ok_or_else(|| AppError::ServiceUnavailable(
|
||||
"crawler daemon is disabled; force resync unavailable".into(),
|
||||
))?;
|
||||
@@ -96,8 +95,7 @@ async fn resync_chapter(
|
||||
Path(chapter_id): Path<Uuid>,
|
||||
) -> AppResult<Json<ChapterResyncResponse>> {
|
||||
let resync = state
|
||||
.resync
|
||||
.as_ref()
|
||||
.resync()
|
||||
.ok_or_else(|| AppError::ServiceUnavailable(
|
||||
"crawler daemon is disabled; force resync unavailable".into(),
|
||||
))?;
|
||||
|
||||
239
backend/src/api/admin/settings.rs
Normal file
239
backend/src/api/admin/settings.rs
Normal file
@@ -0,0 +1,239 @@
|
||||
//! Admin endpoints for runtime-editable crawler / analysis configuration.
|
||||
//!
|
||||
//! `GET` returns the current editable settings DTO plus a read-only view of
|
||||
//! the env-managed (host/infra + secret) fields, so the dashboard can show
|
||||
//! the full picture while only letting the operator edit the safe knobs.
|
||||
//!
|
||||
//! `PUT` validates the incoming DTO against the env-derived base (re-parsing
|
||||
//! timezone/time, rebuilding the download allowlist), persists a normalized
|
||||
//! DTO and an audit row in one transaction, then asks the [`DaemonReloader`]
|
||||
//! to gracefully respawn the affected daemon with the new config — so the
|
||||
//! change takes effect without a restart. Validation failures return 422 with
|
||||
//! per-field details (the established `validation_failed` envelope).
|
||||
//!
|
||||
//! `PUT` is a **full replace**, not a patch: the body is deserialized with
|
||||
//! field-level defaults, so any omitted field is reset to its compiled
|
||||
//! default rather than retaining the stored value. The dashboard always
|
||||
//! sends the complete DTO (it round-trips the one it loaded); programmatic
|
||||
//! callers must do the same.
|
||||
|
||||
use axum::extract::State;
|
||||
use axum::routing::get;
|
||||
use axum::{Json, Router};
|
||||
use serde::Serialize;
|
||||
|
||||
use crate::app::AppState;
|
||||
use crate::auth::extractor::RequireAdmin;
|
||||
use crate::config::CrawlerConfig;
|
||||
use crate::crawler::browser::BrowserMode;
|
||||
use crate::error::{AppError, AppResult};
|
||||
use crate::repo;
|
||||
use crate::settings::{
|
||||
AnalysisSettings, CrawlerSettings, FieldErrors, PromptDefaults, KEY_ANALYSIS, KEY_CRAWLER,
|
||||
};
|
||||
|
||||
pub fn routes() -> Router<AppState> {
|
||||
Router::new()
|
||||
.route(
|
||||
"/admin/settings/crawler",
|
||||
get(get_crawler).put(put_crawler),
|
||||
)
|
||||
.route(
|
||||
"/admin/settings/analysis",
|
||||
get(get_analysis).put(put_analysis),
|
||||
)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Response shapes
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Read-only mirror of the crawler fields managed via environment (host/infra
|
||||
/// + secrets), surfaced so the admin sees what's in effect without editing it.
|
||||
#[derive(Debug, Serialize)]
|
||||
struct CrawlerEnvOnly {
|
||||
browser_mode: &'static str,
|
||||
browser_args: Vec<String>,
|
||||
proxy: Option<String>,
|
||||
tor_control_url: Option<String>,
|
||||
/// Whether a TOR control credential (password or cookie file) is set.
|
||||
tor_credentials_configured: bool,
|
||||
/// Whether an initial `CRAWLER_PHPSESSID` is configured (the live session
|
||||
/// is managed separately via the crawler dashboard).
|
||||
session_configured: bool,
|
||||
}
|
||||
|
||||
impl CrawlerEnvOnly {
|
||||
fn from_base(base: &CrawlerConfig) -> Self {
|
||||
Self {
|
||||
browser_mode: match base.browser.mode {
|
||||
BrowserMode::Headed => "headed",
|
||||
BrowserMode::Headless => "headless",
|
||||
},
|
||||
browser_args: base.browser.extra_args.clone(),
|
||||
proxy: base.proxy.clone(),
|
||||
tor_control_url: base.tor_control_url.clone(),
|
||||
tor_credentials_configured: base.tor_control_password.is_some()
|
||||
|| base.tor_control_cookie_path.is_some(),
|
||||
session_configured: base.phpsessid.is_some(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct CrawlerSettingsResponse {
|
||||
editable: CrawlerSettings,
|
||||
env_only: CrawlerEnvOnly,
|
||||
}
|
||||
|
||||
/// Read-only mirror of analysis env-managed fields (just the secret).
|
||||
#[derive(Debug, Serialize)]
|
||||
struct AnalysisEnvOnly {
|
||||
api_key_configured: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct AnalysisSettingsResponse {
|
||||
editable: AnalysisSettings,
|
||||
env_only: AnalysisEnvOnly,
|
||||
/// The compiled prompt defaults so the UI can show placeholders and
|
||||
/// implement per-prompt "reset to default".
|
||||
prompt_defaults: PromptDefaults,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Handlers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn validation_error(errs: FieldErrors) -> AppError {
|
||||
AppError::ValidationFailed {
|
||||
message: "invalid settings".to_string(),
|
||||
details: serde_json::json!({ "fields": errs.errors }),
|
||||
}
|
||||
}
|
||||
|
||||
/// Current effective crawler settings: the stored DTO when present, otherwise
|
||||
/// derived from the env base (the row is seeded at boot, so the stored branch
|
||||
/// is the norm).
|
||||
async fn load_crawler_dto(state: &AppState) -> AppResult<CrawlerSettings> {
|
||||
Ok(match repo::app_settings::get(&state.db, KEY_CRAWLER).await? {
|
||||
Some(v) => serde_json::from_value(v)
|
||||
.unwrap_or_else(|_| CrawlerSettings::from_config(&state.crawler_base)),
|
||||
None => CrawlerSettings::from_config(&state.crawler_base),
|
||||
})
|
||||
}
|
||||
|
||||
async fn load_analysis_dto(state: &AppState) -> AppResult<AnalysisSettings> {
|
||||
Ok(match repo::app_settings::get(&state.db, KEY_ANALYSIS).await? {
|
||||
Some(v) => serde_json::from_value(v)
|
||||
.unwrap_or_else(|_| AnalysisSettings::from_config(&state.analysis_base)),
|
||||
None => AnalysisSettings::from_config(&state.analysis_base),
|
||||
})
|
||||
}
|
||||
|
||||
async fn get_crawler(
|
||||
State(state): State<AppState>,
|
||||
_admin: RequireAdmin,
|
||||
) -> AppResult<Json<CrawlerSettingsResponse>> {
|
||||
let editable = load_crawler_dto(&state).await?;
|
||||
Ok(Json(CrawlerSettingsResponse {
|
||||
editable,
|
||||
env_only: CrawlerEnvOnly::from_base(&state.crawler_base),
|
||||
}))
|
||||
}
|
||||
|
||||
async fn put_crawler(
|
||||
State(state): State<AppState>,
|
||||
admin: RequireAdmin,
|
||||
Json(incoming): Json<CrawlerSettings>,
|
||||
) -> AppResult<Json<CrawlerSettingsResponse>> {
|
||||
// Validate by converting against the env base; this also rebuilds the
|
||||
// download allowlist and re-parses tz / time.
|
||||
let cfg = incoming
|
||||
.to_config(&state.crawler_base)
|
||||
.map_err(validation_error)?;
|
||||
// Persist a normalized DTO (so the stored/returned allowlist reflects the
|
||||
// effective hosts, prompts are canonicalized, etc.).
|
||||
let normalized = CrawlerSettings::from_config(&cfg);
|
||||
let value = serde_json::to_value(&normalized).map_err(|e| AppError::Other(e.into()))?;
|
||||
|
||||
let mut tx = state.db.begin().await?;
|
||||
repo::app_settings::upsert(&mut *tx, KEY_CRAWLER, &value).await?;
|
||||
repo::admin_audit::insert(
|
||||
&mut *tx,
|
||||
admin.0.id,
|
||||
"update_crawler_settings",
|
||||
"settings",
|
||||
None,
|
||||
value.clone(),
|
||||
)
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
|
||||
// Apply live (graceful respawn) when a reloader is wired up.
|
||||
if let Some(reloader) = &state.reloader {
|
||||
reloader
|
||||
.reload_crawler(cfg)
|
||||
.await
|
||||
.map_err(AppError::Other)?;
|
||||
}
|
||||
|
||||
Ok(Json(CrawlerSettingsResponse {
|
||||
editable: normalized,
|
||||
env_only: CrawlerEnvOnly::from_base(&state.crawler_base),
|
||||
}))
|
||||
}
|
||||
|
||||
async fn get_analysis(
|
||||
State(state): State<AppState>,
|
||||
_admin: RequireAdmin,
|
||||
) -> AppResult<Json<AnalysisSettingsResponse>> {
|
||||
let editable = load_analysis_dto(&state).await?;
|
||||
Ok(Json(AnalysisSettingsResponse {
|
||||
editable,
|
||||
env_only: AnalysisEnvOnly {
|
||||
api_key_configured: state.analysis_base.api_key.is_some(),
|
||||
},
|
||||
prompt_defaults: PromptDefaults::get(),
|
||||
}))
|
||||
}
|
||||
|
||||
async fn put_analysis(
|
||||
State(state): State<AppState>,
|
||||
admin: RequireAdmin,
|
||||
Json(incoming): Json<AnalysisSettings>,
|
||||
) -> AppResult<Json<AnalysisSettingsResponse>> {
|
||||
let cfg = incoming
|
||||
.to_config(&state.analysis_base)
|
||||
.map_err(validation_error)?;
|
||||
let normalized = AnalysisSettings::from_config(&cfg);
|
||||
let value = serde_json::to_value(&normalized).map_err(|e| AppError::Other(e.into()))?;
|
||||
|
||||
let mut tx = state.db.begin().await?;
|
||||
repo::app_settings::upsert(&mut *tx, KEY_ANALYSIS, &value).await?;
|
||||
repo::admin_audit::insert(
|
||||
&mut *tx,
|
||||
admin.0.id,
|
||||
"update_analysis_settings",
|
||||
"settings",
|
||||
None,
|
||||
value.clone(),
|
||||
)
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
|
||||
if let Some(reloader) = &state.reloader {
|
||||
reloader
|
||||
.reload_analysis(cfg)
|
||||
.await
|
||||
.map_err(AppError::Other)?;
|
||||
}
|
||||
|
||||
Ok(Json(AnalysisSettingsResponse {
|
||||
editable: normalized,
|
||||
env_only: AnalysisEnvOnly {
|
||||
api_key_configured: state.analysis_base.api_key.is_some(),
|
||||
},
|
||||
prompt_defaults: PromptDefaults::get(),
|
||||
}))
|
||||
}
|
||||
@@ -137,6 +137,7 @@ async fn create(
|
||||
)
|
||||
.await?;
|
||||
|
||||
let mut page_ids: Vec<Uuid> = Vec::with_capacity(pages.len());
|
||||
for (idx, page) in pages.iter().enumerate() {
|
||||
let page_number = (idx + 1) as i32;
|
||||
let nnnn = format!("{:04}", page_number);
|
||||
@@ -145,7 +146,9 @@ async fn create(
|
||||
manga_id, chapter.id, nnnn, page.ext
|
||||
);
|
||||
state.storage.put(&key, &page.bytes).await?;
|
||||
repo::page::create(&mut *tx, chapter.id, page_number, &key, page.mime).await?;
|
||||
let created =
|
||||
repo::page::create(&mut *tx, chapter.id, page_number, &key, page.mime).await?;
|
||||
page_ids.push(created.id);
|
||||
}
|
||||
|
||||
let page_count = pages.len() as i32;
|
||||
@@ -154,6 +157,20 @@ async fn create(
|
||||
|
||||
tx.commit().await?;
|
||||
|
||||
// Enqueue AI content-analysis for each new page. Done after commit so a
|
||||
// rolled-back upload never leaves jobs pointing at nonexistent pages; a
|
||||
// failed enqueue is logged but doesn't fail the upload (the admin
|
||||
// re-enqueue endpoint can backfill).
|
||||
if state.analysis_enabled() {
|
||||
for page_id in page_ids {
|
||||
if let Err(e) =
|
||||
repo::page_analysis::enqueue_for_page(&state.db, page_id, false).await
|
||||
{
|
||||
tracing::warn!(%page_id, error = %e, "failed to enqueue page analysis");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok((StatusCode::CREATED, Json(chapter)))
|
||||
}
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@ use crate::api::pagination::PagedResponse;
|
||||
use crate::app::AppState;
|
||||
use crate::auth::extractor::CurrentUser;
|
||||
use crate::domain::collection::{
|
||||
Collection, CollectionPatch, CollectionSummary, NewCollection,
|
||||
Collection, CollectionPageItem, CollectionPatch, CollectionSummary, NewCollection,
|
||||
};
|
||||
use crate::domain::manga::Manga;
|
||||
use crate::domain::patch::Patch;
|
||||
@@ -27,10 +27,16 @@ pub fn routes() -> Router<AppState> {
|
||||
"/collections/:id/mangas/:manga_id",
|
||||
delete(remove_manga),
|
||||
)
|
||||
.route("/collections/:id/pages", get(list_pages).post(add_page))
|
||||
.route("/collections/:id/pages/:page_id", delete(remove_page))
|
||||
.route(
|
||||
"/mangas/:id/my-collections",
|
||||
get(list_my_collections_containing),
|
||||
)
|
||||
.route(
|
||||
"/pages/:id/my-collections",
|
||||
get(list_my_collections_containing_page),
|
||||
)
|
||||
}
|
||||
|
||||
const MAX_NAME_LEN: usize = 64;
|
||||
@@ -54,11 +60,21 @@ pub struct AddMangaBody {
|
||||
pub manga_id: Uuid,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct AddPageBody {
|
||||
pub page_id: Uuid,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct MangaCollectionIds {
|
||||
pub collection_ids: Vec<Uuid>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct PageCollectionIds {
|
||||
pub collection_ids: Vec<Uuid>,
|
||||
}
|
||||
|
||||
fn validate_name(name: &str) -> AppResult<()> {
|
||||
let trimmed = name.trim();
|
||||
if trimmed.is_empty() {
|
||||
@@ -218,6 +234,57 @@ async fn list_my_collections_containing(
|
||||
Ok(Json(MangaCollectionIds { collection_ids: ids }))
|
||||
}
|
||||
|
||||
async fn list_pages(
|
||||
State(state): State<AppState>,
|
||||
CurrentUser(user): CurrentUser,
|
||||
Path(id): Path<Uuid>,
|
||||
Query(params): Query<ListParams>,
|
||||
) -> AppResult<Json<PagedResponse<CollectionPageItem>>> {
|
||||
require_owner_id(&state, user.id, id).await?;
|
||||
let limit = params.limit.clamp(1, 200);
|
||||
let offset = params.offset.max(0);
|
||||
let (items, total) =
|
||||
repo::collection::list_pages(&state.db, id, limit, offset).await?;
|
||||
Ok(Json(PagedResponse::with_total(items, limit, offset, total)))
|
||||
}
|
||||
|
||||
async fn add_page(
|
||||
State(state): State<AppState>,
|
||||
CurrentUser(user): CurrentUser,
|
||||
Path(id): Path<Uuid>,
|
||||
Json(body): Json<AddPageBody>,
|
||||
) -> AppResult<StatusCode> {
|
||||
require_owner_id(&state, user.id, id).await?;
|
||||
// FK violation in `repo::collection::add_page` maps to NotFound, so
|
||||
// no separate `repo::page::exists` check is needed — the insert is
|
||||
// the existence check.
|
||||
let created = repo::collection::add_page(&state.db, id, body.page_id).await?;
|
||||
Ok(if created { StatusCode::CREATED } else { StatusCode::OK })
|
||||
}
|
||||
|
||||
async fn remove_page(
|
||||
State(state): State<AppState>,
|
||||
CurrentUser(user): CurrentUser,
|
||||
Path((collection_id, page_id)): Path<(Uuid, Uuid)>,
|
||||
) -> AppResult<StatusCode> {
|
||||
require_owner_id(&state, user.id, collection_id).await?;
|
||||
repo::collection::remove_page(&state.db, collection_id, page_id).await?;
|
||||
Ok(StatusCode::NO_CONTENT)
|
||||
}
|
||||
|
||||
async fn list_my_collections_containing_page(
|
||||
State(state): State<AppState>,
|
||||
CurrentUser(user): CurrentUser,
|
||||
Path(page_id): Path<Uuid>,
|
||||
) -> AppResult<Json<PageCollectionIds>> {
|
||||
// Mirrors `list_my_collections_containing` for pages: unknown page
|
||||
// returns an empty list, not 404 — keeps the endpoint side-effect-
|
||||
// free and skips a distinguishing-status oracle.
|
||||
let ids = repo::collection::list_collections_containing_page(&state.db, user.id, page_id)
|
||||
.await?;
|
||||
Ok(Json(PageCollectionIds { collection_ids: ids }))
|
||||
}
|
||||
|
||||
/// Returns the row iff the caller owns it. Both "doesn't exist" and
|
||||
/// "exists but belongs to someone else" surface as `NotFound` so the
|
||||
/// API doesn't disclose collection existence to non-owners — the
|
||||
|
||||
@@ -21,6 +21,7 @@ pub fn routes() -> Router<AppState> {
|
||||
Router::new()
|
||||
.route("/mangas", get(list).post(create))
|
||||
.route("/mangas/:id", get(get_one).patch(update))
|
||||
.route("/mangas/:id/similar", get(list_similar))
|
||||
.route("/mangas/:id/cover", put(put_cover).delete(delete_cover))
|
||||
.route("/mangas/:id/tags", post(attach_tag))
|
||||
.route("/mangas/:id/tags/:tag_id", delete(detach_tag))
|
||||
@@ -40,6 +41,12 @@ pub struct ListParams {
|
||||
pub genre_id: Option<String>,
|
||||
#[serde(default)]
|
||||
pub tag_id: Option<String>,
|
||||
/// Comma-separated content warnings the manga must carry (AND).
|
||||
#[serde(default)]
|
||||
pub cw_include: Option<String>,
|
||||
/// Comma-separated content warnings the manga must NOT carry (any).
|
||||
#[serde(default)]
|
||||
pub cw_exclude: Option<String>,
|
||||
#[serde(default = "default_limit")]
|
||||
pub limit: i64,
|
||||
#[serde(default)]
|
||||
@@ -93,6 +100,8 @@ async fn list(
|
||||
author_ids: parse_uuid_csv("author_id", params.author_id.as_deref())?,
|
||||
genre_ids: parse_uuid_csv("genre_id", params.genre_id.as_deref())?,
|
||||
tag_ids: parse_uuid_csv("tag_id", params.tag_id.as_deref())?,
|
||||
cw_include: crate::api::page_tags::parse_warnings_csv(params.cw_include.as_deref())?,
|
||||
cw_exclude: crate::api::page_tags::parse_warnings_csv(params.cw_exclude.as_deref())?,
|
||||
limit,
|
||||
offset,
|
||||
sort: params.sort,
|
||||
@@ -108,6 +117,28 @@ async fn get_one(
|
||||
Ok(Json(repo::manga::get_detail(&state.db, id).await?))
|
||||
}
|
||||
|
||||
/// How many similar mangas the recommendation section shows.
|
||||
const SIMILAR_LIMIT: i64 = 5;
|
||||
|
||||
/// `GET /api/v1/mangas/:id/similar` — top-`SIMILAR_LIMIT` mangas ranked by
|
||||
/// tag overlap with `:id`, as cards. Read-only and unauthenticated, like
|
||||
/// `get_one`/`list`. Returns a plain `{ items: [...] }` object (not the
|
||||
/// paginated envelope) since this is a fixed top-N, not a collection.
|
||||
///
|
||||
/// The `exists` check is load-bearing: `list_similar` returns an empty list
|
||||
/// for both an untagged manga and a nonexistent id, so without it an unknown
|
||||
/// id would 200 with `[]` instead of 404.
|
||||
async fn list_similar(
|
||||
State(state): State<AppState>,
|
||||
Path(id): Path<Uuid>,
|
||||
) -> AppResult<Json<serde_json::Value>> {
|
||||
if !repo::manga::exists(&state.db, id).await? {
|
||||
return Err(AppError::NotFound);
|
||||
}
|
||||
let items = repo::manga::list_similar(&state.db, id, SIMILAR_LIMIT).await?;
|
||||
Ok(Json(json!({ "items": items })))
|
||||
}
|
||||
|
||||
/// `POST /api/v1/mangas` is multipart/form-data. Parts:
|
||||
///
|
||||
/// - `metadata` (required): JSON body matching `NewManga` — title, optional
|
||||
|
||||
@@ -9,6 +9,7 @@ pub mod genres;
|
||||
pub mod health;
|
||||
pub mod history;
|
||||
pub mod mangas;
|
||||
pub mod page_tags;
|
||||
pub mod pagination;
|
||||
pub mod tags;
|
||||
|
||||
@@ -28,6 +29,7 @@ pub fn routes() -> Router<AppState> {
|
||||
.merge(tags::routes())
|
||||
.merge(authors::routes())
|
||||
.merge(collections::routes())
|
||||
.merge(page_tags::routes())
|
||||
.merge(history::routes())
|
||||
.merge(admin::routes())
|
||||
}
|
||||
|
||||
549
backend/src/api/page_tags.rs
Normal file
549
backend/src/api/page_tags.rs
Normal file
@@ -0,0 +1,549 @@
|
||||
//! Per-page tag endpoints. See `migration 0023` for the underlying
|
||||
//! schema and `repo::page_tag` for the query layer. All endpoints
|
||||
//! require `CurrentUser` — every byte of this data is owned by the
|
||||
//! caller.
|
||||
|
||||
use axum::extract::{Path, Query, State};
|
||||
use axum::http::StatusCode;
|
||||
use axum::routing::{delete, get, post};
|
||||
use axum::{Json, Router};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::json;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::api::pagination::PagedResponse;
|
||||
use crate::app::AppState;
|
||||
use crate::auth::extractor::CurrentUser;
|
||||
use crate::domain::page_analysis::{ContentWarning, PageSearchItem};
|
||||
use crate::domain::page_tag::{
|
||||
NewPageTag, PageTagSummary, TaggedChapterAggregate, TaggedMangaAggregate,
|
||||
TaggedPageItem,
|
||||
};
|
||||
use crate::error::{AppError, AppResult};
|
||||
use crate::repo;
|
||||
use crate::repo::page_analysis::PageSearchQuery;
|
||||
use crate::repo::page_tag::Order;
|
||||
|
||||
pub fn routes() -> Router<AppState> {
|
||||
Router::new()
|
||||
.route("/pages/:id/tags", post(add))
|
||||
.route("/pages/:id/tags/:tag", delete(remove))
|
||||
// GET uses `/my-tags` to mirror the `mangas/:id/my-collections`
|
||||
// convention — the URL says whose tags we're reading even
|
||||
// though the cookie already implies it.
|
||||
.route("/pages/:id/my-tags", get(list_for_page))
|
||||
.route("/me/page-tags", get(list_mine))
|
||||
.route("/me/page-search", get(page_search))
|
||||
.route("/me/page-tags/distinct", get(list_distinct_mine))
|
||||
.route("/me/page-tags/chapters", get(list_chapters_for_tag))
|
||||
.route("/me/page-tags/mangas", get(list_mangas_for_tag))
|
||||
}
|
||||
|
||||
const MAX_TAG_LEN: usize = 64;
|
||||
/// Hard wire-level byte cap, applied before any allocation in
|
||||
/// `normalize_tag`. A 10MB tag in the JSON body would otherwise
|
||||
/// allocate twice in `to_lowercase()` + `split_whitespace().collect()`
|
||||
/// before the 64-char post-normalize cap rejected it. 4 KiB is a
|
||||
/// generous ~64x the post-normalize char cap and well below anything
|
||||
/// a user could fairly call a "tag".
|
||||
const MAX_TAG_BYTES: usize = 4096;
|
||||
const DEFAULT_LIMIT: i64 = 50;
|
||||
const DEFAULT_DISTINCT_LIMIT: i64 = 200;
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct ListMineParams {
|
||||
#[serde(default = "default_limit")]
|
||||
pub limit: i64,
|
||||
#[serde(default)]
|
||||
pub offset: i64,
|
||||
/// Restrict to this exact tag (chip filter in the library tab).
|
||||
#[serde(default)]
|
||||
pub tag: Option<String>,
|
||||
/// Prefix filter (autocomplete-style).
|
||||
#[serde(default)]
|
||||
pub q: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct DistinctParams {
|
||||
#[serde(default)]
|
||||
pub q: Option<String>,
|
||||
#[serde(default = "default_distinct_limit")]
|
||||
pub limit: i64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct TagsResponse {
|
||||
tags: Vec<String>,
|
||||
}
|
||||
|
||||
fn default_limit() -> i64 {
|
||||
DEFAULT_LIMIT
|
||||
}
|
||||
fn default_distinct_limit() -> i64 {
|
||||
DEFAULT_DISTINCT_LIMIT
|
||||
}
|
||||
|
||||
/// Normalize a user-supplied tag for storage. Lowercases, trims,
|
||||
/// collapses internal whitespace, rejects control chars and edge
|
||||
/// colons, caps at 64 chars. Single source of truth so the same input
|
||||
/// produces the same row regardless of which client sent it.
|
||||
/// True for Unicode format / invisible chars that would let two
|
||||
/// visually-identical tags coexist as distinct rows — `"funny"` and
|
||||
/// `"funny\u{200d}"` are different codepoints but render the same, and
|
||||
/// the autocomplete + chip cloud would split them. We hardcode the
|
||||
/// well-known offenders (a subset of the Unicode `Cf` general category
|
||||
/// plus the bidi-override block) so the file stays free of new deps.
|
||||
fn is_invisible_format(c: char) -> bool {
|
||||
matches!(c,
|
||||
// Soft hyphen, Arabic letter mark, Mongolian vowel separator.
|
||||
'\u{00AD}' | '\u{061C}' | '\u{180E}'
|
||||
// ZWSP, ZWNJ, ZWJ, LRM, RLM.
|
||||
| '\u{200B}'..='\u{200F}'
|
||||
// LRE / RLE / PDF / LRO / RLO.
|
||||
| '\u{202A}'..='\u{202E}'
|
||||
// Word joiner, function-application markers, deprecated
|
||||
// formatting (U+206A..U+206F).
|
||||
| '\u{2060}'..='\u{2064}'
|
||||
| '\u{2066}'..='\u{206F}'
|
||||
// BOM / ZWNBSP.
|
||||
| '\u{FEFF}'
|
||||
// Plane-14 LANGUAGE TAG + TAG characters. These can tunnel
|
||||
// ASCII semantics invisibly (the same mechanism used in 2024's
|
||||
// LLM prompt-injection smuggling work). Storing them would
|
||||
// let two visually-identical tags coexist while carrying
|
||||
// distinct hidden payloads.
|
||||
| '\u{E0001}'
|
||||
| '\u{E0020}'..='\u{E007F}'
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn normalize_tag(input: &str) -> AppResult<String> {
|
||||
// Fast-fail on absurd input before allocating in lowercase /
|
||||
// whitespace passes. Cheap and bounds worst-case work.
|
||||
if input.len() > MAX_TAG_BYTES {
|
||||
return Err(AppError::ValidationFailed {
|
||||
message: "tag too long".into(),
|
||||
details: json!({ "tag": format!("max {MAX_TAG_BYTES} bytes") }),
|
||||
});
|
||||
}
|
||||
let trimmed = input.trim();
|
||||
if trimmed.is_empty() {
|
||||
return Err(AppError::ValidationFailed {
|
||||
message: "tag is required".into(),
|
||||
details: json!({ "tag": "required" }),
|
||||
});
|
||||
}
|
||||
if trimmed.chars().any(|c| c.is_control()) {
|
||||
return Err(AppError::ValidationFailed {
|
||||
message: "tag contains control characters".into(),
|
||||
details: json!({ "tag": "control_chars" }),
|
||||
});
|
||||
}
|
||||
if trimmed.chars().any(is_invisible_format) {
|
||||
return Err(AppError::ValidationFailed {
|
||||
message: "tag contains invisible / format characters".into(),
|
||||
details: json!({ "tag": "invisible_chars" }),
|
||||
});
|
||||
}
|
||||
// Reject characters that would break the DELETE URL path
|
||||
// (`/v1/pages/:id/tags/:tag`). axum decodes %2F back to `/` after
|
||||
// routing, so a tag containing one of these would be silently
|
||||
// unreachable to the delete handler — store-only. Reject up front
|
||||
// so every stored tag is removable. `%` and `_` are also rejected
|
||||
// since they are LIKE wildcards in the `list_for_user` prefix
|
||||
// filter and would otherwise let a user accidentally search for
|
||||
// anything.
|
||||
if trimmed.chars().any(|c| matches!(c, '/' | '\\' | '?' | '#' | '%' | '_')) {
|
||||
return Err(AppError::ValidationFailed {
|
||||
message: "tag contains a forbidden character (/ \\ ? # % _)".into(),
|
||||
details: json!({ "tag": "forbidden_chars" }),
|
||||
});
|
||||
}
|
||||
// Lowercase + collapse internal whitespace runs (any kind: spaces,
|
||||
// tabs, fullwidth space, etc.) into a single ASCII space. This
|
||||
// keeps "Foo Bar" and "foo bar" identifying the same tag.
|
||||
let normalized: String = trimmed
|
||||
.to_lowercase()
|
||||
.split_whitespace()
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ");
|
||||
// `namespace:value` is permitted but a bare leading/trailing colon
|
||||
// is nonsense and would break any future split.
|
||||
if normalized.starts_with(':') || normalized.ends_with(':') {
|
||||
return Err(AppError::ValidationFailed {
|
||||
message: "tag must not start or end with ':'".into(),
|
||||
details: json!({ "tag": "edge_colon" }),
|
||||
});
|
||||
}
|
||||
if normalized.chars().count() > MAX_TAG_LEN {
|
||||
// "After normalization" because Unicode case-folding can
|
||||
// expand chars (Turkish capital `İ` → `i\u{307}`, so 33 İ's
|
||||
// pass the wire-level 33-char input but trip this 64 cap).
|
||||
return Err(AppError::ValidationFailed {
|
||||
message: "tag too long after normalization".into(),
|
||||
details: json!({ "tag": format!("max {MAX_TAG_LEN} characters") }),
|
||||
});
|
||||
}
|
||||
Ok(normalized)
|
||||
}
|
||||
|
||||
async fn add(
|
||||
State(state): State<AppState>,
|
||||
CurrentUser(user): CurrentUser,
|
||||
Path(page_id): Path<Uuid>,
|
||||
Json(input): Json<NewPageTag>,
|
||||
) -> AppResult<StatusCode> {
|
||||
let tag = normalize_tag(&input.tag)?;
|
||||
let created = repo::page_tag::upsert(&state.db, user.id, page_id, &tag).await?;
|
||||
Ok(if created { StatusCode::CREATED } else { StatusCode::OK })
|
||||
}
|
||||
|
||||
async fn remove(
|
||||
State(state): State<AppState>,
|
||||
CurrentUser(user): CurrentUser,
|
||||
Path((page_id, tag)): Path<(Uuid, String)>,
|
||||
) -> AppResult<StatusCode> {
|
||||
// Normalize the URL-decoded tag the same way add() did, so
|
||||
// DELETE /pages/.../tags/Funny removes the row stored as "funny".
|
||||
let normalized = normalize_tag(&tag)?;
|
||||
repo::page_tag::remove(&state.db, user.id, page_id, &normalized).await?;
|
||||
Ok(StatusCode::NO_CONTENT)
|
||||
}
|
||||
|
||||
async fn list_for_page(
|
||||
State(state): State<AppState>,
|
||||
CurrentUser(user): CurrentUser,
|
||||
Path(page_id): Path<Uuid>,
|
||||
) -> AppResult<Json<TagsResponse>> {
|
||||
let tags = repo::page_tag::list_for_page(&state.db, user.id, page_id).await?;
|
||||
Ok(Json(TagsResponse { tags }))
|
||||
}
|
||||
|
||||
async fn list_mine(
|
||||
State(state): State<AppState>,
|
||||
CurrentUser(user): CurrentUser,
|
||||
Query(params): Query<ListMineParams>,
|
||||
) -> AppResult<Json<PagedResponse<TaggedPageItem>>> {
|
||||
let limit = params.limit.clamp(1, 200);
|
||||
let offset = params.offset.max(0);
|
||||
// Filters from the wire arrive raw — normalize them so a filter on
|
||||
// "Funny" matches rows stored as "funny".
|
||||
let tag_filter = params
|
||||
.tag
|
||||
.as_deref()
|
||||
.map(normalize_tag)
|
||||
.transpose()?;
|
||||
let prefix_filter = params
|
||||
.q
|
||||
.as_deref()
|
||||
.map(normalize_tag)
|
||||
.transpose()?;
|
||||
let (items, total) = repo::page_tag::list_for_user(
|
||||
&state.db,
|
||||
user.id,
|
||||
tag_filter.as_deref(),
|
||||
prefix_filter.as_deref(),
|
||||
limit,
|
||||
offset,
|
||||
)
|
||||
.await?;
|
||||
Ok(Json(PagedResponse::with_total(items, limit, offset, total)))
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct PageSearchParams {
|
||||
#[serde(default = "default_limit")]
|
||||
pub limit: i64,
|
||||
#[serde(default)]
|
||||
pub offset: i64,
|
||||
/// Comma-separated tags, AND-ed. Matched against the caller's page
|
||||
/// tags ∪ the global auto-tags.
|
||||
#[serde(default)]
|
||||
pub tags: Option<String>,
|
||||
/// Free-text query over the OCR + scene-description search document.
|
||||
#[serde(default)]
|
||||
pub text: Option<String>,
|
||||
/// Comma-separated content warnings the page must carry (all of them).
|
||||
#[serde(default)]
|
||||
pub cw_include: Option<String>,
|
||||
/// Comma-separated content warnings the page must NOT carry (any of).
|
||||
#[serde(default)]
|
||||
pub cw_exclude: Option<String>,
|
||||
}
|
||||
|
||||
/// Split a comma-separated tag list into normalized, deduped tag names.
|
||||
fn parse_tags_csv(raw: Option<&str>) -> AppResult<Vec<String>> {
|
||||
let Some(raw) = raw else { return Ok(Vec::new()) };
|
||||
let mut seen = std::collections::HashSet::new();
|
||||
let mut out = Vec::new();
|
||||
for part in raw.split(',') {
|
||||
if part.trim().is_empty() {
|
||||
continue;
|
||||
}
|
||||
let norm = normalize_tag(part)?;
|
||||
if seen.insert(norm.clone()) {
|
||||
out.push(norm);
|
||||
}
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Split + validate a comma-separated content-warning list against the
|
||||
/// closed vocabulary, returning canonical lowercase names. Unknown values
|
||||
/// are a 422 rather than a silent drop so a typo'd filter is visible.
|
||||
pub(crate) fn parse_warnings_csv(raw: Option<&str>) -> AppResult<Vec<String>> {
|
||||
let Some(raw) = raw else { return Ok(Vec::new()) };
|
||||
let mut seen = std::collections::HashSet::new();
|
||||
let mut out = Vec::new();
|
||||
for part in raw.split(',') {
|
||||
let t = part.trim();
|
||||
if t.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let w = ContentWarning::parse_strict(t).ok_or_else(|| AppError::ValidationFailed {
|
||||
message: format!("unknown content warning {t:?}"),
|
||||
details: json!({ "content_warning": "must be one of sexual|nudity|gore|violence|disturbing" }),
|
||||
})?;
|
||||
let canon = format!("{w:?}").to_lowercase();
|
||||
if seen.insert(canon.clone()) {
|
||||
out.push(canon);
|
||||
}
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Content search over the caller's reachable pages: multi-tag AND (own
|
||||
/// page tags ∪ global auto-tags), weighted OCR/scene text ranking, and
|
||||
/// content-warning include/exclude. At least one positive filter (tags,
|
||||
/// text, or cw_include) is required so the endpoint never dumps every page.
|
||||
async fn page_search(
|
||||
State(state): State<AppState>,
|
||||
CurrentUser(user): CurrentUser,
|
||||
Query(params): Query<PageSearchParams>,
|
||||
) -> AppResult<Json<PagedResponse<PageSearchItem>>> {
|
||||
let limit = params.limit.clamp(1, 200);
|
||||
let offset = params.offset.max(0);
|
||||
let tags = parse_tags_csv(params.tags.as_deref())?;
|
||||
let cw_include = parse_warnings_csv(params.cw_include.as_deref())?;
|
||||
let cw_exclude = parse_warnings_csv(params.cw_exclude.as_deref())?;
|
||||
let text = params
|
||||
.text
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|s| !s.is_empty())
|
||||
.map(str::to_string);
|
||||
|
||||
if tags.is_empty() && text.is_none() && cw_include.is_empty() {
|
||||
return Err(AppError::ValidationFailed {
|
||||
message: "at least one of tags, text, or cw_include is required".into(),
|
||||
details: json!({ "filter": "required" }),
|
||||
});
|
||||
}
|
||||
|
||||
let query = PageSearchQuery {
|
||||
user_id: user.id,
|
||||
tags,
|
||||
text,
|
||||
cw_include,
|
||||
cw_exclude,
|
||||
limit,
|
||||
offset,
|
||||
};
|
||||
let (items, total) = repo::page_analysis::page_search(&state.db, &query).await?;
|
||||
Ok(Json(PagedResponse::with_total(items, limit, offset, total)))
|
||||
}
|
||||
|
||||
async fn list_distinct_mine(
|
||||
State(state): State<AppState>,
|
||||
CurrentUser(user): CurrentUser,
|
||||
Query(params): Query<DistinctParams>,
|
||||
) -> AppResult<Json<DistinctResponse>> {
|
||||
let limit = params.limit.clamp(1, 500);
|
||||
let prefix = params
|
||||
.q
|
||||
.as_deref()
|
||||
.map(normalize_tag)
|
||||
.transpose()?;
|
||||
let items = repo::page_tag::distinct_tags_for_user(
|
||||
&state.db,
|
||||
user.id,
|
||||
prefix.as_deref(),
|
||||
limit,
|
||||
)
|
||||
.await?;
|
||||
Ok(Json(DistinctResponse { items }))
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct DistinctResponse {
|
||||
items: Vec<PageTagSummary>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct AggregateParams {
|
||||
/// Required. Exact tag to aggregate by. Empty / whitespace-only
|
||||
/// inputs are rejected with 422 via `normalize_tag`.
|
||||
pub tag: String,
|
||||
/// `desc` (default) or `asc`. Anything else → 422.
|
||||
#[serde(default)]
|
||||
pub order: Option<String>,
|
||||
#[serde(default = "default_limit")]
|
||||
pub limit: i64,
|
||||
#[serde(default)]
|
||||
pub offset: i64,
|
||||
/// Reserved for the planned OCR text-search input. Accepted on
|
||||
/// the wire so adding OCR later won't break the API shape, but
|
||||
/// rejected with 501 `text_search_not_yet_supported` if non-empty
|
||||
/// until the backend supports it.
|
||||
#[serde(default)]
|
||||
pub text: Option<String>,
|
||||
}
|
||||
|
||||
fn parse_order(raw: Option<&str>) -> AppResult<Order> {
|
||||
match raw.map(str::trim) {
|
||||
None | Some("") | Some("desc") => Ok(Order::Desc),
|
||||
Some("asc") => Ok(Order::Asc),
|
||||
Some(other) => Err(AppError::ValidationFailed {
|
||||
message: format!("order must be 'desc' or 'asc' (got {other:?})"),
|
||||
details: json!({ "order": "invalid" }),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
fn ensure_text_unsupported(text: Option<&str>) -> AppResult<()> {
|
||||
// Future OCR search will plug in here. Until then, return a
|
||||
// distinct code (`text_search_not_yet_supported`) so clients can
|
||||
// detect "feature pending" vs. a generic 4xx — the code is the
|
||||
// wire contract, not the message.
|
||||
if text.is_some_and(|s| !s.trim().is_empty()) {
|
||||
return Err(AppError::NotImplemented {
|
||||
code: "text_search_not_yet_supported",
|
||||
message: "text search is reserved for the planned OCR input but not yet supported",
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn list_chapters_for_tag(
|
||||
State(state): State<AppState>,
|
||||
CurrentUser(user): CurrentUser,
|
||||
Query(params): Query<AggregateParams>,
|
||||
) -> AppResult<Json<PagedResponse<TaggedChapterAggregate>>> {
|
||||
ensure_text_unsupported(params.text.as_deref())?;
|
||||
let tag = normalize_tag(¶ms.tag)?;
|
||||
let order = parse_order(params.order.as_deref())?;
|
||||
let limit = params.limit.clamp(1, 200);
|
||||
let offset = params.offset.max(0);
|
||||
let (items, total) = repo::page_tag::aggregate_chapters_for_tag(
|
||||
&state.db, user.id, &tag, order, limit, offset,
|
||||
)
|
||||
.await?;
|
||||
Ok(Json(PagedResponse::with_total(items, limit, offset, total)))
|
||||
}
|
||||
|
||||
async fn list_mangas_for_tag(
|
||||
State(state): State<AppState>,
|
||||
CurrentUser(user): CurrentUser,
|
||||
Query(params): Query<AggregateParams>,
|
||||
) -> AppResult<Json<PagedResponse<TaggedMangaAggregate>>> {
|
||||
ensure_text_unsupported(params.text.as_deref())?;
|
||||
let tag = normalize_tag(¶ms.tag)?;
|
||||
let order = parse_order(params.order.as_deref())?;
|
||||
let limit = params.limit.clamp(1, 200);
|
||||
let offset = params.offset.max(0);
|
||||
let (items, total) = repo::page_tag::aggregate_mangas_for_tag(
|
||||
&state.db, user.id, &tag, order, limit, offset,
|
||||
)
|
||||
.await?;
|
||||
Ok(Json(PagedResponse::with_total(items, limit, offset, total)))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::normalize_tag;
|
||||
|
||||
#[test]
|
||||
fn lowercases_and_collapses_whitespace() {
|
||||
assert_eq!(normalize_tag("Funny").unwrap(), "funny");
|
||||
assert_eq!(normalize_tag(" Foo Bar ").unwrap(), "foo bar");
|
||||
assert_eq!(normalize_tag("character:Askeladd").unwrap(), "character:askeladd");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_blank_input() {
|
||||
assert!(normalize_tag("").is_err());
|
||||
assert!(normalize_tag(" ").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_control_chars() {
|
||||
assert!(normalize_tag("bad\ntag").is_err());
|
||||
assert!(normalize_tag("a\tb").is_err());
|
||||
// NEL (U+0085, Cc) — rejected via `char::is_control()`. Locked
|
||||
// here so a future stdlib drift surfaces as a test fail rather
|
||||
// than a silent acceptance.
|
||||
assert!(normalize_tag("a\u{0085}b").is_err());
|
||||
// NULL byte.
|
||||
assert!(normalize_tag("a\u{0000}b").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_edge_colon() {
|
||||
assert!(normalize_tag(":foo").is_err());
|
||||
assert!(normalize_tag("foo:").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_invisible_format_chars() {
|
||||
// ZWJ — visually indistinguishable from no-zwj, would split
|
||||
// "funny" and "funny\u{200d}" into distinct rows.
|
||||
assert!(normalize_tag("funny\u{200d}").is_err());
|
||||
// RLO — bidi override could let a tag display backwards.
|
||||
assert!(normalize_tag("a\u{202e}b").is_err());
|
||||
// ZWNBSP / BOM at the boundary survives `trim()`.
|
||||
assert!(normalize_tag("a\u{feff}b").is_err());
|
||||
// ZWSP, RLM also rejected.
|
||||
assert!(normalize_tag("a\u{200b}b").is_err());
|
||||
assert!(normalize_tag("a\u{200f}b").is_err());
|
||||
// Plane-14 LANGUAGE TAG + TAG-character block. These tunnel
|
||||
// ASCII invisibly via the supplementary tag mechanism.
|
||||
assert!(normalize_tag("a\u{E0001}b").is_err());
|
||||
assert!(normalize_tag("a\u{E0020}b").is_err());
|
||||
assert!(normalize_tag("a\u{E007F}b").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_path_breaking_and_like_wildcards() {
|
||||
// `/` would survive %-encoding round-trip but axum decodes
|
||||
// %2F back to `/` after path routing, so the DELETE path
|
||||
// would never match. Same for `?`, `#`, `\`.
|
||||
assert!(normalize_tag("a/b").is_err());
|
||||
assert!(normalize_tag("a\\b").is_err());
|
||||
assert!(normalize_tag("a?b").is_err());
|
||||
assert!(normalize_tag("a#b").is_err());
|
||||
// `%` and `_` are SQL LIKE wildcards in the prefix filter.
|
||||
assert!(normalize_tag("a%b").is_err());
|
||||
assert!(normalize_tag("a_b").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_too_long() {
|
||||
let long = "a".repeat(65);
|
||||
assert!(normalize_tag(&long).is_err());
|
||||
let ok_len = "a".repeat(64);
|
||||
assert!(normalize_tag(&ok_len).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_pathologically_long_input_before_allocating() {
|
||||
// 10 MiB of `a` — should fail-fast on the byte cap, not
|
||||
// allocate twice through the lowercase/whitespace passes.
|
||||
let huge = "a".repeat(10 * 1024 * 1024);
|
||||
assert!(normalize_tag(&huge).is_err());
|
||||
// Just under cap still goes through the rest of the pipeline
|
||||
// and fails on the char count.
|
||||
let near_cap = "a".repeat(super::MAX_TAG_BYTES);
|
||||
assert!(normalize_tag(&near_cap).is_err());
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::AtomicBool;
|
||||
use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
|
||||
use std::sync::RwLock as StdRwLock;
|
||||
|
||||
use anyhow::Context;
|
||||
use async_trait::async_trait;
|
||||
@@ -17,7 +18,7 @@ use tower_http::trace::TraceLayer;
|
||||
use crate::auth::extractor::CurrentUser;
|
||||
use crate::auth::rate_limit::AuthRateLimiter;
|
||||
use crate::error::AppError;
|
||||
use crate::config::{AuthConfig, Config, CrawlerConfig, UploadConfig};
|
||||
use crate::config::{AnalysisConfig, AuthConfig, Config, CrawlerConfig, UploadConfig};
|
||||
use crate::crawler::browser_manager::{self, BrowserManager};
|
||||
use crate::crawler::content::{self, SyncOutcome};
|
||||
use crate::crawler::daemon::{self, ChapterDispatcher, DaemonConfig, MetadataPass};
|
||||
@@ -40,27 +41,235 @@ pub struct AppState {
|
||||
/// One instance per AppState so tests stay isolated across the
|
||||
/// same process.
|
||||
pub auth_limiter: Arc<AuthRateLimiter>,
|
||||
/// Admin-triggered force resync. `None` when the crawler daemon
|
||||
/// is disabled (`CRAWLER_DAEMON=false`); admin handlers gate on
|
||||
/// `.is_some()` and return 503 otherwise. Set by [`build`] from the
|
||||
/// same wiring that builds the daemon's chapter dispatcher, so a
|
||||
/// force resync uses the daemon's BrowserManager + rate limiters.
|
||||
pub resync: Option<Arc<dyn ResyncService>>,
|
||||
/// Runtime-swappable controls (crawler resync/control handles + the
|
||||
/// analysis enable gate). Shared with the [`Supervisors`] so a config
|
||||
/// reload that respawns a daemon updates what handlers see, live —
|
||||
/// without a restart. In tests this starts empty (no daemons).
|
||||
pub runtime: Arc<RuntimeControls>,
|
||||
/// Applies a persisted settings change to the running daemons
|
||||
/// (graceful stop + respawn). `Some` in production ([`build`]); `None`
|
||||
/// in the test harness, where settings still persist but no daemon is
|
||||
/// spawned. See [`DaemonReloader`].
|
||||
pub reloader: Option<Arc<dyn DaemonReloader>>,
|
||||
/// Env-derived crawler config — the **base** the stored settings DTO is
|
||||
/// overlaid onto (carries env-only fields: browser, proxy, TOR, session).
|
||||
pub crawler_base: CrawlerConfig,
|
||||
/// Env-derived analysis config base (carries the env-only vision API key).
|
||||
pub analysis_base: AnalysisConfig,
|
||||
/// Browser origins permitted to issue mutating requests to
|
||||
/// `/api/v1/admin/*`. See [`crate::config::Config::admin_allowed_origins`]
|
||||
/// for the policy. Cloned per-request into the CSRF middleware; the
|
||||
/// `Arc` keeps the clone cheap. Empty list → check is skipped
|
||||
/// (operator opt-out documented in `.env.example`).
|
||||
pub admin_allowed_origins: Arc<Vec<String>>,
|
||||
/// Live analysis-event broadcaster. Always present (the worker and the
|
||||
/// admin enqueue path publish here; the SSE endpoint subscribes). When
|
||||
/// the worker is disabled the channel simply stays quiet.
|
||||
pub analysis_events: Arc<crate::analysis::events::AnalysisEvents>,
|
||||
}
|
||||
|
||||
impl AppState {
|
||||
/// Current crawler control handle, or `None` when the daemon is stopped.
|
||||
pub fn crawler(&self) -> Option<Arc<CrawlerControl>> {
|
||||
self.runtime.crawler_control()
|
||||
}
|
||||
|
||||
/// Current force-resync service, or `None` when the daemon is stopped.
|
||||
pub fn resync(&self) -> Option<Arc<dyn ResyncService>> {
|
||||
self.runtime.resync()
|
||||
}
|
||||
|
||||
/// Whether AI page analysis is currently enabled (read live).
|
||||
pub fn analysis_enabled(&self) -> bool {
|
||||
self.runtime.analysis_enabled()
|
||||
}
|
||||
}
|
||||
|
||||
/// The crawler control handles, swapped atomically when the daemon is
|
||||
/// (re)spawned or stopped. Cloned cheaply on every read.
|
||||
#[derive(Clone, Default)]
|
||||
struct CrawlerControls {
|
||||
resync: Option<Arc<dyn ResyncService>>,
|
||||
control: Option<Arc<CrawlerControl>>,
|
||||
}
|
||||
|
||||
/// Shared, runtime-mutable surface read by handlers and mutated by the
|
||||
/// [`Supervisors`] on a config reload. The analysis gate is an `Arc<AtomicBool>`
|
||||
/// so the crawler's chapter dispatcher (which enqueues `analyze_page` jobs)
|
||||
/// sees toggles live without being respawned.
|
||||
pub struct RuntimeControls {
|
||||
crawler: StdRwLock<CrawlerControls>,
|
||||
analysis_enabled: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
impl RuntimeControls {
|
||||
/// Empty controls (no crawler daemon) with the given initial analysis gate.
|
||||
pub fn new(analysis_enabled: bool) -> Self {
|
||||
Self {
|
||||
crawler: StdRwLock::new(CrawlerControls::default()),
|
||||
analysis_enabled: Arc::new(AtomicBool::new(analysis_enabled)),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn analysis_enabled(&self) -> bool {
|
||||
self.analysis_enabled.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Flip the analysis gate. Used by the [`Supervisors`] on reload and by
|
||||
/// the test harness's stub reloader.
|
||||
pub fn set_analysis_enabled(&self, v: bool) {
|
||||
self.analysis_enabled.store(v, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// The shared gate handle, passed to the crawler chapter dispatcher.
|
||||
fn analysis_gate(&self) -> Arc<AtomicBool> {
|
||||
Arc::clone(&self.analysis_enabled)
|
||||
}
|
||||
|
||||
pub fn crawler_control(&self) -> Option<Arc<CrawlerControl>> {
|
||||
self.crawler.read().unwrap().control.clone()
|
||||
}
|
||||
|
||||
pub fn resync(&self) -> Option<Arc<dyn ResyncService>> {
|
||||
self.crawler.read().unwrap().resync.clone()
|
||||
}
|
||||
|
||||
/// Test helper: install a resync service without a running daemon.
|
||||
pub fn set_resync(&self, resync: Option<Arc<dyn ResyncService>>) {
|
||||
self.crawler.write().unwrap().resync = resync;
|
||||
}
|
||||
|
||||
fn set_crawler(
|
||||
&self,
|
||||
resync: Option<Arc<dyn ResyncService>>,
|
||||
control: Option<Arc<CrawlerControl>>,
|
||||
) {
|
||||
let mut g = self.crawler.write().unwrap();
|
||||
g.resync = resync;
|
||||
g.control = control;
|
||||
}
|
||||
}
|
||||
|
||||
/// Applies a validated settings change to the running daemons by gracefully
|
||||
/// stopping and respawning them with the new effective config (and updating
|
||||
/// the shared [`RuntimeControls`]). Implemented by [`Supervisors`] in
|
||||
/// production; the settings handlers call it after persisting.
|
||||
#[async_trait]
|
||||
pub trait DaemonReloader: Send + Sync {
|
||||
async fn reload_crawler(&self, cfg: CrawlerConfig) -> anyhow::Result<()>;
|
||||
async fn reload_analysis(&self, cfg: AnalysisConfig) -> anyhow::Result<()>;
|
||||
}
|
||||
|
||||
/// Shared handle the admin crawler endpoints use to observe and control
|
||||
/// the running daemon. Bundled so the handlers take one optional field on
|
||||
/// `AppState` rather than many.
|
||||
pub struct CrawlerControl {
|
||||
pub browser_manager: Arc<BrowserManager>,
|
||||
pub session: Arc<crate::crawler::session_control::SessionController>,
|
||||
pub status: crate::crawler::status::StatusHandle,
|
||||
/// Used by the "run metadata pass now" endpoint; `None` when no
|
||||
/// `CRAWLER_START_URL` is configured (cron disabled).
|
||||
pub metadata_pass: Option<Arc<dyn MetadataPass>>,
|
||||
/// Drain budget for a manually-triggered coordinated browser restart.
|
||||
pub drain_deadline: std::time::Duration,
|
||||
/// Held for the duration of a `/admin/crawler/run` pass so a second
|
||||
/// click returns 409 instead of fanning N overlapping metadata passes
|
||||
/// onto the single browser lease. The daemon's daily cron does NOT
|
||||
/// take this lock — cron and operator-triggered are different
|
||||
/// trust modes (cron is single-fire by definition; the lock only
|
||||
/// dedups operator clicks). `Arc` so the spawned task can hold an
|
||||
/// owned guard past the request boundary.
|
||||
pub manual_pass_lock: Arc<tokio::sync::Mutex<()>>,
|
||||
}
|
||||
|
||||
/// Bundle returned by [`build`]. The router is what `axum::serve` consumes;
|
||||
/// the daemon (when enabled) outlives the HTTP server and is awaited via
|
||||
/// [`AppHandle::shutdown`] after the listener has finished gracefully.
|
||||
/// the [`Supervisors`] own the background daemons (when enabled), outlive the
|
||||
/// HTTP server, and are awaited via [`AppHandle::shutdown`] after the listener
|
||||
/// has finished gracefully.
|
||||
pub struct AppHandle {
|
||||
pub router: Router,
|
||||
pub daemon: Option<daemon::DaemonHandle>,
|
||||
pub supervisors: Arc<Supervisors>,
|
||||
}
|
||||
|
||||
impl AppHandle {
|
||||
pub async fn shutdown(self) {
|
||||
if let Some(d) = self.daemon {
|
||||
d.shutdown().await;
|
||||
self.supervisors.shutdown().await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Owns the crawler + analysis daemon handles and respawns them on a config
|
||||
/// reload. Holds the build-time inputs (db, storage, env-base configs, the
|
||||
/// shared event bus and [`RuntimeControls`]) needed to spawn from scratch.
|
||||
pub struct Supervisors {
|
||||
db: PgPool,
|
||||
storage: Arc<dyn Storage>,
|
||||
runtime: Arc<RuntimeControls>,
|
||||
analysis_events: Arc<crate::analysis::events::AnalysisEvents>,
|
||||
/// Serializes reloads + owns the live crawler daemon handle.
|
||||
crawler_handle: tokio::sync::Mutex<Option<daemon::DaemonHandle>>,
|
||||
/// Serializes reloads + owns the live analysis daemon handle.
|
||||
analysis_handle: tokio::sync::Mutex<Option<crate::analysis::daemon::AnalysisDaemonHandle>>,
|
||||
}
|
||||
|
||||
impl Supervisors {
|
||||
pub async fn shutdown(&self) {
|
||||
if let Some(h) = self.crawler_handle.lock().await.take() {
|
||||
h.shutdown().await;
|
||||
}
|
||||
if let Some(h) = self.analysis_handle.lock().await.take() {
|
||||
h.shutdown().await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl DaemonReloader for Supervisors {
|
||||
async fn reload_crawler(&self, cfg: CrawlerConfig) -> anyhow::Result<()> {
|
||||
// Serialize reloads; do the (possibly slow) graceful shutdown under
|
||||
// the handle lock but outside the read-path RwLock so status reads
|
||||
// never block on a draining browser.
|
||||
let mut guard = self.crawler_handle.lock().await;
|
||||
if let Some(h) = guard.take() {
|
||||
h.shutdown().await;
|
||||
}
|
||||
self.runtime.set_crawler(None, None);
|
||||
if cfg.daemon_enabled {
|
||||
let spawned = spawn_crawler_daemon(
|
||||
self.db.clone(),
|
||||
Arc::clone(&self.storage),
|
||||
&cfg,
|
||||
self.runtime.analysis_gate(),
|
||||
)
|
||||
.await?;
|
||||
self.runtime
|
||||
.set_crawler(Some(spawned.resync), Some(spawned.crawler));
|
||||
*guard = Some(spawned.handle);
|
||||
tracing::info!("crawler daemon (re)started from settings");
|
||||
} else {
|
||||
tracing::info!("crawler daemon stopped (disabled in settings)");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn reload_analysis(&self, cfg: AnalysisConfig) -> anyhow::Result<()> {
|
||||
let mut guard = self.analysis_handle.lock().await;
|
||||
if let Some(h) = guard.take() {
|
||||
h.shutdown().await;
|
||||
}
|
||||
self.runtime.set_analysis_enabled(cfg.enabled);
|
||||
if cfg.enabled {
|
||||
let handle = spawn_analysis_daemon(
|
||||
self.db.clone(),
|
||||
Arc::clone(&self.storage),
|
||||
&cfg,
|
||||
Arc::clone(&self.analysis_events),
|
||||
)?;
|
||||
*guard = Some(handle);
|
||||
tracing::info!(model = %cfg.model, "analysis daemon (re)started from settings");
|
||||
} else {
|
||||
tracing::info!("analysis daemon stopped (disabled in settings)");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -80,13 +289,48 @@ pub async fn build(config: Config) -> anyhow::Result<AppHandle> {
|
||||
|
||||
let storage: Arc<dyn Storage> = Arc::new(LocalStorage::new(config.storage_dir.clone()));
|
||||
|
||||
let (daemon, resync) = if config.crawler.daemon_enabled {
|
||||
let spawned = spawn_crawler_daemon(db.clone(), Arc::clone(&storage), &config.crawler).await?;
|
||||
(Some(spawned.handle), Some(spawned.resync))
|
||||
// Seed the settings rows from env on first boot, then load the effective
|
||||
// config (DB overlaid on the env base). The DB is the source of truth
|
||||
// after the first boot; env only fills a missing row.
|
||||
let crawler_cfg = load_effective_crawler(&db, &config.crawler).await?;
|
||||
let analysis_cfg = load_effective_analysis(&db, &config.analysis).await?;
|
||||
|
||||
// Live-event bus shared by the worker (progress) and the admin enqueue
|
||||
// path; the SSE endpoint subscribes. Created unconditionally so the
|
||||
// stream exists even with the worker disabled.
|
||||
let analysis_events = Arc::new(crate::analysis::events::AnalysisEvents::new());
|
||||
|
||||
let runtime = Arc::new(RuntimeControls::new(analysis_cfg.enabled));
|
||||
let supervisors = Arc::new(Supervisors {
|
||||
db: db.clone(),
|
||||
storage: Arc::clone(&storage),
|
||||
runtime: Arc::clone(&runtime),
|
||||
analysis_events: Arc::clone(&analysis_events),
|
||||
crawler_handle: tokio::sync::Mutex::new(None),
|
||||
analysis_handle: tokio::sync::Mutex::new(None),
|
||||
});
|
||||
|
||||
// Initial spawn goes through the same reload path so there's one code
|
||||
// path for "bring the daemon up with this config". A spawn failure is
|
||||
// logged but does NOT abort startup: the persisted settings are the
|
||||
// source of truth and a bad value (e.g. a config that won't launch the
|
||||
// browser) must not wedge the whole server into a boot loop — the
|
||||
// server comes up with that daemon stopped so an admin can fix it via
|
||||
// the settings UI.
|
||||
if crawler_cfg.daemon_enabled {
|
||||
if let Err(e) = supervisors.reload_crawler(crawler_cfg).await {
|
||||
tracing::error!(?e, "crawler daemon failed to start; continuing with it stopped");
|
||||
}
|
||||
} else {
|
||||
tracing::info!("crawler daemon disabled (CRAWLER_DAEMON=false)");
|
||||
(None, None)
|
||||
};
|
||||
tracing::info!("crawler daemon disabled");
|
||||
}
|
||||
if analysis_cfg.enabled {
|
||||
if let Err(e) = supervisors.reload_analysis(analysis_cfg).await {
|
||||
tracing::error!(?e, "analysis worker failed to start; continuing with it stopped");
|
||||
}
|
||||
} else {
|
||||
tracing::info!("analysis worker disabled");
|
||||
}
|
||||
|
||||
let auth_limiter = Arc::new(AuthRateLimiter::new(config.auth.rate_limit));
|
||||
let state = AppState {
|
||||
@@ -95,10 +339,99 @@ pub async fn build(config: Config) -> anyhow::Result<AppHandle> {
|
||||
auth: config.auth.clone(),
|
||||
upload: config.upload.clone(),
|
||||
auth_limiter,
|
||||
resync,
|
||||
runtime,
|
||||
reloader: Some(Arc::clone(&supervisors) as Arc<dyn DaemonReloader>),
|
||||
crawler_base: config.crawler.clone(),
|
||||
analysis_base: config.analysis.clone(),
|
||||
admin_allowed_origins: Arc::new(config.admin_allowed_origins.clone()),
|
||||
analysis_events,
|
||||
};
|
||||
let router = router(state).layer(cors_layer(&config.cors_allowed_origins));
|
||||
Ok(AppHandle { router, daemon })
|
||||
Ok(AppHandle { router, supervisors })
|
||||
}
|
||||
|
||||
/// Seed the crawler settings row from env when absent, then return the
|
||||
/// effective [`CrawlerConfig`] (DB DTO overlaid on the env base). A stored
|
||||
/// DTO that fails to convert (corruption / drift) falls back to the env base
|
||||
/// rather than blocking startup.
|
||||
async fn load_effective_crawler(
|
||||
db: &PgPool,
|
||||
base: &CrawlerConfig,
|
||||
) -> anyhow::Result<CrawlerConfig> {
|
||||
use crate::settings::{CrawlerSettings, KEY_CRAWLER};
|
||||
let dto = match repo::app_settings::get(db, KEY_CRAWLER).await? {
|
||||
Some(v) => serde_json::from_value::<CrawlerSettings>(v)
|
||||
.unwrap_or_else(|_| CrawlerSettings::from_config(base)),
|
||||
None => {
|
||||
let dto = CrawlerSettings::from_config(base);
|
||||
repo::app_settings::seed_if_absent(db, KEY_CRAWLER, &serde_json::to_value(&dto)?)
|
||||
.await?;
|
||||
tracing::info!("seeded crawler settings from env");
|
||||
dto
|
||||
}
|
||||
};
|
||||
Ok(dto.to_config(base).unwrap_or_else(|e| {
|
||||
tracing::warn!(?e, "stored crawler settings invalid; using env base");
|
||||
base.clone()
|
||||
}))
|
||||
}
|
||||
|
||||
/// Analysis counterpart of [`load_effective_crawler`].
|
||||
async fn load_effective_analysis(
|
||||
db: &PgPool,
|
||||
base: &AnalysisConfig,
|
||||
) -> anyhow::Result<AnalysisConfig> {
|
||||
use crate::settings::{AnalysisSettings, KEY_ANALYSIS};
|
||||
let dto = match repo::app_settings::get(db, KEY_ANALYSIS).await? {
|
||||
Some(v) => serde_json::from_value::<AnalysisSettings>(v)
|
||||
.unwrap_or_else(|_| AnalysisSettings::from_config(base)),
|
||||
None => {
|
||||
let dto = AnalysisSettings::from_config(base);
|
||||
repo::app_settings::seed_if_absent(db, KEY_ANALYSIS, &serde_json::to_value(&dto)?)
|
||||
.await?;
|
||||
tracing::info!("seeded analysis settings from env");
|
||||
dto
|
||||
}
|
||||
};
|
||||
Ok(dto.to_config(base).unwrap_or_else(|e| {
|
||||
tracing::warn!(?e, "stored analysis settings invalid; using env base");
|
||||
base.clone()
|
||||
}))
|
||||
}
|
||||
|
||||
/// Spawn the AI content-analysis worker daemon with the given config. Returns
|
||||
/// its handle. Independent of the crawler daemon (works for uploads with the
|
||||
/// crawler off). Uses a plain reqwest client — no cookie jar / proxy.
|
||||
fn spawn_analysis_daemon(
|
||||
db: PgPool,
|
||||
storage: Arc<dyn Storage>,
|
||||
cfg: &AnalysisConfig,
|
||||
events: Arc<crate::analysis::events::AnalysisEvents>,
|
||||
) -> anyhow::Result<crate::analysis::daemon::AnalysisDaemonHandle> {
|
||||
let http = reqwest::Client::builder()
|
||||
.timeout(cfg.request_timeout)
|
||||
.build()
|
||||
.context("build analysis http client")?;
|
||||
let vision = crate::analysis::vision::VisionClient::new(http, cfg);
|
||||
let dispatcher = Arc::new(crate::analysis::daemon::RealAnalyzeDispatcher {
|
||||
db: db.clone(),
|
||||
storage,
|
||||
vision,
|
||||
model: cfg.model.clone(),
|
||||
max_image_bytes: cfg.max_image_bytes,
|
||||
});
|
||||
let handle = crate::analysis::daemon::spawn(
|
||||
db,
|
||||
CancellationToken::new(),
|
||||
crate::analysis::daemon::AnalysisDaemonConfig {
|
||||
dispatcher,
|
||||
workers: cfg.workers,
|
||||
job_timeout: cfg.job_timeout,
|
||||
events,
|
||||
},
|
||||
);
|
||||
tracing::info!(workers = cfg.workers, model = %cfg.model, "analysis worker daemon started");
|
||||
Ok(handle)
|
||||
}
|
||||
|
||||
/// Bundle returned by [`spawn_crawler_daemon`]. The handle owns the
|
||||
@@ -108,18 +441,26 @@ pub async fn build(config: Config) -> anyhow::Result<AppHandle> {
|
||||
struct SpawnedDaemon {
|
||||
handle: daemon::DaemonHandle,
|
||||
resync: Arc<dyn ResyncService>,
|
||||
crawler: Arc<CrawlerControl>,
|
||||
}
|
||||
|
||||
async fn spawn_crawler_daemon(
|
||||
db: PgPool,
|
||||
storage: Arc<dyn Storage>,
|
||||
cfg: &CrawlerConfig,
|
||||
analysis_enabled: Arc<AtomicBool>,
|
||||
) -> anyhow::Result<SpawnedDaemon> {
|
||||
// Reqwest client with cookie jar pre-seeded so CDN image fetches
|
||||
// include PHPSESSID. Same shape as bin/crawler.rs main().
|
||||
// Reqwest client with a shared cookie jar so CDN image fetches include
|
||||
// PHPSESSID. The same `Arc<Jar>` is held by the SessionController, so a
|
||||
// runtime session refresh rewrites it in place. Initial value: a
|
||||
// persisted runtime session (survives restart) takes precedence over
|
||||
// CRAWLER_PHPSESSID env.
|
||||
let cookie_jar = Arc::new(reqwest::cookie::Jar::default());
|
||||
let initial_sid = crate::crawler::session_control::SessionController::load_persisted(&db)
|
||||
.await
|
||||
.or_else(|| cfg.phpsessid.clone());
|
||||
if let (Some(sid), Some(domain), Some(start_url)) =
|
||||
(&cfg.phpsessid, &cfg.cookie_domain, &cfg.start_url)
|
||||
(&initial_sid, &cfg.cookie_domain, &cfg.start_url)
|
||||
{
|
||||
let cookie_str = format!("PHPSESSID={sid}; Domain={domain}; Path=/");
|
||||
let seed_url = reqwest::Url::parse(start_url)
|
||||
@@ -129,7 +470,7 @@ async fn spawn_crawler_daemon(
|
||||
let mut http_builder = reqwest::Client::builder()
|
||||
.timeout(std::time::Duration::from_secs(30))
|
||||
.no_proxy()
|
||||
.cookie_provider(cookie_jar);
|
||||
.cookie_provider(Arc::clone(&cookie_jar));
|
||||
if let Some(ua) = &cfg.user_agent {
|
||||
http_builder = http_builder.user_agent(ua);
|
||||
}
|
||||
@@ -157,6 +498,23 @@ async fn spawn_crawler_daemon(
|
||||
}
|
||||
let tor_recircuit_max = cfg.tor_recircuit_max_attempts;
|
||||
|
||||
// Session controller + sticky session-expired flag. Created before the
|
||||
// browser so the on_launch hook can read the *current* session value
|
||||
// (rather than a value captured at startup), and so a runtime refresh
|
||||
// updates the cookie everywhere.
|
||||
let session_expired = Arc::new(AtomicBool::new(false));
|
||||
let session_controller = crate::crawler::session_control::SessionController::new(
|
||||
initial_sid,
|
||||
Arc::clone(&cookie_jar),
|
||||
cfg.cookie_domain.clone(),
|
||||
cfg.start_url.clone(),
|
||||
db.clone(),
|
||||
Arc::clone(&session_expired),
|
||||
);
|
||||
|
||||
// Live status surface, sized to the worker count.
|
||||
let status = crate::crawler::status::StatusHandle::new(cfg.chapter_workers);
|
||||
|
||||
// Browser manager. on_launch re-injects PHPSESSID on every fresh
|
||||
// chromium spawn so an idle teardown followed by re-launch stays
|
||||
// authenticated without operator action.
|
||||
@@ -165,18 +523,25 @@ async fn spawn_crawler_daemon(
|
||||
let chromium_proxy = crate::crawler::url_utils::chromium_proxy_arg(proxy);
|
||||
launch_opts.extra_args.push(format!("--proxy-server={chromium_proxy}"));
|
||||
}
|
||||
let on_launch = match (&cfg.phpsessid, &cfg.cookie_domain, &cfg.start_url) {
|
||||
(Some(sid), Some(domain), Some(start_url)) => {
|
||||
let sid = sid.clone();
|
||||
let on_launch = match (&cfg.cookie_domain, &cfg.start_url) {
|
||||
(Some(domain), Some(start_url)) => {
|
||||
let domain = domain.clone();
|
||||
let start_url = start_url.clone();
|
||||
let tor_for_launch = tor.as_ref().map(Arc::clone);
|
||||
let sc = Arc::clone(&session_controller);
|
||||
let on_launch: browser_manager::OnLaunch = Arc::new(move |browser| {
|
||||
let sid = sid.clone();
|
||||
let domain = domain.clone();
|
||||
let start_url = start_url.clone();
|
||||
let tor_for_launch = tor_for_launch.as_ref().map(Arc::clone);
|
||||
let sc = Arc::clone(&sc);
|
||||
Box::pin(async move {
|
||||
// Read the *current* session each launch so a runtime
|
||||
// refresh is picked up on the next (re)launch. No session
|
||||
// configured → run unauthenticated (metadata needs no auth).
|
||||
let Some(sid) = sc.current().await else {
|
||||
tracing::info!("on_launch: no session set — skipping inject + probe");
|
||||
return Ok(());
|
||||
};
|
||||
session::inject_phpsessid(&browser, &sid, &domain)
|
||||
.await
|
||||
.context("on_launch: inject_phpsessid")?;
|
||||
@@ -197,8 +562,6 @@ async fn spawn_crawler_daemon(
|
||||
};
|
||||
let browser_manager = BrowserManager::new(launch_opts, cfg.idle_timeout, on_launch);
|
||||
|
||||
let session_expired = Arc::new(AtomicBool::new(false));
|
||||
|
||||
let metadata_pass: Option<Arc<dyn MetadataPass>> = cfg.start_url.as_ref().map(|url| {
|
||||
let m: Arc<dyn MetadataPass> = Arc::new(RealMetadataPass {
|
||||
browser_manager: Arc::clone(&browser_manager),
|
||||
@@ -210,6 +573,8 @@ async fn spawn_crawler_daemon(
|
||||
manga_limit: cfg.manga_limit,
|
||||
download_allowlist: cfg.download_allowlist.clone(),
|
||||
max_image_bytes: cfg.max_image_bytes,
|
||||
metadata_max_consecutive_failures: cfg.metadata_max_consecutive_failures,
|
||||
status: status.clone(),
|
||||
tor: tor.as_ref().map(Arc::clone),
|
||||
});
|
||||
m
|
||||
@@ -223,6 +588,11 @@ async fn spawn_crawler_daemon(
|
||||
rate: Arc::clone(&rate),
|
||||
download_allowlist: cfg.download_allowlist.clone(),
|
||||
max_image_bytes: cfg.max_image_bytes,
|
||||
analysis_enabled,
|
||||
transient_failures: Arc::new(AtomicU32::new(0)),
|
||||
restart_threshold: cfg.browser_restart_threshold,
|
||||
drain_deadline: cfg.job_timeout,
|
||||
status: status.clone(),
|
||||
tor: tor.as_ref().map(Arc::clone),
|
||||
});
|
||||
|
||||
@@ -260,20 +630,32 @@ async fn spawn_crawler_daemon(
|
||||
db,
|
||||
cancel,
|
||||
DaemonConfig {
|
||||
metadata_pass,
|
||||
metadata_pass: metadata_pass.clone(),
|
||||
dispatcher,
|
||||
chapter_workers: cfg.chapter_workers,
|
||||
daily_at: cfg.daily_at,
|
||||
tz: cfg.tz,
|
||||
retention_days: cfg.retention_days,
|
||||
session_expired,
|
||||
status: status.clone(),
|
||||
job_timeout: cfg.job_timeout,
|
||||
extra_tasks: vec![reaper_task, shutdown_task],
|
||||
},
|
||||
);
|
||||
|
||||
let crawler = Arc::new(CrawlerControl {
|
||||
browser_manager: Arc::clone(&browser_manager),
|
||||
session: session_controller,
|
||||
status,
|
||||
metadata_pass,
|
||||
drain_deadline: cfg.job_timeout,
|
||||
manual_pass_lock: Arc::new(tokio::sync::Mutex::new(())),
|
||||
});
|
||||
|
||||
Ok(SpawnedDaemon {
|
||||
handle: daemon_handle,
|
||||
resync,
|
||||
crawler,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -292,6 +674,8 @@ struct RealMetadataPass {
|
||||
manga_limit: usize,
|
||||
download_allowlist: DownloadAllowlist,
|
||||
max_image_bytes: usize,
|
||||
metadata_max_consecutive_failures: u32,
|
||||
status: crate::crawler::status::StatusHandle,
|
||||
tor: Option<Arc<crate::crawler::tor::TorController>>,
|
||||
}
|
||||
|
||||
@@ -309,6 +693,8 @@ impl MetadataPass for RealMetadataPass {
|
||||
false,
|
||||
&self.download_allowlist,
|
||||
self.max_image_bytes,
|
||||
self.metadata_max_consecutive_failures,
|
||||
Some(&self.status),
|
||||
self.tor.as_deref(),
|
||||
)
|
||||
.await;
|
||||
@@ -321,7 +707,8 @@ impl MetadataPass for RealMetadataPass {
|
||||
// errored — the early-stop walk can complete its work and bail
|
||||
// late, and a transient browser failure shouldn't cancel the
|
||||
// residual cover backlog. The backfill has its own per-call cap
|
||||
// so a runaway error stream can't monopolise the tick.
|
||||
// so a runaway error stream can't monopolise the tick. It sets the
|
||||
// CoverBackfill{index,total} phase + current_cover per entry.
|
||||
match pipeline::backfill_missing_covers(
|
||||
&self.browser_manager,
|
||||
&self.db,
|
||||
@@ -331,6 +718,7 @@ impl MetadataPass for RealMetadataPass {
|
||||
pipeline::COVER_BACKFILL_DEFAULT_MAX,
|
||||
&self.download_allowlist,
|
||||
self.max_image_bytes,
|
||||
Some(&self.status),
|
||||
self.tor.as_deref(),
|
||||
)
|
||||
.await
|
||||
@@ -359,6 +747,20 @@ struct RealChapterDispatcher {
|
||||
rate: Arc<HostRateLimiters>,
|
||||
download_allowlist: DownloadAllowlist,
|
||||
max_image_bytes: usize,
|
||||
/// Enqueue `analyze_page` jobs for freshly-crawled pages. Shared gate
|
||||
/// (read live) so toggling analysis at runtime takes effect without a
|
||||
/// crawler respawn. Mirrors the analysis enable setting.
|
||||
analysis_enabled: Arc<AtomicBool>,
|
||||
/// Consecutive transient chapter failures; resets on any success.
|
||||
/// Drives the automatic coordinated browser restart.
|
||||
transient_failures: Arc<std::sync::atomic::AtomicU32>,
|
||||
/// Consecutive-failure count that triggers an auto restart.
|
||||
restart_threshold: u32,
|
||||
/// How long a coordinated restart waits for in-flight leases to drain.
|
||||
drain_deadline: std::time::Duration,
|
||||
/// Live status surface — the dispatcher registers each chapter it
|
||||
/// crawls (with a realtime page count) here.
|
||||
status: crate::crawler::status::StatusHandle,
|
||||
tor: Option<Arc<crate::crawler::tor::TorController>>,
|
||||
}
|
||||
|
||||
@@ -374,10 +776,21 @@ impl ChapterDispatcher for RealChapterDispatcher {
|
||||
let row = repo::chapter::dispatch_target(&self.db, chapter_id)
|
||||
.await
|
||||
.context("look up chapter for dispatch")?;
|
||||
let Some((manga_id, source_url)) = row else {
|
||||
let Some((manga_id, source_url, manga_title, chapter_number)) = row else {
|
||||
// Chapter (or its source row) is gone — ack done.
|
||||
return Ok(SyncOutcome::Skipped);
|
||||
};
|
||||
// Register the chapter as crawling now (live status). The
|
||||
// guard removes it on every exit path — success, panic, or
|
||||
// the worker's outer-timeout drop.
|
||||
let _active = self.status.begin_chapter(crate::crawler::status::ActiveChapter {
|
||||
manga_id,
|
||||
manga_title,
|
||||
chapter_id,
|
||||
chapter_number,
|
||||
pages_done: 0,
|
||||
pages_total: None,
|
||||
});
|
||||
let lease = self.browser_manager.acquire().await?;
|
||||
let result = content::sync_chapter_content(
|
||||
&lease,
|
||||
@@ -392,14 +805,38 @@ impl ChapterDispatcher for RealChapterDispatcher {
|
||||
&self.download_allowlist,
|
||||
self.max_image_bytes,
|
||||
self.tor.as_deref(),
|
||||
Some(&self.status),
|
||||
self.analysis_enabled.load(Ordering::Relaxed),
|
||||
)
|
||||
.await;
|
||||
drop(lease);
|
||||
match result {
|
||||
Ok(outcome) => Ok(outcome),
|
||||
Ok(outcome) => {
|
||||
// Any successful dispatch (including a clean Skipped)
|
||||
// means the browser is healthy — reset the streak.
|
||||
self.transient_failures.store(0, Ordering::Release);
|
||||
Ok(outcome)
|
||||
}
|
||||
Err(e) => {
|
||||
let streak = self.transient_failures.fetch_add(1, Ordering::AcqRel) + 1;
|
||||
if crate::crawler::nav::anyhow_looks_browser_dead(&e) {
|
||||
// Hard browser-dead: lazy invalidate (next acquire
|
||||
// relaunches). Reset the streak — we're recovering.
|
||||
self.browser_manager.invalidate().await;
|
||||
self.transient_failures.store(0, Ordering::Release);
|
||||
} else if self.restart_threshold > 0 && streak >= self.restart_threshold {
|
||||
// Persistent transients that TOR recircuit couldn't
|
||||
// fix — proactively restart Chromium.
|
||||
tracing::warn!(
|
||||
streak,
|
||||
threshold = self.restart_threshold,
|
||||
"auto browser restart: consecutive transient chapter failures"
|
||||
);
|
||||
let _ = self
|
||||
.browser_manager
|
||||
.coordinated_restart(self.drain_deadline)
|
||||
.await;
|
||||
self.transient_failures.store(0, Ordering::Release);
|
||||
}
|
||||
Err(e)
|
||||
}
|
||||
@@ -419,6 +856,11 @@ pub fn router(state: AppState) -> Router {
|
||||
let max_request_bytes = state.upload.max_request_bytes;
|
||||
Router::new()
|
||||
.nest("/api/v1", crate::api::routes())
|
||||
.layer(middleware::from_fn(admin_no_store_guard))
|
||||
.layer(middleware::from_fn_with_state(
|
||||
state.clone(),
|
||||
admin_csrf_guard,
|
||||
))
|
||||
.layer(middleware::from_fn_with_state(
|
||||
state.clone(),
|
||||
private_mode_guard,
|
||||
@@ -428,6 +870,113 @@ pub fn router(state: AppState) -> Router {
|
||||
.layer(TraceLayer::new_for_http())
|
||||
}
|
||||
|
||||
/// Path prefix the admin-only middlewares scope themselves to. The router
|
||||
/// already nests `/api/v1`, so callers see `/api/v1/admin/...`.
|
||||
const ADMIN_PATH_PREFIX: &str = "/api/v1/admin/";
|
||||
|
||||
/// CSRF defence for cookie-authenticated admin mutations. The session
|
||||
/// cookie is `SameSite=Lax`, which still permits top-level form-POSTs
|
||||
/// from a malicious page — this middleware rejects such requests by
|
||||
/// comparing the request's `Origin` (with `Referer` as fallback) against
|
||||
/// the configured allowlist. Safe methods (`GET`/`HEAD`/`OPTIONS`) are
|
||||
/// always allowed. Requests with neither `Origin` nor `Referer` are
|
||||
/// allowed (non-browser callers like curl can't be a CSRF vector). When
|
||||
/// the allowlist is empty the check is skipped entirely (operator
|
||||
/// opt-out — documented in `.env.example`).
|
||||
async fn admin_csrf_guard(
|
||||
State(state): State<AppState>,
|
||||
req: Request,
|
||||
next: Next,
|
||||
) -> Result<Response, AppError> {
|
||||
if !req.uri().path().starts_with(ADMIN_PATH_PREFIX) {
|
||||
return Ok(next.run(req).await);
|
||||
}
|
||||
if matches!(
|
||||
*req.method(),
|
||||
Method::GET | Method::HEAD | Method::OPTIONS
|
||||
) {
|
||||
return Ok(next.run(req).await);
|
||||
}
|
||||
if state.admin_allowed_origins.is_empty() {
|
||||
return Ok(next.run(req).await);
|
||||
}
|
||||
let headers = req.headers();
|
||||
let origin = headers.get("origin").and_then(|v| v.to_str().ok());
|
||||
let referer = headers.get("referer").and_then(|v| v.to_str().ok());
|
||||
// No Origin AND no Referer → server-to-server / curl / extension.
|
||||
// Browsers always send one or the other on a cross-site POST.
|
||||
let Some(candidate) = origin.or(referer) else {
|
||||
return Ok(next.run(req).await);
|
||||
};
|
||||
if origin_in_allowlist(candidate, &state.admin_allowed_origins) {
|
||||
return Ok(next.run(req).await);
|
||||
}
|
||||
tracing::warn!(
|
||||
candidate = %truncate_for_log(candidate, 64),
|
||||
path = %req.uri().path(),
|
||||
"admin CSRF: rejecting mutation with disallowed origin"
|
||||
);
|
||||
Err(AppError::Forbidden)
|
||||
}
|
||||
|
||||
/// Match `candidate` (an `Origin` value, or a `Referer` URL whose
|
||||
/// origin we'll extract) against `allowed`. `Origin` is `scheme://host[:port]`
|
||||
/// with no path; `Referer` is a full URL — compare by parsing both and
|
||||
/// matching scheme + host + port.
|
||||
fn origin_in_allowlist(candidate: &str, allowed: &[String]) -> bool {
|
||||
let cand_origin = parse_origin(candidate);
|
||||
let Some(cand) = cand_origin else { return false };
|
||||
allowed
|
||||
.iter()
|
||||
.filter_map(|a| parse_origin(a))
|
||||
.any(|a| a == cand)
|
||||
}
|
||||
|
||||
/// Extract the origin (`scheme://host[:port]`) from an `Origin` header
|
||||
/// value or a `Referer` URL. Returns `None` when the input doesn't parse
|
||||
/// as a URL with a host.
|
||||
fn parse_origin(raw: &str) -> Option<String> {
|
||||
let url = reqwest::Url::parse(raw.trim()).ok()?;
|
||||
let host = url.host_str()?;
|
||||
let scheme = url.scheme();
|
||||
let port_str = match (url.port(), scheme) {
|
||||
(Some(p), "http") if p == 80 => String::new(),
|
||||
(Some(p), "https") if p == 443 => String::new(),
|
||||
(Some(p), _) => format!(":{p}"),
|
||||
(None, _) => String::new(),
|
||||
};
|
||||
Some(format!("{scheme}://{host}{port_str}"))
|
||||
}
|
||||
|
||||
fn truncate_for_log(s: &str, max: usize) -> &str {
|
||||
let end = s
|
||||
.char_indices()
|
||||
.take(max)
|
||||
.last()
|
||||
.map(|(i, c)| i + c.len_utf8())
|
||||
.unwrap_or(0);
|
||||
&s[..end.min(s.len())]
|
||||
}
|
||||
|
||||
/// Forbids intermediaries (CDN, browser bfcache, reverse proxy with a
|
||||
/// permissive default) from caching admin responses. Defence-in-depth
|
||||
/// for cookie-authenticated reads — even though the responses already
|
||||
/// vary on cookie, a misconfigured cache layer in front of the
|
||||
/// SvelteKit container could leak a logged-in admin's view to another
|
||||
/// session. Headers added on response so the rest of the API is
|
||||
/// unaffected.
|
||||
async fn admin_no_store_guard(req: Request, next: Next) -> Response {
|
||||
let is_admin_path = req.uri().path().starts_with(ADMIN_PATH_PREFIX);
|
||||
let mut resp = next.run(req).await;
|
||||
if is_admin_path {
|
||||
resp.headers_mut().insert(
|
||||
axum::http::header::CACHE_CONTROL,
|
||||
HeaderValue::from_static("no-store"),
|
||||
);
|
||||
}
|
||||
resp
|
||||
}
|
||||
|
||||
/// Paths reachable anonymously even when `PRIVATE_MODE=true`. Login and
|
||||
/// logout are needed for the auth flow itself; `/health` is reserved
|
||||
/// for load-balancer probes; `/auth/config` lets the frontend decide
|
||||
|
||||
@@ -303,6 +303,11 @@ async fn run(
|
||||
skip_chapters,
|
||||
allowlist.as_ref(),
|
||||
max_image_bytes,
|
||||
// Circuit-breaker disabled for the operator-driven CLI: a manual
|
||||
// sweep should push through transient failures, not self-abort.
|
||||
0,
|
||||
// No live status surface for the one-shot CLI.
|
||||
None,
|
||||
tor.as_deref(),
|
||||
)
|
||||
.await?;
|
||||
@@ -412,6 +417,10 @@ async fn sync_bookmarked_chapter_content(
|
||||
allowlist.as_ref(),
|
||||
max_image_bytes,
|
||||
tor.as_deref(),
|
||||
// CLI one-shot — no live status surface.
|
||||
None,
|
||||
// Standalone CLI doesn't drive the analysis worker.
|
||||
false,
|
||||
)
|
||||
.await;
|
||||
drop(lease);
|
||||
|
||||
@@ -66,6 +66,203 @@ impl Default for UploadConfig {
|
||||
}
|
||||
}
|
||||
|
||||
/// How the worker asks the model to constrain its output. OpenAI-compatible
|
||||
/// servers disagree here: LM Studio accepts only `json_schema` or `text`
|
||||
/// (NOT `json_object`); vanilla OpenAI/vLLM accept `json_object` too. The
|
||||
/// default `JsonSchema` is the most portable and the most reliable — it
|
||||
/// also keeps "thinking" models (e.g. Gemma) from emitting an empty
|
||||
/// `content` with the answer buried in `reasoning_content`.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum ResponseFormat {
|
||||
JsonSchema,
|
||||
JsonObject,
|
||||
/// Send no `response_format` — rely on the prompt + the parser's
|
||||
/// fence/brace extraction. Needed for servers that reject the field.
|
||||
None,
|
||||
}
|
||||
|
||||
impl ResponseFormat {
|
||||
fn from_str(s: &str) -> ResponseFormat {
|
||||
match s.trim().to_lowercase().as_str() {
|
||||
"json_object" => ResponseFormat::JsonObject,
|
||||
"none" | "text" | "off" | "" => ResponseFormat::None,
|
||||
// Default (incl. "json_schema" and anything unrecognized).
|
||||
_ => ResponseFormat::JsonSchema,
|
||||
}
|
||||
}
|
||||
|
||||
/// Canonical wire string, the inverse of [`Self::from_str`] for the
|
||||
/// three modes the API exposes.
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
ResponseFormat::JsonSchema => "json_schema",
|
||||
ResponseFormat::JsonObject => "json_object",
|
||||
ResponseFormat::None => "none",
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse a wire value, rejecting unknown modes (stricter than
|
||||
/// [`Self::from_str`], which the env loader uses to stay lenient).
|
||||
pub fn parse_strict(s: &str) -> Option<ResponseFormat> {
|
||||
match s.trim().to_lowercase().as_str() {
|
||||
"json_schema" => Some(ResponseFormat::JsonSchema),
|
||||
"json_object" => Some(ResponseFormat::JsonObject),
|
||||
"none" => Some(ResponseFormat::None),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// AI content-analysis worker configuration: the enable gate, the local
|
||||
/// OpenAI-compatible vision endpoint, and the worker / request knobs.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct AnalysisConfig {
|
||||
/// Master switch (`ANALYSIS_ENABLED`). When `false`, no analysis jobs
|
||||
/// are enqueued and no worker runs. Defaults to `false`.
|
||||
pub enabled: bool,
|
||||
/// Number of concurrent analysis workers (`ANALYSIS_WORKERS`).
|
||||
pub workers: usize,
|
||||
/// OpenAI-compatible chat/completions URL (`ANALYSIS_VISION_URL`).
|
||||
pub endpoint: String,
|
||||
/// Model id to request (`ANALYSIS_MODEL`).
|
||||
pub model: String,
|
||||
/// Optional bearer token (`ANALYSIS_API_KEY`); local servers usually
|
||||
/// don't need one.
|
||||
pub api_key: Option<String>,
|
||||
/// Per-request HTTP timeout (`ANALYSIS_REQUEST_TIMEOUT_SECS`).
|
||||
pub request_timeout: Duration,
|
||||
/// Whole-job timeout in the worker (`ANALYSIS_JOB_TIMEOUT_SECS`).
|
||||
pub job_timeout: Duration,
|
||||
/// Output token cap sent as `max_tokens` (`ANALYSIS_MAX_TOKENS`).
|
||||
/// Must be generous enough to hold the full JSON for a text-dense page
|
||||
/// — too low truncates the response mid-object (`finish_reason:
|
||||
/// length`) and the parse fails. The page image + prompt are only a few
|
||||
/// hundred tokens, so a large output budget still fits an 8k window.
|
||||
pub max_tokens: u32,
|
||||
/// Per-slice / per-image pixel budget (`ANALYSIS_MAX_PIXELS`). The model
|
||||
/// resizes any input to roughly this many pixels anyway, so we slice/fit
|
||||
/// to it at native resolution rather than pre-squashing by a fixed edge.
|
||||
pub max_pixels: u32,
|
||||
/// Minimum slice height (px), the aspect guard for very wide pages
|
||||
/// (`ANALYSIS_MIN_SLICE_HEIGHT`). Implies `max_slice_width =
|
||||
/// max_pixels / min_slice_height`.
|
||||
pub min_slice_height: u32,
|
||||
/// Vertical overlap between adjacent slices as a fraction of slice height
|
||||
/// (`ANALYSIS_SLICE_OVERLAP`), so text straddling a cut survives.
|
||||
pub slice_overlap: f64,
|
||||
/// Slice only when `height/width` exceeds this (`ANALYSIS_TALL_ASPECT`);
|
||||
/// normal-aspect pages take a single combined call.
|
||||
pub tall_aspect_threshold: f64,
|
||||
/// Hard cap on slices per page (`ANALYSIS_MAX_SLICES`); beyond it slices
|
||||
/// grow coarser (and get downscaled to budget) rather than multiplying.
|
||||
pub max_slices: usize,
|
||||
/// Hard cap on a page image's stored size; larger pages are skipped
|
||||
/// (`ANALYSIS_MAX_IMAGE_BYTES`).
|
||||
pub max_image_bytes: usize,
|
||||
/// Output-constraint mode (`ANALYSIS_RESPONSE_FORMAT`):
|
||||
/// `json_schema` (default) | `json_object` | `none`.
|
||||
pub response_format: ResponseFormat,
|
||||
/// Sampling `frequency_penalty` sent with each request
|
||||
/// (`ANALYSIS_FREQUENCY_PENALTY`). A small positive value discourages
|
||||
/// the repetition loops that otherwise run small models into the token
|
||||
/// ceiling. `0` omits the field.
|
||||
pub frequency_penalty: f64,
|
||||
/// Sampling `temperature` sent with each request (`ANALYSIS_TEMPERATURE`).
|
||||
/// Defaults to `0.0` (deterministic), which is the most reliable for
|
||||
/// structured JSON output; some models behave better with a small
|
||||
/// positive value.
|
||||
pub temperature: f64,
|
||||
/// System prompt for the single-call (normal-aspect) analysis path
|
||||
/// (`ANALYSIS_SYSTEM_PROMPT`). Defaults to
|
||||
/// [`crate::analysis::prompt::SYSTEM_PROMPT_DEFAULT`].
|
||||
pub system_prompt: String,
|
||||
/// Pass-A OCR-only prompt for tall-page slices (`ANALYSIS_OCR_PROMPT`).
|
||||
/// Defaults to [`crate::analysis::prompt::OCR_PROMPT_DEFAULT`].
|
||||
pub ocr_prompt: String,
|
||||
/// Pass-B grounding prompt — tags/scene/safety (`ANALYSIS_GROUNDING_PROMPT`).
|
||||
/// Defaults to [`crate::analysis::prompt::GROUNDING_PROMPT_DEFAULT`].
|
||||
pub grounding_prompt: String,
|
||||
}
|
||||
|
||||
impl Default for AnalysisConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
enabled: false,
|
||||
workers: 1,
|
||||
endpoint: "http://localhost:8000/v1/chat/completions".to_string(),
|
||||
model: String::new(),
|
||||
api_key: None,
|
||||
request_timeout: Duration::from_secs(120),
|
||||
// A sliced long page is N sequential calls under one job, so the
|
||||
// whole-job budget must be generous (request_timeout stays
|
||||
// per-call).
|
||||
job_timeout: Duration::from_secs(600),
|
||||
max_tokens: 4096,
|
||||
max_pixels: 1_000_000,
|
||||
min_slice_height: 640,
|
||||
slice_overlap: 0.12,
|
||||
tall_aspect_threshold: 1.6,
|
||||
max_slices: 16,
|
||||
max_image_bytes: 8 * 1024 * 1024,
|
||||
response_format: ResponseFormat::JsonSchema,
|
||||
frequency_penalty: 0.3,
|
||||
temperature: 0.0,
|
||||
system_prompt: crate::analysis::prompt::SYSTEM_PROMPT_DEFAULT.to_string(),
|
||||
ocr_prompt: crate::analysis::prompt::OCR_PROMPT_DEFAULT.to_string(),
|
||||
grounding_prompt: crate::analysis::prompt::GROUNDING_PROMPT_DEFAULT.to_string(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl AnalysisConfig {
|
||||
pub fn from_env() -> Self {
|
||||
let d = AnalysisConfig::default();
|
||||
Self {
|
||||
enabled: env_bool("ANALYSIS_ENABLED", d.enabled),
|
||||
workers: env_usize("ANALYSIS_WORKERS", d.workers).max(1),
|
||||
endpoint: std::env::var("ANALYSIS_VISION_URL").unwrap_or(d.endpoint),
|
||||
model: std::env::var("ANALYSIS_MODEL").unwrap_or(d.model),
|
||||
api_key: std::env::var("ANALYSIS_API_KEY")
|
||||
.ok()
|
||||
.filter(|s| !s.is_empty()),
|
||||
request_timeout: Duration::from_secs(env_u64(
|
||||
"ANALYSIS_REQUEST_TIMEOUT_SECS",
|
||||
d.request_timeout.as_secs(),
|
||||
)),
|
||||
job_timeout: Duration::from_secs(env_u64(
|
||||
"ANALYSIS_JOB_TIMEOUT_SECS",
|
||||
d.job_timeout.as_secs(),
|
||||
)),
|
||||
max_tokens: env_u64("ANALYSIS_MAX_TOKENS", d.max_tokens as u64) as u32,
|
||||
max_pixels: env_u64("ANALYSIS_MAX_PIXELS", d.max_pixels as u64) as u32,
|
||||
min_slice_height: env_u64("ANALYSIS_MIN_SLICE_HEIGHT", d.min_slice_height as u64)
|
||||
.max(1) as u32,
|
||||
slice_overlap: env_f64("ANALYSIS_SLICE_OVERLAP", d.slice_overlap).clamp(0.0, 0.9),
|
||||
tall_aspect_threshold: env_f64("ANALYSIS_TALL_ASPECT", d.tall_aspect_threshold)
|
||||
.max(1.0),
|
||||
max_slices: env_usize("ANALYSIS_MAX_SLICES", d.max_slices).max(1),
|
||||
max_image_bytes: env_usize("ANALYSIS_MAX_IMAGE_BYTES", d.max_image_bytes),
|
||||
response_format: std::env::var("ANALYSIS_RESPONSE_FORMAT")
|
||||
.map(|s| ResponseFormat::from_str(&s))
|
||||
.unwrap_or(d.response_format),
|
||||
frequency_penalty: env_f64("ANALYSIS_FREQUENCY_PENALTY", d.frequency_penalty),
|
||||
temperature: env_f64("ANALYSIS_TEMPERATURE", d.temperature),
|
||||
system_prompt: env_prompt("ANALYSIS_SYSTEM_PROMPT", d.system_prompt),
|
||||
ocr_prompt: env_prompt("ANALYSIS_OCR_PROMPT", d.ocr_prompt),
|
||||
grounding_prompt: env_prompt("ANALYSIS_GROUNDING_PROMPT", d.grounding_prompt),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Read a prompt override from env, falling back to the default when unset
|
||||
/// or blank (so `ANALYSIS_SYSTEM_PROMPT=` doesn't wipe the prompt).
|
||||
fn env_prompt(name: &str, default: String) -> String {
|
||||
std::env::var(name)
|
||||
.ok()
|
||||
.filter(|s| !s.trim().is_empty())
|
||||
.unwrap_or(default)
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Config {
|
||||
pub database_url: String,
|
||||
@@ -74,7 +271,22 @@ pub struct Config {
|
||||
pub auth: AuthConfig,
|
||||
pub upload: UploadConfig,
|
||||
pub cors_allowed_origins: Vec<String>,
|
||||
/// Origins (scheme + host[:port]) that may issue browser-driven
|
||||
/// mutating requests to `/api/v1/admin/*`. Defends against
|
||||
/// SameSite=Lax CSRF on the admin cookie: a top-level form POST from
|
||||
/// a malicious page still carries the cookie, but the middleware
|
||||
/// rejects it when the `Origin` (or `Referer` fallback) is absent
|
||||
/// from this list. Sourced from `ADMIN_ALLOWED_ORIGINS`
|
||||
/// (comma-separated). Leave empty to skip the check entirely
|
||||
/// (curl / server-to-server callers send neither header, so they
|
||||
/// pass; same-origin browser requests don't have Origin set on
|
||||
/// same-origin POSTs in some browsers either — operators on a
|
||||
/// same-origin deploy can leave this empty, but doing so removes
|
||||
/// the CSRF defence). Safe methods (GET/HEAD/OPTIONS) never trigger
|
||||
/// the check.
|
||||
pub admin_allowed_origins: Vec<String>,
|
||||
pub crawler: CrawlerConfig,
|
||||
pub analysis: AnalysisConfig,
|
||||
/// `(username, password)` for the admin user provisioned at startup
|
||||
/// when both `ADMIN_USERNAME` and `ADMIN_PASSWORD` are set. `None`
|
||||
/// skips the bootstrap entirely. See `repo::user::bootstrap_admin`
|
||||
@@ -132,6 +344,19 @@ pub struct CrawlerConfig {
|
||||
/// (full sweep up to the source's own bound). Sourced from
|
||||
/// `CRAWLER_LIMIT`, mirroring the CLI binary.
|
||||
pub manga_limit: usize,
|
||||
/// Hard upper bound on a single chapter-content job dispatch. A job
|
||||
/// exceeding this is acked failed (exponential backoff) instead of
|
||||
/// wedging a worker. Defaults to 600s. `CRAWLER_JOB_TIMEOUT_SECS`.
|
||||
pub job_timeout: Duration,
|
||||
/// Consecutive `fetch_manga` failures that abort a metadata pass
|
||||
/// (circuit-breaker for a source outage). The pass does NOT mark a
|
||||
/// clean exit, so the next tick does a recovery sweep. Defaults to
|
||||
/// 10. `CRAWLER_METADATA_MAX_CONSECUTIVE_FAILURES`.
|
||||
pub metadata_max_consecutive_failures: u32,
|
||||
/// Consecutive transient chapter failures (after TOR recircuit is
|
||||
/// exhausted) that trigger an automatic coordinated browser restart.
|
||||
/// Defaults to 3. `CRAWLER_BROWSER_RESTART_THRESHOLD`.
|
||||
pub browser_restart_threshold: u32,
|
||||
}
|
||||
|
||||
impl Default for CrawlerConfig {
|
||||
@@ -159,6 +384,9 @@ impl Default for CrawlerConfig {
|
||||
download_allowlist: DownloadAllowlist::new(),
|
||||
max_image_bytes: DEFAULT_MAX_IMAGE_BYTES,
|
||||
manga_limit: 0,
|
||||
job_timeout: Duration::from_secs(600),
|
||||
metadata_max_consecutive_failures: 10,
|
||||
browser_restart_threshold: 3,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -205,7 +433,17 @@ impl Config {
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_default(),
|
||||
admin_allowed_origins: std::env::var("ADMIN_ALLOWED_ORIGINS")
|
||||
.ok()
|
||||
.map(|s| {
|
||||
s.split(',')
|
||||
.map(|o| o.trim().to_string())
|
||||
.filter(|o| !o.is_empty())
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_default(),
|
||||
crawler: CrawlerConfig::from_env()?,
|
||||
analysis: AnalysisConfig::from_env(),
|
||||
admin_bootstrap: admin_bootstrap_from_env(),
|
||||
})
|
||||
}
|
||||
@@ -283,6 +521,13 @@ impl CrawlerConfig {
|
||||
download_allowlist,
|
||||
max_image_bytes: env_usize("CRAWLER_MAX_IMAGE_BYTES", DEFAULT_MAX_IMAGE_BYTES),
|
||||
manga_limit: env_usize("CRAWLER_LIMIT", 0),
|
||||
job_timeout: Duration::from_secs(env_u64("CRAWLER_JOB_TIMEOUT_SECS", 600).max(1)),
|
||||
metadata_max_consecutive_failures: env_u64(
|
||||
"CRAWLER_METADATA_MAX_CONSECUTIVE_FAILURES",
|
||||
10,
|
||||
) as u32,
|
||||
browser_restart_threshold: env_u64("CRAWLER_BROWSER_RESTART_THRESHOLD", 3).max(1)
|
||||
as u32,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -349,6 +594,13 @@ fn env_i64(name: &str, default: i64) -> i64 {
|
||||
.unwrap_or(default)
|
||||
}
|
||||
|
||||
fn env_f64(name: &str, default: f64) -> f64 {
|
||||
std::env::var(name)
|
||||
.ok()
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(default)
|
||||
}
|
||||
|
||||
fn env_usize(name: &str, default: usize) -> usize {
|
||||
std::env::var(name)
|
||||
.ok()
|
||||
@@ -384,6 +636,112 @@ mod tests {
|
||||
assert_eq!(cfg.manga_limit, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reliability_knobs_default_when_unset() {
|
||||
let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner());
|
||||
std::env::remove_var("CRAWLER_JOB_TIMEOUT_SECS");
|
||||
std::env::remove_var("CRAWLER_METADATA_MAX_CONSECUTIVE_FAILURES");
|
||||
std::env::remove_var("CRAWLER_BROWSER_RESTART_THRESHOLD");
|
||||
let cfg = CrawlerConfig::from_env().expect("from_env");
|
||||
assert_eq!(cfg.job_timeout, Duration::from_secs(600));
|
||||
assert_eq!(cfg.metadata_max_consecutive_failures, 10);
|
||||
assert_eq!(cfg.browser_restart_threshold, 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reliability_knobs_parse_from_env() {
|
||||
let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner());
|
||||
std::env::set_var("CRAWLER_JOB_TIMEOUT_SECS", "120");
|
||||
std::env::set_var("CRAWLER_METADATA_MAX_CONSECUTIVE_FAILURES", "5");
|
||||
std::env::set_var("CRAWLER_BROWSER_RESTART_THRESHOLD", "7");
|
||||
let cfg = CrawlerConfig::from_env().expect("from_env");
|
||||
std::env::remove_var("CRAWLER_JOB_TIMEOUT_SECS");
|
||||
std::env::remove_var("CRAWLER_METADATA_MAX_CONSECUTIVE_FAILURES");
|
||||
std::env::remove_var("CRAWLER_BROWSER_RESTART_THRESHOLD");
|
||||
assert_eq!(cfg.job_timeout, Duration::from_secs(120));
|
||||
assert_eq!(cfg.metadata_max_consecutive_failures, 5);
|
||||
assert_eq!(cfg.browser_restart_threshold, 7);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn analysis_config_defaults_when_unset() {
|
||||
let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner());
|
||||
for k in [
|
||||
"ANALYSIS_ENABLED",
|
||||
"ANALYSIS_WORKERS",
|
||||
"ANALYSIS_VISION_URL",
|
||||
"ANALYSIS_MODEL",
|
||||
"ANALYSIS_API_KEY",
|
||||
"ANALYSIS_MAX_TOKENS",
|
||||
"ANALYSIS_MAX_PIXELS",
|
||||
"ANALYSIS_MIN_SLICE_HEIGHT",
|
||||
"ANALYSIS_SLICE_OVERLAP",
|
||||
"ANALYSIS_TALL_ASPECT",
|
||||
"ANALYSIS_MAX_SLICES",
|
||||
"ANALYSIS_RESPONSE_FORMAT",
|
||||
"ANALYSIS_FREQUENCY_PENALTY",
|
||||
] {
|
||||
std::env::remove_var(k);
|
||||
}
|
||||
let cfg = AnalysisConfig::from_env();
|
||||
assert!(!cfg.enabled);
|
||||
assert_eq!(cfg.workers, 1);
|
||||
assert_eq!(cfg.max_pixels, 1_000_000);
|
||||
assert_eq!(cfg.min_slice_height, 640);
|
||||
assert_eq!(cfg.max_slices, 16);
|
||||
assert_eq!(cfg.tall_aspect_threshold, 1.6);
|
||||
assert_eq!(cfg.frequency_penalty, 0.3);
|
||||
assert!(cfg.api_key.is_none());
|
||||
assert_eq!(cfg.response_format, ResponseFormat::JsonSchema);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn analysis_response_format_parses_modes() {
|
||||
let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner());
|
||||
for (raw, want) in [
|
||||
("json_object", ResponseFormat::JsonObject),
|
||||
("none", ResponseFormat::None),
|
||||
("text", ResponseFormat::None),
|
||||
("json_schema", ResponseFormat::JsonSchema),
|
||||
("anything-else", ResponseFormat::JsonSchema),
|
||||
] {
|
||||
std::env::set_var("ANALYSIS_RESPONSE_FORMAT", raw);
|
||||
assert_eq!(AnalysisConfig::from_env().response_format, want, "raw={raw}");
|
||||
}
|
||||
std::env::remove_var("ANALYSIS_RESPONSE_FORMAT");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn analysis_config_parses_from_env() {
|
||||
let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner());
|
||||
std::env::set_var("ANALYSIS_ENABLED", "true");
|
||||
std::env::set_var("ANALYSIS_WORKERS", "4");
|
||||
std::env::set_var("ANALYSIS_VISION_URL", "http://vis/v1/chat");
|
||||
std::env::set_var("ANALYSIS_MODEL", "qwen2-vl");
|
||||
std::env::set_var("ANALYSIS_MAX_PIXELS", "768000");
|
||||
std::env::set_var("ANALYSIS_MAX_SLICES", "8");
|
||||
std::env::set_var("ANALYSIS_SLICE_OVERLAP", "0.2");
|
||||
let cfg = AnalysisConfig::from_env();
|
||||
for k in [
|
||||
"ANALYSIS_ENABLED",
|
||||
"ANALYSIS_WORKERS",
|
||||
"ANALYSIS_VISION_URL",
|
||||
"ANALYSIS_MODEL",
|
||||
"ANALYSIS_MAX_PIXELS",
|
||||
"ANALYSIS_MAX_SLICES",
|
||||
"ANALYSIS_SLICE_OVERLAP",
|
||||
] {
|
||||
std::env::remove_var(k);
|
||||
}
|
||||
assert!(cfg.enabled);
|
||||
assert_eq!(cfg.workers, 4);
|
||||
assert_eq!(cfg.endpoint, "http://vis/v1/chat");
|
||||
assert_eq!(cfg.model, "qwen2-vl");
|
||||
assert_eq!(cfg.max_pixels, 768_000);
|
||||
assert_eq!(cfg.max_slices, 8);
|
||||
assert_eq!(cfg.slice_overlap, 0.2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn private_mode_env_parses_true() {
|
||||
let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner());
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
//! until [`BrowserManager::shutdown`].
|
||||
|
||||
use std::ops::Deref;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::atomic::{AtomicBool, AtomicU8, AtomicUsize, Ordering};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
@@ -71,12 +71,42 @@ impl ActiveTracker {
|
||||
}
|
||||
}
|
||||
|
||||
/// Lifecycle gate for a coordinated browser restart. `acquire()` parks
|
||||
/// while not [`RestartPhase::Healthy`] so no new navigation starts mid-
|
||||
/// restart; long-lived lease holders (the metadata pass) cooperate by
|
||||
/// checking [`BrowserManager::is_restart_pending`] at safe boundaries.
|
||||
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
|
||||
pub enum RestartPhase {
|
||||
/// Normal operation — acquires proceed.
|
||||
Healthy,
|
||||
/// Restart requested; new acquires park, waiting for in-flight leases
|
||||
/// to drain.
|
||||
Draining,
|
||||
/// Chromium is being closed + relaunched.
|
||||
Restarting,
|
||||
}
|
||||
|
||||
const PHASE_HEALTHY: u8 = 0;
|
||||
const PHASE_DRAINING: u8 = 1;
|
||||
const PHASE_RESTARTING: u8 = 2;
|
||||
|
||||
pub struct BrowserManager {
|
||||
inner: Mutex<Inner>,
|
||||
active: Arc<ActiveTracker>,
|
||||
launch_opts: LaunchOptions,
|
||||
idle_timeout: Duration,
|
||||
on_launch: OnLaunch,
|
||||
/// Coarse lifecycle phase (one of the `PHASE_*` constants).
|
||||
phase: AtomicU8,
|
||||
/// Woken when the phase returns to `Healthy` so parked acquires resume.
|
||||
resume: Notify,
|
||||
/// Serialises coordinated restarts so concurrent requests collapse into
|
||||
/// a single relaunch.
|
||||
restart_lock: Mutex<()>,
|
||||
/// Result of the most recent relaunch, so a caller that coalesced into
|
||||
/// an in-progress restart reports that restart's real outcome instead
|
||||
/// of a blind success.
|
||||
last_restart_ok: AtomicBool,
|
||||
}
|
||||
|
||||
struct Inner {
|
||||
@@ -99,28 +129,72 @@ impl BrowserManager {
|
||||
launch_opts,
|
||||
idle_timeout,
|
||||
on_launch,
|
||||
phase: AtomicU8::new(PHASE_HEALTHY),
|
||||
resume: Notify::new(),
|
||||
restart_lock: Mutex::new(()),
|
||||
last_restart_ok: AtomicBool::new(true),
|
||||
})
|
||||
}
|
||||
|
||||
/// Current restart phase.
|
||||
pub fn phase(&self) -> RestartPhase {
|
||||
match self.phase.load(Ordering::Acquire) {
|
||||
PHASE_DRAINING => RestartPhase::Draining,
|
||||
PHASE_RESTARTING => RestartPhase::Restarting,
|
||||
_ => RestartPhase::Healthy,
|
||||
}
|
||||
}
|
||||
|
||||
fn set_phase(&self, phase: RestartPhase) {
|
||||
let v = match phase {
|
||||
RestartPhase::Healthy => PHASE_HEALTHY,
|
||||
RestartPhase::Draining => PHASE_DRAINING,
|
||||
RestartPhase::Restarting => PHASE_RESTARTING,
|
||||
};
|
||||
self.phase.store(v, Ordering::Release);
|
||||
}
|
||||
|
||||
/// Whether a coordinated restart is in progress. Long-lived lease
|
||||
/// holders poll this at safe boundaries and yield their lease so the
|
||||
/// drain can complete promptly.
|
||||
pub fn is_restart_pending(&self) -> bool {
|
||||
self.phase() != RestartPhase::Healthy
|
||||
}
|
||||
|
||||
/// Launch Chromium into `guard`, running the `on_launch` hook before
|
||||
/// publishing the handle so a probe failure doesn't leave a half-
|
||||
/// initialised browser behind.
|
||||
async fn launch_into(&self, guard: &mut Inner) -> anyhow::Result<()> {
|
||||
let handle = browser::launch(self.launch_opts.clone())
|
||||
.await
|
||||
.context("BrowserManager: launch chromium")?;
|
||||
let shared = handle.shared();
|
||||
if let Err(e) = (self.on_launch)(Arc::clone(&shared)).await {
|
||||
let _ = handle.close().await;
|
||||
return Err(e.context("BrowserManager: on_launch hook failed"));
|
||||
}
|
||||
guard.handle = Some(handle);
|
||||
guard.shared = Some(shared);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Acquire a shared browser lease. The first acquire after a teardown
|
||||
/// launches a fresh Chromium (and runs `on_launch`); subsequent acquires
|
||||
/// while a process is alive just bump the counter and clone the `Arc`.
|
||||
pub async fn acquire(&self) -> anyhow::Result<BrowserLease> {
|
||||
// Park while a coordinated restart is draining/relaunching so no new
|
||||
// navigation starts against a browser that's about to be torn down.
|
||||
// The short sleep fallback guarantees liveness even if a `resume`
|
||||
// notification is missed (classic Notify lost-wakeup).
|
||||
while self.phase() != RestartPhase::Healthy {
|
||||
tokio::select! {
|
||||
_ = self.resume.notified() => {}
|
||||
_ = tokio::time::sleep(Duration::from_millis(100)) => {}
|
||||
}
|
||||
}
|
||||
let mut guard = self.inner.lock().await;
|
||||
if guard.handle.is_none() {
|
||||
let handle = browser::launch(self.launch_opts.clone())
|
||||
.await
|
||||
.context("BrowserManager: launch chromium")?;
|
||||
let shared = handle.shared();
|
||||
// Run the on-launch hook before publishing the handle so a session
|
||||
// probe failure doesn't leave a half-initialized browser behind.
|
||||
if let Err(e) = (self.on_launch)(Arc::clone(&shared)).await {
|
||||
// Close the just-launched browser since we won't be using it.
|
||||
let _ = handle.close().await;
|
||||
return Err(e.context("BrowserManager: on_launch hook failed"));
|
||||
}
|
||||
guard.handle = Some(handle);
|
||||
guard.shared = Some(shared);
|
||||
self.launch_into(&mut guard).await?;
|
||||
}
|
||||
let browser = guard
|
||||
.shared
|
||||
@@ -134,6 +208,51 @@ impl BrowserManager {
|
||||
})
|
||||
}
|
||||
|
||||
/// Coordinated restart: block new acquires, wait for in-flight leases
|
||||
/// to drain (up to `drain_deadline`, then force), close + relaunch
|
||||
/// Chromium (re-running `on_launch` → re-inject session + probe), then
|
||||
/// resume parked acquirers. Concurrent calls collapse into one
|
||||
/// relaunch. The phase is always returned to `Healthy` — even if the
|
||||
/// relaunch errors — so a failed restart never permanently wedges
|
||||
/// acquisition (the next acquire retries the launch lazily).
|
||||
pub async fn coordinated_restart(&self, drain_deadline: Duration) -> anyhow::Result<()> {
|
||||
// Dedup: if a restart is already running, wait for it and report
|
||||
// that restart's real outcome (not a blind success).
|
||||
let _restart_guard = match self.restart_lock.try_lock() {
|
||||
Ok(g) => g,
|
||||
Err(_) => {
|
||||
let _ = self.restart_lock.lock().await;
|
||||
return if self.last_restart_ok.load(Ordering::Acquire) {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(anyhow::anyhow!("a concurrent coordinated browser restart failed"))
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
self.set_phase(RestartPhase::Draining);
|
||||
await_drain(&self.active, drain_deadline).await;
|
||||
|
||||
self.set_phase(RestartPhase::Restarting);
|
||||
let relaunch = {
|
||||
let mut guard = self.inner.lock().await;
|
||||
guard.shared = None;
|
||||
if let Some(handle) = guard.handle.take() {
|
||||
let _ = handle.close().await;
|
||||
}
|
||||
self.launch_into(&mut guard).await
|
||||
};
|
||||
|
||||
self.last_restart_ok.store(relaunch.is_ok(), Ordering::Release);
|
||||
self.set_phase(RestartPhase::Healthy);
|
||||
self.resume.notify_waiters();
|
||||
match &relaunch {
|
||||
Ok(()) => tracing::info!("BrowserManager: coordinated restart complete"),
|
||||
Err(e) => tracing::error!(error = ?e, "BrowserManager: coordinated restart relaunch failed"),
|
||||
}
|
||||
relaunch.context("coordinated_restart: relaunch")
|
||||
}
|
||||
|
||||
/// Forcefully close the cached browser regardless of active count.
|
||||
/// Used on daemon shutdown. After this returns the next acquire will
|
||||
/// re-launch from scratch.
|
||||
@@ -176,6 +295,29 @@ impl BrowserManager {
|
||||
}
|
||||
}
|
||||
|
||||
/// Wait for the active-lease count to reach zero, up to `deadline`. Wakes
|
||||
/// on the tracker's idle signal and re-checks on a short poll so a missed
|
||||
/// signal can't strand the drain. Returns when drained or when the
|
||||
/// deadline elapses (the caller then force-restarts). Extracted as a free
|
||||
/// fn so the timing logic is unit-testable without launching Chromium.
|
||||
async fn await_drain(active: &Arc<ActiveTracker>, deadline: Duration) {
|
||||
let start = tokio::time::Instant::now();
|
||||
while active.current() > 0 {
|
||||
let Some(remaining) = deadline.checked_sub(start.elapsed()) else {
|
||||
tracing::warn!(
|
||||
active = active.current(),
|
||||
"coordinated_restart: drain deadline exceeded — forcing relaunch"
|
||||
);
|
||||
return;
|
||||
};
|
||||
let nap = remaining.min(Duration::from_millis(250));
|
||||
tokio::select! {
|
||||
_ = active.idle_signal().notified() => {}
|
||||
_ = tokio::time::sleep(nap) => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Background reaper. Returns immediately when `idle_timeout == 0`.
|
||||
/// Otherwise spawns a task that:
|
||||
/// 1. Waits on `idle_signal` (woken when active hits zero).
|
||||
@@ -270,6 +412,63 @@ mod tests {
|
||||
mgr.invalidate().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn await_drain_returns_immediately_when_already_idle() {
|
||||
let active = ActiveTracker::new();
|
||||
let start = tokio::time::Instant::now();
|
||||
await_drain(&active, Duration::from_secs(5)).await;
|
||||
assert!(start.elapsed() < Duration::from_millis(200), "no wait when idle");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn await_drain_completes_when_lease_released() {
|
||||
let active = ActiveTracker::new();
|
||||
active.acquire();
|
||||
let bg = {
|
||||
let a = Arc::clone(&active);
|
||||
tokio::spawn(async move {
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
a.release();
|
||||
})
|
||||
};
|
||||
// Generous deadline; should return shortly after the release, not
|
||||
// at the deadline.
|
||||
let start = tokio::time::Instant::now();
|
||||
await_drain(&active, Duration::from_secs(5)).await;
|
||||
assert!(start.elapsed() < Duration::from_secs(2), "drained on release");
|
||||
assert_eq!(active.current(), 0);
|
||||
bg.await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn await_drain_force_returns_after_deadline_when_stuck() {
|
||||
let active = ActiveTracker::new();
|
||||
active.acquire(); // never released
|
||||
let start = tokio::time::Instant::now();
|
||||
await_drain(&active, Duration::from_millis(300)).await;
|
||||
let elapsed = start.elapsed();
|
||||
assert!(elapsed >= Duration::from_millis(250), "waited ~deadline: {elapsed:?}");
|
||||
assert!(elapsed < Duration::from_secs(2), "but not forever: {elapsed:?}");
|
||||
assert_eq!(active.current(), 1, "still held — caller force-restarts");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn phase_transitions_reflect_is_restart_pending() {
|
||||
let mgr = BrowserManager::new(
|
||||
crate::crawler::browser::LaunchOptions::default(),
|
||||
Duration::ZERO,
|
||||
noop_on_launch(),
|
||||
);
|
||||
assert_eq!(mgr.phase(), RestartPhase::Healthy);
|
||||
assert!(!mgr.is_restart_pending());
|
||||
mgr.set_phase(RestartPhase::Draining);
|
||||
assert!(mgr.is_restart_pending());
|
||||
mgr.set_phase(RestartPhase::Restarting);
|
||||
assert!(mgr.is_restart_pending());
|
||||
mgr.set_phase(RestartPhase::Healthy);
|
||||
assert!(!mgr.is_restart_pending());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn active_tracker_signals_idle_only_on_zero_transition() {
|
||||
let tracker = ActiveTracker::new();
|
||||
|
||||
@@ -18,9 +18,9 @@ use uuid::Uuid;
|
||||
|
||||
use crate::crawler::detect::PageError;
|
||||
use crate::crawler::rate_limit::HostRateLimiters;
|
||||
use crate::crawler::safety::{fetch_bytes_capped, looks_like_image, DownloadAllowlist};
|
||||
use crate::crawler::safety::{fetch_stream, looks_like_image, DownloadAllowlist};
|
||||
use crate::crawler::session::{self, ChapterProbe};
|
||||
use crate::storage::Storage;
|
||||
use crate::storage::{Storage, StorageError};
|
||||
|
||||
/// Parse the chapter page DOM and return the page images in `pageN`
|
||||
/// order. Filters out the loader `<img class="loading">` and any
|
||||
@@ -186,11 +186,17 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
/// Fetch all images for one chapter and persist them atomically. On
|
||||
/// any error after the first storage put, the DB transaction rolls
|
||||
/// back so the chapter stays at `page_count = 0` and is retried on the
|
||||
/// next run. Bytes already written to storage become orphans; a future
|
||||
/// reaper sweeps them.
|
||||
/// Fetch one chapter's images and persist them. Each image is
|
||||
/// streamed straight to storage via `Storage::put_stream` after a
|
||||
/// short prefix is peeked off the body for content-type sniffing —
|
||||
/// peak memory per concurrent dispatch is one HTTP chunk plus the
|
||||
/// sniff prefix, not a full multi-MB image. The per-image size cap
|
||||
/// (`CRAWLER_MAX_IMAGE_BYTES`) is enforced inside the stream so a
|
||||
/// server that omits Content-Length still can't exhaust memory. The
|
||||
/// page rows + `page_count` are then written in one short transaction.
|
||||
/// On any failure the chapter stays at `page_count = 0` (no partial
|
||||
/// rows) and the blobs already written are deleted best-effort by
|
||||
/// [`cleanup_orphans`], so a retry starts clean.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub async fn sync_chapter_content(
|
||||
browser: &chromiumoxide::Browser,
|
||||
@@ -205,6 +211,14 @@ pub async fn sync_chapter_content(
|
||||
allowlist: &DownloadAllowlist,
|
||||
max_image_bytes: usize,
|
||||
tor: Option<&crate::crawler::tor::TorController>,
|
||||
// Optional live-status sink for the realtime page counter. The daemon
|
||||
// dispatcher passes the shared handle (the chapter has already been
|
||||
// registered via `begin_chapter`); the CLI / admin resync pass `None`.
|
||||
progress: Option<&crate::crawler::status::StatusHandle>,
|
||||
// When `true`, enqueue an `analyze_page` job per persisted page. The
|
||||
// daemon dispatcher passes the configured flag; CLI / resync pass
|
||||
// `false`.
|
||||
enqueue_analysis: bool,
|
||||
) -> anyhow::Result<SyncOutcome> {
|
||||
// Skip if already fetched, unless caller explicitly forces.
|
||||
if !force_refetch {
|
||||
@@ -260,80 +274,245 @@ pub async fn sync_chapter_content(
|
||||
// Resolve image URLs against the chapter URL (they may be relative).
|
||||
let base = reqwest::Url::parse(source_url).context("parse chapter URL")?;
|
||||
|
||||
// Fetch every image bytes-first into memory before writing
|
||||
// anything. Lets us bail the whole chapter cleanly if any image
|
||||
// fails — DB stays at page_count=0, no partial rows persisted.
|
||||
let mut fetched: Vec<(i32, Vec<u8>, &'static str)> = Vec::with_capacity(images.len());
|
||||
// Stream each image straight to storage as it's fetched, capping peak
|
||||
// memory at a single image rather than the whole chapter. Track the
|
||||
// keys written so they can be rolled back if a later page (or the
|
||||
// final DB commit) fails — preserving the all-or-nothing guarantee
|
||||
// without holding a DB transaction open across the network puts
|
||||
// (which matters once `Storage` is backed by S3).
|
||||
let total = images.len();
|
||||
// Publish the now-known page total so the dashboard shows "0/N".
|
||||
if let Some(p) = progress {
|
||||
p.set_chapter_pages(chapter_id, 0, Some(total));
|
||||
}
|
||||
let mut written_keys: Vec<String> = Vec::with_capacity(total);
|
||||
let mut stored: Vec<StoredPage> = Vec::with_capacity(total);
|
||||
for img in &images {
|
||||
let url = base.join(&img.url).with_context(|| {
|
||||
format!("join image URL {} onto {source_url}", img.url)
|
||||
})?;
|
||||
rate.wait_for(url.as_str()).await?;
|
||||
let bytes = fetch_bytes_capped(
|
||||
match download_and_store_page(
|
||||
storage,
|
||||
http,
|
||||
url.as_str(),
|
||||
Some(source_url),
|
||||
rate,
|
||||
&base,
|
||||
source_url,
|
||||
manga_id,
|
||||
chapter_id,
|
||||
img,
|
||||
allowlist,
|
||||
max_image_bytes,
|
||||
)
|
||||
.await?
|
||||
.to_vec();
|
||||
// Reject any non-image response: the only valid output of an
|
||||
// image URL is an image. `infer` returns None on truncated
|
||||
// bytes too, which also wants to be a failure not a silent
|
||||
// `.bin` extension.
|
||||
if !looks_like_image(&bytes) {
|
||||
anyhow::bail!(
|
||||
"image URL {url} returned non-image bytes \
|
||||
(first 16: {:?}); refusing to store as binary blob",
|
||||
&bytes.get(..16.min(bytes.len()))
|
||||
);
|
||||
.await
|
||||
{
|
||||
Ok(page) => {
|
||||
written_keys.push(page.storage_key.clone());
|
||||
stored.push(page);
|
||||
// Live page counter: push the climbing count to subscribers.
|
||||
if let Some(p) = progress {
|
||||
p.set_chapter_pages(chapter_id, stored.len(), Some(total));
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
cleanup_orphans(storage, &written_keys).await;
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
let ext = infer::get(&bytes)
|
||||
.map(|k| k.extension())
|
||||
.expect("looks_like_image asserted infer succeeded");
|
||||
fetched.push((img.page_number, bytes, ext));
|
||||
}
|
||||
|
||||
// Atomic write: storage puts + page row inserts + page_count
|
||||
// update, all in one transaction. If anything fails, rollback +
|
||||
// the chapter is retried next run. Storage orphans the bytes; a
|
||||
// reaper sweeps them later.
|
||||
let mut tx = db.begin().await.context("open chapter sync tx")?;
|
||||
for (page_number, bytes, ext) in &fetched {
|
||||
let key = format!(
|
||||
"mangas/{manga_id}/chapters/{chapter_id}/pages/{:04}.{ext}",
|
||||
page_number
|
||||
// Short transaction: page rows + page_count only, no network I/O. On
|
||||
// failure, roll back the stored bytes so the chapter stays at
|
||||
// page_count=0 and is retried cleanly next run.
|
||||
if let Err(e) = persist_pages(db, chapter_id, &stored, enqueue_analysis).await {
|
||||
cleanup_orphans(storage, &written_keys).await;
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
Ok(SyncOutcome::Fetched { pages: stored.len() })
|
||||
}
|
||||
|
||||
/// A page image that has been written to storage and is awaiting its DB
|
||||
/// row. Carries everything `persist_pages` needs.
|
||||
pub(crate) struct StoredPage {
|
||||
page_number: i32,
|
||||
storage_key: String,
|
||||
content_type: String,
|
||||
}
|
||||
|
||||
/// Bytes accumulated for content-type sniffing. `infer` only needs the
|
||||
/// first few bytes for image formats (the longest signature in our
|
||||
/// allow-list is AVIF at 12 bytes), but we read up to this many so a
|
||||
/// fragmented TCP frame still produces a confident sniff and the
|
||||
/// "first 16 bytes" diagnostic in the error path is useful.
|
||||
const SNIFF_PREFIX_BYTES: usize = 64;
|
||||
|
||||
/// Download a single page image, validate it's really an image, and
|
||||
/// stream it to storage. Returns the storage key + content type. Does
|
||||
/// not touch the DB — persistence is batched into one short transaction
|
||||
/// afterward.
|
||||
///
|
||||
/// Streaming path: we peek the first [`SNIFF_PREFIX_BYTES`] from the
|
||||
/// HTTP body to determine the file extension (and thus the storage
|
||||
/// key), then re-emit those bytes followed by the rest of the response
|
||||
/// stream via `Storage::put_stream`. Peak memory per concurrent
|
||||
/// dispatch is one HTTP chunk (~16 KiB) plus the sniff prefix, not a
|
||||
/// full multi-MB image. The per-image cap is enforced as bytes flow.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn download_and_store_page(
|
||||
storage: &dyn Storage,
|
||||
http: &reqwest::Client,
|
||||
rate: &HostRateLimiters,
|
||||
base: &reqwest::Url,
|
||||
source_url: &str,
|
||||
manga_id: Uuid,
|
||||
chapter_id: Uuid,
|
||||
img: &ChapterImage,
|
||||
allowlist: &DownloadAllowlist,
|
||||
max_image_bytes: usize,
|
||||
) -> anyhow::Result<StoredPage> {
|
||||
use futures_util::StreamExt as _;
|
||||
let url = base
|
||||
.join(&img.url)
|
||||
.with_context(|| format!("join image URL {} onto {source_url}", img.url))?;
|
||||
rate.wait_for(url.as_str()).await?;
|
||||
let resp = fetch_stream(http, url.as_str(), Some(source_url), allowlist).await?;
|
||||
let mut body = resp.bytes_stream();
|
||||
|
||||
// Drain chunks until we have enough bytes to sniff confidently
|
||||
// (or the body is shorter than the prefix). Enforces the per-image
|
||||
// cap on the prefix accumulation too.
|
||||
let mut prefix = bytes::BytesMut::new();
|
||||
while prefix.len() < SNIFF_PREFIX_BYTES {
|
||||
let Some(chunk) = body.next().await else { break };
|
||||
let chunk = chunk
|
||||
.with_context(|| format!("stream chunk for {url}"))?;
|
||||
if prefix.len().saturating_add(chunk.len()) > max_image_bytes {
|
||||
anyhow::bail!(
|
||||
"image {url} exceeds {max_image_bytes}-byte cap (received >{}+{})",
|
||||
prefix.len(),
|
||||
chunk.len()
|
||||
);
|
||||
}
|
||||
prefix.extend_from_slice(&chunk);
|
||||
}
|
||||
let prefix = prefix.freeze();
|
||||
|
||||
// Reject any non-image response: the only valid output of an image
|
||||
// URL is an image. `infer` returns None on truncated bytes too,
|
||||
// which is also a failure not a silent `.bin` extension.
|
||||
if !looks_like_image(&prefix) {
|
||||
anyhow::bail!(
|
||||
"image URL {url} returned non-image bytes \
|
||||
(first 16: {:?}); refusing to store as binary blob",
|
||||
&prefix.get(..16.min(prefix.len()))
|
||||
);
|
||||
storage
|
||||
.put(&key, bytes)
|
||||
.await
|
||||
.with_context(|| format!("put {key}"))?;
|
||||
// (chapter_id, page_number) is unique — re-runs idempotent.
|
||||
sqlx::query(
|
||||
}
|
||||
let ext = infer::get(&prefix)
|
||||
.map(|k| k.extension())
|
||||
.expect("looks_like_image asserted infer succeeded");
|
||||
let key = format!(
|
||||
"mangas/{manga_id}/chapters/{chapter_id}/pages/{:04}.{ext}",
|
||||
img.page_number
|
||||
);
|
||||
|
||||
// Build a single stream of (prefix + remaining body) and pipe it
|
||||
// straight to storage. The cap is enforced via a running total in
|
||||
// the stream adapter so a server that omits Content-Length still
|
||||
// can't exhaust memory.
|
||||
let prefix_stream = futures_util::stream::once(async move {
|
||||
Ok::<bytes::Bytes, StorageError>(prefix)
|
||||
});
|
||||
let prefix_len = SNIFF_PREFIX_BYTES.min(max_image_bytes);
|
||||
let mut remaining = max_image_bytes.saturating_sub(prefix_len);
|
||||
let url_for_err = url.clone();
|
||||
let rest_stream = body.map(move |frame| match frame {
|
||||
Ok(chunk) => {
|
||||
if chunk.len() > remaining {
|
||||
return Err(StorageError::Io(std::io::Error::other(format!(
|
||||
"image {url_for_err} exceeds {max_image_bytes}-byte cap"
|
||||
))));
|
||||
}
|
||||
remaining -= chunk.len();
|
||||
Ok(chunk)
|
||||
}
|
||||
Err(e) => Err(StorageError::Io(std::io::Error::other(format!(
|
||||
"stream chunk for {url_for_err}: {e}"
|
||||
)))),
|
||||
});
|
||||
let combined = prefix_stream.chain(rest_stream);
|
||||
storage
|
||||
.put_stream(&key, Box::pin(combined))
|
||||
.await
|
||||
.with_context(|| format!("put_stream {key}"))?;
|
||||
Ok(StoredPage {
|
||||
page_number: img.page_number,
|
||||
storage_key: key,
|
||||
content_type: format!("image/{ext}"),
|
||||
})
|
||||
}
|
||||
|
||||
/// Persist the page rows + chapter `page_count` in one short transaction.
|
||||
/// `(chapter_id, page_number)` is unique so re-runs are idempotent.
|
||||
pub(crate) async fn persist_pages(
|
||||
db: &PgPool,
|
||||
chapter_id: Uuid,
|
||||
stored: &[StoredPage],
|
||||
enqueue_analysis: bool,
|
||||
) -> anyhow::Result<()> {
|
||||
let mut tx = db.begin().await.context("open chapter sync tx")?;
|
||||
let mut page_ids: Vec<Uuid> = Vec::with_capacity(stored.len());
|
||||
for page in stored {
|
||||
let (id,): (Uuid,) = sqlx::query_as(
|
||||
"INSERT INTO pages (chapter_id, page_number, storage_key, content_type)
|
||||
VALUES ($1, $2, $3, $4)
|
||||
ON CONFLICT (chapter_id, page_number) DO UPDATE
|
||||
SET storage_key = EXCLUDED.storage_key,
|
||||
content_type = EXCLUDED.content_type",
|
||||
content_type = EXCLUDED.content_type
|
||||
RETURNING id",
|
||||
)
|
||||
.bind(chapter_id)
|
||||
.bind(page_number)
|
||||
.bind(&key)
|
||||
.bind(format!("image/{ext}"))
|
||||
.execute(&mut *tx)
|
||||
.bind(page.page_number)
|
||||
.bind(&page.storage_key)
|
||||
.bind(&page.content_type)
|
||||
.fetch_one(&mut *tx)
|
||||
.await
|
||||
.with_context(|| format!("insert page row {page_number}"))?;
|
||||
.with_context(|| format!("insert page row {}", page.page_number))?;
|
||||
page_ids.push(id);
|
||||
}
|
||||
sqlx::query("UPDATE chapters SET page_count = $1 WHERE id = $2")
|
||||
.bind(fetched.len() as i32)
|
||||
.bind(stored.len() as i32)
|
||||
.bind(chapter_id)
|
||||
.execute(&mut *tx)
|
||||
.await
|
||||
.context("update page_count")?;
|
||||
tx.commit().await.context("commit chapter sync")?;
|
||||
|
||||
Ok(SyncOutcome::Fetched { pages: fetched.len() })
|
||||
// Enqueue AI content-analysis for the crawled pages once their rows
|
||||
// are committed. Best-effort: a failed enqueue is logged, never fatal
|
||||
// (the admin re-enqueue endpoint can backfill). Gated by the caller so
|
||||
// jobs don't pile up when the analysis worker is disabled.
|
||||
if enqueue_analysis {
|
||||
for page_id in page_ids {
|
||||
if let Err(e) =
|
||||
crate::repo::page_analysis::enqueue_for_page(db, page_id, false).await
|
||||
{
|
||||
tracing::warn!(%page_id, error = %e, "failed to enqueue page analysis after crawl");
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Best-effort delete of partially-written page blobs after a chapter sync
|
||||
/// fails, so a retry doesn't accumulate orphans. Errors are logged, not
|
||||
/// raised — a leftover blob is harmless and a future reaper can sweep it.
|
||||
pub(crate) async fn cleanup_orphans(storage: &dyn Storage, keys: &[String]) {
|
||||
for key in keys {
|
||||
if let Err(e) = storage.delete(key).await {
|
||||
tracing::warn!(
|
||||
%key,
|
||||
error = ?e,
|
||||
"failed to delete orphaned page blob after chapter sync failure"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Suppress unused-import warning for `session::registrable_domain`
|
||||
@@ -347,6 +526,132 @@ fn _keep_session_in_scope() {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::storage::LocalStorage;
|
||||
|
||||
#[tokio::test]
|
||||
async fn cleanup_orphans_deletes_written_keys() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let storage = LocalStorage::new(dir.path());
|
||||
let keys = vec![
|
||||
"mangas/m/chapters/c/pages/0001.jpg".to_string(),
|
||||
"mangas/m/chapters/c/pages/0002.jpg".to_string(),
|
||||
];
|
||||
for k in &keys {
|
||||
storage.put(k, b"\xff\xd8\xff\xe0 jpeg-ish").await.unwrap();
|
||||
assert!(storage.exists(k).await.unwrap());
|
||||
}
|
||||
cleanup_orphans(&storage, &keys).await;
|
||||
for k in &keys {
|
||||
assert!(!storage.exists(k).await.unwrap(), "{k} should be deleted");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cleanup_orphans_tolerates_missing_keys() {
|
||||
// A key that was never written (e.g. the put itself failed) must
|
||||
// not make cleanup error — it's best-effort.
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let storage = LocalStorage::new(dir.path());
|
||||
cleanup_orphans(&storage, &["never/written.jpg".to_string()]).await;
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn persist_pages_inserts_rows_and_sets_page_count(pool: PgPool) {
|
||||
let manga_id = Uuid::new_v4();
|
||||
let chapter_id = Uuid::new_v4();
|
||||
sqlx::query("INSERT INTO mangas (id, title) VALUES ($1, 'T')")
|
||||
.bind(manga_id)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("INSERT INTO chapters (id, manga_id, number) VALUES ($1, $2, 1)")
|
||||
.bind(chapter_id)
|
||||
.bind(manga_id)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let stored = vec![
|
||||
StoredPage {
|
||||
page_number: 1,
|
||||
storage_key: "k/0001.jpg".into(),
|
||||
content_type: "image/jpeg".into(),
|
||||
},
|
||||
StoredPage {
|
||||
page_number: 2,
|
||||
storage_key: "k/0002.jpg".into(),
|
||||
content_type: "image/jpeg".into(),
|
||||
},
|
||||
];
|
||||
persist_pages(&pool, chapter_id, &stored, false).await.unwrap();
|
||||
|
||||
let page_count: i32 =
|
||||
sqlx::query_scalar("SELECT page_count FROM chapters WHERE id = $1")
|
||||
.bind(chapter_id)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(page_count, 2);
|
||||
let rows: i64 =
|
||||
sqlx::query_scalar("SELECT COUNT(*) FROM pages WHERE chapter_id = $1")
|
||||
.bind(chapter_id)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(rows, 2);
|
||||
|
||||
// Idempotent re-run (force refetch path): same rows, page_count stable.
|
||||
persist_pages(&pool, chapter_id, &stored, false).await.unwrap();
|
||||
let rows2: i64 =
|
||||
sqlx::query_scalar("SELECT COUNT(*) FROM pages WHERE chapter_id = $1")
|
||||
.bind(chapter_id)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(rows2, 2, "re-run is idempotent via ON CONFLICT");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn persist_pages_enqueues_analysis_only_when_flag_set(pool: PgPool) {
|
||||
let manga_id = Uuid::new_v4();
|
||||
let chapter_id = Uuid::new_v4();
|
||||
sqlx::query("INSERT INTO mangas (id, title) VALUES ($1, 'T')")
|
||||
.bind(manga_id)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("INSERT INTO chapters (id, manga_id, number) VALUES ($1, $2, 1)")
|
||||
.bind(chapter_id)
|
||||
.bind(manga_id)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let stored = vec![StoredPage {
|
||||
page_number: 1,
|
||||
storage_key: "k/0001.jpg".into(),
|
||||
content_type: "image/jpeg".into(),
|
||||
}];
|
||||
|
||||
// Flag off: no analyze_page jobs.
|
||||
persist_pages(&pool, chapter_id, &stored, false).await.unwrap();
|
||||
let jobs_off: i64 = sqlx::query_scalar(
|
||||
"SELECT COUNT(*) FROM crawler_jobs WHERE payload->>'kind' = 'analyze_page'",
|
||||
)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(jobs_off, 0, "flag off must not enqueue analysis");
|
||||
|
||||
// Flag on: one analyze_page job for the upserted page.
|
||||
persist_pages(&pool, chapter_id, &stored, true).await.unwrap();
|
||||
let jobs_on: i64 = sqlx::query_scalar(
|
||||
"SELECT COUNT(*) FROM crawler_jobs WHERE payload->>'kind' = 'analyze_page'",
|
||||
)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(jobs_on, 1, "flag on enqueues one job per persisted page");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_chapter_pages_skips_loader_and_sorts_by_id() {
|
||||
|
||||
@@ -48,6 +48,7 @@ use tokio_util::sync::CancellationToken;
|
||||
use crate::crawler::content::SyncOutcome;
|
||||
use crate::crawler::jobs::{self, JobPayload, Lease, KIND_SYNC_CHAPTER_CONTENT};
|
||||
use crate::crawler::pipeline;
|
||||
use crate::crawler::status::{Phase, StatusHandle};
|
||||
|
||||
/// Fixed `pg_try_advisory_lock` key. ASCII "MANGALRD" interpreted as a
|
||||
/// big-endian i64. Hardcoded so every replica agrees on the lock identity
|
||||
@@ -56,6 +57,15 @@ pub const CRON_LOCK_KEY: i64 = 0x4D414E47414C5244;
|
||||
|
||||
const STATE_KEY_LAST_TICK: &str = "last_metadata_tick_at";
|
||||
|
||||
/// Lease window handed to `jobs::lease`. Kept short, but continuously
|
||||
/// extended by the per-job heartbeat (see [`WorkerContext::process_lease`])
|
||||
/// so a long-but-healthy job never lapses and gets stolen.
|
||||
const LEASE_DURATION: Duration = Duration::from_secs(60);
|
||||
|
||||
/// How often the heartbeat renews the lease while a job runs. A third of
|
||||
/// the lease window leaves two missed-beat's slack before expiry.
|
||||
const LEASE_HEARTBEAT: Duration = Duration::from_secs(20);
|
||||
|
||||
#[async_trait]
|
||||
pub trait MetadataPass: Send + Sync {
|
||||
async fn run(&self) -> anyhow::Result<pipeline::MetadataStats>;
|
||||
@@ -77,6 +87,13 @@ pub struct DaemonConfig {
|
||||
pub tz: Tz,
|
||||
pub retention_days: u32,
|
||||
pub session_expired: Arc<AtomicBool>,
|
||||
/// Live status surface updated by the cron + workers.
|
||||
pub status: StatusHandle,
|
||||
/// Hard upper bound on a single job's dispatch. A job that exceeds it
|
||||
/// is acked failed (exponential backoff) rather than wedging a worker
|
||||
/// forever. Must exceed [`LEASE_HEARTBEAT`] and the realistic
|
||||
/// single-job runtime.
|
||||
pub job_timeout: Duration,
|
||||
/// Tasks that should run alongside the cron + workers and be cancelled
|
||||
/// on shutdown. Used to hand the daemon ownership of the browser
|
||||
/// manager's idle reaper.
|
||||
@@ -123,6 +140,8 @@ pub fn spawn(pool: PgPool, cancel: CancellationToken, cfg: DaemonConfig) -> Daem
|
||||
tz,
|
||||
retention_days,
|
||||
session_expired,
|
||||
status,
|
||||
job_timeout,
|
||||
extra_tasks,
|
||||
} = cfg;
|
||||
|
||||
@@ -134,6 +153,7 @@ pub fn spawn(pool: PgPool, cancel: CancellationToken, cfg: DaemonConfig) -> Daem
|
||||
tz,
|
||||
retention_days,
|
||||
metadata,
|
||||
status: status.clone(),
|
||||
};
|
||||
join.spawn(async move { ctx.run().await });
|
||||
} else {
|
||||
@@ -146,6 +166,8 @@ pub fn spawn(pool: PgPool, cancel: CancellationToken, cfg: DaemonConfig) -> Daem
|
||||
cancel: cancel.clone(),
|
||||
dispatcher: Arc::clone(&dispatcher),
|
||||
session_expired: Arc::clone(&session_expired),
|
||||
status: status.clone(),
|
||||
job_timeout,
|
||||
id: worker_id,
|
||||
};
|
||||
join.spawn(async move { ctx.run().await });
|
||||
@@ -169,6 +191,7 @@ struct CronContext {
|
||||
tz: Tz,
|
||||
retention_days: u32,
|
||||
metadata: Arc<dyn MetadataPass>,
|
||||
status: StatusHandle,
|
||||
}
|
||||
|
||||
impl CronContext {
|
||||
@@ -196,6 +219,11 @@ impl CronContext {
|
||||
// (NTP step, suspend/resume) don't strand us on a stale instant.
|
||||
let next = next_fire(Utc::now(), self.daily_at, self.tz);
|
||||
let wait = (next - Utc::now()).to_std().unwrap_or(Duration::ZERO);
|
||||
self.status
|
||||
.set_phase(Phase::Idle {
|
||||
next_fire: Some(next),
|
||||
})
|
||||
.await;
|
||||
tracing::info!(
|
||||
next_fire_utc = %next.to_rfc3339(),
|
||||
wait_seconds = wait.as_secs(),
|
||||
@@ -243,9 +271,13 @@ impl CronContext {
|
||||
let metadata = &self.metadata;
|
||||
let pool = &self.pool;
|
||||
let retention_days = self.retention_days;
|
||||
let status = &self.status;
|
||||
let body = async move {
|
||||
match metadata.run().await {
|
||||
Ok(stats) => tracing::info!(?stats, "cron: metadata pass done"),
|
||||
Ok(stats) => {
|
||||
status.record_pass(&stats, Utc::now()).await;
|
||||
tracing::info!(?stats, "cron: metadata pass done");
|
||||
}
|
||||
Err(e) => tracing::error!(?e, "cron: metadata pass failed"),
|
||||
}
|
||||
match pipeline::enqueue_bookmarked_pending(pool).await {
|
||||
@@ -283,6 +315,8 @@ struct WorkerContext {
|
||||
cancel: CancellationToken,
|
||||
dispatcher: Arc<dyn ChapterDispatcher>,
|
||||
session_expired: Arc<AtomicBool>,
|
||||
status: StatusHandle,
|
||||
job_timeout: Duration,
|
||||
id: usize,
|
||||
}
|
||||
|
||||
@@ -303,7 +337,7 @@ impl WorkerContext {
|
||||
&self.pool,
|
||||
Some(KIND_SYNC_CHAPTER_CONTENT),
|
||||
1,
|
||||
Duration::from_secs(60),
|
||||
LEASE_DURATION,
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -341,9 +375,59 @@ impl WorkerContext {
|
||||
}
|
||||
}
|
||||
|
||||
let outcome = AssertUnwindSafe(self.dispatcher.dispatch(lease.payload.clone()))
|
||||
.catch_unwind()
|
||||
.await;
|
||||
// Heartbeat: keep the lease fresh while the (potentially long)
|
||||
// dispatch runs, so a slow-but-healthy job is never re-leased and
|
||||
// never inflates `attempts` toward `max_attempts`. Stops itself
|
||||
// once the job is no longer ours (renew returns false).
|
||||
let heartbeat = {
|
||||
let hb_pool = self.pool.clone();
|
||||
let hb_id = lease.id;
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
tokio::time::sleep(LEASE_HEARTBEAT).await;
|
||||
match jobs::renew(&hb_pool, hb_id, LEASE_DURATION).await {
|
||||
Ok(true) => {}
|
||||
Ok(false) => break,
|
||||
Err(e) => {
|
||||
tracing::warn!(lease_id = %hb_id, ?e, "heartbeat renew failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
};
|
||||
|
||||
// The "currently crawling" chapter (with its live page count) is
|
||||
// registered by the dispatcher itself (RealChapterDispatcher) so it
|
||||
// carries the manga/chapter identity + page progress and is removed
|
||||
// via an RAII guard on every exit path.
|
||||
|
||||
// Outer timeout: a dispatch that exceeds `job_timeout` is acked
|
||||
// failed (exponential backoff) rather than wedging the worker.
|
||||
let dispatch = AssertUnwindSafe(self.dispatcher.dispatch(lease.payload.clone()))
|
||||
.catch_unwind();
|
||||
let outcome = tokio::time::timeout(self.job_timeout, dispatch).await;
|
||||
heartbeat.abort();
|
||||
|
||||
let outcome = match outcome {
|
||||
Ok(o) => o,
|
||||
Err(_elapsed) => {
|
||||
tracing::warn!(
|
||||
worker = self.id,
|
||||
lease_id = %lease.id,
|
||||
timeout_secs = self.job_timeout.as_secs(),
|
||||
"worker: dispatch timed out — ack failed"
|
||||
);
|
||||
let _ = jobs::ack_failed(
|
||||
&self.pool,
|
||||
lease.id,
|
||||
"dispatch timed out",
|
||||
lease.attempts,
|
||||
lease.max_attempts,
|
||||
)
|
||||
.await;
|
||||
return;
|
||||
}
|
||||
};
|
||||
match outcome {
|
||||
Ok(Ok(SyncOutcome::Fetched { .. } | SyncOutcome::Skipped)) => {
|
||||
let _ = jobs::ack_done(&self.pool, lease.id).await;
|
||||
@@ -355,6 +439,8 @@ impl WorkerContext {
|
||||
"session expired — workers will idle until restart"
|
||||
);
|
||||
self.session_expired.store(true, Ordering::Release);
|
||||
// Push the session-expired flip to live status subscribers.
|
||||
self.status.poke();
|
||||
let _ = jobs::release(&self.pool, lease.id).await;
|
||||
}
|
||||
Ok(Err(e)) => {
|
||||
|
||||
@@ -34,6 +34,15 @@ pub enum JobPayload {
|
||||
chapter_id: Uuid,
|
||||
source_chapter_key: String,
|
||||
},
|
||||
/// Run AI content-analysis (OCR, auto-tags, scene description, NSFW
|
||||
/// moderation) on a single page image via the local vision model.
|
||||
/// `force` re-analyzes a page that is already `done` (manual admin
|
||||
/// re-trigger); otherwise the worker skips it.
|
||||
AnalyzePage {
|
||||
page_id: Uuid,
|
||||
#[serde(default)]
|
||||
force: bool,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, sqlx::Type, Serialize, Deserialize)]
|
||||
@@ -52,6 +61,10 @@ pub enum JobState {
|
||||
/// without re-spelling the literal.
|
||||
pub const KIND_SYNC_CHAPTER_CONTENT: &str = "sync_chapter_content";
|
||||
|
||||
/// Kind discriminator for AI page-analysis jobs. The analysis daemon
|
||||
/// leases with this filter so it never contends with crawl jobs.
|
||||
pub const KIND_ANALYZE_PAGE: &str = "analyze_page";
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum EnqueueResult {
|
||||
Inserted(Uuid),
|
||||
@@ -66,16 +79,33 @@ pub struct Lease {
|
||||
pub max_attempts: i32,
|
||||
}
|
||||
|
||||
/// Exponential backoff for `ack_failed` retries. `attempts` is the
|
||||
/// post-increment value reported by `lease()` (so the first failure has
|
||||
/// `attempts == 1` and waits 60s, the second 120s, etc.). Capped at 1h to
|
||||
/// avoid runaway long sleeps that would outlive the daemon process.
|
||||
fn backoff_for(attempts: i32) -> Duration {
|
||||
/// Deterministic exponential backoff base for `ack_failed` retries.
|
||||
/// `attempts` is the post-increment value reported by `lease()` (so the
|
||||
/// first failure has `attempts == 1` and waits 60s, the second 120s,
|
||||
/// etc.). Capped at 1h to avoid runaway long sleeps that would outlive
|
||||
/// the daemon process. Jitter is applied separately by [`apply_jitter`].
|
||||
fn backoff_base(attempts: i32) -> Duration {
|
||||
let shift = attempts.saturating_sub(1).clamp(0, 20) as u32;
|
||||
let secs = 60u64.saturating_mul(1u64 << shift);
|
||||
Duration::from_secs(secs.min(3600))
|
||||
}
|
||||
|
||||
/// Apply ±20% jitter to a backoff duration. `jitter` is a fraction in
|
||||
/// `[0.0, 1.0)` (e.g. `rand::random::<f64>()`), mapped to a multiplier in
|
||||
/// `[0.8, 1.2)`. Pure so the bounds stay unit-testable. Spreading retries
|
||||
/// avoids a thundering herd when a source outage fails many jobs at once.
|
||||
fn apply_jitter(base: Duration, jitter: f64) -> Duration {
|
||||
let frac = jitter.clamp(0.0, 1.0);
|
||||
let mult = 0.8 + 0.4 * frac; // [0.8, 1.2)
|
||||
Duration::from_secs((base.as_secs_f64() * mult).round() as u64)
|
||||
}
|
||||
|
||||
/// Jittered exponential backoff for `ack_failed`. Wraps [`backoff_base`]
|
||||
/// with a random ±20% spread.
|
||||
fn backoff_for(attempts: i32) -> Duration {
|
||||
apply_jitter(backoff_base(attempts), rand::random::<f64>())
|
||||
}
|
||||
|
||||
/// Insert a new pending job. For `SyncChapterContent` payloads the
|
||||
/// partial unique index `crawler_jobs_chapter_content_dedup_idx` blocks
|
||||
/// a second `(pending|running)` insert per chapter_id, returning
|
||||
@@ -159,6 +189,35 @@ pub async fn lease(
|
||||
Ok(leases)
|
||||
}
|
||||
|
||||
/// Extend the lease on a still-owned `running` job. Returns `true` if the
|
||||
/// row was updated (we still hold the lease), `false` if the job is no
|
||||
/// longer `running` (re-leased after a missed heartbeat, or already
|
||||
/// acked) — the caller's heartbeat loop should stop. The `state =
|
||||
/// 'running'` guard mirrors [`ack_done`]'s rationale.
|
||||
///
|
||||
/// This is the heartbeat primitive: a worker renews periodically while a
|
||||
/// long-but-healthy job runs so `leased_until` never lapses, which would
|
||||
/// otherwise let another worker steal the in-flight job and spuriously
|
||||
/// inflate `attempts` toward `max_attempts`.
|
||||
pub async fn renew(
|
||||
pool: &PgPool,
|
||||
lease_id: Uuid,
|
||||
lease_duration: Duration,
|
||||
) -> sqlx::Result<bool> {
|
||||
let lease_ms: i64 = lease_duration.as_millis().min(i64::MAX as u128) as i64;
|
||||
let res = sqlx::query(
|
||||
"UPDATE crawler_jobs \
|
||||
SET leased_until = now() + ($2::bigint || ' milliseconds')::interval, \
|
||||
updated_at = now() \
|
||||
WHERE id = $1 AND state = 'running'",
|
||||
)
|
||||
.bind(lease_id)
|
||||
.bind(lease_ms)
|
||||
.execute(pool)
|
||||
.await?;
|
||||
Ok(res.rows_affected() > 0)
|
||||
}
|
||||
|
||||
/// Mark a leased job as successfully completed. The `state = 'running'`
|
||||
/// predicate guards against a late ack from a worker whose lease expired
|
||||
/// and was already re-leased by another worker: without it, the late ack
|
||||
@@ -278,19 +337,71 @@ mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn backoff_grows_exponentially_and_caps_at_one_hour() {
|
||||
fn analyze_page_payload_round_trips_through_tagged_json() {
|
||||
let page_id = Uuid::new_v4();
|
||||
let payload = JobPayload::AnalyzePage {
|
||||
page_id,
|
||||
force: true,
|
||||
};
|
||||
let json = serde_json::to_value(&payload).unwrap();
|
||||
assert_eq!(json["kind"], "analyze_page");
|
||||
assert_eq!(json["page_id"], page_id.to_string());
|
||||
assert_eq!(json["force"], true);
|
||||
|
||||
// `force` defaults to false when an older enqueue omitted it.
|
||||
let legacy = serde_json::json!({ "kind": "analyze_page", "page_id": page_id });
|
||||
match serde_json::from_value::<JobPayload>(legacy).unwrap() {
|
||||
JobPayload::AnalyzePage { page_id: pid, force } => {
|
||||
assert_eq!(pid, page_id);
|
||||
assert!(!force);
|
||||
}
|
||||
other => panic!("expected AnalyzePage, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn backoff_base_grows_exponentially_and_caps_at_one_hour() {
|
||||
// attempts == 1 → 60s, doubling each step.
|
||||
assert_eq!(backoff_for(1), Duration::from_secs(60));
|
||||
assert_eq!(backoff_for(2), Duration::from_secs(120));
|
||||
assert_eq!(backoff_for(3), Duration::from_secs(240));
|
||||
assert_eq!(backoff_for(4), Duration::from_secs(480));
|
||||
assert_eq!(backoff_for(5), Duration::from_secs(960));
|
||||
assert_eq!(backoff_for(6), Duration::from_secs(1920));
|
||||
assert_eq!(backoff_base(1), Duration::from_secs(60));
|
||||
assert_eq!(backoff_base(2), Duration::from_secs(120));
|
||||
assert_eq!(backoff_base(3), Duration::from_secs(240));
|
||||
assert_eq!(backoff_base(4), Duration::from_secs(480));
|
||||
assert_eq!(backoff_base(5), Duration::from_secs(960));
|
||||
assert_eq!(backoff_base(6), Duration::from_secs(1920));
|
||||
// 7th: 60 * 64 = 3840 → capped to 3600.
|
||||
assert_eq!(backoff_for(7), Duration::from_secs(3600));
|
||||
assert_eq!(backoff_for(20), Duration::from_secs(3600));
|
||||
assert_eq!(backoff_base(7), Duration::from_secs(3600));
|
||||
assert_eq!(backoff_base(20), Duration::from_secs(3600));
|
||||
// Garbage / zero / negatives stay sane.
|
||||
assert_eq!(backoff_for(0), Duration::from_secs(60));
|
||||
assert_eq!(backoff_for(-5), Duration::from_secs(60));
|
||||
assert_eq!(backoff_base(0), Duration::from_secs(60));
|
||||
assert_eq!(backoff_base(-5), Duration::from_secs(60));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn apply_jitter_stays_within_plus_minus_twenty_percent() {
|
||||
let base = Duration::from_secs(100);
|
||||
// Lower bound (jitter = 0.0) → 0.8x.
|
||||
assert_eq!(apply_jitter(base, 0.0), Duration::from_secs(80));
|
||||
// Midpoint (jitter = 0.5) → 1.0x.
|
||||
assert_eq!(apply_jitter(base, 0.5), Duration::from_secs(100));
|
||||
// Upper end (jitter → 1.0) → ~1.2x.
|
||||
assert_eq!(apply_jitter(base, 1.0), Duration::from_secs(120));
|
||||
// Out-of-range inputs are clamped, never panic.
|
||||
assert_eq!(apply_jitter(base, -3.0), Duration::from_secs(80));
|
||||
assert_eq!(apply_jitter(base, 9.0), Duration::from_secs(120));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn backoff_for_random_jitter_stays_in_band() {
|
||||
// The production wrapper draws its own randomness; assert the
|
||||
// result for a mid-range attempt always lands within the jitter
|
||||
// band of the base, across many draws.
|
||||
let base = backoff_base(3).as_secs_f64(); // 240s
|
||||
for _ in 0..1000 {
|
||||
let v = backoff_for(3).as_secs_f64();
|
||||
assert!(
|
||||
v >= base * 0.8 - 1.0 && v <= base * 1.2 + 1.0,
|
||||
"jittered backoff {v} outside band of base {base}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,6 +26,8 @@ pub mod rate_limit;
|
||||
pub mod resync;
|
||||
pub mod safety;
|
||||
pub mod session;
|
||||
pub mod session_control;
|
||||
pub mod source;
|
||||
pub mod status;
|
||||
pub mod tor;
|
||||
pub mod url_utils;
|
||||
|
||||
@@ -65,6 +65,17 @@ pub(crate) fn should_mark_clean_exit(
|
||||
walked_to_completion || hit_stop_condition
|
||||
}
|
||||
|
||||
/// Circuit-breaker: abort the walk once `consecutive` `fetch_manga`
|
||||
/// failures reach `threshold`. A `threshold` of 0 disables the breaker
|
||||
/// (unbounded — the legacy behaviour). When it fires the caller must NOT
|
||||
/// mark a clean exit, so the next tick does a recovery sweep over the
|
||||
/// catalog tail the aborted pass never reached.
|
||||
///
|
||||
/// Pure so the rule is unit-testable without the walker.
|
||||
pub(crate) fn should_abort_pass(consecutive: u32, threshold: u32) -> bool {
|
||||
threshold > 0 && consecutive >= threshold
|
||||
}
|
||||
|
||||
/// Runs the discover → fetch → upsert → cover → chapter-list-diff pipeline
|
||||
/// for the target source. Pure metadata; chapter content is enqueued as
|
||||
/// separate `SyncChapterContent` jobs by the caller after this returns.
|
||||
@@ -103,6 +114,8 @@ pub async fn run_metadata_pass(
|
||||
skip_chapters: bool,
|
||||
allowlist: &DownloadAllowlist,
|
||||
max_image_bytes: usize,
|
||||
max_consecutive_failures: u32,
|
||||
status: Option<&crate::crawler::status::StatusHandle>,
|
||||
tor: Option<&crate::crawler::tor::TorController>,
|
||||
) -> anyhow::Result<MetadataStats> {
|
||||
let lease = browser_manager
|
||||
@@ -110,6 +123,9 @@ pub async fn run_metadata_pass(
|
||||
.await
|
||||
.context("acquire browser lease for metadata pass")?;
|
||||
let browser_ref: &chromiumoxide::Browser = &lease;
|
||||
if let Some(s) = status {
|
||||
s.set_phase(crate::crawler::status::Phase::WalkingList).await;
|
||||
}
|
||||
|
||||
let source = {
|
||||
let s = TargetSource::new(start_url.to_string());
|
||||
@@ -165,6 +181,11 @@ pub async fn run_metadata_pass(
|
||||
let mut walked_to_completion = false;
|
||||
let mut hit_limit = false;
|
||||
let mut hit_stop_condition = false;
|
||||
// Circuit-breaker state: consecutive fetch_manga failures. A sustained
|
||||
// run abort (source outage) leaves the pass un-clean → recovery sweep
|
||||
// next tick.
|
||||
let mut consecutive_failures = 0u32;
|
||||
let mut hit_failure_breaker = false;
|
||||
|
||||
'outer: loop {
|
||||
let batch = match walker.next_batch(&ctx).await? {
|
||||
@@ -175,6 +196,17 @@ pub async fn run_metadata_pass(
|
||||
}
|
||||
};
|
||||
for r in batch {
|
||||
// Cooperative checkpoint: if a coordinated browser restart is
|
||||
// pending, yield our (long-lived) lease so the drain can
|
||||
// proceed instead of stalling for the rest of the walk. The
|
||||
// pass exits un-clean, so the next tick recovery-sweeps the
|
||||
// tail we didn't reach.
|
||||
if browser_manager.is_restart_pending() {
|
||||
tracing::info!(
|
||||
"metadata pass: browser restart pending — yielding (recovery sweep next tick)"
|
||||
);
|
||||
break 'outer;
|
||||
}
|
||||
if max_refs.map(|m| stats.discovered >= m).unwrap_or(false) {
|
||||
hit_limit = true;
|
||||
tracing::info!(cap = ?max_refs, "max_results reached; halting walk");
|
||||
@@ -198,13 +230,24 @@ pub async fn run_metadata_pass(
|
||||
continue;
|
||||
}
|
||||
stats.discovered += 1;
|
||||
if let Some(s) = status {
|
||||
s.set_phase(crate::crawler::status::Phase::FetchingMetadata {
|
||||
index: stats.discovered,
|
||||
total: max_refs,
|
||||
title: r.title.clone(),
|
||||
})
|
||||
.await;
|
||||
}
|
||||
tracing::info!(
|
||||
idx = stats.discovered,
|
||||
key = %r.source_manga_key,
|
||||
"fetching metadata"
|
||||
);
|
||||
let manga = match source.fetch_manga(&ctx, &r).await {
|
||||
Ok(m) => m,
|
||||
Ok(m) => {
|
||||
consecutive_failures = 0;
|
||||
m
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
key = %r.source_manga_key,
|
||||
@@ -213,6 +256,17 @@ pub async fn run_metadata_pass(
|
||||
"fetch_manga failed"
|
||||
);
|
||||
stats.mangas_failed += 1;
|
||||
consecutive_failures += 1;
|
||||
if should_abort_pass(consecutive_failures, max_consecutive_failures) {
|
||||
hit_failure_breaker = true;
|
||||
tracing::error!(
|
||||
consecutive_failures,
|
||||
threshold = max_consecutive_failures,
|
||||
"metadata pass: too many consecutive fetch_manga failures; \
|
||||
aborting (recovery sweep on next tick)"
|
||||
);
|
||||
break 'outer;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
};
|
||||
@@ -295,7 +349,16 @@ pub async fn run_metadata_pass(
|
||||
|| matches!(upsert.status, repo::crawler::UpsertStatus::Updated);
|
||||
if needs_cover {
|
||||
if let Some(cover_url) = manga.cover_url.as_deref() {
|
||||
match download_and_store_cover(
|
||||
// RAII: the guard clears `current_cover` on every
|
||||
// exit path (success, panic, future early-return).
|
||||
// Mirrors the chapter-side ChapterGuard.
|
||||
let _cover_guard = status.map(|s| {
|
||||
s.begin_cover(crate::crawler::status::CoverTarget {
|
||||
manga_id: upsert.manga_id,
|
||||
manga_title: manga.title.clone(),
|
||||
})
|
||||
});
|
||||
let cover_result = download_and_store_cover(
|
||||
db,
|
||||
storage,
|
||||
http,
|
||||
@@ -306,8 +369,8 @@ pub async fn run_metadata_pass(
|
||||
allowlist,
|
||||
max_image_bytes,
|
||||
)
|
||||
.await
|
||||
{
|
||||
.await;
|
||||
match cover_result {
|
||||
Ok(()) => stats.covers_fetched += 1,
|
||||
Err(e) => tracing::warn!(
|
||||
manga_id = %upsert.manga_id,
|
||||
@@ -390,6 +453,7 @@ pub async fn run_metadata_pass(
|
||||
walked_to_completion,
|
||||
hit_limit,
|
||||
hit_stop_condition,
|
||||
hit_failure_breaker,
|
||||
exited_cleanly,
|
||||
"metadata pass complete"
|
||||
);
|
||||
@@ -560,6 +624,7 @@ pub async fn backfill_missing_covers(
|
||||
max_mangas: usize,
|
||||
allowlist: &DownloadAllowlist,
|
||||
max_image_bytes: usize,
|
||||
status: Option<&crate::crawler::status::StatusHandle>,
|
||||
tor: Option<&crate::crawler::tor::TorController>,
|
||||
) -> anyhow::Result<CoverBackfillStats> {
|
||||
let mut stats = CoverBackfillStats::default();
|
||||
@@ -582,8 +647,13 @@ pub async fn backfill_missing_covers(
|
||||
let browser_ref: &chromiumoxide::Browser = &lease;
|
||||
let ctx = FetchContext { browser: browser_ref, rate, tor };
|
||||
|
||||
for entry in entries {
|
||||
let total = entries.len();
|
||||
for (index, entry) in entries.into_iter().enumerate() {
|
||||
stats.considered += 1;
|
||||
if let Some(s) = status {
|
||||
s.set_phase(crate::crawler::status::Phase::CoverBackfill { index, total })
|
||||
.await;
|
||||
}
|
||||
// Metadata-only TargetSource: skip chapter-list parsing so a
|
||||
// missing-cover refetch doesn't soft-drop chapters on a partial
|
||||
// render. Cover URL alone is what we need.
|
||||
@@ -593,8 +663,8 @@ pub async fn backfill_missing_covers(
|
||||
title: String::new(),
|
||||
url: entry.source_url.clone(),
|
||||
};
|
||||
let cover_url = match source.fetch_manga(&ctx, &r).await {
|
||||
Ok(manga) => manga.cover_url,
|
||||
let manga = match source.fetch_manga(&ctx, &r).await {
|
||||
Ok(manga) => manga,
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
manga_id = %entry.manga_id,
|
||||
@@ -606,7 +676,7 @@ pub async fn backfill_missing_covers(
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let Some(cover_url) = cover_url else {
|
||||
let Some(cover_url) = manga.cover_url.clone() else {
|
||||
tracing::warn!(
|
||||
manga_id = %entry.manga_id,
|
||||
url = %entry.source_url,
|
||||
@@ -615,7 +685,16 @@ pub async fn backfill_missing_covers(
|
||||
stats.failed += 1;
|
||||
continue;
|
||||
};
|
||||
match download_and_store_cover(
|
||||
// RAII guard: clears the live current_cover on every exit path,
|
||||
// including a panic inside download_and_store_cover. Mirrors the
|
||||
// chapter-side ChapterGuard.
|
||||
let _cover_guard = status.map(|s| {
|
||||
s.begin_cover(crate::crawler::status::CoverTarget {
|
||||
manga_id: entry.manga_id,
|
||||
manga_title: manga.title.clone(),
|
||||
})
|
||||
});
|
||||
let cover_result = download_and_store_cover(
|
||||
db,
|
||||
storage,
|
||||
http,
|
||||
@@ -626,8 +705,8 @@ pub async fn backfill_missing_covers(
|
||||
allowlist,
|
||||
max_image_bytes,
|
||||
)
|
||||
.await
|
||||
{
|
||||
.await;
|
||||
match cover_result {
|
||||
Ok(()) => stats.fetched += 1,
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
@@ -756,6 +835,18 @@ mod tests {
|
||||
assert!(!should_stop(false, UpsertStatus::New, None));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn abort_pass_fires_at_threshold_and_respects_disable() {
|
||||
// Disabled (0) never fires, no matter how many failures.
|
||||
assert!(!should_abort_pass(0, 0));
|
||||
assert!(!should_abort_pass(100, 0));
|
||||
// Below threshold: keep going.
|
||||
assert!(!should_abort_pass(9, 10));
|
||||
// At/above threshold: abort.
|
||||
assert!(should_abort_pass(10, 10));
|
||||
assert!(should_abort_pass(11, 10));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clean_exit_when_walked_to_completion() {
|
||||
// End-of-walk reached the catalog tail — the recovery flag may
|
||||
|
||||
@@ -235,7 +235,7 @@ impl ResyncService for RealResyncService {
|
||||
let row = repo::chapter::dispatch_target(&self.db, chapter_id)
|
||||
.await
|
||||
.context("look up chapter_sources for resync")?;
|
||||
let Some((manga_id, source_url)) = row else {
|
||||
let Some((manga_id, source_url, _title, _number)) = row else {
|
||||
return Err(ResyncError::NoChapterSource.into());
|
||||
};
|
||||
|
||||
@@ -257,6 +257,11 @@ impl ResyncService for RealResyncService {
|
||||
&self.download_allowlist,
|
||||
self.max_image_bytes,
|
||||
self.tor.as_deref(),
|
||||
// Admin resync isn't a daemon worker slot — no live status.
|
||||
None,
|
||||
// Resync re-fetches existing pages (same ids); analysis isn't
|
||||
// re-enqueued here — use the admin force re-analyze endpoint.
|
||||
false,
|
||||
)
|
||||
.await;
|
||||
drop(lease);
|
||||
|
||||
@@ -91,6 +91,16 @@ impl DownloadAllowlist {
|
||||
self.hosts.is_empty()
|
||||
}
|
||||
|
||||
/// The explicitly-allowed hosts (lowercased). Empty when `allow_any`.
|
||||
pub fn hosts(&self) -> &[String] {
|
||||
&self.hosts
|
||||
}
|
||||
|
||||
/// Whether the host check is bypassed entirely (`CRAWLER_ALLOW_ANY_HOST`).
|
||||
pub fn is_allow_any(&self) -> bool {
|
||||
self.allow_any
|
||||
}
|
||||
|
||||
pub fn contains(&self, host: &str) -> bool {
|
||||
if self.allow_any {
|
||||
return true;
|
||||
@@ -241,6 +251,31 @@ pub async fn fetch_bytes_capped(
|
||||
.with_context(|| format!("download body for {url}"))
|
||||
}
|
||||
|
||||
/// Send `req` and return the response body as a stream after the
|
||||
/// safety check + 2xx status check. Caller owns chunking, capping, and
|
||||
/// piping to storage. Used by `download_and_store_page` so peak memory
|
||||
/// stays at one chunk per concurrent dispatch instead of one full
|
||||
/// image.
|
||||
pub async fn fetch_stream(
|
||||
http: &reqwest::Client,
|
||||
url: &str,
|
||||
referer: Option<&str>,
|
||||
allow: &DownloadAllowlist,
|
||||
) -> anyhow::Result<reqwest::Response> {
|
||||
is_safe_url(url, allow).with_context(|| format!("reject unsafe URL {url}"))?;
|
||||
let mut req = http.get(url);
|
||||
if let Some(r) = referer {
|
||||
req = req.header(reqwest::header::REFERER, r);
|
||||
}
|
||||
let resp = req
|
||||
.send()
|
||||
.await
|
||||
.with_context(|| format!("GET {url}"))?
|
||||
.error_for_status()
|
||||
.with_context(|| format!("non-2xx for {url}"))?;
|
||||
Ok(resp)
|
||||
}
|
||||
|
||||
/// True when `bytes` sniffs as one of the *renderable* image formats
|
||||
/// the `/files/*key` endpoint can serve with a correct Content-Type:
|
||||
/// JPEG, PNG, WebP, GIF, AVIF. Matches the upload pipeline's
|
||||
|
||||
184
backend/src/crawler/session_control.rs
Normal file
184
backend/src/crawler/session_control.rs
Normal file
@@ -0,0 +1,184 @@
|
||||
//! Runtime-updatable crawler session (PHPSESSID).
|
||||
//!
|
||||
//! At startup the session comes from `CRAWLER_PHPSESSID`, but it expires
|
||||
//! and previously needed a container restart to refresh. This controller
|
||||
//! lets an admin push a fresh cookie at runtime: it rewrites the reqwest
|
||||
//! cookie jar (CDN image fetches), updates the in-memory value the browser
|
||||
//! `on_launch` hook reads, persists it to `crawler_state` (so it survives
|
||||
//! a restart), and clears the sticky `session_expired` flag. A subsequent
|
||||
//! coordinated browser restart re-runs `on_launch`, re-injecting the new
|
||||
//! cookie into Chromium and re-probing.
|
||||
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::Arc;
|
||||
|
||||
use anyhow::Context;
|
||||
use sqlx::PgPool;
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
use crate::repo;
|
||||
|
||||
pub struct SessionController {
|
||||
/// Current PHPSESSID — what `on_launch` injects into a fresh browser.
|
||||
phpsessid: RwLock<Option<String>>,
|
||||
/// The same `Arc<Jar>` handed to the reqwest client; updating it here
|
||||
/// updates the client's cookies (the jar is internally mutable).
|
||||
cookie_jar: Arc<reqwest::cookie::Jar>,
|
||||
cookie_domain: Option<String>,
|
||||
start_url: Option<String>,
|
||||
db: PgPool,
|
||||
session_expired: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
impl SessionController {
|
||||
pub fn new(
|
||||
initial: Option<String>,
|
||||
cookie_jar: Arc<reqwest::cookie::Jar>,
|
||||
cookie_domain: Option<String>,
|
||||
start_url: Option<String>,
|
||||
db: PgPool,
|
||||
session_expired: Arc<AtomicBool>,
|
||||
) -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
phpsessid: RwLock::new(initial),
|
||||
cookie_jar,
|
||||
cookie_domain,
|
||||
start_url,
|
||||
db,
|
||||
session_expired,
|
||||
})
|
||||
}
|
||||
|
||||
/// The PHPSESSID a fresh browser should inject (None when unset).
|
||||
pub async fn current(&self) -> Option<String> {
|
||||
self.phpsessid.read().await.clone()
|
||||
}
|
||||
|
||||
/// Whether the sticky session-expired flag is set (chapter workers
|
||||
/// idle while true).
|
||||
pub fn is_expired(&self) -> bool {
|
||||
self.session_expired.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
/// Clear the session-expired flag without changing the cookie — used
|
||||
/// when the operator knows the session is fine and wants workers to
|
||||
/// resume immediately.
|
||||
pub fn clear_expired(&self) {
|
||||
self.session_expired.store(false, Ordering::Release);
|
||||
}
|
||||
|
||||
/// Update the session everywhere: reqwest jar, in-memory value, and
|
||||
/// persisted `crawler_state`. Clears the session-expired flag. Does
|
||||
/// NOT relaunch the browser — the caller triggers a coordinated
|
||||
/// restart so `on_launch` re-injects + re-probes.
|
||||
pub async fn update(&self, sid: &str) -> anyhow::Result<()> {
|
||||
let sid = sid.trim().to_string();
|
||||
anyhow::ensure!(!sid.is_empty(), "PHPSESSID must not be empty");
|
||||
// The value is spliced into a cookie string and a CDP CookieParam.
|
||||
// PHPSESSID values produced by PHP are URL-safe base64 alphanumerics
|
||||
// plus a small set of punctuation depending on session.sid_bits_per_
|
||||
// character. An allow-list (rather than a blocklist of control chars
|
||||
// + `;,`) makes the check robust against future cookie syntax
|
||||
// extensions and forces a paste that includes whitespace, quotes,
|
||||
// backslashes, etc. — typical signs of a botched copy-paste — to
|
||||
// be rejected early.
|
||||
anyhow::ensure!(
|
||||
sid.chars().all(is_phpsessid_char),
|
||||
"PHPSESSID contains invalid characters"
|
||||
);
|
||||
|
||||
if let (Some(domain), Some(start_url)) = (&self.cookie_domain, &self.start_url) {
|
||||
let cookie_str = format!("PHPSESSID={sid}; Domain={domain}; Path=/");
|
||||
let seed_url =
|
||||
reqwest::Url::parse(start_url).context("parse start_url for cookie seed")?;
|
||||
self.cookie_jar.add_cookie_str(&cookie_str, &seed_url);
|
||||
}
|
||||
*self.phpsessid.write().await = Some(sid.clone());
|
||||
repo::crawler::runtime_session_persist(&self.db, &sid)
|
||||
.await
|
||||
.context("persist runtime session")?;
|
||||
self.session_expired.store(false, Ordering::Release);
|
||||
tracing::info!("crawler session updated at runtime");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Read a persisted runtime session (if any) from `crawler_state`.
|
||||
/// Called at startup so a mid-day refresh survives a restart.
|
||||
pub async fn load_persisted(db: &PgPool) -> Option<String> {
|
||||
repo::crawler::runtime_session_load(db).await.ok().flatten()
|
||||
}
|
||||
}
|
||||
|
||||
/// Characters allowed in a PHPSESSID. PHP's session.sid_bits_per_character
|
||||
/// produces alphanumerics plus `-` and `,` in the lowest-bit mode, but our
|
||||
/// audit rejects `,` (cookie delimiter) — operators paste from a browser
|
||||
/// devtools snapshot, which never embeds raw commas in the SID itself.
|
||||
/// Underscore is allowed because some sources customise their session
|
||||
/// alphabet.
|
||||
fn is_phpsessid_char(c: char) -> bool {
|
||||
c.is_ascii_alphanumeric() || matches!(c, '-' | '_')
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn controller(db: PgPool) -> Arc<SessionController> {
|
||||
SessionController::new(
|
||||
None,
|
||||
Arc::new(reqwest::cookie::Jar::default()),
|
||||
Some("example.com".into()),
|
||||
Some("https://example.com/".into()),
|
||||
db,
|
||||
Arc::new(AtomicBool::new(true)),
|
||||
)
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn update_rejects_empty_and_control_chars(pool: PgPool) {
|
||||
let c = controller(pool);
|
||||
assert!(c.update(" ").await.is_err(), "empty rejected");
|
||||
assert!(c.update("abc\r\ndef").await.is_err(), "CRLF rejected");
|
||||
assert!(c.update("ab;Domain=evil").await.is_err(), "semicolon rejected");
|
||||
assert!(c.update("x,y").await.is_err(), "comma rejected");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn update_rejects_non_alphanumeric_pastes(pool: PgPool) {
|
||||
// Allow-list tightening (M6): pastes that include whitespace,
|
||||
// quotes, slashes, backslashes, `=`, etc. are typical signs of a
|
||||
// botched copy-paste and must be rejected outright.
|
||||
let c = controller(pool);
|
||||
for bad in ["ab cd", "ab\"cd", "ab=cd", "ab/cd", "ab\\cd", "ab+cd", "ab.cd"] {
|
||||
assert!(c.update(bad).await.is_err(), "{bad:?} should be rejected");
|
||||
}
|
||||
// Allowed cases (sanity): plain alphanumerics, '-' and '_'.
|
||||
assert!(c.update("abc_DEF-123").await.is_ok());
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn update_persists_and_clears_expired_then_round_trips(pool: PgPool) {
|
||||
let c = controller(pool.clone());
|
||||
c.update("good-sid-123").await.unwrap();
|
||||
assert_eq!(c.current().await.as_deref(), Some("good-sid-123"));
|
||||
assert!(!c.is_expired(), "update clears the expired flag");
|
||||
// Persisted to crawler_state and readable by a fresh load.
|
||||
assert_eq!(
|
||||
SessionController::load_persisted(&pool).await.as_deref(),
|
||||
Some("good-sid-123")
|
||||
);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn clear_expired_flips_sticky_flag_without_touching_session(pool: PgPool) {
|
||||
// The flag starts `true` per `controller(pool)`'s test wiring.
|
||||
let c = controller(pool);
|
||||
assert!(c.is_expired(), "test fixture starts with the flag set");
|
||||
c.clear_expired();
|
||||
assert!(!c.is_expired(), "clear_expired flips the sticky flag to false");
|
||||
assert!(
|
||||
c.current().await.is_none(),
|
||||
"clear_expired does not invent a session"
|
||||
);
|
||||
}
|
||||
}
|
||||
456
backend/src/crawler/status.rs
Normal file
456
backend/src/crawler/status.rs
Normal file
@@ -0,0 +1,456 @@
|
||||
//! Live, in-process crawler status.
|
||||
//!
|
||||
//! The metadata pass runs inline in the cron tick (it is not a
|
||||
//! `crawler_jobs` row), so without this surface "what is the crawler doing
|
||||
//! right now" is unanswerable from the dashboard. The daemon publishes its
|
||||
//! current [`Phase`], the chapters being crawled right now (with a live
|
||||
//! page count), and the cover being fetched into a shared [`StatusHandle`];
|
||||
//! the admin endpoint reads a [`CrawlerStatus`] snapshot and composes it
|
||||
//! with DB-derived counts + the session/browser flags.
|
||||
//!
|
||||
//! NOTE: this is per-process state. The deployment is a single server
|
||||
//! (see CLAUDE.md), so an in-memory handle is sufficient; durable signals
|
||||
//! (last-pass summary, runtime session) are persisted in `crawler_state`.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde::Serialize;
|
||||
use tokio::sync::{watch, RwLock};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::crawler::pipeline::MetadataStats;
|
||||
|
||||
/// What the daemon's metadata pass is doing right now. Serialised with an
|
||||
/// internal `state` tag so the frontend can switch on it.
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
#[serde(tag = "state", rename_all = "snake_case")]
|
||||
pub enum Phase {
|
||||
/// Sleeping until the next scheduled metadata pass.
|
||||
Idle { next_fire: Option<DateTime<Utc>> },
|
||||
/// Walking the source catalog list pages.
|
||||
WalkingList,
|
||||
/// Fetching one manga's metadata. `index`/`total` drive a progress bar
|
||||
/// (`total` is `None` when the source size is unknown / uncapped).
|
||||
FetchingMetadata {
|
||||
index: usize,
|
||||
total: Option<usize>,
|
||||
title: String,
|
||||
},
|
||||
/// Backfilling covers that failed on first attempt. `index`/`total`
|
||||
/// track progress through this tick's batch.
|
||||
CoverBackfill { index: usize, total: usize },
|
||||
}
|
||||
|
||||
/// A chapter being downloaded right now, with a live page count. Keyed in
|
||||
/// the status by `chapter_id`; inserted by the dispatcher when a job starts
|
||||
/// and removed (via an RAII guard) when it finishes, panics, or times out.
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
pub struct ActiveChapter {
|
||||
pub manga_id: Uuid,
|
||||
pub manga_title: String,
|
||||
pub chapter_id: Uuid,
|
||||
pub chapter_number: i32,
|
||||
pub pages_done: usize,
|
||||
/// `None` until the chapter page list has been parsed.
|
||||
pub pages_total: Option<usize>,
|
||||
}
|
||||
|
||||
/// The manga whose cover is being downloaded right now.
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
pub struct CoverTarget {
|
||||
pub manga_id: Uuid,
|
||||
pub manga_title: String,
|
||||
}
|
||||
|
||||
/// Summary of the most recent metadata pass (persisted across restarts in
|
||||
/// `crawler_state` by the cron; mirrored here for the live read).
|
||||
#[derive(Clone, Debug, Serialize, Default)]
|
||||
pub struct LastPass {
|
||||
pub at: Option<DateTime<Utc>>,
|
||||
pub discovered: usize,
|
||||
pub upserted: usize,
|
||||
pub covers_fetched: usize,
|
||||
pub mangas_failed: usize,
|
||||
}
|
||||
|
||||
/// A point-in-time snapshot returned by [`StatusHandle::snapshot`]. The
|
||||
/// session/browser/queue fields are composed at read time by the endpoint
|
||||
/// (they live elsewhere), so they are not stored here.
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
pub struct CrawlerStatus {
|
||||
pub phase: Phase,
|
||||
/// Number of configured chapter workers (for "N busy / M workers").
|
||||
pub worker_count: usize,
|
||||
/// Chapters being downloaded right now, with live page counts.
|
||||
pub active_chapters: Vec<ActiveChapter>,
|
||||
pub last_pass: LastPass,
|
||||
/// The cover being downloaded right now, if any.
|
||||
pub current_cover: Option<CoverTarget>,
|
||||
}
|
||||
|
||||
/// Scalar status state held under the async `RwLock`. Active chapters and
|
||||
/// the current cover live in separate sync maps so per-page updates and
|
||||
/// RAII removal don't need to `.await` (removal happens in `Drop`).
|
||||
#[derive(Clone, Debug)]
|
||||
struct Scalar {
|
||||
phase: Phase,
|
||||
worker_count: usize,
|
||||
last_pass: LastPass,
|
||||
}
|
||||
|
||||
/// Cloneable handle the daemon tasks use to publish status. Cheap to clone
|
||||
/// (`Arc`). All writers funnel through the helper methods so locking stays
|
||||
/// localised. Every mutation bumps a `watch` version so SSE subscribers
|
||||
/// get pushed an update instead of polling.
|
||||
#[derive(Clone)]
|
||||
pub struct StatusHandle {
|
||||
scalar: Arc<RwLock<Scalar>>,
|
||||
/// Currently-downloading chapters keyed by `chapter_id`. A sync mutex so
|
||||
/// the RAII [`ChapterGuard`]'s `Drop` can remove without `.await`.
|
||||
active: Arc<Mutex<HashMap<Uuid, ActiveChapter>>>,
|
||||
/// The cover being downloaded right now (if any). Sync mutex so the
|
||||
/// RAII [`CoverGuard`]'s `Drop` can clear without `.await`, which is
|
||||
/// what makes the cleared-on-panic guarantee hold.
|
||||
current_cover: Arc<Mutex<Option<CoverTarget>>>,
|
||||
/// Monotonic version bumped on every change. SSE handlers `subscribe()`
|
||||
/// and `await .changed()` for instant pushes; `watch` has no
|
||||
/// lost-wakeup so a change between snapshots is never missed.
|
||||
version: Arc<watch::Sender<u64>>,
|
||||
}
|
||||
|
||||
/// Lock the active map, recovering from a poisoned mutex. The map values
|
||||
/// are plain structs and we never hold the lock across a panic-prone
|
||||
/// section, so resuming on poison is safe — but log it so a real poison
|
||||
/// (which signals a panic-in-critical-section bug somewhere) doesn't pass
|
||||
/// in silence.
|
||||
fn lock_active(
|
||||
m: &Mutex<HashMap<Uuid, ActiveChapter>>,
|
||||
) -> std::sync::MutexGuard<'_, HashMap<Uuid, ActiveChapter>> {
|
||||
m.lock().unwrap_or_else(|e| {
|
||||
tracing::warn!(
|
||||
"status::lock_active recovered from a poisoned mutex — \
|
||||
this implies a panic somewhere holding the lock"
|
||||
);
|
||||
e.into_inner()
|
||||
})
|
||||
}
|
||||
|
||||
/// Same shape as [`lock_active`] but for the single-slot cover mutex.
|
||||
fn lock_cover(
|
||||
m: &Mutex<Option<CoverTarget>>,
|
||||
) -> std::sync::MutexGuard<'_, Option<CoverTarget>> {
|
||||
m.lock().unwrap_or_else(|e| {
|
||||
tracing::warn!(
|
||||
"status::lock_cover recovered from a poisoned mutex — \
|
||||
this implies a panic somewhere holding the lock"
|
||||
);
|
||||
e.into_inner()
|
||||
})
|
||||
}
|
||||
|
||||
impl StatusHandle {
|
||||
pub fn new(num_workers: usize) -> Self {
|
||||
let (version, _rx) = watch::channel(0u64);
|
||||
Self {
|
||||
scalar: Arc::new(RwLock::new(Scalar {
|
||||
phase: Phase::Idle { next_fire: None },
|
||||
worker_count: num_workers.max(1),
|
||||
last_pass: LastPass::default(),
|
||||
})),
|
||||
active: Arc::new(Mutex::new(HashMap::new())),
|
||||
current_cover: Arc::new(Mutex::new(None)),
|
||||
version: Arc::new(version),
|
||||
}
|
||||
}
|
||||
|
||||
fn bump(&self) {
|
||||
self.version.send_modify(|v| *v = v.wrapping_add(1));
|
||||
}
|
||||
|
||||
/// A receiver whose `.changed()` resolves on the next status change.
|
||||
pub fn subscribe(&self) -> watch::Receiver<u64> {
|
||||
self.version.subscribe()
|
||||
}
|
||||
|
||||
/// Signal a change without mutating in-memory state — used when an
|
||||
/// *external* signal the live snapshot reflects (browser phase,
|
||||
/// session-expired flag, queue counts) has changed, so subscribers
|
||||
/// recompose promptly.
|
||||
pub fn poke(&self) {
|
||||
self.bump();
|
||||
}
|
||||
|
||||
pub async fn set_phase(&self, phase: Phase) {
|
||||
self.scalar.write().await.phase = phase;
|
||||
self.bump();
|
||||
}
|
||||
|
||||
/// Register a cover-fetch as in flight; returns a guard that clears
|
||||
/// the current cover when dropped (on completion, panic-unwind, or
|
||||
/// any future early-return). Last-writer-wins: a guard only clears
|
||||
/// the slot when it still holds the cover it set (so overlapping
|
||||
/// guards — not used today, but defensive — don't clobber each
|
||||
/// other).
|
||||
pub fn begin_cover(&self, target: CoverTarget) -> CoverGuard {
|
||||
let manga_id = target.manga_id;
|
||||
*lock_cover(&self.current_cover) = Some(target);
|
||||
self.bump();
|
||||
CoverGuard {
|
||||
current_cover: Arc::clone(&self.current_cover),
|
||||
version: Arc::clone(&self.version),
|
||||
manga_id,
|
||||
}
|
||||
}
|
||||
|
||||
/// Register a chapter as crawling now; returns a guard that removes it
|
||||
/// when dropped (on completion, panic-unwind, or timeout-drop).
|
||||
pub fn begin_chapter(&self, chapter: ActiveChapter) -> ChapterGuard {
|
||||
let id = chapter.chapter_id;
|
||||
lock_active(&self.active).insert(id, chapter);
|
||||
self.bump();
|
||||
ChapterGuard {
|
||||
active: Arc::clone(&self.active),
|
||||
version: Arc::clone(&self.version),
|
||||
chapter_id: id,
|
||||
}
|
||||
}
|
||||
|
||||
/// Update the live page count of an in-flight chapter. Sync (no
|
||||
/// `.await`) so it's cheap to call once per stored page.
|
||||
pub fn set_chapter_pages(&self, chapter_id: Uuid, done: usize, total: Option<usize>) {
|
||||
{
|
||||
let mut map = lock_active(&self.active);
|
||||
if let Some(c) = map.get_mut(&chapter_id) {
|
||||
c.pages_done = done;
|
||||
c.pages_total = total;
|
||||
}
|
||||
}
|
||||
self.bump();
|
||||
}
|
||||
|
||||
/// Record a finished metadata pass. Stamps `at` with `now`.
|
||||
pub async fn record_pass(&self, stats: &MetadataStats, at: DateTime<Utc>) {
|
||||
self.scalar.write().await.last_pass = LastPass {
|
||||
at: Some(at),
|
||||
discovered: stats.discovered,
|
||||
upserted: stats.upserted,
|
||||
covers_fetched: stats.covers_fetched,
|
||||
mangas_failed: stats.mangas_failed,
|
||||
};
|
||||
self.bump();
|
||||
}
|
||||
|
||||
/// Seed the last-pass summary from a persisted `crawler_state` value on
|
||||
/// startup so the dashboard isn't blank until the first tick.
|
||||
pub async fn set_last_pass(&self, last: LastPass) {
|
||||
self.scalar.write().await.last_pass = last;
|
||||
self.bump();
|
||||
}
|
||||
|
||||
pub async fn snapshot(&self) -> CrawlerStatus {
|
||||
let scalar = self.scalar.read().await.clone();
|
||||
let mut active_chapters: Vec<ActiveChapter> =
|
||||
lock_active(&self.active).values().cloned().collect();
|
||||
// Stable, readable order: by chapter number then id.
|
||||
active_chapters.sort_by(|a, b| {
|
||||
a.chapter_number
|
||||
.cmp(&b.chapter_number)
|
||||
.then(a.chapter_id.cmp(&b.chapter_id))
|
||||
});
|
||||
let current_cover = lock_cover(&self.current_cover).clone();
|
||||
CrawlerStatus {
|
||||
phase: scalar.phase,
|
||||
worker_count: scalar.worker_count,
|
||||
active_chapters,
|
||||
last_pass: scalar.last_pass,
|
||||
current_cover,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// RAII handle clearing the [`CoverTarget`] from the live status when the
|
||||
/// cover-fetch finishes, panics, or is dropped on any early-return.
|
||||
pub struct CoverGuard {
|
||||
current_cover: Arc<Mutex<Option<CoverTarget>>>,
|
||||
version: Arc<watch::Sender<u64>>,
|
||||
/// Manga id whose cover this guard registered. The drop only clears
|
||||
/// the slot when the stored value still matches — defends against a
|
||||
/// hypothetical newer guard clobbering this one's clear.
|
||||
manga_id: Uuid,
|
||||
}
|
||||
|
||||
impl Drop for CoverGuard {
|
||||
fn drop(&mut self) {
|
||||
let mut slot = lock_cover(&self.current_cover);
|
||||
if slot.as_ref().map(|c| c.manga_id) == Some(self.manga_id) {
|
||||
*slot = None;
|
||||
}
|
||||
self.version.send_modify(|v| *v = v.wrapping_add(1));
|
||||
}
|
||||
}
|
||||
|
||||
/// RAII handle removing an [`ActiveChapter`] from the live status when the
|
||||
/// chapter dispatch finishes, panics, or is dropped on timeout.
|
||||
pub struct ChapterGuard {
|
||||
active: Arc<Mutex<HashMap<Uuid, ActiveChapter>>>,
|
||||
version: Arc<watch::Sender<u64>>,
|
||||
chapter_id: Uuid,
|
||||
}
|
||||
|
||||
impl Drop for ChapterGuard {
|
||||
fn drop(&mut self) {
|
||||
lock_active(&self.active).remove(&self.chapter_id);
|
||||
self.version.send_modify(|v| *v = v.wrapping_add(1));
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn sample_chapter(n: i32) -> ActiveChapter {
|
||||
ActiveChapter {
|
||||
manga_id: Uuid::new_v4(),
|
||||
manga_title: "M".into(),
|
||||
chapter_id: Uuid::new_v4(),
|
||||
chapter_number: n,
|
||||
pages_done: 0,
|
||||
pages_total: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn begin_chapter_shows_in_snapshot_and_guard_removes_on_drop() {
|
||||
let h = StatusHandle::new(2);
|
||||
let chap = sample_chapter(7);
|
||||
let cid = chap.chapter_id;
|
||||
{
|
||||
let _guard = h.begin_chapter(chap);
|
||||
let snap = h.snapshot().await;
|
||||
assert_eq!(snap.active_chapters.len(), 1);
|
||||
assert_eq!(snap.active_chapters[0].chapter_id, cid);
|
||||
assert_eq!(snap.worker_count, 2);
|
||||
}
|
||||
// Guard dropped → entry removed.
|
||||
let snap = h.snapshot().await;
|
||||
assert!(snap.active_chapters.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn set_chapter_pages_updates_live_count() {
|
||||
let h = StatusHandle::new(1);
|
||||
let chap = sample_chapter(1);
|
||||
let cid = chap.chapter_id;
|
||||
let _guard = h.begin_chapter(chap);
|
||||
h.set_chapter_pages(cid, 3, Some(20));
|
||||
let snap = h.snapshot().await;
|
||||
assert_eq!(snap.active_chapters[0].pages_done, 3);
|
||||
assert_eq!(snap.active_chapters[0].pages_total, Some(20));
|
||||
// Updating an unknown chapter is a no-op, not a panic.
|
||||
h.set_chapter_pages(Uuid::new_v4(), 9, Some(9));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn snapshot_sorts_active_chapters_by_number() {
|
||||
let h = StatusHandle::new(2);
|
||||
let _g1 = h.begin_chapter(sample_chapter(5));
|
||||
let _g2 = h.begin_chapter(sample_chapter(2));
|
||||
let snap = h.snapshot().await;
|
||||
assert_eq!(snap.active_chapters[0].chapter_number, 2);
|
||||
assert_eq!(snap.active_chapters[1].chapter_number, 5);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cover_guard_sets_then_clears_on_drop() {
|
||||
let h = StatusHandle::new(1);
|
||||
let mid = Uuid::new_v4();
|
||||
{
|
||||
let _g = h.begin_cover(CoverTarget {
|
||||
manga_id: mid,
|
||||
manga_title: "One Piece".into(),
|
||||
});
|
||||
assert_eq!(
|
||||
h.snapshot().await.current_cover.map(|c| c.manga_id),
|
||||
Some(mid)
|
||||
);
|
||||
}
|
||||
assert!(h.snapshot().await.current_cover.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cover_guard_clears_on_panic_drop() {
|
||||
// Simulate a download_and_store_cover panic: the guard is on the
|
||||
// stack, the panic unwinds, and the slot must still be cleared.
|
||||
let h = StatusHandle::new(1);
|
||||
let mid = Uuid::new_v4();
|
||||
let h2 = h.clone();
|
||||
let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
|
||||
let _g = h2.begin_cover(CoverTarget {
|
||||
manga_id: mid,
|
||||
manga_title: "K-On!".into(),
|
||||
});
|
||||
panic!("simulated cover-download panic");
|
||||
}));
|
||||
assert!(result.is_err());
|
||||
assert!(h.snapshot().await.current_cover.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cover_guard_does_not_clobber_a_newer_target() {
|
||||
// Defensive: if a *newer* begin_cover ran before the older
|
||||
// guard's drop fires, the drop must not clear the newer
|
||||
// target. (No code today produces overlapping guards, but the
|
||||
// invariant prevents a future caller from quietly breaking the
|
||||
// live-cover surface.)
|
||||
let h = StatusHandle::new(1);
|
||||
let older = Uuid::new_v4();
|
||||
let newer = Uuid::new_v4();
|
||||
let g_old = h.begin_cover(CoverTarget {
|
||||
manga_id: older,
|
||||
manga_title: "old".into(),
|
||||
});
|
||||
let _g_new = h.begin_cover(CoverTarget {
|
||||
manga_id: newer,
|
||||
manga_title: "new".into(),
|
||||
});
|
||||
drop(g_old);
|
||||
assert_eq!(
|
||||
h.snapshot().await.current_cover.map(|c| c.manga_id),
|
||||
Some(newer)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn record_pass_captures_stats_and_timestamp() {
|
||||
let h = StatusHandle::new(1);
|
||||
let stats = MetadataStats {
|
||||
discovered: 5,
|
||||
upserted: 3,
|
||||
covers_fetched: 2,
|
||||
mangas_failed: 1,
|
||||
};
|
||||
let at = Utc::now();
|
||||
h.record_pass(&stats, at).await;
|
||||
let snap = h.snapshot().await;
|
||||
assert_eq!(snap.last_pass.discovered, 5);
|
||||
assert_eq!(snap.last_pass.upserted, 3);
|
||||
assert_eq!(snap.last_pass.at, Some(at));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn subscribe_resolves_on_mutation_poke_and_chapter_change() {
|
||||
let h = StatusHandle::new(1);
|
||||
let mut rx = h.subscribe();
|
||||
h.set_phase(Phase::WalkingList).await;
|
||||
rx.changed().await.unwrap();
|
||||
h.poke();
|
||||
rx.changed().await.unwrap();
|
||||
// begin_chapter + guard drop each bump the version.
|
||||
let g = h.begin_chapter(sample_chapter(1));
|
||||
rx.changed().await.unwrap();
|
||||
drop(g);
|
||||
rx.changed().await.unwrap();
|
||||
}
|
||||
}
|
||||
@@ -33,6 +33,22 @@ pub struct CollectionSummary {
|
||||
pub sample_covers: Vec<String>,
|
||||
}
|
||||
|
||||
/// Row returned by `GET /collections/:id/pages`. Joins through
|
||||
/// `chapters` and `mangas` so the collection detail view can render a
|
||||
/// thumbnail + breadcrumb without per-row follow-up fetches.
|
||||
#[derive(Debug, Clone, Serialize, FromRow)]
|
||||
pub struct CollectionPageItem {
|
||||
pub page_id: Uuid,
|
||||
pub chapter_id: Uuid,
|
||||
pub manga_id: Uuid,
|
||||
pub page_number: i32,
|
||||
pub chapter_number: i32,
|
||||
pub chapter_title: Option<String>,
|
||||
pub manga_title: String,
|
||||
pub storage_key: String,
|
||||
pub added_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub struct NewCollection {
|
||||
pub name: String,
|
||||
|
||||
@@ -5,6 +5,7 @@ use uuid::Uuid;
|
||||
|
||||
use super::author::AuthorRef;
|
||||
use super::genre::GenreRef;
|
||||
use super::page_analysis::ContentWarning;
|
||||
use super::patch::Patch;
|
||||
use super::tag::TagRef;
|
||||
|
||||
@@ -32,7 +33,8 @@ pub struct MangaCard {
|
||||
}
|
||||
|
||||
/// Shape returned by `GET /mangas/:id`. Adds user-added tags on top of
|
||||
/// the card fields.
|
||||
/// the card fields, plus the deduped content warnings derived by the
|
||||
/// analysis worker across all the manga's pages.
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct MangaDetail {
|
||||
#[serde(flatten)]
|
||||
@@ -40,6 +42,7 @@ pub struct MangaDetail {
|
||||
pub authors: Vec<AuthorRef>,
|
||||
pub genres: Vec<GenreRef>,
|
||||
pub tags: Vec<TagRef>,
|
||||
pub content_warnings: Vec<ContentWarning>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Default)]
|
||||
|
||||
@@ -7,6 +7,8 @@ pub mod collection;
|
||||
pub mod genre;
|
||||
pub mod manga;
|
||||
pub mod page;
|
||||
pub mod page_analysis;
|
||||
pub mod page_tag;
|
||||
pub mod patch;
|
||||
pub mod read_progress;
|
||||
pub mod session;
|
||||
@@ -21,10 +23,18 @@ pub use api_token::ApiToken;
|
||||
pub use author::{Author, AuthorRef, AuthorWithCount};
|
||||
pub use bookmark::{Bookmark, BookmarkSummary};
|
||||
pub use chapter::Chapter;
|
||||
pub use collection::{Collection, CollectionSummary};
|
||||
pub use collection::{Collection, CollectionPageItem, CollectionSummary};
|
||||
pub use genre::{Genre, GenreRef};
|
||||
pub use manga::{Manga, MangaCard, MangaDetail};
|
||||
pub use page::Page;
|
||||
pub use page_analysis::{
|
||||
AnalysisStatus, ContentWarning, OcrKind, OcrResult, PageAnalysis, PageSearchItem,
|
||||
SafetyFlag, VisionAnalysis,
|
||||
};
|
||||
pub use page_tag::{
|
||||
NewPageTag, PageTagSummary, TaggedChapterAggregate, TaggedMangaAggregate,
|
||||
TaggedPageItem,
|
||||
};
|
||||
pub use patch::Patch;
|
||||
pub use read_progress::{ReadProgress, ReadProgressForManga, ReadProgressSummary};
|
||||
pub use session::Session;
|
||||
|
||||
293
backend/src/domain/page_analysis.rs
Normal file
293
backend/src/domain/page_analysis.rs
Normal file
@@ -0,0 +1,293 @@
|
||||
//! AI page-analysis domain types.
|
||||
//!
|
||||
//! Two distinct shapes live here:
|
||||
//!
|
||||
//! * The **persisted** row types (`PageAnalysis`) and the closed
|
||||
//! vocabularies (`OcrKind`, `ContentWarning`, `AnalysisStatus`) that the
|
||||
//! `page_analysis` / `page_ocr_text` / `page_content_warnings` tables
|
||||
//! constrain via CHECKs.
|
||||
//! * The **vision-response** DTOs (`VisionAnalysis` and friends) that the
|
||||
//! worker deserializes from the local model's JSON. These are
|
||||
//! deliberately lenient — `kind` and `content_type` arrive as free
|
||||
//! strings because a small local model can emit anything; mapping to the
|
||||
//! closed enums (and dropping the unmappable) happens at persist time via
|
||||
//! [`OcrKind::from_model_str`] / [`ContentWarning::from_model_str`].
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sqlx::FromRow;
|
||||
use uuid::Uuid;
|
||||
|
||||
/// Lifecycle of a page's analysis row.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, sqlx::Type, Serialize, Deserialize)]
|
||||
#[sqlx(type_name = "text", rename_all = "snake_case")]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum AnalysisStatus {
|
||||
Pending,
|
||||
Done,
|
||||
Failed,
|
||||
}
|
||||
|
||||
/// Kind of an OCR'd text piece. Drives the search-ranking weight:
|
||||
/// `Speech`/`Title` → A, `Narration`/`Thought`/`Caption` → B, `Sfx` → D
|
||||
/// (the scene description is weighted C separately). See
|
||||
/// [`OcrKind::weight`].
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, sqlx::Type, Serialize, Deserialize)]
|
||||
#[sqlx(type_name = "text", rename_all = "snake_case")]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum OcrKind {
|
||||
Speech,
|
||||
Thought,
|
||||
Narration,
|
||||
Sfx,
|
||||
Title,
|
||||
Caption,
|
||||
}
|
||||
|
||||
impl OcrKind {
|
||||
/// Postgres `tsvector` weight label for this kind. The default
|
||||
/// `ts_rank` weights `{D,C,B,A} = {0.1,0.2,0.4,1.0}` then realize the
|
||||
/// intended relevance ordering: speech/title most important, sfx least.
|
||||
pub fn weight(self) -> char {
|
||||
match self {
|
||||
OcrKind::Speech | OcrKind::Title => 'A',
|
||||
OcrKind::Narration | OcrKind::Thought | OcrKind::Caption => 'B',
|
||||
OcrKind::Sfx => 'D',
|
||||
}
|
||||
}
|
||||
|
||||
/// Map a model-supplied kind string onto the closed vocabulary. Unknown
|
||||
/// or unparseable kinds fall back to `Narration` (the neutral
|
||||
/// mid-weight bucket) rather than dropping the text — losing the
|
||||
/// transcription entirely is worse than mis-weighting it.
|
||||
pub fn from_model_str(raw: &str) -> OcrKind {
|
||||
match raw.trim().to_lowercase().as_str() {
|
||||
"speech" => OcrKind::Speech,
|
||||
"thought" => OcrKind::Thought,
|
||||
"narration" => OcrKind::Narration,
|
||||
"sfx" => OcrKind::Sfx,
|
||||
"title" => OcrKind::Title,
|
||||
"caption" => OcrKind::Caption,
|
||||
_ => OcrKind::Narration,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A content-warning category from the closed moderation vocabulary.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, sqlx::Type, Serialize, Deserialize)]
|
||||
#[sqlx(type_name = "text", rename_all = "snake_case")]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ContentWarning {
|
||||
Sexual,
|
||||
Nudity,
|
||||
Gore,
|
||||
Violence,
|
||||
Disturbing,
|
||||
}
|
||||
|
||||
impl ContentWarning {
|
||||
/// Map a model-supplied content-type string onto the closed
|
||||
/// vocabulary, or `None` if it isn't one we recognize. Unlike OCR
|
||||
/// kinds, an unknown warning is *dropped* — flagging a page with a
|
||||
/// category we can't filter on is meaningless.
|
||||
pub fn from_model_str(raw: &str) -> Option<ContentWarning> {
|
||||
match raw.trim().to_lowercase().as_str() {
|
||||
"sexual" => Some(ContentWarning::Sexual),
|
||||
"nudity" => Some(ContentWarning::Nudity),
|
||||
"gore" => Some(ContentWarning::Gore),
|
||||
"violence" => Some(ContentWarning::Violence),
|
||||
"disturbing" => Some(ContentWarning::Disturbing),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse a wire/query-param value strictly (no fallback). Used by the
|
||||
/// API layer to validate `cw_include` / `cw_exclude` filters.
|
||||
pub fn parse_strict(raw: &str) -> Option<ContentWarning> {
|
||||
ContentWarning::from_model_str(raw)
|
||||
}
|
||||
}
|
||||
|
||||
/// One result row from the page content-search (`GET /v1/me/page-search`).
|
||||
/// One row per matching page, carrying the breadcrumb plus the moderation
|
||||
/// flags and the text-search rank so the UI can badge and order results.
|
||||
#[derive(Debug, Clone, Serialize, FromRow)]
|
||||
pub struct PageSearchItem {
|
||||
pub page_id: Uuid,
|
||||
pub chapter_id: Uuid,
|
||||
pub manga_id: Uuid,
|
||||
pub page_number: i32,
|
||||
pub chapter_number: i32,
|
||||
pub chapter_title: Option<String>,
|
||||
pub manga_title: String,
|
||||
pub storage_key: String,
|
||||
pub is_nsfw: bool,
|
||||
/// Deduped content warnings on this page (canonical lowercase names).
|
||||
pub content_warnings: Vec<String>,
|
||||
/// `ts_rank` against the text query; `0` for tag/warning-only searches.
|
||||
pub rank: f32,
|
||||
}
|
||||
|
||||
/// Analysis coverage for one manga (admin overview): how many of its pages
|
||||
/// have a completed analysis vs. how many exist.
|
||||
#[derive(Debug, Clone, Serialize, FromRow)]
|
||||
pub struct MangaCoverage {
|
||||
pub manga_id: Uuid,
|
||||
pub title: String,
|
||||
pub total_pages: i64,
|
||||
pub analyzed_pages: i64,
|
||||
}
|
||||
|
||||
/// Analysis coverage for one chapter.
|
||||
#[derive(Debug, Clone, Serialize, FromRow)]
|
||||
pub struct ChapterCoverage {
|
||||
pub chapter_id: Uuid,
|
||||
pub number: i32,
|
||||
pub title: Option<String>,
|
||||
pub total_pages: i64,
|
||||
pub analyzed_pages: i64,
|
||||
}
|
||||
|
||||
/// Per-page analysis status for a chapter's page grid. `status` is one of
|
||||
/// `done` | `failed` | `queued` (a pending/running job) | `none`.
|
||||
#[derive(Debug, Clone, Serialize, FromRow)]
|
||||
pub struct PageStatusItem {
|
||||
pub page_id: Uuid,
|
||||
pub page_number: i32,
|
||||
pub status: String,
|
||||
}
|
||||
|
||||
/// One OCR line in the page-detail view.
|
||||
#[derive(Debug, Clone, Serialize, FromRow)]
|
||||
pub struct OcrLine {
|
||||
pub kind: OcrKind,
|
||||
pub text: String,
|
||||
}
|
||||
|
||||
/// Full analysis result for one page, for the admin detail modal. `status`
|
||||
/// is `done` | `failed` | `none` (no analysis row yet).
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct PageAnalysisDetail {
|
||||
pub page_id: Uuid,
|
||||
pub page_number: i32,
|
||||
pub chapter_id: Uuid,
|
||||
pub manga_id: Uuid,
|
||||
pub status: String,
|
||||
pub is_nsfw: bool,
|
||||
pub scene_description: Option<String>,
|
||||
pub model: Option<String>,
|
||||
pub error: Option<String>,
|
||||
pub analyzed_at: Option<DateTime<Utc>>,
|
||||
pub ocr: Vec<OcrLine>,
|
||||
pub tags: Vec<String>,
|
||||
pub content_warnings: Vec<ContentWarning>,
|
||||
}
|
||||
|
||||
/// One persisted `page_analysis` row.
|
||||
#[derive(Debug, Clone, Serialize, FromRow)]
|
||||
pub struct PageAnalysis {
|
||||
pub page_id: Uuid,
|
||||
pub status: AnalysisStatus,
|
||||
pub scene_description: Option<String>,
|
||||
pub is_nsfw: bool,
|
||||
pub model: Option<String>,
|
||||
pub error: Option<String>,
|
||||
pub analyzed_at: Option<DateTime<Utc>>,
|
||||
}
|
||||
|
||||
// --- Vision-response DTOs (deserialized from the local model) ---------
|
||||
|
||||
/// The full JSON object the vision model returns for one page. Lenient by
|
||||
/// design: see the module docs.
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)]
|
||||
pub struct VisionAnalysis {
|
||||
#[serde(default)]
|
||||
pub ocr_results: Vec<OcrResult>,
|
||||
#[serde(default)]
|
||||
pub tagging_results: Vec<String>,
|
||||
#[serde(default)]
|
||||
pub scene_description: String,
|
||||
#[serde(default)]
|
||||
pub safety_flag: SafetyFlag,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, Deserialize, Serialize, PartialEq)]
|
||||
pub struct OcrResult {
|
||||
#[serde(default)]
|
||||
pub text: String,
|
||||
/// Free string from the model; mapped via [`OcrKind::from_model_str`].
|
||||
#[serde(default)]
|
||||
pub kind: String,
|
||||
/// Optional vertical center of the text as a fraction (0.0 top … 1.0
|
||||
/// bottom) of the slice image, requested in the OCR pass to dedup
|
||||
/// duplicates across slice seams by position. `None` for the combined
|
||||
/// single-call path. Internal-only — not persisted.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub y: Option<f64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, Deserialize, Serialize, PartialEq)]
|
||||
pub struct SafetyFlag {
|
||||
#[serde(default)]
|
||||
pub is_nsfw: bool,
|
||||
/// Free strings; mapped via [`ContentWarning::from_model_str`].
|
||||
#[serde(default)]
|
||||
pub content_type: Vec<String>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn ocr_kind_weights_match_spec() {
|
||||
assert_eq!(OcrKind::Speech.weight(), 'A');
|
||||
assert_eq!(OcrKind::Title.weight(), 'A');
|
||||
assert_eq!(OcrKind::Narration.weight(), 'B');
|
||||
assert_eq!(OcrKind::Thought.weight(), 'B');
|
||||
assert_eq!(OcrKind::Caption.weight(), 'B');
|
||||
assert_eq!(OcrKind::Sfx.weight(), 'D');
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ocr_kind_from_model_str_falls_back_to_narration() {
|
||||
assert_eq!(OcrKind::from_model_str("SPEECH"), OcrKind::Speech);
|
||||
assert_eq!(OcrKind::from_model_str(" sfx "), OcrKind::Sfx);
|
||||
assert_eq!(OcrKind::from_model_str("dialogue"), OcrKind::Narration);
|
||||
assert_eq!(OcrKind::from_model_str(""), OcrKind::Narration);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn content_warning_from_model_str_drops_unknown() {
|
||||
assert_eq!(
|
||||
ContentWarning::from_model_str("Sexual"),
|
||||
Some(ContentWarning::Sexual)
|
||||
);
|
||||
assert_eq!(
|
||||
ContentWarning::from_model_str(" gore"),
|
||||
Some(ContentWarning::Gore)
|
||||
);
|
||||
assert_eq!(ContentWarning::from_model_str("spicy"), None);
|
||||
assert_eq!(ContentWarning::from_model_str(""), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn vision_analysis_deserializes_sample_and_tolerates_missing_fields() {
|
||||
let json = r#"{
|
||||
"ocr_results": [{"text":"Hi","kind":"speech"}],
|
||||
"tagging_results": ["action","city"],
|
||||
"scene_description": "A street.",
|
||||
"safety_flag": {"is_nsfw": false, "content_type": []}
|
||||
}"#;
|
||||
let v: VisionAnalysis = serde_json::from_str(json).unwrap();
|
||||
assert_eq!(v.ocr_results.len(), 1);
|
||||
assert_eq!(v.tagging_results, vec!["action", "city"]);
|
||||
assert!(!v.safety_flag.is_nsfw);
|
||||
|
||||
// Missing optional fields default rather than failing the parse.
|
||||
let sparse: VisionAnalysis = serde_json::from_str("{}").unwrap();
|
||||
assert!(sparse.ocr_results.is_empty());
|
||||
assert_eq!(sparse.scene_description, "");
|
||||
assert!(!sparse.safety_flag.is_nsfw);
|
||||
}
|
||||
}
|
||||
63
backend/src/domain/page_tag.rs
Normal file
63
backend/src/domain/page_tag.rs
Normal file
@@ -0,0 +1,63 @@
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sqlx::FromRow;
|
||||
use uuid::Uuid;
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub struct NewPageTag {
|
||||
pub tag: String,
|
||||
}
|
||||
|
||||
/// Returned by `GET /v1/me/page-tags`. Joins through chapters and
|
||||
/// mangas so the library Page-tags tab can render the breadcrumb
|
||||
/// without follow-up requests.
|
||||
#[derive(Debug, Clone, Serialize, FromRow)]
|
||||
pub struct TaggedPageItem {
|
||||
pub tag: String,
|
||||
pub page_id: Uuid,
|
||||
pub chapter_id: Uuid,
|
||||
pub manga_id: Uuid,
|
||||
pub page_number: i32,
|
||||
pub chapter_number: i32,
|
||||
pub chapter_title: Option<String>,
|
||||
pub manga_title: String,
|
||||
pub storage_key: String,
|
||||
pub tagged_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
/// Distinct-tags histogram row. Used both for autocomplete in the
|
||||
/// "Add tag" sheet and for the chip cloud in the library Page-tags
|
||||
/// tab.
|
||||
#[derive(Debug, Clone, Serialize, FromRow)]
|
||||
pub struct PageTagSummary {
|
||||
pub tag: String,
|
||||
pub count: i64,
|
||||
}
|
||||
|
||||
/// One chapter (with breadcrumb) ranked by how many of its pages the
|
||||
/// caller has tagged with a given tag. Returned by the `/search`
|
||||
/// page's Chapters tab via `GET /v1/me/page-tags/chapters`.
|
||||
#[derive(Debug, Clone, Serialize, FromRow)]
|
||||
pub struct TaggedChapterAggregate {
|
||||
pub chapter_id: Uuid,
|
||||
pub manga_id: Uuid,
|
||||
pub manga_title: String,
|
||||
pub chapter_number: i32,
|
||||
pub chapter_title: Option<String>,
|
||||
pub match_count: i64,
|
||||
/// Up to 3 storage keys of matching pages in this chapter,
|
||||
/// page-number ascending. Powers the thumbnail strip in the row.
|
||||
pub sample_storage_keys: Vec<String>,
|
||||
}
|
||||
|
||||
/// One manga ranked by how many of its pages (across all chapters)
|
||||
/// the caller has tagged with a given tag. Returned by the `/search`
|
||||
/// page's Mangas tab.
|
||||
#[derive(Debug, Clone, Serialize, FromRow)]
|
||||
pub struct TaggedMangaAggregate {
|
||||
pub manga_id: Uuid,
|
||||
pub manga_title: String,
|
||||
pub manga_cover_image_path: Option<String>,
|
||||
pub match_count: i64,
|
||||
pub sample_storage_keys: Vec<String>,
|
||||
}
|
||||
@@ -38,6 +38,16 @@ pub enum AppError {
|
||||
message: String,
|
||||
details: serde_json::Value,
|
||||
},
|
||||
/// 501 — the wire shape is accepted but the feature isn't built yet.
|
||||
/// Carries a `&'static str` snake_case code so clients can detect
|
||||
/// the specific pending feature (`text_search_not_yet_supported`,
|
||||
/// etc.) without parsing the message. Used today by the `?text=`
|
||||
/// reservation on the page-tag aggregation endpoints.
|
||||
#[error("not implemented: {code}")]
|
||||
NotImplemented {
|
||||
code: &'static str,
|
||||
message: &'static str,
|
||||
},
|
||||
#[error(transparent)]
|
||||
Database(#[from] sqlx::Error),
|
||||
#[error(transparent)]
|
||||
@@ -64,6 +74,7 @@ impl AppError {
|
||||
AppError::ServiceUnavailable(_) => "service_unavailable",
|
||||
AppError::TooManyRequests { .. } => "too_many_requests",
|
||||
AppError::ValidationFailed { .. } => "validation_failed",
|
||||
AppError::NotImplemented { code, .. } => code,
|
||||
AppError::Database(sqlx::Error::RowNotFound) => "not_found",
|
||||
AppError::Database(_) => "internal_error",
|
||||
AppError::Storage(StorageError::NotFound) => "not_found",
|
||||
@@ -124,6 +135,11 @@ impl IntoResponse for AppError {
|
||||
message.clone(),
|
||||
Some(details.clone()),
|
||||
),
|
||||
AppError::NotImplemented { message, .. } => (
|
||||
StatusCode::NOT_IMPLEMENTED,
|
||||
(*message).to_string(),
|
||||
None,
|
||||
),
|
||||
AppError::Database(sqlx::Error::RowNotFound) => {
|
||||
(StatusCode::NOT_FOUND, "not found".to_string(), None)
|
||||
}
|
||||
@@ -180,5 +196,15 @@ mod tests {
|
||||
assert_eq!(AppError::Storage(StorageError::NotFound).code(), "not_found");
|
||||
assert_eq!(AppError::Database(sqlx::Error::RowNotFound).code(), "not_found");
|
||||
assert_eq!(AppError::Other(anyhow::anyhow!("oops")).code(), "internal_error");
|
||||
// NotImplemented carries the code through so each pending
|
||||
// feature gets its own stable identifier on the wire.
|
||||
assert_eq!(
|
||||
AppError::NotImplemented {
|
||||
code: "text_search_not_yet_supported",
|
||||
message: "x"
|
||||
}
|
||||
.code(),
|
||||
"text_search_not_yet_supported"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
pub mod analysis;
|
||||
pub mod api;
|
||||
pub mod app;
|
||||
pub mod auth;
|
||||
@@ -6,5 +7,6 @@ pub mod crawler;
|
||||
pub mod domain;
|
||||
pub mod error;
|
||||
pub mod repo;
|
||||
pub mod settings;
|
||||
pub mod storage;
|
||||
pub mod upload;
|
||||
|
||||
@@ -21,7 +21,8 @@ async fn main() -> anyhow::Result<()> {
|
||||
|
||||
let config = mangalord::config::Config::from_env()?;
|
||||
let addr: SocketAddr = config.bind_address.parse()?;
|
||||
let mangalord::app::AppHandle { router, daemon } = mangalord::app::build(config).await?;
|
||||
let mangalord::app::AppHandle { router, supervisors } =
|
||||
mangalord::app::build(config).await?;
|
||||
|
||||
tracing::info!(%addr, "mangalord listening");
|
||||
let listener = tokio::net::TcpListener::bind(addr).await?;
|
||||
@@ -29,19 +30,17 @@ async fn main() -> anyhow::Result<()> {
|
||||
.with_graceful_shutdown(shutdown_signal())
|
||||
.await?;
|
||||
|
||||
// Drain background tasks (crawler daemon) before exiting so Chromium
|
||||
// gets a clean shutdown rather than relying on kill-on-drop. Bounded
|
||||
// by a timeout so a wedged shutdown path can't trap the process.
|
||||
if let Some(d) = daemon {
|
||||
if tokio::time::timeout(CRAWLER_SHUTDOWN_TIMEOUT, d.shutdown())
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
tracing::warn!(
|
||||
timeout_s = CRAWLER_SHUTDOWN_TIMEOUT.as_secs(),
|
||||
"crawler daemon shutdown exceeded timeout; abandoning"
|
||||
);
|
||||
}
|
||||
// Drain background daemons (crawler + analysis) before exiting so
|
||||
// Chromium gets a clean shutdown rather than relying on kill-on-drop.
|
||||
// Bounded by a timeout so a wedged shutdown path can't trap the process.
|
||||
if tokio::time::timeout(CRAWLER_SHUTDOWN_TIMEOUT, supervisors.shutdown())
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
tracing::warn!(
|
||||
timeout_s = CRAWLER_SHUTDOWN_TIMEOUT.as_secs(),
|
||||
"daemon shutdown exceeded timeout; abandoning"
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
54
backend/src/repo/app_settings.rs
Normal file
54
backend/src/repo/app_settings.rs
Normal file
@@ -0,0 +1,54 @@
|
||||
//! Persistence for runtime-editable application settings (`app_settings`),
|
||||
//! a small key-value JSONB table (one row per subsystem group). Mirrors the
|
||||
//! `crawler_state` access style: plain async fns over `&PgPool`, the `value`
|
||||
//! a `serde_json::Value` the caller (de)serializes into a settings DTO.
|
||||
|
||||
use sqlx::{PgExecutor, PgPool};
|
||||
|
||||
/// Fetch the raw JSONB for a settings group, or `None` if unset.
|
||||
pub async fn get(pool: &PgPool, key: &str) -> sqlx::Result<Option<serde_json::Value>> {
|
||||
sqlx::query_scalar("SELECT value FROM app_settings WHERE key = $1")
|
||||
.bind(key)
|
||||
.fetch_optional(pool)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Insert-or-replace the JSONB for a settings group, stamping `updated_at`.
|
||||
/// Generic over the executor so it can run inside the same transaction as
|
||||
/// the matching admin-audit insert.
|
||||
pub async fn upsert<'e, E: PgExecutor<'e>>(
|
||||
executor: E,
|
||||
key: &str,
|
||||
value: &serde_json::Value,
|
||||
) -> sqlx::Result<()> {
|
||||
sqlx::query(
|
||||
"INSERT INTO app_settings (key, value, updated_at) \
|
||||
VALUES ($1, $2, now()) \
|
||||
ON CONFLICT (key) DO UPDATE \
|
||||
SET value = EXCLUDED.value, updated_at = now()",
|
||||
)
|
||||
.bind(key)
|
||||
.bind(value)
|
||||
.execute(executor)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Write the value only if the row is absent (the env → DB boot seed). Returns
|
||||
/// `true` when a row was inserted, `false` when one already existed. Uses
|
||||
/// `ON CONFLICT DO NOTHING` so a concurrent boot can't race two seeds.
|
||||
pub async fn seed_if_absent(
|
||||
pool: &PgPool,
|
||||
key: &str,
|
||||
value: &serde_json::Value,
|
||||
) -> sqlx::Result<bool> {
|
||||
let result = sqlx::query(
|
||||
"INSERT INTO app_settings (key, value) VALUES ($1, $2) \
|
||||
ON CONFLICT (key) DO NOTHING",
|
||||
)
|
||||
.bind(key)
|
||||
.bind(value)
|
||||
.execute(pool)
|
||||
.await?;
|
||||
Ok(result.rows_affected() > 0)
|
||||
}
|
||||
@@ -138,14 +138,18 @@ pub async fn page_count(pool: &PgPool, id: Uuid) -> sqlx::Result<Option<i32>> {
|
||||
/// filter — this resolver stays in lockstep so a chapter that was
|
||||
/// dropped between enqueue and lease isn't dispatched against a stale
|
||||
/// URL.
|
||||
/// Returns `(manga_id, source_url, manga_title, chapter_number)`. The
|
||||
/// title + number feed the live "currently crawling" status; the rest is
|
||||
/// what the dispatcher needs to do the work.
|
||||
pub async fn dispatch_target(
|
||||
pool: &PgPool,
|
||||
chapter_id: Uuid,
|
||||
) -> sqlx::Result<Option<(Uuid, String)>> {
|
||||
) -> sqlx::Result<Option<(Uuid, String, String, i32)>> {
|
||||
sqlx::query_as(
|
||||
"SELECT c.manga_id, cs.source_url \
|
||||
"SELECT c.manga_id, cs.source_url, m.title, c.number \
|
||||
FROM chapters c \
|
||||
JOIN chapter_sources cs ON cs.chapter_id = c.id \
|
||||
JOIN mangas m ON m.id = c.manga_id \
|
||||
WHERE c.id = $1 \
|
||||
AND cs.dropped_at IS NULL \
|
||||
ORDER BY cs.last_seen_at DESC \
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
use sqlx::PgPool;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::domain::collection::{Collection, CollectionSummary};
|
||||
use crate::domain::collection::{Collection, CollectionPageItem, CollectionSummary};
|
||||
use crate::domain::manga::Manga;
|
||||
use crate::error::{AppError, AppResult};
|
||||
|
||||
@@ -278,3 +278,132 @@ pub async fn list_collections_containing(
|
||||
.await?;
|
||||
Ok(rows.into_iter().map(|(id,)| id).collect())
|
||||
}
|
||||
|
||||
/// Add a page to a collection. Same `(true → 201, false → 200)`
|
||||
/// idempotency contract as `add_manga`. FK violations (page deleted
|
||||
/// between the handler's existence check and this insert) surface as
|
||||
/// `NotFound`, not a 500.
|
||||
pub async fn add_page(
|
||||
pool: &PgPool,
|
||||
collection_id: Uuid,
|
||||
page_id: Uuid,
|
||||
) -> AppResult<bool> {
|
||||
let mut tx = pool.begin().await?;
|
||||
let inserted = sqlx::query(
|
||||
r#"
|
||||
INSERT INTO collection_pages (collection_id, page_id)
|
||||
VALUES ($1, $2)
|
||||
ON CONFLICT DO NOTHING
|
||||
"#,
|
||||
)
|
||||
.bind(collection_id)
|
||||
.bind(page_id)
|
||||
.execute(&mut *tx)
|
||||
.await
|
||||
.map_err(|e| match e {
|
||||
sqlx::Error::Database(ref db_err) if db_err.is_foreign_key_violation() => {
|
||||
AppError::NotFound
|
||||
}
|
||||
other => AppError::Database(other),
|
||||
})?;
|
||||
let rows_affected = inserted.rows_affected();
|
||||
if rows_affected > 0 {
|
||||
sqlx::query("UPDATE collections SET updated_at = now() WHERE id = $1")
|
||||
.bind(collection_id)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
}
|
||||
tx.commit().await?;
|
||||
Ok(rows_affected > 0)
|
||||
}
|
||||
|
||||
pub async fn remove_page(
|
||||
pool: &PgPool,
|
||||
collection_id: Uuid,
|
||||
page_id: Uuid,
|
||||
) -> AppResult<()> {
|
||||
let mut tx = pool.begin().await?;
|
||||
let rows_affected = sqlx::query(
|
||||
"DELETE FROM collection_pages WHERE collection_id = $1 AND page_id = $2",
|
||||
)
|
||||
.bind(collection_id)
|
||||
.bind(page_id)
|
||||
.execute(&mut *tx)
|
||||
.await?
|
||||
.rows_affected();
|
||||
if rows_affected > 0 {
|
||||
sqlx::query("UPDATE collections SET updated_at = now() WHERE id = $1")
|
||||
.bind(collection_id)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
}
|
||||
tx.commit().await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Paged list of `collection_id`'s pages, JOINed through chapters and
|
||||
/// mangas so each row carries the breadcrumb the detail view needs.
|
||||
pub async fn list_pages(
|
||||
pool: &PgPool,
|
||||
collection_id: Uuid,
|
||||
limit: i64,
|
||||
offset: i64,
|
||||
) -> AppResult<(Vec<CollectionPageItem>, i64)> {
|
||||
let rows = sqlx::query_as::<_, CollectionPageItem>(
|
||||
r#"
|
||||
SELECT
|
||||
p.id AS page_id,
|
||||
p.chapter_id AS chapter_id,
|
||||
ch.manga_id AS manga_id,
|
||||
p.page_number AS page_number,
|
||||
ch.number AS chapter_number,
|
||||
ch.title AS chapter_title,
|
||||
m.title AS manga_title,
|
||||
p.storage_key AS storage_key,
|
||||
cp.added_at AS added_at
|
||||
FROM collection_pages cp
|
||||
JOIN pages p ON p.id = cp.page_id
|
||||
JOIN chapters ch ON ch.id = p.chapter_id
|
||||
JOIN mangas m ON m.id = ch.manga_id
|
||||
WHERE cp.collection_id = $1
|
||||
ORDER BY cp.added_at DESC, p.id
|
||||
LIMIT $2 OFFSET $3
|
||||
"#,
|
||||
)
|
||||
.bind(collection_id)
|
||||
.bind(limit)
|
||||
.bind(offset)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
let (total,): (i64,) =
|
||||
sqlx::query_as("SELECT count(*) FROM collection_pages WHERE collection_id = $1")
|
||||
.bind(collection_id)
|
||||
.fetch_one(pool)
|
||||
.await?;
|
||||
Ok((rows, total))
|
||||
}
|
||||
|
||||
/// Which of `user_id`'s collections currently contain `page_id`?
|
||||
/// Powers the reader context menu's "In N collections" line and the
|
||||
/// "Add to collection" pre-check.
|
||||
pub async fn list_collections_containing_page(
|
||||
pool: &PgPool,
|
||||
user_id: Uuid,
|
||||
page_id: Uuid,
|
||||
) -> AppResult<Vec<Uuid>> {
|
||||
let rows: Vec<(Uuid,)> = sqlx::query_as(
|
||||
r#"
|
||||
SELECT c.id
|
||||
FROM collections c
|
||||
JOIN collection_pages cp ON cp.collection_id = c.id
|
||||
WHERE c.user_id = $1
|
||||
AND cp.page_id = $2
|
||||
ORDER BY c.updated_at DESC
|
||||
"#,
|
||||
)
|
||||
.bind(user_id)
|
||||
.bind(page_id)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
Ok(rows.into_iter().map(|(id,)| id).collect())
|
||||
}
|
||||
|
||||
@@ -17,8 +17,9 @@
|
||||
//! Each public function is a transaction boundary so a partial failure
|
||||
//! mid-call leaves the DB in its pre-call state.
|
||||
|
||||
use chrono::Utc;
|
||||
use sqlx::{PgPool, Postgres, Transaction};
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde::Serialize;
|
||||
use sqlx::{FromRow, PgPool, Postgres, Transaction};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::crawler::source::{SourceChapterRef, SourceManga};
|
||||
@@ -618,3 +619,424 @@ pub async fn last_run_completed_cleanly(
|
||||
.unwrap_or(true))
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Dead-letter jobs: admin observability + requeue.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// A `dead` crawler job joined to its chapter/manga context for the admin
|
||||
/// dead-letter view. Chapter columns are `Option` because the join is
|
||||
/// best-effort (the chapter may have been removed since the job died, or
|
||||
/// the job may be a non-chapter kind).
|
||||
#[derive(Debug, Clone, Serialize, FromRow)]
|
||||
pub struct DeadJob {
|
||||
pub id: Uuid,
|
||||
pub kind: String,
|
||||
pub chapter_id: Option<Uuid>,
|
||||
pub manga_id: Option<Uuid>,
|
||||
pub manga_title: Option<String>,
|
||||
pub chapter_number: Option<i32>,
|
||||
pub attempts: i32,
|
||||
pub max_attempts: i32,
|
||||
pub last_error: Option<String>,
|
||||
pub updated_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
/// Paginated list of `dead` jobs, newest-failed first, joined to chapter +
|
||||
/// manga context. `search` filters on manga title (case-insensitive
|
||||
/// substring). Returns the page slice plus the unfiltered-by-page total.
|
||||
pub async fn list_dead_jobs(
|
||||
pool: &PgPool,
|
||||
search: Option<&str>,
|
||||
limit: i64,
|
||||
offset: i64,
|
||||
) -> sqlx::Result<(Vec<DeadJob>, i64)> {
|
||||
let search_pat = search
|
||||
.map(|s| format!("%{}%", s.trim()))
|
||||
.filter(|p| p.len() > 2);
|
||||
|
||||
let items: Vec<DeadJob> = sqlx::query_as(
|
||||
r#"
|
||||
SELECT
|
||||
cj.id,
|
||||
cj.payload->>'kind' AS kind,
|
||||
(cj.payload->>'chapter_id')::uuid AS chapter_id,
|
||||
c.manga_id AS manga_id,
|
||||
m.title AS manga_title,
|
||||
c.number AS chapter_number,
|
||||
cj.attempts,
|
||||
cj.max_attempts,
|
||||
cj.last_error,
|
||||
cj.updated_at
|
||||
FROM crawler_jobs cj
|
||||
LEFT JOIN chapters c ON c.id = (cj.payload->>'chapter_id')::uuid
|
||||
LEFT JOIN mangas m ON m.id = c.manga_id
|
||||
WHERE cj.state = 'dead'
|
||||
AND ($1::text IS NULL OR m.title ILIKE $1)
|
||||
ORDER BY cj.updated_at DESC
|
||||
LIMIT $2 OFFSET $3
|
||||
"#,
|
||||
)
|
||||
.bind(&search_pat)
|
||||
.bind(limit)
|
||||
.bind(offset)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
|
||||
let total: i64 = sqlx::query_scalar(
|
||||
r#"
|
||||
SELECT COUNT(*)
|
||||
FROM crawler_jobs cj
|
||||
LEFT JOIN chapters c ON c.id = (cj.payload->>'chapter_id')::uuid
|
||||
LEFT JOIN mangas m ON m.id = c.manga_id
|
||||
WHERE cj.state = 'dead'
|
||||
AND ($1::text IS NULL OR m.title ILIKE $1)
|
||||
"#,
|
||||
)
|
||||
.bind(&search_pat)
|
||||
.fetch_one(pool)
|
||||
.await?;
|
||||
|
||||
Ok((items, total))
|
||||
}
|
||||
|
||||
/// An in-flight chapter-content job (`pending` or `running`) joined to its
|
||||
/// chapter + manga, for the "queued chapters" admin view.
|
||||
#[derive(Debug, Clone, Serialize, FromRow)]
|
||||
pub struct ActiveJob {
|
||||
pub id: Uuid,
|
||||
pub chapter_id: Option<Uuid>,
|
||||
pub manga_id: Option<Uuid>,
|
||||
pub manga_title: Option<String>,
|
||||
pub chapter_number: Option<i32>,
|
||||
/// `"pending"` or `"running"`.
|
||||
pub state: String,
|
||||
pub attempts: i32,
|
||||
pub max_attempts: i32,
|
||||
pub updated_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
/// Paginated list of `pending`/`running` chapter-content jobs (which
|
||||
/// chapters of which mangas are queued or being crawled). Running first,
|
||||
/// then by scheduled order. `search` filters on manga title.
|
||||
pub async fn list_active_jobs(
|
||||
pool: &PgPool,
|
||||
search: Option<&str>,
|
||||
limit: i64,
|
||||
offset: i64,
|
||||
) -> sqlx::Result<(Vec<ActiveJob>, i64)> {
|
||||
let search_pat = search
|
||||
.map(|s| format!("%{}%", s.trim()))
|
||||
.filter(|p| p.len() > 2);
|
||||
|
||||
let items: Vec<ActiveJob> = sqlx::query_as(
|
||||
r#"
|
||||
SELECT
|
||||
cj.id,
|
||||
(cj.payload->>'chapter_id')::uuid AS chapter_id,
|
||||
c.manga_id AS manga_id,
|
||||
m.title AS manga_title,
|
||||
c.number AS chapter_number,
|
||||
cj.state,
|
||||
cj.attempts,
|
||||
cj.max_attempts,
|
||||
cj.updated_at
|
||||
FROM crawler_jobs cj
|
||||
LEFT JOIN chapters c ON c.id = (cj.payload->>'chapter_id')::uuid
|
||||
LEFT JOIN mangas m ON m.id = c.manga_id
|
||||
WHERE cj.state IN ('pending','running')
|
||||
AND cj.payload->>'kind' = 'sync_chapter_content'
|
||||
AND ($1::text IS NULL OR m.title ILIKE $1)
|
||||
ORDER BY (cj.state = 'running') DESC, cj.scheduled_at, cj.created_at
|
||||
LIMIT $2 OFFSET $3
|
||||
"#,
|
||||
)
|
||||
.bind(&search_pat)
|
||||
.bind(limit)
|
||||
.bind(offset)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
|
||||
let total: i64 = sqlx::query_scalar(
|
||||
r#"
|
||||
SELECT COUNT(*)
|
||||
FROM crawler_jobs cj
|
||||
LEFT JOIN chapters c ON c.id = (cj.payload->>'chapter_id')::uuid
|
||||
LEFT JOIN mangas m ON m.id = c.manga_id
|
||||
WHERE cj.state IN ('pending','running')
|
||||
AND cj.payload->>'kind' = 'sync_chapter_content'
|
||||
AND ($1::text IS NULL OR m.title ILIKE $1)
|
||||
"#,
|
||||
)
|
||||
.bind(&search_pat)
|
||||
.fetch_one(pool)
|
||||
.await?;
|
||||
|
||||
Ok((items, total))
|
||||
}
|
||||
|
||||
/// A manga whose cover is still missing (queued for cover fetch).
|
||||
#[derive(Debug, Clone, Serialize, FromRow)]
|
||||
pub struct MissingCoverRow {
|
||||
pub manga_id: Uuid,
|
||||
pub manga_title: String,
|
||||
}
|
||||
|
||||
/// Count mangas with no cover yet but a live source row — the cover
|
||||
/// backlog the metadata pass + backfill drain.
|
||||
pub async fn count_missing_covers(pool: &PgPool) -> sqlx::Result<i64> {
|
||||
sqlx::query_scalar(
|
||||
r#"
|
||||
SELECT COUNT(*) FROM mangas m
|
||||
WHERE m.cover_image_path IS NULL
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM manga_sources ms
|
||||
WHERE ms.manga_id = m.id AND ms.dropped_at IS NULL
|
||||
)
|
||||
"#,
|
||||
)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Paginated list of mangas queued for a cover fetch (no cover yet + a live
|
||||
/// source), with titles. `search` filters on title. Freshest source first.
|
||||
pub async fn list_missing_cover_mangas(
|
||||
pool: &PgPool,
|
||||
search: Option<&str>,
|
||||
limit: i64,
|
||||
offset: i64,
|
||||
) -> sqlx::Result<(Vec<MissingCoverRow>, i64)> {
|
||||
let search_pat = search
|
||||
.map(|s| format!("%{}%", s.trim()))
|
||||
.filter(|p| p.len() > 2);
|
||||
|
||||
let items: Vec<MissingCoverRow> = sqlx::query_as(
|
||||
r#"
|
||||
SELECT m.id AS manga_id, m.title AS manga_title
|
||||
FROM mangas m
|
||||
WHERE m.cover_image_path IS NULL
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM manga_sources ms
|
||||
WHERE ms.manga_id = m.id AND ms.dropped_at IS NULL
|
||||
)
|
||||
AND ($1::text IS NULL OR m.title ILIKE $1)
|
||||
ORDER BY m.updated_at DESC
|
||||
LIMIT $2 OFFSET $3
|
||||
"#,
|
||||
)
|
||||
.bind(&search_pat)
|
||||
.bind(limit)
|
||||
.bind(offset)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
|
||||
let total: i64 = sqlx::query_scalar(
|
||||
r#"
|
||||
SELECT COUNT(*) FROM mangas m
|
||||
WHERE m.cover_image_path IS NULL
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM manga_sources ms
|
||||
WHERE ms.manga_id = m.id AND ms.dropped_at IS NULL
|
||||
)
|
||||
AND ($1::text IS NULL OR m.title ILIKE $1)
|
||||
"#,
|
||||
)
|
||||
.bind(&search_pat)
|
||||
.fetch_one(pool)
|
||||
.await?;
|
||||
|
||||
Ok((items, total))
|
||||
}
|
||||
|
||||
/// Scope of a dead-job requeue.
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum RequeueScope {
|
||||
/// Every dead job.
|
||||
All,
|
||||
/// Dead jobs whose chapter belongs to this manga.
|
||||
Manga(Uuid),
|
||||
/// Dead jobs for a single chapter.
|
||||
Chapter(Uuid),
|
||||
/// A single dead job by its id.
|
||||
Job(Uuid),
|
||||
}
|
||||
|
||||
/// Requeue dead jobs back to `pending` with a fresh attempt budget. This is
|
||||
/// an explicit operator override, so it bypasses the dead-letter quarantine
|
||||
/// the enqueue helpers honour (we act directly on the row). Returns the
|
||||
/// number of rows requeued.
|
||||
///
|
||||
/// Two invariants protect the partial unique dedup index
|
||||
/// `crawler_jobs_chapter_content_dedup_idx` (one `pending|running`
|
||||
/// sync_chapter_content job per chapter):
|
||||
/// 1. A chapter that already has a live (`pending|running`) job is
|
||||
/// skipped entirely (`NO_LIVE_DUP`).
|
||||
/// 2. When a chapter has *multiple* dead jobs, only the newest is
|
||||
/// revived (`DISTINCT ON` the chapter key) — without this, flipping
|
||||
/// two dead rows for the same chapter to `pending` in one statement
|
||||
/// would violate the index and abort the whole requeue. Non-chapter
|
||||
/// jobs fall back to their row id so each stays distinct.
|
||||
pub async fn requeue_dead_jobs(pool: &PgPool, scope: RequeueScope) -> sqlx::Result<u64> {
|
||||
// One full-shape SQL string per scope. Previously the scope
|
||||
// predicate was spliced via `format!()` from a `&'static str`
|
||||
// match — structurally safe today but fragile against a later
|
||||
// refactor accidentally interpolating a non-literal. Four fixed
|
||||
// queries cost a few duplicated lines but are immune to that
|
||||
// class of bug, and each can be reviewed independently. The CTE
|
||||
// body (DISTINCT ON dedup + NOT EXISTS guard) is identical
|
||||
// everywhere so the duplication is mechanical, not semantic.
|
||||
let q = match scope {
|
||||
RequeueScope::All => sqlx::query(REQUEUE_DEAD_SQL_ALL),
|
||||
RequeueScope::Manga(id) => sqlx::query(REQUEUE_DEAD_SQL_MANGA).bind(id),
|
||||
RequeueScope::Chapter(id) => sqlx::query(REQUEUE_DEAD_SQL_CHAPTER).bind(id),
|
||||
RequeueScope::Job(id) => sqlx::query(REQUEUE_DEAD_SQL_JOB).bind(id),
|
||||
};
|
||||
Ok(q.execute(pool).await?.rows_affected())
|
||||
}
|
||||
|
||||
/// Common shell of the requeue CTE. Each scope variant inlines the
|
||||
/// shell and substitutes its own WHERE clause; the duplication is the
|
||||
/// price of avoiding runtime SQL string assembly. The dedup guarantees
|
||||
/// documented on [`requeue_dead_jobs`] live in the `DISTINCT ON` and
|
||||
/// the `NOT EXISTS` block — keep those identical across the four
|
||||
/// constants below when editing.
|
||||
const REQUEUE_DEAD_SQL_ALL: &str = r#"
|
||||
WITH pick AS (
|
||||
SELECT DISTINCT ON (COALESCE(cj.payload->>'chapter_id', cj.id::text)) cj.id
|
||||
FROM crawler_jobs cj
|
||||
WHERE cj.state = 'dead'
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM crawler_jobs live
|
||||
WHERE live.payload->>'kind' = 'sync_chapter_content'
|
||||
AND live.payload->>'chapter_id' = cj.payload->>'chapter_id'
|
||||
AND live.state IN ('pending','running')
|
||||
)
|
||||
ORDER BY COALESCE(cj.payload->>'chapter_id', cj.id::text), cj.updated_at DESC
|
||||
)
|
||||
UPDATE crawler_jobs
|
||||
SET state = 'pending', attempts = 0, leased_until = NULL,
|
||||
last_error = NULL, scheduled_at = now(), updated_at = now()
|
||||
FROM pick
|
||||
WHERE crawler_jobs.id = pick.id
|
||||
"#;
|
||||
|
||||
const REQUEUE_DEAD_SQL_MANGA: &str = r#"
|
||||
WITH pick AS (
|
||||
SELECT DISTINCT ON (COALESCE(cj.payload->>'chapter_id', cj.id::text)) cj.id
|
||||
FROM crawler_jobs cj
|
||||
WHERE cj.state = 'dead'
|
||||
AND (cj.payload->>'chapter_id')::uuid IN
|
||||
(SELECT id FROM chapters WHERE manga_id = $1)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM crawler_jobs live
|
||||
WHERE live.payload->>'kind' = 'sync_chapter_content'
|
||||
AND live.payload->>'chapter_id' = cj.payload->>'chapter_id'
|
||||
AND live.state IN ('pending','running')
|
||||
)
|
||||
ORDER BY COALESCE(cj.payload->>'chapter_id', cj.id::text), cj.updated_at DESC
|
||||
)
|
||||
UPDATE crawler_jobs
|
||||
SET state = 'pending', attempts = 0, leased_until = NULL,
|
||||
last_error = NULL, scheduled_at = now(), updated_at = now()
|
||||
FROM pick
|
||||
WHERE crawler_jobs.id = pick.id
|
||||
"#;
|
||||
|
||||
const REQUEUE_DEAD_SQL_CHAPTER: &str = r#"
|
||||
WITH pick AS (
|
||||
SELECT DISTINCT ON (COALESCE(cj.payload->>'chapter_id', cj.id::text)) cj.id
|
||||
FROM crawler_jobs cj
|
||||
WHERE cj.state = 'dead'
|
||||
AND (cj.payload->>'chapter_id')::uuid = $1
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM crawler_jobs live
|
||||
WHERE live.payload->>'kind' = 'sync_chapter_content'
|
||||
AND live.payload->>'chapter_id' = cj.payload->>'chapter_id'
|
||||
AND live.state IN ('pending','running')
|
||||
)
|
||||
ORDER BY COALESCE(cj.payload->>'chapter_id', cj.id::text), cj.updated_at DESC
|
||||
)
|
||||
UPDATE crawler_jobs
|
||||
SET state = 'pending', attempts = 0, leased_until = NULL,
|
||||
last_error = NULL, scheduled_at = now(), updated_at = now()
|
||||
FROM pick
|
||||
WHERE crawler_jobs.id = pick.id
|
||||
"#;
|
||||
|
||||
const REQUEUE_DEAD_SQL_JOB: &str = r#"
|
||||
WITH pick AS (
|
||||
SELECT DISTINCT ON (COALESCE(cj.payload->>'chapter_id', cj.id::text)) cj.id
|
||||
FROM crawler_jobs cj
|
||||
WHERE cj.state = 'dead'
|
||||
AND cj.id = $1
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM crawler_jobs live
|
||||
WHERE live.payload->>'kind' = 'sync_chapter_content'
|
||||
AND live.payload->>'chapter_id' = cj.payload->>'chapter_id'
|
||||
AND live.state IN ('pending','running')
|
||||
)
|
||||
ORDER BY COALESCE(cj.payload->>'chapter_id', cj.id::text), cj.updated_at DESC
|
||||
)
|
||||
UPDATE crawler_jobs
|
||||
SET state = 'pending', attempts = 0, leased_until = NULL,
|
||||
last_error = NULL, scheduled_at = now(), updated_at = now()
|
||||
FROM pick
|
||||
WHERE crawler_jobs.id = pick.id
|
||||
"#;
|
||||
|
||||
/// `crawler_state` key under which the runtime session value (an admin
|
||||
/// pushed PHPSESSID) is persisted. Survives a backend restart so a
|
||||
/// mid-day refresh isn't lost.
|
||||
const STATE_KEY_RUNTIME_SESSION: &str = "runtime_session";
|
||||
|
||||
/// Read the persisted runtime PHPSESSID (if any). The payload shape is
|
||||
/// `{ "phpsessid": "<value>" }`; anything else returns `None`.
|
||||
pub async fn runtime_session_load(pool: &PgPool) -> sqlx::Result<Option<String>> {
|
||||
let row: Option<serde_json::Value> =
|
||||
sqlx::query_scalar("SELECT value FROM crawler_state WHERE key = $1")
|
||||
.bind(STATE_KEY_RUNTIME_SESSION)
|
||||
.fetch_optional(pool)
|
||||
.await?;
|
||||
Ok(row.and_then(|v| {
|
||||
v.get("phpsessid")
|
||||
.and_then(|s| s.as_str())
|
||||
.map(|s| s.to_string())
|
||||
}))
|
||||
}
|
||||
|
||||
/// Persist a fresh runtime PHPSESSID, replacing any previous value.
|
||||
pub async fn runtime_session_persist(pool: &PgPool, sid: &str) -> sqlx::Result<()> {
|
||||
sqlx::query(
|
||||
"INSERT INTO crawler_state (key, value, updated_at) \
|
||||
VALUES ($1, $2, now()) \
|
||||
ON CONFLICT (key) DO UPDATE \
|
||||
SET value = EXCLUDED.value, updated_at = now()",
|
||||
)
|
||||
.bind(STATE_KEY_RUNTIME_SESSION)
|
||||
.bind(serde_json::json!({ "phpsessid": sid }))
|
||||
.execute(pool)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Count crawler jobs grouped by state — drives the dashboard queue
|
||||
/// gauges. Returns `(pending, running, dead)`.
|
||||
pub async fn job_state_counts(pool: &PgPool) -> sqlx::Result<(i64, i64, i64)> {
|
||||
let rows: Vec<(String, i64)> =
|
||||
sqlx::query_as("SELECT state, COUNT(*) FROM crawler_jobs GROUP BY state")
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
let mut pending = 0;
|
||||
let mut running = 0;
|
||||
let mut dead = 0;
|
||||
for (state, n) in rows {
|
||||
match state.as_str() {
|
||||
"pending" => pending = n,
|
||||
"running" => running = n,
|
||||
"dead" => dead = n,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
Ok((pending, running, dead))
|
||||
}
|
||||
|
||||
|
||||
@@ -35,13 +35,44 @@ pub struct ListQuery {
|
||||
pub author_ids: Vec<Uuid>,
|
||||
pub genre_ids: Vec<Uuid>,
|
||||
pub tag_ids: Vec<Uuid>,
|
||||
/// Content warnings (canonical lowercase names) the manga must carry on
|
||||
/// at least one page each — AND across the list.
|
||||
pub cw_include: Vec<String>,
|
||||
/// Content warnings the manga must NOT carry on any page.
|
||||
pub cw_exclude: Vec<String>,
|
||||
pub limit: i64,
|
||||
pub offset: i64,
|
||||
pub sort: ListSort,
|
||||
}
|
||||
|
||||
const SELECT_COLS: &str =
|
||||
"id, title, status, alt_titles, description, cover_image_path, created_at, updated_at";
|
||||
/// Single source of truth for the `mangas` columns that hydrate a [`Manga`],
|
||||
/// so the plain and join-aliased select lists stay in lockstep with the
|
||||
/// struct's `FromRow`.
|
||||
const MANGA_COLS: [&str; 8] = [
|
||||
"id",
|
||||
"title",
|
||||
"status",
|
||||
"alt_titles",
|
||||
"description",
|
||||
"cover_image_path",
|
||||
"created_at",
|
||||
"updated_at",
|
||||
];
|
||||
|
||||
/// `MANGA_COLS` rendered as a select list. A non-empty `alias` qualifies each
|
||||
/// column (`"m"` → `m.id, m.title, …`) for queries that join other tables;
|
||||
/// an empty alias yields the bare names used by single-table queries.
|
||||
fn manga_cols(alias: &str) -> String {
|
||||
if alias.is_empty() {
|
||||
MANGA_COLS.join(", ")
|
||||
} else {
|
||||
MANGA_COLS
|
||||
.iter()
|
||||
.map(|c| format!("{alias}.{c}"))
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ")
|
||||
}
|
||||
}
|
||||
|
||||
/// Shared WHERE used by both the rows and the count queries. Filters
|
||||
/// are AND across facets: every supplied author_id (or genre, or tag)
|
||||
@@ -81,6 +112,21 @@ const FILTER_WHERE: &str = r#"
|
||||
WHERE mt.manga_id = mangas.id AND mt.tag_id = req.id
|
||||
)
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM unnest($6::text[]) AS req(w)
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM page_content_warnings pw
|
||||
JOIN pages p ON p.id = pw.page_id
|
||||
JOIN chapters c ON c.id = p.chapter_id
|
||||
WHERE c.manga_id = mangas.id AND pw.warning = req.w
|
||||
)
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM page_content_warnings pw
|
||||
JOIN pages p ON p.id = pw.page_id
|
||||
JOIN chapters c ON c.id = p.chapter_id
|
||||
WHERE c.manga_id = mangas.id AND pw.warning = ANY($7::text[])
|
||||
)
|
||||
"#;
|
||||
|
||||
/// Returns the page of mangas matching `query` plus the unfiltered total
|
||||
@@ -100,12 +146,13 @@ pub async fn list(pool: &PgPool, query: &ListQuery) -> AppResult<(Vec<Manga>, i6
|
||||
|
||||
let list_sql = format!(
|
||||
r#"
|
||||
SELECT {SELECT_COLS}
|
||||
SELECT {cols}
|
||||
FROM mangas
|
||||
WHERE {FILTER_WHERE}
|
||||
ORDER BY {order_by}
|
||||
LIMIT $6 OFFSET $7
|
||||
"#
|
||||
LIMIT $8 OFFSET $9
|
||||
"#,
|
||||
cols = manga_cols(""),
|
||||
);
|
||||
|
||||
let rows = sqlx::query_as::<_, Manga>(&list_sql)
|
||||
@@ -114,6 +161,8 @@ pub async fn list(pool: &PgPool, query: &ListQuery) -> AppResult<(Vec<Manga>, i6
|
||||
.bind(&query.author_ids)
|
||||
.bind(&query.genre_ids)
|
||||
.bind(&query.tag_ids)
|
||||
.bind(&query.cw_include)
|
||||
.bind(&query.cw_exclude)
|
||||
.bind(query.limit)
|
||||
.bind(query.offset)
|
||||
.fetch_all(pool)
|
||||
@@ -131,6 +180,8 @@ pub async fn list(pool: &PgPool, query: &ListQuery) -> AppResult<(Vec<Manga>, i6
|
||||
.bind(&query.author_ids)
|
||||
.bind(&query.genre_ids)
|
||||
.bind(&query.tag_ids)
|
||||
.bind(&query.cw_include)
|
||||
.bind(&query.cw_exclude)
|
||||
.fetch_one(pool)
|
||||
.await?;
|
||||
|
||||
@@ -145,9 +196,73 @@ pub async fn list_cards(
|
||||
query: &ListQuery,
|
||||
) -> AppResult<(Vec<MangaCard>, i64)> {
|
||||
let (rows, total) = list(pool, query).await?;
|
||||
let cards = cards_from_rows(pool, rows).await?;
|
||||
Ok((cards, total))
|
||||
}
|
||||
|
||||
/// Top-`limit` mangas ranked by tag similarity to `id`, as cards.
|
||||
///
|
||||
/// Similarity is the Jaccard index of the two tag sets —
|
||||
/// `shared / (|base| + |other| - shared)` — so a heavily-tagged manga
|
||||
/// sharing many generic tags does not crowd out genuinely-close matches.
|
||||
/// (A simpler raw shared-tag count is one ORDER BY term away, but it
|
||||
/// biases toward mangas with large tag sets; see the tie-break below.)
|
||||
///
|
||||
/// The candidate set comes from the self-join on `manga_tags`, which by
|
||||
/// construction only includes mangas sharing at least one tag and never
|
||||
/// the source itself. A source with no tags — or no overlap — yields an
|
||||
/// empty result. Existence of `id` is the caller's concern (the empty
|
||||
/// result here cannot distinguish "no tags" from "no such manga").
|
||||
pub async fn list_similar(
|
||||
pool: &PgPool,
|
||||
id: Uuid,
|
||||
limit: i64,
|
||||
) -> AppResult<Vec<MangaCard>> {
|
||||
let sql = format!(
|
||||
r#"
|
||||
SELECT {cols}
|
||||
FROM manga_tags base
|
||||
JOIN manga_tags other
|
||||
ON other.tag_id = base.tag_id
|
||||
AND other.manga_id <> base.manga_id
|
||||
JOIN mangas m ON m.id = other.manga_id
|
||||
WHERE base.manga_id = $1
|
||||
GROUP BY m.id
|
||||
ORDER BY
|
||||
count(*)::float
|
||||
/ ( (SELECT count(*) FROM manga_tags WHERE manga_id = $1)
|
||||
+ (SELECT count(*) FROM manga_tags WHERE manga_id = m.id)
|
||||
- count(*) ) DESC,
|
||||
count(*) DESC,
|
||||
m.updated_at DESC,
|
||||
lower(m.title) ASC,
|
||||
m.id
|
||||
LIMIT $2
|
||||
"#,
|
||||
// The query joins `manga_tags`, so the mangas columns need the `m.` alias.
|
||||
cols = manga_cols("m"),
|
||||
);
|
||||
|
||||
let rows = sqlx::query_as::<_, Manga>(&sql)
|
||||
.bind(id)
|
||||
.bind(limit)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
|
||||
cards_from_rows(pool, rows).await
|
||||
}
|
||||
|
||||
/// Hydrate a batch of `Manga` rows into `MangaCard`s by attaching their
|
||||
/// authors and genres in two batched round-trips. The input order is
|
||||
/// preserved (callers rely on this to keep list/ranking order), so we
|
||||
/// iterate `rows` rather than the id-ordered `BTreeMap`s.
|
||||
async fn cards_from_rows(pool: &PgPool, rows: Vec<Manga>) -> AppResult<Vec<MangaCard>> {
|
||||
let ids: Vec<Uuid> = rows.iter().map(|m| m.id).collect();
|
||||
let mut authors = repo::author::load_for_mangas(pool, &ids).await?;
|
||||
let mut genres = repo::genre::load_for_mangas(pool, &ids).await?;
|
||||
// Authors and genres are independent reads — load them concurrently.
|
||||
let (mut authors, mut genres) = tokio::try_join!(
|
||||
repo::author::load_for_mangas(pool, &ids),
|
||||
repo::genre::load_for_mangas(pool, &ids),
|
||||
)?;
|
||||
let cards = rows
|
||||
.into_iter()
|
||||
.map(|manga| MangaCard {
|
||||
@@ -156,12 +271,13 @@ pub async fn list_cards(
|
||||
manga,
|
||||
})
|
||||
.collect();
|
||||
Ok((cards, total))
|
||||
Ok(cards)
|
||||
}
|
||||
|
||||
pub async fn get(pool: &PgPool, id: Uuid) -> AppResult<Manga> {
|
||||
sqlx::query_as::<_, Manga>(&format!(
|
||||
"SELECT {SELECT_COLS} FROM mangas WHERE id = $1"
|
||||
"SELECT {} FROM mangas WHERE id = $1",
|
||||
manga_cols("")
|
||||
))
|
||||
.bind(id)
|
||||
.fetch_optional(pool)
|
||||
@@ -174,7 +290,8 @@ pub async fn get_detail(pool: &PgPool, id: Uuid) -> AppResult<MangaDetail> {
|
||||
let authors = repo::author::list_for_manga(pool, id).await?;
|
||||
let genres = repo::genre::list_for_manga(pool, id).await?;
|
||||
let tags = repo::tag::list_for_manga(pool, id).await?;
|
||||
Ok(MangaDetail { manga, authors, genres, tags })
|
||||
let content_warnings = repo::page_analysis::warnings_for_manga(pool, id).await?;
|
||||
Ok(MangaDetail { manga, authors, genres, tags, content_warnings })
|
||||
}
|
||||
|
||||
/// Insert just the manga row. Relations (authors, genres) are written
|
||||
@@ -198,8 +315,9 @@ pub async fn create<'e, E: PgExecutor<'e>>(
|
||||
r#"
|
||||
INSERT INTO mangas (title, status, description, alt_titles, uploaded_by)
|
||||
VALUES ($1, $2, $3, $4, $5)
|
||||
RETURNING {SELECT_COLS}
|
||||
"#
|
||||
RETURNING {cols}
|
||||
"#,
|
||||
cols = manga_cols(""),
|
||||
))
|
||||
.bind(title)
|
||||
.bind(status)
|
||||
@@ -234,8 +352,9 @@ pub async fn update_basics(
|
||||
alt_titles = COALESCE($6, alt_titles),
|
||||
updated_at = now()
|
||||
WHERE id = $1
|
||||
RETURNING {SELECT_COLS}
|
||||
"#
|
||||
RETURNING {cols}
|
||||
"#,
|
||||
cols = manga_cols(""),
|
||||
))
|
||||
.bind(id)
|
||||
.bind(title)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
pub mod admin_audit;
|
||||
pub mod admin_view;
|
||||
pub mod api_token;
|
||||
pub mod app_settings;
|
||||
pub mod author;
|
||||
pub mod bookmark;
|
||||
pub mod chapter;
|
||||
@@ -9,6 +10,8 @@ pub mod crawler;
|
||||
pub mod genre;
|
||||
pub mod manga;
|
||||
pub mod page;
|
||||
pub mod page_analysis;
|
||||
pub mod page_tag;
|
||||
pub mod read_progress;
|
||||
pub mod session;
|
||||
pub mod tag;
|
||||
|
||||
@@ -29,6 +29,35 @@ pub async fn create<'e, E: PgExecutor<'e>>(
|
||||
Ok(row)
|
||||
}
|
||||
|
||||
/// Fetch a single page by id. `None` when it doesn't exist (e.g. deleted
|
||||
/// between an analysis job's enqueue and its dispatch).
|
||||
pub async fn find_by_id(pool: &PgPool, id: Uuid) -> AppResult<Option<Page>> {
|
||||
let row = sqlx::query_as::<_, Page>(
|
||||
"SELECT id, chapter_id, page_number, storage_key, content_type \
|
||||
FROM pages WHERE id = $1",
|
||||
)
|
||||
.bind(id)
|
||||
.fetch_optional(pool)
|
||||
.await?;
|
||||
Ok(row)
|
||||
}
|
||||
|
||||
/// Resolve a page's breadcrumb `(manga_id, chapter_id, page_number)` for
|
||||
/// live event payloads. `None` if the page doesn't exist.
|
||||
pub async fn locate(
|
||||
pool: &PgPool,
|
||||
page_id: Uuid,
|
||||
) -> AppResult<Option<(Uuid, Uuid, i32)>> {
|
||||
let row: Option<(Uuid, Uuid, i32)> = sqlx::query_as(
|
||||
"SELECT c.manga_id, p.chapter_id, p.page_number \
|
||||
FROM pages p JOIN chapters c ON c.id = p.chapter_id WHERE p.id = $1",
|
||||
)
|
||||
.bind(page_id)
|
||||
.fetch_optional(pool)
|
||||
.await?;
|
||||
Ok(row)
|
||||
}
|
||||
|
||||
pub async fn list_for_chapter(pool: &PgPool, chapter_id: Uuid) -> AppResult<Vec<Page>> {
|
||||
let rows = sqlx::query_as::<_, Page>(
|
||||
r#"
|
||||
|
||||
591
backend/src/repo/page_analysis.rs
Normal file
591
backend/src/repo/page_analysis.rs
Normal file
@@ -0,0 +1,591 @@
|
||||
//! Persistence for AI page-analysis results.
|
||||
//!
|
||||
//! [`persist_analysis`] is the single transactional writer: it replaces a
|
||||
//! page's OCR text, global auto-tags, and content warnings, upserts the
|
||||
//! `page_analysis` row, and computes the kind-weighted `search_doc`
|
||||
//! tsvector — all atomically, so re-analysis (delete+reinsert) is
|
||||
//! idempotent and never leaves a half-written page. It deliberately does
|
||||
//! NOT touch the per-user `page_tags` table.
|
||||
//!
|
||||
//! Auto-tag names resolve to the shared `tags` vocabulary via
|
||||
//! [`crate::repo::tag::upsert_by_name`], the same path manga tags and
|
||||
//! personal page tags use.
|
||||
|
||||
use sqlx::PgPool;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::crawler::jobs::{self, JobPayload};
|
||||
use crate::domain::page_analysis::{
|
||||
ChapterCoverage, ContentWarning, MangaCoverage, OcrKind, OcrLine, PageAnalysis,
|
||||
PageAnalysisDetail, PageSearchItem, PageStatusItem, VisionAnalysis,
|
||||
};
|
||||
use crate::error::AppResult;
|
||||
|
||||
/// Filter set for [`page_search`]. `tags` are AND-ed (a page must carry
|
||||
/// every one, satisfiable by EITHER the user's page tag or a global auto
|
||||
/// tag); `text` ranks via the weighted tsvector; `cw_include` requires all
|
||||
/// listed warnings; `cw_exclude` rejects any. All string values arrive
|
||||
/// pre-normalized (tags lowercased, warnings validated) from the handler.
|
||||
#[derive(Debug, Default, Clone)]
|
||||
pub struct PageSearchQuery {
|
||||
pub user_id: uuid::Uuid,
|
||||
pub tags: Vec<String>,
|
||||
pub text: Option<String>,
|
||||
pub cw_include: Vec<String>,
|
||||
pub cw_exclude: Vec<String>,
|
||||
pub limit: i64,
|
||||
pub offset: i64,
|
||||
}
|
||||
|
||||
/// Longest tag the shared `tags` table accepts (`upsert_by_name` enforces
|
||||
/// the same bound). Auto-tags over this are dropped here rather than
|
||||
/// aborting the whole page's analysis on one bad model output.
|
||||
const MAX_TAG_CHARS: usize = 64;
|
||||
|
||||
/// Enqueue an `analyze_page` job for `page_id`. `force` re-analyzes a page
|
||||
/// that is already `done`. Enqueue is idempotent at the job level only in
|
||||
/// that duplicate pending jobs are harmless — processing is idempotent.
|
||||
pub async fn enqueue_for_page(pool: &PgPool, page_id: Uuid, force: bool) -> AppResult<()> {
|
||||
jobs::enqueue(pool, &JobPayload::AnalyzePage { page_id, force }).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// How wide a bulk re-enqueue reaches.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub enum ReenqueueScope {
|
||||
/// Every page in the library.
|
||||
All,
|
||||
/// Every page across all chapters of one manga.
|
||||
Manga(uuid::Uuid),
|
||||
/// Every page of one chapter.
|
||||
Chapter(uuid::Uuid),
|
||||
}
|
||||
|
||||
/// Bulk-enqueue `analyze_page` jobs for existing pages within `scope` — the
|
||||
/// admin backfill / re-analyze path.
|
||||
///
|
||||
/// When `only_unanalyzed` is true, pages that already have a `done`
|
||||
/// analysis row are skipped and the enqueued jobs carry `force = false`.
|
||||
/// When false, ALL in-scope pages are enqueued with `force = true` so the
|
||||
/// worker re-analyzes even already-done pages (otherwise the worker's
|
||||
/// skip-if-done net would no-op them). Pages with a pending/running
|
||||
/// `analyze_page` job are always skipped so repeated calls don't pile up
|
||||
/// duplicates. Returns the number of jobs enqueued.
|
||||
pub async fn enqueue_pages(
|
||||
pool: &PgPool,
|
||||
scope: ReenqueueScope,
|
||||
only_unanalyzed: bool,
|
||||
) -> AppResult<u64> {
|
||||
// Scope predicate; the bound uuid (when present) is always $2.
|
||||
let scope_clause = match scope {
|
||||
ReenqueueScope::All => "",
|
||||
ReenqueueScope::Manga(_) => {
|
||||
"AND p.chapter_id IN (SELECT id FROM chapters WHERE manga_id = $2)"
|
||||
}
|
||||
ReenqueueScope::Chapter(_) => "AND p.chapter_id = $2",
|
||||
};
|
||||
let sql = format!(
|
||||
r#"
|
||||
INSERT INTO crawler_jobs (payload)
|
||||
SELECT jsonb_build_object('kind', 'analyze_page', 'page_id', p.id, 'force', NOT $1)
|
||||
FROM pages p
|
||||
WHERE ($1 = false OR NOT EXISTS (
|
||||
SELECT 1 FROM page_analysis pa
|
||||
WHERE pa.page_id = p.id AND pa.status = 'done'))
|
||||
{scope_clause}
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM crawler_jobs j
|
||||
WHERE j.payload->>'kind' = 'analyze_page'
|
||||
AND j.payload->>'page_id' = p.id::text
|
||||
AND j.state IN ('pending', 'running'))
|
||||
"#
|
||||
);
|
||||
let query = sqlx::query(&sql).bind(only_unanalyzed);
|
||||
let query = match scope {
|
||||
ReenqueueScope::All => query,
|
||||
ReenqueueScope::Manga(id) | ReenqueueScope::Chapter(id) => query.bind(id),
|
||||
};
|
||||
Ok(query.execute(pool).await?.rows_affected())
|
||||
}
|
||||
|
||||
/// Per-manga analysis coverage for the admin overview. Only mangas that
|
||||
/// have at least one page appear (nothing to analyze otherwise). `search`
|
||||
/// filters by title (case-insensitive substring). Ordered by title.
|
||||
/// Returns the page plus the total matching-manga count for pagination.
|
||||
pub async fn manga_coverage(
|
||||
pool: &PgPool,
|
||||
search: Option<&str>,
|
||||
limit: i64,
|
||||
offset: i64,
|
||||
) -> AppResult<(Vec<MangaCoverage>, i64)> {
|
||||
let rows = sqlx::query_as::<_, MangaCoverage>(
|
||||
r#"
|
||||
SELECT m.id AS manga_id, m.title,
|
||||
count(p.id) AS total_pages,
|
||||
count(*) FILTER (WHERE pa.status = 'done') AS analyzed_pages
|
||||
FROM mangas m
|
||||
JOIN chapters c ON c.manga_id = m.id
|
||||
JOIN pages p ON p.chapter_id = c.id
|
||||
LEFT JOIN page_analysis pa ON pa.page_id = p.id
|
||||
WHERE ($1::text IS NULL OR m.title ILIKE '%' || $1 || '%')
|
||||
GROUP BY m.id, m.title
|
||||
ORDER BY lower(m.title), m.id
|
||||
LIMIT $2 OFFSET $3
|
||||
"#,
|
||||
)
|
||||
.bind(search)
|
||||
.bind(limit)
|
||||
.bind(offset)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
|
||||
let (total,): (i64,) = sqlx::query_as(
|
||||
r#"
|
||||
SELECT count(*) FROM (
|
||||
SELECT m.id
|
||||
FROM mangas m
|
||||
JOIN chapters c ON c.manga_id = m.id
|
||||
JOIN pages p ON p.chapter_id = c.id
|
||||
WHERE ($1::text IS NULL OR m.title ILIKE '%' || $1 || '%')
|
||||
GROUP BY m.id
|
||||
) x
|
||||
"#,
|
||||
)
|
||||
.bind(search)
|
||||
.fetch_one(pool)
|
||||
.await?;
|
||||
Ok((rows, total))
|
||||
}
|
||||
|
||||
/// Per-chapter analysis coverage for one manga, ordered by chapter number.
|
||||
pub async fn chapter_coverage(
|
||||
pool: &PgPool,
|
||||
manga_id: uuid::Uuid,
|
||||
) -> AppResult<Vec<ChapterCoverage>> {
|
||||
let rows = sqlx::query_as::<_, ChapterCoverage>(
|
||||
r#"
|
||||
SELECT c.id AS chapter_id, c.number, c.title,
|
||||
count(p.id) AS total_pages,
|
||||
count(*) FILTER (WHERE pa.status = 'done') AS analyzed_pages
|
||||
FROM chapters c
|
||||
JOIN pages p ON p.chapter_id = c.id
|
||||
LEFT JOIN page_analysis pa ON pa.page_id = p.id
|
||||
WHERE c.manga_id = $1
|
||||
GROUP BY c.id, c.number, c.title
|
||||
ORDER BY c.number, c.id
|
||||
"#,
|
||||
)
|
||||
.bind(manga_id)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
Ok(rows)
|
||||
}
|
||||
|
||||
/// Per-page status grid for a chapter. `status` resolves to the analysis
|
||||
/// row's state (`done`/`failed`), else `queued` when a pending/running
|
||||
/// analyze_page job exists, else `none`.
|
||||
pub async fn chapter_page_status(
|
||||
pool: &PgPool,
|
||||
chapter_id: uuid::Uuid,
|
||||
) -> AppResult<Vec<PageStatusItem>> {
|
||||
let rows = sqlx::query_as::<_, PageStatusItem>(
|
||||
r#"
|
||||
SELECT p.id AS page_id, p.page_number,
|
||||
COALESCE(
|
||||
pa.status,
|
||||
CASE WHEN EXISTS (
|
||||
SELECT 1 FROM crawler_jobs j
|
||||
WHERE j.payload->>'kind' = 'analyze_page'
|
||||
AND j.payload->>'page_id' = p.id::text
|
||||
AND j.state IN ('pending', 'running')
|
||||
) THEN 'queued' ELSE 'none' END
|
||||
) AS status
|
||||
FROM pages p
|
||||
LEFT JOIN page_analysis pa ON pa.page_id = p.id
|
||||
WHERE p.chapter_id = $1
|
||||
ORDER BY p.page_number
|
||||
"#,
|
||||
)
|
||||
.bind(chapter_id)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
Ok(rows)
|
||||
}
|
||||
|
||||
/// Full analysis detail for one page. `None` when the page itself doesn't
|
||||
/// exist; an existing-but-unanalyzed page returns `status = "none"` with
|
||||
/// empty OCR/tags/warnings so the UI can show "not analyzed yet".
|
||||
pub async fn page_detail(
|
||||
pool: &PgPool,
|
||||
page_id: uuid::Uuid,
|
||||
) -> AppResult<Option<PageAnalysisDetail>> {
|
||||
let Some((page_number, chapter_id, manga_id)): Option<(i32, uuid::Uuid, uuid::Uuid)> =
|
||||
sqlx::query_as(
|
||||
"SELECT p.page_number, p.chapter_id, c.manga_id \
|
||||
FROM pages p JOIN chapters c ON c.id = p.chapter_id WHERE p.id = $1",
|
||||
)
|
||||
.bind(page_id)
|
||||
.fetch_optional(pool)
|
||||
.await?
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
let analysis = load(pool, page_id).await?;
|
||||
|
||||
let ocr = sqlx::query_as::<_, OcrLine>(
|
||||
"SELECT kind, text FROM page_ocr_text WHERE page_id = $1 ORDER BY ord, id",
|
||||
)
|
||||
.bind(page_id)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
|
||||
let tags: Vec<String> = sqlx::query_scalar(
|
||||
"SELECT t.name FROM page_auto_tags pat JOIN tags t ON t.id = pat.tag_id \
|
||||
WHERE pat.page_id = $1 ORDER BY t.name",
|
||||
)
|
||||
.bind(page_id)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
|
||||
let content_warnings = sqlx::query_scalar::<_, ContentWarning>(
|
||||
"SELECT warning FROM page_content_warnings WHERE page_id = $1 ORDER BY warning",
|
||||
)
|
||||
.bind(page_id)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
|
||||
Ok(Some(PageAnalysisDetail {
|
||||
page_id,
|
||||
page_number,
|
||||
chapter_id,
|
||||
manga_id,
|
||||
status: analysis
|
||||
.as_ref()
|
||||
.map(|a| format!("{:?}", a.status).to_lowercase())
|
||||
.unwrap_or_else(|| "none".to_string()),
|
||||
is_nsfw: analysis.as_ref().map(|a| a.is_nsfw).unwrap_or(false),
|
||||
scene_description: analysis.as_ref().and_then(|a| a.scene_description.clone()),
|
||||
model: analysis.as_ref().and_then(|a| a.model.clone()),
|
||||
error: analysis.as_ref().and_then(|a| a.error.clone()),
|
||||
analyzed_at: analysis.as_ref().and_then(|a| a.analyzed_at),
|
||||
ocr,
|
||||
tags,
|
||||
content_warnings,
|
||||
}))
|
||||
}
|
||||
|
||||
/// Deduplicated union of content warnings across all of a manga's pages,
|
||||
/// ordered alphabetically. Powers the manga-detail content-warning banner.
|
||||
pub async fn warnings_for_manga(
|
||||
pool: &PgPool,
|
||||
manga_id: uuid::Uuid,
|
||||
) -> AppResult<Vec<ContentWarning>> {
|
||||
let rows = sqlx::query_scalar::<_, ContentWarning>(
|
||||
r#"
|
||||
SELECT DISTINCT pw.warning
|
||||
FROM page_content_warnings pw
|
||||
JOIN pages p ON p.id = pw.page_id
|
||||
JOIN chapters c ON c.id = p.chapter_id
|
||||
WHERE c.manga_id = $1
|
||||
ORDER BY pw.warning
|
||||
"#,
|
||||
)
|
||||
.bind(manga_id)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
Ok(rows)
|
||||
}
|
||||
|
||||
/// Load a page's analysis row, if it has one.
|
||||
pub async fn load(pool: &PgPool, page_id: Uuid) -> AppResult<Option<PageAnalysis>> {
|
||||
let row = sqlx::query_as::<_, PageAnalysis>(
|
||||
r#"
|
||||
SELECT page_id, status, scene_description, is_nsfw, model, error, analyzed_at
|
||||
FROM page_analysis
|
||||
WHERE page_id = $1
|
||||
"#,
|
||||
)
|
||||
.bind(page_id)
|
||||
.fetch_optional(pool)
|
||||
.await?;
|
||||
Ok(row)
|
||||
}
|
||||
|
||||
/// Record that analysis failed terminally for a page (retries exhausted /
|
||||
/// dead-lettered). Leaves a `failed` row so the page's state is
|
||||
/// observable; it simply contributes nothing to search.
|
||||
pub async fn mark_failed(pool: &PgPool, page_id: Uuid, error: &str) -> AppResult<()> {
|
||||
sqlx::query(
|
||||
r#"
|
||||
INSERT INTO page_analysis (page_id, status, error)
|
||||
VALUES ($1, 'failed', $2)
|
||||
ON CONFLICT (page_id) DO UPDATE
|
||||
SET status = 'failed', error = EXCLUDED.error, updated_at = now()
|
||||
"#,
|
||||
)
|
||||
.bind(page_id)
|
||||
.bind(error)
|
||||
.execute(pool)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Persist a completed analysis for a page, replacing any previous result.
|
||||
///
|
||||
/// The model's free-form `kind` / `content_type` strings are mapped onto
|
||||
/// the closed vocabularies here ([`OcrKind::from_model_str`] /
|
||||
/// [`ContentWarning::from_model_str`]); unrecognized warnings are dropped,
|
||||
/// unknown OCR kinds fall back to `narration`. Empty / over-long tags and
|
||||
/// empty OCR text are skipped so one bad item never aborts the page.
|
||||
pub async fn persist_analysis(
|
||||
pool: &PgPool,
|
||||
page_id: Uuid,
|
||||
analysis: &VisionAnalysis,
|
||||
model: &str,
|
||||
) -> AppResult<()> {
|
||||
// Kind-weighted text buckets for the tsvector: A=speech/title,
|
||||
// B=narration/thought/caption, C=scene, D=sfx.
|
||||
let mut bucket_a = String::new();
|
||||
let mut bucket_b = String::new();
|
||||
let bucket_c = analysis.scene_description.trim().to_string();
|
||||
let mut bucket_d = String::new();
|
||||
|
||||
let mut ocr_rows: Vec<(OcrKind, String)> = Vec::new();
|
||||
for r in &analysis.ocr_results {
|
||||
let text = r.text.trim();
|
||||
if text.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let kind = OcrKind::from_model_str(&r.kind);
|
||||
match kind.weight() {
|
||||
'A' => push_token(&mut bucket_a, text),
|
||||
'B' => push_token(&mut bucket_b, text),
|
||||
'D' => push_token(&mut bucket_d, text),
|
||||
_ => {}
|
||||
}
|
||||
ocr_rows.push((kind, text.to_string()));
|
||||
}
|
||||
|
||||
// Dedup tags case-insensitively, preserving first-seen order; drop
|
||||
// empties and over-long names rather than failing the transaction.
|
||||
let mut seen_tags = std::collections::HashSet::new();
|
||||
let mut tags: Vec<String> = Vec::new();
|
||||
for raw in &analysis.tagging_results {
|
||||
let t = raw.trim();
|
||||
if t.is_empty() || t.chars().count() > MAX_TAG_CHARS {
|
||||
continue;
|
||||
}
|
||||
if seen_tags.insert(t.to_lowercase()) {
|
||||
tags.push(t.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
// Map + dedup warnings.
|
||||
let mut seen_warn = std::collections::HashSet::new();
|
||||
let mut warnings: Vec<ContentWarning> = Vec::new();
|
||||
for raw in &analysis.safety_flag.content_type {
|
||||
if let Some(w) = ContentWarning::from_model_str(raw) {
|
||||
if seen_warn.insert(w) {
|
||||
warnings.push(w);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut tx = pool.begin().await?;
|
||||
|
||||
sqlx::query("DELETE FROM page_ocr_text WHERE page_id = $1")
|
||||
.bind(page_id)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
sqlx::query("DELETE FROM page_auto_tags WHERE page_id = $1")
|
||||
.bind(page_id)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
sqlx::query("DELETE FROM page_content_warnings WHERE page_id = $1")
|
||||
.bind(page_id)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
|
||||
for (ord, (kind, text)) in ocr_rows.iter().enumerate() {
|
||||
sqlx::query(
|
||||
"INSERT INTO page_ocr_text (page_id, kind, text, ord) VALUES ($1, $2, $3, $4)",
|
||||
)
|
||||
.bind(page_id)
|
||||
.bind(kind)
|
||||
.bind(text)
|
||||
.bind(ord as i32)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
}
|
||||
|
||||
for tag in &tags {
|
||||
let tag_row = crate::repo::tag::upsert_by_name(&mut *tx, tag).await?;
|
||||
sqlx::query(
|
||||
"INSERT INTO page_auto_tags (page_id, tag_id) VALUES ($1, $2) \
|
||||
ON CONFLICT (page_id, tag_id) DO NOTHING",
|
||||
)
|
||||
.bind(page_id)
|
||||
.bind(tag_row.id)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
}
|
||||
|
||||
for w in &warnings {
|
||||
sqlx::query(
|
||||
"INSERT INTO page_content_warnings (page_id, warning) VALUES ($1, $2) \
|
||||
ON CONFLICT (page_id, warning) DO NOTHING",
|
||||
)
|
||||
.bind(page_id)
|
||||
.bind(w)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
}
|
||||
|
||||
let scene = if bucket_c.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(bucket_c.as_str())
|
||||
};
|
||||
sqlx::query(
|
||||
r#"
|
||||
INSERT INTO page_analysis
|
||||
(page_id, status, scene_description, is_nsfw, model, analyzed_at, search_doc)
|
||||
VALUES
|
||||
($1, 'done', $2, $3, $4, now(),
|
||||
setweight(to_tsvector('simple', $5), 'A') ||
|
||||
setweight(to_tsvector('simple', $6), 'B') ||
|
||||
setweight(to_tsvector('simple', $7), 'C') ||
|
||||
setweight(to_tsvector('simple', $8), 'D'))
|
||||
ON CONFLICT (page_id) DO UPDATE SET
|
||||
status = 'done',
|
||||
scene_description = EXCLUDED.scene_description,
|
||||
is_nsfw = EXCLUDED.is_nsfw,
|
||||
model = EXCLUDED.model,
|
||||
analyzed_at = now(),
|
||||
error = NULL,
|
||||
search_doc = EXCLUDED.search_doc,
|
||||
updated_at = now()
|
||||
"#,
|
||||
)
|
||||
.bind(page_id)
|
||||
.bind(scene)
|
||||
.bind(analysis.safety_flag.is_nsfw)
|
||||
.bind(model)
|
||||
.bind(&bucket_a)
|
||||
.bind(&bucket_b)
|
||||
.bind(&bucket_c)
|
||||
.bind(&bucket_d)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
|
||||
tx.commit().await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Content search over pages: multi-tag AND across (user page tags ∪
|
||||
/// global auto tags), optional weighted text search over the OCR/scene
|
||||
/// document, and content-warning include/exclude. Returns one row per
|
||||
/// matching page plus the total for pagination.
|
||||
///
|
||||
/// Empty `tags` makes the tag clause vacuously true (text- or
|
||||
/// warning-only search), mirroring the manga search's `unnest` idiom. The
|
||||
/// caller is responsible for requiring at least one positive filter so
|
||||
/// this never degenerates into "every page".
|
||||
pub async fn page_search(
|
||||
pool: &PgPool,
|
||||
q: &PageSearchQuery,
|
||||
) -> AppResult<(Vec<PageSearchItem>, i64)> {
|
||||
// `text` participates in three places ($3): the rank, the match
|
||||
// predicate, and the order key. Empty string disables text filtering.
|
||||
let text = q.text.as_deref().unwrap_or("").trim();
|
||||
|
||||
const WHERE: &str = r#"
|
||||
NOT EXISTS (
|
||||
SELECT 1 FROM unnest($2::text[]) AS req(name)
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM page_tags ut
|
||||
JOIN tags t ON t.id = ut.tag_id
|
||||
WHERE ut.page_id = p.id AND ut.user_id = $1 AND lower(t.name) = req.name
|
||||
UNION ALL
|
||||
SELECT 1 FROM page_auto_tags at
|
||||
JOIN tags t ON t.id = at.tag_id
|
||||
WHERE at.page_id = p.id AND lower(t.name) = req.name
|
||||
)
|
||||
)
|
||||
AND ($3 = '' OR pa.search_doc @@ plainto_tsquery('simple', $3))
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM unnest($4::text[]) AS req(w)
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM page_content_warnings pw
|
||||
WHERE pw.page_id = p.id AND pw.warning = req.w
|
||||
)
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM page_content_warnings pw
|
||||
WHERE pw.page_id = p.id AND pw.warning = ANY($5::text[])
|
||||
)
|
||||
"#;
|
||||
|
||||
let rows_sql = format!(
|
||||
r#"
|
||||
SELECT
|
||||
p.id AS page_id,
|
||||
p.chapter_id AS chapter_id,
|
||||
ch.manga_id AS manga_id,
|
||||
p.page_number AS page_number,
|
||||
ch.number AS chapter_number,
|
||||
ch.title AS chapter_title,
|
||||
m.title AS manga_title,
|
||||
p.storage_key AS storage_key,
|
||||
COALESCE(pa.is_nsfw, false) AS is_nsfw,
|
||||
COALESCE(
|
||||
(SELECT array_agg(pw.warning ORDER BY pw.warning)
|
||||
FROM page_content_warnings pw WHERE pw.page_id = p.id),
|
||||
ARRAY[]::text[]
|
||||
) AS content_warnings,
|
||||
COALESCE(ts_rank(pa.search_doc, plainto_tsquery('simple', $3)), 0)::real AS rank
|
||||
FROM pages p
|
||||
JOIN chapters ch ON ch.id = p.chapter_id
|
||||
JOIN mangas m ON m.id = ch.manga_id
|
||||
LEFT JOIN page_analysis pa ON pa.page_id = p.id
|
||||
WHERE {WHERE}
|
||||
ORDER BY (CASE WHEN $3 = '' THEN 0 ELSE 1 END) DESC, rank DESC, p.id
|
||||
LIMIT $6 OFFSET $7
|
||||
"#
|
||||
);
|
||||
let rows = sqlx::query_as::<_, PageSearchItem>(&rows_sql)
|
||||
.bind(q.user_id)
|
||||
.bind(&q.tags)
|
||||
.bind(text)
|
||||
.bind(&q.cw_include)
|
||||
.bind(&q.cw_exclude)
|
||||
.bind(q.limit)
|
||||
.bind(q.offset)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
|
||||
let count_sql = format!(
|
||||
"SELECT count(*) FROM pages p \
|
||||
JOIN chapters ch ON ch.id = p.chapter_id \
|
||||
JOIN mangas m ON m.id = ch.manga_id \
|
||||
LEFT JOIN page_analysis pa ON pa.page_id = p.id \
|
||||
WHERE {WHERE}"
|
||||
);
|
||||
let (total,): (i64,) = sqlx::query_as(&count_sql)
|
||||
.bind(q.user_id)
|
||||
.bind(&q.tags)
|
||||
.bind(text)
|
||||
.bind(&q.cw_include)
|
||||
.bind(&q.cw_exclude)
|
||||
.fetch_one(pool)
|
||||
.await?;
|
||||
|
||||
Ok((rows, total))
|
||||
}
|
||||
|
||||
/// Append a token to a tsvector text bucket with a trailing space.
|
||||
fn push_token(bucket: &mut String, text: &str) {
|
||||
bucket.push_str(text);
|
||||
bucket.push(' ');
|
||||
}
|
||||
401
backend/src/repo/page_tag.rs
Normal file
401
backend/src/repo/page_tag.rs
Normal file
@@ -0,0 +1,401 @@
|
||||
//! Per-user, per-page tag persistence.
|
||||
//!
|
||||
//! Same plain-function pattern as the rest of `repo`. Tag names resolve
|
||||
//! to the shared `tags` table (the same one `manga_tags` uses) via
|
||||
//! [`crate::repo::tag::upsert_by_name`]; `page_tags` links rows by
|
||||
//! `tag_id` (see migration 0024). The API still speaks tag *names* — the
|
||||
//! name<->id resolution lives entirely here. Idempotent upserts via
|
||||
//! `ON CONFLICT DO NOTHING` (the caller distinguishes 201/200 from the
|
||||
//! returned `bool`), FK-violation remap to NotFound so the handler can
|
||||
//! return 404 when the page was deleted between an existence check and
|
||||
//! the insert.
|
||||
|
||||
use sqlx::PgPool;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::domain::page_tag::{
|
||||
PageTagSummary, TaggedChapterAggregate, TaggedMangaAggregate, TaggedPageItem,
|
||||
};
|
||||
use crate::error::{AppError, AppResult};
|
||||
|
||||
/// Sort direction for the per-chapter / per-manga aggregations. We
|
||||
/// inline this into the SQL via `format!()` because Postgres won't
|
||||
/// accept ASC/DESC as a parameter — and the value space is closed
|
||||
/// (just two variants), so there's no SQL-injection vector.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub enum Order {
|
||||
Desc,
|
||||
Asc,
|
||||
}
|
||||
|
||||
impl Order {
|
||||
fn as_sql(self) -> &'static str {
|
||||
match self {
|
||||
Order::Desc => "DESC",
|
||||
Order::Asc => "ASC",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Insert a tag for `(user_id, page_id)`. The tag name is resolved to
|
||||
/// (or created in) the shared `tags` table first — the same table and
|
||||
/// `upsert_by_name` helper the manga-tag path uses — then linked by
|
||||
/// `tag_id`. Returns `true` if a new link was inserted (handler → 201),
|
||||
/// `false` if the tag was already present (handler → 200).
|
||||
pub async fn upsert(
|
||||
pool: &PgPool,
|
||||
user_id: Uuid,
|
||||
page_id: Uuid,
|
||||
tag: &str,
|
||||
) -> AppResult<bool> {
|
||||
let mut tx = pool.begin().await?;
|
||||
let tag_row = crate::repo::tag::upsert_by_name(&mut *tx, tag).await?;
|
||||
let result = sqlx::query(
|
||||
r#"
|
||||
INSERT INTO page_tags (user_id, page_id, tag_id)
|
||||
VALUES ($1, $2, $3)
|
||||
ON CONFLICT (user_id, page_id, tag_id) DO NOTHING
|
||||
"#,
|
||||
)
|
||||
.bind(user_id)
|
||||
.bind(page_id)
|
||||
.bind(tag_row.id)
|
||||
.execute(&mut *tx)
|
||||
.await
|
||||
.map_err(|e| match e {
|
||||
sqlx::Error::Database(ref db_err) if db_err.is_foreign_key_violation() => {
|
||||
AppError::NotFound
|
||||
}
|
||||
other => AppError::Database(other),
|
||||
})?;
|
||||
tx.commit().await?;
|
||||
Ok(result.rows_affected() > 0)
|
||||
}
|
||||
|
||||
/// Remove a tag from `(user_id, page_id)`. The tag arrives as a name; we
|
||||
/// resolve it to the shared `tags` row case-insensitively in a subquery
|
||||
/// so a no-op delete (unknown name) simply matches nothing.
|
||||
pub async fn remove(
|
||||
pool: &PgPool,
|
||||
user_id: Uuid,
|
||||
page_id: Uuid,
|
||||
tag: &str,
|
||||
) -> AppResult<()> {
|
||||
sqlx::query(
|
||||
r#"
|
||||
DELETE FROM page_tags
|
||||
WHERE user_id = $1
|
||||
AND page_id = $2
|
||||
AND tag_id = (SELECT id FROM tags WHERE lower(name) = lower($3))
|
||||
"#,
|
||||
)
|
||||
.bind(user_id)
|
||||
.bind(page_id)
|
||||
.bind(tag)
|
||||
.execute(pool)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// `user_id`'s tags on `page_id`, oldest-first so the context-menu
|
||||
/// summary line reads in the order the user added them.
|
||||
pub async fn list_for_page(
|
||||
pool: &PgPool,
|
||||
user_id: Uuid,
|
||||
page_id: Uuid,
|
||||
) -> AppResult<Vec<String>> {
|
||||
let rows: Vec<(String,)> = sqlx::query_as(
|
||||
r#"
|
||||
SELECT t.name AS tag
|
||||
FROM page_tags pt
|
||||
JOIN tags t ON t.id = pt.tag_id
|
||||
WHERE pt.user_id = $1 AND pt.page_id = $2
|
||||
ORDER BY pt.created_at ASC, t.name
|
||||
"#,
|
||||
)
|
||||
.bind(user_id)
|
||||
.bind(page_id)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
Ok(rows.into_iter().map(|(t,)| t).collect())
|
||||
}
|
||||
|
||||
/// Escape a string for use as a LIKE pattern fragment: `%`, `_`, and
|
||||
/// `\` get a leading backslash so they're matched literally rather
|
||||
/// than as wildcards / escapes. The matching queries below pair this
|
||||
/// with `ESCAPE '\'` for explicitness — a single backslash, since the
|
||||
/// SQL lives in a raw string and Postgres treats `\\` in a single-
|
||||
/// quoted literal as one backslash under `standard_conforming_strings`.
|
||||
///
|
||||
/// The public API rejects `%`/`_`/`\` in `normalize_tag` before
|
||||
/// they reach this repo, so this is defence-in-depth — a future
|
||||
/// internal caller (worker, CLI) that bypasses the normalizer can't
|
||||
/// turn a prefix filter into a wildcard search by accident.
|
||||
fn escape_like_prefix(s: &str) -> String {
|
||||
let mut out = String::with_capacity(s.len());
|
||||
for ch in s.chars() {
|
||||
if matches!(ch, '\\' | '%' | '_') {
|
||||
out.push('\\');
|
||||
}
|
||||
out.push(ch);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Paged list of `user_id`'s tagged pages, with breadcrumb. When
|
||||
/// `tag_filter` is `Some(_)`, restrict to that exact tag (used by the
|
||||
/// library Page-tags chip filter). `prefix_filter` does a `LIKE
|
||||
/// 'prefix%'` against the normalized tag (used by autocomplete when
|
||||
/// the user is typing a chip). The prefix is LIKE-escaped here so
|
||||
/// stray `%` / `_` from a bypass-the-API caller don't widen the
|
||||
/// pattern.
|
||||
pub async fn list_for_user(
|
||||
pool: &PgPool,
|
||||
user_id: Uuid,
|
||||
tag_filter: Option<&str>,
|
||||
prefix_filter: Option<&str>,
|
||||
limit: i64,
|
||||
offset: i64,
|
||||
) -> AppResult<(Vec<TaggedPageItem>, i64)> {
|
||||
let escaped_prefix = prefix_filter.map(escape_like_prefix);
|
||||
let rows = sqlx::query_as::<_, TaggedPageItem>(
|
||||
r#"
|
||||
SELECT
|
||||
t.name AS tag,
|
||||
p.id AS page_id,
|
||||
p.chapter_id AS chapter_id,
|
||||
ch.manga_id AS manga_id,
|
||||
p.page_number AS page_number,
|
||||
ch.number AS chapter_number,
|
||||
ch.title AS chapter_title,
|
||||
m.title AS manga_title,
|
||||
p.storage_key AS storage_key,
|
||||
pt.created_at AS tagged_at
|
||||
FROM page_tags pt
|
||||
JOIN tags t ON t.id = pt.tag_id
|
||||
JOIN pages p ON p.id = pt.page_id
|
||||
JOIN chapters ch ON ch.id = p.chapter_id
|
||||
JOIN mangas m ON m.id = ch.manga_id
|
||||
WHERE pt.user_id = $1
|
||||
AND ($2::text IS NULL OR lower(t.name) = $2)
|
||||
AND ($3::text IS NULL OR lower(t.name) LIKE $3 || '%' ESCAPE '\')
|
||||
ORDER BY pt.created_at DESC, pt.id
|
||||
LIMIT $4 OFFSET $5
|
||||
"#,
|
||||
)
|
||||
.bind(user_id)
|
||||
.bind(tag_filter)
|
||||
.bind(escaped_prefix.as_deref())
|
||||
.bind(limit)
|
||||
.bind(offset)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
|
||||
let escaped_prefix = prefix_filter.map(escape_like_prefix);
|
||||
let (total,): (i64,) = sqlx::query_as(
|
||||
r#"
|
||||
SELECT count(*)
|
||||
FROM page_tags pt
|
||||
JOIN tags t ON t.id = pt.tag_id
|
||||
WHERE pt.user_id = $1
|
||||
AND ($2::text IS NULL OR lower(t.name) = $2)
|
||||
AND ($3::text IS NULL OR lower(t.name) LIKE $3 || '%' ESCAPE '\')
|
||||
"#,
|
||||
)
|
||||
.bind(user_id)
|
||||
.bind(tag_filter)
|
||||
.bind(escaped_prefix.as_deref())
|
||||
.fetch_one(pool)
|
||||
.await?;
|
||||
Ok((rows, total))
|
||||
}
|
||||
|
||||
/// Distinct tag list for the user, with per-tag counts. When `prefix`
|
||||
/// is `Some(_)`, restrict to tags starting with that prefix — drives
|
||||
/// the autocomplete dropdown in the "Add tag" sheet. The prefix is
|
||||
/// LIKE-escaped here so stray `%` / `_` from a bypass-the-API caller
|
||||
/// don't widen the pattern.
|
||||
pub async fn distinct_tags_for_user(
|
||||
pool: &PgPool,
|
||||
user_id: Uuid,
|
||||
prefix: Option<&str>,
|
||||
limit: i64,
|
||||
) -> AppResult<Vec<PageTagSummary>> {
|
||||
let escaped_prefix = prefix.map(escape_like_prefix);
|
||||
let rows = sqlx::query_as::<_, PageTagSummary>(
|
||||
r#"
|
||||
SELECT t.name AS tag, count(*) AS count
|
||||
FROM page_tags pt
|
||||
JOIN tags t ON t.id = pt.tag_id
|
||||
WHERE pt.user_id = $1
|
||||
AND ($2::text IS NULL OR lower(t.name) LIKE $2 || '%' ESCAPE '\')
|
||||
GROUP BY t.name
|
||||
ORDER BY count DESC, t.name
|
||||
LIMIT $3
|
||||
"#,
|
||||
)
|
||||
.bind(user_id)
|
||||
.bind(escaped_prefix.as_deref())
|
||||
.bind(limit)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
Ok(rows)
|
||||
}
|
||||
|
||||
/// Paged list of chapters that contain pages tagged `tag` for `user_id`,
|
||||
/// ranked by `match_count`. Each row carries up to 3 sample page
|
||||
/// storage keys (page-number ascending) so the row can render a
|
||||
/// thumbnail strip without a follow-up fetch.
|
||||
///
|
||||
/// `order` is inlined via `format!()` — the enum value space is
|
||||
/// closed (`ASC` / `DESC`) so this is not a SQL-injection vector.
|
||||
pub async fn aggregate_chapters_for_tag(
|
||||
pool: &PgPool,
|
||||
user_id: Uuid,
|
||||
tag: &str,
|
||||
order: Order,
|
||||
limit: i64,
|
||||
offset: i64,
|
||||
) -> AppResult<(Vec<TaggedChapterAggregate>, i64)> {
|
||||
let sql = format!(
|
||||
r#"
|
||||
SELECT
|
||||
ch.id AS chapter_id,
|
||||
ch.manga_id AS manga_id,
|
||||
m.title AS manga_title,
|
||||
ch.number AS chapter_number,
|
||||
ch.title AS chapter_title,
|
||||
count(*) AS match_count,
|
||||
COALESCE(
|
||||
(
|
||||
SELECT array_agg(p2.storage_key ORDER BY p2.page_number ASC)
|
||||
FROM (
|
||||
SELECT p.storage_key, p.page_number
|
||||
FROM pages p
|
||||
JOIN page_tags pt2 ON pt2.page_id = p.id
|
||||
JOIN tags t2 ON t2.id = pt2.tag_id
|
||||
WHERE p.chapter_id = ch.id
|
||||
AND pt2.user_id = $1
|
||||
AND lower(t2.name) = $2
|
||||
ORDER BY p.page_number ASC
|
||||
LIMIT 3
|
||||
) p2
|
||||
),
|
||||
ARRAY[]::text[]
|
||||
) AS sample_storage_keys
|
||||
FROM page_tags pt
|
||||
JOIN tags t ON t.id = pt.tag_id
|
||||
JOIN pages p ON p.id = pt.page_id
|
||||
JOIN chapters ch ON ch.id = p.chapter_id
|
||||
JOIN mangas m ON m.id = ch.manga_id
|
||||
WHERE pt.user_id = $1
|
||||
AND lower(t.name) = $2
|
||||
GROUP BY ch.id, ch.manga_id, m.title, ch.number, ch.title
|
||||
ORDER BY match_count {dir}, ch.id
|
||||
LIMIT $3 OFFSET $4
|
||||
"#,
|
||||
dir = order.as_sql(),
|
||||
);
|
||||
let rows = sqlx::query_as::<_, TaggedChapterAggregate>(&sql)
|
||||
.bind(user_id)
|
||||
.bind(tag)
|
||||
.bind(limit)
|
||||
.bind(offset)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
|
||||
let (total,): (i64,) = sqlx::query_as(
|
||||
r#"
|
||||
SELECT count(*) FROM (
|
||||
SELECT 1
|
||||
FROM page_tags pt
|
||||
JOIN tags t ON t.id = pt.tag_id
|
||||
JOIN pages p ON p.id = pt.page_id
|
||||
WHERE pt.user_id = $1 AND lower(t.name) = $2
|
||||
GROUP BY p.chapter_id
|
||||
) c
|
||||
"#,
|
||||
)
|
||||
.bind(user_id)
|
||||
.bind(tag)
|
||||
.fetch_one(pool)
|
||||
.await?;
|
||||
Ok((rows, total))
|
||||
}
|
||||
|
||||
/// Paged list of mangas containing pages tagged `tag` for `user_id`,
|
||||
/// ranked by `match_count` summed across all their chapters.
|
||||
pub async fn aggregate_mangas_for_tag(
|
||||
pool: &PgPool,
|
||||
user_id: Uuid,
|
||||
tag: &str,
|
||||
order: Order,
|
||||
limit: i64,
|
||||
offset: i64,
|
||||
) -> AppResult<(Vec<TaggedMangaAggregate>, i64)> {
|
||||
let sql = format!(
|
||||
r#"
|
||||
SELECT
|
||||
m.id AS manga_id,
|
||||
m.title AS manga_title,
|
||||
m.cover_image_path AS manga_cover_image_path,
|
||||
count(*) AS match_count,
|
||||
COALESCE(
|
||||
(
|
||||
SELECT array_agg(p2.storage_key ORDER BY p2.page_number ASC)
|
||||
FROM (
|
||||
SELECT p.storage_key, p.page_number
|
||||
FROM pages p
|
||||
JOIN chapters ch2 ON ch2.id = p.chapter_id
|
||||
JOIN page_tags pt2 ON pt2.page_id = p.id
|
||||
JOIN tags t2 ON t2.id = pt2.tag_id
|
||||
WHERE ch2.manga_id = m.id
|
||||
AND pt2.user_id = $1
|
||||
AND lower(t2.name) = $2
|
||||
ORDER BY p.page_number ASC
|
||||
LIMIT 3
|
||||
) p2
|
||||
),
|
||||
ARRAY[]::text[]
|
||||
) AS sample_storage_keys
|
||||
FROM page_tags pt
|
||||
JOIN tags t ON t.id = pt.tag_id
|
||||
JOIN pages p ON p.id = pt.page_id
|
||||
JOIN chapters ch ON ch.id = p.chapter_id
|
||||
JOIN mangas m ON m.id = ch.manga_id
|
||||
WHERE pt.user_id = $1
|
||||
AND lower(t.name) = $2
|
||||
GROUP BY m.id, m.title, m.cover_image_path
|
||||
ORDER BY match_count {dir}, m.id
|
||||
LIMIT $3 OFFSET $4
|
||||
"#,
|
||||
dir = order.as_sql(),
|
||||
);
|
||||
let rows = sqlx::query_as::<_, TaggedMangaAggregate>(&sql)
|
||||
.bind(user_id)
|
||||
.bind(tag)
|
||||
.bind(limit)
|
||||
.bind(offset)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
|
||||
let (total,): (i64,) = sqlx::query_as(
|
||||
r#"
|
||||
SELECT count(*) FROM (
|
||||
SELECT 1
|
||||
FROM page_tags pt
|
||||
JOIN tags t ON t.id = pt.tag_id
|
||||
JOIN pages p ON p.id = pt.page_id
|
||||
JOIN chapters ch ON ch.id = p.chapter_id
|
||||
WHERE pt.user_id = $1 AND lower(t.name) = $2
|
||||
GROUP BY ch.manga_id
|
||||
) m
|
||||
"#,
|
||||
)
|
||||
.bind(user_id)
|
||||
.bind(tag)
|
||||
.fetch_one(pool)
|
||||
.await?;
|
||||
Ok((rows, total))
|
||||
}
|
||||
|
||||
643
backend/src/settings.rs
Normal file
643
backend/src/settings.rs
Normal file
@@ -0,0 +1,643 @@
|
||||
//! Serializable, admin-editable settings DTOs for the crawler and analysis
|
||||
//! subsystems, and their conversion to/from the runtime [`CrawlerConfig`] /
|
||||
//! [`AnalysisConfig`].
|
||||
//!
|
||||
//! These DTOs are the boundary persisted in the `app_settings` table (one
|
||||
//! JSONB row per group) and exchanged over the admin API. They carry **only
|
||||
//! the operationally-safe, UI-editable fields** as primitive types — the
|
||||
//! runtime configs additionally hold env-only/structural fields (browser
|
||||
//! launch options, proxy, TOR control, the vision API key, the cookie
|
||||
//! domain) that are never persisted here.
|
||||
//!
|
||||
//! Flow: at boot, [`CrawlerSettings::from_config`] /
|
||||
//! [`AnalysisSettings::from_config`] derive the env-seed DTO; it is written to
|
||||
//! the DB only when the row is absent. Thereafter the DB row is the source of
|
||||
//! truth, and [`CrawlerSettings::to_config`] / [`AnalysisSettings::to_config`]
|
||||
//! overlay it onto the env-derived **base** (which carries the env-only
|
||||
//! fields) to produce the effective config the daemons are (re)spawned with.
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use chrono::NaiveTime;
|
||||
use chrono_tz::Tz;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::analysis::prompt::{
|
||||
GROUNDING_PROMPT_DEFAULT, OCR_PROMPT_DEFAULT, SYSTEM_PROMPT_DEFAULT,
|
||||
};
|
||||
use crate::config::{AnalysisConfig, CrawlerConfig, ResponseFormat};
|
||||
use crate::crawler::safety::DownloadAllowlist;
|
||||
|
||||
/// `app_settings.key` for the crawler group.
|
||||
pub const KEY_CRAWLER: &str = "crawler";
|
||||
/// `app_settings.key` for the analysis group.
|
||||
pub const KEY_ANALYSIS: &str = "analysis";
|
||||
|
||||
/// One field-level validation failure, surfaced to the UI per input.
|
||||
#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
|
||||
pub struct FieldError {
|
||||
pub field: String,
|
||||
pub message: String,
|
||||
}
|
||||
|
||||
/// Collected validation failures for a settings payload. Empty == valid.
|
||||
#[derive(Debug, Clone, Serialize, Default, PartialEq, Eq)]
|
||||
pub struct FieldErrors {
|
||||
pub errors: Vec<FieldError>,
|
||||
}
|
||||
|
||||
impl FieldErrors {
|
||||
fn push(&mut self, field: &str, message: impl Into<String>) {
|
||||
self.errors.push(FieldError {
|
||||
field: field.to_string(),
|
||||
message: message.into(),
|
||||
});
|
||||
}
|
||||
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.errors.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Crawler
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Admin-editable crawler-daemon settings. Mirrors the operational `CRAWLER_*`
|
||||
/// env vars; host/infra and session fields stay env-only.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(default)]
|
||||
pub struct CrawlerSettings {
|
||||
pub daemon_enabled: bool,
|
||||
/// Daily metadata-pass time, `"HH:MM"` (24h).
|
||||
pub daily_at: String,
|
||||
/// IANA timezone for the daily schedule.
|
||||
pub tz: String,
|
||||
pub idle_timeout_secs: u64,
|
||||
pub chapter_workers: u64,
|
||||
pub retention_days: u32,
|
||||
pub start_url: Option<String>,
|
||||
pub rate_ms: u64,
|
||||
pub cdn_host: Option<String>,
|
||||
pub cdn_rate_ms: u64,
|
||||
/// Domain the session cookie (PHPSESSID) is scoped to when injecting it
|
||||
/// into the browser / CDN requests. Pairs with `start_url`.
|
||||
pub cookie_domain: Option<String>,
|
||||
pub user_agent: Option<String>,
|
||||
/// Hosts the crawler may download images from (in addition to the
|
||||
/// auto-seeded start-url / CDN hosts). Ignored when `allow_any_host`.
|
||||
pub download_allowlist: Vec<String>,
|
||||
pub allow_any_host: bool,
|
||||
pub max_image_bytes: u64,
|
||||
/// Max manga detail fetches per metadata pass; `0` = unlimited.
|
||||
pub manga_limit: u64,
|
||||
pub job_timeout_secs: u64,
|
||||
pub metadata_max_consecutive_failures: u32,
|
||||
pub browser_restart_threshold: u32,
|
||||
}
|
||||
|
||||
impl Default for CrawlerSettings {
|
||||
fn default() -> Self {
|
||||
Self::from_config(&CrawlerConfig::default())
|
||||
}
|
||||
}
|
||||
|
||||
impl CrawlerSettings {
|
||||
/// Derive the DTO from a runtime config (used to seed the DB from env).
|
||||
pub fn from_config(c: &CrawlerConfig) -> Self {
|
||||
Self {
|
||||
daemon_enabled: c.daemon_enabled,
|
||||
daily_at: c.daily_at.format("%H:%M").to_string(),
|
||||
tz: c.tz.name().to_string(),
|
||||
idle_timeout_secs: c.idle_timeout.as_secs(),
|
||||
chapter_workers: c.chapter_workers as u64,
|
||||
retention_days: c.retention_days,
|
||||
start_url: c.start_url.clone(),
|
||||
rate_ms: c.rate_ms,
|
||||
cdn_host: c.cdn_host.clone(),
|
||||
cdn_rate_ms: c.cdn_rate_ms,
|
||||
cookie_domain: c.cookie_domain.clone(),
|
||||
user_agent: c.user_agent.clone(),
|
||||
download_allowlist: c.download_allowlist.hosts().to_vec(),
|
||||
allow_any_host: c.download_allowlist.is_allow_any(),
|
||||
max_image_bytes: c.max_image_bytes as u64,
|
||||
manga_limit: c.manga_limit as u64,
|
||||
job_timeout_secs: c.job_timeout.as_secs(),
|
||||
metadata_max_consecutive_failures: c.metadata_max_consecutive_failures,
|
||||
browser_restart_threshold: c.browser_restart_threshold,
|
||||
}
|
||||
}
|
||||
|
||||
/// Overlay the DTO onto an env-derived `base` (which carries the env-only
|
||||
/// fields: browser, proxy, TOR, session, cookie domain) to produce the
|
||||
/// effective runtime config. Validates and returns all field errors at
|
||||
/// once on failure.
|
||||
pub fn to_config(&self, base: &CrawlerConfig) -> Result<CrawlerConfig, FieldErrors> {
|
||||
let mut errs = FieldErrors::default();
|
||||
|
||||
let daily_at = match NaiveTime::parse_from_str(&self.daily_at, "%H:%M") {
|
||||
Ok(t) => t,
|
||||
Err(_) => {
|
||||
errs.push("daily_at", "must be HH:MM (24-hour)");
|
||||
base.daily_at
|
||||
}
|
||||
};
|
||||
let tz: Tz = match self.tz.parse() {
|
||||
Ok(t) => t,
|
||||
Err(_) => {
|
||||
errs.push("tz", "must be a valid IANA timezone (e.g. UTC, Europe/Berlin)");
|
||||
base.tz
|
||||
}
|
||||
};
|
||||
if self.chapter_workers < 1 {
|
||||
errs.push("chapter_workers", "must be at least 1");
|
||||
}
|
||||
if let Some(url) = self.start_url.as_deref().map(str::trim).filter(|s| !s.is_empty()) {
|
||||
if reqwest::Url::parse(url).is_err() {
|
||||
errs.push("start_url", "must be a valid absolute URL");
|
||||
}
|
||||
}
|
||||
if self.job_timeout_secs < 1 {
|
||||
errs.push("job_timeout_secs", "must be at least 1 second");
|
||||
}
|
||||
|
||||
if !errs.is_empty() {
|
||||
return Err(errs);
|
||||
}
|
||||
|
||||
let start_url = self
|
||||
.start_url
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|s| !s.is_empty())
|
||||
.map(str::to_string);
|
||||
let cdn_host = self
|
||||
.cdn_host
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|s| !s.is_empty())
|
||||
.map(str::to_string);
|
||||
|
||||
let download_allowlist = build_allowlist(
|
||||
self.allow_any_host,
|
||||
start_url.as_deref(),
|
||||
cdn_host.as_deref(),
|
||||
&self.download_allowlist,
|
||||
);
|
||||
|
||||
Ok(CrawlerConfig {
|
||||
daemon_enabled: self.daemon_enabled,
|
||||
daily_at,
|
||||
tz,
|
||||
idle_timeout: Duration::from_secs(self.idle_timeout_secs),
|
||||
chapter_workers: (self.chapter_workers as usize).max(1),
|
||||
retention_days: self.retention_days,
|
||||
start_url,
|
||||
rate_ms: self.rate_ms,
|
||||
cdn_host,
|
||||
cdn_rate_ms: self.cdn_rate_ms,
|
||||
user_agent: self
|
||||
.user_agent
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|s| !s.is_empty())
|
||||
.map(str::to_string),
|
||||
cookie_domain: self
|
||||
.cookie_domain
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|s| !s.is_empty())
|
||||
.map(str::to_string),
|
||||
download_allowlist,
|
||||
max_image_bytes: self.max_image_bytes as usize,
|
||||
manga_limit: self.manga_limit as usize,
|
||||
job_timeout: Duration::from_secs(self.job_timeout_secs.max(1)),
|
||||
metadata_max_consecutive_failures: self.metadata_max_consecutive_failures,
|
||||
browser_restart_threshold: self.browser_restart_threshold,
|
||||
// Env-only / structural fields preserved from the base.
|
||||
phpsessid: base.phpsessid.clone(),
|
||||
proxy: base.proxy.clone(),
|
||||
tor_control_url: base.tor_control_url.clone(),
|
||||
tor_control_password: base.tor_control_password.clone(),
|
||||
tor_control_cookie_path: base.tor_control_cookie_path.clone(),
|
||||
tor_recircuit_max_attempts: base.tor_recircuit_max_attempts,
|
||||
browser: base.browser.clone(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Rebuild a [`DownloadAllowlist`] from the DTO, always seeding the start-url
|
||||
/// and CDN hosts (mirrors `config::build_download_allowlist`).
|
||||
fn build_allowlist(
|
||||
allow_any: bool,
|
||||
start_url: Option<&str>,
|
||||
cdn_host: Option<&str>,
|
||||
extras: &[String],
|
||||
) -> DownloadAllowlist {
|
||||
if allow_any {
|
||||
return DownloadAllowlist::allow_any();
|
||||
}
|
||||
let mut allow = DownloadAllowlist::new();
|
||||
if let Some(url) = start_url {
|
||||
if let Ok(parsed) = reqwest::Url::parse(url) {
|
||||
if let Some(h) = parsed.host_str() {
|
||||
allow = allow.allow(h);
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(host) = cdn_host {
|
||||
allow = allow.allow(host);
|
||||
}
|
||||
for h in extras {
|
||||
let trimmed = h.trim();
|
||||
if !trimmed.is_empty() {
|
||||
allow = allow.allow(trimmed);
|
||||
}
|
||||
}
|
||||
allow
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Analysis
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Admin-editable analysis-worker settings. The vision `api_key` is env-only
|
||||
/// and never carried here. Prompts are `Option<String>`: `None` means "use
|
||||
/// the compiled default" (the UI's "reset to default").
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||
#[serde(default)]
|
||||
pub struct AnalysisSettings {
|
||||
pub enabled: bool,
|
||||
pub workers: u64,
|
||||
pub endpoint: String,
|
||||
pub model: String,
|
||||
pub request_timeout_secs: u64,
|
||||
pub job_timeout_secs: u64,
|
||||
pub max_tokens: u32,
|
||||
pub max_pixels: u32,
|
||||
pub min_slice_height: u32,
|
||||
pub slice_overlap: f64,
|
||||
pub tall_aspect_threshold: f64,
|
||||
pub max_slices: u64,
|
||||
pub max_image_bytes: u64,
|
||||
/// `"json_schema" | "json_object" | "none"`.
|
||||
pub response_format: String,
|
||||
pub frequency_penalty: f64,
|
||||
pub temperature: f64,
|
||||
pub system_prompt: Option<String>,
|
||||
pub ocr_prompt: Option<String>,
|
||||
pub grounding_prompt: Option<String>,
|
||||
}
|
||||
|
||||
impl Default for AnalysisSettings {
|
||||
fn default() -> Self {
|
||||
Self::from_config(&AnalysisConfig::default())
|
||||
}
|
||||
}
|
||||
|
||||
impl AnalysisSettings {
|
||||
pub fn from_config(c: &AnalysisConfig) -> Self {
|
||||
// Capture a prompt only when it diverges from the compiled default,
|
||||
// so an unmodified config seeds as `None` ("use default").
|
||||
let opt = |cur: &str, def: &str| (cur != def).then(|| cur.to_string());
|
||||
Self {
|
||||
enabled: c.enabled,
|
||||
workers: c.workers as u64,
|
||||
endpoint: c.endpoint.clone(),
|
||||
model: c.model.clone(),
|
||||
request_timeout_secs: c.request_timeout.as_secs(),
|
||||
job_timeout_secs: c.job_timeout.as_secs(),
|
||||
max_tokens: c.max_tokens,
|
||||
max_pixels: c.max_pixels,
|
||||
min_slice_height: c.min_slice_height,
|
||||
slice_overlap: c.slice_overlap,
|
||||
tall_aspect_threshold: c.tall_aspect_threshold,
|
||||
max_slices: c.max_slices as u64,
|
||||
max_image_bytes: c.max_image_bytes as u64,
|
||||
response_format: c.response_format.as_str().to_string(),
|
||||
frequency_penalty: c.frequency_penalty,
|
||||
temperature: c.temperature,
|
||||
system_prompt: opt(&c.system_prompt, SYSTEM_PROMPT_DEFAULT),
|
||||
ocr_prompt: opt(&c.ocr_prompt, OCR_PROMPT_DEFAULT),
|
||||
grounding_prompt: opt(&c.grounding_prompt, GROUNDING_PROMPT_DEFAULT),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn to_config(&self, base: &AnalysisConfig) -> Result<AnalysisConfig, FieldErrors> {
|
||||
let mut errs = FieldErrors::default();
|
||||
|
||||
if self.workers < 1 {
|
||||
errs.push("workers", "must be at least 1");
|
||||
}
|
||||
if self.endpoint.trim().is_empty() || reqwest::Url::parse(self.endpoint.trim()).is_err() {
|
||||
errs.push("endpoint", "must be a valid absolute URL");
|
||||
}
|
||||
if self.enabled && self.model.trim().is_empty() {
|
||||
errs.push("model", "required when analysis is enabled");
|
||||
}
|
||||
if self.max_tokens < 1 {
|
||||
errs.push("max_tokens", "must be at least 1");
|
||||
}
|
||||
if self.request_timeout_secs < 1 {
|
||||
errs.push("request_timeout_secs", "must be at least 1 second");
|
||||
}
|
||||
if self.job_timeout_secs < 1 {
|
||||
errs.push("job_timeout_secs", "must be at least 1 second");
|
||||
}
|
||||
if self.max_pixels < 1 {
|
||||
errs.push("max_pixels", "must be at least 1");
|
||||
}
|
||||
if self.max_image_bytes < 1 {
|
||||
errs.push("max_image_bytes", "must be greater than 0");
|
||||
}
|
||||
if !(0.0..=0.9).contains(&self.slice_overlap) {
|
||||
errs.push("slice_overlap", "must be between 0.0 and 0.9");
|
||||
}
|
||||
if self.tall_aspect_threshold < 1.0 {
|
||||
errs.push("tall_aspect_threshold", "must be at least 1.0");
|
||||
}
|
||||
if self.min_slice_height < 1 {
|
||||
errs.push("min_slice_height", "must be at least 1");
|
||||
}
|
||||
if self.max_slices < 1 {
|
||||
errs.push("max_slices", "must be at least 1");
|
||||
}
|
||||
if self.temperature < 0.0 {
|
||||
errs.push("temperature", "must be 0 or greater");
|
||||
}
|
||||
let response_format = match ResponseFormat::parse_strict(&self.response_format) {
|
||||
Some(rf) => rf,
|
||||
None => {
|
||||
errs.push(
|
||||
"response_format",
|
||||
"must be one of: json_schema, json_object, none",
|
||||
);
|
||||
base.response_format
|
||||
}
|
||||
};
|
||||
|
||||
if !errs.is_empty() {
|
||||
return Err(errs);
|
||||
}
|
||||
|
||||
let prompt = |o: &Option<String>, def: &str| {
|
||||
o.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|s| !s.is_empty())
|
||||
.unwrap_or(def)
|
||||
.to_string()
|
||||
};
|
||||
|
||||
Ok(AnalysisConfig {
|
||||
enabled: self.enabled,
|
||||
workers: (self.workers as usize).max(1),
|
||||
endpoint: self.endpoint.trim().to_string(),
|
||||
model: self.model.trim().to_string(),
|
||||
request_timeout: Duration::from_secs(self.request_timeout_secs),
|
||||
job_timeout: Duration::from_secs(self.job_timeout_secs),
|
||||
max_tokens: self.max_tokens,
|
||||
max_pixels: self.max_pixels,
|
||||
min_slice_height: self.min_slice_height.max(1),
|
||||
slice_overlap: self.slice_overlap.clamp(0.0, 0.9),
|
||||
tall_aspect_threshold: self.tall_aspect_threshold.max(1.0),
|
||||
max_slices: (self.max_slices as usize).max(1),
|
||||
max_image_bytes: self.max_image_bytes as usize,
|
||||
response_format,
|
||||
frequency_penalty: self.frequency_penalty,
|
||||
temperature: self.temperature.max(0.0),
|
||||
system_prompt: prompt(&self.system_prompt, SYSTEM_PROMPT_DEFAULT),
|
||||
ocr_prompt: prompt(&self.ocr_prompt, OCR_PROMPT_DEFAULT),
|
||||
grounding_prompt: prompt(&self.grounding_prompt, GROUNDING_PROMPT_DEFAULT),
|
||||
// Env-only secret preserved from the base.
|
||||
api_key: base.api_key.clone(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Prompt defaults, returned by the API so the UI can render placeholders and
|
||||
/// implement "reset to default".
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct PromptDefaults {
|
||||
pub system_prompt: &'static str,
|
||||
pub ocr_prompt: &'static str,
|
||||
pub grounding_prompt: &'static str,
|
||||
}
|
||||
|
||||
impl PromptDefaults {
|
||||
pub fn get() -> Self {
|
||||
Self {
|
||||
system_prompt: SYSTEM_PROMPT_DEFAULT,
|
||||
ocr_prompt: OCR_PROMPT_DEFAULT,
|
||||
grounding_prompt: GROUNDING_PROMPT_DEFAULT,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
// --- crawler round-trip & validation ---
|
||||
|
||||
#[test]
|
||||
fn crawler_round_trips_through_dto() {
|
||||
let mut base = CrawlerConfig::default();
|
||||
base.start_url = Some("https://example.com/".to_string());
|
||||
base.tz = Tz::Europe__Berlin;
|
||||
base.chapter_workers = 3;
|
||||
base.cookie_domain = Some("example.com".to_string());
|
||||
let dto = CrawlerSettings::from_config(&base);
|
||||
let back = dto.to_config(&base).expect("valid");
|
||||
assert_eq!(back.tz, Tz::Europe__Berlin);
|
||||
assert_eq!(back.chapter_workers, 3);
|
||||
assert_eq!(back.start_url.as_deref(), Some("https://example.com/"));
|
||||
// cookie_domain is now an editable DTO field, round-tripping like the rest.
|
||||
assert_eq!(back.cookie_domain.as_deref(), Some("example.com"));
|
||||
assert_eq!(dto.daily_at, "00:00");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn crawler_cookie_domain_is_editable() {
|
||||
let base = CrawlerConfig::default();
|
||||
let dto = CrawlerSettings {
|
||||
cookie_domain: Some("new.example.org".to_string()),
|
||||
..CrawlerSettings::from_config(&base)
|
||||
};
|
||||
assert_eq!(
|
||||
dto.to_config(&base).unwrap().cookie_domain.as_deref(),
|
||||
Some("new.example.org")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn crawler_overlay_preserves_env_only_fields() {
|
||||
let mut base = CrawlerConfig::default();
|
||||
base.proxy = Some("socks5://127.0.0.1:9050".to_string());
|
||||
base.tor_control_password = Some("secret".to_string());
|
||||
base.phpsessid = Some("abc123".to_string());
|
||||
// A DTO that knows nothing about the env-only fields.
|
||||
let dto = CrawlerSettings {
|
||||
rate_ms: 2000,
|
||||
..CrawlerSettings::from_config(&base)
|
||||
};
|
||||
let out = dto.to_config(&base).expect("valid");
|
||||
assert_eq!(out.rate_ms, 2000);
|
||||
assert_eq!(out.proxy.as_deref(), Some("socks5://127.0.0.1:9050"));
|
||||
assert_eq!(out.tor_control_password.as_deref(), Some("secret"));
|
||||
assert_eq!(out.phpsessid.as_deref(), Some("abc123"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn crawler_rejects_bad_tz_and_time() {
|
||||
let base = CrawlerConfig::default();
|
||||
let dto = CrawlerSettings {
|
||||
tz: "Mars/Phobos".to_string(),
|
||||
daily_at: "9am".to_string(),
|
||||
..CrawlerSettings::from_config(&base)
|
||||
};
|
||||
let errs = dto.to_config(&base).unwrap_err();
|
||||
let fields: Vec<_> = errs.errors.iter().map(|e| e.field.as_str()).collect();
|
||||
assert!(fields.contains(&"tz"));
|
||||
assert!(fields.contains(&"daily_at"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn crawler_allowlist_seeds_start_and_cdn_hosts() {
|
||||
let base = CrawlerConfig::default();
|
||||
let dto = CrawlerSettings {
|
||||
start_url: Some("https://catalog.example.com/".to_string()),
|
||||
cdn_host: Some("cdn.example.com".to_string()),
|
||||
download_allowlist: vec!["extra.example.net".to_string()],
|
||||
allow_any_host: false,
|
||||
..CrawlerSettings::from_config(&base)
|
||||
};
|
||||
let out = dto.to_config(&base).expect("valid");
|
||||
assert!(out.download_allowlist.contains("catalog.example.com"));
|
||||
assert!(out.download_allowlist.contains("cdn.example.com"));
|
||||
assert!(out.download_allowlist.contains("extra.example.net"));
|
||||
assert!(!out.download_allowlist.is_allow_any());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn crawler_allow_any_host_bypasses_list() {
|
||||
let base = CrawlerConfig::default();
|
||||
let dto = CrawlerSettings {
|
||||
allow_any_host: true,
|
||||
..CrawlerSettings::from_config(&base)
|
||||
};
|
||||
let out = dto.to_config(&base).expect("valid");
|
||||
assert!(out.download_allowlist.is_allow_any());
|
||||
}
|
||||
|
||||
// --- analysis round-trip, validation & prompt defaults ---
|
||||
|
||||
#[test]
|
||||
fn analysis_round_trips_and_omits_default_prompts() {
|
||||
let base = AnalysisConfig::default();
|
||||
let dto = AnalysisSettings::from_config(&base);
|
||||
// Unmodified prompts seed as None ("use default").
|
||||
assert!(dto.system_prompt.is_none());
|
||||
assert!(dto.ocr_prompt.is_none());
|
||||
assert!(dto.grounding_prompt.is_none());
|
||||
assert_eq!(dto.response_format, "json_schema");
|
||||
let back = dto.to_config(&base).expect("valid");
|
||||
assert_eq!(back.system_prompt, SYSTEM_PROMPT_DEFAULT);
|
||||
assert_eq!(back.response_format, ResponseFormat::JsonSchema);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn analysis_captures_env_prompt_override() {
|
||||
let mut base = AnalysisConfig::default();
|
||||
base.system_prompt = "custom env prompt".to_string();
|
||||
let dto = AnalysisSettings::from_config(&base);
|
||||
assert_eq!(dto.system_prompt.as_deref(), Some("custom env prompt"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn analysis_prompt_override_and_reset() {
|
||||
let base = AnalysisConfig::default();
|
||||
// Override applied.
|
||||
let dto = AnalysisSettings {
|
||||
system_prompt: Some("OVERRIDE".to_string()),
|
||||
..AnalysisSettings::from_config(&base)
|
||||
};
|
||||
assert_eq!(dto.to_config(&base).unwrap().system_prompt, "OVERRIDE");
|
||||
// None / blank falls back to the compiled default.
|
||||
let dto = AnalysisSettings {
|
||||
system_prompt: Some(" ".to_string()),
|
||||
..AnalysisSettings::from_config(&base)
|
||||
};
|
||||
assert_eq!(dto.to_config(&base).unwrap().system_prompt, SYSTEM_PROMPT_DEFAULT);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn analysis_overlay_preserves_api_key() {
|
||||
let mut base = AnalysisConfig::default();
|
||||
base.api_key = Some("sk-secret".to_string());
|
||||
let dto = AnalysisSettings::from_config(&base);
|
||||
assert_eq!(dto.to_config(&base).unwrap().api_key.as_deref(), Some("sk-secret"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn analysis_rejects_out_of_range() {
|
||||
let base = AnalysisConfig::default();
|
||||
let dto = AnalysisSettings {
|
||||
workers: 0,
|
||||
slice_overlap: 1.5,
|
||||
tall_aspect_threshold: 0.5,
|
||||
response_format: "yaml".to_string(),
|
||||
endpoint: "not a url".to_string(),
|
||||
request_timeout_secs: 0,
|
||||
job_timeout_secs: 0,
|
||||
max_pixels: 0,
|
||||
max_image_bytes: 0,
|
||||
..AnalysisSettings::from_config(&base)
|
||||
};
|
||||
let errs = dto.to_config(&base).unwrap_err();
|
||||
let fields: Vec<_> = errs.errors.iter().map(|e| e.field.as_str()).collect();
|
||||
for f in [
|
||||
"workers",
|
||||
"slice_overlap",
|
||||
"tall_aspect_threshold",
|
||||
"response_format",
|
||||
"endpoint",
|
||||
"request_timeout_secs",
|
||||
"job_timeout_secs",
|
||||
"max_pixels",
|
||||
"max_image_bytes",
|
||||
] {
|
||||
assert!(fields.contains(&f), "missing error for {f}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn analysis_requires_model_only_when_enabled() {
|
||||
let base = AnalysisConfig::default();
|
||||
let disabled = AnalysisSettings {
|
||||
enabled: false,
|
||||
model: "".to_string(),
|
||||
..AnalysisSettings::from_config(&base)
|
||||
};
|
||||
assert!(disabled.to_config(&base).is_ok());
|
||||
let enabled = AnalysisSettings {
|
||||
enabled: true,
|
||||
model: "".to_string(),
|
||||
..AnalysisSettings::from_config(&base)
|
||||
};
|
||||
let errs = enabled.to_config(&base).unwrap_err();
|
||||
assert!(errs.errors.iter().any(|e| e.field == "model"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dtos_serialize_to_json_and_back() {
|
||||
let c = CrawlerSettings::default();
|
||||
let v = serde_json::to_value(&c).unwrap();
|
||||
let back: CrawlerSettings = serde_json::from_value(v).unwrap();
|
||||
assert_eq!(c, back);
|
||||
|
||||
let a = AnalysisSettings::default();
|
||||
let v = serde_json::to_value(&a).unwrap();
|
||||
let back: AnalysisSettings = serde_json::from_value(v).unwrap();
|
||||
assert_eq!(a, back);
|
||||
}
|
||||
}
|
||||
@@ -1,10 +1,12 @@
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use futures_util::StreamExt as _;
|
||||
use tokio::fs;
|
||||
use tokio::io::AsyncWriteExt as _;
|
||||
use tokio_util::io::ReaderStream;
|
||||
|
||||
use super::{Storage, StorageError, StreamingFile};
|
||||
use super::{PutByteStream, Storage, StorageError, StreamingFile};
|
||||
|
||||
pub struct LocalStorage {
|
||||
root: PathBuf,
|
||||
@@ -45,6 +47,49 @@ impl Storage for LocalStorage {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn put_stream(
|
||||
&self,
|
||||
key: &str,
|
||||
mut stream: PutByteStream<'_>,
|
||||
) -> Result<u64, StorageError> {
|
||||
let path = self.resolve(key)?;
|
||||
if let Some(parent) = path.parent() {
|
||||
fs::create_dir_all(parent).await?;
|
||||
}
|
||||
// Atomic install via temp + rename. A failure mid-stream
|
||||
// removes the temp so nothing is visible at `key`. The temp
|
||||
// name uses a UUID suffix so concurrent puts of the same key
|
||||
// (e.g. two workers racing a retry) don't clobber each
|
||||
// other's in-progress file before the rename.
|
||||
let tmp = path.with_extension(format!(
|
||||
"{}.tmp.{}",
|
||||
path.extension().and_then(|e| e.to_str()).unwrap_or(""),
|
||||
uuid::Uuid::new_v4().simple()
|
||||
));
|
||||
let mut written: u64 = 0;
|
||||
let result: Result<(), StorageError> = async {
|
||||
let mut f = fs::File::create(&tmp).await?;
|
||||
while let Some(chunk) = stream.next().await {
|
||||
let chunk = chunk?;
|
||||
f.write_all(&chunk).await?;
|
||||
written = written.saturating_add(chunk.len() as u64);
|
||||
}
|
||||
// fsync before rename so a power-loss can't leave a
|
||||
// zero-byte file at the destination.
|
||||
f.sync_all().await?;
|
||||
fs::rename(&tmp, &path).await?;
|
||||
Ok(())
|
||||
}
|
||||
.await;
|
||||
if let Err(e) = result {
|
||||
// Best-effort cleanup; ignore "no such file" if the temp
|
||||
// was never created.
|
||||
let _ = fs::remove_file(&tmp).await;
|
||||
return Err(e);
|
||||
}
|
||||
Ok(written)
|
||||
}
|
||||
|
||||
async fn get(&self, key: &str) -> Result<Vec<u8>, StorageError> {
|
||||
let path = self.resolve(key)?;
|
||||
match fs::read(&path).await {
|
||||
@@ -142,6 +187,52 @@ mod tests {
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn put_stream_writes_full_body_and_removes_temp_on_error() {
|
||||
use bytes::Bytes;
|
||||
use futures_util::stream;
|
||||
|
||||
let dir = tempdir().unwrap();
|
||||
let s = LocalStorage::new(dir.path());
|
||||
|
||||
// Success: a 3-chunk stream writes the concatenation and
|
||||
// returns the right byte count.
|
||||
let chunks: Vec<Result<Bytes, StorageError>> = vec![
|
||||
Ok(Bytes::from_static(b"alpha-")),
|
||||
Ok(Bytes::from_static(b"beta-")),
|
||||
Ok(Bytes::from_static(b"gamma")),
|
||||
];
|
||||
let bytes_written = s
|
||||
.put_stream("streamed/ok.bin", Box::pin(stream::iter(chunks)))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(bytes_written, b"alpha-beta-gamma".len() as u64);
|
||||
assert_eq!(s.get("streamed/ok.bin").await.unwrap(), b"alpha-beta-gamma");
|
||||
|
||||
// Failure mid-stream: nothing is visible at the destination
|
||||
// and no .tmp file is left behind.
|
||||
let chunks: Vec<Result<Bytes, StorageError>> = vec![
|
||||
Ok(Bytes::from_static(b"good")),
|
||||
Err(StorageError::Io(std::io::Error::other("boom"))),
|
||||
];
|
||||
let err = s
|
||||
.put_stream("streamed/bad.bin", Box::pin(stream::iter(chunks)))
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(matches!(err, StorageError::Io(_)));
|
||||
assert!(matches!(
|
||||
s.get("streamed/bad.bin").await,
|
||||
Err(StorageError::NotFound)
|
||||
));
|
||||
// No stray temp file in the streamed/ directory.
|
||||
let entries: Vec<_> = std::fs::read_dir(dir.path().join("streamed"))
|
||||
.unwrap()
|
||||
.filter_map(|e| e.ok())
|
||||
.map(|e| e.file_name().to_string_lossy().to_string())
|
||||
.collect();
|
||||
assert_eq!(entries, vec!["ok.bin"]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn get_stream_emits_multiple_chunks_for_large_files() {
|
||||
use futures_util::StreamExt as _;
|
||||
|
||||
@@ -31,6 +31,13 @@ pub enum StorageError {
|
||||
/// object-safe regardless of the concrete reader behind it.
|
||||
pub type ByteStream = Pin<Box<dyn Stream<Item = io::Result<Bytes>> + Send>>;
|
||||
|
||||
/// Boxed byte stream accepted by `Storage::put_stream`. The item type
|
||||
/// is fallible so a producer (e.g. an HTTP body) can surface a transport
|
||||
/// error mid-stream without breaking the trait shape; the storage impl
|
||||
/// is responsible for not installing a partial blob on such an error.
|
||||
pub type PutByteStream<'a> =
|
||||
Pin<Box<dyn Stream<Item = Result<Bytes, StorageError>> + Send + 'a>>;
|
||||
|
||||
pub struct StreamingFile {
|
||||
pub stream: ByteStream,
|
||||
pub size_bytes: u64,
|
||||
@@ -39,6 +46,34 @@ pub struct StreamingFile {
|
||||
#[async_trait]
|
||||
pub trait Storage: Send + Sync {
|
||||
async fn put(&self, key: &str, bytes: &[u8]) -> Result<(), StorageError>;
|
||||
/// Stream a blob to storage without holding the entire body in
|
||||
/// memory. The chapter-content download path uses this so peak
|
||||
/// memory stays at one chunk per concurrent dispatch (not one full
|
||||
/// page image). The contract is atomic: a stream that errors mid-way
|
||||
/// must leave nothing visible at `key` — implementations should
|
||||
/// write to a temp location and rename only on the successful
|
||||
/// drain. Returns the total bytes written on success.
|
||||
///
|
||||
/// The default implementation buffers the stream into memory and
|
||||
/// calls `put`, so backends without a native streaming write still
|
||||
/// satisfy the contract (at the cost of peak memory). LocalStorage
|
||||
/// overrides this to do a temp-file rename; a future S3Storage
|
||||
/// would override with a multi-part upload.
|
||||
async fn put_stream(
|
||||
&self,
|
||||
key: &str,
|
||||
mut stream: PutByteStream<'_>,
|
||||
) -> Result<u64, StorageError> {
|
||||
use futures_util::StreamExt as _;
|
||||
let mut buf: Vec<u8> = Vec::new();
|
||||
while let Some(chunk) = stream.next().await {
|
||||
let chunk = chunk?;
|
||||
buf.extend_from_slice(&chunk);
|
||||
}
|
||||
let len = buf.len() as u64;
|
||||
self.put(key, &buf).await?;
|
||||
Ok(len)
|
||||
}
|
||||
/// Reads the entire blob into memory. Convenient for small assets
|
||||
/// (covers, thumbnails). For pages and other large blobs, use
|
||||
/// `get_stream` so axum can pipe bytes straight to the client.
|
||||
|
||||
333
backend/tests/analysis_worker.rs
Normal file
333
backend/tests/analysis_worker.rs
Normal file
@@ -0,0 +1,333 @@
|
||||
//! Integration tests for the analysis worker daemon: it leases only
|
||||
//! `analyze_page` jobs, acks done on success, skips already-done pages
|
||||
//! (unless forced), and on terminal failure writes a `failed` row.
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use mangalord::analysis::daemon::{self, test_support::CountingDispatcher, AnalysisDaemonConfig};
|
||||
use mangalord::domain::page_analysis::{
|
||||
AnalysisStatus, SafetyFlag, VisionAnalysis,
|
||||
};
|
||||
use mangalord::repo::page_analysis;
|
||||
use sqlx::PgPool;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use uuid::Uuid;
|
||||
|
||||
async fn seed_page(pool: &PgPool) -> Uuid {
|
||||
let manga_id: Uuid =
|
||||
sqlx::query_scalar("INSERT INTO mangas (title) VALUES ('M') RETURNING id")
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let chapter_id: Uuid = sqlx::query_scalar(
|
||||
"INSERT INTO chapters (manga_id, number) VALUES ($1, 1) RETURNING id",
|
||||
)
|
||||
.bind(manga_id)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query_scalar(
|
||||
"INSERT INTO pages (chapter_id, page_number, storage_key, content_type) \
|
||||
VALUES ($1, 1, 'k/1.png', 'image/png') RETURNING id",
|
||||
)
|
||||
.bind(chapter_id)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
async fn job_state(pool: &PgPool, page_id: Uuid) -> Option<String> {
|
||||
sqlx::query_scalar(
|
||||
"SELECT state FROM crawler_jobs \
|
||||
WHERE payload->>'kind' = 'analyze_page' AND payload->>'page_id' = $1",
|
||||
)
|
||||
.bind(page_id.to_string())
|
||||
.fetch_optional(pool)
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
/// Poll until the page's analyze_page job reaches `want`, or fail after 5s.
|
||||
async fn wait_for_state(pool: &PgPool, page_id: Uuid, want: &str) {
|
||||
let deadline = Duration::from_secs(5);
|
||||
let result = tokio::time::timeout(deadline, async {
|
||||
loop {
|
||||
if job_state(pool, page_id).await.as_deref() == Some(want) {
|
||||
return;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
})
|
||||
.await;
|
||||
assert!(result.is_ok(), "job for {page_id} never reached state {want}");
|
||||
}
|
||||
|
||||
fn spawn_with(
|
||||
pool: &PgPool,
|
||||
dispatcher: Arc<CountingDispatcher>,
|
||||
) -> (daemon::AnalysisDaemonHandle, CancellationToken) {
|
||||
let cancel = CancellationToken::new();
|
||||
let handle = daemon::spawn(
|
||||
pool.clone(),
|
||||
cancel.clone(),
|
||||
AnalysisDaemonConfig {
|
||||
dispatcher,
|
||||
workers: 1,
|
||||
job_timeout: Duration::from_secs(5),
|
||||
events: std::sync::Arc::new(
|
||||
mangalord::analysis::events::AnalysisEvents::new(),
|
||||
),
|
||||
},
|
||||
);
|
||||
(handle, cancel)
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn worker_dispatches_and_acks_done(pool: PgPool) {
|
||||
let page_id = seed_page(&pool).await;
|
||||
page_analysis::enqueue_for_page(&pool, page_id, false)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let dispatcher = CountingDispatcher::ok();
|
||||
let (handle, _c) = spawn_with(&pool, dispatcher.clone());
|
||||
wait_for_state(&pool, page_id, "done").await;
|
||||
handle.shutdown().await;
|
||||
|
||||
assert_eq!(dispatcher.call_count(), 1);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn worker_skips_already_done_page_unless_forced(pool: PgPool) {
|
||||
let page_id = seed_page(&pool).await;
|
||||
// Pre-mark the page analyzed.
|
||||
page_analysis::persist_analysis(
|
||||
&pool,
|
||||
page_id,
|
||||
&VisionAnalysis {
|
||||
ocr_results: vec![],
|
||||
tagging_results: vec![],
|
||||
scene_description: String::new(),
|
||||
safety_flag: SafetyFlag::default(),
|
||||
},
|
||||
"m",
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
page_analysis::enqueue_for_page(&pool, page_id, false)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let dispatcher = CountingDispatcher::ok();
|
||||
let (handle, _c) = spawn_with(&pool, dispatcher.clone());
|
||||
wait_for_state(&pool, page_id, "done").await;
|
||||
handle.shutdown().await;
|
||||
|
||||
assert_eq!(
|
||||
dispatcher.call_count(),
|
||||
0,
|
||||
"a non-forced job for a done page must skip dispatch"
|
||||
);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn worker_reanalyzes_done_page_when_forced(pool: PgPool) {
|
||||
let page_id = seed_page(&pool).await;
|
||||
page_analysis::persist_analysis(
|
||||
&pool,
|
||||
page_id,
|
||||
&VisionAnalysis {
|
||||
ocr_results: vec![],
|
||||
tagging_results: vec![],
|
||||
scene_description: String::new(),
|
||||
safety_flag: SafetyFlag::default(),
|
||||
},
|
||||
"m",
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
page_analysis::enqueue_for_page(&pool, page_id, true)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let dispatcher = CountingDispatcher::ok();
|
||||
let (handle, _c) = spawn_with(&pool, dispatcher.clone());
|
||||
wait_for_state(&pool, page_id, "done").await;
|
||||
handle.shutdown().await;
|
||||
|
||||
assert_eq!(dispatcher.call_count(), 1, "force must re-dispatch");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn worker_marks_failed_row_on_terminal_failure(pool: PgPool) {
|
||||
let page_id = seed_page(&pool).await;
|
||||
page_analysis::enqueue_for_page(&pool, page_id, false)
|
||||
.await
|
||||
.unwrap();
|
||||
// Make the first failure terminal (no backoff wait in the test).
|
||||
sqlx::query(
|
||||
"UPDATE crawler_jobs SET max_attempts = 1 \
|
||||
WHERE payload->>'page_id' = $1",
|
||||
)
|
||||
.bind(page_id.to_string())
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let dispatcher = CountingDispatcher::failing();
|
||||
let (handle, _c) = spawn_with(&pool, dispatcher.clone());
|
||||
wait_for_state(&pool, page_id, "dead").await;
|
||||
handle.shutdown().await;
|
||||
|
||||
let row = page_analysis::load(&pool, page_id).await.unwrap().unwrap();
|
||||
assert_eq!(row.status, AnalysisStatus::Failed);
|
||||
assert!(row.error.is_some());
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn worker_isolates_dispatcher_panics(pool: PgPool) {
|
||||
let page_id = seed_page(&pool).await;
|
||||
page_analysis::enqueue_for_page(&pool, page_id, false)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query(
|
||||
"UPDATE crawler_jobs SET max_attempts = 1 WHERE payload->>'page_id' = $1",
|
||||
)
|
||||
.bind(page_id.to_string())
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let dispatcher = CountingDispatcher::panicking();
|
||||
let (handle, _c) = spawn_with(&pool, dispatcher.clone());
|
||||
wait_for_state(&pool, page_id, "dead").await;
|
||||
handle.shutdown().await;
|
||||
|
||||
// The worker survived the panic and recorded a failed row.
|
||||
let row = page_analysis::load(&pool, page_id).await.unwrap().unwrap();
|
||||
assert_eq!(row.status, AnalysisStatus::Failed);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn worker_publishes_started_and_completed_events(pool: PgPool) {
|
||||
let page_id = seed_page(&pool).await;
|
||||
page_analysis::enqueue_for_page(&pool, page_id, false)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let events = Arc::new(mangalord::analysis::events::AnalysisEvents::new());
|
||||
let mut rx = events.subscribe();
|
||||
let cancel = CancellationToken::new();
|
||||
let handle = daemon::spawn(
|
||||
pool.clone(),
|
||||
cancel,
|
||||
AnalysisDaemonConfig {
|
||||
dispatcher: CountingDispatcher::ok(),
|
||||
workers: 1,
|
||||
job_timeout: Duration::from_secs(5),
|
||||
events: events.clone(),
|
||||
},
|
||||
);
|
||||
|
||||
let mut kinds = Vec::new();
|
||||
let _ = tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while let Ok(ev) = rx.recv().await {
|
||||
let v = serde_json::to_value(&ev).unwrap();
|
||||
let kind = v["kind"].as_str().unwrap().to_string();
|
||||
// Each event must carry the page breadcrumb.
|
||||
assert_eq!(v["page_id"].as_str().unwrap(), page_id.to_string());
|
||||
let done = kind == "completed";
|
||||
kinds.push(kind);
|
||||
if done {
|
||||
break;
|
||||
}
|
||||
}
|
||||
})
|
||||
.await;
|
||||
handle.shutdown().await;
|
||||
|
||||
assert!(kinds.contains(&"started".to_string()), "expected a started event");
|
||||
assert!(
|
||||
kinds.contains(&"completed".to_string()),
|
||||
"expected a completed event"
|
||||
);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn worker_publishes_failed_event_on_dispatch_error(pool: PgPool) {
|
||||
let page_id = seed_page(&pool).await;
|
||||
page_analysis::enqueue_for_page(&pool, page_id, false)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("UPDATE crawler_jobs SET max_attempts = 1 WHERE payload->>'page_id' = $1")
|
||||
.bind(page_id.to_string())
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let events = Arc::new(mangalord::analysis::events::AnalysisEvents::new());
|
||||
let mut rx = events.subscribe();
|
||||
let cancel = CancellationToken::new();
|
||||
let handle = daemon::spawn(
|
||||
pool.clone(),
|
||||
cancel,
|
||||
AnalysisDaemonConfig {
|
||||
dispatcher: CountingDispatcher::failing(),
|
||||
workers: 1,
|
||||
job_timeout: Duration::from_secs(5),
|
||||
events: events.clone(),
|
||||
},
|
||||
);
|
||||
|
||||
let mut saw_failed = false;
|
||||
let _ = tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while let Ok(ev) = rx.recv().await {
|
||||
let v = serde_json::to_value(&ev).unwrap();
|
||||
if v["kind"] == "failed" {
|
||||
saw_failed = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
})
|
||||
.await;
|
||||
handle.shutdown().await;
|
||||
assert!(saw_failed, "expected a failed event");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn worker_ignores_non_analyze_jobs(pool: PgPool) {
|
||||
// A crawl job must be left untouched by the analysis worker.
|
||||
use mangalord::crawler::jobs::{self, JobPayload};
|
||||
let crawl = jobs::enqueue(
|
||||
&pool,
|
||||
&JobPayload::SyncManga {
|
||||
source_id: "s".into(),
|
||||
source_manga_key: "k".into(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let crawl_id = match crawl {
|
||||
jobs::EnqueueResult::Inserted(id) => id,
|
||||
jobs::EnqueueResult::Skipped => panic!("expected insert"),
|
||||
};
|
||||
|
||||
let page_id = seed_page(&pool).await;
|
||||
page_analysis::enqueue_for_page(&pool, page_id, false)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let dispatcher = CountingDispatcher::ok();
|
||||
let (handle, _c) = spawn_with(&pool, dispatcher.clone());
|
||||
wait_for_state(&pool, page_id, "done").await;
|
||||
handle.shutdown().await;
|
||||
|
||||
let crawl_state: String =
|
||||
sqlx::query_scalar("SELECT state FROM crawler_jobs WHERE id = $1")
|
||||
.bind(crawl_id)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(crawl_state, "pending", "crawl job must be left for the crawler");
|
||||
}
|
||||
603
backend/tests/api_admin_analysis.rs
Normal file
603
backend/tests/api_admin_analysis.rs
Normal file
@@ -0,0 +1,603 @@
|
||||
//! Integration tests for the AI-analysis enqueue wiring: chapter upload
|
||||
//! enqueues `analyze_page` jobs, and the admin backfill / force-reanalyze
|
||||
//! endpoints. All gated on `analysis_enabled` (the `harness_with_analysis`
|
||||
//! variant turns it on; the default harness leaves it off → 503).
|
||||
|
||||
mod common;
|
||||
|
||||
use axum::http::StatusCode;
|
||||
use axum::Router;
|
||||
use serde_json::json;
|
||||
use sqlx::PgPool;
|
||||
use tower::ServiceExt;
|
||||
|
||||
use mangalord::repo;
|
||||
|
||||
async fn seed_admin(pool: &PgPool, app: &Router) -> String {
|
||||
let (username, cookie) = common::register_user(app).await;
|
||||
let u = repo::user::find_by_username(pool, &username)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
repo::user::set_is_admin_unchecked(pool, u.id, true)
|
||||
.await
|
||||
.unwrap();
|
||||
cookie
|
||||
}
|
||||
|
||||
/// Seed a manga → chapter with `n` pages, returning (manga_id, chapter_id,
|
||||
/// page_ids).
|
||||
async fn seed_manga_chapter_pages(
|
||||
pool: &PgPool,
|
||||
n: i32,
|
||||
) -> (uuid::Uuid, uuid::Uuid, Vec<uuid::Uuid>) {
|
||||
let manga_id: uuid::Uuid =
|
||||
sqlx::query_scalar("INSERT INTO mangas (title) VALUES ('M') RETURNING id")
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let chapter_id: uuid::Uuid = sqlx::query_scalar(
|
||||
"INSERT INTO chapters (manga_id, number) VALUES ($1, 1) RETURNING id",
|
||||
)
|
||||
.bind(manga_id)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut ids = Vec::new();
|
||||
for i in 1..=n {
|
||||
let id: uuid::Uuid = sqlx::query_scalar(
|
||||
"INSERT INTO pages (chapter_id, page_number, storage_key, content_type) \
|
||||
VALUES ($1, $2, $3, 'image/png') RETURNING id",
|
||||
)
|
||||
.bind(chapter_id)
|
||||
.bind(i)
|
||||
.bind(format!("k/{manga_id}/{i}.png"))
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
ids.push(id);
|
||||
}
|
||||
(manga_id, chapter_id, ids)
|
||||
}
|
||||
|
||||
/// Convenience wrapper for tests that only need the page ids.
|
||||
async fn seed_pages(pool: &PgPool, n: i32) -> Vec<uuid::Uuid> {
|
||||
seed_manga_chapter_pages(pool, n).await.2
|
||||
}
|
||||
|
||||
async fn analyze_job_count(pool: &PgPool) -> i64 {
|
||||
sqlx::query_scalar(
|
||||
"SELECT COUNT(*) FROM crawler_jobs WHERE payload->>'kind' = 'analyze_page'",
|
||||
)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn chapter_upload_enqueues_one_analysis_job_per_page(pool: PgPool) {
|
||||
let h = common::harness_with_analysis(pool.clone());
|
||||
let (_user, cookie) = common::register_user(&h.app).await;
|
||||
let manga_id = common::seed_manga_via_api(&h.app, &cookie, "Uploaded").await;
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::post_multipart_with_cookie(
|
||||
&format!("/api/v1/mangas/{manga_id}/chapters"),
|
||||
common::MultipartBuilder::new()
|
||||
.add_json("metadata", json!({ "number": 1, "title": "Ch 1" }))
|
||||
.add_file("page", "p1.png", "image/png", &common::fake_png_bytes())
|
||||
.add_file("page", "p2.png", "image/png", &common::fake_png_bytes()),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::CREATED);
|
||||
|
||||
assert_eq!(analyze_job_count(&pool).await, 2);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn chapter_upload_does_not_enqueue_when_analysis_disabled(pool: PgPool) {
|
||||
// Default harness has analysis_enabled = false.
|
||||
let h = common::harness(pool.clone());
|
||||
let (_user, cookie) = common::register_user(&h.app).await;
|
||||
let manga_id = common::seed_manga_via_api(&h.app, &cookie, "Uploaded").await;
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::post_multipart_with_cookie(
|
||||
&format!("/api/v1/mangas/{manga_id}/chapters"),
|
||||
common::MultipartBuilder::new()
|
||||
.add_json("metadata", json!({ "number": 1 }))
|
||||
.add_file("page", "p1.png", "image/png", &common::fake_png_bytes()),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::CREATED);
|
||||
assert_eq!(analyze_job_count(&pool).await, 0);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn reenqueue_is_forbidden_for_non_admin(pool: PgPool) {
|
||||
let h = common::harness_with_analysis(pool.clone());
|
||||
let (_user, cookie) = common::register_user(&h.app).await;
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::post_json_with_cookie(
|
||||
"/api/v1/admin/analysis/reenqueue",
|
||||
json!({}),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn reenqueue_returns_503_when_analysis_disabled(pool: PgPool) {
|
||||
let h = common::harness(pool.clone()); // analysis disabled
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::post_json_with_cookie(
|
||||
"/api/v1/admin/analysis/reenqueue",
|
||||
json!({}),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::SERVICE_UNAVAILABLE);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn reenqueue_backfills_existing_pages(pool: PgPool) {
|
||||
let h = common::harness_with_analysis(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
seed_pages(&pool, 3).await;
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::post_json_with_cookie(
|
||||
"/api/v1/admin/analysis/reenqueue",
|
||||
json!({ "only_unanalyzed": true }),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = common::body_json(resp).await;
|
||||
assert_eq!(body["enqueued"], 3);
|
||||
assert_eq!(analyze_job_count(&pool).await, 3);
|
||||
|
||||
// Idempotent: a second call enqueues nothing (pending jobs already exist).
|
||||
let resp2 = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::post_json_with_cookie(
|
||||
"/api/v1/admin/analysis/reenqueue",
|
||||
json!({ "only_unanalyzed": true }),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body2 = common::body_json(resp2).await;
|
||||
assert_eq!(body2["enqueued"], 0);
|
||||
assert_eq!(analyze_job_count(&pool).await, 3);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn reenqueue_scoped_to_manga(pool: PgPool) {
|
||||
let h = common::harness_with_analysis(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
let (manga_a, _ch_a, _pa) = seed_manga_chapter_pages(&pool, 2).await;
|
||||
seed_manga_chapter_pages(&pool, 3).await; // a different manga
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::post_json_with_cookie(
|
||||
"/api/v1/admin/analysis/reenqueue",
|
||||
json!({ "manga_id": manga_a }),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = common::body_json(resp).await;
|
||||
assert_eq!(body["enqueued"], 2, "only manga_a's pages enqueued");
|
||||
assert_eq!(analyze_job_count(&pool).await, 2);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn reenqueue_scoped_to_chapter(pool: PgPool) {
|
||||
let h = common::harness_with_analysis(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
let (_m, chapter_id, _ids) = seed_manga_chapter_pages(&pool, 4).await;
|
||||
seed_manga_chapter_pages(&pool, 5).await; // unrelated chapter
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::post_json_with_cookie(
|
||||
"/api/v1/admin/analysis/reenqueue",
|
||||
json!({ "chapter_id": chapter_id }),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = common::body_json(resp).await;
|
||||
assert_eq!(body["enqueued"], 4);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn reenqueue_including_analyzed_sets_force_and_covers_done_pages(pool: PgPool) {
|
||||
use mangalord::domain::page_analysis::{SafetyFlag, VisionAnalysis};
|
||||
|
||||
let h = common::harness_with_analysis(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
let ids = seed_pages(&pool, 2).await;
|
||||
// First page already analyzed.
|
||||
repo::page_analysis::persist_analysis(
|
||||
&pool,
|
||||
ids[0],
|
||||
&VisionAnalysis {
|
||||
ocr_results: vec![],
|
||||
tagging_results: vec![],
|
||||
scene_description: String::new(),
|
||||
safety_flag: SafetyFlag::default(),
|
||||
},
|
||||
"m",
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// only_unanalyzed=false → include the done page, with force=true.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::post_json_with_cookie(
|
||||
"/api/v1/admin/analysis/reenqueue",
|
||||
json!({ "only_unanalyzed": false }),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = common::body_json(resp).await;
|
||||
assert_eq!(body["enqueued"], 2, "both pages enqueued (incl. the done one)");
|
||||
|
||||
// The job for the already-done page must carry force=true so the worker
|
||||
// re-analyzes instead of skipping it.
|
||||
let force: bool = sqlx::query_scalar(
|
||||
"SELECT (payload->>'force')::boolean FROM crawler_jobs \
|
||||
WHERE payload->>'page_id' = $1",
|
||||
)
|
||||
.bind(ids[0].to_string())
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(force, "include-analyzed jobs must force re-analysis");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn reenqueue_manga_and_chapter_are_mutually_exclusive(pool: PgPool) {
|
||||
let h = common::harness_with_analysis(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::post_json_with_cookie(
|
||||
"/api/v1/admin/analysis/reenqueue",
|
||||
json!({ "manga_id": uuid::Uuid::new_v4(), "chapter_id": uuid::Uuid::new_v4() }),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::UNPROCESSABLE_ENTITY);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn reenqueue_unknown_scope_target_is_404(pool: PgPool) {
|
||||
let h = common::harness_with_analysis(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
|
||||
for body in [
|
||||
json!({ "manga_id": uuid::Uuid::new_v4() }),
|
||||
json!({ "chapter_id": uuid::Uuid::new_v4() }),
|
||||
] {
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::post_json_with_cookie(
|
||||
"/api/v1/admin/analysis/reenqueue",
|
||||
body,
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::NOT_FOUND);
|
||||
}
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn reenqueue_only_unanalyzed_skips_done_pages(pool: PgPool) {
|
||||
use mangalord::domain::page_analysis::{SafetyFlag, VisionAnalysis};
|
||||
|
||||
let h = common::harness_with_analysis(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
let ids = seed_pages(&pool, 2).await;
|
||||
|
||||
// Mark the first page as already analyzed.
|
||||
repo::page_analysis::persist_analysis(
|
||||
&pool,
|
||||
ids[0],
|
||||
&VisionAnalysis {
|
||||
ocr_results: vec![],
|
||||
tagging_results: vec![],
|
||||
scene_description: String::new(),
|
||||
safety_flag: SafetyFlag::default(),
|
||||
},
|
||||
"m",
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::post_json_with_cookie(
|
||||
"/api/v1/admin/analysis/reenqueue",
|
||||
json!({ "only_unanalyzed": true }),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = common::body_json(resp).await;
|
||||
assert_eq!(body["enqueued"], 1, "only the un-analyzed page is enqueued");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn force_analyze_page_enqueues_with_force_flag(pool: PgPool) {
|
||||
let h = common::harness_with_analysis(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
let ids = seed_pages(&pool, 1).await;
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::post_json_with_cookie(
|
||||
&format!("/api/v1/admin/pages/{}/analyze", ids[0]),
|
||||
json!({}),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
|
||||
let payload: serde_json::Value = sqlx::query_scalar(
|
||||
"SELECT payload FROM crawler_jobs WHERE payload->>'kind' = 'analyze_page'",
|
||||
)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(payload["force"].as_bool(), Some(true));
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn force_analyze_unknown_page_is_404(pool: PgPool) {
|
||||
let h = common::harness_with_analysis(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::post_json_with_cookie(
|
||||
&format!("/api/v1/admin/pages/{}/analyze", uuid::Uuid::new_v4()),
|
||||
json!({}),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::NOT_FOUND);
|
||||
}
|
||||
|
||||
// ---- coverage / inspection -------------------------------------------------
|
||||
|
||||
async fn get_json(h: &common::Harness, cookie: &str, uri: &str) -> serde_json::Value {
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::get_with_cookie(uri, cookie))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK, "GET {uri} failed");
|
||||
common::body_json(resp).await
|
||||
}
|
||||
|
||||
/// Seed a manga (1 chapter, 2 pages), analyze the first, return ids.
|
||||
async fn seed_partial(pool: &PgPool) -> (uuid::Uuid, uuid::Uuid, uuid::Uuid, uuid::Uuid) {
|
||||
use mangalord::domain::page_analysis::{OcrResult, SafetyFlag, VisionAnalysis};
|
||||
let (manga_id, chapter_id, ids) = seed_manga_chapter_pages(pool, 2).await;
|
||||
repo::page_analysis::persist_analysis(
|
||||
pool,
|
||||
ids[0],
|
||||
&VisionAnalysis {
|
||||
ocr_results: vec![OcrResult {
|
||||
text: "Hello".into(),
|
||||
kind: "speech".into(),
|
||||
y: None,
|
||||
}],
|
||||
tagging_results: vec!["action".into(), "city".into()],
|
||||
scene_description: "A rainy street.".into(),
|
||||
safety_flag: SafetyFlag {
|
||||
is_nsfw: true,
|
||||
content_type: vec!["gore".into()],
|
||||
},
|
||||
},
|
||||
"test-model",
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
(manga_id, chapter_id, ids[0], ids[1])
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn coverage_mangas_reports_analyzed_over_total(pool: PgPool) {
|
||||
// Coverage works with the worker disabled — use the plain harness.
|
||||
let h = common::harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
let (manga_id, _ch, _p1, _p2) = seed_partial(&pool).await;
|
||||
|
||||
let body = get_json(&h, &cookie, "/api/v1/admin/analysis/mangas?search=M").await;
|
||||
let item = &body["items"][0];
|
||||
assert_eq!(item["manga_id"].as_str().unwrap(), manga_id.to_string());
|
||||
assert_eq!(item["total_pages"], 2);
|
||||
assert_eq!(item["analyzed_pages"], 1);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn coverage_chapters_lists_per_chapter_counts(pool: PgPool) {
|
||||
let h = common::harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
let (manga_id, chapter_id, _p1, _p2) = seed_partial(&pool).await;
|
||||
|
||||
let body = get_json(
|
||||
&h,
|
||||
&cookie,
|
||||
&format!("/api/v1/admin/analysis/mangas/{manga_id}/chapters"),
|
||||
)
|
||||
.await;
|
||||
let item = &body["items"][0];
|
||||
assert_eq!(item["chapter_id"].as_str().unwrap(), chapter_id.to_string());
|
||||
assert_eq!(item["total_pages"], 2);
|
||||
assert_eq!(item["analyzed_pages"], 1);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn chapter_pages_reports_per_page_status(pool: PgPool) {
|
||||
let h = common::harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
let (_m, chapter_id, p1, p2) = seed_partial(&pool).await;
|
||||
|
||||
let body = get_json(
|
||||
&h,
|
||||
&cookie,
|
||||
&format!("/api/v1/admin/analysis/chapters/{chapter_id}/pages"),
|
||||
)
|
||||
.await;
|
||||
let items = body["items"].as_array().unwrap();
|
||||
assert_eq!(items.len(), 2);
|
||||
let status_for = |id: uuid::Uuid| {
|
||||
items
|
||||
.iter()
|
||||
.find(|i| i["page_id"].as_str().unwrap() == id.to_string())
|
||||
.unwrap()["status"]
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.to_string()
|
||||
};
|
||||
assert_eq!(status_for(p1), "done");
|
||||
assert_eq!(status_for(p2), "none");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn page_detail_returns_full_result(pool: PgPool) {
|
||||
let h = common::harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
let (_m, _ch, p1, p2) = seed_partial(&pool).await;
|
||||
|
||||
let d = get_json(&h, &cookie, &format!("/api/v1/admin/analysis/pages/{p1}")).await;
|
||||
assert_eq!(d["status"], "done");
|
||||
assert_eq!(d["is_nsfw"], true);
|
||||
assert_eq!(d["scene_description"], "A rainy street.");
|
||||
assert_eq!(d["model"], "test-model");
|
||||
assert_eq!(d["ocr"][0]["kind"], "speech");
|
||||
assert_eq!(d["ocr"][0]["text"], "Hello");
|
||||
let tags: Vec<&str> = d["tags"].as_array().unwrap().iter().map(|t| t.as_str().unwrap()).collect();
|
||||
assert!(tags.contains(&"action") && tags.contains(&"city"));
|
||||
assert_eq!(d["content_warnings"][0], "gore");
|
||||
|
||||
// Unanalyzed page → status "none", empty result.
|
||||
let d2 = get_json(&h, &cookie, &format!("/api/v1/admin/analysis/pages/{p2}")).await;
|
||||
assert_eq!(d2["status"], "none");
|
||||
assert_eq!(d2["ocr"].as_array().unwrap().len(), 0);
|
||||
assert_eq!(d2["tags"].as_array().unwrap().len(), 0);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn page_detail_unknown_page_is_404(pool: PgPool) {
|
||||
let h = common::harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::get_with_cookie(
|
||||
&format!("/api/v1/admin/analysis/pages/{}", uuid::Uuid::new_v4()),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::NOT_FOUND);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn status_stream_is_admin_gated_and_event_stream(pool: PgPool) {
|
||||
let h = common::harness(pool.clone());
|
||||
|
||||
// Non-admin → 403 (don't consume the streaming body).
|
||||
let (_u, cookie) = common::register_user(&h.app).await;
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::get_with_cookie(
|
||||
"/api/v1/admin/analysis/status/stream",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
|
||||
|
||||
// Admin → 200 with an SSE content type.
|
||||
let admin_cookie = seed_admin(&pool, &h.app).await;
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::get_with_cookie(
|
||||
"/api/v1/admin/analysis/status/stream",
|
||||
&admin_cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let ct = resp
|
||||
.headers()
|
||||
.get(axum::http::header::CONTENT_TYPE)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.unwrap_or_default();
|
||||
assert!(ct.starts_with("text/event-stream"), "got content-type {ct:?}");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn coverage_requires_admin(pool: PgPool) {
|
||||
let h = common::harness(pool.clone());
|
||||
let (_u, cookie) = common::register_user(&h.app).await;
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::get_with_cookie(
|
||||
"/api/v1/admin/analysis/mangas",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
|
||||
}
|
||||
654
backend/tests/api_admin_crawler.rs
Normal file
654
backend/tests/api_admin_crawler.rs
Normal file
@@ -0,0 +1,654 @@
|
||||
//! Integration tests for the admin crawler observability/control API.
|
||||
//!
|
||||
//! The default test harness wires `AppState.crawler = None` (no daemon),
|
||||
//! so the *control* endpoints return 503 and the *read* endpoints that
|
||||
//! work off the DB (status shell, dead-jobs list/requeue) still function.
|
||||
//! This is exactly the production "daemon disabled" posture.
|
||||
|
||||
mod common;
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use axum::http::StatusCode;
|
||||
use axum::Router;
|
||||
use http_body_util::BodyExt;
|
||||
use serde_json::json;
|
||||
use sqlx::PgPool;
|
||||
use tower::ServiceExt;
|
||||
use uuid::Uuid;
|
||||
|
||||
use common::{
|
||||
body_json, get, get_with_cookie, harness, harness_with_admin_origins,
|
||||
post_json_with_cookie, post_json_with_cookie_origin, register_user,
|
||||
};
|
||||
|
||||
async fn seed_admin(pool: &PgPool, app: &Router) -> String {
|
||||
let (username, cookie) = register_user(app).await;
|
||||
let u = mangalord::repo::user::find_by_username(pool, &username)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
mangalord::repo::user::set_is_admin_unchecked(pool, u.id, true)
|
||||
.await
|
||||
.unwrap();
|
||||
cookie
|
||||
}
|
||||
|
||||
async fn seed_dead_job(pool: &PgPool, title: &str) -> Uuid {
|
||||
let manga_id = Uuid::new_v4();
|
||||
let chapter_id = Uuid::new_v4();
|
||||
sqlx::query("INSERT INTO mangas (id, title) VALUES ($1, $2)")
|
||||
.bind(manga_id)
|
||||
.bind(title)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("INSERT INTO chapters (id, manga_id, number) VALUES ($1, $2, 1)")
|
||||
.bind(chapter_id)
|
||||
.bind(manga_id)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let job_id = Uuid::new_v4();
|
||||
sqlx::query(
|
||||
"INSERT INTO crawler_jobs (id, payload, state, attempts, last_error) \
|
||||
VALUES ($1, $2, 'dead', 5, 'boom')",
|
||||
)
|
||||
.bind(job_id)
|
||||
.bind(json!({
|
||||
"kind": "sync_chapter_content",
|
||||
"source_id": "target",
|
||||
"chapter_id": chapter_id,
|
||||
"source_chapter_key": "k",
|
||||
}))
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
job_id
|
||||
}
|
||||
|
||||
/// Seed a chapter-content job in a given state ('pending'/'running').
|
||||
async fn seed_job(pool: &PgPool, title: &str, state: &str) {
|
||||
let manga_id = Uuid::new_v4();
|
||||
let chapter_id = Uuid::new_v4();
|
||||
sqlx::query("INSERT INTO mangas (id, title) VALUES ($1, $2)")
|
||||
.bind(manga_id)
|
||||
.bind(title)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("INSERT INTO chapters (id, manga_id, number) VALUES ($1, $2, 1)")
|
||||
.bind(chapter_id)
|
||||
.bind(manga_id)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("INSERT INTO crawler_jobs (id, payload, state) VALUES ($1, $2, $3)")
|
||||
.bind(Uuid::new_v4())
|
||||
.bind(json!({
|
||||
"kind": "sync_chapter_content",
|
||||
"source_id": "target",
|
||||
"chapter_id": chapter_id,
|
||||
"source_chapter_key": "k",
|
||||
}))
|
||||
.bind(state)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
/// Seed a manga with no cover + a live source row (queued for cover fetch).
|
||||
async fn seed_missing_cover(pool: &PgPool, title: &str) {
|
||||
let manga_id = Uuid::new_v4();
|
||||
sqlx::query("INSERT INTO mangas (id, title, cover_image_path) VALUES ($1, $2, NULL)")
|
||||
.bind(manga_id)
|
||||
.bind(title)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("INSERT INTO sources (id, name, base_url) VALUES ('target','T','http://x') ON CONFLICT DO NOTHING")
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query(
|
||||
"INSERT INTO manga_sources (source_id, source_manga_key, manga_id, source_url) \
|
||||
VALUES ('target', $1, $2, 'http://x/m')",
|
||||
)
|
||||
.bind(format!("k-{manga_id}"))
|
||||
.bind(manga_id)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn active_jobs_and_covers_lists_over_http(pool: PgPool) {
|
||||
seed_job(&pool, "Naruto", "pending").await;
|
||||
seed_job(&pool, "Bleach", "running").await;
|
||||
seed_missing_cover(&pool, "One Piece").await;
|
||||
let h = harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
|
||||
// Queued/active chapters.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get_with_cookie("/api/v1/admin/crawler/active-jobs", &cookie))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = body_json(resp).await;
|
||||
assert_eq!(body["page"]["total"], 2);
|
||||
|
||||
// Queued covers.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get_with_cookie("/api/v1/admin/crawler/covers", &cookie))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = body_json(resp).await;
|
||||
assert_eq!(body["page"]["total"], 1);
|
||||
assert_eq!(body["items"][0]["manga_title"], "One Piece");
|
||||
|
||||
// Both are admin-gated.
|
||||
let (_u, plain) = register_user(&h.app).await;
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get_with_cookie("/api/v1/admin/crawler/active-jobs", &plain))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn get_status_requires_admin(pool: PgPool) {
|
||||
let h = harness(pool);
|
||||
// Unauthenticated → 401.
|
||||
let resp = h.app.clone().oneshot(get("/api/v1/admin/crawler")).await.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::UNAUTHORIZED);
|
||||
|
||||
// Authenticated non-admin → 403.
|
||||
let (_u, cookie) = register_user(&h.app).await;
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get_with_cookie("/api/v1/admin/crawler", &cookie))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn get_status_reports_disabled_daemon_with_queue_counts(pool: PgPool) {
|
||||
seed_dead_job(&pool, "Naruto").await;
|
||||
let h = harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get_with_cookie("/api/v1/admin/crawler", &cookie))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = body_json(resp).await;
|
||||
assert_eq!(body["daemon"], "disabled");
|
||||
assert_eq!(body["queue"]["dead"], 1);
|
||||
assert_eq!(body["browser"], "down");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn control_endpoints_return_503_when_daemon_disabled(pool: PgPool) {
|
||||
let h = harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
for uri in [
|
||||
"/api/v1/admin/crawler/run",
|
||||
"/api/v1/admin/crawler/browser/restart",
|
||||
"/api/v1/admin/crawler/session/clear-expired",
|
||||
] {
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(post_json_with_cookie(uri, json!({}), &cookie))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
resp.status(),
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"{uri} should be 503 when daemon disabled"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn status_stream_requires_admin(pool: PgPool) {
|
||||
let h = harness(pool);
|
||||
// Unauthenticated → 401.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get("/api/v1/admin/crawler/stream"))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::UNAUTHORIZED);
|
||||
// Non-admin → 403.
|
||||
let (_u, cookie) = register_user(&h.app).await;
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get_with_cookie("/api/v1/admin/crawler/stream", &cookie))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn status_stream_emits_initial_event(pool: PgPool) {
|
||||
let h = harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get_with_cookie("/api/v1/admin/crawler/stream", &cookie))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let ct = resp
|
||||
.headers()
|
||||
.get(axum::http::header::CONTENT_TYPE)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.unwrap_or_default()
|
||||
.to_string();
|
||||
assert!(ct.starts_with("text/event-stream"), "content-type was {ct:?}");
|
||||
|
||||
// Accumulate frames (the immediate snapshot may arrive split across
|
||||
// frames) until the status payload appears, with an overall timeout so
|
||||
// the never-ending stream can't hang the test.
|
||||
let mut body = resp.into_body();
|
||||
let mut acc = String::new();
|
||||
let deadline = tokio::time::timeout(Duration::from_secs(5), async {
|
||||
loop {
|
||||
let Some(frame) = body.frame().await else { break };
|
||||
if let Ok(data) = frame.expect("frame ok").into_data() {
|
||||
acc.push_str(&String::from_utf8_lossy(&data));
|
||||
if acc.contains("\"daemon\"") {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
.await;
|
||||
assert!(deadline.is_ok(), "did not receive status within 5s; got: {acc:?}");
|
||||
assert!(acc.contains("\"daemon\""), "missing status payload: {acc}");
|
||||
assert!(acc.contains("status"), "missing SSE event name: {acc}");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn mutating_endpoints_reject_non_admin(pool: PgPool) {
|
||||
let h = harness(pool);
|
||||
// A logged-in non-admin must be forbidden from a mutating endpoint.
|
||||
let (_u, cookie) = register_user(&h.app).await;
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(post_json_with_cookie(
|
||||
"/api/v1/admin/crawler/dead-jobs/requeue",
|
||||
json!({ "scope": "all" }),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// CSRF Origin/Referer allowlist (T2)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn csrf_rejects_mutation_with_cross_origin_header(pool: PgPool) {
|
||||
let h = harness_with_admin_origins(
|
||||
pool.clone(),
|
||||
vec!["http://localhost:3000".to_string()],
|
||||
);
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(post_json_with_cookie_origin(
|
||||
"/api/v1/admin/crawler/session/clear-expired",
|
||||
json!({}),
|
||||
&cookie,
|
||||
Some("https://evil.example.com"),
|
||||
None,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn csrf_allows_mutation_with_allowed_origin(pool: PgPool) {
|
||||
let h = harness_with_admin_origins(
|
||||
pool.clone(),
|
||||
vec!["http://localhost:3000".to_string()],
|
||||
);
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(post_json_with_cookie_origin(
|
||||
"/api/v1/admin/crawler/session/clear-expired",
|
||||
json!({}),
|
||||
&cookie,
|
||||
Some("http://localhost:3000"),
|
||||
None,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
// Daemon disabled in the harness → 503 from the handler; the CSRF
|
||||
// gate must have let the request through before that response.
|
||||
assert_eq!(resp.status(), StatusCode::SERVICE_UNAVAILABLE);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn csrf_allows_mutation_without_origin_or_referer(pool: PgPool) {
|
||||
// curl/server-to-server callers send neither — they can't be a CSRF
|
||||
// vector since there's no third-party browser context.
|
||||
let h = harness_with_admin_origins(
|
||||
pool.clone(),
|
||||
vec!["http://localhost:3000".to_string()],
|
||||
);
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(post_json_with_cookie(
|
||||
"/api/v1/admin/crawler/session/clear-expired",
|
||||
json!({}),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::SERVICE_UNAVAILABLE);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn csrf_falls_back_to_referer_when_origin_missing(pool: PgPool) {
|
||||
let h = harness_with_admin_origins(
|
||||
pool.clone(),
|
||||
vec!["http://localhost:3000".to_string()],
|
||||
);
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(post_json_with_cookie_origin(
|
||||
"/api/v1/admin/crawler/session/clear-expired",
|
||||
json!({}),
|
||||
&cookie,
|
||||
None,
|
||||
Some("http://localhost:3000/admin/crawler"),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::SERVICE_UNAVAILABLE);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn csrf_skipped_on_safe_methods(pool: PgPool) {
|
||||
let h = harness_with_admin_origins(
|
||||
pool.clone(),
|
||||
vec!["http://localhost:3000".to_string()],
|
||||
);
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
// GET from a hostile origin is fine — browsers can't mutate via GET.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::get_with_cookie_origin(
|
||||
"/api/v1/admin/crawler",
|
||||
&cookie,
|
||||
Some("https://evil.example.com"),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn csrf_disabled_when_allowlist_empty(pool: PgPool) {
|
||||
// Default harness has admin_allowed_origins = empty → operator
|
||||
// opt-out documented in .env.example. A cross-origin POST passes
|
||||
// through to the handler.
|
||||
let h = harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(post_json_with_cookie_origin(
|
||||
"/api/v1/admin/crawler/session/clear-expired",
|
||||
json!({}),
|
||||
&cookie,
|
||||
Some("https://evil.example.com"),
|
||||
None,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::SERVICE_UNAVAILABLE);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Cache-Control: no-store on admin responses (S3)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn admin_responses_have_no_store(pool: PgPool) {
|
||||
let h = harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get_with_cookie("/api/v1/admin/crawler", &cookie))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let cc = resp
|
||||
.headers()
|
||||
.get(axum::http::header::CACHE_CONTROL)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.unwrap_or_default()
|
||||
.to_string();
|
||||
assert!(
|
||||
cc.contains("no-store"),
|
||||
"admin response missing Cache-Control: no-store (got {cc:?})"
|
||||
);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn non_admin_responses_unaffected_by_no_store(pool: PgPool) {
|
||||
let h = harness(pool);
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get("/api/v1/health"))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let cc = resp
|
||||
.headers()
|
||||
.get(axum::http::header::CACHE_CONTROL)
|
||||
.and_then(|v| v.to_str().ok());
|
||||
assert!(
|
||||
cc.is_none() || !cc.unwrap_or_default().contains("no-store"),
|
||||
"non-admin response should not get no-store (got {cc:?})"
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// scope=all confirm:true guard (S1)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn requeue_all_without_confirm_rejected(pool: PgPool) {
|
||||
seed_dead_job(&pool, "X").await;
|
||||
let h = harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(post_json_with_cookie(
|
||||
"/api/v1/admin/crawler/dead-jobs/requeue",
|
||||
json!({ "scope": "all" }),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::BAD_REQUEST);
|
||||
// The dead row must NOT have been touched.
|
||||
let state: String = sqlx::query_scalar("SELECT state FROM crawler_jobs LIMIT 1")
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(state, "dead");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn requeue_all_with_confirm_flips_dead_pile(pool: PgPool) {
|
||||
seed_dead_job(&pool, "X").await;
|
||||
seed_dead_job(&pool, "Y").await;
|
||||
let h = harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(post_json_with_cookie(
|
||||
"/api/v1/admin/crawler/dead-jobs/requeue",
|
||||
json!({ "scope": "all", "confirm": true }),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let pending: i64 = sqlx::query_scalar(
|
||||
"SELECT COUNT(*) FROM crawler_jobs WHERE state = 'pending'",
|
||||
)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(pending, 2);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// admin_audit target_id + PHPSESSID fingerprint (S2)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn requeue_chapter_audit_records_chapter_target_id(pool: PgPool) {
|
||||
let job_id = seed_dead_job(&pool, "Bleach").await;
|
||||
let chapter_id: Uuid = sqlx::query_scalar(
|
||||
"SELECT (payload->>'chapter_id')::uuid FROM crawler_jobs WHERE id = $1",
|
||||
)
|
||||
.bind(job_id)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let h = harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(post_json_with_cookie(
|
||||
"/api/v1/admin/crawler/dead-jobs/requeue",
|
||||
json!({ "scope": "chapter", "chapter_id": chapter_id }),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let (target_kind, target_id): (String, Option<Uuid>) = sqlx::query_as(
|
||||
"SELECT target_kind, target_id FROM admin_audit \
|
||||
WHERE action = 'crawler_dead_jobs_requeue' \
|
||||
ORDER BY created_at DESC LIMIT 1",
|
||||
)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(target_kind, "chapter");
|
||||
assert_eq!(target_id, Some(chapter_id));
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn requeue_all_audit_omits_target_id_but_logs_count(pool: PgPool) {
|
||||
seed_dead_job(&pool, "X").await;
|
||||
let h = harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
let _ = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(post_json_with_cookie(
|
||||
"/api/v1/admin/crawler/dead-jobs/requeue",
|
||||
json!({ "scope": "all", "confirm": true }),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let (target_kind, target_id, payload): (String, Option<Uuid>, serde_json::Value) =
|
||||
sqlx::query_as(
|
||||
"SELECT target_kind, target_id, payload FROM admin_audit \
|
||||
WHERE action = 'crawler_dead_jobs_requeue' \
|
||||
ORDER BY created_at DESC LIMIT 1",
|
||||
)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(target_kind, "crawler");
|
||||
assert_eq!(target_id, None);
|
||||
assert_eq!(payload["scope"], "all");
|
||||
assert_eq!(payload["requeued"], 1);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn dead_jobs_list_and_requeue_over_http(pool: PgPool) {
|
||||
let job_id = seed_dead_job(&pool, "Bleach").await;
|
||||
let h = harness(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
|
||||
// List.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get_with_cookie("/api/v1/admin/crawler/dead-jobs", &cookie))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = body_json(resp).await;
|
||||
assert_eq!(body["page"]["total"], 1);
|
||||
assert_eq!(body["items"][0]["manga_title"], "Bleach");
|
||||
|
||||
// Requeue the single job.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(post_json_with_cookie(
|
||||
"/api/v1/admin/crawler/dead-jobs/requeue",
|
||||
json!({ "scope": "job", "job_id": job_id }),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = body_json(resp).await;
|
||||
assert_eq!(body["requeued"], 1);
|
||||
|
||||
let state: String = sqlx::query_scalar("SELECT state FROM crawler_jobs WHERE id = $1")
|
||||
.bind(job_id)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(state, "pending");
|
||||
}
|
||||
@@ -50,6 +50,10 @@ fn admin_test_router(pool: PgPool) -> (Router, TempDir) {
|
||||
upload: UploadConfig::default(),
|
||||
auth_limiter,
|
||||
resync: None,
|
||||
crawler: None,
|
||||
admin_allowed_origins: Arc::new(Vec::new()),
|
||||
analysis_enabled: false,
|
||||
analysis_events: Arc::new(mangalord::analysis::events::AnalysisEvents::new()),
|
||||
};
|
||||
let app = Router::new()
|
||||
.nest("/api/v1", api::routes())
|
||||
|
||||
272
backend/tests/api_admin_settings.rs
Normal file
272
backend/tests/api_admin_settings.rs
Normal file
@@ -0,0 +1,272 @@
|
||||
//! Integration tests for the runtime-editable settings endpoints
|
||||
//! (`/api/v1/admin/settings/{crawler,analysis}`):
|
||||
//!
|
||||
//! * the `RequireAdmin` gate,
|
||||
//! * `GET` returns the editable DTO + the read-only env-managed view (and the
|
||||
//! analysis prompt defaults), never the secret,
|
||||
//! * `PUT` validates (422 with per-field details on bad input), persists, and
|
||||
//! writes an `admin_audit` row,
|
||||
//! * `PUT` invokes the daemon reloader with the converted config and moves the
|
||||
//! analysis enable gate,
|
||||
//! * the repo-level env→DB seed is idempotent.
|
||||
|
||||
mod common;
|
||||
|
||||
use axum::http::StatusCode;
|
||||
use axum::Router;
|
||||
use serde_json::json;
|
||||
use sqlx::PgPool;
|
||||
use tower::ServiceExt;
|
||||
use uuid::Uuid;
|
||||
|
||||
use mangalord::repo;
|
||||
|
||||
async fn seed_admin(pool: &PgPool, app: &Router) -> (String, String, Uuid) {
|
||||
let (username, cookie) = common::register_user(app).await;
|
||||
let u = repo::user::find_by_username(pool, &username)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
repo::user::set_is_admin_unchecked(pool, u.id, true).await.unwrap();
|
||||
(username, cookie, u.id)
|
||||
}
|
||||
|
||||
// ---- RequireAdmin gate -----------------------------------------------------
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn get_settings_requires_admin(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
let (_u, cookie) = common::register_user(&h.app).await;
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::get_with_cookie("/api/v1/admin/settings/crawler", &cookie))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn get_settings_rejects_anonymous(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
// No cookie at all → not logged in.
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::get("/api/v1/admin/settings/crawler"))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::UNAUTHORIZED);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn put_settings_requires_admin(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
let (_u, cookie) = common::register_user(&h.app).await;
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::put_json_with_cookie(
|
||||
"/api/v1/admin/settings/analysis",
|
||||
json!({ "enabled": false }),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
|
||||
}
|
||||
|
||||
// ---- GET shape -------------------------------------------------------------
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn get_crawler_returns_editable_and_env_only(pool: PgPool) {
|
||||
let h = common::harness(pool.clone());
|
||||
let (_u, cookie, _id) = seed_admin(&pool, &h.app).await;
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::get_with_cookie("/api/v1/admin/settings/crawler", &cookie))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = common::body_json(resp).await;
|
||||
// Editable knobs present with their defaults.
|
||||
assert_eq!(body["editable"]["daily_at"], "00:00");
|
||||
assert_eq!(body["editable"]["chapter_workers"], 1);
|
||||
// Env-managed view present and read-only.
|
||||
assert_eq!(body["env_only"]["browser_mode"], "headless");
|
||||
assert_eq!(body["env_only"]["session_configured"], false);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn get_analysis_returns_defaults_and_no_secret(pool: PgPool) {
|
||||
let h = common::harness(pool.clone());
|
||||
let (_u, cookie, _id) = seed_admin(&pool, &h.app).await;
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::get_with_cookie("/api/v1/admin/settings/analysis", &cookie))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = common::body_json(resp).await;
|
||||
// Unmodified prompts seed as null ("use default").
|
||||
assert!(body["editable"]["system_prompt"].is_null());
|
||||
// Compiled defaults exposed for the UI placeholder / reset.
|
||||
assert!(body["prompt_defaults"]["system_prompt"]
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.contains("manga"));
|
||||
// The secret is never echoed; only a boolean indicator.
|
||||
assert!(body["editable"].get("api_key").is_none());
|
||||
assert_eq!(body["env_only"]["api_key_configured"], false);
|
||||
}
|
||||
|
||||
// ---- PUT validation --------------------------------------------------------
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn put_crawler_invalid_tz_returns_422(pool: PgPool) {
|
||||
let h = common::harness(pool.clone());
|
||||
let (_u, cookie, _id) = seed_admin(&pool, &h.app).await;
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::put_json_with_cookie(
|
||||
"/api/v1/admin/settings/crawler",
|
||||
json!({ "tz": "Mars/Phobos", "daily_at": "9am" }),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::UNPROCESSABLE_ENTITY);
|
||||
let body = common::body_json(resp).await;
|
||||
let fields: Vec<&str> = body["error"]["details"]["fields"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|e| e["field"].as_str().unwrap())
|
||||
.collect();
|
||||
assert!(fields.contains(&"tz"));
|
||||
assert!(fields.contains(&"daily_at"));
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn put_analysis_invalid_returns_422(pool: PgPool) {
|
||||
let h = common::harness(pool.clone());
|
||||
let (_u, cookie, _id) = seed_admin(&pool, &h.app).await;
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::put_json_with_cookie(
|
||||
"/api/v1/admin/settings/analysis",
|
||||
json!({ "workers": 0, "slice_overlap": 2.0, "endpoint": "nope" }),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::UNPROCESSABLE_ENTITY);
|
||||
let body = common::body_json(resp).await;
|
||||
let fields: Vec<&str> = body["error"]["details"]["fields"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|e| e["field"].as_str().unwrap())
|
||||
.collect();
|
||||
assert!(fields.contains(&"workers"));
|
||||
assert!(fields.contains(&"slice_overlap"));
|
||||
assert!(fields.contains(&"endpoint"));
|
||||
}
|
||||
|
||||
// ---- PUT persistence + audit ----------------------------------------------
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn put_crawler_persists_and_audits(pool: PgPool) {
|
||||
let h = common::harness(pool.clone());
|
||||
let (_u, cookie, admin_id) = seed_admin(&pool, &h.app).await;
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::put_json_with_cookie(
|
||||
"/api/v1/admin/settings/crawler",
|
||||
json!({ "rate_ms": 2500, "chapter_workers": 4, "start_url": "https://example.com/" }),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = common::body_json(resp).await;
|
||||
assert_eq!(body["editable"]["rate_ms"], 2500);
|
||||
assert_eq!(body["editable"]["chapter_workers"], 4);
|
||||
// Allowlist normalized to include the start-url host.
|
||||
let allow: Vec<&str> = body["editable"]["download_allowlist"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|v| v.as_str().unwrap())
|
||||
.collect();
|
||||
assert!(allow.contains(&"example.com"));
|
||||
|
||||
// A second GET reflects the persisted change.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::get_with_cookie("/api/v1/admin/settings/crawler", &cookie))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = common::body_json(resp).await;
|
||||
assert_eq!(body["editable"]["rate_ms"], 2500);
|
||||
|
||||
// An audit row landed.
|
||||
let (action, kind): (String, String) = sqlx::query_as(
|
||||
"SELECT action, target_kind FROM admin_audit WHERE actor_user_id = $1 ORDER BY at DESC LIMIT 1",
|
||||
)
|
||||
.bind(admin_id)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(action, "update_crawler_settings");
|
||||
assert_eq!(kind, "settings");
|
||||
}
|
||||
|
||||
// ---- PUT triggers reload ---------------------------------------------------
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn put_analysis_triggers_reload_and_flips_gate(pool: PgPool) {
|
||||
let (h, reloader) = common::harness_with_settings_reloader(pool.clone());
|
||||
let (_u, cookie, _id) = seed_admin(&pool, &h.app).await;
|
||||
|
||||
// Gate starts closed.
|
||||
assert!(!reloader.runtime.analysis_enabled());
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::put_json_with_cookie(
|
||||
"/api/v1/admin/settings/analysis",
|
||||
json!({ "enabled": true, "model": "qwen2-vl", "temperature": 0.4 }),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
|
||||
// The reloader was invoked with the converted config, and the shared gate
|
||||
// flipped on — both without a restart.
|
||||
assert!(reloader.runtime.analysis_enabled());
|
||||
let applied = reloader.analysis.lock().unwrap().clone().expect("reload called");
|
||||
assert!(applied.enabled);
|
||||
assert_eq!(applied.model, "qwen2-vl");
|
||||
assert_eq!(applied.temperature, 0.4);
|
||||
}
|
||||
|
||||
// ---- repo-level seed -------------------------------------------------------
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn seed_if_absent_is_idempotent(pool: PgPool) {
|
||||
let v1 = json!({ "rate_ms": 1000 });
|
||||
let v2 = json!({ "rate_ms": 9999 });
|
||||
// First seed inserts.
|
||||
assert!(repo::app_settings::seed_if_absent(&pool, "crawler", &v1).await.unwrap());
|
||||
// Second seed is a no-op (row already present) and does not overwrite.
|
||||
assert!(!repo::app_settings::seed_if_absent(&pool, "crawler", &v2).await.unwrap());
|
||||
let got = repo::app_settings::get(&pool, "crawler").await.unwrap().unwrap();
|
||||
assert_eq!(got["rate_ms"], 1000);
|
||||
// upsert does overwrite.
|
||||
repo::app_settings::upsert(&pool, "crawler", &v2).await.unwrap();
|
||||
let got = repo::app_settings::get(&pool, "crawler").await.unwrap().unwrap();
|
||||
assert_eq!(got["rate_ms"], 9999);
|
||||
}
|
||||
331
backend/tests/api_collection_pages.rs
Normal file
331
backend/tests/api_collection_pages.rs
Normal file
@@ -0,0 +1,331 @@
|
||||
//! Integration tests for the page-level collection endpoints:
|
||||
//!
|
||||
//! - `POST /v1/collections/:id/pages`
|
||||
//! - `DELETE /v1/collections/:id/pages/:page_id`
|
||||
//! - `GET /v1/collections/:id/pages`
|
||||
//! - `GET /v1/pages/:id/my-collections`
|
||||
//!
|
||||
//! These exercise migration 0023's `collection_pages` table end-to-end,
|
||||
//! plus the owner-only / 404-non-existence-leak / idempotency / cascade
|
||||
//! invariants that mirror `api_collections.rs`.
|
||||
|
||||
mod common;
|
||||
|
||||
use axum::http::StatusCode;
|
||||
use serde_json::{json, Value};
|
||||
use sqlx::PgPool;
|
||||
use tower::ServiceExt;
|
||||
use uuid::Uuid;
|
||||
|
||||
use common::MultipartBuilder;
|
||||
|
||||
async fn create_collection(app: &axum::Router, cookie: &str, name: &str) -> Value {
|
||||
let resp = app
|
||||
.clone()
|
||||
.oneshot(common::post_json_with_cookie(
|
||||
"/api/v1/collections",
|
||||
json!({ "name": name }),
|
||||
cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::CREATED);
|
||||
common::body_json(resp).await
|
||||
}
|
||||
|
||||
/// Create a chapter with a single page so we have a real `pages.id` to
|
||||
/// attach to. Returns `(chapter_id, page_id)`.
|
||||
async fn seed_chapter_with_page(
|
||||
app: &axum::Router,
|
||||
cookie: &str,
|
||||
manga_id: Uuid,
|
||||
number: i32,
|
||||
) -> (String, String) {
|
||||
let resp = app
|
||||
.clone()
|
||||
.oneshot(common::post_multipart_with_cookie(
|
||||
&format!("/api/v1/mangas/{manga_id}/chapters"),
|
||||
MultipartBuilder::new()
|
||||
.add_json("metadata", json!({ "number": number }))
|
||||
.add_file("page", "1.png", "image/png", &common::fake_png_bytes()),
|
||||
cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::CREATED);
|
||||
let chapter = common::body_json(resp).await;
|
||||
let chapter_id = chapter["id"].as_str().unwrap().to_string();
|
||||
|
||||
let resp = app
|
||||
.clone()
|
||||
.oneshot(common::get_with_cookie(
|
||||
&format!("/api/v1/mangas/{manga_id}/chapters/{chapter_id}/pages"),
|
||||
cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = common::body_json(resp).await;
|
||||
let page_id = body["pages"][0]["id"].as_str().unwrap().to_string();
|
||||
(chapter_id, page_id)
|
||||
}
|
||||
|
||||
fn id_of(v: &Value) -> String {
|
||||
v["id"].as_str().unwrap().to_string()
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn add_page_then_list_returns_breadcrumb(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
let (_, cookie) = common::register_user(&h.app).await;
|
||||
let manga_id = common::seed_manga_via_api(&h.app, &cookie, "Berserk").await;
|
||||
let (chapter_id, page_id) = seed_chapter_with_page(&h.app, &cookie, manga_id, 1).await;
|
||||
let coll = create_collection(&h.app, &cookie, "Favorite panels").await;
|
||||
let coll_id = id_of(&coll);
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::post_json_with_cookie(
|
||||
&format!("/api/v1/collections/{coll_id}/pages"),
|
||||
json!({ "page_id": page_id }),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::CREATED);
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::get_with_cookie(
|
||||
&format!("/api/v1/collections/{coll_id}/pages"),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = common::body_json(resp).await;
|
||||
let items = body["items"].as_array().unwrap();
|
||||
assert_eq!(items.len(), 1);
|
||||
let item = &items[0];
|
||||
assert_eq!(item["page_id"], page_id);
|
||||
assert_eq!(item["chapter_id"], chapter_id);
|
||||
assert_eq!(item["manga_id"], manga_id.to_string());
|
||||
assert_eq!(item["manga_title"], "Berserk");
|
||||
assert_eq!(item["chapter_number"], 1);
|
||||
assert_eq!(item["page_number"], 1);
|
||||
assert!(
|
||||
item["storage_key"].as_str().unwrap().starts_with(&format!(
|
||||
"mangas/{manga_id}/chapters/{chapter_id}/pages/"
|
||||
)),
|
||||
"unexpected storage_key: {}",
|
||||
item["storage_key"]
|
||||
);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn add_page_is_idempotent_and_picks_201_then_200(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
let (_, cookie) = common::register_user(&h.app).await;
|
||||
let manga_id = common::seed_manga_via_api(&h.app, &cookie, "M").await;
|
||||
let (_, page_id) = seed_chapter_with_page(&h.app, &cookie, manga_id, 1).await;
|
||||
let coll = create_collection(&h.app, &cookie, "C").await;
|
||||
let coll_id = id_of(&coll);
|
||||
|
||||
let req = || {
|
||||
common::post_json_with_cookie(
|
||||
&format!("/api/v1/collections/{coll_id}/pages"),
|
||||
json!({ "page_id": page_id }),
|
||||
&cookie,
|
||||
)
|
||||
};
|
||||
let first = h.app.clone().oneshot(req()).await.unwrap();
|
||||
assert_eq!(first.status(), StatusCode::CREATED);
|
||||
let second = h.app.oneshot(req()).await.unwrap();
|
||||
assert_eq!(second.status(), StatusCode::OK);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn add_page_returns_404_when_page_missing(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
let (_, cookie) = common::register_user(&h.app).await;
|
||||
let coll = create_collection(&h.app, &cookie, "C").await;
|
||||
let coll_id = id_of(&coll);
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::post_json_with_cookie(
|
||||
&format!("/api/v1/collections/{coll_id}/pages"),
|
||||
json!({ "page_id": Uuid::new_v4().to_string() }),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::NOT_FOUND);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn add_page_to_other_users_collection_is_404(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
let (_, a) = common::register_user(&h.app).await;
|
||||
let (_, b) = common::register_user(&h.app).await;
|
||||
|
||||
let manga_id = common::seed_manga_via_api(&h.app, &b, "M").await;
|
||||
let (_, page_id) = seed_chapter_with_page(&h.app, &b, manga_id, 1).await;
|
||||
let coll_a = create_collection(&h.app, &a, "A's").await;
|
||||
let coll_a_id = id_of(&coll_a);
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::post_json_with_cookie(
|
||||
&format!("/api/v1/collections/{coll_a_id}/pages"),
|
||||
json!({ "page_id": page_id }),
|
||||
&b,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
// Same non-leak semantic as the manga-level handler.
|
||||
assert_eq!(resp.status(), StatusCode::NOT_FOUND);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn remove_page_is_idempotent(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
let (_, cookie) = common::register_user(&h.app).await;
|
||||
let manga_id = common::seed_manga_via_api(&h.app, &cookie, "M").await;
|
||||
let (_, page_id) = seed_chapter_with_page(&h.app, &cookie, manga_id, 1).await;
|
||||
let coll = create_collection(&h.app, &cookie, "C").await;
|
||||
let coll_id = id_of(&coll);
|
||||
|
||||
let _ = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::post_json_with_cookie(
|
||||
&format!("/api/v1/collections/{coll_id}/pages"),
|
||||
json!({ "page_id": page_id }),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let first = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::delete_with_cookie(
|
||||
&format!("/api/v1/collections/{coll_id}/pages/{page_id}"),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(first.status(), StatusCode::NO_CONTENT);
|
||||
let second = h
|
||||
.app
|
||||
.oneshot(common::delete_with_cookie(
|
||||
&format!("/api/v1/collections/{coll_id}/pages/{page_id}"),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(second.status(), StatusCode::NO_CONTENT);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn my_collections_for_page_lists_only_owned(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
let (_, a) = common::register_user(&h.app).await;
|
||||
let (_, b) = common::register_user(&h.app).await;
|
||||
|
||||
let manga_id = common::seed_manga_via_api(&h.app, &a, "M").await;
|
||||
let (_, page_id) = seed_chapter_with_page(&h.app, &a, manga_id, 1).await;
|
||||
|
||||
let a_coll = create_collection(&h.app, &a, "A").await;
|
||||
let b_coll = create_collection(&h.app, &b, "B").await;
|
||||
let a_coll_id = id_of(&a_coll);
|
||||
let b_coll_id = id_of(&b_coll);
|
||||
|
||||
for (coll, cookie) in [(&a_coll_id, &a), (&b_coll_id, &b)] {
|
||||
let _ = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::post_json_with_cookie(
|
||||
&format!("/api/v1/collections/{coll}/pages"),
|
||||
json!({ "page_id": page_id }),
|
||||
cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::get_with_cookie(
|
||||
&format!("/api/v1/pages/{page_id}/my-collections"),
|
||||
&a,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = common::body_json(resp).await;
|
||||
let ids: Vec<&str> = body["collection_ids"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|v| v.as_str().unwrap())
|
||||
.collect();
|
||||
assert_eq!(ids, vec![a_coll_id.as_str()]);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn my_collections_for_unknown_page_returns_empty_list(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
let (_, cookie) = common::register_user(&h.app).await;
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::get_with_cookie(
|
||||
&format!("/api/v1/pages/{}/my-collections", Uuid::new_v4()),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = common::body_json(resp).await;
|
||||
assert_eq!(body["collection_ids"], json!([]));
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn add_page_requires_authentication(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
let (_, cookie) = common::register_user(&h.app).await;
|
||||
let manga_id = common::seed_manga_via_api(&h.app, &cookie, "M").await;
|
||||
let (_, page_id) = seed_chapter_with_page(&h.app, &cookie, manga_id, 1).await;
|
||||
let coll = create_collection(&h.app, &cookie, "C").await;
|
||||
let coll_id = id_of(&coll);
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::post_json(
|
||||
&format!("/api/v1/collections/{coll_id}/pages"),
|
||||
json!({ "page_id": page_id }),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::UNAUTHORIZED);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn list_pages_in_others_collection_is_404(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
let (_, a) = common::register_user(&h.app).await;
|
||||
let (_, b) = common::register_user(&h.app).await;
|
||||
let coll = create_collection(&h.app, &a, "Mine").await;
|
||||
let coll_id = id_of(&coll);
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::get_with_cookie(
|
||||
&format!("/api/v1/collections/{coll_id}/pages"),
|
||||
&b,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::NOT_FOUND);
|
||||
}
|
||||
140
backend/tests/api_manga_warnings.rs
Normal file
140
backend/tests/api_manga_warnings.rs
Normal file
@@ -0,0 +1,140 @@
|
||||
//! Integration tests for manga-level content warnings: the deduped union
|
||||
//! on the detail endpoint and the include/exclude filters on the list.
|
||||
|
||||
mod common;
|
||||
|
||||
use axum::http::StatusCode;
|
||||
use axum::Router;
|
||||
use sqlx::PgPool;
|
||||
use tower::ServiceExt;
|
||||
use uuid::Uuid;
|
||||
|
||||
use mangalord::domain::page_analysis::{SafetyFlag, VisionAnalysis};
|
||||
use mangalord::repo;
|
||||
|
||||
/// Seed a manga whose pages carry the given per-page warning sets, and
|
||||
/// return the manga id.
|
||||
async fn seed_manga(pool: &PgPool, title: &str, pages: &[&[&str]]) -> Uuid {
|
||||
let manga_id: Uuid =
|
||||
sqlx::query_scalar("INSERT INTO mangas (title) VALUES ($1) RETURNING id")
|
||||
.bind(title)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let chapter_id: Uuid =
|
||||
sqlx::query_scalar("INSERT INTO chapters (manga_id, number) VALUES ($1, 1) RETURNING id")
|
||||
.bind(manga_id)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
for (i, warnings) in pages.iter().enumerate() {
|
||||
let page_id: Uuid = sqlx::query_scalar(
|
||||
"INSERT INTO pages (chapter_id, page_number, storage_key, content_type) \
|
||||
VALUES ($1, $2, $3, 'image/png') RETURNING id",
|
||||
)
|
||||
.bind(chapter_id)
|
||||
.bind((i + 1) as i32)
|
||||
.bind(format!("k/{i}.png"))
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let analysis = VisionAnalysis {
|
||||
ocr_results: vec![],
|
||||
tagging_results: vec![],
|
||||
scene_description: String::new(),
|
||||
safety_flag: SafetyFlag {
|
||||
is_nsfw: !warnings.is_empty(),
|
||||
content_type: warnings.iter().map(|w| w.to_string()).collect(),
|
||||
},
|
||||
};
|
||||
repo::page_analysis::persist_analysis(pool, page_id, &analysis, "m")
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
manga_id
|
||||
}
|
||||
|
||||
async fn list_ids(app: &Router, query: &str) -> Vec<String> {
|
||||
let resp = app
|
||||
.clone()
|
||||
.oneshot(common::get(&format!("/api/v1/mangas?{query}")))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = common::body_json(resp).await;
|
||||
body["items"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|m| m["id"].as_str().unwrap().to_string())
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn detail_returns_deduped_warning_union(pool: PgPool) {
|
||||
let h = common::harness(pool.clone());
|
||||
// Two pages: one sexual+gore, one gore — union is [gore, sexual].
|
||||
let manga_id = seed_manga(&pool, "M", &[&["sexual", "gore"], &["gore"]]).await;
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::get(&format!("/api/v1/mangas/{manga_id}")))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = common::body_json(resp).await;
|
||||
let warnings: Vec<String> = body["content_warnings"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|w| w.as_str().unwrap().to_string())
|
||||
.collect();
|
||||
assert_eq!(warnings, vec!["gore", "sexual"], "deduped + alphabetical");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn detail_has_empty_warnings_when_unflagged(pool: PgPool) {
|
||||
let h = common::harness(pool.clone());
|
||||
let manga_id = seed_manga(&pool, "Clean", &[&[]]).await;
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::get(&format!("/api/v1/mangas/{manga_id}")))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = common::body_json(resp).await;
|
||||
assert_eq!(body["content_warnings"].as_array().unwrap().len(), 0);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn list_filters_by_content_warning(pool: PgPool) {
|
||||
let h = common::harness(pool.clone());
|
||||
let gory = seed_manga(&pool, "Gory", &[&["gore"]]).await;
|
||||
let sexy = seed_manga(&pool, "Sexy", &[&["sexual"]]).await;
|
||||
let clean = seed_manga(&pool, "Clean", &[&[]]).await;
|
||||
|
||||
// include=gore → only the gory manga.
|
||||
let ids = list_ids(&h.app, "cw_include=gore").await;
|
||||
assert_eq!(ids, vec![gory.to_string()]);
|
||||
|
||||
// exclude=gore → the others (sexy + clean), not gory.
|
||||
let mut ids = list_ids(&h.app, "cw_exclude=gore").await;
|
||||
ids.sort();
|
||||
let mut want = vec![sexy.to_string(), clean.to_string()];
|
||||
want.sort();
|
||||
assert_eq!(ids, want);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn list_rejects_unknown_warning(pool: PgPool) {
|
||||
let h = common::harness(pool.clone());
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::get("/api/v1/mangas?cw_include=spicy"))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::UNPROCESSABLE_ENTITY);
|
||||
}
|
||||
@@ -315,3 +315,166 @@ async fn get_unknown_id_is_404_with_envelope(pool: PgPool) {
|
||||
let msg = body["error"]["message"].as_str().expect("message is string");
|
||||
assert!(!msg.is_empty(), "message should be non-empty");
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------
|
||||
// GET /v1/mangas/:id/similar — recommendation by tag overlap (Jaccard).
|
||||
// ----------------------------------------------------------------------------
|
||||
|
||||
/// Attach a tag to a manga via the public API. Asserts the call succeeds
|
||||
/// (201 created or 200 already-attached) so test setup failures surface loudly.
|
||||
async fn attach_tag(app: &axum::Router, cookie: &str, manga_id: uuid::Uuid, name: &str) {
|
||||
let resp = app
|
||||
.clone()
|
||||
.oneshot(common::post_json_with_cookie(
|
||||
&format!("/api/v1/mangas/{manga_id}/tags"),
|
||||
json!({ "name": name }),
|
||||
cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
resp.status() == StatusCode::CREATED || resp.status() == StatusCode::OK,
|
||||
"attach_tag({name}) failed: {}",
|
||||
resp.status()
|
||||
);
|
||||
}
|
||||
|
||||
fn ids(body: &serde_json::Value) -> Vec<String> {
|
||||
body["items"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|m| m["id"].as_str().unwrap().to_string())
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn similar_ranks_by_jaccard(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
let (_, cookie) = common::register_user(&h.app).await;
|
||||
|
||||
let a = common::seed_manga_via_api(&h.app, &cookie, "Source").await;
|
||||
let b = common::seed_manga_via_api(&h.app, &cookie, "Perfect Match").await;
|
||||
let c = common::seed_manga_via_api(&h.app, &cookie, "Tight Subset").await;
|
||||
let d = common::seed_manga_via_api(&h.app, &cookie, "Broad Overlap").await;
|
||||
let e = common::seed_manga_via_api(&h.app, &cookie, "No Overlap").await;
|
||||
|
||||
// Source A: t1..t4
|
||||
for t in ["t1", "t2", "t3", "t4"] {
|
||||
attach_tag(&h.app, &cookie, a, t).await;
|
||||
}
|
||||
// B shares all 4 -> Jaccard 4/4 = 1.0
|
||||
for t in ["t1", "t2", "t3", "t4"] {
|
||||
attach_tag(&h.app, &cookie, b, t).await;
|
||||
}
|
||||
// C has t1,t2 -> shares 2 -> Jaccard 2/4 = 0.5
|
||||
for t in ["t1", "t2"] {
|
||||
attach_tag(&h.app, &cookie, c, t).await;
|
||||
}
|
||||
// D has t1..t8 -> shares 4 -> Jaccard 4/8 = 0.5 (tie with C; loses on shared-count tie-break)
|
||||
for t in ["t1", "t2", "t3", "t4", "t5", "t6", "t7", "t8"] {
|
||||
attach_tag(&h.app, &cookie, d, t).await;
|
||||
}
|
||||
// E disjoint -> excluded
|
||||
for t in ["z1", "z2"] {
|
||||
attach_tag(&h.app, &cookie, e, t).await;
|
||||
}
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::get(&format!("/api/v1/mangas/{a}/similar")))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = common::body_json(resp).await;
|
||||
let got = ids(&body);
|
||||
|
||||
let b = b.to_string();
|
||||
let c = c.to_string();
|
||||
let d = d.to_string();
|
||||
// B (1.0) first; then C and D both 0.5 but C wins the shared-count tie-break (D shares 4 too,
|
||||
// wait — D shares 4, C shares 2). Tie-break is more-shared-first, so D precedes C.
|
||||
assert_eq!(got, vec![b, d, c], "ranked B(1.0), D(0.5,4 shared), C(0.5,2 shared)");
|
||||
// Self and the disjoint manga never appear.
|
||||
assert!(!got.contains(&a.to_string()));
|
||||
assert!(!got.contains(&e.to_string()));
|
||||
// Cards carry the enriched shape.
|
||||
assert!(body["items"][0]["authors"].is_array());
|
||||
assert!(body["items"][0]["genres"].is_array());
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn similar_caps_at_five_and_excludes_self(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
let (_, cookie) = common::register_user(&h.app).await;
|
||||
|
||||
let source = common::seed_manga_via_api(&h.app, &cookie, "Source").await;
|
||||
attach_tag(&h.app, &cookie, source, "shared").await;
|
||||
|
||||
for i in 0..7 {
|
||||
let m = common::seed_manga_via_api(&h.app, &cookie, &format!("Neighbor {i}")).await;
|
||||
attach_tag(&h.app, &cookie, m, "shared").await;
|
||||
}
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::get(&format!("/api/v1/mangas/{source}/similar")))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = common::body_json(resp).await;
|
||||
let got = ids(&body);
|
||||
assert_eq!(got.len(), 5, "capped at 5");
|
||||
assert!(!got.contains(&source.to_string()), "self excluded");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn similar_empty_when_no_tags(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
let (_, cookie) = common::register_user(&h.app).await;
|
||||
let source = common::seed_manga_via_api(&h.app, &cookie, "Untagged").await;
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::get(&format!("/api/v1/mangas/{source}/similar")))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = common::body_json(resp).await;
|
||||
assert_eq!(body["items"], json!([]));
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn similar_empty_when_no_overlap(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
let (_, cookie) = common::register_user(&h.app).await;
|
||||
let source = common::seed_manga_via_api(&h.app, &cookie, "Source").await;
|
||||
let other = common::seed_manga_via_api(&h.app, &cookie, "Unrelated").await;
|
||||
attach_tag(&h.app, &cookie, source, "alpha").await;
|
||||
attach_tag(&h.app, &cookie, other, "beta").await;
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::get(&format!("/api/v1/mangas/{source}/similar")))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = common::body_json(resp).await;
|
||||
assert_eq!(body["items"], json!([]));
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn similar_404_for_unknown_manga(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::get(&format!(
|
||||
"/api/v1/mangas/{}/similar",
|
||||
uuid::Uuid::new_v4()
|
||||
)))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::NOT_FOUND);
|
||||
let body = common::body_json(resp).await;
|
||||
assert_eq!(body["error"]["code"], "not_found");
|
||||
}
|
||||
|
||||
253
backend/tests/api_page_analysis.rs
Normal file
253
backend/tests/api_page_analysis.rs
Normal file
@@ -0,0 +1,253 @@
|
||||
//! Integration tests for `repo::page_analysis` — the transactional writer
|
||||
//! that persists AI page-analysis output (OCR text, global auto-tags,
|
||||
//! content warnings, the weighted search tsvector) and the `analyze_page`
|
||||
//! job enqueue. Each `#[sqlx::test]` gets a fresh migrated DB.
|
||||
|
||||
use mangalord::domain::page_analysis::{
|
||||
AnalysisStatus, OcrResult, SafetyFlag, VisionAnalysis,
|
||||
};
|
||||
use mangalord::repo::page_analysis;
|
||||
use sqlx::PgPool;
|
||||
use uuid::Uuid;
|
||||
|
||||
/// Seed a manga → chapter → page chain and return the page id.
|
||||
async fn seed_page(pool: &PgPool) -> Uuid {
|
||||
let manga_id: Uuid =
|
||||
sqlx::query_scalar("INSERT INTO mangas (title) VALUES ('M') RETURNING id")
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let chapter_id: Uuid = sqlx::query_scalar(
|
||||
"INSERT INTO chapters (manga_id, number) VALUES ($1, 1) RETURNING id",
|
||||
)
|
||||
.bind(manga_id)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query_scalar(
|
||||
"INSERT INTO pages (chapter_id, page_number, storage_key, content_type) \
|
||||
VALUES ($1, 1, 'mangas/x/p1.png', 'image/png') RETURNING id",
|
||||
)
|
||||
.bind(chapter_id)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn analysis(
|
||||
ocr: &[(&str, &str)],
|
||||
tags: &[&str],
|
||||
scene: &str,
|
||||
nsfw: bool,
|
||||
warnings: &[&str],
|
||||
) -> VisionAnalysis {
|
||||
VisionAnalysis {
|
||||
ocr_results: ocr
|
||||
.iter()
|
||||
.map(|(text, kind)| OcrResult {
|
||||
text: (*text).to_string(),
|
||||
kind: (*kind).to_string(),
|
||||
y: None,
|
||||
})
|
||||
.collect(),
|
||||
tagging_results: tags.iter().map(|t| t.to_string()).collect(),
|
||||
scene_description: scene.to_string(),
|
||||
safety_flag: SafetyFlag {
|
||||
is_nsfw: nsfw,
|
||||
content_type: warnings.iter().map(|w| w.to_string()).collect(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
async fn count(pool: &PgPool, table: &str, page_id: Uuid) -> i64 {
|
||||
sqlx::query_scalar(&format!("SELECT count(*) FROM {table} WHERE page_id = $1"))
|
||||
.bind(page_id)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn persist_writes_ocr_tags_warnings_and_search_doc(pool: PgPool) {
|
||||
let page_id = seed_page(&pool).await;
|
||||
let a = analysis(
|
||||
&[("Hello", "speech"), ("BOOM", "sfx")],
|
||||
&["action", "city"],
|
||||
"A rainy street at night.",
|
||||
true,
|
||||
&["violence"],
|
||||
);
|
||||
|
||||
page_analysis::persist_analysis(&pool, page_id, &a, "test-model")
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(count(&pool, "page_ocr_text", page_id).await, 2);
|
||||
assert_eq!(count(&pool, "page_auto_tags", page_id).await, 2);
|
||||
assert_eq!(count(&pool, "page_content_warnings", page_id).await, 1);
|
||||
|
||||
let row = page_analysis::load(&pool, page_id).await.unwrap().unwrap();
|
||||
assert_eq!(row.status, AnalysisStatus::Done);
|
||||
assert!(row.is_nsfw);
|
||||
assert_eq!(row.scene_description.as_deref(), Some("A rainy street at night."));
|
||||
assert_eq!(row.model.as_deref(), Some("test-model"));
|
||||
|
||||
// search_doc must be populated (non-null, non-empty).
|
||||
let has_doc: bool = sqlx::query_scalar(
|
||||
"SELECT search_doc IS NOT NULL AND search_doc != ''::tsvector \
|
||||
FROM page_analysis WHERE page_id = $1",
|
||||
)
|
||||
.bind(page_id)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(has_doc, "search_doc should be a populated tsvector");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn persist_is_idempotent_and_replaces_prior_result(pool: PgPool) {
|
||||
let page_id = seed_page(&pool).await;
|
||||
|
||||
page_analysis::persist_analysis(
|
||||
&pool,
|
||||
page_id,
|
||||
&analysis(&[("one", "speech")], &["alpha", "beta"], "first", true, &["gore"]),
|
||||
"m1",
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Re-analysis with different output fully replaces the prior rows.
|
||||
page_analysis::persist_analysis(
|
||||
&pool,
|
||||
page_id,
|
||||
&analysis(&[("two", "narration")], &["beta", "gamma"], "second", false, &[]),
|
||||
"m2",
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Exactly one analysis row, reflecting the second pass.
|
||||
let total: i64 = sqlx::query_scalar("SELECT count(*) FROM page_analysis WHERE page_id = $1")
|
||||
.bind(page_id)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(total, 1);
|
||||
|
||||
let row = page_analysis::load(&pool, page_id).await.unwrap().unwrap();
|
||||
assert!(!row.is_nsfw);
|
||||
assert_eq!(row.scene_description.as_deref(), Some("second"));
|
||||
assert_eq!(row.model.as_deref(), Some("m2"));
|
||||
|
||||
// Auto-tags are {beta, gamma}, NOT the union with the first pass.
|
||||
let tag_names: Vec<String> = sqlx::query_scalar(
|
||||
"SELECT t.name FROM page_auto_tags pat JOIN tags t ON t.id = pat.tag_id \
|
||||
WHERE pat.page_id = $1 ORDER BY t.name",
|
||||
)
|
||||
.bind(page_id)
|
||||
.fetch_all(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(tag_names, vec!["beta", "gamma"]);
|
||||
|
||||
// Warnings cleared (second pass had none).
|
||||
assert_eq!(count(&pool, "page_content_warnings", page_id).await, 0);
|
||||
assert_eq!(count(&pool, "page_ocr_text", page_id).await, 1);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn persist_leaves_user_page_tags_untouched(pool: PgPool) {
|
||||
let page_id = seed_page(&pool).await;
|
||||
let user_id: Uuid = sqlx::query_scalar(
|
||||
"INSERT INTO users (username, password_hash) VALUES ('alice', 'x') RETURNING id",
|
||||
)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// A personal page tag the user added.
|
||||
mangalord::repo::page_tag::upsert(&pool, user_id, page_id, "favourite")
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// The auto-tagger proposes its own (overlapping + new) tags.
|
||||
page_analysis::persist_analysis(
|
||||
&pool,
|
||||
page_id,
|
||||
&analysis(&[], &["favourite", "robot"], "", false, &[]),
|
||||
"m",
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// The user's personal tag survives untouched.
|
||||
let mine = mangalord::repo::page_tag::list_for_page(&pool, user_id, page_id)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(mine, vec!["favourite"]);
|
||||
|
||||
// And the global auto-tags are separate rows.
|
||||
assert_eq!(count(&pool, "page_auto_tags", page_id).await, 2);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn enqueue_for_page_inserts_analyze_page_job(pool: PgPool) {
|
||||
let page_id = seed_page(&pool).await;
|
||||
|
||||
page_analysis::enqueue_for_page(&pool, page_id, false)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let payload: serde_json::Value = sqlx::query_scalar(
|
||||
"SELECT payload FROM crawler_jobs WHERE payload->>'kind' = 'analyze_page'",
|
||||
)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(payload["page_id"].as_str().unwrap(), page_id.to_string());
|
||||
assert_eq!(payload["force"].as_bool(), Some(false));
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn mark_failed_writes_a_failed_row(pool: PgPool) {
|
||||
let page_id = seed_page(&pool).await;
|
||||
|
||||
page_analysis::mark_failed(&pool, page_id, "model timed out")
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let row = page_analysis::load(&pool, page_id).await.unwrap().unwrap();
|
||||
assert_eq!(row.status, AnalysisStatus::Failed);
|
||||
assert_eq!(row.error.as_deref(), Some("model timed out"));
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn search_doc_ranks_speech_above_sfx(pool: PgPool) {
|
||||
let page_id = seed_page(&pool).await;
|
||||
// "alpha" is speech (weight A), "beta" is sfx (weight D).
|
||||
page_analysis::persist_analysis(
|
||||
&pool,
|
||||
page_id,
|
||||
&analysis(&[("alpha", "speech"), ("beta", "sfx")], &[], "", false, &[]),
|
||||
"m",
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let (ra, rb): (f32, f32) = sqlx::query_as(
|
||||
"SELECT ts_rank(search_doc, plainto_tsquery('simple', 'alpha')), \
|
||||
ts_rank(search_doc, plainto_tsquery('simple', 'beta')) \
|
||||
FROM page_analysis WHERE page_id = $1",
|
||||
)
|
||||
.bind(page_id)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(ra > 0.0 && rb > 0.0, "both terms should match the doc");
|
||||
assert!(
|
||||
ra > rb,
|
||||
"speech (weight A) must rank higher than sfx (weight D): {ra} vs {rb}"
|
||||
);
|
||||
}
|
||||
266
backend/tests/api_page_search.rs
Normal file
266
backend/tests/api_page_search.rs
Normal file
@@ -0,0 +1,266 @@
|
||||
//! Integration tests for the page content-search endpoint
|
||||
//! `GET /v1/me/page-search`: multi-tag AND across (user ∪ auto) tags,
|
||||
//! weighted OCR/scene text ranking, and content-warning include/exclude.
|
||||
|
||||
mod common;
|
||||
|
||||
use axum::http::StatusCode;
|
||||
use axum::Router;
|
||||
use serde_json::json;
|
||||
use sqlx::PgPool;
|
||||
use tower::ServiceExt;
|
||||
use uuid::Uuid;
|
||||
|
||||
use mangalord::domain::page_analysis::{OcrResult, SafetyFlag, VisionAnalysis};
|
||||
use mangalord::repo;
|
||||
|
||||
/// Seed a manga + chapter and return the chapter id for adding pages.
|
||||
async fn seed_chapter(pool: &PgPool, title: &str) -> Uuid {
|
||||
let manga_id: Uuid =
|
||||
sqlx::query_scalar("INSERT INTO mangas (title) VALUES ($1) RETURNING id")
|
||||
.bind(title)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query_scalar("INSERT INTO chapters (manga_id, number) VALUES ($1, 1) RETURNING id")
|
||||
.bind(manga_id)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
async fn add_page(pool: &PgPool, chapter_id: Uuid, n: i32) -> Uuid {
|
||||
sqlx::query_scalar(
|
||||
"INSERT INTO pages (chapter_id, page_number, storage_key, content_type) \
|
||||
VALUES ($1, $2, $3, 'image/png') RETURNING id",
|
||||
)
|
||||
.bind(chapter_id)
|
||||
.bind(n)
|
||||
.bind(format!("k/{n}.png"))
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn analysis(
|
||||
ocr: &[(&str, &str)],
|
||||
auto_tags: &[&str],
|
||||
nsfw: bool,
|
||||
warnings: &[&str],
|
||||
) -> VisionAnalysis {
|
||||
VisionAnalysis {
|
||||
ocr_results: ocr
|
||||
.iter()
|
||||
.map(|(t, k)| OcrResult {
|
||||
text: (*t).into(),
|
||||
kind: (*k).into(),
|
||||
y: None,
|
||||
})
|
||||
.collect(),
|
||||
tagging_results: auto_tags.iter().map(|t| t.to_string()).collect(),
|
||||
scene_description: String::new(),
|
||||
safety_flag: SafetyFlag {
|
||||
is_nsfw: nsfw,
|
||||
content_type: warnings.iter().map(|w| w.to_string()).collect(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
async fn user_id_for(pool: &PgPool, username: &str) -> Uuid {
|
||||
repo::user::find_by_username(pool, username)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.id
|
||||
}
|
||||
|
||||
async fn search(app: &Router, cookie: &str, query: &str) -> serde_json::Value {
|
||||
let resp = app
|
||||
.clone()
|
||||
.oneshot(common::get_with_cookie(
|
||||
&format!("/api/v1/me/page-search?{query}"),
|
||||
cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK, "search {query} failed");
|
||||
common::body_json(resp).await
|
||||
}
|
||||
|
||||
fn page_ids(body: &serde_json::Value) -> Vec<String> {
|
||||
body["items"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|i| i["page_id"].as_str().unwrap().to_string())
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn multi_tag_and_across_user_and_auto_tags(pool: PgPool) {
|
||||
let h = common::harness(pool.clone());
|
||||
let (username, cookie) = common::register_user(&h.app).await;
|
||||
let uid = user_id_for(&pool, &username).await;
|
||||
|
||||
let ch = seed_chapter(&pool, "M").await;
|
||||
let p1 = add_page(&pool, ch, 1).await;
|
||||
let p2 = add_page(&pool, ch, 2).await;
|
||||
|
||||
// p1: auto-tag "b"; user adds tag "a" → has both a (user) and b (auto).
|
||||
repo::page_analysis::persist_analysis(&pool, p1, &analysis(&[], &["b"], false, &[]), "m")
|
||||
.await
|
||||
.unwrap();
|
||||
repo::page_tag::upsert(&pool, uid, p1, "a").await.unwrap();
|
||||
// p2: only auto-tag "a".
|
||||
repo::page_analysis::persist_analysis(&pool, p2, &analysis(&[], &["a"], false, &[]), "m")
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// tags=a,b (AND) → only p1 satisfies both.
|
||||
let body = search(&h.app, &cookie, "tags=a,b").await;
|
||||
assert_eq!(page_ids(&body), vec![p1.to_string()]);
|
||||
|
||||
// tags=a alone → both (p1 via user tag, p2 via auto tag).
|
||||
let body = search(&h.app, &cookie, "tags=a").await;
|
||||
let mut ids = page_ids(&body);
|
||||
ids.sort();
|
||||
let mut want = vec![p1.to_string(), p2.to_string()];
|
||||
want.sort();
|
||||
assert_eq!(ids, want);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn text_search_ranks_speech_above_sfx(pool: PgPool) {
|
||||
let h = common::harness(pool.clone());
|
||||
let (_u, cookie) = common::register_user(&h.app).await;
|
||||
let ch = seed_chapter(&pool, "M").await;
|
||||
let speech_page = add_page(&pool, ch, 1).await;
|
||||
let sfx_page = add_page(&pool, ch, 2).await;
|
||||
|
||||
repo::page_analysis::persist_analysis(
|
||||
&pool,
|
||||
speech_page,
|
||||
&analysis(&[("alpha", "speech")], &[], false, &[]),
|
||||
"m",
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
repo::page_analysis::persist_analysis(
|
||||
&pool,
|
||||
sfx_page,
|
||||
&analysis(&[("alpha", "sfx")], &[], false, &[]),
|
||||
"m",
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let body = search(&h.app, &cookie, "text=alpha").await;
|
||||
let ids = page_ids(&body);
|
||||
assert_eq!(ids.len(), 2, "both pages match the term");
|
||||
assert_eq!(
|
||||
ids[0],
|
||||
speech_page.to_string(),
|
||||
"speech (weight A) must rank above sfx (weight D)"
|
||||
);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn content_warning_include_and_exclude(pool: PgPool) {
|
||||
let h = common::harness(pool.clone());
|
||||
let (_u, cookie) = common::register_user(&h.app).await;
|
||||
let ch = seed_chapter(&pool, "M").await;
|
||||
let sexual = add_page(&pool, ch, 1).await;
|
||||
let gore = add_page(&pool, ch, 2).await;
|
||||
|
||||
repo::page_analysis::persist_analysis(
|
||||
&pool,
|
||||
sexual,
|
||||
&analysis(&[("x", "speech")], &["nsfw-tag"], true, &["sexual"]),
|
||||
"m",
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
repo::page_analysis::persist_analysis(
|
||||
&pool,
|
||||
gore,
|
||||
&analysis(&[("x", "speech")], &["nsfw-tag"], true, &["gore"]),
|
||||
"m",
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// include sexual → only the sexual page.
|
||||
let body = search(&h.app, &cookie, "tags=nsfw-tag&cw_include=sexual").await;
|
||||
assert_eq!(page_ids(&body), vec![sexual.to_string()]);
|
||||
// and it carries the warning + nsfw flag in the row.
|
||||
assert_eq!(body["items"][0]["is_nsfw"], true);
|
||||
assert_eq!(body["items"][0]["content_warnings"][0], "sexual");
|
||||
|
||||
// exclude sexual → only the gore page.
|
||||
let body = search(&h.app, &cookie, "tags=nsfw-tag&cw_exclude=sexual").await;
|
||||
assert_eq!(page_ids(&body), vec![gore.to_string()]);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn text_only_search_needs_no_tags(pool: PgPool) {
|
||||
let h = common::harness(pool.clone());
|
||||
let (_u, cookie) = common::register_user(&h.app).await;
|
||||
let ch = seed_chapter(&pool, "M").await;
|
||||
let p = add_page(&pool, ch, 1).await;
|
||||
repo::page_analysis::persist_analysis(
|
||||
&pool,
|
||||
p,
|
||||
&analysis(&[("dragon", "speech")], &[], false, &[]),
|
||||
"m",
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// The previously-501 text param now returns results (no tags needed).
|
||||
let body = search(&h.app, &cookie, "text=dragon").await;
|
||||
assert_eq!(page_ids(&body), vec![p.to_string()]);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn search_without_any_filter_is_rejected(pool: PgPool) {
|
||||
let h = common::harness(pool.clone());
|
||||
let (_u, cookie) = common::register_user(&h.app).await;
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::get_with_cookie("/api/v1/me/page-search", &cookie))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::UNPROCESSABLE_ENTITY);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn unknown_content_warning_is_rejected(pool: PgPool) {
|
||||
let h = common::harness(pool.clone());
|
||||
let (_u, cookie) = common::register_user(&h.app).await;
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::get_with_cookie(
|
||||
"/api/v1/me/page-search?cw_include=spicy",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::UNPROCESSABLE_ENTITY);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn requires_authentication(pool: PgPool) {
|
||||
let h = common::harness(pool.clone());
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::get("/api/v1/me/page-search?text=x"))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::UNAUTHORIZED);
|
||||
let _ = json!({}); // silence unused import in some configs
|
||||
}
|
||||
712
backend/tests/api_page_tags.rs
Normal file
712
backend/tests/api_page_tags.rs
Normal file
@@ -0,0 +1,712 @@
|
||||
//! Integration tests for the page-tag endpoints:
|
||||
//!
|
||||
//! - `POST /v1/pages/:id/tags`
|
||||
//! - `DELETE /v1/pages/:id/tags/:tag`
|
||||
//! - `GET /v1/pages/:id/my-tags`
|
||||
//! - `GET /v1/me/page-tags`
|
||||
//! - `GET /v1/me/page-tags/distinct`
|
||||
//! - `GET /v1/me/page-tags/chapters`
|
||||
//! - `GET /v1/me/page-tags/mangas`
|
||||
//!
|
||||
//! Together with `repo::page_tag` and migration 0023's `page_tags`
|
||||
//! table. Validation behaviour and the page-deletion cascade are
|
||||
//! pinned here.
|
||||
|
||||
mod common;
|
||||
|
||||
use axum::http::StatusCode;
|
||||
use serde_json::{json, Value};
|
||||
use sqlx::PgPool;
|
||||
use tower::ServiceExt;
|
||||
use uuid::Uuid;
|
||||
|
||||
use common::MultipartBuilder;
|
||||
|
||||
async fn seed_chapter_with_page(
|
||||
app: &axum::Router,
|
||||
cookie: &str,
|
||||
manga_id: Uuid,
|
||||
number: i32,
|
||||
) -> (String, String) {
|
||||
let resp = app
|
||||
.clone()
|
||||
.oneshot(common::post_multipart_with_cookie(
|
||||
&format!("/api/v1/mangas/{manga_id}/chapters"),
|
||||
MultipartBuilder::new()
|
||||
.add_json("metadata", json!({ "number": number }))
|
||||
.add_file("page", "1.png", "image/png", &common::fake_png_bytes()),
|
||||
cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::CREATED);
|
||||
let chapter = common::body_json(resp).await;
|
||||
let chapter_id = chapter["id"].as_str().unwrap().to_string();
|
||||
|
||||
let resp = app
|
||||
.clone()
|
||||
.oneshot(common::get_with_cookie(
|
||||
&format!("/api/v1/mangas/{manga_id}/chapters/{chapter_id}/pages"),
|
||||
cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = common::body_json(resp).await;
|
||||
let page_id = body["pages"][0]["id"].as_str().unwrap().to_string();
|
||||
(chapter_id, page_id)
|
||||
}
|
||||
|
||||
/// Like `seed_chapter_with_page` but uploads `n` pages so the
|
||||
/// aggregation tests can tag a subset and verify match counts.
|
||||
/// Returns `(chapter_id, page_ids_in_order)`.
|
||||
async fn seed_chapter_with_n_pages(
|
||||
app: &axum::Router,
|
||||
cookie: &str,
|
||||
manga_id: Uuid,
|
||||
number: i32,
|
||||
n: usize,
|
||||
) -> (String, Vec<String>) {
|
||||
let mut builder = MultipartBuilder::new()
|
||||
.add_json("metadata", json!({ "number": number }));
|
||||
for i in 0..n {
|
||||
let fname = format!("{:04}.png", i + 1);
|
||||
builder = builder.add_file("page", &fname, "image/png", &common::fake_png_bytes());
|
||||
}
|
||||
let resp = app
|
||||
.clone()
|
||||
.oneshot(common::post_multipart_with_cookie(
|
||||
&format!("/api/v1/mangas/{manga_id}/chapters"),
|
||||
builder,
|
||||
cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::CREATED);
|
||||
let chapter = common::body_json(resp).await;
|
||||
let chapter_id = chapter["id"].as_str().unwrap().to_string();
|
||||
|
||||
let resp = app
|
||||
.clone()
|
||||
.oneshot(common::get_with_cookie(
|
||||
&format!("/api/v1/mangas/{manga_id}/chapters/{chapter_id}/pages"),
|
||||
cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = common::body_json(resp).await;
|
||||
let page_ids: Vec<String> = body["pages"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|p| p["id"].as_str().unwrap().to_string())
|
||||
.collect();
|
||||
(chapter_id, page_ids)
|
||||
}
|
||||
|
||||
async fn add_tag(
|
||||
app: &axum::Router,
|
||||
cookie: &str,
|
||||
page_id: &str,
|
||||
tag: &str,
|
||||
) -> StatusCode {
|
||||
let resp = app
|
||||
.clone()
|
||||
.oneshot(common::post_json_with_cookie(
|
||||
&format!("/api/v1/pages/{page_id}/tags"),
|
||||
json!({ "tag": tag }),
|
||||
cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
resp.status()
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn add_then_list_returns_normalized_tag(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
let (_, cookie) = common::register_user(&h.app).await;
|
||||
let manga_id = common::seed_manga_via_api(&h.app, &cookie, "M").await;
|
||||
let (_, page_id) = seed_chapter_with_page(&h.app, &cookie, manga_id, 1).await;
|
||||
|
||||
assert_eq!(add_tag(&h.app, &cookie, &page_id, " Funny ").await, StatusCode::CREATED);
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::get_with_cookie(
|
||||
&format!("/api/v1/pages/{page_id}/my-tags"),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = common::body_json(resp).await;
|
||||
// Stored as normalized "funny".
|
||||
assert_eq!(body["tags"], json!(["funny"]));
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn add_is_idempotent_picks_201_then_200(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
let (_, cookie) = common::register_user(&h.app).await;
|
||||
let manga_id = common::seed_manga_via_api(&h.app, &cookie, "M").await;
|
||||
let (_, page_id) = seed_chapter_with_page(&h.app, &cookie, manga_id, 1).await;
|
||||
|
||||
assert_eq!(add_tag(&h.app, &cookie, &page_id, "funny").await, StatusCode::CREATED);
|
||||
// Even after re-casing, the normalized form collides → 200.
|
||||
assert_eq!(add_tag(&h.app, &cookie, &page_id, "FUNNY").await, StatusCode::OK);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn add_unknown_page_is_404(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
let (_, cookie) = common::register_user(&h.app).await;
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::post_json_with_cookie(
|
||||
&format!("/api/v1/pages/{}/tags", Uuid::new_v4()),
|
||||
json!({ "tag": "funny" }),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::NOT_FOUND);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn add_blank_tag_is_422(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
let (_, cookie) = common::register_user(&h.app).await;
|
||||
let manga_id = common::seed_manga_via_api(&h.app, &cookie, "M").await;
|
||||
let (_, page_id) = seed_chapter_with_page(&h.app, &cookie, manga_id, 1).await;
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::post_json_with_cookie(
|
||||
&format!("/api/v1/pages/{page_id}/tags"),
|
||||
json!({ "tag": " " }),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::UNPROCESSABLE_ENTITY);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn add_tag_too_long_is_422(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
let (_, cookie) = common::register_user(&h.app).await;
|
||||
let manga_id = common::seed_manga_via_api(&h.app, &cookie, "M").await;
|
||||
let (_, page_id) = seed_chapter_with_page(&h.app, &cookie, manga_id, 1).await;
|
||||
|
||||
let long = "a".repeat(65);
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::post_json_with_cookie(
|
||||
&format!("/api/v1/pages/{page_id}/tags"),
|
||||
json!({ "tag": long }),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::UNPROCESSABLE_ENTITY);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn add_tag_with_invisible_format_char_is_422(pool: PgPool) {
|
||||
// Without the format-char guard, `"funny"` and `"funny\u{200d}"`
|
||||
// (zero-width joiner appended) would be stored as distinct rows,
|
||||
// visually identical to the user and indistinguishable in the
|
||||
// chip cloud. Same risk for bidi overrides and BOM.
|
||||
let h = common::harness(pool);
|
||||
let (_, cookie) = common::register_user(&h.app).await;
|
||||
let manga_id = common::seed_manga_via_api(&h.app, &cookie, "M").await;
|
||||
let (_, page_id) = seed_chapter_with_page(&h.app, &cookie, manga_id, 1).await;
|
||||
|
||||
for bad in [
|
||||
"funny\u{200d}", // ZWJ
|
||||
"a\u{202e}b", // RLO
|
||||
"a\u{feff}b", // ZWNBSP / BOM
|
||||
"a\u{200b}b", // ZWSP
|
||||
] {
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::post_json_with_cookie(
|
||||
&format!("/api/v1/pages/{page_id}/tags"),
|
||||
json!({ "tag": bad }),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
resp.status(),
|
||||
StatusCode::UNPROCESSABLE_ENTITY,
|
||||
"expected 422 for tag containing invisible char (bytes: {:?})",
|
||||
bad.as_bytes()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn add_tag_with_path_breaking_char_is_422(pool: PgPool) {
|
||||
// Without this guard, `"a/b"` would be storable via POST (JSON
|
||||
// body) but unremovable via DELETE (axum decodes %2F back to `/`
|
||||
// and the `:tag` segment never matches). The 422 keeps every
|
||||
// stored tag round-trippable.
|
||||
let h = common::harness(pool);
|
||||
let (_, cookie) = common::register_user(&h.app).await;
|
||||
let manga_id = common::seed_manga_via_api(&h.app, &cookie, "M").await;
|
||||
let (_, page_id) = seed_chapter_with_page(&h.app, &cookie, manga_id, 1).await;
|
||||
|
||||
for bad in ["a/b", "a?b", "a#b", "a%b", "a_b", "a\\b"] {
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::post_json_with_cookie(
|
||||
&format!("/api/v1/pages/{page_id}/tags"),
|
||||
json!({ "tag": bad }),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
resp.status(),
|
||||
StatusCode::UNPROCESSABLE_ENTITY,
|
||||
"expected 422 for tag {bad:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn add_tag_with_control_char_is_422(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
let (_, cookie) = common::register_user(&h.app).await;
|
||||
let manga_id = common::seed_manga_via_api(&h.app, &cookie, "M").await;
|
||||
let (_, page_id) = seed_chapter_with_page(&h.app, &cookie, manga_id, 1).await;
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::post_json_with_cookie(
|
||||
&format!("/api/v1/pages/{page_id}/tags"),
|
||||
json!({ "tag": "bad\ntag" }),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::UNPROCESSABLE_ENTITY);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn remove_normalizes_url_tag(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
let (_, cookie) = common::register_user(&h.app).await;
|
||||
let manga_id = common::seed_manga_via_api(&h.app, &cookie, "M").await;
|
||||
let (_, page_id) = seed_chapter_with_page(&h.app, &cookie, manga_id, 1).await;
|
||||
|
||||
assert_eq!(add_tag(&h.app, &cookie, &page_id, "funny").await, StatusCode::CREATED);
|
||||
|
||||
// DELETE arrives with the original case; the handler renormalizes
|
||||
// so it still matches "funny" in storage.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::delete_with_cookie(
|
||||
&format!("/api/v1/pages/{page_id}/tags/FUNNY"),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::NO_CONTENT);
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::get_with_cookie(
|
||||
&format!("/api/v1/pages/{page_id}/my-tags"),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body: Value = common::body_json(resp).await;
|
||||
assert_eq!(body["tags"], json!([]));
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn list_mine_filters_by_tag(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
let (_, cookie) = common::register_user(&h.app).await;
|
||||
let manga_id = common::seed_manga_via_api(&h.app, &cookie, "M").await;
|
||||
let (_, page_id) = seed_chapter_with_page(&h.app, &cookie, manga_id, 1).await;
|
||||
|
||||
assert_eq!(add_tag(&h.app, &cookie, &page_id, "funny").await, StatusCode::CREATED);
|
||||
assert_eq!(add_tag(&h.app, &cookie, &page_id, "fight").await, StatusCode::CREATED);
|
||||
assert_eq!(add_tag(&h.app, &cookie, &page_id, "panel-of-the-day").await, StatusCode::CREATED);
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::get_with_cookie("/api/v1/me/page-tags?tag=fight", &cookie))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = common::body_json(resp).await;
|
||||
let items = body["items"].as_array().unwrap();
|
||||
assert_eq!(items.len(), 1);
|
||||
assert_eq!(items[0]["tag"], "fight");
|
||||
assert_eq!(items[0]["page_id"], page_id);
|
||||
assert_eq!(items[0]["manga_title"], "M");
|
||||
|
||||
// Prefix filter ?q=fu matches "funny".
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::get_with_cookie("/api/v1/me/page-tags?q=fu", &cookie))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = common::body_json(resp).await;
|
||||
let tags: Vec<&str> = body["items"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|v| v["tag"].as_str().unwrap())
|
||||
.collect();
|
||||
assert_eq!(tags, vec!["funny"]);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn distinct_lists_counts_per_tag(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
let (_, cookie) = common::register_user(&h.app).await;
|
||||
let manga_id = common::seed_manga_via_api(&h.app, &cookie, "M").await;
|
||||
let (_, p1) = seed_chapter_with_page(&h.app, &cookie, manga_id, 1).await;
|
||||
let (_, p2) = seed_chapter_with_page(&h.app, &cookie, manga_id, 2).await;
|
||||
|
||||
// "funny" appears twice; "fight" once. Distinct should reflect that.
|
||||
assert_eq!(add_tag(&h.app, &cookie, &p1, "funny").await, StatusCode::CREATED);
|
||||
assert_eq!(add_tag(&h.app, &cookie, &p2, "funny").await, StatusCode::CREATED);
|
||||
assert_eq!(add_tag(&h.app, &cookie, &p1, "fight").await, StatusCode::CREATED);
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::get_with_cookie("/api/v1/me/page-tags/distinct", &cookie))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = common::body_json(resp).await;
|
||||
let items = body["items"].as_array().unwrap();
|
||||
// Ordered by count DESC, tag ASC.
|
||||
assert_eq!(items[0]["tag"], "funny");
|
||||
assert_eq!(items[0]["count"], 2);
|
||||
assert_eq!(items[1]["tag"], "fight");
|
||||
assert_eq!(items[1]["count"], 1);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn tags_are_per_user_only(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
let (_, a) = common::register_user(&h.app).await;
|
||||
let (_, b) = common::register_user(&h.app).await;
|
||||
let manga_id = common::seed_manga_via_api(&h.app, &a, "M").await;
|
||||
let (_, page_id) = seed_chapter_with_page(&h.app, &a, manga_id, 1).await;
|
||||
|
||||
assert_eq!(add_tag(&h.app, &a, &page_id, "funny").await, StatusCode::CREATED);
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::get_with_cookie(
|
||||
&format!("/api/v1/pages/{page_id}/my-tags"),
|
||||
&b,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = common::body_json(resp).await;
|
||||
assert_eq!(body["tags"], json!([]));
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn page_tag_and_manga_tag_share_one_tags_row(pool: PgPool) {
|
||||
// The point of the normalization refactor: page tags now reference
|
||||
// the shared `tags` table. A manga tag "Funny" and a page tag
|
||||
// "funny" (case-folded) must resolve to a single `tags` row, and
|
||||
// `page_tags.tag_id` must point at the same row as `manga_tags`.
|
||||
let db = pool.clone();
|
||||
let h = common::harness(pool);
|
||||
let (_, cookie) = common::register_user(&h.app).await;
|
||||
let manga_id = common::seed_manga_via_api(&h.app, &cookie, "M").await;
|
||||
let (_, page_id) = seed_chapter_with_page(&h.app, &cookie, manga_id, 1).await;
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::post_json_with_cookie(
|
||||
&format!("/api/v1/mangas/{manga_id}/tags"),
|
||||
json!({ "name": "Funny" }),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::CREATED);
|
||||
|
||||
assert_eq!(add_tag(&h.app, &cookie, &page_id, "funny").await, StatusCode::CREATED);
|
||||
|
||||
let (tag_count,): (i64,) =
|
||||
sqlx::query_as("SELECT count(*) FROM tags WHERE lower(name) = 'funny'")
|
||||
.fetch_one(&db)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(tag_count, 1, "page tag and manga tag must share one tags row");
|
||||
|
||||
let (shared,): (bool,) = sqlx::query_as(
|
||||
"SELECT (SELECT tag_id FROM page_tags LIMIT 1) \
|
||||
= (SELECT tag_id FROM manga_tags LIMIT 1)",
|
||||
)
|
||||
.fetch_one(&db)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
shared,
|
||||
"page_tags.tag_id must reference the same row as manga_tags.tag_id"
|
||||
);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn adding_a_page_tag_creates_a_shared_tags_row(pool: PgPool) {
|
||||
// Even with no manga tag in play, adding a page tag must populate
|
||||
// the shared `tags` lookup table (one row per distinct name).
|
||||
let db = pool.clone();
|
||||
let h = common::harness(pool);
|
||||
let (_, cookie) = common::register_user(&h.app).await;
|
||||
let manga_id = common::seed_manga_via_api(&h.app, &cookie, "M").await;
|
||||
let (_, page_id) = seed_chapter_with_page(&h.app, &cookie, manga_id, 1).await;
|
||||
|
||||
assert_eq!(add_tag(&h.app, &cookie, &page_id, "funny").await, StatusCode::CREATED);
|
||||
|
||||
let (name,): (String,) =
|
||||
sqlx::query_as("SELECT name FROM tags WHERE lower(name) = 'funny'")
|
||||
.fetch_one(&db)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(name, "funny");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn tags_require_authentication(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::post_json(
|
||||
&format!("/api/v1/pages/{}/tags", Uuid::new_v4()),
|
||||
json!({ "tag": "funny" }),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::UNAUTHORIZED);
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------
|
||||
// Aggregation endpoints (`/me/page-tags/{chapters,mangas}`).
|
||||
// -------------------------------------------------------------------
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn chapters_aggregate_groups_and_ranks_by_match_count(pool: PgPool) {
|
||||
// Chapter A in manga Berserk gets 3 pages tagged "funny";
|
||||
// chapter B (same manga) gets 1. Desc order → A first.
|
||||
let h = common::harness(pool);
|
||||
let (_, cookie) = common::register_user(&h.app).await;
|
||||
let manga_id = common::seed_manga_via_api(&h.app, &cookie, "Berserk").await;
|
||||
|
||||
let (chapter_a, pages_a) =
|
||||
seed_chapter_with_n_pages(&h.app, &cookie, manga_id, 1, 3).await;
|
||||
let (chapter_b, pages_b) =
|
||||
seed_chapter_with_n_pages(&h.app, &cookie, manga_id, 2, 2).await;
|
||||
|
||||
for p in &pages_a {
|
||||
assert_eq!(add_tag(&h.app, &cookie, p, "funny").await, StatusCode::CREATED);
|
||||
}
|
||||
assert_eq!(add_tag(&h.app, &cookie, &pages_b[0], "funny").await, StatusCode::CREATED);
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::get_with_cookie(
|
||||
"/api/v1/me/page-tags/chapters?tag=funny",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = common::body_json(resp).await;
|
||||
let items = body["items"].as_array().unwrap();
|
||||
assert_eq!(items.len(), 2);
|
||||
assert_eq!(items[0]["chapter_id"], chapter_a);
|
||||
assert_eq!(items[0]["match_count"], 3);
|
||||
// Up to 3 sample storage keys, ordered by page_number ASC.
|
||||
let samples_a = items[0]["sample_storage_keys"].as_array().unwrap();
|
||||
assert_eq!(samples_a.len(), 3);
|
||||
assert_eq!(items[1]["chapter_id"], chapter_b);
|
||||
assert_eq!(items[1]["match_count"], 1);
|
||||
assert_eq!(body["page"]["total"], 2);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn chapters_aggregate_respects_asc_order(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
let (_, cookie) = common::register_user(&h.app).await;
|
||||
let manga_id = common::seed_manga_via_api(&h.app, &cookie, "M").await;
|
||||
let (chapter_a, pages_a) =
|
||||
seed_chapter_with_n_pages(&h.app, &cookie, manga_id, 1, 3).await;
|
||||
let (chapter_b, pages_b) =
|
||||
seed_chapter_with_n_pages(&h.app, &cookie, manga_id, 2, 1).await;
|
||||
for p in &pages_a {
|
||||
let _ = add_tag(&h.app, &cookie, p, "funny").await;
|
||||
}
|
||||
let _ = add_tag(&h.app, &cookie, &pages_b[0], "funny").await;
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::get_with_cookie(
|
||||
"/api/v1/me/page-tags/chapters?tag=funny&order=asc",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = common::body_json(resp).await;
|
||||
let items = body["items"].as_array().unwrap();
|
||||
// Lowest count first.
|
||||
assert_eq!(items[0]["chapter_id"], chapter_b);
|
||||
assert_eq!(items[0]["match_count"], 1);
|
||||
assert_eq!(items[1]["chapter_id"], chapter_a);
|
||||
assert_eq!(items[1]["match_count"], 3);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn mangas_aggregate_sums_across_chapters(pool: PgPool) {
|
||||
// Two chapters in the same manga, each contributes to the same
|
||||
// manga's match_count.
|
||||
let h = common::harness(pool);
|
||||
let (_, cookie) = common::register_user(&h.app).await;
|
||||
let manga_id = common::seed_manga_via_api(&h.app, &cookie, "Berserk").await;
|
||||
let (_, pages_a) =
|
||||
seed_chapter_with_n_pages(&h.app, &cookie, manga_id, 1, 3).await;
|
||||
let (_, pages_b) =
|
||||
seed_chapter_with_n_pages(&h.app, &cookie, manga_id, 2, 2).await;
|
||||
for p in pages_a.iter().chain(pages_b.iter()) {
|
||||
let _ = add_tag(&h.app, &cookie, p, "funny").await;
|
||||
}
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::get_with_cookie(
|
||||
"/api/v1/me/page-tags/mangas?tag=funny",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = common::body_json(resp).await;
|
||||
let items = body["items"].as_array().unwrap();
|
||||
assert_eq!(items.len(), 1);
|
||||
assert_eq!(items[0]["manga_id"], manga_id.to_string());
|
||||
assert_eq!(items[0]["match_count"], 5);
|
||||
assert_eq!(items[0]["manga_title"], "Berserk");
|
||||
let samples = items[0]["sample_storage_keys"].as_array().unwrap();
|
||||
assert_eq!(samples.len(), 3, "manga sample preview capped at 3");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn aggregate_other_users_tags_are_excluded(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
let (_, a) = common::register_user(&h.app).await;
|
||||
let (_, b) = common::register_user(&h.app).await;
|
||||
let manga_id = common::seed_manga_via_api(&h.app, &a, "M").await;
|
||||
let (_, pages) = seed_chapter_with_n_pages(&h.app, &a, manga_id, 1, 2).await;
|
||||
// A tags everything funny; B tags nothing.
|
||||
for p in &pages {
|
||||
let _ = add_tag(&h.app, &a, p, "funny").await;
|
||||
}
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::get_with_cookie(
|
||||
"/api/v1/me/page-tags/chapters?tag=funny",
|
||||
&b,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = common::body_json(resp).await;
|
||||
assert_eq!(body["items"], json!([]));
|
||||
assert_eq!(body["page"]["total"], 0);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn aggregate_unknown_tag_returns_empty_paged_response(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
let (_, cookie) = common::register_user(&h.app).await;
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::get_with_cookie(
|
||||
"/api/v1/me/page-tags/chapters?tag=neverused",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = common::body_json(resp).await;
|
||||
assert_eq!(body["items"], json!([]));
|
||||
assert_eq!(body["page"]["total"], 0);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn aggregate_rejects_missing_tag(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
let (_, cookie) = common::register_user(&h.app).await;
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::get_with_cookie(
|
||||
"/api/v1/me/page-tags/chapters?tag=",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::UNPROCESSABLE_ENTITY);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn aggregate_rejects_invalid_order(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
let (_, cookie) = common::register_user(&h.app).await;
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::get_with_cookie(
|
||||
"/api/v1/me/page-tags/chapters?tag=funny&order=sideways",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::UNPROCESSABLE_ENTITY);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn aggregate_with_text_param_is_501_with_stable_code(pool: PgPool) {
|
||||
// OCR text search isn't built yet; the param is accepted so adding
|
||||
// OCR won't break the wire shape, but rejected with a distinct
|
||||
// status + code. The code is the wire contract — clients pin on
|
||||
// `text_search_not_yet_supported`, not the message.
|
||||
let h = common::harness(pool);
|
||||
let (_, cookie) = common::register_user(&h.app).await;
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::get_with_cookie(
|
||||
"/api/v1/me/page-tags/chapters?tag=funny&text=guts",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::NOT_IMPLEMENTED);
|
||||
let body = common::body_json(resp).await;
|
||||
assert_eq!(body["error"]["code"], "text_search_not_yet_supported");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn aggregate_requires_authentication(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::get("/api/v1/me/page-tags/chapters?tag=funny"))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::UNAUTHORIZED);
|
||||
}
|
||||
@@ -14,10 +14,10 @@ use sqlx::PgPool;
|
||||
use tempfile::TempDir;
|
||||
use tower::ServiceExt;
|
||||
|
||||
use mangalord::app::{router, AppState};
|
||||
use mangalord::app::{router, AppState, RuntimeControls};
|
||||
use mangalord::auth::rate_limit::AuthRateLimiter;
|
||||
use mangalord::config::{AuthConfig, UploadConfig};
|
||||
use mangalord::storage::{LocalStorage, Storage, StorageError, StreamingFile};
|
||||
use mangalord::config::{AnalysisConfig, AuthConfig, CrawlerConfig, UploadConfig};
|
||||
use mangalord::storage::{LocalStorage, PutByteStream, Storage, StorageError, StreamingFile};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
@@ -76,8 +76,16 @@ fn harness_with_auth_config(
|
||||
auth_limiter,
|
||||
// Default harness has no crawler daemon wired up; admin resync
|
||||
// handlers return 503 in this config. Tests that need a stub
|
||||
// resync service swap it in via `harness_with_resync`.
|
||||
resync: None,
|
||||
// resync service swap it in via `harness_with_resync`. No reloader,
|
||||
// so settings still persist but spawn no daemon.
|
||||
runtime: Arc::new(RuntimeControls::new(false)),
|
||||
reloader: None,
|
||||
crawler_base: CrawlerConfig::default(),
|
||||
analysis_base: AnalysisConfig::default(),
|
||||
// Empty allowlist = CSRF check skipped. The CSRF-specific test
|
||||
// harness `harness_with_admin_origins` overrides this.
|
||||
admin_allowed_origins: Arc::new(Vec::new()),
|
||||
analysis_events: Arc::new(mangalord::analysis::events::AnalysisEvents::new()),
|
||||
};
|
||||
Harness { app: router(state), _storage_dir: storage_dir }
|
||||
}
|
||||
@@ -142,6 +150,8 @@ pub fn harness_with_resync(
|
||||
..AuthConfig::default()
|
||||
};
|
||||
let auth_limiter = Arc::new(AuthRateLimiter::new(auth.rate_limit));
|
||||
let runtime = Arc::new(RuntimeControls::new(false));
|
||||
runtime.set_resync(Some(resync));
|
||||
let state = AppState {
|
||||
db: pool,
|
||||
storage,
|
||||
@@ -151,7 +161,142 @@ pub fn harness_with_resync(
|
||||
max_file_bytes: 256 * 1024,
|
||||
},
|
||||
auth_limiter,
|
||||
resync: Some(resync),
|
||||
runtime,
|
||||
reloader: None,
|
||||
crawler_base: CrawlerConfig::default(),
|
||||
analysis_base: AnalysisConfig::default(),
|
||||
admin_allowed_origins: Arc::new(Vec::new()),
|
||||
analysis_events: Arc::new(mangalord::analysis::events::AnalysisEvents::new()),
|
||||
};
|
||||
Harness {
|
||||
app: router(state),
|
||||
_storage_dir: storage_dir,
|
||||
}
|
||||
}
|
||||
|
||||
/// Like [`harness`] but flips `analysis_enabled` on so the page-create
|
||||
/// paths enqueue `analyze_page` jobs and the admin analysis endpoints are
|
||||
/// active (rather than returning 503).
|
||||
pub fn harness_with_analysis(pool: PgPool) -> Harness {
|
||||
let storage_dir = tempfile::tempdir().expect("tempdir");
|
||||
let storage = Arc::new(LocalStorage::new(storage_dir.path()));
|
||||
let auth = AuthConfig {
|
||||
cookie_secure: false,
|
||||
..AuthConfig::default()
|
||||
};
|
||||
let auth_limiter = Arc::new(AuthRateLimiter::new(auth.rate_limit));
|
||||
let state = AppState {
|
||||
db: pool,
|
||||
storage,
|
||||
auth,
|
||||
upload: UploadConfig {
|
||||
max_request_bytes: 4 * 1024 * 1024,
|
||||
max_file_bytes: 256 * 1024,
|
||||
},
|
||||
auth_limiter,
|
||||
runtime: Arc::new(RuntimeControls::new(true)),
|
||||
reloader: None,
|
||||
crawler_base: CrawlerConfig::default(),
|
||||
analysis_base: AnalysisConfig::default(),
|
||||
admin_allowed_origins: Arc::new(Vec::new()),
|
||||
analysis_events: Arc::new(mangalord::analysis::events::AnalysisEvents::new()),
|
||||
};
|
||||
Harness {
|
||||
app: router(state),
|
||||
_storage_dir: storage_dir,
|
||||
}
|
||||
}
|
||||
|
||||
/// A [`DaemonReloader`] stub that records the configs it was asked to apply
|
||||
/// and flips the shared analysis gate, without spawning any real daemon. Lets
|
||||
/// settings tests assert that a `PUT` triggers a reload with the converted
|
||||
/// config (and that the analysis enable gate moves).
|
||||
pub struct StubReloader {
|
||||
pub runtime: Arc<RuntimeControls>,
|
||||
pub crawler: std::sync::Mutex<Option<CrawlerConfig>>,
|
||||
pub analysis: std::sync::Mutex<Option<AnalysisConfig>>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl mangalord::app::DaemonReloader for StubReloader {
|
||||
async fn reload_crawler(&self, cfg: CrawlerConfig) -> anyhow::Result<()> {
|
||||
*self.crawler.lock().unwrap() = Some(cfg);
|
||||
Ok(())
|
||||
}
|
||||
async fn reload_analysis(&self, cfg: AnalysisConfig) -> anyhow::Result<()> {
|
||||
self.runtime.set_analysis_enabled(cfg.enabled);
|
||||
*self.analysis.lock().unwrap() = Some(cfg);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Like [`harness`] but wires a [`StubReloader`] so the settings `PUT`
|
||||
/// endpoints exercise the reload path. Returns the harness plus the shared
|
||||
/// stub so the test can inspect what was applied.
|
||||
pub fn harness_with_settings_reloader(pool: PgPool) -> (Harness, Arc<StubReloader>) {
|
||||
let storage_dir = tempfile::tempdir().expect("tempdir");
|
||||
let storage = Arc::new(LocalStorage::new(storage_dir.path()));
|
||||
let auth = AuthConfig {
|
||||
cookie_secure: false,
|
||||
..AuthConfig::default()
|
||||
};
|
||||
let auth_limiter = Arc::new(AuthRateLimiter::new(auth.rate_limit));
|
||||
let runtime = Arc::new(RuntimeControls::new(false));
|
||||
let reloader = Arc::new(StubReloader {
|
||||
runtime: Arc::clone(&runtime),
|
||||
crawler: std::sync::Mutex::new(None),
|
||||
analysis: std::sync::Mutex::new(None),
|
||||
});
|
||||
let state = AppState {
|
||||
db: pool,
|
||||
storage,
|
||||
auth,
|
||||
upload: UploadConfig {
|
||||
max_request_bytes: 4 * 1024 * 1024,
|
||||
max_file_bytes: 256 * 1024,
|
||||
},
|
||||
auth_limiter,
|
||||
runtime,
|
||||
reloader: Some(Arc::clone(&reloader) as Arc<dyn mangalord::app::DaemonReloader>),
|
||||
crawler_base: CrawlerConfig::default(),
|
||||
analysis_base: AnalysisConfig::default(),
|
||||
admin_allowed_origins: Arc::new(Vec::new()),
|
||||
analysis_events: Arc::new(mangalord::analysis::events::AnalysisEvents::new()),
|
||||
};
|
||||
(
|
||||
Harness {
|
||||
app: router(state),
|
||||
_storage_dir: storage_dir,
|
||||
},
|
||||
reloader,
|
||||
)
|
||||
}
|
||||
|
||||
/// Like [`harness`] but configures an admin CSRF allowlist so the
|
||||
/// `/admin/*` mutating endpoints reject cross-origin browser POSTs.
|
||||
/// Used by the admin CSRF integration tests.
|
||||
pub fn harness_with_admin_origins(pool: PgPool, origins: Vec<String>) -> Harness {
|
||||
let storage_dir = tempfile::tempdir().expect("tempdir");
|
||||
let storage = Arc::new(LocalStorage::new(storage_dir.path()));
|
||||
let auth_limiter = Arc::new(AuthRateLimiter::new(Default::default()));
|
||||
let state = AppState {
|
||||
db: pool,
|
||||
storage,
|
||||
auth: AuthConfig {
|
||||
cookie_secure: false,
|
||||
..AuthConfig::default()
|
||||
},
|
||||
upload: UploadConfig {
|
||||
max_request_bytes: 4 * 1024 * 1024,
|
||||
max_file_bytes: 256 * 1024,
|
||||
},
|
||||
auth_limiter,
|
||||
runtime: Arc::new(RuntimeControls::new(false)),
|
||||
reloader: None,
|
||||
crawler_base: CrawlerConfig::default(),
|
||||
analysis_base: AnalysisConfig::default(),
|
||||
admin_allowed_origins: Arc::new(origins),
|
||||
analysis_events: Arc::new(mangalord::analysis::events::AnalysisEvents::new()),
|
||||
};
|
||||
Harness {
|
||||
app: router(state),
|
||||
@@ -189,6 +334,22 @@ impl Storage for FailingStorage {
|
||||
}
|
||||
self.inner.put(key, bytes).await
|
||||
}
|
||||
async fn put_stream(
|
||||
&self,
|
||||
key: &str,
|
||||
stream: PutByteStream<'_>,
|
||||
) -> Result<u64, StorageError> {
|
||||
// Count put_stream towards the same fail-index so tests that
|
||||
// expect "the Nth put fails" don't care which entry point
|
||||
// the caller took.
|
||||
let n = self.counter.fetch_add(1, Ordering::SeqCst);
|
||||
if n == self.fail_on_put_index {
|
||||
return Err(StorageError::Io(std::io::Error::other(
|
||||
"FailingStorage: injected put_stream failure",
|
||||
)));
|
||||
}
|
||||
self.inner.put_stream(key, stream).await
|
||||
}
|
||||
async fn get(&self, key: &str) -> Result<Vec<u8>, StorageError> {
|
||||
self.inner.get(key).await
|
||||
}
|
||||
@@ -251,6 +412,44 @@ pub fn post_json_with_cookie(
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
/// Same as [`post_json_with_cookie`] but also attaches `Origin` (and
|
||||
/// optionally `Referer`) headers. Used by the admin CSRF tests to drive
|
||||
/// the cross-origin reject + allowed-origin accept paths.
|
||||
pub fn post_json_with_cookie_origin(
|
||||
uri: &str,
|
||||
body: serde_json::Value,
|
||||
cookie: &str,
|
||||
origin: Option<&str>,
|
||||
referer: Option<&str>,
|
||||
) -> Request<Body> {
|
||||
let mut b = Request::builder()
|
||||
.method("POST")
|
||||
.uri(uri)
|
||||
.header(header::CONTENT_TYPE, "application/json")
|
||||
.header(header::COOKIE, cookie);
|
||||
if let Some(o) = origin {
|
||||
b = b.header(header::ORIGIN, o);
|
||||
}
|
||||
if let Some(r) = referer {
|
||||
b = b.header(header::REFERER, r);
|
||||
}
|
||||
b.body(Body::from(body.to_string())).unwrap()
|
||||
}
|
||||
|
||||
pub fn get_with_cookie_origin(
|
||||
uri: &str,
|
||||
cookie: &str,
|
||||
origin: Option<&str>,
|
||||
) -> Request<Body> {
|
||||
let mut b = Request::builder()
|
||||
.uri(uri)
|
||||
.header(header::COOKIE, cookie);
|
||||
if let Some(o) = origin {
|
||||
b = b.header(header::ORIGIN, o);
|
||||
}
|
||||
b.body(Body::empty()).unwrap()
|
||||
}
|
||||
|
||||
pub fn post_json_with_bearer(
|
||||
uri: &str,
|
||||
body: serde_json::Value,
|
||||
|
||||
@@ -40,6 +40,8 @@ fn make_cfg(
|
||||
tz: Tz::UTC,
|
||||
retention_days: 7,
|
||||
session_expired,
|
||||
status: mangalord::crawler::status::StatusHandle::new(workers),
|
||||
job_timeout: Duration::from_secs(60),
|
||||
extra_tasks: Vec::new(),
|
||||
}
|
||||
}
|
||||
@@ -88,6 +90,52 @@ impl ChapterDispatcher for PanickingDispatcher {
|
||||
}
|
||||
}
|
||||
|
||||
/// Never completes — used to verify the worker's outer dispatch timeout.
|
||||
struct HangingDispatcher {
|
||||
seen: AtomicUsize,
|
||||
}
|
||||
#[async_trait::async_trait]
|
||||
impl ChapterDispatcher for HangingDispatcher {
|
||||
async fn dispatch(&self, _payload: JobPayload) -> anyhow::Result<SyncOutcome> {
|
||||
self.seen.fetch_add(1, Ordering::AcqRel);
|
||||
std::future::pending::<()>().await;
|
||||
unreachable!("hanging dispatcher never resolves");
|
||||
}
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn worker_times_out_a_hung_dispatch_and_acks_failed(pool: PgPool) {
|
||||
enqueue_chapter_job(&pool).await;
|
||||
let dispatcher = Arc::new(HangingDispatcher {
|
||||
seen: AtomicUsize::new(0),
|
||||
});
|
||||
let session_expired = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
let cancel = CancellationToken::new();
|
||||
let mut cfg = make_cfg(None, dispatcher.clone(), session_expired, 1);
|
||||
cfg.job_timeout = Duration::from_millis(300);
|
||||
let handle = daemon::spawn(pool.clone(), cancel.clone(), cfg);
|
||||
|
||||
// The hung job should time out and return to pending with backoff
|
||||
// (attempts=1 < max=5). Poll for the recorded error.
|
||||
let mut timed_out = false;
|
||||
for _ in 0..40 {
|
||||
let n: i64 = sqlx::query_scalar(
|
||||
"SELECT COUNT(*) FROM crawler_jobs WHERE last_error = 'dispatch timed out'",
|
||||
)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
if n == 1 {
|
||||
timed_out = true;
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
handle.shutdown().await;
|
||||
assert!(timed_out, "hung dispatch must be acked failed with a timeout error");
|
||||
assert!(dispatcher.seen.load(Ordering::Acquire) >= 1);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn workers_drain_jobs_through_dispatcher(pool: PgPool) {
|
||||
enqueue_chapter_job(&pool).await;
|
||||
|
||||
304
backend/tests/crawler_dead_jobs.rs
Normal file
304
backend/tests/crawler_dead_jobs.rs
Normal file
@@ -0,0 +1,304 @@
|
||||
//! Integration tests for the dead-letter admin queries in
|
||||
//! `repo::crawler`: listing dead jobs with manga/chapter context and the
|
||||
//! scoped requeue (all / per-manga / single) used by the admin dashboard.
|
||||
|
||||
use mangalord::repo::crawler::{self, RequeueScope};
|
||||
use serde_json::json;
|
||||
use sqlx::PgPool;
|
||||
use uuid::Uuid;
|
||||
|
||||
/// Seed a manga with no cover + a live source row (so it's "queued for a
|
||||
/// cover fetch"). Returns the manga id.
|
||||
async fn seed_missing_cover(pool: &PgPool, title: &str) -> Uuid {
|
||||
let manga_id = Uuid::new_v4();
|
||||
sqlx::query("INSERT INTO mangas (id, title, cover_image_path) VALUES ($1, $2, NULL)")
|
||||
.bind(manga_id)
|
||||
.bind(title)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("INSERT INTO sources (id, name, base_url) VALUES ('target', 'T', 'http://x') ON CONFLICT DO NOTHING")
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query(
|
||||
"INSERT INTO manga_sources (source_id, source_manga_key, manga_id, source_url) \
|
||||
VALUES ('target', $1, $2, 'http://x/m')",
|
||||
)
|
||||
.bind(format!("k-{manga_id}"))
|
||||
.bind(manga_id)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
manga_id
|
||||
}
|
||||
|
||||
/// Seed a manga + chapter and return their ids.
|
||||
async fn seed_chapter(pool: &PgPool, title: &str, number: i32) -> (Uuid, Uuid) {
|
||||
let manga_id = Uuid::new_v4();
|
||||
let chapter_id = Uuid::new_v4();
|
||||
sqlx::query("INSERT INTO mangas (id, title) VALUES ($1, $2)")
|
||||
.bind(manga_id)
|
||||
.bind(title)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("INSERT INTO chapters (id, manga_id, number) VALUES ($1, $2, $3)")
|
||||
.bind(chapter_id)
|
||||
.bind(manga_id)
|
||||
.bind(number)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
(manga_id, chapter_id)
|
||||
}
|
||||
|
||||
/// Insert a crawler_jobs row in a given state for a chapter-content job.
|
||||
async fn insert_job(pool: &PgPool, chapter_id: Uuid, state: &str, attempts: i32) -> Uuid {
|
||||
let id = Uuid::new_v4();
|
||||
let payload = json!({
|
||||
"kind": "sync_chapter_content",
|
||||
"source_id": "target",
|
||||
"chapter_id": chapter_id,
|
||||
"source_chapter_key": "k",
|
||||
});
|
||||
sqlx::query(
|
||||
"INSERT INTO crawler_jobs (id, payload, state, attempts, last_error) \
|
||||
VALUES ($1, $2, $3, $4, 'boom')",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(payload)
|
||||
.bind(state)
|
||||
.bind(attempts)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
id
|
||||
}
|
||||
|
||||
async fn state_of(pool: &PgPool, id: Uuid) -> String {
|
||||
sqlx::query_scalar::<_, String>("SELECT state FROM crawler_jobs WHERE id = $1")
|
||||
.bind(id)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn list_dead_jobs_returns_context_and_total(pool: PgPool) {
|
||||
let (_m, c1) = seed_chapter(&pool, "Naruto", 700).await;
|
||||
insert_job(&pool, c1, "dead", 5).await;
|
||||
// A non-dead job must not appear.
|
||||
let (_m2, c2) = seed_chapter(&pool, "Bleach", 1).await;
|
||||
insert_job(&pool, c2, "pending", 0).await;
|
||||
|
||||
let (items, total) = crawler::list_dead_jobs(&pool, None, 50, 0).await.unwrap();
|
||||
assert_eq!(total, 1);
|
||||
assert_eq!(items.len(), 1);
|
||||
let row = &items[0];
|
||||
assert_eq!(row.manga_title.as_deref(), Some("Naruto"));
|
||||
assert_eq!(row.chapter_number, Some(700));
|
||||
assert_eq!(row.attempts, 5);
|
||||
assert_eq!(row.last_error.as_deref(), Some("boom"));
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn list_dead_jobs_filters_by_title_search(pool: PgPool) {
|
||||
let (_m, c1) = seed_chapter(&pool, "Naruto", 700).await;
|
||||
insert_job(&pool, c1, "dead", 5).await;
|
||||
let (_m2, c2) = seed_chapter(&pool, "One Piece", 1).await;
|
||||
insert_job(&pool, c2, "dead", 5).await;
|
||||
|
||||
let (items, total) = crawler::list_dead_jobs(&pool, Some("piece"), 50, 0)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(total, 1);
|
||||
assert_eq!(items[0].manga_title.as_deref(), Some("One Piece"));
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn requeue_all_resets_dead_jobs_to_pending(pool: PgPool) {
|
||||
let (_m, c1) = seed_chapter(&pool, "A", 1).await;
|
||||
let (_m2, c2) = seed_chapter(&pool, "B", 1).await;
|
||||
let j1 = insert_job(&pool, c1, "dead", 5).await;
|
||||
let j2 = insert_job(&pool, c2, "dead", 5).await;
|
||||
|
||||
let n = crawler::requeue_dead_jobs(&pool, RequeueScope::All)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(n, 2);
|
||||
assert_eq!(state_of(&pool, j1).await, "pending");
|
||||
assert_eq!(state_of(&pool, j2).await, "pending");
|
||||
let attempts: i32 = sqlx::query_scalar("SELECT attempts FROM crawler_jobs WHERE id = $1")
|
||||
.bind(j1)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(attempts, 0, "attempts reset on requeue");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn requeue_by_manga_scopes_to_that_manga(pool: PgPool) {
|
||||
let (m1, c1) = seed_chapter(&pool, "A", 1).await;
|
||||
let (_m2, c2) = seed_chapter(&pool, "B", 1).await;
|
||||
let j1 = insert_job(&pool, c1, "dead", 5).await;
|
||||
let j2 = insert_job(&pool, c2, "dead", 5).await;
|
||||
|
||||
let n = crawler::requeue_dead_jobs(&pool, RequeueScope::Manga(m1))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(n, 1);
|
||||
assert_eq!(state_of(&pool, j1).await, "pending");
|
||||
assert_eq!(state_of(&pool, j2).await, "dead", "other manga untouched");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn requeue_by_chapter_scopes_to_that_chapter(pool: PgPool) {
|
||||
let (_m, c1) = seed_chapter(&pool, "A", 1).await;
|
||||
let (_m2, c2) = seed_chapter(&pool, "A", 2).await;
|
||||
let j1 = insert_job(&pool, c1, "dead", 5).await;
|
||||
let j2 = insert_job(&pool, c2, "dead", 5).await;
|
||||
|
||||
let n = crawler::requeue_dead_jobs(&pool, RequeueScope::Chapter(c1))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(n, 1);
|
||||
assert_eq!(state_of(&pool, j1).await, "pending");
|
||||
assert_eq!(state_of(&pool, j2).await, "dead", "other chapter untouched");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn requeue_single_job(pool: PgPool) {
|
||||
let (_m, c1) = seed_chapter(&pool, "A", 1).await;
|
||||
let (_m2, c2) = seed_chapter(&pool, "B", 1).await;
|
||||
let j1 = insert_job(&pool, c1, "dead", 5).await;
|
||||
let j2 = insert_job(&pool, c2, "dead", 5).await;
|
||||
|
||||
let n = crawler::requeue_dead_jobs(&pool, RequeueScope::Job(j1))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(n, 1);
|
||||
assert_eq!(state_of(&pool, j1).await, "pending");
|
||||
assert_eq!(state_of(&pool, j2).await, "dead");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn requeue_skips_dead_when_live_job_exists_for_same_chapter(pool: PgPool) {
|
||||
let (_m, c1) = seed_chapter(&pool, "A", 1).await;
|
||||
let dead = insert_job(&pool, c1, "dead", 5).await;
|
||||
// A live pending job for the SAME chapter already exists.
|
||||
insert_job(&pool, c1, "pending", 0).await;
|
||||
|
||||
let n = crawler::requeue_dead_jobs(&pool, RequeueScope::All)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(n, 0, "must not resurrect a dead job that has a live counterpart");
|
||||
assert_eq!(state_of(&pool, dead).await, "dead");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn requeue_with_two_dead_jobs_for_one_chapter_revives_one_not_500(pool: PgPool) {
|
||||
// Regression: two dead jobs for the SAME chapter must not both flip to
|
||||
// pending in one statement — that would violate the partial unique
|
||||
// dedup index and abort the whole requeue.
|
||||
let (manga_id, c1) = seed_chapter(&pool, "A", 1).await;
|
||||
let older = insert_job(&pool, c1, "dead", 5).await;
|
||||
let newer = insert_job(&pool, c1, "dead", 5).await;
|
||||
// Make `newer` unambiguously newer.
|
||||
sqlx::query("UPDATE crawler_jobs SET updated_at = now() - interval '1 hour' WHERE id = $1")
|
||||
.bind(older)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
for scope in [RequeueScope::All, RequeueScope::Manga(manga_id), RequeueScope::Chapter(c1)] {
|
||||
// Reset to two-dead before each scope variant.
|
||||
sqlx::query("UPDATE crawler_jobs SET state = 'dead' WHERE id = ANY($1)")
|
||||
.bind(vec![older, newer])
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let n = crawler::requeue_dead_jobs(&pool, scope)
|
||||
.await
|
||||
.expect("requeue must not error on duplicate dead jobs");
|
||||
assert_eq!(n, 1, "exactly one dead job per chapter is revived");
|
||||
// The newest one is the survivor; the other stays dead.
|
||||
assert_eq!(state_of(&pool, newer).await, "pending");
|
||||
assert_eq!(state_of(&pool, older).await, "dead");
|
||||
}
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn list_active_jobs_returns_pending_and_running_running_first(pool: PgPool) {
|
||||
let (_m, c1) = seed_chapter(&pool, "Naruto", 700).await;
|
||||
let (_m2, c2) = seed_chapter(&pool, "Bleach", 10).await;
|
||||
insert_job(&pool, c1, "pending", 0).await;
|
||||
insert_job(&pool, c2, "running", 1).await;
|
||||
// A dead + a done job must NOT appear.
|
||||
let (_m3, c3) = seed_chapter(&pool, "Gone", 1).await;
|
||||
insert_job(&pool, c3, "dead", 5).await;
|
||||
|
||||
let (items, total) = crawler::list_active_jobs(&pool, None, 50, 0).await.unwrap();
|
||||
assert_eq!(total, 2);
|
||||
assert_eq!(items.len(), 2);
|
||||
// Running first.
|
||||
assert_eq!(items[0].state, "running");
|
||||
assert_eq!(items[0].manga_title.as_deref(), Some("Bleach"));
|
||||
assert_eq!(items[1].state, "pending");
|
||||
assert_eq!(items[1].chapter_number, Some(700));
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn list_active_jobs_filters_by_title(pool: PgPool) {
|
||||
let (_m, c1) = seed_chapter(&pool, "Naruto", 1).await;
|
||||
let (_m2, c2) = seed_chapter(&pool, "One Piece", 1).await;
|
||||
insert_job(&pool, c1, "pending", 0).await;
|
||||
insert_job(&pool, c2, "pending", 0).await;
|
||||
let (items, total) = crawler::list_active_jobs(&pool, Some("piece"), 50, 0)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(total, 1);
|
||||
assert_eq!(items[0].manga_title.as_deref(), Some("One Piece"));
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn missing_covers_count_and_list(pool: PgPool) {
|
||||
seed_missing_cover(&pool, "Naruto").await;
|
||||
seed_missing_cover(&pool, "Bleach").await;
|
||||
// A manga WITH a cover must not be counted.
|
||||
let with_cover = Uuid::new_v4();
|
||||
sqlx::query("INSERT INTO mangas (id, title, cover_image_path) VALUES ($1, 'Done', 'k.jpg')")
|
||||
.bind(with_cover)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(crawler::count_missing_covers(&pool).await.unwrap(), 2);
|
||||
|
||||
let (items, total) = crawler::list_missing_cover_mangas(&pool, None, 50, 0)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(total, 2);
|
||||
assert_eq!(items.len(), 2);
|
||||
|
||||
let (items, total) = crawler::list_missing_cover_mangas(&pool, Some("naru"), 50, 0)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(total, 1);
|
||||
assert_eq!(items[0].manga_title, "Naruto");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn job_state_counts_groups_by_state(pool: PgPool) {
|
||||
let (_m, c1) = seed_chapter(&pool, "A", 1).await;
|
||||
let (_m2, c2) = seed_chapter(&pool, "B", 1).await;
|
||||
let (_m3, c3) = seed_chapter(&pool, "C", 1).await;
|
||||
insert_job(&pool, c1, "pending", 0).await;
|
||||
insert_job(&pool, c2, "dead", 5).await;
|
||||
insert_job(&pool, c3, "dead", 5).await;
|
||||
|
||||
let (pending, running, dead) = crawler::job_state_counts(&pool).await.unwrap();
|
||||
assert_eq!(pending, 1);
|
||||
assert_eq!(running, 0);
|
||||
assert_eq!(dead, 2);
|
||||
}
|
||||
@@ -185,6 +185,68 @@ async fn lease_marks_running_and_bumps_attempts_and_sets_leased_until(pool: PgPo
|
||||
assert!(leased_until > chrono::Utc::now());
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn renew_extends_leased_until_while_running(pool: PgPool) {
|
||||
let id = match jobs::enqueue(&pool, &chapter_content_payload(Uuid::new_v4()))
|
||||
.await
|
||||
.unwrap()
|
||||
{
|
||||
EnqueueResult::Inserted(id) => id,
|
||||
EnqueueResult::Skipped => unreachable!(),
|
||||
};
|
||||
|
||||
// Lease with a short window, then collapse leased_until to the recent
|
||||
// past so the renew is unambiguously an extension.
|
||||
let leases = jobs::lease(&pool, None, 1, Duration::from_secs(5))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(leases.len(), 1);
|
||||
sqlx::query("UPDATE crawler_jobs SET leased_until = now() - interval '1 second' WHERE id = $1")
|
||||
.bind(id)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let still_owned = jobs::renew(&pool, id, Duration::from_secs(120))
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(still_owned, "renew on a running job returns true");
|
||||
|
||||
let leased_until: chrono::DateTime<chrono::Utc> =
|
||||
sqlx::query_scalar("SELECT leased_until FROM crawler_jobs WHERE id = $1")
|
||||
.bind(id)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
leased_until > chrono::Utc::now() + chrono::Duration::seconds(60),
|
||||
"leased_until pushed ~120s into the future"
|
||||
);
|
||||
assert_eq!(job_state(&pool, id).await, "running");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn renew_is_noop_once_job_no_longer_running(pool: PgPool) {
|
||||
let id = match jobs::enqueue(&pool, &chapter_content_payload(Uuid::new_v4()))
|
||||
.await
|
||||
.unwrap()
|
||||
{
|
||||
EnqueueResult::Inserted(id) => id,
|
||||
EnqueueResult::Skipped => unreachable!(),
|
||||
};
|
||||
let leases = jobs::lease(&pool, None, 1, Duration::from_secs(60))
|
||||
.await
|
||||
.unwrap();
|
||||
// Job completes — heartbeat should now see it's no longer ours.
|
||||
jobs::ack_done(&pool, leases[0].id).await.unwrap();
|
||||
|
||||
let still_owned = jobs::renew(&pool, id, Duration::from_secs(120))
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(!still_owned, "renew on a non-running job returns false");
|
||||
assert_eq!(job_state(&pool, id).await, "done");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn lease_with_kind_filter_only_matches_that_kind(pool: PgPool) {
|
||||
let manga_id = match jobs::enqueue(&pool, &sync_manga_payload("foo"))
|
||||
|
||||
@@ -109,7 +109,7 @@ async fn dispatch_target_prefers_most_recent_live_source(pool: PgPool) {
|
||||
seed_chapter_with_two_live_sources(&pool).await;
|
||||
|
||||
let row = dispatch_target(&pool, chapter_id).await.unwrap();
|
||||
let (_manga_id, source_url) =
|
||||
let (_manga_id, source_url, _title, _number) =
|
||||
row.expect("two live sources should yield a dispatch target");
|
||||
assert_eq!(
|
||||
source_url, new_url,
|
||||
@@ -133,7 +133,7 @@ async fn dispatch_target_skips_dropped_sources(pool: PgPool) {
|
||||
.unwrap();
|
||||
|
||||
let row = dispatch_target(&pool, chapter_id).await.unwrap();
|
||||
let (_manga_id, source_url) =
|
||||
let (_manga_id, source_url, _title, _number) =
|
||||
row.expect("a single live source should still yield a dispatch target");
|
||||
assert!(
|
||||
source_url != new_url,
|
||||
|
||||
236
frontend/audit.mjs
Normal file
236
frontend/audit.mjs
Normal file
@@ -0,0 +1,236 @@
|
||||
import { chromium } from '@playwright/test';
|
||||
|
||||
const MID = 'a1111111-1111-1111-1111-111111111111';
|
||||
const CID = 'c1111111-1111-1111-1111-111111111111';
|
||||
|
||||
const browser = await chromium.launch();
|
||||
const ctx = await browser.newContext({ viewport: { width: 390, height: 844 }, deviceScaleFactor: 1 });
|
||||
const page = await ctx.newPage();
|
||||
|
||||
page.on('pageerror', e => console.log(' PAGE ERROR:', e.message));
|
||||
|
||||
// Generic stubs so every page renders deterministically without a backend
|
||||
async function mock() {
|
||||
await page.unrouteAll().catch(() => {});
|
||||
await page.route('**/api/v1/auth/config', r => r.fulfill({ status: 200, contentType: 'application/json', body: '{"self_register_enabled":true,"private_mode":false}' }));
|
||||
await page.route('**/api/v1/auth/me', r => r.fulfill({ status: 401, contentType: 'application/json', body: '{"error":{"code":"x","message":"x"}}' }));
|
||||
await page.route('**/api/v1/auth/me/preferences', r => r.fulfill({ status: 401, contentType: 'application/json', body: '{}' }));
|
||||
await page.route('**/api/v1/me/bookmarks*', r => r.fulfill({ status: 401, contentType: 'application/json', body: '{}' }));
|
||||
await page.route('**/api/v1/me/collections*', r => r.fulfill({ status: 401, contentType: 'application/json', body: '{}' }));
|
||||
await page.route('**/api/v1/me/read-progress*', r => r.fulfill({ status: 401, contentType: 'application/json', body: '{}' }));
|
||||
await page.route('**/api/v1/mangas/**', r => {
|
||||
const url = new URL(r.request().url());
|
||||
const segs = url.pathname.split('/');
|
||||
const last = segs[segs.length - 1];
|
||||
if (last === MID) {
|
||||
return r.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify({
|
||||
id: MID,
|
||||
title: 'A Very Long Manga Title That Should Ellipsize On Narrow Screens',
|
||||
status: 'ongoing',
|
||||
alt_titles: ['Alt One', 'Alt Two'],
|
||||
// Realistic worst case: a normal-length lorem-ipsum prefix
|
||||
// followed by a long unbreakable token (e.g. a crawl-source
|
||||
// URL or a romanized title with no separators). The token
|
||||
// is what pushed the description past the screen edge on
|
||||
// user devices.
|
||||
description: 'Lorem ipsum dolor sit amet '.repeat(8) + ' AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA',
|
||||
cover_image_path: `mangas/${MID}/cover.png`,
|
||||
created_at: '2026-01-01T00:00:00Z', updated_at: '2026-01-01T00:00:00Z',
|
||||
authors: [{ id: 'au1', name: 'Kentaro-Miura-with-an-extremely-long-author-name' }],
|
||||
genres: [{ id: 'g1', name: 'Action' }, { id: 'g2', name: 'Dark Fantasy' }, { id: 'g3', name: 'Adventure' }],
|
||||
// Mix of normal-length and one extreme tag with no soft-break
|
||||
// opportunities — exercises chip wrap behavior under stress.
|
||||
tags: [
|
||||
{ id: 't1', name: 'psychological', added_by: null },
|
||||
{ id: 't2', name: 'school', added_by: null },
|
||||
{ id: 't3', name: 'SuperLongUnbrokenTagNameThatDoesNotFitInAnyReasonableChipWidth', added_by: null }
|
||||
]
|
||||
}) });
|
||||
}
|
||||
if (last === CID) {
|
||||
return r.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify({ id: CID, manga_id: MID, number: 1, title: 'The Brand', page_count: 3, created_at: '2026-01-01T00:00:00Z' }) });
|
||||
}
|
||||
if (last === 'pages') {
|
||||
return r.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify({ pages: [
|
||||
{ id: 'p1', chapter_id: CID, page_number: 1, storage_key: 'x.png', content_type: 'image/png' },
|
||||
{ id: 'p2', chapter_id: CID, page_number: 2, storage_key: 'y.png', content_type: 'image/png' },
|
||||
{ id: 'p3', chapter_id: CID, page_number: 3, storage_key: 'z.png', content_type: 'image/png' }
|
||||
] }) });
|
||||
}
|
||||
// Mangas list or chapters list
|
||||
if (url.pathname.endsWith('/chapters')) {
|
||||
return r.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify({
|
||||
items: [{ id: CID, manga_id: MID, number: 1, title: 'The Brand', page_count: 3, created_at: '2026-01-01T00:00:00Z' }],
|
||||
page: { limit: 50, offset: 0, total: 1 }
|
||||
}) });
|
||||
}
|
||||
const items = Array.from({ length: 8 }, (_, i) => ({
|
||||
id: `m${i + 1}`, title: `Manga ${i + 1} with a long title that ellipsizes`,
|
||||
status: 'ongoing', alt_titles: [], description: null, cover_image_path: null,
|
||||
created_at: '2026-01-01T00:00:00Z', updated_at: '2026-01-01T00:00:00Z',
|
||||
authors: [{ id: 'a1', name: 'Author' }], genres: [{ id: 'g1', name: 'Action' }], tags: []
|
||||
}));
|
||||
return r.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify({ items, page: { limit: 50, offset: 0, total: 8 } }) });
|
||||
});
|
||||
await page.route('**/api/v1/genres*', r => r.fulfill({ status: 200, contentType: 'application/json', body: '[]' }));
|
||||
await page.route('**/api/v1/files/**', r => r.fulfill({ status: 200, contentType: 'image/png', body: Buffer.from('89504e470d0a1a0a0000000d49484452000000010000000108060000001f15c4890000000d49444154789c63000100000005000158a3b62a0000000049454e44ae426082', 'hex') }));
|
||||
}
|
||||
|
||||
const routes = [
|
||||
['/', 'catalog'],
|
||||
['/login', 'login'],
|
||||
['/register', 'register'],
|
||||
['/upload', 'upload'],
|
||||
[`/manga/${MID}`, 'detail'],
|
||||
[`/manga/${MID}/chapter/${CID}`, 'reader'],
|
||||
['/library', 'library'],
|
||||
['/profile', 'profile-overview'],
|
||||
['/profile/account','account'],
|
||||
['/profile/preferences','preferences'],
|
||||
['/bookmarks', 'bookmarks-top'],
|
||||
['/collections', 'collections-top']
|
||||
];
|
||||
|
||||
// Identify elements that "kiss" the viewport edges — their right
|
||||
// edge is essentially flush with the screen edge. For interactive
|
||||
// controls (buttons, anchors, inputs, .chip*) this is almost always
|
||||
// a design bug rather than intentional full-bleed chrome.
|
||||
function bleeders(thresholdPx) {
|
||||
const vw = window.innerWidth;
|
||||
const bad = [];
|
||||
const candidates = document.querySelectorAll('button, a, input, select, textarea, [class*="chip"], [class*="card"], h1, h2, h3, p');
|
||||
for (const el of candidates) {
|
||||
const r = el.getBoundingClientRect();
|
||||
if (r.width === 0) continue;
|
||||
// Skip elements whose ancestor is itself position:fixed at the
|
||||
// edges — that's intentional chrome (bottom nav, CTA bar).
|
||||
let p = el;
|
||||
let inFixed = false;
|
||||
while (p) {
|
||||
const cs = getComputedStyle(p);
|
||||
if (cs.position === 'fixed') { inFixed = true; break; }
|
||||
p = p.parentElement;
|
||||
}
|
||||
if (inFixed) continue;
|
||||
// Touching right edge?
|
||||
if (vw - r.right < thresholdPx) {
|
||||
const tag = el.tagName.toLowerCase();
|
||||
const cls = (typeof el.className === 'string' ? el.className : '').split(' ').filter(c => !!c && !c.startsWith('s-')).slice(0, 2).join('.');
|
||||
const id = el.dataset.testid ?? cls;
|
||||
bad.push(`right-bleed ${tag}.${id} right=${Math.round(r.right)}`);
|
||||
}
|
||||
// Touching left edge?
|
||||
if (r.left < thresholdPx) {
|
||||
const tag = el.tagName.toLowerCase();
|
||||
const cls = (typeof el.className === 'string' ? el.className : '').split(' ').filter(c => !!c && !c.startsWith('s-')).slice(0, 2).join('.');
|
||||
const id = el.dataset.testid ?? cls;
|
||||
bad.push(`left-bleed ${tag}.${id} left=${Math.round(r.left)}`);
|
||||
}
|
||||
}
|
||||
// de-dup
|
||||
return [...new Set(bad)];
|
||||
}
|
||||
|
||||
for (const [path, name] of routes) {
|
||||
await mock();
|
||||
try {
|
||||
await page.goto('http://localhost:5173' + path, { waitUntil: 'networkidle', timeout: 8000 });
|
||||
} catch (e) {
|
||||
console.log(`${name.padEnd(20)} NAV ERROR: ${e.message.slice(0, 60)}`);
|
||||
continue;
|
||||
}
|
||||
await page.waitForTimeout(400);
|
||||
const m = await page.evaluate(() => {
|
||||
const all = Array.from(document.querySelectorAll('*'));
|
||||
const vw = window.innerWidth;
|
||||
const offenders = [];
|
||||
for (const el of all) {
|
||||
const r = el.getBoundingClientRect();
|
||||
if (r.right > vw + 0.5 || r.width > vw + 0.5) {
|
||||
const tag = el.tagName.toLowerCase();
|
||||
const className = typeof el.className === 'string'
|
||||
? el.className.split(' ').filter(c => !!c).slice(0, 2).join('.')
|
||||
: '';
|
||||
const id = el.dataset.testid ?? className;
|
||||
offenders.push(`${tag}.${id} right=${Math.round(r.right)} w=${Math.round(r.width)}`);
|
||||
}
|
||||
}
|
||||
// Detect horizontal overflow from scroll measurement, but skip
|
||||
// elements that intentionally clip via overflow:hidden — that's
|
||||
// ellipsization, not a layout bug.
|
||||
const scrollers = [];
|
||||
for (const el of all) {
|
||||
if (el.scrollWidth > el.clientWidth + 0.5) {
|
||||
const cs = getComputedStyle(el);
|
||||
if (cs.overflowX === 'hidden' || cs.overflowX === 'scroll' || cs.overflowX === 'auto') continue;
|
||||
const tag = el.tagName.toLowerCase();
|
||||
const className = typeof el.className === 'string'
|
||||
? el.className.split(' ').filter(c => !!c).slice(0, 2).join('.')
|
||||
: '';
|
||||
const id = el.dataset.testid ?? className;
|
||||
// Walk up to find the offending child too
|
||||
let child = '';
|
||||
for (const c of el.children) {
|
||||
const r = c.getBoundingClientRect();
|
||||
const er = el.getBoundingClientRect();
|
||||
if (r.right > er.right + 0.5) {
|
||||
child = ` child:${c.tagName.toLowerCase()}.${typeof c.className === 'string' ? c.className.split(' ').filter(x => !!x).slice(0, 2).join('.') : ''}(right=${Math.round(r.right)})`;
|
||||
break;
|
||||
}
|
||||
}
|
||||
scrollers.push(`${tag}.${id} scrollW=${el.scrollWidth} clientW=${el.clientWidth}${child}`);
|
||||
}
|
||||
}
|
||||
return {
|
||||
innerWidth: vw,
|
||||
docW: document.documentElement.scrollWidth,
|
||||
bodyW: document.body.scrollWidth,
|
||||
offenders: offenders.slice(0, 8),
|
||||
scrollers: scrollers.slice(0, 8)
|
||||
};
|
||||
});
|
||||
// Also flag elements that touch the viewport edges (no inset gutter)
|
||||
const bleed = await page.evaluate((b) => {
|
||||
// re-execute the bleeders function here because page context is fresh
|
||||
const vw = window.innerWidth;
|
||||
const threshold = 4;
|
||||
const bad = [];
|
||||
const candidates = document.querySelectorAll('button, a, input, select, textarea, [class*="chip"], [class*="card"], h1, h2, h3, p');
|
||||
for (const el of candidates) {
|
||||
const r = el.getBoundingClientRect();
|
||||
if (r.width === 0) continue;
|
||||
let p = el;
|
||||
let inFixed = false;
|
||||
while (p) {
|
||||
const cs = getComputedStyle(p);
|
||||
if (cs.position === 'fixed') { inFixed = true; break; }
|
||||
p = p.parentElement;
|
||||
}
|
||||
if (inFixed) continue;
|
||||
const tag = el.tagName.toLowerCase();
|
||||
const cls = (typeof el.className === 'string' ? el.className : '').split(' ').filter(c => !!c && !c.startsWith('s-')).slice(0, 2).join('.');
|
||||
const id = el.dataset.testid ?? cls;
|
||||
if (vw - r.right < threshold) bad.push(`right-bleed ${tag}.${id} right=${Math.round(r.right)}`);
|
||||
if (r.left < threshold) bad.push(`left-bleed ${tag}.${id} left=${Math.round(r.left)}`);
|
||||
}
|
||||
return [...new Set(bad)];
|
||||
});
|
||||
|
||||
const safe = m.docW <= m.innerWidth && m.bodyW <= m.innerWidth && m.offenders.length === 0 && m.scrollers.length === 0 && bleed.length === 0;
|
||||
console.log(`${name.padEnd(20)} ${safe ? '✅' : '⚠️ '} doc=${m.docW} body=${m.bodyW} inner=${m.innerWidth}`);
|
||||
if (m.offenders.length) {
|
||||
console.log(' --- elements past viewport ---');
|
||||
for (const o of m.offenders) console.log(' ', o);
|
||||
}
|
||||
if (m.scrollers.length) {
|
||||
console.log(' --- horizontal scrollers ---');
|
||||
for (const s of m.scrollers) console.log(' ', s);
|
||||
}
|
||||
if (bleed.length) {
|
||||
console.log(' --- edge bleeders (touch viewport) ---');
|
||||
for (const b of bleed.slice(0, 12)) console.log(' ', b);
|
||||
}
|
||||
await page.screenshot({ path: `/tmp/audit-${name}.png`, fullPage: false });
|
||||
}
|
||||
|
||||
await browser.close();
|
||||
295
frontend/e2e/admin-analysis.spec.ts
Normal file
295
frontend/e2e/admin-analysis.spec.ts
Normal file
@@ -0,0 +1,295 @@
|
||||
import { test, expect, type Page } from '@playwright/test';
|
||||
|
||||
// E2E for the admin Analysis section: coverage overview + badges, drill
|
||||
// manga → chapter → page, the page-detail modal, and the enqueue actions.
|
||||
// Fully mocked.
|
||||
|
||||
const DESKTOP = { width: 1280, height: 720 } as const;
|
||||
|
||||
const mangaId = 'a9999999-9999-9999-9999-999999999999';
|
||||
const chapterId = 'c9999999-9999-9999-9999-999999999999';
|
||||
const pageDone = 'p1111111-1111-1111-1111-111111111111';
|
||||
const pageNone = 'p2222222-2222-2222-2222-222222222222';
|
||||
|
||||
const adminUser = {
|
||||
id: 'u11111111-1111-1111-1111-111111111111',
|
||||
username: 'admin',
|
||||
created_at: '2026-01-01T00:00:00Z',
|
||||
is_admin: true
|
||||
};
|
||||
|
||||
const systemStats = {
|
||||
disk: null,
|
||||
memory: { total_bytes: 1, used_bytes: 0, percent_used: 0 },
|
||||
cpu: { percent_used: 0 },
|
||||
alerts: []
|
||||
};
|
||||
|
||||
type Captured = { reenqueue: Record<string, unknown> | null; analyzeCalls: number };
|
||||
|
||||
async function mockAdmin(page: Page, cap: Captured) {
|
||||
await page.route('**/api/v1/auth/config', (r) =>
|
||||
r.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ self_register_enabled: true, private_mode: false })
|
||||
})
|
||||
);
|
||||
await page.route('**/api/v1/auth/me', (r) =>
|
||||
r.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ user: adminUser })
|
||||
})
|
||||
);
|
||||
await page.route('**/api/v1/auth/me/preferences', (r) =>
|
||||
r.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ reader_mode: 'single', reader_page_gap: 'small' })
|
||||
})
|
||||
);
|
||||
await page.route('**/api/v1/me/bookmarks*', (r) =>
|
||||
r.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ items: [], page: { limit: 50, offset: 0, total: 0 } })
|
||||
})
|
||||
);
|
||||
await page.route('**/api/v1/admin/system', (r) =>
|
||||
r.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify(systemStats)
|
||||
})
|
||||
);
|
||||
|
||||
// Coverage overview. Registered before the more specific routes below
|
||||
// so the later-registered (more specific) globs win for their URLs.
|
||||
await page.route('**/api/v1/admin/analysis/mangas**', (r) =>
|
||||
r.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({
|
||||
items: [
|
||||
{
|
||||
manga_id: mangaId,
|
||||
title: 'Berserk',
|
||||
total_pages: 2,
|
||||
analyzed_pages: 1
|
||||
}
|
||||
],
|
||||
page: { limit: 25, offset: 0, total: 1 }
|
||||
})
|
||||
})
|
||||
);
|
||||
await page.route('**/api/v1/admin/analysis/mangas/*/chapters', (r) =>
|
||||
r.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({
|
||||
items: [
|
||||
{
|
||||
chapter_id: chapterId,
|
||||
number: 1,
|
||||
title: 'The Brand',
|
||||
total_pages: 2,
|
||||
analyzed_pages: 1
|
||||
}
|
||||
]
|
||||
})
|
||||
})
|
||||
);
|
||||
await page.route('**/api/v1/admin/analysis/chapters/*/pages', (r) =>
|
||||
r.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({
|
||||
items: [
|
||||
{ page_id: pageDone, page_number: 1, status: 'done' },
|
||||
{ page_id: pageNone, page_number: 2, status: 'none' }
|
||||
]
|
||||
})
|
||||
})
|
||||
);
|
||||
await page.route(`**/api/v1/admin/analysis/pages/${pageDone}`, (r) =>
|
||||
r.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({
|
||||
page_id: pageDone,
|
||||
page_number: 1,
|
||||
chapter_id: chapterId,
|
||||
manga_id: mangaId,
|
||||
status: 'done',
|
||||
is_nsfw: true,
|
||||
scene_description: 'A rainy street at night.',
|
||||
model: 'test-model',
|
||||
error: null,
|
||||
analyzed_at: '2026-06-13T12:00:00Z',
|
||||
ocr: [{ kind: 'speech', text: 'Hello there' }],
|
||||
tags: ['action', 'city'],
|
||||
content_warnings: ['gore']
|
||||
})
|
||||
})
|
||||
);
|
||||
await page.route(`**/api/v1/admin/analysis/pages/${pageNone}`, (r) =>
|
||||
r.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({
|
||||
page_id: pageNone,
|
||||
page_number: 2,
|
||||
chapter_id: chapterId,
|
||||
manga_id: mangaId,
|
||||
status: 'none',
|
||||
is_nsfw: false,
|
||||
scene_description: null,
|
||||
model: null,
|
||||
error: null,
|
||||
analyzed_at: null,
|
||||
ocr: [],
|
||||
tags: [],
|
||||
content_warnings: []
|
||||
})
|
||||
})
|
||||
);
|
||||
|
||||
// Default: keep the SSE connection pending (no events) so tests that
|
||||
// don't care about live updates don't trigger reconnect churn. The
|
||||
// live-updates test overrides this with a fulfilling stream.
|
||||
await page.route(
|
||||
'**/api/v1/admin/analysis/status/stream',
|
||||
() => new Promise(() => {})
|
||||
);
|
||||
|
||||
await page.route('**/api/v1/admin/analysis/reenqueue', (r) => {
|
||||
cap.reenqueue = JSON.parse(r.request().postData() ?? '{}');
|
||||
return r.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ enqueued: 12 })
|
||||
});
|
||||
});
|
||||
await page.route(`**/api/v1/admin/pages/${pageNone}/analyze`, (r) => {
|
||||
cap.analyzeCalls += 1;
|
||||
return r.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ enqueued: true })
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
test.describe('/admin/analysis', () => {
|
||||
test('overview shows coverage badges and queues the whole library', async ({
|
||||
page
|
||||
}) => {
|
||||
const cap: Captured = { reenqueue: null, analyzeCalls: 0 };
|
||||
await mockAdmin(page, cap);
|
||||
await page.setViewportSize(DESKTOP);
|
||||
await page.goto('/admin/analysis');
|
||||
|
||||
const badge = page.getByTestId(`admin-analysis-coverage-manga-${mangaId}`);
|
||||
await expect(badge).toBeVisible();
|
||||
await expect(badge).toContainText('Partial 1/2');
|
||||
|
||||
await page.getByTestId('admin-analysis-enqueue-library').click();
|
||||
await expect(page.getByTestId('admin-analysis-notice')).toContainText(
|
||||
'Enqueued 12 pages'
|
||||
);
|
||||
expect(cap.reenqueue).toEqual({ only_unanalyzed: true });
|
||||
});
|
||||
|
||||
test('drill manga → chapter → page and inspect the result', async ({ page }) => {
|
||||
const cap: Captured = { reenqueue: null, analyzeCalls: 0 };
|
||||
await mockAdmin(page, cap);
|
||||
await page.setViewportSize(DESKTOP);
|
||||
await page.goto('/admin/analysis');
|
||||
|
||||
await page.getByTestId(`admin-analysis-expand-${mangaId}`).click();
|
||||
await expect(
|
||||
page.getByTestId(`admin-analysis-coverage-chapter-${chapterId}`)
|
||||
).toContainText('Partial 1/2');
|
||||
|
||||
await page.getByTestId(`admin-analysis-expand-chapter-${chapterId}`).click();
|
||||
const doneChip = page.getByTestId(`admin-analysis-page-${pageDone}`);
|
||||
await expect(doneChip).toHaveAttribute('data-status', 'done');
|
||||
|
||||
await doneChip.click();
|
||||
const modal = page.getByTestId('admin-analysis-detail');
|
||||
await expect(modal).toBeVisible();
|
||||
await expect(
|
||||
page.getByTestId('admin-analysis-detail-status')
|
||||
).toContainText('Analyzed');
|
||||
await expect(modal).toContainText('Hello there');
|
||||
await expect(modal).toContainText('action');
|
||||
await expect(page.getByTestId('admin-analysis-detail-warnings')).toContainText(
|
||||
'gore'
|
||||
);
|
||||
});
|
||||
|
||||
test('live SSE events drive the indicator and activity ticker', async ({
|
||||
page
|
||||
}) => {
|
||||
const cap: Captured = { reenqueue: null, analyzeCalls: 0 };
|
||||
await mockAdmin(page, cap);
|
||||
// Override the default hanging stream with one that emits two frames.
|
||||
const frames =
|
||||
`event: analysis\ndata: ${JSON.stringify({
|
||||
kind: 'started',
|
||||
page_id: pageNone,
|
||||
manga_id: mangaId,
|
||||
chapter_id: chapterId,
|
||||
page_number: 2
|
||||
})}\n\n` +
|
||||
`event: analysis\ndata: ${JSON.stringify({
|
||||
kind: 'completed',
|
||||
page_id: pageNone,
|
||||
manga_id: mangaId,
|
||||
chapter_id: chapterId,
|
||||
page_number: 2
|
||||
})}\n\n`;
|
||||
let served = false;
|
||||
await page.route('**/api/v1/admin/analysis/status/stream', (r) => {
|
||||
if (served) return new Promise(() => {}); // hang on reconnect
|
||||
served = true;
|
||||
return r.fulfill({
|
||||
status: 200,
|
||||
headers: { 'content-type': 'text/event-stream' },
|
||||
body: frames
|
||||
});
|
||||
});
|
||||
|
||||
await page.setViewportSize(DESKTOP);
|
||||
await page.goto('/admin/analysis');
|
||||
|
||||
// The two SSE frames are parsed and applied to the activity ticker.
|
||||
// (The mocked stream closes after its body, so the live pill flips
|
||||
// back to "Reconnecting…" — the ticker is the durable signal.)
|
||||
const tick = page.getByTestId('admin-analysis-ticker');
|
||||
await expect(tick).toContainText('Analyzing page 2');
|
||||
await expect(tick).toContainText('Analyzed page 2');
|
||||
});
|
||||
|
||||
test('queue an unanalyzed page from its detail modal', async ({ page }) => {
|
||||
const cap: Captured = { reenqueue: null, analyzeCalls: 0 };
|
||||
await mockAdmin(page, cap);
|
||||
await page.setViewportSize(DESKTOP);
|
||||
await page.goto('/admin/analysis');
|
||||
|
||||
await page.getByTestId(`admin-analysis-expand-${mangaId}`).click();
|
||||
await page.getByTestId(`admin-analysis-expand-chapter-${chapterId}`).click();
|
||||
await page.getByTestId(`admin-analysis-page-${pageNone}`).click();
|
||||
|
||||
await expect(page.getByTestId('admin-analysis-detail-status')).toContainText(
|
||||
'Not analyzed'
|
||||
);
|
||||
await page.getByTestId('admin-analysis-detail-queue').click();
|
||||
|
||||
await expect(page.getByTestId('admin-analysis-notice')).toContainText(
|
||||
'Queued page 2'
|
||||
);
|
||||
expect(cap.analyzeCalls).toBe(1);
|
||||
});
|
||||
});
|
||||
224
frontend/e2e/back-nav-flow.spec.ts
Normal file
224
frontend/e2e/back-nav-flow.spec.ts
Normal file
@@ -0,0 +1,224 @@
|
||||
import { test, expect, type Page } from '@playwright/test';
|
||||
|
||||
// Regression spec for the reader-back-loop bug: previously the reader's
|
||||
// back arrow was a plain `<a href="/manga/{id}">`, which PUSHED a new
|
||||
// history entry every tap, so browser-back kept ping-ponging between
|
||||
// detail and reader instead of walking out to home. The reader now
|
||||
// intercepts left-click and does `history.back()` when there's
|
||||
// same-origin history to pop.
|
||||
|
||||
const MID = 'a1111111-1111-1111-1111-111111111111';
|
||||
const CID = 'c1111111-1111-1111-1111-111111111111';
|
||||
|
||||
const mangaBody = {
|
||||
id: MID,
|
||||
title: 'Berserk',
|
||||
status: 'ongoing',
|
||||
alt_titles: [],
|
||||
description: 'Short.',
|
||||
cover_image_path: null,
|
||||
created_at: '2026-01-01T00:00:00Z',
|
||||
updated_at: '2026-01-01T00:00:00Z',
|
||||
authors: [],
|
||||
genres: [],
|
||||
tags: []
|
||||
};
|
||||
|
||||
async function mockApis(page: Page) {
|
||||
await page.route('**/api/v1/auth/config', (r) =>
|
||||
r.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ self_register_enabled: true, private_mode: false })
|
||||
})
|
||||
);
|
||||
await page.route('**/api/v1/auth/me', (r) =>
|
||||
r.fulfill({
|
||||
status: 401,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ error: { code: 'x', message: 'x' } })
|
||||
})
|
||||
);
|
||||
await page.route('**/api/v1/auth/me/preferences', (r) =>
|
||||
r.fulfill({ status: 401, contentType: 'application/json', body: '{}' })
|
||||
);
|
||||
await page.route('**/api/v1/me/bookmarks*', (r) =>
|
||||
r.fulfill({ status: 401, contentType: 'application/json', body: '{}' })
|
||||
);
|
||||
await page.route('**/api/v1/me/read-progress*', (r) =>
|
||||
r.fulfill({ status: 404, contentType: 'application/json', body: '{}' })
|
||||
);
|
||||
await page.route('**/api/v1/genres*', (r) =>
|
||||
r.fulfill({ status: 200, contentType: 'application/json', body: '[]' })
|
||||
);
|
||||
// Catalog returns a single card whose href points at MID so the
|
||||
// SPA-click walk lands on the manga we've mocked downstream.
|
||||
// `**/api/v1/mangas` alone doesn't intercept query strings
|
||||
// (`?limit=&sort=`), so the catalog request would fall through
|
||||
// to a real backend if one is running on :8080 and seed the
|
||||
// page with real-manga IDs that don't match the mock body.
|
||||
// `**/api/v1/mangas?**` (and the same suffix on the catch-all
|
||||
// below) makes the intercept query-tolerant.
|
||||
await page.route('**/api/v1/mangas?**', (r) =>
|
||||
r.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({
|
||||
items: [mangaBody],
|
||||
page: { limit: 50, offset: 0, total: 1 }
|
||||
})
|
||||
})
|
||||
);
|
||||
await page.route('**/api/v1/mangas/**', (r) => {
|
||||
const u = new URL(r.request().url());
|
||||
const last = u.pathname.split('/').pop();
|
||||
if (last === CID)
|
||||
return r.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({
|
||||
id: CID,
|
||||
manga_id: MID,
|
||||
number: 1,
|
||||
title: 'Ch.1',
|
||||
page_count: 1,
|
||||
created_at: '2026-01-01T00:00:00Z'
|
||||
})
|
||||
});
|
||||
if (last === 'pages')
|
||||
return r.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({
|
||||
pages: [
|
||||
{
|
||||
id: 'p',
|
||||
chapter_id: CID,
|
||||
page_number: 1,
|
||||
storage_key: 'x.png',
|
||||
content_type: 'image/png'
|
||||
}
|
||||
]
|
||||
})
|
||||
});
|
||||
if (u.pathname.endsWith('/chapters'))
|
||||
return r.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({
|
||||
items: [
|
||||
{
|
||||
id: CID,
|
||||
manga_id: MID,
|
||||
number: 1,
|
||||
title: 'Ch.1',
|
||||
page_count: 1,
|
||||
created_at: '2026-01-01T00:00:00Z'
|
||||
}
|
||||
],
|
||||
page: { limit: 50, offset: 0, total: 1 }
|
||||
})
|
||||
});
|
||||
return r.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify(mangaBody)
|
||||
});
|
||||
});
|
||||
const png = Buffer.from(
|
||||
'89504e470d0a1a0a0000000d49484452000000010000000108060000001f15c4890000000d49444154789c63000100000005000158a3b62a0000000049454e44ae426082',
|
||||
'hex'
|
||||
);
|
||||
await page.route('**/api/v1/files/**', (r) =>
|
||||
r.fulfill({ status: 200, contentType: 'image/png', body: png })
|
||||
);
|
||||
}
|
||||
|
||||
test.describe('back-nav flow', () => {
|
||||
|
||||
test('phone viewport: reader back pops history (does not push), then detail back returns to home', async ({
|
||||
page
|
||||
}) => {
|
||||
await mockApis(page);
|
||||
await page.setViewportSize({ width: 390, height: 844 });
|
||||
|
||||
await page.goto('/');
|
||||
const initialLen = await page.evaluate(() => window.history.length);
|
||||
|
||||
// Walk: home → detail → reader USING SPA NAVIGATION (link clicks,
|
||||
// not window.location.href). The bug only manifests on SPA nav
|
||||
// because `document.referrer` stays empty across pushState — so
|
||||
// a hard navigation would silently mask the regression.
|
||||
await page.locator('[data-testid="manga-list"] a').first().click();
|
||||
await page.waitForURL(/\/manga\/[^/]+$/);
|
||||
|
||||
await page.locator('[data-testid="chapter-list"] a').first().click();
|
||||
await page.waitForURL(/\/chapter\//);
|
||||
|
||||
const lenAtReader = await page.evaluate(() => window.history.length);
|
||||
expect(lenAtReader).toBe(initialLen + 2);
|
||||
|
||||
// Tap reader back → goes back to detail WITHOUT pushing a new entry.
|
||||
await page.getByTestId('back-to-manga').click();
|
||||
await page.waitForURL(/\/manga\/[^/]+$/);
|
||||
const lenAtDetail = await page.evaluate(() => window.history.length);
|
||||
expect(lenAtDetail).toBe(lenAtReader);
|
||||
|
||||
// Tap detail back → walks out to home.
|
||||
await page.getByTestId('detail-back').click();
|
||||
await page.waitForURL((url) => url.pathname === '/');
|
||||
});
|
||||
|
||||
test('reader cover+title pushes detail when arrived from a non-detail page', async ({
|
||||
page
|
||||
}) => {
|
||||
// The smart-cover-title fix: if the user got to the reader from
|
||||
// a page OTHER than this manga's detail (search, library,
|
||||
// direct link, etc.), the cover+title should push the detail
|
||||
// page so browser-back returns to the reader. The arrow stays
|
||||
// a pure "go back".
|
||||
//
|
||||
// We simulate "non-detail arrival" with a deep-link load
|
||||
// straight to the reader. That covers cold-tab + shared-link +
|
||||
// search-result cases — afterNavigate fires with from=null and
|
||||
// lastInternalPath stays null, so the cover+title click sees
|
||||
// a mismatch and pushes the detail page.
|
||||
await mockApis(page);
|
||||
await page.setViewportSize({ width: 390, height: 844 });
|
||||
|
||||
await page.goto(`/manga/${MID}/chapter/${CID}`);
|
||||
await page.waitForSelector('[data-testid="back-to-manga"]');
|
||||
const lenAtReader = await page.evaluate(() => window.history.length);
|
||||
|
||||
await page.getByTestId('back-to-manga').click();
|
||||
await page.waitForURL(/\/manga\/[^/]+$/);
|
||||
const lenAfter = await page.evaluate(() => window.history.length);
|
||||
// PUSH (not pop) — lenAtReader + 1.
|
||||
expect(lenAfter).toBe(lenAtReader + 1);
|
||||
|
||||
// Browser-back from detail returns to reader (proves it was a
|
||||
// push, not a replace).
|
||||
await page.goBack();
|
||||
await page.waitForURL(/\/chapter\//);
|
||||
});
|
||||
|
||||
test('reader arrow always pops history', async ({ page }) => {
|
||||
await mockApis(page);
|
||||
await page.setViewportSize({ width: 390, height: 844 });
|
||||
|
||||
await page.goto('/');
|
||||
await page.locator('[data-testid="manga-list"] a').first().click();
|
||||
await page.waitForURL(/\/manga\/[^/]+$/);
|
||||
await page.locator('[data-testid="chapter-list"] a').first().click();
|
||||
await page.waitForURL(/\/chapter\//);
|
||||
|
||||
const lenAtReader = await page.evaluate(() => window.history.length);
|
||||
|
||||
// Click the arrow — always pops, regardless of previous page.
|
||||
await page.getByTestId('reader-back-arrow').click();
|
||||
await page.waitForURL(/\/manga\/[^/]+$/);
|
||||
const lenAfter = await page.evaluate(() => window.history.length);
|
||||
expect(lenAfter).toBe(lenAtReader);
|
||||
});
|
||||
|
||||
});
|
||||
244
frontend/e2e/mobile-account-library.spec.ts
Normal file
244
frontend/e2e/mobile-account-library.spec.ts
Normal file
@@ -0,0 +1,244 @@
|
||||
import { test, expect, type Page } from '@playwright/test';
|
||||
|
||||
// Phase 5: Account becomes an inset-grouped hub on mobile (Profile /
|
||||
// Preferences / Change password + red Log out at the bottom) and
|
||||
// /library hosts a SegmentedControl over Bookmarks / Collections /
|
||||
// History. Profile-layout horizontal tabs are hidden on mobile. Desktop
|
||||
// is unchanged.
|
||||
|
||||
const MOBILE = { width: 390, height: 844 } as const;
|
||||
const DESKTOP = { width: 1280, height: 720 } as const;
|
||||
|
||||
async function mockSession(
|
||||
page: Page,
|
||||
opts: { authed?: boolean; logoutCalls?: { count: number } } = {}
|
||||
) {
|
||||
const authed = opts.authed ?? false;
|
||||
await page.route('**/api/v1/auth/config', (route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ self_register_enabled: true, private_mode: false })
|
||||
})
|
||||
);
|
||||
await page.route('**/api/v1/auth/me', (route) =>
|
||||
route.fulfill({
|
||||
status: authed ? 200 : 401,
|
||||
contentType: 'application/json',
|
||||
body: authed
|
||||
? JSON.stringify({
|
||||
user: {
|
||||
id: 'u1',
|
||||
username: 'fabian',
|
||||
created_at: '2026-01-05T00:00:00Z',
|
||||
is_admin: false
|
||||
}
|
||||
})
|
||||
: JSON.stringify({
|
||||
error: { code: 'unauthenticated', message: 'unauthenticated' }
|
||||
})
|
||||
})
|
||||
);
|
||||
await page.route('**/api/v1/auth/me/preferences', (route) =>
|
||||
route.fulfill({
|
||||
status: authed ? 200 : 401,
|
||||
contentType: 'application/json',
|
||||
body: authed
|
||||
? JSON.stringify({
|
||||
reader_mode: 'single',
|
||||
reader_page_gap: 'none',
|
||||
updated_at: '2026-01-05T00:00:00Z'
|
||||
})
|
||||
: JSON.stringify({ error: { code: 'x', message: 'x' } })
|
||||
})
|
||||
);
|
||||
await page.route('**/api/v1/auth/logout', (route) => {
|
||||
if (opts.logoutCalls) opts.logoutCalls.count += 1;
|
||||
return route.fulfill({ status: 204, body: '' });
|
||||
});
|
||||
}
|
||||
|
||||
async function mockLibraryData(page: Page, opts: { authed?: boolean } = {}) {
|
||||
const status = opts.authed ? 200 : 401;
|
||||
const emptyPage = JSON.stringify({
|
||||
items: [],
|
||||
page: { limit: 50, offset: 0, total: 0 }
|
||||
});
|
||||
const unauth = JSON.stringify({
|
||||
error: { code: 'unauthenticated', message: 'unauthenticated' }
|
||||
});
|
||||
await page.route('**/api/v1/me/bookmarks*', (route) =>
|
||||
route.fulfill({
|
||||
status,
|
||||
contentType: 'application/json',
|
||||
body: opts.authed ? emptyPage : unauth
|
||||
})
|
||||
);
|
||||
await page.route('**/api/v1/me/collections*', (route) =>
|
||||
route.fulfill({
|
||||
status,
|
||||
contentType: 'application/json',
|
||||
body: opts.authed ? emptyPage : unauth
|
||||
})
|
||||
);
|
||||
await page.route('**/api/v1/me/read-progress*', (route) =>
|
||||
route.fulfill({
|
||||
status,
|
||||
contentType: 'application/json',
|
||||
body: opts.authed ? emptyPage : unauth
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
test.describe('mobile library', () => {
|
||||
test('phone viewport: BottomNav Library tab navigates to /library', async ({
|
||||
page
|
||||
}) => {
|
||||
await mockSession(page);
|
||||
await mockLibraryData(page);
|
||||
await page.route('**/api/v1/mangas*', (route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({
|
||||
items: [],
|
||||
page: { limit: 50, offset: 0, total: 0 }
|
||||
})
|
||||
})
|
||||
);
|
||||
await page.setViewportSize(MOBILE);
|
||||
await page.goto('/');
|
||||
|
||||
await page.getByTestId('bottom-nav-library').click();
|
||||
await expect(page).toHaveURL(/\/library$/);
|
||||
await expect(page.getByTestId('library-tabs')).toBeVisible();
|
||||
});
|
||||
|
||||
test('phone viewport: SegmentedControl swaps Library sub-tabs and updates ?tab=', async ({
|
||||
page
|
||||
}) => {
|
||||
await mockSession(page, { authed: true });
|
||||
await mockLibraryData(page, { authed: true });
|
||||
await page.setViewportSize(MOBILE);
|
||||
await page.goto('/library');
|
||||
|
||||
// Default sub-tab is bookmarks — URL has no `tab=` param.
|
||||
await expect(page).toHaveURL(/\/library$/);
|
||||
await expect(page.getByTestId('library-bookmarks-empty')).toBeVisible();
|
||||
|
||||
await page
|
||||
.getByTestId('library-tabs')
|
||||
.getByRole('radio', { name: 'Collections' })
|
||||
.click();
|
||||
await expect(page).toHaveURL(/\?tab=collections$/);
|
||||
await expect(page.getByTestId('library-collections-empty')).toBeVisible();
|
||||
|
||||
await page
|
||||
.getByTestId('library-tabs')
|
||||
.getByRole('radio', { name: 'History' })
|
||||
.click();
|
||||
await expect(page).toHaveURL(/\?tab=history$/);
|
||||
await expect(page.getByTestId('library-history-empty')).toBeVisible();
|
||||
|
||||
// Returning to Bookmarks clears the param entirely.
|
||||
await page
|
||||
.getByTestId('library-tabs')
|
||||
.getByRole('radio', { name: 'Bookmarks' })
|
||||
.click();
|
||||
await expect(page).toHaveURL(/\/library$/);
|
||||
});
|
||||
|
||||
test('phone viewport on /library unauth: sign-in prompt, no list', async ({
|
||||
page
|
||||
}) => {
|
||||
await mockSession(page);
|
||||
await mockLibraryData(page);
|
||||
await page.setViewportSize(MOBILE);
|
||||
await page.goto('/library');
|
||||
|
||||
await expect(page.getByTestId('library-signin')).toBeVisible();
|
||||
await expect(page.getByTestId('library-bookmarks-empty')).toHaveCount(0);
|
||||
});
|
||||
});
|
||||
|
||||
test.describe('mobile account hub', () => {
|
||||
test('phone viewport authed: inset-grouped hub renders with profile / preferences / change-password / logout rows', async ({
|
||||
page
|
||||
}) => {
|
||||
await mockSession(page, { authed: true });
|
||||
await page.setViewportSize(MOBILE);
|
||||
await page.goto('/profile/account');
|
||||
|
||||
await expect(page.getByTestId('account-hub')).toBeVisible();
|
||||
await expect(page.getByTestId('account-username')).toHaveText('fabian');
|
||||
await expect(page.getByTestId('account-row-profile')).toBeVisible();
|
||||
await expect(page.getByTestId('account-row-preferences')).toBeVisible();
|
||||
await expect(page.getByTestId('account-row-change-password')).toBeVisible();
|
||||
await expect(page.getByTestId('account-row-logout')).toBeVisible();
|
||||
// Desktop card MUST not render — only the hub view is active
|
||||
// on mobile so we don't double-up the password form testids.
|
||||
await expect(page.getByTestId('account-desktop-card')).toHaveCount(0);
|
||||
});
|
||||
|
||||
test('phone viewport authed: Change password row opens the bottom sheet', async ({
|
||||
page
|
||||
}) => {
|
||||
await mockSession(page, { authed: true });
|
||||
await page.setViewportSize(MOBILE);
|
||||
await page.goto('/profile/account');
|
||||
|
||||
await expect(page.getByTestId('password-sheet')).toBeHidden();
|
||||
await page.getByTestId('account-row-change-password').click();
|
||||
await expect(page.getByTestId('password-sheet')).toBeVisible();
|
||||
await expect(
|
||||
page.getByTestId('password-sheet').getByTestId('current-password')
|
||||
).toBeVisible();
|
||||
});
|
||||
|
||||
test('phone viewport authed: Log out row fires /auth/logout and routes to /login', async ({
|
||||
page
|
||||
}) => {
|
||||
const logoutCalls = { count: 0 };
|
||||
await mockSession(page, { authed: true, logoutCalls });
|
||||
await page.setViewportSize(MOBILE);
|
||||
await page.goto('/profile/account');
|
||||
|
||||
await page.getByTestId('account-row-logout').click();
|
||||
|
||||
await expect(page).toHaveURL(/\/login$/);
|
||||
expect(logoutCalls.count).toBe(1);
|
||||
});
|
||||
|
||||
test('phone viewport unauth: sign-in CTA, no hub', async ({ page }) => {
|
||||
await mockSession(page);
|
||||
await page.setViewportSize(MOBILE);
|
||||
await page.goto('/profile/account');
|
||||
|
||||
await expect(page.getByTestId('account-signin')).toBeVisible();
|
||||
await expect(page.getByTestId('account-hub')).toHaveCount(0);
|
||||
});
|
||||
|
||||
test('phone viewport authed: profile horizontal tabs are hidden', async ({
|
||||
page
|
||||
}) => {
|
||||
await mockSession(page, { authed: true });
|
||||
await page.setViewportSize(MOBILE);
|
||||
await page.goto('/profile/account');
|
||||
|
||||
await expect(page.getByTestId('tab-account')).toBeHidden();
|
||||
});
|
||||
|
||||
test('desktop viewport authed: existing password card visible, hub is not', async ({
|
||||
page
|
||||
}) => {
|
||||
await mockSession(page, { authed: true });
|
||||
await page.setViewportSize(DESKTOP);
|
||||
await page.goto('/profile/account');
|
||||
|
||||
await expect(page.getByTestId('account-desktop-card')).toBeVisible();
|
||||
await expect(page.getByTestId('password-form')).toBeVisible();
|
||||
await expect(page.getByTestId('account-hub')).toHaveCount(0);
|
||||
// Desktop profile tabs remain.
|
||||
await expect(page.getByTestId('tab-account')).toBeVisible();
|
||||
});
|
||||
});
|
||||
105
frontend/e2e/mobile-chrome.spec.ts
Normal file
105
frontend/e2e/mobile-chrome.spec.ts
Normal file
@@ -0,0 +1,105 @@
|
||||
import { test, expect, type Page } from '@playwright/test';
|
||||
|
||||
// Mobile chrome contract: the AppBar + BottomNav are visible on phone
|
||||
// viewports and the desktop header is hidden. On desktop, the reverse.
|
||||
// On the reader route, both mobile bars are hidden so the reader's own
|
||||
// chrome owns the screen. The 640px cutover is the project's single
|
||||
// existing breakpoint and is also used here.
|
||||
|
||||
const MOBILE = { width: 390, height: 844 } as const;
|
||||
const DESKTOP = { width: 1280, height: 720 } as const;
|
||||
|
||||
async function mockAnonymous(page: Page) {
|
||||
await page.route('**/api/v1/auth/config', async (route) => {
|
||||
await route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ self_register_enabled: true, private_mode: false })
|
||||
});
|
||||
});
|
||||
await page.route('**/api/v1/auth/me', async (route) => {
|
||||
await route.fulfill({
|
||||
status: 401,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ error: { code: 'unauthenticated', message: 'unauthenticated' } })
|
||||
});
|
||||
});
|
||||
await page.route('**/api/v1/mangas*', async (route) => {
|
||||
await route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ items: [], page: { limit: 50, offset: 0, total: 0 } })
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
test.describe('mobile chrome', () => {
|
||||
test('phone viewport on /: bottom nav visible, desktop header hidden', async ({ page }) => {
|
||||
await mockAnonymous(page);
|
||||
await page.setViewportSize(MOBILE);
|
||||
await page.goto('/');
|
||||
|
||||
await expect(page.getByTestId('bottom-nav')).toBeVisible();
|
||||
await expect(page.getByTestId('mobile-app-bar')).toBeVisible();
|
||||
await expect(page.locator('header.desktop-header')).toBeHidden();
|
||||
});
|
||||
|
||||
test('desktop viewport on /: desktop header visible, mobile chrome hidden', async ({ page }) => {
|
||||
await mockAnonymous(page);
|
||||
await page.setViewportSize(DESKTOP);
|
||||
await page.goto('/');
|
||||
|
||||
await expect(page.locator('header.desktop-header')).toBeVisible();
|
||||
await expect(page.getByTestId('bottom-nav')).toBeHidden();
|
||||
await expect(page.getByTestId('mobile-app-bar')).toBeHidden();
|
||||
});
|
||||
|
||||
test('phone viewport: Home tab carries aria-current on /', async ({ page }) => {
|
||||
await mockAnonymous(page);
|
||||
await page.setViewportSize(MOBILE);
|
||||
await page.goto('/');
|
||||
|
||||
const home = page.getByTestId('bottom-nav-home');
|
||||
await expect(home).toHaveAttribute('aria-current', 'page');
|
||||
});
|
||||
|
||||
test('phone viewport: tapping Library navigates to /library and marks itself active', async ({ page }) => {
|
||||
await mockAnonymous(page);
|
||||
await page.route('**/api/v1/me/bookmarks*', (route) =>
|
||||
route.fulfill({
|
||||
status: 401,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ error: { code: 'x', message: 'x' } })
|
||||
})
|
||||
);
|
||||
await page.route('**/api/v1/me/collections*', (route) =>
|
||||
route.fulfill({
|
||||
status: 401,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ error: { code: 'x', message: 'x' } })
|
||||
})
|
||||
);
|
||||
await page.route('**/api/v1/me/read-progress*', (route) =>
|
||||
route.fulfill({
|
||||
status: 401,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ error: { code: 'x', message: 'x' } })
|
||||
})
|
||||
);
|
||||
await page.setViewportSize(MOBILE);
|
||||
await page.goto('/');
|
||||
|
||||
await page.getByTestId('bottom-nav-library').click();
|
||||
await expect(page).toHaveURL(/\/library$/);
|
||||
await expect(page.getByTestId('bottom-nav-library')).toHaveAttribute('aria-current', 'page');
|
||||
});
|
||||
|
||||
test('phone viewport: login route hides the mobile chrome (auth gateway pattern)', async ({ page }) => {
|
||||
await mockAnonymous(page);
|
||||
await page.setViewportSize(MOBILE);
|
||||
await page.goto('/login');
|
||||
|
||||
await expect(page.getByTestId('bottom-nav')).toBeHidden();
|
||||
await expect(page.getByTestId('mobile-app-bar')).toBeHidden();
|
||||
});
|
||||
});
|
||||
178
frontend/e2e/mobile-list-search.spec.ts
Normal file
178
frontend/e2e/mobile-list-search.spec.ts
Normal file
@@ -0,0 +1,178 @@
|
||||
import { test, expect, type Page } from '@playwright/test';
|
||||
|
||||
// Phase 2: the catalog (/) gets a mobile chrome — search input full
|
||||
// width, Filter and Sort as chip buttons that open bottom sheets, and
|
||||
// the active-filter row exposes selected facets as removable chips. The
|
||||
// inline desktop filter panel is hidden on mobile.
|
||||
|
||||
const MOBILE = { width: 390, height: 844 } as const;
|
||||
const DESKTOP = { width: 1280, height: 720 } as const;
|
||||
|
||||
async function mockAnonymous(page: Page) {
|
||||
await page.route('**/api/v1/auth/config', async (route) => {
|
||||
await route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ self_register_enabled: true, private_mode: false })
|
||||
});
|
||||
});
|
||||
await page.route('**/api/v1/auth/me', async (route) => {
|
||||
await route.fulfill({
|
||||
status: 401,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ error: { code: 'unauthenticated', message: 'unauthenticated' } })
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function mockCatalog(
|
||||
page: Page,
|
||||
opts: { genres?: { id: string; name: string }[]; capture?: (url: URL) => void } = {}
|
||||
) {
|
||||
const genres = opts.genres ?? [
|
||||
{ id: 'g-action', name: 'Action' },
|
||||
{ id: 'g-romance', name: 'Romance' }
|
||||
];
|
||||
await page.route('**/api/v1/genres*', async (route) => {
|
||||
await route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify(genres)
|
||||
});
|
||||
});
|
||||
await page.route('**/api/v1/mangas*', async (route) => {
|
||||
const url = new URL(route.request().url());
|
||||
opts.capture?.(url);
|
||||
await route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({
|
||||
items: [],
|
||||
page: { limit: 50, offset: 0, total: 0 }
|
||||
})
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
test.describe('mobile catalog chrome', () => {
|
||||
test('phone viewport: Sort chip is visible, inline desktop select is hidden', async ({
|
||||
page
|
||||
}) => {
|
||||
await mockAnonymous(page);
|
||||
await mockCatalog(page);
|
||||
await page.setViewportSize(MOBILE);
|
||||
await page.goto('/');
|
||||
|
||||
await expect(page.getByTestId('sort-chip')).toBeVisible();
|
||||
await expect(page.getByTestId('sort-select')).toBeHidden();
|
||||
});
|
||||
|
||||
test('desktop viewport: inline Sort select is visible, mobile chip is hidden', async ({
|
||||
page
|
||||
}) => {
|
||||
await mockAnonymous(page);
|
||||
await mockCatalog(page);
|
||||
await page.setViewportSize(DESKTOP);
|
||||
await page.goto('/');
|
||||
|
||||
await expect(page.getByTestId('sort-select')).toBeVisible();
|
||||
await expect(page.getByTestId('sort-chip')).toBeHidden();
|
||||
});
|
||||
|
||||
// `empty` only renders after onMount's listMangas() resolves, which
|
||||
// means hydration is complete and click handlers are attached. Without
|
||||
// this gate, Playwright dispatches the click against the SSR'd static
|
||||
// button before Svelte takes over and the state mutation is dropped.
|
||||
async function waitForHydration(page: Page) {
|
||||
await expect(page.getByTestId('empty')).toBeVisible();
|
||||
}
|
||||
|
||||
test('phone viewport: Filter chip opens the bottom sheet, not the inline panel', async ({
|
||||
page
|
||||
}) => {
|
||||
await mockAnonymous(page);
|
||||
await mockCatalog(page);
|
||||
await page.setViewportSize(MOBILE);
|
||||
await page.goto('/');
|
||||
await waitForHydration(page);
|
||||
|
||||
await expect(page.getByTestId('filter-sheet')).toBeHidden();
|
||||
await page.getByTestId('filters-toggle').click();
|
||||
|
||||
await expect(page.getByTestId('filter-sheet')).toBeVisible();
|
||||
// Inline panel must not render on mobile — the snippet gated by
|
||||
// !isMobileViewport means the same form is never on the page twice.
|
||||
await expect(page.getByTestId('filters-panel')).toHaveCount(0);
|
||||
});
|
||||
|
||||
test('phone viewport: picking a genre updates URL + shows an active-filter chip', async ({
|
||||
page
|
||||
}) => {
|
||||
// The API uses `genre_id` (singular, comma-joined) while the URL
|
||||
// we surface to the user uses `genres` — verify both paths.
|
||||
let lastGenreIdParam: string | null = null;
|
||||
await mockAnonymous(page);
|
||||
await mockCatalog(page, {
|
||||
capture: (url) => {
|
||||
lastGenreIdParam = url.searchParams.get('genre_id');
|
||||
}
|
||||
});
|
||||
await page.setViewportSize(MOBILE);
|
||||
await page.goto('/');
|
||||
await waitForHydration(page);
|
||||
|
||||
await page.getByTestId('filters-toggle').click();
|
||||
await page.getByTestId('genre-filter-Action').click();
|
||||
|
||||
await expect(page).toHaveURL(/genres=g-action/);
|
||||
await expect(page.getByTestId('active-filter-genre-Action')).toBeVisible();
|
||||
expect(lastGenreIdParam).toBe('g-action');
|
||||
});
|
||||
|
||||
test('phone viewport: removing an active-filter chip clears that facet', async ({
|
||||
page
|
||||
}) => {
|
||||
await mockAnonymous(page);
|
||||
await mockCatalog(page);
|
||||
await page.setViewportSize(MOBILE);
|
||||
await page.goto('/?genres=g-action');
|
||||
await waitForHydration(page);
|
||||
|
||||
// The chip appears after hydrateFromUrl resolves the genre id.
|
||||
const chip = page.getByTestId('active-filter-genre-Action');
|
||||
await expect(chip).toBeVisible();
|
||||
|
||||
// Chip's remove button has an aria-label "Remove genre Action".
|
||||
await chip.getByRole('button', { name: 'Remove genre Action' }).click();
|
||||
|
||||
await expect(chip).toHaveCount(0);
|
||||
await expect(page).toHaveURL((url) => !url.search.includes('genres='));
|
||||
});
|
||||
|
||||
test('phone viewport: Sort sheet swaps the sort and dismisses on pick', async ({
|
||||
page
|
||||
}) => {
|
||||
let lastSortParam: string | null = null;
|
||||
await mockAnonymous(page);
|
||||
await mockCatalog(page, {
|
||||
capture: (url) => {
|
||||
lastSortParam = url.searchParams.get('sort');
|
||||
}
|
||||
});
|
||||
await page.setViewportSize(MOBILE);
|
||||
await page.goto('/');
|
||||
await waitForHydration(page);
|
||||
|
||||
await page.getByTestId('sort-chip').click();
|
||||
await expect(page.getByTestId('sort-sheet')).toBeVisible();
|
||||
|
||||
// Use click(), not check(): the onchange handler closes the sheet
|
||||
// synchronously so the radio is gone before check() can verify the
|
||||
// `checked` state.
|
||||
await page.getByTestId('sort-sheet').getByRole('radio', { name: /Title/ }).click();
|
||||
|
||||
await expect(page.getByTestId('sort-sheet')).toBeHidden();
|
||||
await expect(page).toHaveURL(/sort=title/);
|
||||
expect(lastSortParam).toBe('title');
|
||||
});
|
||||
});
|
||||
287
frontend/e2e/mobile-manga-detail.spec.ts
Normal file
287
frontend/e2e/mobile-manga-detail.spec.ts
Normal file
@@ -0,0 +1,287 @@
|
||||
import { test, expect, type Page } from '@playwright/test';
|
||||
|
||||
// Phase 3: the manga detail page gains a mobile hero (blurred backdrop
|
||||
// + transparent app bar), a sticky bottom CTA whose wording reflects
|
||||
// read progress, a 3-line description clamp with Read more, and an
|
||||
// overflow Sheet that hosts secondary actions (Edit / Upload chapter /
|
||||
// Add to collection / Force resync). Desktop layout is preserved.
|
||||
|
||||
const MOBILE = { width: 390, height: 844 } as const;
|
||||
const DESKTOP = { width: 1280, height: 720 } as const;
|
||||
|
||||
const mangaId = 'a1111111-1111-1111-1111-111111111111';
|
||||
const firstChapterId = 'c1111111-1111-1111-1111-111111111111';
|
||||
const latestChapterId = 'c2222222-2222-2222-2222-222222222222';
|
||||
|
||||
// Chapters arrive newest-first from the API (source_index DESC) — the
|
||||
// detail page's "Read first chapter" CTA targets the last element.
|
||||
const chaptersFixture = [
|
||||
{
|
||||
id: latestChapterId,
|
||||
manga_id: mangaId,
|
||||
number: 2,
|
||||
title: 'Second',
|
||||
page_count: 10,
|
||||
created_at: '2026-02-01T00:00:00Z'
|
||||
},
|
||||
{
|
||||
id: firstChapterId,
|
||||
manga_id: mangaId,
|
||||
number: 1,
|
||||
title: 'The Brand',
|
||||
page_count: 8,
|
||||
created_at: '2026-01-01T00:00:00Z'
|
||||
}
|
||||
];
|
||||
|
||||
const shortDescription = 'A brief synopsis.';
|
||||
const longDescription =
|
||||
'This is an extended description that runs over multiple lines so that ' +
|
||||
'the clamp + Read more behavior on mobile has actual content to truncate. ' +
|
||||
'Long-form descriptions are common for manga that have been crawled from ' +
|
||||
'public sources with editorial blurbs, so the catalog has to handle them ' +
|
||||
'gracefully without pushing the chapter list below the fold on phones.';
|
||||
|
||||
function mangaFixture(description: string = shortDescription) {
|
||||
return {
|
||||
id: mangaId,
|
||||
title: 'Berserk',
|
||||
status: 'ongoing',
|
||||
alt_titles: [],
|
||||
description,
|
||||
cover_image_path: `mangas/${mangaId}/cover.png`,
|
||||
created_at: '2026-01-01T00:00:00Z',
|
||||
updated_at: '2026-01-01T00:00:00Z',
|
||||
authors: [{ id: 'au1', name: 'Kentaro Miura' }],
|
||||
genres: [],
|
||||
tags: []
|
||||
};
|
||||
}
|
||||
|
||||
async function mockDetail(
|
||||
page: Page,
|
||||
opts: {
|
||||
description?: string;
|
||||
readProgress?: {
|
||||
chapter_id: string;
|
||||
chapter_number: number;
|
||||
page: number;
|
||||
} | null;
|
||||
authed?: boolean;
|
||||
similar?: Array<Record<string, unknown>>;
|
||||
} = {}
|
||||
) {
|
||||
const authed = opts.authed ?? false;
|
||||
|
||||
await page.route('**/api/v1/auth/config', (route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ self_register_enabled: true, private_mode: false })
|
||||
})
|
||||
);
|
||||
await page.route('**/api/v1/auth/me', (route) =>
|
||||
route.fulfill({
|
||||
status: authed ? 200 : 401,
|
||||
contentType: 'application/json',
|
||||
body: authed
|
||||
? JSON.stringify({
|
||||
user: {
|
||||
id: 'u1',
|
||||
username: 'reader',
|
||||
created_at: '2026-01-01T00:00:00Z',
|
||||
is_admin: false
|
||||
}
|
||||
})
|
||||
: JSON.stringify({
|
||||
error: { code: 'unauthenticated', message: 'unauthenticated' }
|
||||
})
|
||||
})
|
||||
);
|
||||
await page.route('**/api/v1/auth/me/preferences', (route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({})
|
||||
})
|
||||
);
|
||||
await page.route('**/api/v1/me/bookmarks*', (route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ items: [], page: { limit: 50, offset: 0, total: 0 } })
|
||||
})
|
||||
);
|
||||
await page.route(`**/api/v1/mangas/${mangaId}`, (route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify(mangaFixture(opts.description))
|
||||
})
|
||||
);
|
||||
await page.route(`**/api/v1/mangas/${mangaId}/chapters*`, (route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({
|
||||
items: chaptersFixture,
|
||||
page: { limit: 50, offset: 0, total: chaptersFixture.length }
|
||||
})
|
||||
})
|
||||
);
|
||||
await page.route(`**/api/v1/mangas/${mangaId}/similar`, (route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ items: opts.similar ?? [] })
|
||||
})
|
||||
);
|
||||
await page.route(`**/api/v1/me/read-progress/${mangaId}`, (route) =>
|
||||
route.fulfill({
|
||||
status: opts.readProgress ? 200 : 404,
|
||||
contentType: 'application/json',
|
||||
body: opts.readProgress
|
||||
? JSON.stringify(opts.readProgress)
|
||||
: JSON.stringify({ error: { code: 'not_found', message: 'not found' } })
|
||||
})
|
||||
);
|
||||
// 1x1 transparent PNG so cover image requests don't 404 noisily.
|
||||
const png = Buffer.from(
|
||||
'89504e470d0a1a0a0000000d49484452000000010000000108060000001f15c4890000000d49444154789c63000100000005000158a3b62a0000000049454e44ae426082',
|
||||
'hex'
|
||||
);
|
||||
await page.route('**/api/v1/files/**', (route) =>
|
||||
route.fulfill({ status: 200, contentType: 'image/png', body: png })
|
||||
);
|
||||
}
|
||||
|
||||
test.describe('mobile manga detail', () => {
|
||||
test('phone viewport: with no progress, CTA reads "Read first chapter" and links to the oldest chapter', async ({
|
||||
page
|
||||
}) => {
|
||||
await mockDetail(page);
|
||||
await page.setViewportSize(MOBILE);
|
||||
await page.goto(`/manga/${mangaId}`);
|
||||
|
||||
await expect(page.getByTestId('mobile-hero')).toBeVisible();
|
||||
|
||||
const cta = page.getByTestId('continue-cta');
|
||||
await expect(cta).toHaveText('Read first chapter');
|
||||
await expect(cta).toHaveAttribute(
|
||||
'href',
|
||||
`/manga/${mangaId}/chapter/${firstChapterId}`
|
||||
);
|
||||
});
|
||||
|
||||
test('phone viewport: with progress, CTA reads "Continue {chapterLabel}" and links to the in-progress chapter', async ({
|
||||
page
|
||||
}) => {
|
||||
await mockDetail(page, {
|
||||
authed: true,
|
||||
readProgress: { chapter_id: latestChapterId, chapter_number: 2, page: 5 }
|
||||
});
|
||||
await page.setViewportSize(MOBILE);
|
||||
await page.goto(`/manga/${mangaId}`);
|
||||
|
||||
const cta = page.getByTestId('continue-cta');
|
||||
// chapterLabel returns the title when present, falling back to
|
||||
// "Chapter N" only when empty — the fixture has title "Second".
|
||||
await expect(cta).toHaveText(/Continue\s+Second/);
|
||||
await expect(cta).toHaveAttribute(
|
||||
'href',
|
||||
`/manga/${mangaId}/chapter/${latestChapterId}`
|
||||
);
|
||||
});
|
||||
|
||||
test('phone viewport: short description shows no Read more toggle', async ({ page }) => {
|
||||
await mockDetail(page, { description: shortDescription });
|
||||
await page.setViewportSize(MOBILE);
|
||||
await page.goto(`/manga/${mangaId}`);
|
||||
|
||||
await expect(page.getByTestId('manga-description')).toBeVisible();
|
||||
await expect(page.getByTestId('read-more-toggle')).toHaveCount(0);
|
||||
});
|
||||
|
||||
test('phone viewport: long description shows Read more, expands on tap, collapses on second tap', async ({
|
||||
page
|
||||
}) => {
|
||||
await mockDetail(page, { description: longDescription });
|
||||
await page.setViewportSize(MOBILE);
|
||||
await page.goto(`/manga/${mangaId}`);
|
||||
|
||||
const toggle = page.getByTestId('read-more-toggle');
|
||||
await expect(toggle).toHaveText('Read more');
|
||||
|
||||
await toggle.click();
|
||||
await expect(toggle).toHaveText('Read less');
|
||||
|
||||
await toggle.click();
|
||||
await expect(toggle).toHaveText('Read more');
|
||||
});
|
||||
|
||||
test('phone viewport: overflow ⋯ opens the actions sheet with Edit / Upload / Add-to-collection rows', async ({
|
||||
page
|
||||
}) => {
|
||||
await mockDetail(page, { authed: true });
|
||||
await page.setViewportSize(MOBILE);
|
||||
await page.goto(`/manga/${mangaId}`);
|
||||
|
||||
await expect(page.getByTestId('detail-overflow-sheet')).toBeHidden();
|
||||
await page.getByTestId('detail-overflow').click();
|
||||
|
||||
const sheet = page.getByTestId('detail-overflow-sheet');
|
||||
await expect(sheet).toBeVisible();
|
||||
await expect(sheet.getByTestId('overflow-edit')).toBeVisible();
|
||||
await expect(sheet.getByTestId('overflow-upload-chapter')).toBeVisible();
|
||||
await expect(sheet.getByTestId('overflow-add-to-collection')).toBeVisible();
|
||||
});
|
||||
|
||||
test('desktop viewport: mobile hero is hidden, existing layout + action-row stays', async ({
|
||||
page
|
||||
}) => {
|
||||
await mockDetail(page, { authed: true });
|
||||
await page.setViewportSize(DESKTOP);
|
||||
await page.goto(`/manga/${mangaId}`);
|
||||
|
||||
// Hero block exists in DOM but is hidden by CSS at >640px.
|
||||
await expect(page.getByTestId('mobile-hero')).toBeHidden();
|
||||
// The desktop title/cover/action surfaces remain visible.
|
||||
await expect(page.getByTestId('manga-title')).toBeVisible();
|
||||
await expect(page.getByTestId('bookmark-toggle')).toBeVisible();
|
||||
});
|
||||
|
||||
test('renders the Similar section with recommended cards when present', async ({
|
||||
page
|
||||
}) => {
|
||||
await mockDetail(page, {
|
||||
similar: [
|
||||
{
|
||||
id: 'b2222222-2222-2222-2222-222222222222',
|
||||
title: 'Vinland Saga',
|
||||
status: 'ongoing',
|
||||
alt_titles: [],
|
||||
description: null,
|
||||
cover_image_path: null,
|
||||
created_at: '2026-01-01T00:00:00Z',
|
||||
updated_at: '2026-01-01T00:00:00Z',
|
||||
authors: [{ id: 'au2', name: 'Makoto Yukimura' }],
|
||||
genres: []
|
||||
}
|
||||
]
|
||||
});
|
||||
await page.goto(`/manga/${mangaId}`);
|
||||
|
||||
const section = page.getByTestId('similar-section');
|
||||
await expect(section).toBeVisible();
|
||||
await expect(section.getByText('Vinland Saga')).toBeVisible();
|
||||
});
|
||||
|
||||
test('omits the Similar section when there are no recommendations', async ({ page }) => {
|
||||
await mockDetail(page, { similar: [] });
|
||||
await page.goto(`/manga/${mangaId}`);
|
||||
|
||||
// The chapter list is the anchor that the page has rendered.
|
||||
await expect(page.getByTestId('manga-title')).toBeVisible();
|
||||
await expect(page.getByTestId('similar-section')).toHaveCount(0);
|
||||
});
|
||||
});
|
||||
295
frontend/e2e/mobile-reader.spec.ts
Normal file
295
frontend/e2e/mobile-reader.spec.ts
Normal file
@@ -0,0 +1,295 @@
|
||||
import { test, expect, type Page } from '@playwright/test';
|
||||
|
||||
// Phase 4: the reader gains a mobile chrome — invisible tap zones for
|
||||
// prev/next/toggle, a chapter-jump bottom sheet, a reader-settings
|
||||
// sheet (mode + gap + brightness), a fixed bottom page scrubber, a
|
||||
// brightness overlay driven by a CSS variable, and an idle-timer
|
||||
// auto-hide for the chrome after 3s of inactivity. Desktop chrome is
|
||||
// preserved above 640px.
|
||||
|
||||
const MOBILE = { width: 390, height: 844 } as const;
|
||||
const DESKTOP = { width: 1280, height: 720 } as const;
|
||||
|
||||
const mangaId = 'a9999999-9999-9999-9999-999999999999';
|
||||
const chapterAId = 'c9999999-9999-9999-9999-999999999999';
|
||||
const chapterBId = 'c8888888-8888-8888-8888-888888888888';
|
||||
|
||||
const mangaFixture = {
|
||||
id: mangaId,
|
||||
title: 'Berserk',
|
||||
status: 'ongoing',
|
||||
alt_titles: [],
|
||||
description: null,
|
||||
cover_image_path: `mangas/${mangaId}/cover.png`,
|
||||
created_at: '2026-01-01T00:00:00Z',
|
||||
updated_at: '2026-01-01T00:00:00Z',
|
||||
authors: [{ id: 'au1', name: 'Kentaro Miura' }],
|
||||
genres: [],
|
||||
tags: []
|
||||
};
|
||||
|
||||
const chaptersFixture = [
|
||||
{
|
||||
id: chapterAId,
|
||||
manga_id: mangaId,
|
||||
number: 1,
|
||||
title: 'The Brand',
|
||||
page_count: 3,
|
||||
created_at: '2026-01-01T00:00:00Z'
|
||||
},
|
||||
{
|
||||
id: chapterBId,
|
||||
manga_id: mangaId,
|
||||
number: 2,
|
||||
title: 'Second',
|
||||
page_count: 3,
|
||||
created_at: '2026-02-01T00:00:00Z'
|
||||
}
|
||||
];
|
||||
|
||||
const pagesFixture = [
|
||||
{
|
||||
id: 'p11111111-1111-1111-1111-111111111111',
|
||||
chapter_id: chapterAId,
|
||||
page_number: 1,
|
||||
storage_key: `mangas/${mangaId}/chapters/${chapterAId}/pages/0001.png`,
|
||||
content_type: 'image/png'
|
||||
},
|
||||
{
|
||||
id: 'p22222222-1111-1111-1111-111111111111',
|
||||
chapter_id: chapterAId,
|
||||
page_number: 2,
|
||||
storage_key: `mangas/${mangaId}/chapters/${chapterAId}/pages/0002.png`,
|
||||
content_type: 'image/png'
|
||||
},
|
||||
{
|
||||
id: 'p33333333-1111-1111-1111-111111111111',
|
||||
chapter_id: chapterAId,
|
||||
page_number: 3,
|
||||
storage_key: `mangas/${mangaId}/chapters/${chapterAId}/pages/0003.png`,
|
||||
content_type: 'image/png'
|
||||
}
|
||||
];
|
||||
|
||||
async function mockReader(page: Page) {
|
||||
await page.route('**/api/v1/auth/config', (route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ self_register_enabled: true, private_mode: false })
|
||||
})
|
||||
);
|
||||
await page.route('**/api/v1/auth/me', (route) =>
|
||||
route.fulfill({
|
||||
status: 401,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({
|
||||
error: { code: 'unauthenticated', message: 'unauthenticated' }
|
||||
})
|
||||
})
|
||||
);
|
||||
await page.route('**/api/v1/auth/me/preferences', (route) =>
|
||||
route.fulfill({ status: 401, contentType: 'application/json', body: '{}' })
|
||||
);
|
||||
await page.route('**/api/v1/me/bookmarks*', (route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ items: [], page: { limit: 50, offset: 0, total: 0 } })
|
||||
})
|
||||
);
|
||||
await page.route(`**/api/v1/mangas/${mangaId}`, (route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify(mangaFixture)
|
||||
})
|
||||
);
|
||||
// Pages endpoint must come before the single-chapter handler
|
||||
// because Playwright matches more recently-registered routes
|
||||
// first — registering pages last keeps it on top.
|
||||
await page.route(`**/api/v1/mangas/${mangaId}/chapters`, (route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({
|
||||
items: chaptersFixture,
|
||||
page: { limit: 50, offset: 0, total: chaptersFixture.length }
|
||||
})
|
||||
})
|
||||
);
|
||||
await page.route(`**/api/v1/mangas/${mangaId}/chapters\\?*`, (route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({
|
||||
items: chaptersFixture,
|
||||
page: { limit: 50, offset: 0, total: chaptersFixture.length }
|
||||
})
|
||||
})
|
||||
);
|
||||
await page.route(`**/api/v1/mangas/${mangaId}/chapters/${chapterAId}`, (route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify(chaptersFixture[0])
|
||||
})
|
||||
);
|
||||
await page.route(`**/api/v1/mangas/${mangaId}/chapters/${chapterBId}`, (route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify(chaptersFixture[1])
|
||||
})
|
||||
);
|
||||
await page.route(
|
||||
`**/api/v1/mangas/${mangaId}/chapters/${chapterAId}/pages`,
|
||||
(route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ pages: pagesFixture })
|
||||
})
|
||||
);
|
||||
await page.route(
|
||||
`**/api/v1/mangas/${mangaId}/chapters/${chapterBId}/pages`,
|
||||
(route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ pages: pagesFixture })
|
||||
})
|
||||
);
|
||||
await page.route(`**/api/v1/me/read-progress/${mangaId}`, (route) =>
|
||||
route.fulfill({
|
||||
status: 404,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ error: { code: 'not_found', message: 'not found' } })
|
||||
})
|
||||
);
|
||||
const png = Buffer.from(
|
||||
'89504e470d0a1a0a0000000d49484452000000010000000108060000001f15c4890000000d49444154789c63000100000005000158a3b62a0000000049454e44ae426082',
|
||||
'hex'
|
||||
);
|
||||
await page.route('**/api/v1/files/**', (route) =>
|
||||
route.fulfill({ status: 200, contentType: 'image/png', body: png })
|
||||
);
|
||||
}
|
||||
|
||||
test.describe('mobile reader', () => {
|
||||
test('phone viewport: tap right advances a page, tap left goes back', async ({
|
||||
page
|
||||
}) => {
|
||||
await mockReader(page);
|
||||
await page.setViewportSize(MOBILE);
|
||||
await page.goto(`/manga/${mangaId}/chapter/${chapterAId}`);
|
||||
|
||||
await expect(page.getByTestId('page-indicator')).toHaveText('Page 1 / 3');
|
||||
|
||||
await page.getByTestId('reader-tap-right').click();
|
||||
await expect(page.getByTestId('page-indicator')).toHaveText('Page 2 / 3');
|
||||
|
||||
await page.getByTestId('reader-tap-left').click();
|
||||
await expect(page.getByTestId('page-indicator')).toHaveText('Page 1 / 3');
|
||||
});
|
||||
|
||||
test('phone viewport: tap center toggles focus mode (chrome slides off)', async ({
|
||||
page
|
||||
}) => {
|
||||
await mockReader(page);
|
||||
await page.setViewportSize(MOBILE);
|
||||
await page.goto(`/manga/${mangaId}/chapter/${chapterAId}`);
|
||||
|
||||
const html = page.locator('html');
|
||||
await expect(html).not.toHaveAttribute('data-reader-fullscreen', 'true');
|
||||
|
||||
await page.getByTestId('reader-tap-center').click();
|
||||
await expect(html).toHaveAttribute('data-reader-fullscreen', 'true');
|
||||
|
||||
await page.getByTestId('reader-tap-center').click();
|
||||
await expect(html).not.toHaveAttribute('data-reader-fullscreen', 'true');
|
||||
});
|
||||
|
||||
test('phone viewport: chrome auto-hides after 3s of inactivity in single mode', async ({
|
||||
page
|
||||
}) => {
|
||||
await mockReader(page);
|
||||
await page.setViewportSize(MOBILE);
|
||||
await page.goto(`/manga/${mangaId}/chapter/${chapterAId}`);
|
||||
|
||||
const html = page.locator('html');
|
||||
// Default state: chrome visible.
|
||||
await expect(html).not.toHaveAttribute('data-reader-fullscreen', 'true');
|
||||
// After 3s of idle the timer flips the fullscreen flag (the
|
||||
// existing focus-mode CSS does the slide-off).
|
||||
await expect(html).toHaveAttribute('data-reader-fullscreen', 'true', {
|
||||
timeout: 6000
|
||||
});
|
||||
});
|
||||
|
||||
test('phone viewport: chapter-jump button opens the sheet and lists all chapters', async ({
|
||||
page
|
||||
}) => {
|
||||
await mockReader(page);
|
||||
await page.setViewportSize(MOBILE);
|
||||
await page.goto(`/manga/${mangaId}/chapter/${chapterAId}`);
|
||||
|
||||
await expect(page.getByTestId('chapter-jump-sheet')).toBeHidden();
|
||||
await page.getByTestId('reader-chapter-jump').click();
|
||||
|
||||
const sheet = page.getByTestId('chapter-jump-sheet');
|
||||
await expect(sheet).toBeVisible();
|
||||
await expect(sheet.getByTestId(`chapter-jump-${chapterAId}`)).toBeVisible();
|
||||
await expect(sheet.getByTestId(`chapter-jump-${chapterBId}`)).toBeVisible();
|
||||
});
|
||||
|
||||
test('phone viewport: settings sheet swaps mode to Continuous and the continuous container appears', async ({
|
||||
page
|
||||
}) => {
|
||||
await mockReader(page);
|
||||
await page.setViewportSize(MOBILE);
|
||||
await page.goto(`/manga/${mangaId}/chapter/${chapterAId}`);
|
||||
|
||||
await page.getByTestId('reader-settings-btn').click();
|
||||
await expect(page.getByTestId('reader-settings-sheet')).toBeVisible();
|
||||
|
||||
await page
|
||||
.getByTestId('reader-settings-sheet')
|
||||
.getByRole('radio', { name: 'Continuous' })
|
||||
.click();
|
||||
|
||||
await expect(page.getByTestId('reader-continuous')).toBeVisible();
|
||||
});
|
||||
|
||||
test('phone viewport: brightness slider drives the --reader-dim CSS variable', async ({
|
||||
page
|
||||
}) => {
|
||||
await mockReader(page);
|
||||
await page.setViewportSize(MOBILE);
|
||||
await page.goto(`/manga/${mangaId}/chapter/${chapterAId}`);
|
||||
|
||||
await page.getByTestId('reader-settings-btn').click();
|
||||
|
||||
// Drive the slider via fill — Playwright's fill on a range
|
||||
// input sets value and dispatches input events.
|
||||
await page.getByTestId('settings-brightness').fill('0.5');
|
||||
|
||||
const dim = await page.evaluate(() =>
|
||||
getComputedStyle(document.documentElement).getPropertyValue('--reader-dim').trim()
|
||||
);
|
||||
// brightness 0.5 → dim = (1 - 0.5) * 0.7 = 0.35
|
||||
expect(Number.parseFloat(dim)).toBeCloseTo(0.35, 2);
|
||||
});
|
||||
|
||||
test('desktop viewport: tap zones hidden, chapter select visible, settings button hidden', async ({
|
||||
page
|
||||
}) => {
|
||||
await mockReader(page);
|
||||
await page.setViewportSize(DESKTOP);
|
||||
await page.goto(`/manga/${mangaId}/chapter/${chapterAId}`);
|
||||
|
||||
await expect(page.getByTestId('reader-tap')).toHaveCount(0);
|
||||
await expect(page.getByTestId('reader-chapter-select')).toBeVisible();
|
||||
await expect(page.getByTestId('reader-settings-btn')).toBeHidden();
|
||||
});
|
||||
});
|
||||
377
frontend/e2e/page-context-menu.spec.ts
Normal file
377
frontend/e2e/page-context-menu.spec.ts
Normal file
@@ -0,0 +1,377 @@
|
||||
import { test, expect, type Page } from '@playwright/test';
|
||||
|
||||
// E2E for the per-page collection + tag flow added in v0.61.0. Mocks
|
||||
// the entire API so the spec runs without a backend. The same
|
||||
// fixtures + mockReader scaffolding as `mobile-reader.spec.ts`;
|
||||
// kept inline so the file stays self-contained.
|
||||
|
||||
const MOBILE = { width: 390, height: 844 } as const;
|
||||
const DESKTOP = { width: 1280, height: 720 } as const;
|
||||
|
||||
const mangaId = 'a9999999-9999-9999-9999-999999999999';
|
||||
const chapterId = 'c9999999-9999-9999-9999-999999999999';
|
||||
const pageId = 'p11111111-1111-1111-1111-111111111111';
|
||||
const collectionId = 'cc111111-1111-1111-1111-111111111111';
|
||||
|
||||
const mangaFixture = {
|
||||
id: mangaId,
|
||||
title: 'Berserk',
|
||||
status: 'ongoing',
|
||||
alt_titles: [],
|
||||
description: null,
|
||||
cover_image_path: `mangas/${mangaId}/cover.png`,
|
||||
created_at: '2026-01-01T00:00:00Z',
|
||||
updated_at: '2026-01-01T00:00:00Z',
|
||||
authors: [{ id: 'au1', name: 'Kentaro Miura' }],
|
||||
genres: [],
|
||||
tags: []
|
||||
};
|
||||
|
||||
const chapterFixture = {
|
||||
id: chapterId,
|
||||
manga_id: mangaId,
|
||||
number: 1,
|
||||
title: 'The Brand',
|
||||
page_count: 1,
|
||||
created_at: '2026-01-01T00:00:00Z'
|
||||
};
|
||||
|
||||
const pagesFixture = [
|
||||
{
|
||||
id: pageId,
|
||||
chapter_id: chapterId,
|
||||
page_number: 1,
|
||||
storage_key: `mangas/${mangaId}/chapters/${chapterId}/pages/0001.png`,
|
||||
content_type: 'image/png'
|
||||
}
|
||||
];
|
||||
|
||||
const userFixture = {
|
||||
id: 'u11111111-1111-1111-1111-111111111111',
|
||||
username: 'tester',
|
||||
created_at: '2026-01-01T00:00:00Z',
|
||||
is_admin: false
|
||||
};
|
||||
|
||||
const collectionFixture = {
|
||||
id: collectionId,
|
||||
user_id: userFixture.id,
|
||||
name: 'Favorite panels',
|
||||
description: null,
|
||||
created_at: '2026-01-01T00:00:00Z',
|
||||
updated_at: '2026-01-01T00:00:00Z',
|
||||
manga_count: 0,
|
||||
sample_covers: []
|
||||
};
|
||||
|
||||
/**
|
||||
* Wire the full mock surface for the reader plus the new
|
||||
* `pages/:id/my-collections`, `pages/:id/my-tags`, `me/collections`,
|
||||
* and `collections/:id/pages` endpoints. `myCollectionsState` lets a
|
||||
* test mutate what /my-collections returns mid-flow so the "re-open
|
||||
* menu after add → In 1 collection" assertion is exercised.
|
||||
*/
|
||||
async function mockReader(
|
||||
page: Page,
|
||||
state: {
|
||||
collectionsContainingPage: string[];
|
||||
tagsOnPage: string[];
|
||||
admin?: boolean;
|
||||
analyzeCalls?: number;
|
||||
}
|
||||
) {
|
||||
await page.route('**/api/v1/auth/config', (route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ self_register_enabled: true, private_mode: false })
|
||||
})
|
||||
);
|
||||
await page.route('**/api/v1/auth/me', (route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({
|
||||
user: { ...userFixture, is_admin: state.admin ?? false }
|
||||
})
|
||||
})
|
||||
);
|
||||
// Admin force-analyze endpoint — counts invocations for assertions.
|
||||
await page.route(`**/api/v1/admin/pages/${pageId}/analyze`, (route) => {
|
||||
state.analyzeCalls = (state.analyzeCalls ?? 0) + 1;
|
||||
return route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ enqueued: true })
|
||||
});
|
||||
});
|
||||
await page.route('**/api/v1/auth/me/preferences', (route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({
|
||||
reader_mode: 'single',
|
||||
reader_page_gap: 'small'
|
||||
})
|
||||
})
|
||||
);
|
||||
await page.route('**/api/v1/me/bookmarks*', (route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ items: [], page: { limit: 50, offset: 0, total: 0 } })
|
||||
})
|
||||
);
|
||||
await page.route(`**/api/v1/mangas/${mangaId}`, (route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify(mangaFixture)
|
||||
})
|
||||
);
|
||||
await page.route(`**/api/v1/mangas/${mangaId}/chapters`, (route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({
|
||||
items: [chapterFixture],
|
||||
page: { limit: 50, offset: 0, total: 1 }
|
||||
})
|
||||
})
|
||||
);
|
||||
await page.route(`**/api/v1/mangas/${mangaId}/chapters\\?*`, (route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({
|
||||
items: [chapterFixture],
|
||||
page: { limit: 50, offset: 0, total: 1 }
|
||||
})
|
||||
})
|
||||
);
|
||||
await page.route(`**/api/v1/mangas/${mangaId}/chapters/${chapterId}`, (route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify(chapterFixture)
|
||||
})
|
||||
);
|
||||
await page.route(
|
||||
`**/api/v1/mangas/${mangaId}/chapters/${chapterId}/pages`,
|
||||
(route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ pages: pagesFixture })
|
||||
})
|
||||
);
|
||||
await page.route(`**/api/v1/me/read-progress/${mangaId}`, (route) =>
|
||||
route.fulfill({
|
||||
status: 404,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ error: { code: 'not_found', message: 'not found' } })
|
||||
})
|
||||
);
|
||||
|
||||
// The new endpoints — these are what the context menu hits.
|
||||
await page.route(`**/api/v1/pages/${pageId}/my-collections`, (route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ collection_ids: state.collectionsContainingPage })
|
||||
})
|
||||
);
|
||||
await page.route(`**/api/v1/pages/${pageId}/my-tags`, (route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ tags: state.tagsOnPage })
|
||||
})
|
||||
);
|
||||
await page.route('**/api/v1/me/collections*', (route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({
|
||||
items: [collectionFixture],
|
||||
page: { limit: 200, offset: 0, total: 1 }
|
||||
})
|
||||
})
|
||||
);
|
||||
await page.route(
|
||||
`**/api/v1/collections/${collectionId}/pages`,
|
||||
async (route) => {
|
||||
if (route.request().method() === 'POST') {
|
||||
state.collectionsContainingPage = [collectionId];
|
||||
return route.fulfill({ status: 201, body: '' });
|
||||
}
|
||||
return route.continue();
|
||||
}
|
||||
);
|
||||
|
||||
const png = Buffer.from(
|
||||
'89504e470d0a1a0a0000000d49484452000000010000000108060000001f15c4890000000d49444154789c63000100000005000158a3b62a0000000049454e44ae426082',
|
||||
'hex'
|
||||
);
|
||||
await page.route('**/api/v1/files/**', (route) =>
|
||||
route.fulfill({ status: 200, contentType: 'image/png', body: png })
|
||||
);
|
||||
}
|
||||
|
||||
test.describe('page context menu (desktop right-click)', () => {
|
||||
test('right-click opens the menu with empty-state hints', async ({ page }) => {
|
||||
const state = { collectionsContainingPage: [], tagsOnPage: [] };
|
||||
await mockReader(page, state);
|
||||
await page.setViewportSize(DESKTOP);
|
||||
await page.goto(`/manga/${mangaId}/chapter/${chapterId}`);
|
||||
|
||||
await expect(page.getByTestId('reader-page')).toBeVisible();
|
||||
await expect(page.getByTestId('page-context-menu')).toBeHidden();
|
||||
|
||||
await page.getByTestId('reader-page').click({ button: 'right' });
|
||||
|
||||
const menu = page.getByTestId('page-context-menu');
|
||||
await expect(menu).toBeVisible();
|
||||
await expect(
|
||||
page.getByTestId('page-context-collections-line')
|
||||
).toHaveText('Not in any collection');
|
||||
await expect(page.getByTestId('page-context-tags-line')).toHaveText(
|
||||
'No tags yet'
|
||||
);
|
||||
});
|
||||
|
||||
test('Escape closes the menu', async ({ page }) => {
|
||||
await mockReader(page, { collectionsContainingPage: [], tagsOnPage: [] });
|
||||
await page.setViewportSize(DESKTOP);
|
||||
await page.goto(`/manga/${mangaId}/chapter/${chapterId}`);
|
||||
|
||||
await page.getByTestId('reader-page').click({ button: 'right' });
|
||||
await expect(page.getByTestId('page-context-menu')).toBeVisible();
|
||||
|
||||
await page.keyboard.press('Escape');
|
||||
await expect(page.getByTestId('page-context-menu')).toBeHidden();
|
||||
});
|
||||
|
||||
test('add-to-collection → modal → toggle → re-open menu shows "In 1 collection"', async ({
|
||||
page
|
||||
}) => {
|
||||
const state = { collectionsContainingPage: [], tagsOnPage: [] };
|
||||
await mockReader(page, state);
|
||||
await page.setViewportSize(DESKTOP);
|
||||
await page.goto(`/manga/${mangaId}/chapter/${chapterId}`);
|
||||
|
||||
await page.getByTestId('reader-page').click({ button: 'right' });
|
||||
await page.getByTestId('page-context-add-to-collection').click();
|
||||
|
||||
// Modal opens with the user's one collection unchecked.
|
||||
const modal = page.getByTestId('add-to-collection-modal');
|
||||
await expect(modal).toBeVisible();
|
||||
const checkbox = modal.getByTestId(`collection-toggle-${collectionId}`);
|
||||
await expect(checkbox).not.toBeChecked();
|
||||
|
||||
await checkbox.check();
|
||||
// The mock flips containing-page state on POST. Close + re-open
|
||||
// the menu to verify the contextual line reflects the new
|
||||
// server state.
|
||||
await page.keyboard.press('Escape');
|
||||
await expect(modal).toBeHidden();
|
||||
|
||||
await page.getByTestId('reader-page').click({ button: 'right' });
|
||||
await expect(page.getByTestId('page-context-collections-line')).toHaveText(
|
||||
'In 1 collection'
|
||||
);
|
||||
});
|
||||
|
||||
test('non-admin does not see the Queue for analysis action', async ({ page }) => {
|
||||
await mockReader(page, { collectionsContainingPage: [], tagsOnPage: [] });
|
||||
await page.setViewportSize(DESKTOP);
|
||||
await page.goto(`/manga/${mangaId}/chapter/${chapterId}`);
|
||||
|
||||
await page.getByTestId('reader-page').click({ button: 'right' });
|
||||
await expect(page.getByTestId('page-context-menu')).toBeVisible();
|
||||
await expect(page.getByTestId('page-context-analyze')).toBeHidden();
|
||||
});
|
||||
|
||||
test('admin can queue the page for analysis from the menu', async ({ page }) => {
|
||||
const state = {
|
||||
collectionsContainingPage: [],
|
||||
tagsOnPage: [],
|
||||
admin: true,
|
||||
analyzeCalls: 0
|
||||
};
|
||||
await mockReader(page, state);
|
||||
await page.setViewportSize(DESKTOP);
|
||||
await page.goto(`/manga/${mangaId}/chapter/${chapterId}`);
|
||||
|
||||
await page.getByTestId('reader-page').click({ button: 'right' });
|
||||
const item = page.getByTestId('page-context-analyze');
|
||||
await expect(item).toBeVisible();
|
||||
await expect(item).toContainText('Queue for analysis');
|
||||
|
||||
await item.click();
|
||||
await expect(item).toContainText('Queued for analysis');
|
||||
expect(state.analyzeCalls).toBe(1);
|
||||
});
|
||||
|
||||
test('Shift+right-click falls through to the browser native menu', async ({
|
||||
page
|
||||
}) => {
|
||||
await mockReader(page, { collectionsContainingPage: [], tagsOnPage: [] });
|
||||
await page.setViewportSize(DESKTOP);
|
||||
await page.goto(`/manga/${mangaId}/chapter/${chapterId}`);
|
||||
|
||||
// Hold Shift while right-clicking. The reader's
|
||||
// `oncontextmenu` returns early on shiftKey, so the in-app
|
||||
// menu must NOT open. (Playwright doesn't surface the
|
||||
// native browser menu, but we can assert ours stays hidden.)
|
||||
await page.keyboard.down('Shift');
|
||||
await page.getByTestId('reader-page').click({ button: 'right' });
|
||||
await page.keyboard.up('Shift');
|
||||
|
||||
await expect(page.getByTestId('page-context-menu')).toBeHidden();
|
||||
});
|
||||
});
|
||||
|
||||
test.describe('page action sheet (mobile long-press)', () => {
|
||||
test('long-press on a page image opens the action sheet', async ({ page }) => {
|
||||
await mockReader(page, { collectionsContainingPage: [], tagsOnPage: [] });
|
||||
await page.setViewportSize(MOBILE);
|
||||
await page.goto(`/manga/${mangaId}/chapter/${chapterId}`);
|
||||
|
||||
// Switch to continuous mode so the per-image long-press
|
||||
// handler is wired (single mode goes through TapZone). Both
|
||||
// codepaths funnel into the same Sheet, but the per-image
|
||||
// wiring is the riskier of the two — it's the one the audit
|
||||
// flagged for the multitouch fix.
|
||||
await page.getByTestId('reader-settings-btn').click();
|
||||
await page
|
||||
.getByTestId('reader-settings-sheet')
|
||||
.getByRole('radio', { name: 'Continuous' })
|
||||
.click();
|
||||
// Close the settings sheet so its scrim isn't blocking.
|
||||
await page.keyboard.press('Escape');
|
||||
|
||||
const pageEl = page.getByTestId('reader-page-1');
|
||||
await expect(pageEl).toBeVisible();
|
||||
|
||||
// Synthesize a touch pointerdown, wait past the 450ms timer.
|
||||
const box = await pageEl.boundingBox();
|
||||
if (!box) throw new Error('page image has no bounding box');
|
||||
const cx = box.x + box.width / 2;
|
||||
const cy = box.y + box.height / 2;
|
||||
|
||||
await pageEl.dispatchEvent('pointerdown', {
|
||||
pointerType: 'touch',
|
||||
clientX: cx,
|
||||
clientY: cy,
|
||||
bubbles: true
|
||||
});
|
||||
await page.waitForTimeout(550);
|
||||
|
||||
await expect(page.getByTestId('page-action-sheet')).toBeVisible();
|
||||
await expect(page.getByTestId('page-action-add-to-collection')).toBeVisible();
|
||||
await expect(page.getByTestId('page-action-add-tag')).toBeVisible();
|
||||
await expect(page.getByTestId('page-action-save-image')).toBeVisible();
|
||||
await expect(page.getByTestId('page-action-copy-link')).toBeVisible();
|
||||
});
|
||||
});
|
||||
250
frontend/e2e/reader-page-deep-link.spec.ts
Normal file
250
frontend/e2e/reader-page-deep-link.spec.ts
Normal file
@@ -0,0 +1,250 @@
|
||||
import { test, expect, type Page } from '@playwright/test';
|
||||
|
||||
// E2E for the `?page=N` deep-link path in both reader modes. The
|
||||
// single-mode case has been working since v0.x; the continuous-mode
|
||||
// scroll-to-page landed in v0.62.0 alongside the /search Pages tab.
|
||||
|
||||
const mangaId = 'a9999999-9999-9999-9999-999999999999';
|
||||
const chapterId = 'c9999999-9999-9999-9999-999999999999';
|
||||
const chapterBId = 'c8888888-8888-8888-8888-888888888888';
|
||||
|
||||
const sixPages = Array.from({ length: 6 }, (_, i) => ({
|
||||
id: `p${i + 1}1111111-1111-1111-1111-111111111111`,
|
||||
chapter_id: chapterId,
|
||||
page_number: i + 1,
|
||||
storage_key: `mangas/${mangaId}/chapters/${chapterId}/pages/000${i + 1}.png`,
|
||||
content_type: 'image/png'
|
||||
}));
|
||||
|
||||
const fourPages = Array.from({ length: 4 }, (_, i) => ({
|
||||
id: `p${i + 1}2222222-2222-2222-2222-222222222222`,
|
||||
chapter_id: chapterBId,
|
||||
page_number: i + 1,
|
||||
storage_key: `mangas/${mangaId}/chapters/${chapterBId}/pages/000${i + 1}.png`,
|
||||
content_type: 'image/png'
|
||||
}));
|
||||
|
||||
const userFixture = {
|
||||
id: 'u11111111-1111-1111-1111-111111111111',
|
||||
username: 'tester',
|
||||
created_at: '2026-01-01T00:00:00Z',
|
||||
is_admin: false
|
||||
};
|
||||
|
||||
const mangaFixture = {
|
||||
id: mangaId,
|
||||
title: 'Berserk',
|
||||
status: 'ongoing',
|
||||
alt_titles: [],
|
||||
description: null,
|
||||
cover_image_path: `mangas/${mangaId}/cover.png`,
|
||||
created_at: '2026-01-01T00:00:00Z',
|
||||
updated_at: '2026-01-01T00:00:00Z',
|
||||
authors: [{ id: 'au1', name: 'Kentaro Miura' }],
|
||||
genres: [],
|
||||
tags: []
|
||||
};
|
||||
|
||||
const chapterFixture = {
|
||||
id: chapterId,
|
||||
manga_id: mangaId,
|
||||
number: 1,
|
||||
title: 'The Brand',
|
||||
page_count: sixPages.length,
|
||||
created_at: '2026-01-01T00:00:00Z'
|
||||
};
|
||||
|
||||
const chapterBFixture = {
|
||||
id: chapterBId,
|
||||
manga_id: mangaId,
|
||||
number: 2,
|
||||
title: 'Guardians of Desire',
|
||||
page_count: fourPages.length,
|
||||
created_at: '2026-01-02T00:00:00Z'
|
||||
};
|
||||
|
||||
async function mockReader(page: Page, mode: 'single' | 'continuous') {
|
||||
await page.route('**/api/v1/auth/config', (route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ self_register_enabled: true, private_mode: false })
|
||||
})
|
||||
);
|
||||
await page.route('**/api/v1/auth/me', (route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ user: userFixture })
|
||||
})
|
||||
);
|
||||
// Critical for this spec: the server-stored preference seeds
|
||||
// `preferences.readerMode` at hydration time. The new continuous-
|
||||
// mode scroll-to-page effect re-fires when `mode` flips from
|
||||
// its initial 'single' default to the hydrated value.
|
||||
await page.route('**/api/v1/auth/me/preferences', (route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ reader_mode: mode, reader_page_gap: 'small' })
|
||||
})
|
||||
);
|
||||
await page.route('**/api/v1/me/bookmarks*', (route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ items: [], page: { limit: 50, offset: 0, total: 0 } })
|
||||
})
|
||||
);
|
||||
await page.route(`**/api/v1/mangas/${mangaId}`, (route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify(mangaFixture)
|
||||
})
|
||||
);
|
||||
await page.route(`**/api/v1/mangas/${mangaId}/chapters*`, (route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({
|
||||
items: [chapterFixture, chapterBFixture],
|
||||
page: { limit: 50, offset: 0, total: 2 }
|
||||
})
|
||||
})
|
||||
);
|
||||
await page.route(`**/api/v1/mangas/${mangaId}/chapters/${chapterId}`, (route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify(chapterFixture)
|
||||
})
|
||||
);
|
||||
await page.route(`**/api/v1/mangas/${mangaId}/chapters/${chapterBId}`, (route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify(chapterBFixture)
|
||||
})
|
||||
);
|
||||
await page.route(
|
||||
`**/api/v1/mangas/${mangaId}/chapters/${chapterId}/pages`,
|
||||
(route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ pages: sixPages })
|
||||
})
|
||||
);
|
||||
await page.route(
|
||||
`**/api/v1/mangas/${mangaId}/chapters/${chapterBId}/pages`,
|
||||
(route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ pages: fourPages })
|
||||
})
|
||||
);
|
||||
await page.route(`**/api/v1/me/read-progress/${mangaId}`, (route) =>
|
||||
route.fulfill({
|
||||
status: 404,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ error: { code: 'not_found', message: 'not found' } })
|
||||
})
|
||||
);
|
||||
const png = Buffer.from(
|
||||
'89504e470d0a1a0a0000000d49484452000000010000000108060000001f15c4890000000d49444154789c63000100000005000158a3b62a0000000049454e44ae426082',
|
||||
'hex'
|
||||
);
|
||||
await page.route('**/api/v1/files/**', (route) =>
|
||||
route.fulfill({ status: 200, contentType: 'image/png', body: png })
|
||||
);
|
||||
}
|
||||
|
||||
test.describe('reader ?page=N deep link', () => {
|
||||
test('continuous mode: pages 1..N are eager-loaded so the scroll target has settled height', async ({
|
||||
page
|
||||
}) => {
|
||||
await mockReader(page, 'continuous');
|
||||
await page.goto(`/manga/${mangaId}/chapter/${chapterId}?page=4`);
|
||||
|
||||
await expect(page.getByTestId('reader-continuous')).toBeVisible();
|
||||
|
||||
// Pages 1..=4 (1-indexed in the testid, 0-indexed in the
|
||||
// template; initialIndex = 3 means we eager-load 0..=3, i.e.
|
||||
// testids 1..4). Without this guard, page 2+ would be lazy
|
||||
// and their 0×0 placeholders would let the scroll target
|
||||
// appear far above its final position.
|
||||
for (const n of [1, 2, 3, 4]) {
|
||||
await expect(page.getByTestId(`reader-page-${n}`)).toHaveAttribute(
|
||||
'loading',
|
||||
'eager'
|
||||
);
|
||||
}
|
||||
// Pages beyond the target stay lazy.
|
||||
for (const n of [5, 6]) {
|
||||
await expect(page.getByTestId(`reader-page-${n}`)).toHaveAttribute(
|
||||
'loading',
|
||||
'lazy'
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test('continuous mode: no ?page= eager-loads only the first two', async ({
|
||||
page
|
||||
}) => {
|
||||
await mockReader(page, 'continuous');
|
||||
await page.goto(`/manga/${mangaId}/chapter/${chapterId}`);
|
||||
|
||||
await expect(page.getByTestId('reader-continuous')).toBeVisible();
|
||||
await expect(page.getByTestId('reader-page-1')).toHaveAttribute(
|
||||
'loading',
|
||||
'eager'
|
||||
);
|
||||
await expect(page.getByTestId('reader-page-2')).toHaveAttribute(
|
||||
'loading',
|
||||
'eager'
|
||||
);
|
||||
await expect(page.getByTestId('reader-page-3')).toHaveAttribute(
|
||||
'loading',
|
||||
'lazy'
|
||||
);
|
||||
});
|
||||
|
||||
test('single mode: ?page=N opens at the requested page', async ({ page }) => {
|
||||
await mockReader(page, 'single');
|
||||
await page.goto(`/manga/${mangaId}/chapter/${chapterId}?page=5`);
|
||||
|
||||
await expect(page.getByTestId('reader-page')).toBeVisible();
|
||||
await expect(page.getByTestId('page-indicator')).toHaveText('Page 5 / 6');
|
||||
});
|
||||
|
||||
test('in-reader chapter selector resets state for the new chapter', async ({
|
||||
page
|
||||
}) => {
|
||||
// Deep-link into chapter A at page 5. SvelteKit reuses the
|
||||
// component when we navigate to chapter B via the chapter
|
||||
// selector, so `index` (and the read-progress sentinel)
|
||||
// have to be reset by the page's chapter-change effect.
|
||||
// Without it, page-indicator would read "Page 5 / 4" (old
|
||||
// index, new pages.length) and the next progress flush would
|
||||
// poison chapter B's stored read-progress with chapter A's
|
||||
// high-water mark.
|
||||
await mockReader(page, 'single');
|
||||
await page.goto(`/manga/${mangaId}/chapter/${chapterId}?page=5`);
|
||||
await expect(page.getByTestId('page-indicator')).toHaveText(
|
||||
'Page 5 / 6'
|
||||
);
|
||||
|
||||
await page
|
||||
.getByTestId('reader-chapter-select')
|
||||
.selectOption(chapterBId);
|
||||
|
||||
await expect(page).toHaveURL(
|
||||
new RegExp(`/manga/${mangaId}/chapter/${chapterBId}$`)
|
||||
);
|
||||
await expect(page.getByTestId('page-indicator')).toHaveText(
|
||||
'Page 1 / 4'
|
||||
);
|
||||
});
|
||||
});
|
||||
305
frontend/e2e/search.spec.ts
Normal file
305
frontend/e2e/search.spec.ts
Normal file
@@ -0,0 +1,305 @@
|
||||
import { test, expect, type Page } from '@playwright/test';
|
||||
|
||||
// E2E for the /search page shipped in v0.62.0. Five scenarios against
|
||||
// mocked endpoints — no backend needed.
|
||||
|
||||
const DESKTOP = { width: 1280, height: 720 } as const;
|
||||
|
||||
const userFixture = {
|
||||
id: 'u11111111-1111-1111-1111-111111111111',
|
||||
username: 'tester',
|
||||
created_at: '2026-01-01T00:00:00Z',
|
||||
is_admin: false
|
||||
};
|
||||
|
||||
const mangaId = 'a9999999-9999-9999-9999-999999999999';
|
||||
const chapterId = 'c9999999-9999-9999-9999-999999999999';
|
||||
const pageId = 'p11111111-1111-1111-1111-111111111111';
|
||||
|
||||
const distinct = [
|
||||
{ tag: 'funny', count: 38 },
|
||||
{ tag: 'fight', count: 24 }
|
||||
];
|
||||
|
||||
const pagesResponse = {
|
||||
items: [
|
||||
{
|
||||
tag: 'funny',
|
||||
page_id: pageId,
|
||||
chapter_id: chapterId,
|
||||
manga_id: mangaId,
|
||||
page_number: 5,
|
||||
chapter_number: 1,
|
||||
chapter_title: null,
|
||||
manga_title: 'Berserk',
|
||||
storage_key: `mangas/${mangaId}/chapters/${chapterId}/pages/0005.png`,
|
||||
tagged_at: '2026-01-01T00:00:00Z'
|
||||
}
|
||||
],
|
||||
page: { limit: 100, offset: 0, total: 1 }
|
||||
};
|
||||
|
||||
const chaptersDesc = {
|
||||
items: [
|
||||
{
|
||||
chapter_id: chapterId,
|
||||
manga_id: mangaId,
|
||||
manga_title: 'Berserk',
|
||||
chapter_number: 1,
|
||||
chapter_title: null,
|
||||
match_count: 12,
|
||||
sample_storage_keys: [
|
||||
`mangas/${mangaId}/chapters/${chapterId}/pages/0001.png`,
|
||||
`mangas/${mangaId}/chapters/${chapterId}/pages/0002.png`,
|
||||
`mangas/${mangaId}/chapters/${chapterId}/pages/0003.png`
|
||||
]
|
||||
},
|
||||
{
|
||||
chapter_id: 'c8888888-8888-8888-8888-888888888888',
|
||||
manga_id: mangaId,
|
||||
manga_title: 'Berserk',
|
||||
chapter_number: 2,
|
||||
chapter_title: null,
|
||||
match_count: 3,
|
||||
sample_storage_keys: []
|
||||
}
|
||||
],
|
||||
page: { limit: 100, offset: 0, total: 2 }
|
||||
};
|
||||
|
||||
// Same fixture reversed by the mock when `order=asc` is requested,
|
||||
// so the test can assert the order flip end-to-end.
|
||||
const chaptersAsc = {
|
||||
items: [...chaptersDesc.items].reverse(),
|
||||
page: chaptersDesc.page
|
||||
};
|
||||
|
||||
const mangasResponse = {
|
||||
items: [
|
||||
{
|
||||
manga_id: mangaId,
|
||||
manga_title: 'Berserk',
|
||||
manga_cover_image_path: `mangas/${mangaId}/cover.png`,
|
||||
match_count: 28,
|
||||
sample_storage_keys: [
|
||||
`mangas/${mangaId}/chapters/${chapterId}/pages/0005.png`
|
||||
]
|
||||
}
|
||||
],
|
||||
page: { limit: 100, offset: 0, total: 1 }
|
||||
};
|
||||
|
||||
const pageSearchResponse = {
|
||||
items: [
|
||||
{
|
||||
page_id: pageId,
|
||||
chapter_id: chapterId,
|
||||
manga_id: mangaId,
|
||||
page_number: 5,
|
||||
chapter_number: 1,
|
||||
chapter_title: null,
|
||||
manga_title: 'Berserk',
|
||||
storage_key: `mangas/${mangaId}/chapters/${chapterId}/pages/0005.png`,
|
||||
is_nsfw: true,
|
||||
content_warnings: ['gore'],
|
||||
rank: 0.9
|
||||
}
|
||||
],
|
||||
page: { limit: 100, offset: 0, total: 1 }
|
||||
};
|
||||
|
||||
async function mockSearch(page: Page) {
|
||||
await page.route('**/api/v1/auth/config', (route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ self_register_enabled: true, private_mode: false })
|
||||
})
|
||||
);
|
||||
await page.route('**/api/v1/auth/me', (route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ user: userFixture })
|
||||
})
|
||||
);
|
||||
await page.route('**/api/v1/auth/me/preferences', (route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ reader_mode: 'single', reader_page_gap: 'small' })
|
||||
})
|
||||
);
|
||||
await page.route('**/api/v1/me/bookmarks*', (route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ items: [], page: { limit: 50, offset: 0, total: 0 } })
|
||||
})
|
||||
);
|
||||
await page.route('**/api/v1/me/page-tags/distinct*', (route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ items: distinct })
|
||||
})
|
||||
);
|
||||
await page.route('**/api/v1/me/page-search*', (route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify(pageSearchResponse)
|
||||
})
|
||||
);
|
||||
await page.route('**/api/v1/me/page-tags?**', (route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify(pagesResponse)
|
||||
})
|
||||
);
|
||||
await page.route('**/api/v1/me/page-tags/chapters*', (route) => {
|
||||
const u = new URL(route.request().url());
|
||||
const body =
|
||||
u.searchParams.get('order') === 'asc' ? chaptersAsc : chaptersDesc;
|
||||
return route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify(body)
|
||||
});
|
||||
});
|
||||
await page.route('**/api/v1/me/page-tags/mangas*', (route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify(mangasResponse)
|
||||
})
|
||||
);
|
||||
|
||||
// PNG stub for fileUrl(storage_key) thumbnails.
|
||||
const png = Buffer.from(
|
||||
'89504e470d0a1a0a0000000d49484452000000010000000108060000001f15c4890000000d49444154789c63000100000005000158a3b62a0000000049454e44ae426082',
|
||||
'hex'
|
||||
);
|
||||
await page.route('**/api/v1/files/**', (route) =>
|
||||
route.fulfill({ status: 200, contentType: 'image/png', body: png })
|
||||
);
|
||||
}
|
||||
|
||||
test.describe('/search', () => {
|
||||
test('empty /search renders the chip cloud; clicking a chip sets ?tag=', async ({
|
||||
page
|
||||
}) => {
|
||||
await mockSearch(page);
|
||||
await page.setViewportSize(DESKTOP);
|
||||
await page.goto('/search');
|
||||
|
||||
await expect(page.getByTestId('search-chip-cloud')).toBeVisible();
|
||||
await expect(page.getByTestId('search-chip-funny')).toBeVisible();
|
||||
await expect(page.getByTestId('search-chip-fight')).toBeVisible();
|
||||
|
||||
await page.getByTestId('search-chip-funny').click();
|
||||
await expect(page).toHaveURL(/[?&]tag=funny/);
|
||||
await expect(page.getByTestId('search-active-tag')).toContainText('funny');
|
||||
});
|
||||
|
||||
test('Pages tab shows results; clicking a row navigates to reader at ?page=N', async ({
|
||||
page
|
||||
}) => {
|
||||
await mockSearch(page);
|
||||
await page.setViewportSize(DESKTOP);
|
||||
await page.goto('/search?tag=funny');
|
||||
|
||||
await expect(page.getByTestId('search-pages-list')).toBeVisible();
|
||||
const row = page.getByTestId(`search-page-row-${pageId}`);
|
||||
await expect(row).toBeVisible();
|
||||
|
||||
// The breadcrumb link goes to the reader at ?page=5.
|
||||
const breadcrumbLink = row.locator('a.target');
|
||||
await expect(breadcrumbLink).toHaveAttribute(
|
||||
'href',
|
||||
`/manga/${mangaId}/chapter/${chapterId}?page=5`
|
||||
);
|
||||
});
|
||||
|
||||
test('Chapters tab calls /chapters endpoint and renders ranked rows', async ({
|
||||
page
|
||||
}) => {
|
||||
await mockSearch(page);
|
||||
await page.setViewportSize(DESKTOP);
|
||||
await page.goto('/search?tag=funny&view=chapters');
|
||||
|
||||
await expect(page.getByTestId('search-chapters-list')).toBeVisible();
|
||||
// Default desc order — first row = highest match count (12).
|
||||
const rows = page.getByTestId('search-chapters-list').locator('li');
|
||||
await expect(rows.nth(0)).toContainText('12 pages');
|
||||
await expect(rows.nth(1)).toContainText('3 pages');
|
||||
|
||||
// Chapter row links to the reader at the chapter root.
|
||||
const firstTitle = rows.nth(0).locator('a.title').first();
|
||||
await expect(firstTitle).toHaveAttribute(
|
||||
'href',
|
||||
`/manga/${mangaId}/chapter/${chapterId}`
|
||||
);
|
||||
});
|
||||
|
||||
test('Order toggle flips chapter rows', async ({ page }) => {
|
||||
await mockSearch(page);
|
||||
await page.setViewportSize(DESKTOP);
|
||||
await page.goto('/search?tag=funny&view=chapters');
|
||||
|
||||
// Click the "Fewest pages" segmented control.
|
||||
await page
|
||||
.getByTestId('search-sort')
|
||||
.getByRole('radio', { name: 'Fewest pages' })
|
||||
.click();
|
||||
|
||||
await expect(page).toHaveURL(/[?&]order=asc/);
|
||||
const rows = page.getByTestId('search-chapters-list').locator('li');
|
||||
await expect(rows.nth(0)).toContainText('3 pages');
|
||||
await expect(rows.nth(1)).toContainText('12 pages');
|
||||
});
|
||||
|
||||
test('Mangas tab renders rows linking to manga detail', async ({ page }) => {
|
||||
await mockSearch(page);
|
||||
await page.setViewportSize(DESKTOP);
|
||||
await page.goto('/search?tag=funny&view=mangas');
|
||||
|
||||
await expect(page.getByTestId('search-mangas-list')).toBeVisible();
|
||||
const row = page.getByTestId(`search-manga-row-${mangaId}`);
|
||||
await expect(row).toContainText('Berserk');
|
||||
await expect(row).toContainText('28 pages');
|
||||
await expect(row.locator('a.title')).toHaveAttribute(
|
||||
'href',
|
||||
`/manga/${mangaId}`
|
||||
);
|
||||
});
|
||||
|
||||
test('text search sets ?text= and renders page results', async ({ page }) => {
|
||||
await mockSearch(page);
|
||||
await page.setViewportSize(DESKTOP);
|
||||
await page.goto('/search');
|
||||
|
||||
await page.getByTestId('search-text-input').fill('dragon');
|
||||
await page.getByTestId('search-text-input').press('Enter');
|
||||
|
||||
await expect(page).toHaveURL(/text=dragon/);
|
||||
await expect(page.getByTestId('search-content-list')).toBeVisible();
|
||||
await expect(
|
||||
page.getByTestId(`search-result-${pageId}`)
|
||||
).toBeVisible();
|
||||
});
|
||||
|
||||
test('content-warning toggle sets ?cw_include= and searches', async ({
|
||||
page
|
||||
}) => {
|
||||
await mockSearch(page);
|
||||
await page.setViewportSize(DESKTOP);
|
||||
await page.goto('/search');
|
||||
|
||||
await page.getByTestId('cw-toggle-gore').click();
|
||||
|
||||
await expect(page).toHaveURL(/cw_include=gore/);
|
||||
await expect(page.getByTestId('search-content-list')).toBeVisible();
|
||||
});
|
||||
});
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "mangalord-frontend",
|
||||
"version": "0.52.0",
|
||||
"version": "0.80.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
|
||||
@@ -3,7 +3,10 @@
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<link rel="icon" href="%sveltekit.assets%/favicon.ico" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<meta
|
||||
name="viewport"
|
||||
content="width=device-width, initial-scale=1, viewport-fit=cover"
|
||||
/>
|
||||
<title>Mangalord</title>
|
||||
<script>
|
||||
(function () {
|
||||
|
||||
@@ -7,7 +7,7 @@ import {
|
||||
afterEach,
|
||||
type MockInstance
|
||||
} from 'vitest';
|
||||
import { handle } from './hooks.server';
|
||||
import { handle, shouldBypassProxyTimeout } from './hooks.server';
|
||||
|
||||
// `BACKEND_URL` is read at module load time, so the values used in the
|
||||
// asserts below assume the test env didn't set it. `?? 'http://localhost:8080'`
|
||||
@@ -192,3 +192,37 @@ describe('hooks.server proxy', () => {
|
||||
expect(init.signal?.aborted).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// ----------------------------------------------------------------------------
|
||||
// T1 — SSE bypass for the wall-clock proxy timeout.
|
||||
// ----------------------------------------------------------------------------
|
||||
|
||||
describe('shouldBypassProxyTimeout', () => {
|
||||
it('returns true for an SSE Accept header', () => {
|
||||
const h = new Headers({ accept: 'text/event-stream' });
|
||||
expect(shouldBypassProxyTimeout(h)).toBe(true);
|
||||
});
|
||||
|
||||
it('matches case-insensitively', () => {
|
||||
const h = new Headers({ accept: 'TEXT/EVENT-STREAM' });
|
||||
expect(shouldBypassProxyTimeout(h)).toBe(true);
|
||||
});
|
||||
|
||||
it('matches when SSE is one of several Accept values', () => {
|
||||
const h = new Headers({ accept: 'text/event-stream, application/json' });
|
||||
expect(shouldBypassProxyTimeout(h)).toBe(true);
|
||||
});
|
||||
|
||||
it('returns false for plain JSON / HTML requests', () => {
|
||||
expect(
|
||||
shouldBypassProxyTimeout(new Headers({ accept: 'application/json' }))
|
||||
).toBe(false);
|
||||
expect(shouldBypassProxyTimeout(new Headers({ accept: 'text/html' }))).toBe(
|
||||
false
|
||||
);
|
||||
});
|
||||
|
||||
it('returns false when Accept is absent', () => {
|
||||
expect(shouldBypassProxyTimeout(new Headers())).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -46,6 +46,11 @@ const HOP_BY_HOP_HEADERS = [
|
||||
* tighter upstream proxy may want to lower it. A future improvement
|
||||
* is an idle-based timeout (reset per chunk) instead of this
|
||||
* wall-clock budget — that's a fair bit more code, deferred.
|
||||
*
|
||||
* SSE streams (`Accept: text/event-stream`) are exempt — see
|
||||
* `shouldBypassProxyTimeout`. A wall-clock abort on a long-lived
|
||||
* stream would tear the connection down every 5 min and force the
|
||||
* browser to reconnect, which flickers the admin dashboard.
|
||||
*/
|
||||
const PROXY_TIMEOUT_MS = (() => {
|
||||
const raw = process.env.BACKEND_PROXY_TIMEOUT_MS;
|
||||
@@ -53,6 +58,18 @@ const PROXY_TIMEOUT_MS = (() => {
|
||||
return Number.isFinite(n) && n > 0 ? n : 300_000;
|
||||
})();
|
||||
|
||||
/**
|
||||
* Whether the proxy should skip its wall-clock timeout for this
|
||||
* request. SSE clients open a single connection and stay subscribed
|
||||
* for the lifetime of the page; aborting on a wall-clock budget would
|
||||
* tear down the live admin dashboard every 5 min. Exported for unit
|
||||
* test coverage.
|
||||
*/
|
||||
export function shouldBypassProxyTimeout(headers: Headers): boolean {
|
||||
const accept = headers.get('accept') ?? '';
|
||||
return accept.toLowerCase().includes('text/event-stream');
|
||||
}
|
||||
|
||||
export const handle: Handle = async ({ event, resolve }) => {
|
||||
if (event.url.pathname.startsWith('/api/')) {
|
||||
const target = `${BACKEND_URL}${event.url.pathname}${event.url.search}`;
|
||||
@@ -63,9 +80,14 @@ export const handle: Handle = async ({ event, resolve }) => {
|
||||
// AbortController times the upstream fetch out so a backend
|
||||
// wedged on a slow DB query doesn't keep the browser request
|
||||
// hanging forever. The `signal` is also wired into the
|
||||
// RequestInit so the body stream is cancelled cleanly.
|
||||
// RequestInit so the body stream is cancelled cleanly. For
|
||||
// SSE streams the timer is suppressed so a long-lived stream
|
||||
// isn't torn down on the 5-minute mark — see T1 in the audit.
|
||||
const bypassTimeout = shouldBypassProxyTimeout(event.request.headers);
|
||||
const ctrl = new AbortController();
|
||||
const timeoutHandle = setTimeout(() => ctrl.abort(), PROXY_TIMEOUT_MS);
|
||||
const timeoutHandle = bypassTimeout
|
||||
? null
|
||||
: setTimeout(() => ctrl.abort(), PROXY_TIMEOUT_MS);
|
||||
|
||||
const init: RequestInit & { duplex?: 'half' } = {
|
||||
method: event.request.method,
|
||||
@@ -91,7 +113,7 @@ export const handle: Handle = async ({ event, resolve }) => {
|
||||
// the real cause. Emit the standard envelope with a
|
||||
// dedicated code instead.
|
||||
console.error('Proxy to backend failed:', e);
|
||||
clearTimeout(timeoutHandle);
|
||||
if (timeoutHandle) clearTimeout(timeoutHandle);
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
error: {
|
||||
@@ -106,7 +128,7 @@ export const handle: Handle = async ({ event, resolve }) => {
|
||||
);
|
||||
}
|
||||
|
||||
clearTimeout(timeoutHandle);
|
||||
if (timeoutHandle) clearTimeout(timeoutHandle);
|
||||
return new Response(upstream.body, {
|
||||
status: upstream.status,
|
||||
statusText: upstream.statusText,
|
||||
|
||||
@@ -16,7 +16,24 @@ import {
|
||||
listAdminChapters,
|
||||
getSystemStats,
|
||||
resyncManga,
|
||||
resyncChapter
|
||||
resyncChapter,
|
||||
getCrawlerStatus,
|
||||
crawlerStatusStreamUrl,
|
||||
runCrawlerPass,
|
||||
restartCrawlerBrowser,
|
||||
updateCrawlerSession,
|
||||
clearCrawlerSessionExpired,
|
||||
listDeadJobs,
|
||||
requeueDeadJobs,
|
||||
listActiveJobs,
|
||||
listMissingCovers,
|
||||
reenqueueAnalysis,
|
||||
analyzePage,
|
||||
getAnalysisMangaCoverage,
|
||||
getAnalysisChapterCoverage,
|
||||
getAnalysisChapterPages,
|
||||
getAnalysisPageDetail,
|
||||
analysisStatusStreamUrl
|
||||
} from './admin';
|
||||
|
||||
function ok(body: unknown, status = 200): Response {
|
||||
@@ -329,3 +346,245 @@ describe('admin api client', () => {
|
||||
expect(got.pages).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('admin crawler api client', () => {
|
||||
let fetchSpy: MockInstance<typeof globalThis.fetch>;
|
||||
beforeEach(() => {
|
||||
fetchSpy = vi.spyOn(globalThis, 'fetch');
|
||||
});
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
const statusFixture = {
|
||||
daemon: 'running',
|
||||
phase: { state: 'fetching_metadata', index: 3, total: 10, title: 'One Piece' },
|
||||
worker_count: 2,
|
||||
active_chapters: [
|
||||
{
|
||||
manga_id: 'm-1',
|
||||
manga_title: 'Bleach',
|
||||
chapter_id: 'c-1',
|
||||
chapter_number: 12,
|
||||
pages_done: 4,
|
||||
pages_total: 20
|
||||
}
|
||||
],
|
||||
current_cover: { manga_id: 'm-2', manga_title: 'Naruto' },
|
||||
covers_queued: 7,
|
||||
last_pass: { at: null, discovered: 0, upserted: 0, covers_fetched: 0, mangas_failed: 0 },
|
||||
session: { expired: false, configured: true },
|
||||
browser: 'healthy',
|
||||
queue: { pending: 2, running: 1, dead: 4 }
|
||||
};
|
||||
|
||||
it('crawlerStatusStreamUrl points at the SSE endpoint under the API base', () => {
|
||||
expect(crawlerStatusStreamUrl()).toMatch(/\/v1\/admin\/crawler\/stream$/);
|
||||
});
|
||||
|
||||
it('getCrawlerStatus GETs /v1/admin/crawler with live chapter/cover fields', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(ok(statusFixture));
|
||||
const s = await getCrawlerStatus();
|
||||
expect(s.queue.dead).toBe(4);
|
||||
expect(s.phase?.state).toBe('fetching_metadata');
|
||||
expect(s.active_chapters[0].pages_done).toBe(4);
|
||||
expect(s.active_chapters[0].pages_total).toBe(20);
|
||||
expect(s.current_cover?.manga_title).toBe('Naruto');
|
||||
expect(s.covers_queued).toBe(7);
|
||||
const url = fetchSpy.mock.calls[0][0] as string;
|
||||
expect(url).toMatch(/\/v1\/admin\/crawler$/);
|
||||
});
|
||||
|
||||
it('listActiveJobs GETs /v1/admin/crawler/active-jobs with search', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(
|
||||
ok({ items: [], page: { limit: 20, offset: 0, total: 0 } })
|
||||
);
|
||||
await listActiveJobs({ search: 'bleach' });
|
||||
const url = fetchSpy.mock.calls[0][0] as string;
|
||||
expect(url).toMatch(/\/v1\/admin\/crawler\/active-jobs\?/);
|
||||
expect(url).toContain('search=bleach');
|
||||
});
|
||||
|
||||
it('listMissingCovers GETs /v1/admin/crawler/covers', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(
|
||||
ok({ items: [{ manga_id: 'm-1', manga_title: 'X' }], page: { limit: 20, offset: 0, total: 1 } })
|
||||
);
|
||||
const r = await listMissingCovers();
|
||||
expect(r.items[0].manga_title).toBe('X');
|
||||
expect(fetchSpy.mock.calls[0][0]).toMatch(/\/v1\/admin\/crawler\/covers$/);
|
||||
});
|
||||
|
||||
it('runCrawlerPass POSTs /v1/admin/crawler/run', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(ok({ started: true }));
|
||||
const r = await runCrawlerPass();
|
||||
expect(r.started).toBe(true);
|
||||
const init = fetchSpy.mock.calls[0][1] as RequestInit;
|
||||
expect(init.method).toBe('POST');
|
||||
expect(fetchSpy.mock.calls[0][0]).toMatch(/\/v1\/admin\/crawler\/run$/);
|
||||
});
|
||||
|
||||
it('restartCrawlerBrowser POSTs the restart endpoint', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(ok({ ok: true, error: null }));
|
||||
const r = await restartCrawlerBrowser();
|
||||
expect(r.ok).toBe(true);
|
||||
expect(fetchSpy.mock.calls[0][0]).toMatch(/\/v1\/admin\/crawler\/browser\/restart$/);
|
||||
});
|
||||
|
||||
it('updateCrawlerSession POSTs the phpsessid body', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(ok({ valid: true, error: null }));
|
||||
const r = await updateCrawlerSession('abc123');
|
||||
expect(r.valid).toBe(true);
|
||||
const init = fetchSpy.mock.calls[0][1] as RequestInit;
|
||||
expect(init.method).toBe('POST');
|
||||
expect(JSON.parse(init.body as string)).toEqual({ phpsessid: 'abc123' });
|
||||
});
|
||||
|
||||
it('clearCrawlerSessionExpired POSTs clear-expired', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(ok({ cleared: true }));
|
||||
const r = await clearCrawlerSessionExpired();
|
||||
expect(r.cleared).toBe(true);
|
||||
expect(fetchSpy.mock.calls[0][0]).toMatch(/\/v1\/admin\/crawler\/session\/clear-expired$/);
|
||||
});
|
||||
|
||||
it('listDeadJobs forwards search + pagination', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(
|
||||
ok({ items: [], page: { limit: 20, offset: 20, total: 0 } })
|
||||
);
|
||||
await listDeadJobs({ search: 'naruto', limit: 20, offset: 20 });
|
||||
const url = fetchSpy.mock.calls[0][0] as string;
|
||||
expect(url).toContain('search=naruto');
|
||||
expect(url).toContain('offset=20');
|
||||
});
|
||||
|
||||
it('requeueDeadJobs POSTs the scope body', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(ok({ requeued: 3 }));
|
||||
const r = await requeueDeadJobs({ scope: 'manga', manga_id: 'm-9' });
|
||||
expect(r.requeued).toBe(3);
|
||||
const init = fetchSpy.mock.calls[0][1] as RequestInit;
|
||||
expect(JSON.parse(init.body as string)).toEqual({ scope: 'manga', manga_id: 'm-9' });
|
||||
});
|
||||
|
||||
it('requeueDeadJobs serialises chapter scope verbatim', async () => {
|
||||
// F7 — the per-chapter requeue button in /admin/mangas sends this
|
||||
// exact shape; pin it so a future rename of `chapter_id` doesn't
|
||||
// silently break the inline button.
|
||||
fetchSpy.mockResolvedValueOnce(ok({ requeued: 1 }));
|
||||
const r = await requeueDeadJobs({ scope: 'chapter', chapter_id: 'c-7' });
|
||||
expect(r.requeued).toBe(1);
|
||||
const init = fetchSpy.mock.calls[0][1] as RequestInit;
|
||||
expect(JSON.parse(init.body as string)).toEqual({
|
||||
scope: 'chapter',
|
||||
chapter_id: 'c-7'
|
||||
});
|
||||
});
|
||||
|
||||
it('requeueDeadJobs serialises job + all (with confirm) scopes', async () => {
|
||||
// Round out the variant coverage: { scope: 'job', job_id } and
|
||||
// { scope: 'all', confirm: true }. The audit (S1) requires the
|
||||
// confirm flag on the wire for scope=all; this pins it so a
|
||||
// future refactor can't drop it.
|
||||
fetchSpy.mockResolvedValueOnce(ok({ requeued: 1 }));
|
||||
await requeueDeadJobs({ scope: 'job', job_id: 'j-1' });
|
||||
expect(JSON.parse(fetchSpy.mock.calls[0][1]!.body as string)).toEqual({
|
||||
scope: 'job',
|
||||
job_id: 'j-1'
|
||||
});
|
||||
fetchSpy.mockResolvedValueOnce(ok({ requeued: 99 }));
|
||||
await requeueDeadJobs({ scope: 'all', confirm: true });
|
||||
expect(JSON.parse(fetchSpy.mock.calls[1][1]!.body as string)).toEqual({
|
||||
scope: 'all',
|
||||
confirm: true
|
||||
});
|
||||
});
|
||||
|
||||
it('surfaces a 503 as ApiError', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(envelope(503, 'service_unavailable', 'disabled'));
|
||||
await expect(runCrawlerPass()).rejects.toMatchObject({ status: 503 });
|
||||
});
|
||||
|
||||
it('reenqueueAnalysis defaults to whole-library, exclude-analyzed', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(ok({ enqueued: 42 }));
|
||||
const r = await reenqueueAnalysis();
|
||||
expect(r.enqueued).toBe(42);
|
||||
const url = fetchSpy.mock.calls[0][0] as string;
|
||||
expect(url).toMatch(/\/v1\/admin\/analysis\/reenqueue$/);
|
||||
expect(JSON.parse(fetchSpy.mock.calls[0][1]!.body as string)).toEqual({
|
||||
only_unanalyzed: true
|
||||
});
|
||||
});
|
||||
|
||||
it('reenqueueAnalysis scopes to a manga and can include analyzed', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(ok({ enqueued: 7 }));
|
||||
await reenqueueAnalysis({ mangaId: 'm1', onlyUnanalyzed: false });
|
||||
expect(JSON.parse(fetchSpy.mock.calls[0][1]!.body as string)).toEqual({
|
||||
only_unanalyzed: false,
|
||||
manga_id: 'm1'
|
||||
});
|
||||
});
|
||||
|
||||
it('reenqueueAnalysis scopes to a chapter', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(ok({ enqueued: 3 }));
|
||||
await reenqueueAnalysis({ chapterId: 'c1' });
|
||||
expect(JSON.parse(fetchSpy.mock.calls[0][1]!.body as string)).toEqual({
|
||||
only_unanalyzed: true,
|
||||
chapter_id: 'c1'
|
||||
});
|
||||
});
|
||||
|
||||
it('analyzePage posts to the force-analyze endpoint', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(ok({ enqueued: true }));
|
||||
const r = await analyzePage('p1');
|
||||
expect(r.enqueued).toBe(true);
|
||||
const url = fetchSpy.mock.calls[0][0] as string;
|
||||
expect(url).toMatch(/\/v1\/admin\/pages\/p1\/analyze$/);
|
||||
expect(fetchSpy.mock.calls[0][1]!.method).toBe('POST');
|
||||
});
|
||||
|
||||
it('getAnalysisMangaCoverage passes search + pagination', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(
|
||||
ok({ items: [], page: { limit: 25, offset: 0, total: 0 } })
|
||||
);
|
||||
await getAnalysisMangaCoverage({ search: 'ber', limit: 25, offset: 50 });
|
||||
const url = fetchSpy.mock.calls[0][0] as string;
|
||||
expect(url).toContain('/v1/admin/analysis/mangas?');
|
||||
expect(url).toContain('search=ber');
|
||||
expect(url).toContain('limit=25');
|
||||
expect(url).toContain('offset=50');
|
||||
});
|
||||
|
||||
it('getAnalysisChapterCoverage unwraps items', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(
|
||||
ok({ items: [{ chapter_id: 'c1', number: 1, title: null, total_pages: 2, analyzed_pages: 1 }] })
|
||||
);
|
||||
const items = await getAnalysisChapterCoverage('m1');
|
||||
const url = fetchSpy.mock.calls[0][0] as string;
|
||||
expect(url).toMatch(/\/v1\/admin\/analysis\/mangas\/m1\/chapters$/);
|
||||
expect(items).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('getAnalysisChapterPages unwraps items', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(
|
||||
ok({ items: [{ page_id: 'p1', page_number: 1, status: 'done' }] })
|
||||
);
|
||||
const items = await getAnalysisChapterPages('c1');
|
||||
const url = fetchSpy.mock.calls[0][0] as string;
|
||||
expect(url).toMatch(/\/v1\/admin\/analysis\/chapters\/c1\/pages$/);
|
||||
expect(items[0].status).toBe('done');
|
||||
});
|
||||
|
||||
it('getAnalysisPageDetail hits the page endpoint', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(
|
||||
ok({ page_id: 'p1', status: 'none', ocr: [], tags: [], content_warnings: [] })
|
||||
);
|
||||
const d = await getAnalysisPageDetail('p1');
|
||||
const url = fetchSpy.mock.calls[0][0] as string;
|
||||
expect(url).toMatch(/\/v1\/admin\/analysis\/pages\/p1$/);
|
||||
expect(d.status).toBe('none');
|
||||
});
|
||||
|
||||
it('analysisStatusStreamUrl points at the SSE endpoint', () => {
|
||||
expect(analysisStatusStreamUrl()).toMatch(
|
||||
/\/v1\/admin\/analysis\/status\/stream$/
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -3,10 +3,11 @@
|
||||
// won't reach these routes). 403s thrown here propagate up to the
|
||||
// /admin layout, which renders the framework error page.
|
||||
|
||||
import { request, type Page } from './client';
|
||||
import { request, apiUrl, type Page } from './client';
|
||||
import type { User } from './auth';
|
||||
import type { MangaDetail } from './mangas';
|
||||
import type { Chapter } from './chapters';
|
||||
import type { ContentWarning } from './page_tags';
|
||||
|
||||
// ---- users -----------------------------------------------------------------
|
||||
|
||||
@@ -214,3 +215,451 @@ export async function resyncChapter(id: string): Promise<ChapterResyncResponse>
|
||||
{ method: 'POST' }
|
||||
);
|
||||
}
|
||||
|
||||
// ---- crawler observability + control ---------------------------------------
|
||||
|
||||
/** Current daemon activity. Discriminated on `state`. */
|
||||
export type CrawlerPhase =
|
||||
| { state: 'idle'; next_fire: string | null }
|
||||
| { state: 'walking_list' }
|
||||
| { state: 'fetching_metadata'; index: number; total: number | null; title: string }
|
||||
| { state: 'cover_backfill'; index: number; total: number };
|
||||
|
||||
/** A chapter being crawled right now, with a live page count. */
|
||||
export type ActiveChapter = {
|
||||
manga_id: string;
|
||||
manga_title: string;
|
||||
chapter_id: string;
|
||||
chapter_number: number;
|
||||
pages_done: number;
|
||||
pages_total: number | null;
|
||||
};
|
||||
|
||||
export type CrawlerLastPass = {
|
||||
at: string | null;
|
||||
discovered: number;
|
||||
upserted: number;
|
||||
covers_fetched: number;
|
||||
mangas_failed: number;
|
||||
};
|
||||
|
||||
export type CrawlerStatus = {
|
||||
daemon: 'running' | 'disabled';
|
||||
phase: CrawlerPhase | null;
|
||||
worker_count: number;
|
||||
active_chapters: ActiveChapter[];
|
||||
current_cover: { manga_id: string; manga_title: string } | null;
|
||||
covers_queued: number;
|
||||
last_pass: CrawlerLastPass;
|
||||
session: { expired: boolean; configured: boolean };
|
||||
browser: 'healthy' | 'draining' | 'restarting' | 'down';
|
||||
queue: { pending: number; running: number; dead: number };
|
||||
};
|
||||
|
||||
export async function getCrawlerStatus(): Promise<CrawlerStatus> {
|
||||
return request<CrawlerStatus>('/v1/admin/crawler');
|
||||
}
|
||||
|
||||
/** URL of the Server-Sent Events live-status stream. Open with
|
||||
* `new EventSource(...)` while the crawler page is mounted and close it on
|
||||
* navigate-away so the subscription is scoped to the active page. Each
|
||||
* message is a named `status` event whose `data` is a {@link CrawlerStatus}. */
|
||||
export function crawlerStatusStreamUrl(): string {
|
||||
return apiUrl('/v1/admin/crawler/stream');
|
||||
}
|
||||
|
||||
/** POST /v1/admin/crawler/run — trigger an out-of-cycle metadata pass. */
|
||||
export async function runCrawlerPass(): Promise<{ started: boolean }> {
|
||||
return request('/v1/admin/crawler/run', { method: 'POST' });
|
||||
}
|
||||
|
||||
/** POST /v1/admin/crawler/browser/restart — coordinated Chromium restart. */
|
||||
export async function restartCrawlerBrowser(): Promise<{ ok: boolean; error: string | null }> {
|
||||
return request('/v1/admin/crawler/browser/restart', { method: 'POST' });
|
||||
}
|
||||
|
||||
/** POST /v1/admin/crawler/session — refresh PHPSESSID and re-probe. */
|
||||
export async function updateCrawlerSession(
|
||||
phpsessid: string
|
||||
): Promise<{ valid: boolean; error: string | null }> {
|
||||
return request('/v1/admin/crawler/session', {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify({ phpsessid })
|
||||
});
|
||||
}
|
||||
|
||||
/** POST /v1/admin/crawler/session/clear-expired — resume idled workers. */
|
||||
export async function clearCrawlerSessionExpired(): Promise<{ cleared: boolean }> {
|
||||
return request('/v1/admin/crawler/session/clear-expired', { method: 'POST' });
|
||||
}
|
||||
|
||||
export type DeadJob = {
|
||||
id: string;
|
||||
kind: string;
|
||||
chapter_id: string | null;
|
||||
manga_id: string | null;
|
||||
manga_title: string | null;
|
||||
chapter_number: number | null;
|
||||
attempts: number;
|
||||
max_attempts: number;
|
||||
last_error: string | null;
|
||||
updated_at: string;
|
||||
};
|
||||
|
||||
export type DeadJobsPage = { items: DeadJob[]; page: Page };
|
||||
|
||||
export async function listDeadJobs(
|
||||
opts?: {
|
||||
search?: string;
|
||||
limit?: number;
|
||||
offset?: number;
|
||||
},
|
||||
init?: RequestInit
|
||||
): Promise<DeadJobsPage> {
|
||||
const params = new URLSearchParams();
|
||||
if (opts?.search) params.set('search', opts.search);
|
||||
if (opts?.limit != null) params.set('limit', String(opts.limit));
|
||||
if (opts?.offset != null) params.set('offset', String(opts.offset));
|
||||
const qs = params.toString();
|
||||
return request<DeadJobsPage>(
|
||||
`/v1/admin/crawler/dead-jobs${qs ? `?${qs}` : ''}`,
|
||||
init
|
||||
);
|
||||
}
|
||||
|
||||
/** Requeue scope: all dead jobs, one manga's, one chapter's, or a single job.
|
||||
* `scope: 'all'` requires an explicit `confirm: true` so a careless
|
||||
* click (or CSRF bait) can't flip the entire dead pile. */
|
||||
export type RequeueScope =
|
||||
| { scope: 'all'; confirm: true }
|
||||
| { scope: 'manga'; manga_id: string }
|
||||
| { scope: 'chapter'; chapter_id: string }
|
||||
| { scope: 'job'; job_id: string };
|
||||
|
||||
export async function requeueDeadJobs(scope: RequeueScope): Promise<{ requeued: number }> {
|
||||
return request('/v1/admin/crawler/dead-jobs/requeue', {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify(scope)
|
||||
});
|
||||
}
|
||||
|
||||
/** A queued/running chapter-content job (which chapters are queued). */
|
||||
export type ActiveJob = {
|
||||
id: string;
|
||||
chapter_id: string | null;
|
||||
manga_id: string | null;
|
||||
manga_title: string | null;
|
||||
chapter_number: number | null;
|
||||
state: 'pending' | 'running';
|
||||
attempts: number;
|
||||
max_attempts: number;
|
||||
updated_at: string;
|
||||
};
|
||||
|
||||
export type ActiveJobsPage = { items: ActiveJob[]; page: Page };
|
||||
|
||||
/** GET /v1/admin/crawler/active-jobs — which chapters of which mangas are
|
||||
* queued or running now. */
|
||||
export async function listActiveJobs(
|
||||
opts?: {
|
||||
search?: string;
|
||||
limit?: number;
|
||||
offset?: number;
|
||||
},
|
||||
init?: RequestInit
|
||||
): Promise<ActiveJobsPage> {
|
||||
const params = new URLSearchParams();
|
||||
if (opts?.search) params.set('search', opts.search);
|
||||
if (opts?.limit != null) params.set('limit', String(opts.limit));
|
||||
if (opts?.offset != null) params.set('offset', String(opts.offset));
|
||||
const qs = params.toString();
|
||||
return request<ActiveJobsPage>(
|
||||
`/v1/admin/crawler/active-jobs${qs ? `?${qs}` : ''}`,
|
||||
init
|
||||
);
|
||||
}
|
||||
|
||||
/** A manga queued for a cover fetch (no cover yet + a live source). */
|
||||
export type MissingCover = { manga_id: string; manga_title: string };
|
||||
export type MissingCoversPage = { items: MissingCover[]; page: Page };
|
||||
|
||||
/** GET /v1/admin/crawler/covers — which manga covers are queued. */
|
||||
export async function listMissingCovers(
|
||||
opts?: {
|
||||
search?: string;
|
||||
limit?: number;
|
||||
offset?: number;
|
||||
},
|
||||
init?: RequestInit
|
||||
): Promise<MissingCoversPage> {
|
||||
const params = new URLSearchParams();
|
||||
if (opts?.search) params.set('search', opts.search);
|
||||
if (opts?.limit != null) params.set('limit', String(opts.limit));
|
||||
if (opts?.offset != null) params.set('offset', String(opts.offset));
|
||||
const qs = params.toString();
|
||||
return request<MissingCoversPage>(
|
||||
`/v1/admin/crawler/covers${qs ? `?${qs}` : ''}`,
|
||||
init
|
||||
);
|
||||
}
|
||||
|
||||
// ---- AI content analysis ---------------------------------------------------
|
||||
|
||||
/** Options for the scoped analysis re-enqueue. `mangaId` and `chapterId`
|
||||
* are mutually exclusive; omit both to target the whole library. */
|
||||
export type ReenqueueAnalysisOptions = {
|
||||
/** Skip pages that already have a completed analysis (default true).
|
||||
* When false, in-scope pages are force re-analyzed. */
|
||||
onlyUnanalyzed?: boolean;
|
||||
mangaId?: string;
|
||||
chapterId?: string;
|
||||
};
|
||||
|
||||
/** Bulk-enqueue `analyze_page` jobs for all / a manga's / a chapter's
|
||||
* pages. Returns how many jobs were enqueued. Requires ANALYSIS_ENABLED
|
||||
* on the backend (503 otherwise). */
|
||||
export async function reenqueueAnalysis(
|
||||
opts: ReenqueueAnalysisOptions = {}
|
||||
): Promise<{ enqueued: number }> {
|
||||
const body: Record<string, unknown> = {
|
||||
only_unanalyzed: opts.onlyUnanalyzed ?? true
|
||||
};
|
||||
if (opts.mangaId) body.manga_id = opts.mangaId;
|
||||
if (opts.chapterId) body.chapter_id = opts.chapterId;
|
||||
return request<{ enqueued: number }>('/v1/admin/analysis/reenqueue', {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify(body)
|
||||
});
|
||||
}
|
||||
|
||||
/** Force re-analysis of a single page (used by the reader context menu). */
|
||||
export async function analyzePage(pageId: string): Promise<{ enqueued: boolean }> {
|
||||
return request<{ enqueued: boolean }>(
|
||||
`/v1/admin/pages/${encodeURIComponent(pageId)}/analyze`,
|
||||
{ method: 'POST' }
|
||||
);
|
||||
}
|
||||
|
||||
// ---- AI content analysis: coverage & inspection ----------------------------
|
||||
|
||||
export type MangaCoverage = {
|
||||
manga_id: string;
|
||||
title: string;
|
||||
total_pages: number;
|
||||
analyzed_pages: number;
|
||||
};
|
||||
|
||||
export type MangaCoveragePage = {
|
||||
items: MangaCoverage[];
|
||||
page: Page;
|
||||
};
|
||||
|
||||
export type ChapterCoverage = {
|
||||
chapter_id: string;
|
||||
number: number;
|
||||
title: string | null;
|
||||
total_pages: number;
|
||||
analyzed_pages: number;
|
||||
};
|
||||
|
||||
/** Per-page status in a chapter grid. */
|
||||
export type PageAnalysisStatus = 'done' | 'failed' | 'queued' | 'none';
|
||||
|
||||
export type PageStatusItem = {
|
||||
page_id: string;
|
||||
page_number: number;
|
||||
status: PageAnalysisStatus;
|
||||
};
|
||||
|
||||
export type OcrLine = { kind: string; text: string };
|
||||
|
||||
export type PageAnalysisDetail = {
|
||||
page_id: string;
|
||||
page_number: number;
|
||||
chapter_id: string;
|
||||
manga_id: string;
|
||||
/** `done` | `failed` | `none` (no analysis row yet). */
|
||||
status: 'done' | 'failed' | 'none';
|
||||
is_nsfw: boolean;
|
||||
scene_description: string | null;
|
||||
model: string | null;
|
||||
error: string | null;
|
||||
analyzed_at: string | null;
|
||||
ocr: OcrLine[];
|
||||
tags: string[];
|
||||
content_warnings: ContentWarning[];
|
||||
};
|
||||
|
||||
/** Paginated per-manga analysis coverage (admin overview). */
|
||||
export async function getAnalysisMangaCoverage(
|
||||
opts: { search?: string; limit?: number; offset?: number } = {}
|
||||
): Promise<MangaCoveragePage> {
|
||||
const params = new URLSearchParams();
|
||||
if (opts.search) params.set('search', opts.search);
|
||||
if (opts.limit != null) params.set('limit', String(opts.limit));
|
||||
if (opts.offset != null) params.set('offset', String(opts.offset));
|
||||
const qs = params.toString();
|
||||
return request<MangaCoveragePage>(
|
||||
`/v1/admin/analysis/mangas${qs ? `?${qs}` : ''}`
|
||||
);
|
||||
}
|
||||
|
||||
export async function getAnalysisChapterCoverage(
|
||||
mangaId: string
|
||||
): Promise<ChapterCoverage[]> {
|
||||
const r = await request<{ items: ChapterCoverage[] }>(
|
||||
`/v1/admin/analysis/mangas/${encodeURIComponent(mangaId)}/chapters`
|
||||
);
|
||||
return r.items;
|
||||
}
|
||||
|
||||
export async function getAnalysisChapterPages(
|
||||
chapterId: string
|
||||
): Promise<PageStatusItem[]> {
|
||||
const r = await request<{ items: PageStatusItem[] }>(
|
||||
`/v1/admin/analysis/chapters/${encodeURIComponent(chapterId)}/pages`
|
||||
);
|
||||
return r.items;
|
||||
}
|
||||
|
||||
export async function getAnalysisPageDetail(
|
||||
pageId: string
|
||||
): Promise<PageAnalysisDetail> {
|
||||
return request<PageAnalysisDetail>(
|
||||
`/v1/admin/analysis/pages/${encodeURIComponent(pageId)}`
|
||||
);
|
||||
}
|
||||
|
||||
/** One live analysis event from the SSE stream. */
|
||||
export type AnalysisEvent =
|
||||
| {
|
||||
kind: 'enqueued';
|
||||
count: number;
|
||||
manga_id: string | null;
|
||||
chapter_id: string | null;
|
||||
}
|
||||
| {
|
||||
kind: 'started' | 'completed' | 'failed';
|
||||
page_id: string;
|
||||
manga_id: string;
|
||||
chapter_id: string;
|
||||
page_number: number;
|
||||
};
|
||||
|
||||
/** URL of the live analysis SSE stream. Open with `new EventSource(...)`
|
||||
* while the admin Analysis page is mounted and close it on navigate-away.
|
||||
* Each message is a named `analysis` event whose `data` is an
|
||||
* {@link AnalysisEvent}; a `lagged` event signals dropped frames. */
|
||||
export function analysisStatusStreamUrl(): string {
|
||||
return apiUrl('/v1/admin/analysis/status/stream');
|
||||
}
|
||||
|
||||
// ---- runtime settings (crawler + analysis) ---------------------------------
|
||||
|
||||
/** Operationally-safe, admin-editable crawler settings. Host/infra and
|
||||
* session/secret fields are managed via environment and surfaced read-only
|
||||
* in {@link CrawlerEnvOnly}. */
|
||||
export type CrawlerSettings = {
|
||||
daemon_enabled: boolean;
|
||||
daily_at: string; // "HH:MM"
|
||||
tz: string; // IANA
|
||||
idle_timeout_secs: number;
|
||||
chapter_workers: number;
|
||||
retention_days: number;
|
||||
start_url: string | null;
|
||||
rate_ms: number;
|
||||
cdn_host: string | null;
|
||||
cdn_rate_ms: number;
|
||||
cookie_domain: string | null;
|
||||
user_agent: string | null;
|
||||
download_allowlist: string[];
|
||||
allow_any_host: boolean;
|
||||
max_image_bytes: number;
|
||||
manga_limit: number;
|
||||
job_timeout_secs: number;
|
||||
metadata_max_consecutive_failures: number;
|
||||
browser_restart_threshold: number;
|
||||
};
|
||||
|
||||
/** Read-only view of crawler fields managed via environment. */
|
||||
export type CrawlerEnvOnly = {
|
||||
browser_mode: string;
|
||||
browser_args: string[];
|
||||
proxy: string | null;
|
||||
tor_control_url: string | null;
|
||||
tor_credentials_configured: boolean;
|
||||
session_configured: boolean;
|
||||
};
|
||||
|
||||
export type CrawlerSettingsResponse = {
|
||||
editable: CrawlerSettings;
|
||||
env_only: CrawlerEnvOnly;
|
||||
};
|
||||
|
||||
/** Admin-editable analysis settings. Prompts are `null` to mean "use the
|
||||
* compiled default" (see {@link PromptDefaults}). The vision API key is
|
||||
* env-only and never returned. */
|
||||
export type AnalysisSettings = {
|
||||
enabled: boolean;
|
||||
workers: number;
|
||||
endpoint: string;
|
||||
model: string;
|
||||
request_timeout_secs: number;
|
||||
job_timeout_secs: number;
|
||||
max_tokens: number;
|
||||
max_pixels: number;
|
||||
min_slice_height: number;
|
||||
slice_overlap: number;
|
||||
tall_aspect_threshold: number;
|
||||
max_slices: number;
|
||||
max_image_bytes: number;
|
||||
response_format: 'json_schema' | 'json_object' | 'none';
|
||||
frequency_penalty: number;
|
||||
temperature: number;
|
||||
system_prompt: string | null;
|
||||
ocr_prompt: string | null;
|
||||
grounding_prompt: string | null;
|
||||
};
|
||||
|
||||
export type PromptDefaults = {
|
||||
system_prompt: string;
|
||||
ocr_prompt: string;
|
||||
grounding_prompt: string;
|
||||
};
|
||||
|
||||
export type AnalysisSettingsResponse = {
|
||||
editable: AnalysisSettings;
|
||||
env_only: { api_key_configured: boolean };
|
||||
prompt_defaults: PromptDefaults;
|
||||
};
|
||||
|
||||
export async function getCrawlerSettings(): Promise<CrawlerSettingsResponse> {
|
||||
return request<CrawlerSettingsResponse>('/v1/admin/settings/crawler');
|
||||
}
|
||||
|
||||
export async function updateCrawlerSettings(
|
||||
settings: CrawlerSettings
|
||||
): Promise<CrawlerSettingsResponse> {
|
||||
return request<CrawlerSettingsResponse>('/v1/admin/settings/crawler', {
|
||||
method: 'PUT',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify(settings)
|
||||
});
|
||||
}
|
||||
|
||||
export async function getAnalysisSettings(): Promise<AnalysisSettingsResponse> {
|
||||
return request<AnalysisSettingsResponse>('/v1/admin/settings/analysis');
|
||||
}
|
||||
|
||||
export async function updateAnalysisSettings(
|
||||
settings: AnalysisSettings
|
||||
): Promise<AnalysisSettingsResponse> {
|
||||
return request<AnalysisSettingsResponse>('/v1/admin/settings/analysis', {
|
||||
method: 'PUT',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify(settings)
|
||||
});
|
||||
}
|
||||
|
||||
@@ -12,18 +12,32 @@ export function fileUrl(key: string): string {
|
||||
return `${BASE}/v1/files/${key}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds an API URL for non-`fetch` consumers (e.g. `EventSource` for SSE),
|
||||
* applying the same `VITE_API_BASE` prefix as `request()`. `path` is the
|
||||
* route after the base, e.g. `/v1/admin/crawler/stream`.
|
||||
*/
|
||||
export function apiUrl(path: string): string {
|
||||
return `${BASE}${path}`;
|
||||
}
|
||||
|
||||
export class ApiError extends Error {
|
||||
constructor(
|
||||
public readonly status: number,
|
||||
public readonly code: string,
|
||||
message: string
|
||||
message: string,
|
||||
/** The error envelope's `details` payload, when present (e.g. the
|
||||
* per-field `{ fields: [...] }` of a `validation_failed` response). */
|
||||
public readonly details?: unknown
|
||||
) {
|
||||
super(message);
|
||||
this.name = 'ApiError';
|
||||
}
|
||||
}
|
||||
|
||||
type ErrorEnvelope = { error?: { code?: unknown; message?: unknown } };
|
||||
type ErrorEnvelope = {
|
||||
error?: { code?: unknown; message?: unknown; details?: unknown };
|
||||
};
|
||||
|
||||
/**
|
||||
* Optional hook fired the first moment `request()` observes a 401 on
|
||||
@@ -50,6 +64,7 @@ export async function request<T>(path: string, init?: RequestInit): Promise<T> {
|
||||
if (!res.ok) {
|
||||
let code = 'http_error';
|
||||
let message = `${res.status} ${res.statusText}`;
|
||||
let details: unknown;
|
||||
const ct = res.headers.get('content-type') ?? '';
|
||||
try {
|
||||
if (ct.includes('application/json')) {
|
||||
@@ -61,6 +76,9 @@ export async function request<T>(path: string, init?: RequestInit): Promise<T> {
|
||||
if (typeof body.error.message === 'string' && body.error.message) {
|
||||
message = body.error.message;
|
||||
}
|
||||
if (body.error.details !== undefined) {
|
||||
details = body.error.details;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
const text = await res.text();
|
||||
@@ -79,7 +97,7 @@ export async function request<T>(path: string, init?: RequestInit): Promise<T> {
|
||||
console.error('on401 hook threw:', e);
|
||||
}
|
||||
}
|
||||
throw new ApiError(res.status, code, message);
|
||||
throw new ApiError(res.status, code, message, details);
|
||||
}
|
||||
// Any empty body (not just 204) returns undefined — the manga-add
|
||||
// endpoint, for instance, signals create-vs-already-present via
|
||||
|
||||
@@ -7,7 +7,8 @@ import {
|
||||
updateMangaCover,
|
||||
deleteMangaCover,
|
||||
attachTag,
|
||||
detachTag
|
||||
detachTag,
|
||||
getSimilarMangas
|
||||
} from './mangas';
|
||||
|
||||
function ok(body: unknown, status = 200): Response {
|
||||
@@ -251,6 +252,29 @@ describe('mangas api client', () => {
|
||||
expect(init.method).toBe('DELETE');
|
||||
});
|
||||
|
||||
it('getSimilarMangas hits /v1/mangas/:id/similar and unwraps items', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(
|
||||
ok({ items: [cardFixture({ id: 's1' }), cardFixture({ id: 's2' })] })
|
||||
);
|
||||
const items = await getSimilarMangas('b1');
|
||||
const url = fetchSpy.mock.calls[0][0] as string;
|
||||
expect(url).toMatch(/\/v1\/mangas\/b1\/similar$/);
|
||||
expect(items.map((m) => m.id)).toEqual(['s1', 's2']);
|
||||
});
|
||||
|
||||
it('getSimilarMangas returns an empty array when there are no matches', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(ok({ items: [] }));
|
||||
expect(await getSimilarMangas('b1')).toEqual([]);
|
||||
});
|
||||
|
||||
it('listMangas serializes content-warning include/exclude filters', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(ok(emptyPage()));
|
||||
await listMangas({ cwInclude: ['gore'], cwExclude: ['sexual', 'nudity'] });
|
||||
const url = fetchSpy.mock.calls[0][0] as string;
|
||||
expect(url).toContain('cw_include=gore');
|
||||
expect(url).toContain('cw_exclude=sexual%2Cnudity');
|
||||
});
|
||||
|
||||
it('getManga throws ApiError carrying the envelope code on non-2xx', async () => {
|
||||
fetchSpy.mockResolvedValue(envelope(404, 'not_found', 'manga not found'));
|
||||
await expect(getManga('missing')).rejects.toMatchObject({
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { request, type Manga, type MangaStatus, type Page } from './client';
|
||||
import type { ContentWarning } from './page_tags';
|
||||
|
||||
export type MangaSort = 'recent' | 'title';
|
||||
|
||||
@@ -17,6 +18,8 @@ export type MangaDetail = Manga & {
|
||||
authors: AuthorRef[];
|
||||
genres: GenreRef[];
|
||||
tags: TagRef[];
|
||||
/** Deduped union of content warnings across the manga's pages. */
|
||||
content_warnings: ContentWarning[];
|
||||
};
|
||||
|
||||
export type ListOptions = {
|
||||
@@ -26,6 +29,9 @@ export type ListOptions = {
|
||||
authorIds?: string[];
|
||||
genreIds?: string[];
|
||||
tagIds?: string[];
|
||||
/** Content warnings the manga must carry (all) / must not carry (any). */
|
||||
cwInclude?: ContentWarning[];
|
||||
cwExclude?: ContentWarning[];
|
||||
limit?: number;
|
||||
offset?: number;
|
||||
sort?: MangaSort;
|
||||
@@ -49,6 +55,12 @@ export async function listMangas(opts: ListOptions = {}): Promise<MangasPage> {
|
||||
if (opts.tagIds && opts.tagIds.length) {
|
||||
params.set('tag_id', opts.tagIds.join(','));
|
||||
}
|
||||
if (opts.cwInclude && opts.cwInclude.length) {
|
||||
params.set('cw_include', opts.cwInclude.join(','));
|
||||
}
|
||||
if (opts.cwExclude && opts.cwExclude.length) {
|
||||
params.set('cw_exclude', opts.cwExclude.join(','));
|
||||
}
|
||||
if (opts.limit != null) params.set('limit', String(opts.limit));
|
||||
if (opts.offset != null) params.set('offset', String(opts.offset));
|
||||
if (opts.sort) params.set('sort', opts.sort);
|
||||
@@ -60,6 +72,20 @@ export async function getManga(id: string): Promise<MangaDetail> {
|
||||
return request<MangaDetail>(`/v1/mangas/${encodeURIComponent(id)}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* GET /v1/mangas/:id/similar — up to 5 mangas ranked by tag overlap with
|
||||
* `id`. Returns a plain `{ items }` object (a fixed top-N, not a paginated
|
||||
* collection), so we unwrap to the card array the page wants.
|
||||
*/
|
||||
export async function getSimilarMangas(id: string): Promise<MangaCard[]> {
|
||||
const res = await request<{ items: MangaCard[] }>(
|
||||
`/v1/mangas/${encodeURIComponent(id)}/similar`
|
||||
);
|
||||
// Defensive: a malformed 200 body (items omitted) must still yield an
|
||||
// array so the page's `similar.length` guard can't throw.
|
||||
return res.items ?? [];
|
||||
}
|
||||
|
||||
export type NewManga = {
|
||||
title: string;
|
||||
status?: MangaStatus;
|
||||
|
||||
101
frontend/src/lib/api/page_collections.test.ts
Normal file
101
frontend/src/lib/api/page_collections.test.ts
Normal file
@@ -0,0 +1,101 @@
|
||||
import {
|
||||
describe,
|
||||
it,
|
||||
expect,
|
||||
vi,
|
||||
beforeEach,
|
||||
afterEach,
|
||||
type MockInstance
|
||||
} from 'vitest';
|
||||
import {
|
||||
addPageToCollection,
|
||||
removePageFromCollection,
|
||||
listCollectionPages,
|
||||
getMyCollectionsContainingPage
|
||||
} from './page_collections';
|
||||
|
||||
function ok(body: unknown, status = 200): Response {
|
||||
return new Response(JSON.stringify(body), {
|
||||
status,
|
||||
headers: { 'content-type': 'application/json' }
|
||||
});
|
||||
}
|
||||
|
||||
function noContent(): Response {
|
||||
return new Response(null, { status: 204 });
|
||||
}
|
||||
|
||||
function pageItemFixture(extra: Record<string, unknown> = {}) {
|
||||
return {
|
||||
page_id: 'p1',
|
||||
chapter_id: 'ch1',
|
||||
manga_id: 'm1',
|
||||
page_number: 1,
|
||||
chapter_number: 1,
|
||||
chapter_title: null,
|
||||
manga_title: 'Berserk',
|
||||
storage_key: 'mangas/m1/chapters/ch1/pages/0001.png',
|
||||
added_at: '2026-01-01T00:00:00Z',
|
||||
...extra
|
||||
};
|
||||
}
|
||||
|
||||
describe('page_collections api client', () => {
|
||||
let fetchSpy: MockInstance<typeof globalThis.fetch>;
|
||||
|
||||
beforeEach(() => {
|
||||
fetchSpy = vi.spyOn(globalThis, 'fetch');
|
||||
});
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it('listCollectionPages hits the breadcrumb endpoint', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(
|
||||
ok({
|
||||
items: [pageItemFixture()],
|
||||
page: { limit: 50, offset: 0, total: 1 }
|
||||
})
|
||||
);
|
||||
const r = await listCollectionPages('c1');
|
||||
expect(r.items[0].manga_title).toBe('Berserk');
|
||||
const url = fetchSpy.mock.calls[0][0] as string;
|
||||
expect(url).toMatch(/\/v1\/collections\/c1\/pages$/);
|
||||
});
|
||||
|
||||
it('listCollectionPages forwards pagination params', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(
|
||||
ok({ items: [], page: { limit: 10, offset: 20, total: 0 } })
|
||||
);
|
||||
await listCollectionPages('c1', { limit: 10, offset: 20 });
|
||||
const url = fetchSpy.mock.calls[0][0] as string;
|
||||
expect(url).toContain('limit=10');
|
||||
expect(url).toContain('offset=20');
|
||||
});
|
||||
|
||||
it('addPageToCollection POSTs { page_id }', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(ok({}, 201));
|
||||
await addPageToCollection('c1', 'p1');
|
||||
const init = fetchSpy.mock.calls[0][1] as RequestInit;
|
||||
expect(init.method).toBe('POST');
|
||||
expect(JSON.parse(init.body as string)).toEqual({ page_id: 'p1' });
|
||||
});
|
||||
|
||||
it('removePageFromCollection DELETEs with both ids encoded', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(noContent());
|
||||
await removePageFromCollection('c with space', 'p1');
|
||||
const url = fetchSpy.mock.calls[0][0] as string;
|
||||
// Path includes encoded "c with space" + page id.
|
||||
expect(url).toContain('/v1/collections/c%20with%20space/pages/p1');
|
||||
const init = fetchSpy.mock.calls[0][1] as RequestInit;
|
||||
expect(init.method).toBe('DELETE');
|
||||
});
|
||||
|
||||
it('getMyCollectionsContainingPage unwraps `collection_ids`', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(ok({ collection_ids: ['c1', 'c2'] }));
|
||||
const ids = await getMyCollectionsContainingPage('p1');
|
||||
expect(ids).toEqual(['c1', 'c2']);
|
||||
const url = fetchSpy.mock.calls[0][0] as string;
|
||||
expect(url).toMatch(/\/v1\/pages\/p1\/my-collections$/);
|
||||
});
|
||||
});
|
||||
68
frontend/src/lib/api/page_collections.ts
Normal file
68
frontend/src/lib/api/page_collections.ts
Normal file
@@ -0,0 +1,68 @@
|
||||
import { request, type Page } from './client';
|
||||
|
||||
/** Row returned by `GET /v1/collections/:id/pages`. */
|
||||
export type CollectionPageItem = {
|
||||
page_id: string;
|
||||
chapter_id: string;
|
||||
manga_id: string;
|
||||
page_number: number;
|
||||
chapter_number: number;
|
||||
chapter_title: string | null;
|
||||
manga_title: string;
|
||||
storage_key: string;
|
||||
added_at: string;
|
||||
};
|
||||
|
||||
export type CollectionPagesPage = {
|
||||
items: CollectionPageItem[];
|
||||
page: Page;
|
||||
};
|
||||
|
||||
export type ListOptions = { limit?: number; offset?: number };
|
||||
|
||||
export async function listCollectionPages(
|
||||
collectionId: string,
|
||||
opts: ListOptions = {}
|
||||
): Promise<CollectionPagesPage> {
|
||||
const params = new URLSearchParams();
|
||||
if (opts.limit != null) params.set('limit', String(opts.limit));
|
||||
if (opts.offset != null) params.set('offset', String(opts.offset));
|
||||
const qs = params.toString();
|
||||
return request<CollectionPagesPage>(
|
||||
`/v1/collections/${encodeURIComponent(collectionId)}/pages${qs ? `?${qs}` : ''}`
|
||||
);
|
||||
}
|
||||
|
||||
export async function addPageToCollection(
|
||||
collectionId: string,
|
||||
pageId: string
|
||||
): Promise<void> {
|
||||
await request<void>(
|
||||
`/v1/collections/${encodeURIComponent(collectionId)}/pages`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify({ page_id: pageId })
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
export async function removePageFromCollection(
|
||||
collectionId: string,
|
||||
pageId: string
|
||||
): Promise<void> {
|
||||
await request<void>(
|
||||
`/v1/collections/${encodeURIComponent(collectionId)}/pages/${encodeURIComponent(pageId)}`,
|
||||
{ method: 'DELETE' }
|
||||
);
|
||||
}
|
||||
|
||||
/** Which of the user's collections currently contain this page. */
|
||||
export async function getMyCollectionsContainingPage(
|
||||
pageId: string
|
||||
): Promise<string[]> {
|
||||
const r = await request<{ collection_ids: string[] }>(
|
||||
`/v1/pages/${encodeURIComponent(pageId)}/my-collections`
|
||||
);
|
||||
return r.collection_ids;
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user