Compare commits
68 Commits
51ea254dde
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
33cc41bacd | ||
|
|
08a9819d76 | ||
|
|
70a6598924 | ||
|
|
e6aefaa804 | ||
|
|
6354a7a8a3 | ||
|
|
60d5a2efea | ||
|
|
90f2398e56 | ||
|
|
e960aae163 | ||
|
|
adce950049 | ||
|
|
bde72e47f4 | ||
|
|
cb757e7b69 | ||
|
|
042e7e9047 | ||
|
|
cc8ae2566f | ||
|
|
4f085f228a | ||
|
|
ac5641bcd3 | ||
|
|
cd5358dc9b | ||
|
|
c31830468c | ||
|
|
75f4481bbd | ||
|
|
958d5bd713 | ||
|
|
f2c9cfe162 | ||
|
|
b5f7467c47 | ||
|
|
9148e23da8 | ||
|
|
f39307232c | ||
|
|
1b7b8a3038 | ||
|
|
253d46c7e5 | ||
|
|
c570e0cc37 | ||
|
|
5784483a57 | ||
|
|
a44511983d | ||
|
|
f8e53809d5 | ||
|
|
0afd202164 | ||
|
|
ba3e5b481b | ||
|
|
3b783c1d9b | ||
|
|
a47b6895c2 | ||
|
|
3ca05dcb58 | ||
|
|
c308eb3eac | ||
|
|
69a9309c54 | ||
|
|
1ed1a134ea | ||
|
|
32d0a7e13b | ||
|
|
d779fa2b97 | ||
|
|
755417730f | ||
|
|
5b1ce581f3 | ||
|
|
c11df3182c | ||
|
|
d749b56d58 | ||
|
|
c24e296f07 | ||
|
|
9508fb8e86 | ||
|
|
ca55712622 | ||
|
|
4fe435cc76 | ||
|
|
f5cb460aec | ||
|
|
f83d49b83e | ||
|
|
cef41ce76a | ||
|
|
7570524e5b | ||
|
|
5b46eab73a | ||
|
|
91d6a50dba | ||
| 134ab54b34 | |||
|
|
f879ce1866 | ||
|
|
bf425cf8e6 | ||
|
|
ff4ca964f5 | ||
|
|
61669aac3f | ||
|
|
795eb76f9a | ||
|
|
ebf0b8289b | ||
|
|
ce9a727c73 | ||
|
|
46134c8760 | ||
|
|
b7d8faadf7 | ||
|
|
b0500e8e48 | ||
|
|
379224bee4 | ||
|
|
1c955458d6 | ||
|
|
987d1ba235 | ||
|
|
c69ff502f7 |
43
.env.example
43
.env.example
@@ -25,6 +25,12 @@ DATABASE_URL=postgres://mangalord:mangalord@postgres:5432/mangalord
|
||||
BIND_ADDRESS=0.0.0.0:8080
|
||||
STORAGE_DIR=/var/lib/mangalord/storage
|
||||
RUST_LOG=info,mangalord=debug,chromiumoxide::conn=off,chromiumoxide::handler=off
|
||||
# Postgres connection-pool sizing. One pool serves HTTP handlers and the
|
||||
# crawler/analysis daemons. DB_MAX_CONNECTIONS caps open connections;
|
||||
# DB_ACQUIRE_TIMEOUT_SECS is how long a request waits for a free connection
|
||||
# before failing fast (rather than hanging on the driver's 30s default).
|
||||
DB_MAX_CONNECTIONS=20
|
||||
DB_ACQUIRE_TIMEOUT_SECS=10
|
||||
|
||||
# ----- Auth / cookies -----
|
||||
# COOKIE_SECURE controls whether the `Secure` flag is set on the session
|
||||
@@ -45,6 +51,13 @@ SESSION_TTL_DAYS=30
|
||||
# rate-limiting reverse proxy that already enforces a budget).
|
||||
AUTH_RATE_PER_SEC=5
|
||||
AUTH_RATE_BURST=10
|
||||
# Trust a proxy-supplied X-Forwarded-For as the client IP for per-IP auth
|
||||
# rate limiting. Enable ONLY when the backend sits behind a trusted proxy
|
||||
# that overrides the header (the compose deploy: SvelteKit forwards the real
|
||||
# peer IP). When false, the header is ignored and a single shared bucket is
|
||||
# used — a directly-exposed backend MUST keep this off or clients could spoof
|
||||
# their IP to dodge the limit.
|
||||
AUTH_TRUSTED_PROXY=false
|
||||
|
||||
# ----- CORS -----
|
||||
# Comma-separated origins allowed to call the API with credentials.
|
||||
@@ -105,8 +118,10 @@ MAX_REQUEST_BYTES=209715200
|
||||
MAX_FILE_BYTES=20971520
|
||||
# Max page images accepted in one chapter upload. Bounds how many parts
|
||||
# the handler will stage before rejecting the request with 413, so a
|
||||
# client can't pin a worker with an unbounded page count. 0 disables the
|
||||
# cap. Default 2000.
|
||||
# client can't pin a worker with an unbounded page count. Default 2000.
|
||||
# Setting 0 disables THIS cap — the total is then bounded only by
|
||||
# MAX_REQUEST_BYTES (the whole-request body limit above), so leave 0 only
|
||||
# if you intend that body limit to be the sole backstop.
|
||||
MAX_PAGES_PER_CHAPTER=2000
|
||||
|
||||
# ----- Crawler download safety -----
|
||||
@@ -178,6 +193,19 @@ CRAWLER_METADATA_MAX_CONSECUTIVE_FAILURES=10
|
||||
# exhausted) that trigger an automatic coordinated browser restart.
|
||||
# Default 3.
|
||||
CRAWLER_BROWSER_RESTART_THRESHOLD=3
|
||||
# CDP Fetch interception that re-validates every headless-browser
|
||||
# navigation/redirect/subresource against the SSRF check (blocks a scraped page
|
||||
# that drives the browser — via a redirect OR page JS/subresource — to an
|
||||
# internal target such as the cloud metadata service or postgres). Default TRUE:
|
||||
# with it off, only the top-level URL string is checked and page JS/subresources
|
||||
# reach internal IPs (the reqwest SafeResolver does not cover Chromium's own
|
||||
# stack). The CDP Fetch hook can't be exercised in CI (no Chromium); before
|
||||
# relying on a fresh deploy, validate it doesn't wedge navigation:
|
||||
# CRAWLER_CHROMIUM_BINARY=/usr/bin/chromium \
|
||||
# cargo test -p mangalord --test crawler_browser_smoke -- --ignored \
|
||||
# ssrf_interception_does_not_wedge_allowed_navigation
|
||||
# Set to false only as a break-glass if the hook destabilizes a deployment.
|
||||
CRAWLER_SSRF_INTERCEPT=true
|
||||
# Path to a system Chromium binary. When set, the crawler skips the
|
||||
# bundled-fetcher download. Required on platforms without a usable
|
||||
# upstream Chromium build (notably Linux_arm64 / Raspberry Pi). On
|
||||
@@ -399,3 +427,14 @@ VISION_MANAGER_DATABASE_URL=
|
||||
# VISION_MEM_LOW_WATERMARK_PCT used% >= this → inhibit starts (keep HIGH - LOW >= ~10 so the band beats jitter). Default 80.
|
||||
# VISION_MEM_YIELD_COOLDOWN Seconds after a pressure-stop during which restart is refused regardless of backlog. Default 300.
|
||||
# VISION_MEM_POLL_INTERVAL Mem sub-poll cadence, seconds (<= VISION_POLL_INTERVAL); catches spikes between backlog ticks. Default 5.
|
||||
|
||||
# ---- Deployment resource bounds & exposure (docker-compose.yml) ----
|
||||
# Host interface the frontend port publishes on. Default 127.0.0.1 so SvelteKit
|
||||
# (plain HTTP) is reachable only by a host-local reverse proxy. Set 0.0.0.0 only
|
||||
# if your TLS terminator runs on a different host.
|
||||
FRONTEND_PUBLISH_ADDR=127.0.0.1
|
||||
# Per-container memory ceilings (docker mem_limit). Generous defaults; tune to
|
||||
# your host. The backend runs OCR + headless Chromium and is the heavy one.
|
||||
BACKEND_MEM_LIMIT=4g
|
||||
FRONTEND_MEM_LIMIT=512m
|
||||
POSTGRES_MEM_LIMIT=1g
|
||||
|
||||
@@ -136,7 +136,7 @@ docker compose -f docker-compose.dev.yml up -d
|
||||
These are first-class slots in the architecture. When adding any of them, plug into the existing seam rather than building parallel infrastructure.
|
||||
|
||||
- **Tags / lists**: new tables joined to `mangas`. New `domain`, `repo`, and `api` modules; the existing manga endpoints do not need to change.
|
||||
- **Per-page collections / tags**: `collections` is heterogeneous — `collection_mangas` holds whole mangas, `collection_pages` holds individual pages (FK to `pages.id`). Per-user page tags live in `page_tags`, which references the **shared** `tags` table by `tag_id` (migration 0024) — the same lookup table `manga_tags` uses, so manga tags and page tags share one global vocabulary. The HTTP contract still speaks tag *names*; `repo::page_tag` resolves name↔id via `repo::tag::upsert_by_name` and applies the stricter page-tag `normalize_tag` (lowercase, collapse whitespace, reject wildcards/control/invisible chars) at the API layer. Both `collection_pages` and `page_tags` cascade-delete with `pages` and `chapters`, so re-uploading a chapter drops saved-page references by design.
|
||||
- **Per-page collections / tags**: `collections` is heterogeneous — `collection_mangas` holds whole mangas, `collection_pages` holds individual pages (FK to `pages.id`). Per-user page tags live in `page_tags`, which references the **shared** `tags` table by `tag_id` (migration 0024) — the same lookup table `manga_tags` uses, so manga tags and page tags share one global vocabulary. The HTTP contract still speaks tag *names*; `repo::page_tag` resolves name↔id via `repo::tag::upsert_by_name` and applies the stricter page-tag `normalize_tag` (lowercase, collapse whitespace, reject wildcards/control/invisible chars) at the API layer. Both `collection_pages` and `page_tags` cascade-delete with `pages` and `chapters`, so a saved-page reference is dropped only when its `pages` row is genuinely deleted — i.e. when the chapter is deleted (cascade), or when a user deletes and re-creates a chapter (the user upload path in `api::chapters::finalize_chapter` inserts a *new* chapter row with fresh page ids). A crawler **re-fetch** does **not** drop saves: `content::persist_pages` upserts pages by `(chapter_id, page_number)` (`ON CONFLICT (…) DO UPDATE … RETURNING id`), preserving each `pages.id`, so collections and page tags keyed on that id survive the re-fetch. (Migration 0023's header comment predates this and describes the cascade as an unconditional "re-upload drops saves"; the checked-in migration text is intentionally left as-is because sqlx checksums applied migrations.)
|
||||
- **Tag-based content search (`/search`)**: the user-facing search surface lives at [frontend/src/routes/search/+page.svelte](frontend/src/routes/search/+page.svelte). Three result views (Pages / Chapters / Mangas) consume the matching `/v1/me/page-tags`, `/v1/me/page-tags/chapters`, and `/v1/me/page-tags/mangas` endpoints. Note the two distinct query-param spaces: `?q=` on `/v1/me/page-tags` is a tag-name prefix (for autocomplete in the "Add tag" sheet); `?text=` on the aggregation endpoints performs **OCR full-text search** — the active OCR backend writes `page_ocr_text` rows and a weighted `search_doc` tsvector, and the aggregation queries JOIN on a `plainto_tsquery` filter ranked by `ts_rank` (see [backend/src/repo/page_analysis.rs](backend/src/repo/page_analysis.rs)). (`text=` was previously reserved and returned 501 `text_search_not_yet_supported`; that placeholder is gone now that the OCR backend is active. The generic `AppError::NotImplemented` 501 mechanism remains for future feature reservations.)
|
||||
- **Full-text / fuzzy search**: enable `pg_trgm` in a migration and add a GIN index on `mangas.title`; swap the `WHERE` in `repo::manga::list` to use `%` operator or `tsvector`. The API shape (`?search=...`) does not change.
|
||||
- **OCR / autotagging**: a background worker (a separate binary or a tokio task spawned in `app::build`) that reads pages from `storage::Storage` and writes tag rows. Do not couple OCR to upload handlers — it runs asynchronously.
|
||||
|
||||
3
backend/Cargo.lock
generated
3
backend/Cargo.lock
generated
@@ -1558,7 +1558,7 @@ checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
|
||||
|
||||
[[package]]
|
||||
name = "mangalord"
|
||||
version = "0.122.1"
|
||||
version = "0.128.25"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"argon2",
|
||||
@@ -1587,7 +1587,6 @@ dependencies = [
|
||||
"serde_json",
|
||||
"sha2",
|
||||
"sqlx",
|
||||
"subtle",
|
||||
"sysinfo",
|
||||
"tempfile",
|
||||
"thiserror 1.0.69",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "mangalord"
|
||||
version = "0.122.1"
|
||||
version = "0.128.25"
|
||||
edition = "2021"
|
||||
default-run = "mangalord"
|
||||
|
||||
@@ -35,7 +35,6 @@ dotenvy = "0.15"
|
||||
argon2 = "0.5"
|
||||
rand = "0.8"
|
||||
sha2 = "0.10"
|
||||
subtle = "2"
|
||||
base64 = "0.22"
|
||||
# Image decode + downscale for the analysis worker (keep the page image
|
||||
# under the local vision model's token budget). Only the manga page formats.
|
||||
@@ -62,6 +61,7 @@ http-body-util = "0.1"
|
||||
mime = "0.3"
|
||||
futures-util = "0.3"
|
||||
tokio = { version = "1", features = ["test-util"] }
|
||||
image = { version = "0.25", default-features = false, features = ["jpeg", "png", "webp"] }
|
||||
|
||||
# Trim debug builds: keep line numbers in panics / backtraces but drop the
|
||||
# full DWARF info (variable-level inspection in gdb/lldb). With a sqlx +
|
||||
|
||||
103
backend/migrations/0036_manga_content_warnings.sql
Normal file
103
backend/migrations/0036_manga_content_warnings.sql
Normal file
@@ -0,0 +1,103 @@
|
||||
-- Denormalized manga -> content-warning set.
|
||||
--
|
||||
-- The list filter previously tested each candidate manga with a correlated
|
||||
-- `page_content_warnings -> pages -> chapters` join, i.e. O(mangas * pages) on
|
||||
-- every filtered list AND its count. This table holds the DISTINCT union of a
|
||||
-- manga's page warnings so the filter is a single indexed lookup.
|
||||
--
|
||||
-- Kept in sync by triggers that recompute an affected manga's set from current
|
||||
-- data (the set is tiny — at most the five moderation labels — so a
|
||||
-- delete-and-reinsert per change is cheap and always correct, sidestepping the
|
||||
-- cascade-ordering hazards of incremental maintenance).
|
||||
|
||||
CREATE TABLE manga_content_warnings (
|
||||
manga_id uuid NOT NULL REFERENCES mangas(id) ON DELETE CASCADE,
|
||||
warning text NOT NULL
|
||||
CHECK (warning IN ('sexual', 'nudity', 'gore', 'violence', 'disturbing')),
|
||||
PRIMARY KEY (manga_id, warning)
|
||||
);
|
||||
|
||||
-- warning -> mangas, for the include/exclude list filters.
|
||||
CREATE INDEX manga_content_warnings_warning_idx ON manga_content_warnings (warning);
|
||||
|
||||
-- Recompute a single manga's warning set from the live per-page rows.
|
||||
CREATE OR REPLACE FUNCTION mcw_refresh_for_manga(mid uuid) RETURNS void AS $$
|
||||
BEGIN
|
||||
-- Skip when the manga is gone (e.g. mid-cascade of a manga delete) so we
|
||||
-- never re-insert a row that would violate the FK / resurrect a deleted set.
|
||||
IF NOT EXISTS (SELECT 1 FROM mangas WHERE id = mid) THEN
|
||||
RETURN;
|
||||
END IF;
|
||||
DELETE FROM manga_content_warnings WHERE manga_id = mid;
|
||||
INSERT INTO manga_content_warnings (manga_id, warning)
|
||||
SELECT DISTINCT mid, pw.warning
|
||||
FROM page_content_warnings pw
|
||||
JOIN pages p ON p.id = pw.page_id
|
||||
JOIN chapters c ON c.id = p.chapter_id
|
||||
WHERE c.manga_id = mid;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
-- page_content_warnings changed for a page: refresh that page's manga.
|
||||
CREATE OR REPLACE FUNCTION mcw_on_pcw_change() RETURNS trigger AS $$
|
||||
DECLARE
|
||||
mid uuid;
|
||||
pid uuid := COALESCE(NEW.page_id, OLD.page_id);
|
||||
BEGIN
|
||||
-- The page (and thus chapter) may already be gone when this fires as part
|
||||
-- of a pages/chapters cascade; in that case the pages/chapters triggers do
|
||||
-- the refresh instead, so a missing join here is harmless.
|
||||
SELECT c.manga_id INTO mid
|
||||
FROM pages p JOIN chapters c ON c.id = p.chapter_id
|
||||
WHERE p.id = pid;
|
||||
IF mid IS NOT NULL THEN
|
||||
PERFORM mcw_refresh_for_manga(mid);
|
||||
END IF;
|
||||
RETURN NULL;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
CREATE TRIGGER mcw_pcw_ins AFTER INSERT ON page_content_warnings
|
||||
FOR EACH ROW EXECUTE FUNCTION mcw_on_pcw_change();
|
||||
CREATE TRIGGER mcw_pcw_del AFTER DELETE ON page_content_warnings
|
||||
FOR EACH ROW EXECUTE FUNCTION mcw_on_pcw_change();
|
||||
|
||||
-- A page was deleted (directly, or via a chapter cascade): its
|
||||
-- page_content_warnings rows are already cascade-gone, so recompute from the
|
||||
-- chapter's manga. If the chapter is gone too, the chapters trigger covers it.
|
||||
CREATE OR REPLACE FUNCTION mcw_on_page_delete() RETURNS trigger AS $$
|
||||
DECLARE
|
||||
mid uuid;
|
||||
BEGIN
|
||||
SELECT c.manga_id INTO mid FROM chapters c WHERE c.id = OLD.chapter_id;
|
||||
IF mid IS NOT NULL THEN
|
||||
PERFORM mcw_refresh_for_manga(mid);
|
||||
END IF;
|
||||
RETURN NULL;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
CREATE TRIGGER mcw_page_del AFTER DELETE ON pages
|
||||
FOR EACH ROW EXECUTE FUNCTION mcw_on_page_delete();
|
||||
|
||||
-- A chapter was deleted (directly, or via a manga cascade): recompute from the
|
||||
-- chapter's manga. `chapters.manga_id` is on the row, so it is always available
|
||||
-- even after the child pages have cascaded; the refresh no-ops when the manga
|
||||
-- itself is being deleted.
|
||||
CREATE OR REPLACE FUNCTION mcw_on_chapter_delete() RETURNS trigger AS $$
|
||||
BEGIN
|
||||
PERFORM mcw_refresh_for_manga(OLD.manga_id);
|
||||
RETURN NULL;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
CREATE TRIGGER mcw_chapter_del AFTER DELETE ON chapters
|
||||
FOR EACH ROW EXECUTE FUNCTION mcw_on_chapter_delete();
|
||||
|
||||
-- Backfill from existing per-page rows.
|
||||
INSERT INTO manga_content_warnings (manga_id, warning)
|
||||
SELECT DISTINCT c.manga_id, pw.warning
|
||||
FROM page_content_warnings pw
|
||||
JOIN pages p ON p.id = pw.page_id
|
||||
JOIN chapters c ON c.id = p.chapter_id
|
||||
ON CONFLICT DO NOTHING;
|
||||
51
backend/migrations/0037_mangas_sort_author.sql
Normal file
51
backend/migrations/0037_mangas_sort_author.sql
Normal file
@@ -0,0 +1,51 @@
|
||||
-- Precomputed author sort key for `?sort=author`.
|
||||
--
|
||||
-- The author sort used a correlated `min(lower(a.name))` subquery as the ORDER
|
||||
-- BY key, evaluated per filter-matching row before LIMIT — it scaled worse than
|
||||
-- the indexed date/title sorts. Materialize the same value on `mangas` so the
|
||||
-- sort is a plain indexed column read.
|
||||
--
|
||||
-- `sort_author` = the alphabetically-first attached author's lowercased name,
|
||||
-- or NULL when the manga has no authors (kept last via NULLS LAST in the query).
|
||||
-- Author names are immutable (authors are upserted by unique lowercased name and
|
||||
-- never renamed), so the value only changes when the manga_authors join changes
|
||||
-- — maintained by the triggers below.
|
||||
|
||||
ALTER TABLE mangas ADD COLUMN sort_author text;
|
||||
|
||||
CREATE INDEX mangas_sort_author_idx ON mangas (sort_author, id);
|
||||
|
||||
-- Backfill from existing links.
|
||||
UPDATE mangas m
|
||||
SET sort_author = (
|
||||
SELECT min(lower(a.name))
|
||||
FROM manga_authors ma
|
||||
JOIN authors a ON a.id = ma.author_id
|
||||
WHERE ma.manga_id = m.id
|
||||
);
|
||||
|
||||
CREATE OR REPLACE FUNCTION refresh_manga_sort_author(mid uuid) RETURNS void AS $$
|
||||
BEGIN
|
||||
UPDATE mangas
|
||||
SET sort_author = (
|
||||
SELECT min(lower(a.name))
|
||||
FROM manga_authors ma
|
||||
JOIN authors a ON a.id = ma.author_id
|
||||
WHERE ma.manga_id = mid
|
||||
)
|
||||
WHERE id = mid;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
CREATE OR REPLACE FUNCTION mangas_sort_author_on_ma_change() RETURNS trigger AS $$
|
||||
BEGIN
|
||||
-- On a manga cascade-delete the UPDATE simply no-ops (row already gone).
|
||||
PERFORM refresh_manga_sort_author(COALESCE(NEW.manga_id, OLD.manga_id));
|
||||
RETURN NULL;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
CREATE TRIGGER manga_authors_sort_author_ins AFTER INSERT ON manga_authors
|
||||
FOR EACH ROW EXECUTE FUNCTION mangas_sort_author_on_ma_change();
|
||||
CREATE TRIGGER manga_authors_sort_author_del AFTER DELETE ON manga_authors
|
||||
FOR EACH ROW EXECUTE FUNCTION mangas_sort_author_on_ma_change();
|
||||
44
backend/migrations/0038_genres_name_lower_unique.sql
Normal file
44
backend/migrations/0038_genres_name_lower_unique.sql
Normal file
@@ -0,0 +1,44 @@
|
||||
-- Enforce case-insensitive genre uniqueness, matching `authors` and `tags`
|
||||
-- (both got `UNIQUE (lower(name))` in 0009). `genres` only had a case-SENSITIVE
|
||||
-- `name UNIQUE`, but the crawler's `sync_genres` treats genres as
|
||||
-- case-insensitive (`WHERE lower(name) = lower($1)`) and can INSERT a new row
|
||||
-- from a source string. Under a race (or across ticks) two different-cased
|
||||
-- strings — "Isekai" vs "isekai" — could both miss the pre-check and both
|
||||
-- insert, since the exact-name unique doesn't collide. That leaves duplicate
|
||||
-- genre rows for one logical genre.
|
||||
|
||||
-- Heal any pre-existing case-variant duplicates before adding the index, so
|
||||
-- `sqlx::migrate!` can't crash on a dirty table (mirrors 0031's pre-dedup).
|
||||
-- Canonical = lowest id per lower(name).
|
||||
|
||||
-- 1. Ensure every manga linked to any case-variant also has the canonical link.
|
||||
-- INSERT ... ON CONFLICT DO NOTHING is collision-proof: SELECT DISTINCT
|
||||
-- collapses multiple variants of one manga to a single canonical row, and the
|
||||
-- ON CONFLICT absorbs a canonical link that already exists. (A prior
|
||||
-- UPDATE-repoint could set two non-canonical rows of the SAME manga to the
|
||||
-- same canonical id in a single statement and violate the manga_genres PK,
|
||||
-- rolling the migration back and wedging startup.)
|
||||
INSERT INTO manga_genres (manga_id, genre_id)
|
||||
SELECT DISTINCT mg.manga_id, k.keep_id
|
||||
FROM manga_genres mg
|
||||
JOIN genres g ON mg.genre_id = g.id
|
||||
JOIN (SELECT lower(name) AS lname, (array_agg(id ORDER BY id))[1] AS keep_id
|
||||
FROM genres GROUP BY lower(name)) k
|
||||
ON lower(g.name) = k.lname
|
||||
WHERE g.id <> k.keep_id
|
||||
ON CONFLICT (manga_id, genre_id) DO NOTHING;
|
||||
|
||||
-- 2. Every non-canonical link now has a canonical sibling — drop the variants.
|
||||
DELETE FROM manga_genres mg
|
||||
USING genres g,
|
||||
(SELECT lower(name) AS lname, (array_agg(id ORDER BY id))[1] AS keep_id
|
||||
FROM genres GROUP BY lower(name)) k
|
||||
WHERE mg.genre_id = g.id AND lower(g.name) = k.lname AND g.id <> k.keep_id;
|
||||
|
||||
-- Remove the now-orphaned duplicate genre rows.
|
||||
DELETE FROM genres g
|
||||
USING (SELECT lower(name) AS lname, (array_agg(id ORDER BY id))[1] AS keep_id
|
||||
FROM genres GROUP BY lower(name)) k
|
||||
WHERE lower(g.name) = k.lname AND g.id <> k.keep_id;
|
||||
|
||||
CREATE UNIQUE INDEX genres_name_lower_uniq ON genres (lower(name));
|
||||
18
backend/migrations/0039_crawler_jobs_running_lease_idx.sql
Normal file
18
backend/migrations/0039_crawler_jobs_running_lease_idx.sql
Normal file
@@ -0,0 +1,18 @@
|
||||
-- Index the crashed-worker-recovery arm of the job lease predicate.
|
||||
--
|
||||
-- lease/lease_kinds match:
|
||||
-- WHERE (state = 'pending' OR (state = 'running' AND leased_until < now()))
|
||||
-- crawler_jobs_ready_idx (0016) is `ON (scheduled_at) WHERE state = 'pending'`,
|
||||
-- so it covers only the pending arm. The `state = 'running' AND leased_until`
|
||||
-- arm had no usable index, so Postgres could not BitmapOr the two arms and
|
||||
-- degraded to a sequential scan of the ENTIRE crawler_jobs table — including all
|
||||
-- done/dead rows not yet reaped — on every lease poll, by every worker,
|
||||
-- continuously (audit H2, the headline performance finding).
|
||||
--
|
||||
-- A partial index on leased_until over just the running rows makes the recovery
|
||||
-- arm index-backed. `state = 'running'` is an immutable predicate (now() stays
|
||||
-- in the query, not the index). The running set is tiny (in-flight jobs only),
|
||||
-- so the index is cheap to maintain.
|
||||
CREATE INDEX crawler_jobs_running_lease_idx
|
||||
ON crawler_jobs (leased_until)
|
||||
WHERE state = 'running';
|
||||
@@ -0,0 +1,9 @@
|
||||
-- Index the retention reaper's scan. reap_terminal deletes
|
||||
-- WHERE state IN ('done','dead') AND updated_at < now() - interval
|
||||
-- which had no supporting index, so each daily sweep sequential-scanned the whole
|
||||
-- crawler_jobs table to find expired terminal rows (audit M5). A partial index on
|
||||
-- updated_at over just the terminal states makes the batched reaper's per-batch
|
||||
-- SELECT index-backed.
|
||||
CREATE INDEX crawler_jobs_terminal_reap_idx
|
||||
ON crawler_jobs (updated_at)
|
||||
WHERE state IN ('done', 'dead');
|
||||
@@ -37,6 +37,22 @@ const LEASE_HEARTBEAT: Duration = Duration::from_secs(20);
|
||||
/// long enough not to hammer `/health` while vision is down.
|
||||
const READINESS_POLL: Duration = Duration::from_secs(2);
|
||||
|
||||
/// Longest an idle analysis worker waits between lease polls, bounding the
|
||||
/// exponential [`idle_backoff`].
|
||||
const IDLE_BACKOFF_CAP: Duration = Duration::from_secs(30);
|
||||
|
||||
/// Exponential idle backoff (1s, 2s, 4s … capped at [`IDLE_BACKOFF_CAP`]).
|
||||
/// `consecutive_empty` is the number of empty lease polls so far (0 on the first
|
||||
/// miss); reset to 0 the moment a job is leased. Replaces the old flat 1s sleep
|
||||
/// so an idle worker isn't firing a `SELECT … FOR UPDATE SKIP LOCKED` lease
|
||||
/// query every second (audit: analysis daemon idle poll). Mirrors the crawler
|
||||
/// daemon's backoff.
|
||||
fn idle_backoff(consecutive_empty: u32) -> Duration {
|
||||
let cap = IDLE_BACKOFF_CAP.as_secs();
|
||||
let secs = 1u64.checked_shl(consecutive_empty).unwrap_or(cap).min(cap);
|
||||
Duration::from_secs(secs)
|
||||
}
|
||||
|
||||
/// The unit of work: analyze one page. Implemented by
|
||||
/// [`RealAnalyzeDispatcher`] in production and stubbed in tests.
|
||||
#[async_trait]
|
||||
@@ -135,6 +151,9 @@ impl WorkerContext {
|
||||
// Last observed readiness, so we log only on transitions (not every
|
||||
// poll). `None` until the first probe.
|
||||
let mut was_ready: Option<bool> = None;
|
||||
// Empty lease polls seen in a row, driving the idle backoff. Reset to 0
|
||||
// the moment a job is leased.
|
||||
let mut consecutive_empty: u32 = 0;
|
||||
loop {
|
||||
if self.cancel.is_cancelled() {
|
||||
tracing::info!(worker = self.id, "analysis worker: shutdown");
|
||||
@@ -178,11 +197,15 @@ impl WorkerContext {
|
||||
}
|
||||
};
|
||||
let Some(lease) = leases.into_iter().next() else {
|
||||
if self.sleep_or_cancel(Duration::from_secs(1)).await {
|
||||
let wait = idle_backoff(consecutive_empty);
|
||||
consecutive_empty = consecutive_empty.saturating_add(1);
|
||||
if self.sleep_or_cancel(wait).await {
|
||||
return;
|
||||
}
|
||||
continue;
|
||||
};
|
||||
// Leased work — drop back to a tight poll cadence.
|
||||
consecutive_empty = 0;
|
||||
self.process_lease(lease).await;
|
||||
}
|
||||
}
|
||||
@@ -393,19 +416,20 @@ impl AnalyzeDispatcher for RealAnalyzeDispatcher {
|
||||
// Page was deleted between enqueue and dispatch — nothing to do.
|
||||
return Ok(());
|
||||
};
|
||||
let bytes = self
|
||||
// Stream through the byte cap so an oversized blob is rejected as it's
|
||||
// read rather than after it's fully buffered into memory (mirrors the
|
||||
// OCR dispatcher).
|
||||
let file = self
|
||||
.storage
|
||||
.get(&page.storage_key)
|
||||
.get_stream(&page.storage_key)
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("read page image {}: {e}", page.storage_key))?;
|
||||
if bytes.len() > self.max_image_bytes {
|
||||
anyhow::bail!(
|
||||
"page image {} is {} bytes, over the {} cap",
|
||||
page.storage_key,
|
||||
bytes.len(),
|
||||
self.max_image_bytes
|
||||
);
|
||||
}
|
||||
let bytes =
|
||||
crate::crawler::safety::accumulate_capped(file.stream, self.max_image_bytes)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
anyhow::anyhow!("page image {} over the byte cap: {e}", page.storage_key)
|
||||
})?;
|
||||
let analysis = self.vision.analyze(&bytes, &page.content_type).await?;
|
||||
repo::page_analysis::persist_analysis(&self.db, page_id, &analysis, &self.model).await?;
|
||||
Ok(())
|
||||
@@ -509,6 +533,17 @@ mod tests {
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::net::TcpListener;
|
||||
|
||||
#[test]
|
||||
fn idle_backoff_grows_exponentially_and_caps() {
|
||||
assert_eq!(idle_backoff(0), Duration::from_secs(1));
|
||||
assert_eq!(idle_backoff(1), Duration::from_secs(2));
|
||||
assert_eq!(idle_backoff(2), Duration::from_secs(4));
|
||||
assert_eq!(idle_backoff(4), Duration::from_secs(16));
|
||||
// Caps at IDLE_BACKOFF_CAP (30s) and never overflows on large counts.
|
||||
assert_eq!(idle_backoff(5), IDLE_BACKOFF_CAP);
|
||||
assert_eq!(idle_backoff(100), IDLE_BACKOFF_CAP);
|
||||
}
|
||||
|
||||
/// Bind an ephemeral port that answers every request with `status_line`
|
||||
/// (e.g. `"200 OK"`), and return its `/health` URL. The probe under test
|
||||
/// only inspects the status code, so a zero-length body is enough.
|
||||
|
||||
@@ -207,19 +207,21 @@ impl AnalyzeDispatcher for OcrAnalyzeDispatcher {
|
||||
// Page was deleted between enqueue and dispatch — nothing to do.
|
||||
return Ok(());
|
||||
};
|
||||
let bytes = self
|
||||
// Stream the blob through the byte cap so the read itself bails once
|
||||
// the running total exceeds `max_image_bytes` — a plain `get()` would
|
||||
// buffer the whole (possibly huge) image into memory before the cap
|
||||
// could reject it.
|
||||
let file = self
|
||||
.storage
|
||||
.get(&page.storage_key)
|
||||
.get_stream(&page.storage_key)
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("read page image {}: {e}", page.storage_key))?;
|
||||
if bytes.len() > self.max_image_bytes {
|
||||
anyhow::bail!(
|
||||
"page image {} is {} bytes, over the {} cap",
|
||||
page.storage_key,
|
||||
bytes.len(),
|
||||
self.max_image_bytes
|
||||
);
|
||||
}
|
||||
let bytes =
|
||||
crate::crawler::safety::accumulate_capped(file.stream, self.max_image_bytes)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
anyhow::anyhow!("page image {} over the byte cap: {e}", page.storage_key)
|
||||
})?;
|
||||
// OCR inference is CPU-bound and synchronous — keep it off the async
|
||||
// worker's runtime thread, and gate it behind the shared permit pool so
|
||||
// ANALYSIS_WORKERS > cores can't oversubscribe the blocking pool.
|
||||
|
||||
@@ -328,10 +328,21 @@ fn ensure_enabled(state: &AppState) -> AppResult<()> {
|
||||
async fn reenqueue(
|
||||
State(state): State<AppState>,
|
||||
admin: RequireAdmin,
|
||||
body: Option<Json<ReenqueueBody>>,
|
||||
raw: axum::body::Bytes,
|
||||
) -> AppResult<Json<ReenqueueResponse>> {
|
||||
ensure_enabled(&state)?;
|
||||
let body = body.map(|b| b.0).unwrap_or_default();
|
||||
// An ABSENT/empty body means the default "All" scope. A PRESENT body must be
|
||||
// valid JSON — `Option<Json<T>>` would silently collapse a malformed body to
|
||||
// None and run the full-library default the caller never intended, so parse
|
||||
// the raw bytes ourselves and 422 on a real parse error.
|
||||
let body: ReenqueueBody = if raw.iter().all(u8::is_ascii_whitespace) {
|
||||
ReenqueueBody::default()
|
||||
} else {
|
||||
serde_json::from_slice(&raw).map_err(|e| AppError::ValidationFailed {
|
||||
message: "reenqueue body is not valid JSON".into(),
|
||||
details: json!({ "body": e.to_string() }),
|
||||
})?
|
||||
};
|
||||
|
||||
// Resolve the scope. manga_id and chapter_id are mutually exclusive;
|
||||
// an unknown target is a 404 rather than a silent zero-enqueue.
|
||||
|
||||
@@ -16,7 +16,7 @@ use crate::api::auth::{validate_password, validate_username};
|
||||
use crate::api::pagination::PagedResponse;
|
||||
use crate::app::AppState;
|
||||
use crate::auth::extractor::RequireAdmin;
|
||||
use crate::auth::password::hash_password;
|
||||
use crate::auth::password::hash_password_async;
|
||||
use crate::domain::User;
|
||||
use crate::error::{AppError, AppResult};
|
||||
use crate::repo;
|
||||
@@ -115,7 +115,7 @@ async fn create_user(
|
||||
// reject (and vice versa).
|
||||
validate_username(username)?;
|
||||
validate_password(&input.password)?;
|
||||
let pwhash = hash_password(&input.password)?;
|
||||
let pwhash = hash_password_async(input.password.clone()).await?;
|
||||
let user = repo::user::admin_create_user(
|
||||
&state.db,
|
||||
actor.id,
|
||||
|
||||
@@ -17,8 +17,8 @@ use serde::{Deserialize, Serialize};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::app::AppState;
|
||||
use crate::auth::extractor::{CurrentUser, SESSION_COOKIE_NAME};
|
||||
use crate::auth::password::{hash_password, verify_password};
|
||||
use crate::auth::extractor::{ClientIp, CurrentUser, SESSION_COOKIE_NAME};
|
||||
use crate::auth::password::{hash_password_async, verify_password_async};
|
||||
use crate::auth::token::{generate_token, hash_token};
|
||||
use crate::config::AuthConfig;
|
||||
use crate::domain::user_preferences::{READER_GAPS, READER_MODES};
|
||||
@@ -38,7 +38,7 @@ pub fn routes() -> Router<AppState> {
|
||||
"/auth/me/preferences",
|
||||
get(get_preferences).patch(update_preferences),
|
||||
)
|
||||
.route("/auth/tokens", post(create_token))
|
||||
.route("/auth/tokens", get(list_tokens).post(create_token))
|
||||
.route("/auth/tokens/:id", delete(delete_token))
|
||||
}
|
||||
|
||||
@@ -107,6 +107,7 @@ pub struct CreatedTokenResponse {
|
||||
|
||||
async fn register(
|
||||
State(state): State<AppState>,
|
||||
ClientIp(client_ip): ClientIp,
|
||||
jar: CookieJar,
|
||||
Json(input): Json<Credentials>,
|
||||
) -> AppResult<impl IntoResponse> {
|
||||
@@ -114,7 +115,7 @@ async fn register(
|
||||
// the toggle can't be probed for the toggle state via timing —
|
||||
// disabled and enabled paths both consume a token, and disabled
|
||||
// returns 403 instead of running argon2.
|
||||
check_auth_rate_limit(&state, "register")?;
|
||||
check_auth_rate_limit(&state, "register", client_ip)?;
|
||||
// Private mode force-blocks self-registration regardless of
|
||||
// ALLOW_SELF_REGISTER — operators of locked-down instances mint
|
||||
// accounts via `POST /admin/users` instead.
|
||||
@@ -125,7 +126,7 @@ async fn register(
|
||||
validate_username(username)?;
|
||||
validate_password(&input.password)?;
|
||||
|
||||
let pwhash = hash_password(&input.password)?;
|
||||
let pwhash = hash_password_async(input.password.clone()).await?;
|
||||
let user = repo::user::create(&state.db, username, &pwhash).await?;
|
||||
let jar = start_session(&state, &user, jar).await?;
|
||||
Ok((StatusCode::CREATED, jar, Json(AuthResponse { user })))
|
||||
@@ -133,10 +134,11 @@ async fn register(
|
||||
|
||||
async fn login(
|
||||
State(state): State<AppState>,
|
||||
ClientIp(client_ip): ClientIp,
|
||||
jar: CookieJar,
|
||||
Json(input): Json<Credentials>,
|
||||
) -> AppResult<impl IntoResponse> {
|
||||
check_auth_rate_limit(&state, "login")?;
|
||||
check_auth_rate_limit(&state, "login", client_ip)?;
|
||||
let username = input.username.trim();
|
||||
if username.is_empty() || input.password.is_empty() {
|
||||
return Err(AppError::InvalidInput(
|
||||
@@ -154,10 +156,11 @@ async fn login(
|
||||
// response time matches the wrong-password branch — otherwise
|
||||
// an attacker can enumerate usernames by timing the no-user
|
||||
// 401 against the wrong-password 401.
|
||||
let _ = verify_password(&input.password, dummy_password_hash());
|
||||
let _ = verify_password_async(input.password.clone(), dummy_password_hash().to_string())
|
||||
.await;
|
||||
return Err(AppError::Unauthenticated);
|
||||
};
|
||||
if !verify_password(&input.password, &user.password_hash) {
|
||||
if !verify_password_async(input.password.clone(), user.password_hash.clone()).await {
|
||||
return Err(AppError::Unauthenticated);
|
||||
}
|
||||
|
||||
@@ -213,19 +216,20 @@ async fn me(CurrentUser(user): CurrentUser) -> AppResult<Json<AuthResponse>> {
|
||||
async fn change_password(
|
||||
State(state): State<AppState>,
|
||||
CurrentUser(user): CurrentUser,
|
||||
ClientIp(client_ip): ClientIp,
|
||||
jar: CookieJar,
|
||||
Json(input): Json<ChangePassword>,
|
||||
) -> AppResult<impl IntoResponse> {
|
||||
check_auth_rate_limit(&state, "change_password")?;
|
||||
check_auth_rate_limit(&state, "change_password", client_ip)?;
|
||||
// Cap current_password before verify_password runs argon2 (same DoS
|
||||
// vector as login). new_password is bounded by validate_password below.
|
||||
reject_oversized_password(&input.current_password)?;
|
||||
if !verify_password(&input.current_password, &user.password_hash) {
|
||||
if !verify_password_async(input.current_password.clone(), user.password_hash.clone()).await {
|
||||
return Err(AppError::Unauthenticated);
|
||||
}
|
||||
validate_password(&input.new_password)?;
|
||||
|
||||
let new_hash = hash_password(&input.new_password)?;
|
||||
let new_hash = hash_password_async(input.new_password.clone()).await?;
|
||||
|
||||
let mut tx = state.db.begin().await?;
|
||||
sqlx::query("UPDATE users SET password_hash = $1 WHERE id = $2")
|
||||
@@ -295,6 +299,22 @@ async fn update_preferences(
|
||||
Ok(Json(saved))
|
||||
}
|
||||
|
||||
/// `GET /auth/tokens` — the caller's bot tokens (newest first). The raw bearer
|
||||
/// is only ever shown once at creation, so this list carries just the metadata
|
||||
/// (name, created/last-used, expiry); `token_hash` is `#[serde(skip)]`.
|
||||
#[derive(Debug, Serialize)]
|
||||
struct TokenListResponse {
|
||||
items: Vec<ApiToken>,
|
||||
}
|
||||
|
||||
async fn list_tokens(
|
||||
State(state): State<AppState>,
|
||||
CurrentUser(user): CurrentUser,
|
||||
) -> AppResult<Json<TokenListResponse>> {
|
||||
let items = repo::api_token::list_for_user(&state.db, user.id).await?;
|
||||
Ok(Json(TokenListResponse { items }))
|
||||
}
|
||||
|
||||
async fn create_token(
|
||||
State(state): State<AppState>,
|
||||
CurrentUser(user): CurrentUser,
|
||||
@@ -416,9 +436,13 @@ fn build_expired_cookie(cfg: &AuthConfig) -> Cookie<'static> {
|
||||
/// any one of them in a tight loop should trip the limit. `endpoint`
|
||||
/// is included in the rate-limit-hit log line so operators can tell
|
||||
/// which endpoint is being probed.
|
||||
fn check_auth_rate_limit(state: &AppState, endpoint: &'static str) -> AppResult<()> {
|
||||
fn check_auth_rate_limit(
|
||||
state: &AppState,
|
||||
endpoint: &'static str,
|
||||
client_ip: Option<std::net::IpAddr>,
|
||||
) -> AppResult<()> {
|
||||
use crate::auth::rate_limit::AcquireResult;
|
||||
match state.auth_limiter.try_acquire() {
|
||||
match state.auth_limiter.try_acquire(client_ip) {
|
||||
AcquireResult::Allowed => Ok(()),
|
||||
AcquireResult::Denied { retry_after_secs } => {
|
||||
tracing::warn!(
|
||||
|
||||
@@ -72,7 +72,7 @@ async fn create(
|
||||
}
|
||||
}
|
||||
|
||||
let bookmark = repo::bookmark::create(
|
||||
let (bookmark, created) = repo::bookmark::create(
|
||||
&state.db,
|
||||
user.id,
|
||||
input.manga_id,
|
||||
@@ -103,7 +103,13 @@ async fn create(
|
||||
}
|
||||
});
|
||||
|
||||
Ok((StatusCode::CREATED, Json(bookmark)))
|
||||
// 201 for a fresh bookmark, 200 when it already existed (idempotent add).
|
||||
let status = if created {
|
||||
StatusCode::CREATED
|
||||
} else {
|
||||
StatusCode::OK
|
||||
};
|
||||
Ok((status, Json(bookmark)))
|
||||
}
|
||||
|
||||
async fn delete_one(
|
||||
|
||||
@@ -13,7 +13,7 @@ use serde::Deserialize;
|
||||
use serde_json::json;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::api::mangas::{next_field, read_field_bytes};
|
||||
use crate::api::mangas::next_field;
|
||||
use crate::api::pagination::PagedResponse;
|
||||
use crate::app::AppState;
|
||||
use crate::auth::extractor::CurrentUser;
|
||||
@@ -107,7 +107,12 @@ async fn create(
|
||||
while let Some(field) = next_field(&mut multipart).await? {
|
||||
match field.name() {
|
||||
Some("metadata") => {
|
||||
let bytes = read_field_bytes(field).await?;
|
||||
let bytes = crate::upload::read_capped(
|
||||
field,
|
||||
crate::upload::MAX_METADATA_BYTES,
|
||||
"metadata",
|
||||
)
|
||||
.await?;
|
||||
metadata = Some(serde_json::from_slice(&bytes).map_err(|e| {
|
||||
AppError::ValidationFailed {
|
||||
message: "metadata is not valid JSON".into(),
|
||||
|
||||
@@ -5,6 +5,13 @@
|
||||
//! The handler uses `Storage::get_stream` so a multi-MB page is piped to
|
||||
//! the client a chunk at a time instead of buffered server-side.
|
||||
//!
|
||||
//! **Thumbnails.** `?w=<px>` serves a width-bounded variant (grids ask for a
|
||||
//! small width so they download ~KB instead of the 1–5 MB original). The width
|
||||
//! snaps to a small allow-list so the number of cached derivatives stays
|
||||
//! bounded; the resized image is cached in storage under a `thumbs/w{W}/` prefix
|
||||
//! and regenerated on demand. Only JPEG/PNG sources are thumbnailed (encoders we
|
||||
//! ship); other formats fall back to the original.
|
||||
//!
|
||||
//! **Auth model — capability URLs by design.** This endpoint is
|
||||
//! deliberately unauthenticated: reads stay public per the project
|
||||
//! brief, and per-page authorisation would require either a per-request
|
||||
@@ -16,41 +23,250 @@
|
||||
//! would gate this endpoint behind a `Storage::owner_of(key)` check;
|
||||
//! the seam is intentional.
|
||||
|
||||
use std::io::Cursor;
|
||||
|
||||
use axum::body::Body;
|
||||
use axum::extract::{Path, State};
|
||||
use axum::extract::{Path, Query, State};
|
||||
use axum::http::{header, HeaderName};
|
||||
use axum::response::{IntoResponse, Response};
|
||||
use axum::routing::get;
|
||||
use axum::Router;
|
||||
use image::imageops::FilterType;
|
||||
use image::{ImageFormat, ImageReader};
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::app::AppState;
|
||||
use crate::error::AppResult;
|
||||
use crate::storage::StorageError;
|
||||
use crate::error::{AppError, AppResult};
|
||||
use crate::storage::{Storage, StorageError};
|
||||
|
||||
/// Widths a thumbnail may be rendered at. A requested width snaps up to the
|
||||
/// smallest of these so the set of cached derivatives stays tiny.
|
||||
const ALLOWED_THUMB_WIDTHS: &[u32] = &[160, 320, 480, 640, 960];
|
||||
|
||||
/// Storage key prefix for cached thumbnails.
|
||||
const THUMB_PREFIX: &str = "thumbs";
|
||||
|
||||
/// Decode allocation cap (mirrors `analysis::ocr`): a tiny file declaring huge
|
||||
/// dimensions is rejected before the decoder allocates, not after (OOM guard).
|
||||
const MAX_THUMB_DECODE_PIXELS: u64 = 40_000_000;
|
||||
|
||||
pub fn routes() -> Router<AppState> {
|
||||
Router::new().route("/files/*key", get(serve))
|
||||
}
|
||||
|
||||
async fn serve(State(state): State<AppState>, Path(key): Path<String>) -> AppResult<Response> {
|
||||
let file = match state.storage.get_stream(&key).await {
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct ServeQuery {
|
||||
/// Requested thumbnail width in pixels; absent = serve the original.
|
||||
w: Option<String>,
|
||||
}
|
||||
|
||||
async fn serve(
|
||||
State(state): State<AppState>,
|
||||
Path(key): Path<String>,
|
||||
Query(q): Query<ServeQuery>,
|
||||
) -> AppResult<Response> {
|
||||
// Thumbnail request: only for source formats we can re-encode; anything
|
||||
// else falls through to serving the original.
|
||||
if let Some(width) = resolve_thumb_width(q.w.as_deref()) {
|
||||
if let Some(fmt) = thumb_format_for(&key) {
|
||||
return serve_thumbnail(&state, &key, width, fmt).await;
|
||||
}
|
||||
}
|
||||
serve_original(&state, &key).await
|
||||
}
|
||||
|
||||
async fn serve_original(state: &AppState, key: &str) -> AppResult<Response> {
|
||||
let file = match state.storage.get_stream(key).await {
|
||||
Ok(f) => f,
|
||||
Err(StorageError::NotFound) => return Err(crate::error::AppError::NotFound),
|
||||
Err(StorageError::NotFound) => return Err(AppError::NotFound),
|
||||
Err(e) => return Err(e.into()),
|
||||
};
|
||||
let ct = content_type_for(&key);
|
||||
// `nosniff` makes the contract explicit: the browser must trust the
|
||||
// Content-Type we declared (and that the magic-byte sniff at upload
|
||||
// time produced) instead of trying to detect HTML/JS in the body.
|
||||
// Belt-and-braces vs. polyglot files that survive the upload sniff.
|
||||
let headers = [
|
||||
(header::CONTENT_TYPE, ct.to_string()),
|
||||
(header::CONTENT_LENGTH, file.size_bytes.to_string()),
|
||||
Ok(image_response(
|
||||
state,
|
||||
content_type_for(key),
|
||||
file.size_bytes.to_string(),
|
||||
Body::from_stream(file.stream),
|
||||
))
|
||||
}
|
||||
|
||||
async fn serve_thumbnail(
|
||||
state: &AppState,
|
||||
key: &str,
|
||||
width: u32,
|
||||
fmt: ImageFormat,
|
||||
) -> AppResult<Response> {
|
||||
let derived = thumb_key(key, width);
|
||||
|
||||
// Serve the cached variant if it exists.
|
||||
match state.storage.get_stream(&derived).await {
|
||||
Ok(f) => {
|
||||
return Ok(image_response(
|
||||
state,
|
||||
content_type_for(key),
|
||||
f.size_bytes.to_string(),
|
||||
Body::from_stream(f.stream),
|
||||
));
|
||||
}
|
||||
Err(StorageError::NotFound) => {}
|
||||
Err(e) => return Err(e.into()),
|
||||
}
|
||||
|
||||
// Generate from the original.
|
||||
let original = match state.storage.get(key).await {
|
||||
Ok(b) => b,
|
||||
Err(StorageError::NotFound) => return Err(AppError::NotFound),
|
||||
Err(e) => return Err(e.into()),
|
||||
};
|
||||
// Resizing is CPU-bound; keep it off the async worker threads.
|
||||
let thumb = match tokio::task::spawn_blocking(move || make_thumbnail(&original, width, fmt))
|
||||
.await
|
||||
.map_err(|e| AppError::Other(anyhow::anyhow!("thumbnail task join: {e}")))?
|
||||
{
|
||||
Ok(t) => t,
|
||||
Err(e) => {
|
||||
// The blob's magic bytes passed upload's sniff but the body won't
|
||||
// decode (corrupt/truncated, or an encoder feature we don't support).
|
||||
// Don't 500 the reader — fall back to streaming the original, which
|
||||
// serves without decoding.
|
||||
tracing::warn!(key, error = %format!("{e:#}"), "thumbnail decode failed; serving original");
|
||||
return serve_original(state, key).await;
|
||||
}
|
||||
};
|
||||
|
||||
// Best-effort cache; a write failure just means we regenerate next time.
|
||||
let _ = state.storage.put(&derived, &thumb).await;
|
||||
|
||||
let len = thumb.len().to_string();
|
||||
Ok(image_response(
|
||||
state,
|
||||
content_type_for(key),
|
||||
len,
|
||||
Body::from(thumb),
|
||||
))
|
||||
}
|
||||
|
||||
/// Shared response builder for both the original and thumbnail paths.
|
||||
fn image_response(
|
||||
state: &AppState,
|
||||
content_type: &str,
|
||||
content_length: String,
|
||||
body: Body,
|
||||
) -> Response {
|
||||
let mut headers = vec![
|
||||
(header::CONTENT_TYPE, content_type.to_string()),
|
||||
(header::CONTENT_LENGTH, content_length),
|
||||
// `nosniff` makes the contract explicit: the browser must trust the
|
||||
// Content-Type we declared (and that the magic-byte sniff at upload
|
||||
// time produced) instead of trying to detect HTML/JS in the body.
|
||||
(
|
||||
HeaderName::from_static("x-content-type-options"),
|
||||
"nosniff".to_string(),
|
||||
),
|
||||
// Blobs are content-addressed by unguessable, immutable keys (a
|
||||
// re-upload mints new UUIDs), so a fetched page/cover never changes.
|
||||
// Cache it for a year and mark it `immutable` so browsers skip
|
||||
// revalidation entirely.
|
||||
//
|
||||
// BUT under PRIVATE_MODE these blobs are auth-gated, so they must NOT be
|
||||
// marked `public`: a shared cache / CDN would store the object and serve
|
||||
// it to unauthenticated clients. Use `private` so only the requesting
|
||||
// user's browser caches it.
|
||||
(
|
||||
header::CACHE_CONTROL,
|
||||
if state.auth.private_mode {
|
||||
"private, max-age=31536000, immutable".to_string()
|
||||
} else {
|
||||
"public, max-age=31536000, immutable".to_string()
|
||||
},
|
||||
),
|
||||
];
|
||||
Ok((headers, Body::from_stream(file.stream)).into_response())
|
||||
// Known image types render inline (covers/pages display in the reader). The
|
||||
// `application/octet-stream` fallback is a blob we couldn't type — it could
|
||||
// be crafted HTML/JS, so force a download rather than let the browser render
|
||||
// it inline. Belt to `nosniff`'s braces.
|
||||
if content_type == "application/octet-stream" {
|
||||
headers.push((
|
||||
header::CONTENT_DISPOSITION,
|
||||
"attachment".to_string(),
|
||||
));
|
||||
}
|
||||
(axum::response::AppendHeaders(headers), body).into_response()
|
||||
}
|
||||
|
||||
/// Parse and clamp a requested thumbnail width. Returns `None` for absent /
|
||||
/// unparseable / zero widths (serve the original); otherwise snaps the request
|
||||
/// up to the smallest allowed width (capped at the largest) so cached variants
|
||||
/// stay bounded.
|
||||
fn resolve_thumb_width(raw: Option<&str>) -> Option<u32> {
|
||||
let requested: u32 = raw?.trim().parse().ok()?;
|
||||
if requested == 0 {
|
||||
return None;
|
||||
}
|
||||
Some(
|
||||
ALLOWED_THUMB_WIDTHS
|
||||
.iter()
|
||||
.copied()
|
||||
.find(|&w| w >= requested)
|
||||
.unwrap_or_else(|| *ALLOWED_THUMB_WIDTHS.last().expect("non-empty")),
|
||||
)
|
||||
}
|
||||
|
||||
/// The re-encode format for a source key, or `None` when it isn't one we ship an
|
||||
/// encoder for (gif/avif → serve the original instead of a broken thumbnail).
|
||||
fn thumb_format_for(key: &str) -> Option<ImageFormat> {
|
||||
match content_type_for(key) {
|
||||
"image/jpeg" => Some(ImageFormat::Jpeg),
|
||||
"image/png" => Some(ImageFormat::Png),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// The storage key a width-`w` thumbnail of `key` is cached under.
|
||||
fn thumb_key(key: &str, width: u32) -> String {
|
||||
format!("{THUMB_PREFIX}/w{width}/{key}")
|
||||
}
|
||||
|
||||
/// Every cached thumbnail key for an original `key`, across all allowed widths.
|
||||
/// Used by the cover handlers to purge stale variants when a cover (whose key is
|
||||
/// reused, unlike content-addressed pages) is replaced or deleted.
|
||||
pub(crate) fn thumbnail_keys(key: &str) -> Vec<String> {
|
||||
ALLOWED_THUMB_WIDTHS
|
||||
.iter()
|
||||
.map(|&w| thumb_key(key, w))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Best-effort deletion of every cached thumbnail for `key`. Call after the
|
||||
/// underlying blob at `key` changes or is removed.
|
||||
pub(crate) async fn purge_thumbnails(storage: &dyn Storage, key: &str) {
|
||||
for derived in thumbnail_keys(key) {
|
||||
let _ = storage.delete(&derived).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Decode, downscale to `width` (aspect-preserving, never upscaling), and
|
||||
/// re-encode in `fmt`. Pure + synchronous so it runs under `spawn_blocking` and
|
||||
/// is unit-testable without a server.
|
||||
fn make_thumbnail(bytes: &[u8], width: u32, fmt: ImageFormat) -> anyhow::Result<Vec<u8>> {
|
||||
use anyhow::Context;
|
||||
let mut reader = ImageReader::new(Cursor::new(bytes))
|
||||
.with_guessed_format()
|
||||
.context("guess image format for thumbnail")?;
|
||||
let mut limits = image::Limits::default();
|
||||
limits.max_alloc = Some(MAX_THUMB_DECODE_PIXELS.saturating_mul(4));
|
||||
reader.limits(limits);
|
||||
let img = reader.decode().context("decode image for thumbnail")?;
|
||||
|
||||
// Only downscale; a source narrower than the target is served as-is.
|
||||
let out = if img.width() > width {
|
||||
img.resize(width, u32::MAX, FilterType::Lanczos3)
|
||||
} else {
|
||||
img
|
||||
};
|
||||
|
||||
let mut buf = Cursor::new(Vec::new());
|
||||
out.write_to(&mut buf, fmt).context("encode thumbnail")?;
|
||||
Ok(buf.into_inner())
|
||||
}
|
||||
|
||||
fn content_type_for(key: &str) -> &'static str {
|
||||
@@ -64,3 +280,68 @@ fn content_type_for(key: &str) -> &'static str {
|
||||
_ => "application/octet-stream",
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn resolve_thumb_width_snaps_and_validates() {
|
||||
// Absent / unparseable / zero → serve the original.
|
||||
assert_eq!(resolve_thumb_width(None), None);
|
||||
assert_eq!(resolve_thumb_width(Some("")), None);
|
||||
assert_eq!(resolve_thumb_width(Some("abc")), None);
|
||||
assert_eq!(resolve_thumb_width(Some("0")), None);
|
||||
// Snap up to the smallest allowed width.
|
||||
assert_eq!(resolve_thumb_width(Some("1")), Some(160));
|
||||
assert_eq!(resolve_thumb_width(Some("160")), Some(160));
|
||||
assert_eq!(resolve_thumb_width(Some("161")), Some(320));
|
||||
assert_eq!(resolve_thumb_width(Some("640")), Some(640));
|
||||
// Above the max → capped at the largest allowed width.
|
||||
assert_eq!(resolve_thumb_width(Some("5000")), Some(960));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn thumb_format_only_for_encodable_sources() {
|
||||
assert_eq!(thumb_format_for("a/b/cover.jpg"), Some(ImageFormat::Jpeg));
|
||||
assert_eq!(thumb_format_for("a/b/cover.png"), Some(ImageFormat::Png));
|
||||
// Formats we can't re-encode fall back to the original.
|
||||
assert_eq!(thumb_format_for("a/b/cover.webp"), None);
|
||||
assert_eq!(thumb_format_for("a/b/cover.gif"), None);
|
||||
assert_eq!(thumb_format_for("a/b/cover.avif"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn thumb_key_is_prefixed_by_width() {
|
||||
assert_eq!(
|
||||
thumb_key("mangas/x/cover.png", 320),
|
||||
"thumbs/w320/mangas/x/cover.png"
|
||||
);
|
||||
assert_eq!(thumbnail_keys("k.png").len(), ALLOWED_THUMB_WIDTHS.len());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn make_thumbnail_downscales_and_preserves_aspect() {
|
||||
// 100x50 red PNG → thumbnail width 40 → 40x20, still decodable PNG.
|
||||
let mut src = Cursor::new(Vec::new());
|
||||
image::RgbImage::from_pixel(100, 50, image::Rgb([255, 0, 0]))
|
||||
.write_to(&mut src, ImageFormat::Png)
|
||||
.unwrap();
|
||||
let out = make_thumbnail(src.get_ref(), 40, ImageFormat::Png).unwrap();
|
||||
let decoded = image::load_from_memory(&out).unwrap();
|
||||
assert_eq!(decoded.width(), 40);
|
||||
assert_eq!(decoded.height(), 20, "aspect ratio preserved");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn make_thumbnail_does_not_upscale() {
|
||||
// A 30px-wide source requested at 40 stays 30 wide (no upscaling).
|
||||
let mut src = Cursor::new(Vec::new());
|
||||
image::RgbImage::from_pixel(30, 30, image::Rgb([0, 255, 0]))
|
||||
.write_to(&mut src, ImageFormat::Png)
|
||||
.unwrap();
|
||||
let out = make_thumbnail(src.get_ref(), 40, ImageFormat::Png).unwrap();
|
||||
let decoded = image::load_from_memory(&out).unwrap();
|
||||
assert_eq!(decoded.width(), 30);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -157,7 +157,9 @@ async fn list(
|
||||
order,
|
||||
};
|
||||
let (items, total) = repo::manga::list_cards(&state.db, &q).await?;
|
||||
Ok(Json(PagedResponse::with_total(items, limit, offset, total)))
|
||||
Ok(Json(PagedResponse::with_optional_total(
|
||||
items, limit, offset, total,
|
||||
)))
|
||||
}
|
||||
|
||||
async fn get_one(
|
||||
@@ -233,11 +235,17 @@ async fn create(
|
||||
while let Some(field) = next_field(&mut multipart).await? {
|
||||
match field.name() {
|
||||
Some("metadata") => {
|
||||
let bytes = read_field_bytes(field).await?;
|
||||
let bytes = crate::upload::read_capped(
|
||||
field,
|
||||
crate::upload::MAX_METADATA_BYTES,
|
||||
"metadata",
|
||||
)
|
||||
.await?;
|
||||
metadata = Some(parse_metadata_json(&bytes)?);
|
||||
}
|
||||
Some("cover") => {
|
||||
let bytes = read_field_bytes(field).await?.to_vec();
|
||||
let bytes =
|
||||
crate::upload::read_capped(field, state.upload.max_file_bytes, "cover").await?;
|
||||
cover = Some(parse_image(bytes, state.upload.max_file_bytes, "cover")?);
|
||||
}
|
||||
_ => continue,
|
||||
@@ -385,7 +393,8 @@ async fn put_cover(
|
||||
let mut cover: Option<UploadedImage> = None;
|
||||
while let Some(field) = next_field(&mut multipart).await? {
|
||||
if field.name() == Some("cover") {
|
||||
let bytes = read_field_bytes(field).await?.to_vec();
|
||||
let bytes =
|
||||
crate::upload::read_capped(field, state.upload.max_file_bytes, "cover").await?;
|
||||
cover = Some(parse_image(bytes, state.upload.max_file_bytes, "cover")?);
|
||||
}
|
||||
}
|
||||
@@ -400,6 +409,17 @@ async fn put_cover(
|
||||
let old_key = repo::manga::get(&state.db, id).await?.cover_image_path;
|
||||
let new_key = format!("mangas/{}/cover.{}", id, img.ext);
|
||||
state.storage.put(&new_key, &img.bytes).await?;
|
||||
// Cover keys are reused (mangas/{id}/cover.{ext}), so a same-extension
|
||||
// replacement overwrites the blob at an existing key — drop any thumbnails
|
||||
// cached for it so we don't serve a stale variant of the old cover.
|
||||
crate::api::files::purge_thumbnails(state.storage.as_ref(), &new_key).await;
|
||||
|
||||
// Commit the DB pointer to the new blob BEFORE removing the old one. If we
|
||||
// deleted the old blob first and this write then failed, the row would point
|
||||
// at a deleted blob (a cover that 404s) and the new blob would be orphaned.
|
||||
// Ordering it first means a failure here leaves the row pointing at the
|
||||
// still-present old blob; only a harmless orphan (new blob) can result.
|
||||
repo::manga::set_cover_image_path(&state.db, id, &new_key, img.bytes.len() as i64).await?;
|
||||
|
||||
if let Some(prev) = old_key.as_deref() {
|
||||
if prev != new_key {
|
||||
@@ -410,10 +430,10 @@ async fn put_cover(
|
||||
Ok(()) | Err(StorageError::NotFound) => {}
|
||||
Err(e) => return Err(e.into()),
|
||||
}
|
||||
// Old key's cached thumbnails are now orphaned.
|
||||
crate::api::files::purge_thumbnails(state.storage.as_ref(), prev).await;
|
||||
}
|
||||
}
|
||||
|
||||
repo::manga::set_cover_image_path(&state.db, id, &new_key, img.bytes.len() as i64).await?;
|
||||
Ok(Json(repo::manga::get_detail(&state.db, id).await?))
|
||||
}
|
||||
|
||||
@@ -431,11 +451,15 @@ async fn delete_cover(
|
||||
}
|
||||
require_can_edit(&state, id, user.id, admin_via_session(&session)).await?;
|
||||
if let Some(key) = repo::manga::get(&state.db, id).await?.cover_image_path {
|
||||
// Clear the DB pointer BEFORE deleting the blob, so a storage-delete
|
||||
// failure can't leave the row pointing at a removed blob. A failure
|
||||
// after the clear only orphans the blob (harmless).
|
||||
repo::manga::clear_cover_image_path(&state.db, id).await?;
|
||||
match state.storage.delete(&key).await {
|
||||
Ok(()) | Err(StorageError::NotFound) => {}
|
||||
Err(e) => return Err(e.into()),
|
||||
}
|
||||
repo::manga::clear_cover_image_path(&state.db, id).await?;
|
||||
crate::api::files::purge_thumbnails(state.storage.as_ref(), &key).await;
|
||||
}
|
||||
Ok(Json(repo::manga::get_detail(&state.db, id).await?))
|
||||
}
|
||||
@@ -666,12 +690,6 @@ pub(crate) async fn next_field(
|
||||
.map_err(map_multipart_error)
|
||||
}
|
||||
|
||||
pub(crate) async fn read_field_bytes(
|
||||
field: axum::extract::multipart::Field<'_>,
|
||||
) -> AppResult<axum::body::Bytes> {
|
||||
field.bytes().await.map_err(map_multipart_error)
|
||||
}
|
||||
|
||||
pub(crate) fn map_multipart_error(e: axum::extract::multipart::MultipartError) -> AppError {
|
||||
let status = e.status();
|
||||
if status == StatusCode::PAYLOAD_TOO_LARGE {
|
||||
|
||||
@@ -34,4 +34,18 @@ impl<T> PagedResponse<T> {
|
||||
page: PageInfo { limit, offset, total: Some(total) },
|
||||
}
|
||||
}
|
||||
|
||||
/// For handlers that compute `total` only on some pages (e.g. the first)
|
||||
/// and leave it `None` elsewhere.
|
||||
pub fn with_optional_total(
|
||||
items: Vec<T>,
|
||||
limit: i64,
|
||||
offset: i64,
|
||||
total: Option<i64>,
|
||||
) -> Self {
|
||||
Self {
|
||||
items,
|
||||
page: PageInfo { limit, offset, total },
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -277,9 +277,15 @@ impl DaemonReloader for Supervisors {
|
||||
}
|
||||
}
|
||||
|
||||
/// How often the background reaper sweeps expired sessions. Hourly is ample:
|
||||
/// the sweep is a single indexed DELETE and expired rows are already invisible
|
||||
/// to auth, so this is purely storage hygiene.
|
||||
const SESSION_GC_INTERVAL: std::time::Duration = std::time::Duration::from_secs(3600);
|
||||
|
||||
pub async fn build(config: Config) -> anyhow::Result<AppHandle> {
|
||||
let db = PgPoolOptions::new()
|
||||
.max_connections(10)
|
||||
.max_connections(config.db.max_connections)
|
||||
.acquire_timeout(config.db.acquire_timeout)
|
||||
.connect(&config.database_url)
|
||||
.await?;
|
||||
sqlx::migrate!("./migrations").run(&db).await?;
|
||||
@@ -336,6 +342,27 @@ pub async fn build(config: Config) -> anyhow::Result<AppHandle> {
|
||||
tracing::info!("analysis worker disabled");
|
||||
}
|
||||
|
||||
// Periodic reaper for lapsed sessions. `find_active` already ignores
|
||||
// expired rows, so this only reclaims storage — without it the table grows
|
||||
// unbounded as sessions lapse. Detached and best-effort: a failed sweep is
|
||||
// logged and retried next tick. Runs regardless of crawler/analysis config
|
||||
// since sessions exist in every deployment.
|
||||
{
|
||||
let db = db.clone();
|
||||
tokio::spawn(async move {
|
||||
let mut ticker = tokio::time::interval(SESSION_GC_INTERVAL);
|
||||
ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
|
||||
loop {
|
||||
ticker.tick().await;
|
||||
match repo::session::delete_expired(&db).await {
|
||||
Ok(0) => {}
|
||||
Ok(n) => tracing::info!(reaped = n, "session gc: removed expired sessions"),
|
||||
Err(e) => tracing::warn!(?e, "session gc sweep failed; retrying next tick"),
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
let auth_limiter = Arc::new(AuthRateLimiter::new(config.auth.rate_limit));
|
||||
let state = AppState {
|
||||
db,
|
||||
@@ -475,6 +502,12 @@ async fn spawn_analysis_daemon(
|
||||
// endpoint is a single admin-configured URL), so the policy
|
||||
// enforces scheme + private-IP only.
|
||||
.redirect(crate::crawler::safety::public_redirect_policy())
|
||||
// NOTE: deliberately no `.dns_resolver(safe_dns_resolver())`.
|
||||
// The vision endpoint is a single operator-configured internal
|
||||
// service (e.g. `mangalord-vision`) that legitimately resolves
|
||||
// to a private Docker IP; a private-IP-rejecting resolver drops
|
||||
// every call. Redirect hops stay guarded by the policy above,
|
||||
// and the URL is operator-set, not attacker-controlled input.
|
||||
.build()
|
||||
.context("build analysis http client")?;
|
||||
let vision = crate::analysis::vision::VisionClient::new(http, cfg);
|
||||
@@ -501,6 +534,8 @@ async fn spawn_analysis_daemon(
|
||||
// uptime + vision health.
|
||||
.no_proxy()
|
||||
.redirect(crate::crawler::safety::public_redirect_policy())
|
||||
// No private-IP resolver: same operator-configured
|
||||
// internal vision host as the analysis client above.
|
||||
.build()
|
||||
.context("build vision readiness http client")?;
|
||||
Some(Arc::new(crate::analysis::daemon::HttpVisionReadiness {
|
||||
@@ -558,6 +593,10 @@ async fn spawn_crawler_daemon(
|
||||
cfg: &CrawlerConfig,
|
||||
analysis_enabled: Arc<AtomicBool>,
|
||||
) -> anyhow::Result<SpawnedDaemon> {
|
||||
// Publish the opt-in browser SSRF-interception toggle so every headless
|
||||
// navigation (via `intercept::open_page`) honours it. Off by default.
|
||||
crate::crawler::intercept::set_enabled(cfg.ssrf_intercept);
|
||||
|
||||
// Reqwest client with a shared cookie jar so CDN image fetches include
|
||||
// PHPSESSID. The same `Arc<Jar>` is held by the SessionController, so a
|
||||
// runtime session refresh rewrites it in place. Initial value: a
|
||||
@@ -593,6 +632,17 @@ async fn spawn_crawler_daemon(
|
||||
http_builder = http_builder
|
||||
.proxy(reqwest::Proxy::all(proxy).with_context(|| format!("parse proxy: {proxy}"))?);
|
||||
}
|
||||
// DNS-rebinding guard: reject hosts that resolve to a private/internal IP,
|
||||
// complementing the string-level allowlist check which can't see
|
||||
// post-resolution addresses. Attached on the direct path AND on http(s)
|
||||
// proxies (reqwest resolves the target itself there). Skipped only for SOCKS
|
||||
// proxies, where the proxy — not reqwest — resolves the target, so the only
|
||||
// name this resolver would see is the proxy's OWN host (legitimately on a
|
||||
// private Docker IP, e.g. `tor` → 172.x); attaching it there rejected every
|
||||
// fetch for zero gain. See `should_attach_safe_resolver`.
|
||||
if crate::crawler::safety::should_attach_safe_resolver(cfg.proxy.as_deref()) {
|
||||
http_builder = http_builder.dns_resolver(crate::crawler::safety::safe_dns_resolver());
|
||||
}
|
||||
let http = http_builder.build().context("build crawler reqwest")?;
|
||||
|
||||
let mut rate = HostRateLimiters::new(std::time::Duration::from_millis(cfg.rate_ms));
|
||||
@@ -966,7 +1016,17 @@ impl ChapterDispatcher for RealChapterDispatcher {
|
||||
pages_done: 0,
|
||||
pages_total: None,
|
||||
});
|
||||
let lease = self.browser_manager.acquire().await?;
|
||||
let lease = match self.browser_manager.acquire().await {
|
||||
Ok(l) => l,
|
||||
Err(e) => {
|
||||
// Browser down / mid-restart: defer the job WITHOUT
|
||||
// burning an attempt (the daemon releases it back to
|
||||
// pending) rather than counting an infrastructure
|
||||
// outage as a per-job failure.
|
||||
tracing::warn!(error = ?e, "dispatch: browser unavailable — deferring job");
|
||||
return Ok(SyncOutcome::BrowserUnavailable);
|
||||
}
|
||||
};
|
||||
let result = content::sync_chapter_content(
|
||||
&lease,
|
||||
&self.db,
|
||||
@@ -1037,7 +1097,15 @@ impl ChapterDispatcher for RealChapterDispatcher {
|
||||
// Scope the lease so it (and the borrowing FetchContext) drop
|
||||
// before any browser-restart handling in the match below.
|
||||
let result = {
|
||||
let lease = self.browser_manager.acquire().await?;
|
||||
let lease = match self.browser_manager.acquire().await {
|
||||
Ok(l) => l,
|
||||
Err(e) => {
|
||||
// See the SyncChapterContent arm: defer without
|
||||
// burning an attempt when the browser is unavailable.
|
||||
tracing::warn!(error = ?e, "dispatch: browser unavailable — deferring job");
|
||||
return Ok(SyncOutcome::BrowserUnavailable);
|
||||
}
|
||||
};
|
||||
let ctx = crate::crawler::source::FetchContext {
|
||||
browser: &lease,
|
||||
rate: &self.rate,
|
||||
|
||||
@@ -70,6 +70,45 @@ impl FromRequestParts<AppState> for CurrentUser {
|
||||
}
|
||||
}
|
||||
|
||||
/// Client IP for per-IP auth rate limiting. Resolves to the first hop of
|
||||
/// `X-Forwarded-For` **only** when [`crate::config::AuthConfig::trusted_proxy`]
|
||||
/// is set (the backend is behind a proxy that overrides the header — the
|
||||
/// compose deploy). Otherwise `None`, so the limiter uses its shared bucket.
|
||||
/// Never fails: a missing or malformed header simply yields `None`.
|
||||
pub struct ClientIp(pub Option<std::net::IpAddr>);
|
||||
|
||||
/// Parse the client IP from an `X-Forwarded-For` value: the left-most hop is
|
||||
/// the original client (later hops are intermediary proxies). Factored out so
|
||||
/// the parsing is unit-testable without constructing a request.
|
||||
pub(crate) fn first_forwarded_ip(header: &str) -> Option<std::net::IpAddr> {
|
||||
header
|
||||
.split(',')
|
||||
.next()
|
||||
.map(str::trim)
|
||||
.filter(|s| !s.is_empty())
|
||||
.and_then(|s| s.parse::<std::net::IpAddr>().ok())
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl FromRequestParts<AppState> for ClientIp {
|
||||
type Rejection = std::convert::Infallible;
|
||||
|
||||
async fn from_request_parts(
|
||||
parts: &mut Parts,
|
||||
state: &AppState,
|
||||
) -> Result<Self, Self::Rejection> {
|
||||
if !state.auth.trusted_proxy {
|
||||
return Ok(ClientIp(None));
|
||||
}
|
||||
let ip = parts
|
||||
.headers
|
||||
.get("x-forwarded-for")
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.and_then(first_forwarded_ip);
|
||||
Ok(ClientIp(ip))
|
||||
}
|
||||
}
|
||||
|
||||
/// Cookie-only authentication. Bot/API tokens are explicitly NOT accepted
|
||||
/// here — this is the substrate for [`RequireAdmin`] and exists precisely
|
||||
/// to keep admin authority out of bearer-token reach.
|
||||
@@ -120,3 +159,34 @@ impl FromRequestParts<AppState> for RequireAdmin {
|
||||
Ok(RequireAdmin(user))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::first_forwarded_ip;
|
||||
use std::net::IpAddr;
|
||||
|
||||
#[test]
|
||||
fn parses_left_most_client_hop() {
|
||||
// The client is the first entry; later entries are intermediary proxies.
|
||||
assert_eq!(
|
||||
first_forwarded_ip("203.0.113.5, 10.0.0.1, 10.0.0.2"),
|
||||
Some("203.0.113.5".parse::<IpAddr>().unwrap())
|
||||
);
|
||||
assert_eq!(
|
||||
first_forwarded_ip(" 198.51.100.9 "),
|
||||
Some("198.51.100.9".parse::<IpAddr>().unwrap())
|
||||
);
|
||||
assert_eq!(
|
||||
first_forwarded_ip("2001:db8::1, 10.0.0.1"),
|
||||
Some("2001:db8::1".parse::<IpAddr>().unwrap())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_empty_or_garbage() {
|
||||
assert_eq!(first_forwarded_ip(""), None);
|
||||
assert_eq!(first_forwarded_ip(" "), None);
|
||||
assert_eq!(first_forwarded_ip("not-an-ip"), None);
|
||||
assert_eq!(first_forwarded_ip(", 10.0.0.1"), None);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,6 +28,25 @@ pub fn verify_password(plain: &str, phc: &str) -> bool {
|
||||
.is_ok()
|
||||
}
|
||||
|
||||
/// Async wrapper around [`hash_password`] that offloads the CPU- and
|
||||
/// memory-heavy Argon2 work to the blocking pool. Async handlers must call
|
||||
/// this rather than the sync primitive: at ~15-50 ms per hash, running it
|
||||
/// inline stalls every other task sharing the runtime worker thread.
|
||||
pub async fn hash_password_async(plain: String) -> AppResult<String> {
|
||||
tokio::task::spawn_blocking(move || hash_password(&plain))
|
||||
.await
|
||||
.map_err(|e| AppError::Other(anyhow::anyhow!("password hash task join: {e}")))?
|
||||
}
|
||||
|
||||
/// Async wrapper around [`verify_password`]. A join failure (blocking pool
|
||||
/// gone at shutdown, panic) resolves to `false` — the caller only ever uses
|
||||
/// the result as a boolean gate, and denying auth is the safe default.
|
||||
pub async fn verify_password_async(plain: String, phc: String) -> bool {
|
||||
tokio::task::spawn_blocking(move || verify_password(&plain, &phc))
|
||||
.await
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -56,4 +75,24 @@ mod tests {
|
||||
let b = hash_password("same").unwrap();
|
||||
assert_ne!(a, b, "two hashes of the same password must differ (salt)");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn async_hash_then_verify_roundtrip() {
|
||||
let phc = hash_password_async("correct horse battery staple".to_string())
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(phc.starts_with("$argon2id$"));
|
||||
assert!(verify_password_async("correct horse battery staple".to_string(), phc).await);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn async_verify_rejects_wrong_password() {
|
||||
let phc = hash_password_async("hunter2".to_string()).await.unwrap();
|
||||
assert!(!verify_password_async("hunter3".to_string(), phc).await);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn async_verify_rejects_malformed_hash() {
|
||||
assert!(!verify_password_async("anything".to_string(), "not a real phc".to_string()).await);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,9 +15,17 @@
|
||||
//! tests run in isolated buckets and won't bleed across `#[sqlx::test]`
|
||||
//! cases that share a process.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::net::IpAddr;
|
||||
use std::sync::Mutex;
|
||||
use std::time::Instant;
|
||||
|
||||
/// Upper bound on distinct client IPs tracked at once, so a spray from many
|
||||
/// spoofed/rotating source IPs can't grow the map without limit. When full,
|
||||
/// idle (refilled-to-burst) buckets are pruned first; if none are idle a new
|
||||
/// IP falls back to the shared global bucket for that request.
|
||||
const MAX_TRACKED_IPS: usize = 10_000;
|
||||
|
||||
/// Tunable limits. `per_sec == 0` disables the limiter — used by the
|
||||
/// test harness and by anyone who wants to opt out via env config.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
@@ -50,6 +58,42 @@ struct Bucket {
|
||||
last_refill: Instant,
|
||||
}
|
||||
|
||||
impl Bucket {
|
||||
fn new(burst: u32) -> Self {
|
||||
Self {
|
||||
tokens: f64::from(burst),
|
||||
last_refill: Instant::now(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Refill by elapsed time then try to consume one token.
|
||||
fn try_take(&mut self, cfg: &RateLimitConfig, now: Instant) -> AcquireResult {
|
||||
let elapsed = now.duration_since(self.last_refill).as_secs_f64();
|
||||
self.tokens =
|
||||
(self.tokens + elapsed * f64::from(cfg.per_sec)).min(f64::from(cfg.burst));
|
||||
self.last_refill = now;
|
||||
if self.tokens >= 1.0 {
|
||||
self.tokens -= 1.0;
|
||||
AcquireResult::Allowed
|
||||
} else {
|
||||
let deficit = 1.0 - self.tokens;
|
||||
let wait_secs = (deficit / f64::from(cfg.per_sec)).ceil() as u64;
|
||||
AcquireResult::Denied {
|
||||
retry_after_secs: wait_secs.max(1),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether this bucket has fully refilled (i.e. the client has been idle).
|
||||
/// Such buckets carry no state worth keeping — dropping and lazily
|
||||
/// recreating one yields an identical full bucket — so they're the safe
|
||||
/// eviction target under memory pressure.
|
||||
fn is_idle(&self, cfg: &RateLimitConfig, now: Instant) -> bool {
|
||||
let elapsed = now.duration_since(self.last_refill).as_secs_f64();
|
||||
(self.tokens + elapsed * f64::from(cfg.per_sec)) >= f64::from(cfg.burst)
|
||||
}
|
||||
}
|
||||
|
||||
/// Outcome of [`AuthRateLimiter::try_acquire`]. When `Denied`, the
|
||||
/// caller can use `retry_after_secs` for a `Retry-After: N` header
|
||||
/// (RFC 6585 §4) so well-behaved clients back off correctly rather
|
||||
@@ -60,51 +104,64 @@ pub enum AcquireResult {
|
||||
Denied { retry_after_secs: u64 },
|
||||
}
|
||||
|
||||
/// Single-bucket token-bucket limiter. `try_acquire` is cheap (one
|
||||
/// mutex acquire, no allocations) so the auth path doesn't pay a real
|
||||
/// cost for the check.
|
||||
/// Token-bucket limiter keyed by client IP, with a shared global bucket for
|
||||
/// requests whose IP is unknown (no trusted proxy).
|
||||
///
|
||||
/// The old design was a single global bucket. That let one attacker at the
|
||||
/// sustained rate deny auth to *every* user (the bucket stayed drained). With
|
||||
/// the SvelteKit proxy now forwarding the peer IP (`X-Forwarded-For`, honoured
|
||||
/// only when `AUTH_TRUSTED_PROXY` is set), each source IP gets its own bucket,
|
||||
/// so an attacker only throttles themselves. When no trustworthy IP is
|
||||
/// available the caller passes `None` and the shared `global` bucket applies —
|
||||
/// exactly the previous behaviour.
|
||||
pub struct AuthRateLimiter {
|
||||
cfg: RateLimitConfig,
|
||||
bucket: Mutex<Bucket>,
|
||||
global: Mutex<Bucket>,
|
||||
per_ip: Mutex<HashMap<IpAddr, Bucket>>,
|
||||
}
|
||||
|
||||
impl AuthRateLimiter {
|
||||
pub fn new(cfg: RateLimitConfig) -> Self {
|
||||
Self {
|
||||
cfg,
|
||||
bucket: Mutex::new(Bucket {
|
||||
tokens: cfg.burst as f64,
|
||||
last_refill: Instant::now(),
|
||||
}),
|
||||
global: Mutex::new(Bucket::new(cfg.burst)),
|
||||
per_ip: Mutex::new(HashMap::new()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Consume one token if available. Returns `Denied` with a
|
||||
/// rounded-up seconds-until-refill so the caller can emit a
|
||||
/// `Retry-After` header.
|
||||
pub fn try_acquire(&self) -> AcquireResult {
|
||||
/// Consume one token from the bucket for `key` (per-IP when `Some`, the
|
||||
/// shared global bucket when `None`). Returns `Denied` with a rounded-up
|
||||
/// seconds-until-refill so the caller can emit a `Retry-After` header.
|
||||
pub fn try_acquire(&self, key: Option<IpAddr>) -> AcquireResult {
|
||||
if self.cfg.per_sec == 0 {
|
||||
return AcquireResult::Allowed;
|
||||
}
|
||||
let now = Instant::now();
|
||||
let mut bucket = self.bucket.lock().expect("rate limiter mutex poisoned");
|
||||
let elapsed = now.duration_since(bucket.last_refill).as_secs_f64();
|
||||
bucket.tokens =
|
||||
(bucket.tokens + elapsed * f64::from(self.cfg.per_sec)).min(f64::from(self.cfg.burst));
|
||||
bucket.last_refill = now;
|
||||
if bucket.tokens >= 1.0 {
|
||||
bucket.tokens -= 1.0;
|
||||
AcquireResult::Allowed
|
||||
} else {
|
||||
// ceil((1 - tokens) / per_sec), minimum 1 — a `Retry-After: 0`
|
||||
// would tell clients to retry immediately, which is what we're
|
||||
// actively trying to discourage.
|
||||
let deficit = 1.0 - bucket.tokens;
|
||||
let wait_secs = (deficit / f64::from(self.cfg.per_sec)).ceil() as u64;
|
||||
AcquireResult::Denied {
|
||||
retry_after_secs: wait_secs.max(1),
|
||||
let Some(ip) = key else {
|
||||
return self
|
||||
.global
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner())
|
||||
.try_take(&self.cfg, now);
|
||||
};
|
||||
let mut map = self.per_ip.lock().unwrap_or_else(|e| e.into_inner());
|
||||
if map.len() >= MAX_TRACKED_IPS && !map.contains_key(&ip) {
|
||||
map.retain(|_, b| !b.is_idle(&self.cfg, now));
|
||||
if map.len() >= MAX_TRACKED_IPS {
|
||||
// Still saturated with active attackers — degrade to the shared
|
||||
// bucket rather than grow unbounded. Worst case is the old
|
||||
// global-bucket behaviour under an extreme distributed flood.
|
||||
drop(map);
|
||||
return self
|
||||
.global
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner())
|
||||
.try_take(&self.cfg, now);
|
||||
}
|
||||
}
|
||||
map.entry(ip)
|
||||
.or_insert_with(|| Bucket::new(self.cfg.burst))
|
||||
.try_take(&self.cfg, now)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -119,7 +176,7 @@ mod tests {
|
||||
burst: 0,
|
||||
});
|
||||
for _ in 0..1000 {
|
||||
assert_eq!(rl.try_acquire(), AcquireResult::Allowed);
|
||||
assert_eq!(rl.try_acquire(None), AcquireResult::Allowed);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -130,10 +187,10 @@ mod tests {
|
||||
per_sec: 1,
|
||||
burst: 3,
|
||||
});
|
||||
assert_eq!(rl.try_acquire(), AcquireResult::Allowed);
|
||||
assert_eq!(rl.try_acquire(), AcquireResult::Allowed);
|
||||
assert_eq!(rl.try_acquire(), AcquireResult::Allowed);
|
||||
match rl.try_acquire() {
|
||||
assert_eq!(rl.try_acquire(None), AcquireResult::Allowed);
|
||||
assert_eq!(rl.try_acquire(None), AcquireResult::Allowed);
|
||||
assert_eq!(rl.try_acquire(None), AcquireResult::Allowed);
|
||||
match rl.try_acquire(None) {
|
||||
AcquireResult::Denied { retry_after_secs } => {
|
||||
// Bucket is at ~0 tokens, refill rate 1/sec → ~1s wait.
|
||||
assert!(
|
||||
@@ -152,11 +209,11 @@ mod tests {
|
||||
per_sec: 10,
|
||||
burst: 1,
|
||||
});
|
||||
assert_eq!(rl.try_acquire(), AcquireResult::Allowed);
|
||||
assert!(matches!(rl.try_acquire(), AcquireResult::Denied { .. }));
|
||||
assert_eq!(rl.try_acquire(None), AcquireResult::Allowed);
|
||||
assert!(matches!(rl.try_acquire(None), AcquireResult::Denied { .. }));
|
||||
std::thread::sleep(std::time::Duration::from_millis(150));
|
||||
assert_eq!(
|
||||
rl.try_acquire(),
|
||||
rl.try_acquire(None),
|
||||
AcquireResult::Allowed,
|
||||
"token should have refilled"
|
||||
);
|
||||
@@ -170,10 +227,95 @@ mod tests {
|
||||
per_sec: 1,
|
||||
burst: 1,
|
||||
});
|
||||
slow.try_acquire();
|
||||
match slow.try_acquire() {
|
||||
slow.try_acquire(None);
|
||||
match slow.try_acquire(None) {
|
||||
AcquireResult::Denied { retry_after_secs } => assert_eq!(retry_after_secs, 1),
|
||||
_ => panic!("expected Denied"),
|
||||
}
|
||||
}
|
||||
|
||||
fn ip(s: &str) -> Option<IpAddr> {
|
||||
Some(s.parse().unwrap())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn per_ip_buckets_are_independent() {
|
||||
// The whole point of the fix: one IP draining its bucket must not deny
|
||||
// a different IP.
|
||||
let rl = AuthRateLimiter::new(RateLimitConfig {
|
||||
per_sec: 1,
|
||||
burst: 2,
|
||||
});
|
||||
let a = ip("203.0.113.7");
|
||||
let b = ip("198.51.100.9");
|
||||
assert_eq!(rl.try_acquire(a), AcquireResult::Allowed);
|
||||
assert_eq!(rl.try_acquire(a), AcquireResult::Allowed);
|
||||
assert!(matches!(rl.try_acquire(a), AcquireResult::Denied { .. }));
|
||||
// b is untouched.
|
||||
assert_eq!(rl.try_acquire(b), AcquireResult::Allowed);
|
||||
assert_eq!(rl.try_acquire(b), AcquireResult::Allowed);
|
||||
assert!(matches!(rl.try_acquire(b), AcquireResult::Denied { .. }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn none_key_shares_the_global_bucket_independent_of_per_ip() {
|
||||
let rl = AuthRateLimiter::new(RateLimitConfig {
|
||||
per_sec: 1,
|
||||
burst: 2,
|
||||
});
|
||||
// Drain the global (None) bucket.
|
||||
assert_eq!(rl.try_acquire(None), AcquireResult::Allowed);
|
||||
assert_eq!(rl.try_acquire(None), AcquireResult::Allowed);
|
||||
assert!(matches!(rl.try_acquire(None), AcquireResult::Denied { .. }));
|
||||
// A real IP is on its own bucket, unaffected by the drained global one.
|
||||
assert_eq!(rl.try_acquire(ip("203.0.113.1")), AcquireResult::Allowed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tracked_ip_map_is_bounded() {
|
||||
let rl = AuthRateLimiter::new(RateLimitConfig {
|
||||
per_sec: 1,
|
||||
burst: 1,
|
||||
});
|
||||
// Distinct IPs beyond the cap must not grow the map without bound —
|
||||
// excess requests fall back to the shared bucket instead.
|
||||
for i in 0..(MAX_TRACKED_IPS as u64 + 50) {
|
||||
let octet_a = (i >> 8) as u8;
|
||||
let octet_b = (i & 0xff) as u8;
|
||||
let addr = format!("10.20.{octet_a}.{octet_b}");
|
||||
let _ = rl.try_acquire(Some(addr.parse().unwrap()));
|
||||
}
|
||||
assert!(rl.per_ip.lock().unwrap().len() <= MAX_TRACKED_IPS);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn survives_a_poisoned_mutex() {
|
||||
// If any thread ever panics while holding a limiter mutex, the lock
|
||||
// becomes poisoned. With the old `.expect(...)` every later auth request
|
||||
// would re-panic — one blip turned into a permanent auth outage. Recover
|
||||
// the guard via `into_inner()` instead so the limiter keeps serving.
|
||||
let rl = AuthRateLimiter::new(RateLimitConfig {
|
||||
per_sec: 5,
|
||||
burst: 5,
|
||||
});
|
||||
|
||||
// Poison per_ip by panicking while holding its guard.
|
||||
let poisoned = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
|
||||
let _g = rl.per_ip.lock().unwrap();
|
||||
panic!("poison the per-IP mutex");
|
||||
}));
|
||||
assert!(poisoned.is_err(), "the panic must unwind");
|
||||
assert!(rl.per_ip.is_poisoned(), "the mutex must now be poisoned");
|
||||
|
||||
// The per-IP path (Some(ip)) must still work despite the poison.
|
||||
assert_eq!(rl.try_acquire(ip("198.51.100.9")), AcquireResult::Allowed);
|
||||
|
||||
// And poison the global bucket too — the None-key path must recover.
|
||||
let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
|
||||
let _g = rl.global.lock().unwrap();
|
||||
panic!("poison the global mutex");
|
||||
}));
|
||||
assert!(rl.global.is_poisoned());
|
||||
assert_eq!(rl.try_acquire(None), AcquireResult::Allowed);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,15 +3,16 @@
|
||||
//! `generate_token` draws 32 bytes from the OS CSPRNG, encodes them as
|
||||
//! URL-safe base64 (no padding), and returns the raw string alongside its
|
||||
//! SHA-256 hash. Storage holds only the hash; the raw value lives in the
|
||||
//! cookie or `Authorization` header. Comparison goes through
|
||||
//! `constant_time_eq` to keep timing side channels off the table.
|
||||
//! cookie or `Authorization` header. Token lookup is an indexed equality on
|
||||
//! that 256-bit hash in the database (`WHERE token_hash = $1`), so there's no
|
||||
//! in-process secret comparison to time-attack: a guess has to match a full
|
||||
//! SHA-256 digest, and the DB index reveals nothing about how close it came.
|
||||
|
||||
use base64::engine::general_purpose::URL_SAFE_NO_PAD;
|
||||
use base64::Engine as _;
|
||||
use rand::rngs::OsRng;
|
||||
use rand::RngCore;
|
||||
use sha2::{Digest, Sha256};
|
||||
use subtle::ConstantTimeEq;
|
||||
|
||||
pub const TOKEN_BYTES: usize = 32;
|
||||
pub const HASH_BYTES: usize = 32;
|
||||
@@ -30,10 +31,6 @@ pub fn hash_token(raw: &str) -> [u8; HASH_BYTES] {
|
||||
hasher.finalize().into()
|
||||
}
|
||||
|
||||
pub fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
|
||||
a.ct_eq(b).into()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -58,11 +55,4 @@ mod tests {
|
||||
assert_eq!(hash_token("abc"), hash_token("abc"));
|
||||
assert_ne!(hash_token("abc"), hash_token("abd"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn constant_time_eq_compares_correctly() {
|
||||
assert!(constant_time_eq(b"abc", b"abc"));
|
||||
assert!(!constant_time_eq(b"abc", b"abd"));
|
||||
assert!(!constant_time_eq(b"abc", b"abcd"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -134,8 +134,25 @@ async fn main() -> anyhow::Result<()> {
|
||||
http_builder = http_builder
|
||||
.proxy(reqwest::Proxy::all(proxy).with_context(|| format!("parse proxy URL: {proxy}"))?);
|
||||
}
|
||||
// DNS-rebinding guard: attached on the direct path AND on http(s) proxies
|
||||
// (reqwest resolves the target itself there), skipped only for SOCKS proxies
|
||||
// where the proxy resolves the target. Use the shared predicate so this CLI
|
||||
// and the daemon (app.rs) stay in lockstep — previously the CLI attached the
|
||||
// resolver only on the fully-direct path, so an http(s) proxy silently lost
|
||||
// the rebinding guard.
|
||||
if mangalord::crawler::safety::should_attach_safe_resolver(proxy_url.as_deref()) {
|
||||
http_builder =
|
||||
http_builder.dns_resolver(mangalord::crawler::safety::safe_dns_resolver());
|
||||
}
|
||||
let http = http_builder.build().context("build http client")?;
|
||||
|
||||
// Opt-in browser SSRF interception (default off), mirroring the daemon.
|
||||
mangalord::crawler::intercept::set_enabled(
|
||||
std::env::var("CRAWLER_SSRF_INTERCEPT")
|
||||
.map(|v| matches!(v.as_str(), "1" | "true" | "TRUE" | "yes"))
|
||||
.unwrap_or(false),
|
||||
);
|
||||
|
||||
let mut options = LaunchOptions::from_env();
|
||||
if let Some(proxy) = &proxy_url {
|
||||
let chromium_proxy = mangalord::crawler::url_utils::chromium_proxy_arg(proxy);
|
||||
@@ -427,6 +444,13 @@ async fn sync_bookmarked_chapter_content(
|
||||
s.fetched += 1;
|
||||
}
|
||||
Ok(SyncOutcome::Skipped) => s.skipped += 1,
|
||||
// Unreachable in the one-shot CLI (it holds its own lease
|
||||
// and never dispatches through the queue), but count it as a
|
||||
// failure for exhaustiveness.
|
||||
Ok(SyncOutcome::BrowserUnavailable) => {
|
||||
tracing::warn!(%chapter_id, "crawler browser unavailable");
|
||||
s.failed += 1;
|
||||
}
|
||||
Ok(SyncOutcome::SessionExpired) => {
|
||||
tracing::error!(
|
||||
%chapter_id,
|
||||
|
||||
@@ -27,6 +27,14 @@ pub struct AuthConfig {
|
||||
/// so a private instance is locked down with a single switch.
|
||||
/// Defaults to `false` (current public behaviour).
|
||||
pub private_mode: bool,
|
||||
/// Whether to trust a proxy-supplied `X-Forwarded-For` header as the
|
||||
/// client IP for per-IP auth rate limiting. Enable ONLY when the backend
|
||||
/// sits behind a trusted reverse proxy that overrides the header (the
|
||||
/// compose deploy: SvelteKit's hooks.server.ts sets it from the real peer
|
||||
/// address). When `false` (default), the header is ignored and the auth
|
||||
/// limiter uses a single shared bucket — a directly-exposed backend must
|
||||
/// keep this off or clients could spoof their IP to dodge the limit.
|
||||
pub trusted_proxy: bool,
|
||||
}
|
||||
|
||||
impl Default for AuthConfig {
|
||||
@@ -42,6 +50,7 @@ impl Default for AuthConfig {
|
||||
rate_limit: crate::auth::rate_limit::RateLimitConfig::default(),
|
||||
allow_self_register: true,
|
||||
private_mode: false,
|
||||
trusted_proxy: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -57,8 +66,10 @@ pub struct UploadConfig {
|
||||
pub max_file_bytes: usize,
|
||||
/// Max page images accepted in one chapter upload. Bounds how many
|
||||
/// parts the handler will stage before giving up, so a client can't
|
||||
/// pin a worker streaming an unbounded page count. `0` disables the
|
||||
/// cap. Defaults to 2000. `MAX_PAGES_PER_CHAPTER`.
|
||||
/// pin a worker streaming an unbounded page count. `0` disables THIS
|
||||
/// cap — the total is then bounded only by `max_request_bytes` (the
|
||||
/// whole-request body limit), which stays the backstop either way.
|
||||
/// Defaults to 2000. `MAX_PAGES_PER_CHAPTER`.
|
||||
pub max_pages_per_chapter: usize,
|
||||
}
|
||||
|
||||
@@ -72,6 +83,45 @@ impl Default for UploadConfig {
|
||||
}
|
||||
}
|
||||
|
||||
/// Postgres connection-pool sizing. One pool backs every HTTP handler plus
|
||||
/// the crawler/analysis daemons, so it must be large enough not to starve
|
||||
/// interactive reads and fail fast (rather than hang on the driver's silent
|
||||
/// 30 s default) when genuinely saturated.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct DbConfig {
|
||||
/// `DB_MAX_CONNECTIONS`. Upper bound on open connections.
|
||||
pub max_connections: u32,
|
||||
/// `DB_ACQUIRE_TIMEOUT_SECS`. How long a caller waits for a free
|
||||
/// connection before erroring — short so overload surfaces as a fast
|
||||
/// 500 instead of a 30 s hang.
|
||||
pub acquire_timeout: Duration,
|
||||
}
|
||||
|
||||
impl Default for DbConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
max_connections: 20,
|
||||
acquire_timeout: Duration::from_secs(10),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl DbConfig {
|
||||
pub fn from_env() -> Self {
|
||||
let default = Self::default();
|
||||
Self {
|
||||
// `.max(1)`: a zero-size pool can never hand out a connection and
|
||||
// would deadlock every query — clamp to at least one.
|
||||
max_connections: env_u64("DB_MAX_CONNECTIONS", default.max_connections.into())
|
||||
.max(1) as u32,
|
||||
acquire_timeout: Duration::from_secs(env_u64(
|
||||
"DB_ACQUIRE_TIMEOUT_SECS",
|
||||
default.acquire_timeout.as_secs(),
|
||||
)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// How the worker asks the model to constrain its output. OpenAI-compatible
|
||||
/// servers disagree here: LM Studio accepts only `json_schema` or `text`
|
||||
/// (NOT `json_object`); vanilla OpenAI/vLLM accept `json_object` too. The
|
||||
@@ -371,6 +421,7 @@ pub struct Config {
|
||||
pub database_url: String,
|
||||
pub bind_address: String,
|
||||
pub storage_dir: PathBuf,
|
||||
pub db: DbConfig,
|
||||
pub auth: AuthConfig,
|
||||
pub upload: UploadConfig,
|
||||
pub cors_allowed_origins: Vec<String>,
|
||||
@@ -471,6 +522,16 @@ pub struct CrawlerConfig {
|
||||
/// exhausted) that trigger an automatic coordinated browser restart.
|
||||
/// Defaults to 3. `CRAWLER_BROWSER_RESTART_THRESHOLD`.
|
||||
pub browser_restart_threshold: u32,
|
||||
/// CDP `Fetch` interception that re-validates every headless-browser
|
||||
/// navigation/redirect/subresource against the SSRF check. Default `true`:
|
||||
/// with it off, only the top-level URL string is validated, so a scraped
|
||||
/// page's JS/subresources (which use Chromium's own network stack, not the
|
||||
/// reqwest `SafeResolver`) can reach internal targets like the cloud
|
||||
/// metadata service or postgres. The Fetch hook can't be exercised in CI (no
|
||||
/// Chromium) — the `#[ignore]`d `ssrf_interception_does_not_wedge_allowed_navigation`
|
||||
/// smoke test validates it against a real binary. `CRAWLER_SSRF_INTERCEPT`;
|
||||
/// set `false` only as a break-glass if the hook destabilizes a deployment.
|
||||
pub ssrf_intercept: bool,
|
||||
}
|
||||
|
||||
impl Default for CrawlerConfig {
|
||||
@@ -503,6 +564,7 @@ impl Default for CrawlerConfig {
|
||||
job_timeout: Duration::from_secs(600),
|
||||
metadata_max_consecutive_failures: 10,
|
||||
browser_restart_threshold: 3,
|
||||
ssrf_intercept: true,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -517,6 +579,7 @@ impl Config {
|
||||
storage_dir: std::env::var("STORAGE_DIR")
|
||||
.unwrap_or_else(|_| "./data/storage".to_string())
|
||||
.into(),
|
||||
db: DbConfig::from_env(),
|
||||
auth: AuthConfig {
|
||||
cookie_secure: env_bool("COOKIE_SECURE", true),
|
||||
cookie_domain: std::env::var("COOKIE_DOMAIN")
|
||||
@@ -535,6 +598,7 @@ impl Config {
|
||||
},
|
||||
allow_self_register: env_bool("ALLOW_SELF_REGISTER", true),
|
||||
private_mode: env_bool("PRIVATE_MODE", false),
|
||||
trusted_proxy: env_bool("AUTH_TRUSTED_PROXY", false),
|
||||
},
|
||||
upload: UploadConfig {
|
||||
max_request_bytes: env_usize("MAX_REQUEST_BYTES", 200 * 1024 * 1024),
|
||||
@@ -666,6 +730,7 @@ impl CrawlerConfig {
|
||||
) as u32,
|
||||
browser_restart_threshold: env_u64("CRAWLER_BROWSER_RESTART_THRESHOLD", 3).max(1)
|
||||
as u32,
|
||||
ssrf_intercept: env_bool("CRAWLER_SSRF_INTERCEPT", true),
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -813,6 +878,37 @@ mod tests {
|
||||
assert_eq!(cfg.browser_restart_threshold, 7);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn db_pool_defaults_when_unset() {
|
||||
let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner());
|
||||
std::env::remove_var("DB_MAX_CONNECTIONS");
|
||||
std::env::remove_var("DB_ACQUIRE_TIMEOUT_SECS");
|
||||
let cfg = DbConfig::from_env();
|
||||
assert_eq!(cfg.max_connections, 20);
|
||||
assert_eq!(cfg.acquire_timeout, Duration::from_secs(10));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn db_pool_parses_from_env() {
|
||||
let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner());
|
||||
std::env::set_var("DB_MAX_CONNECTIONS", "50");
|
||||
std::env::set_var("DB_ACQUIRE_TIMEOUT_SECS", "3");
|
||||
let cfg = DbConfig::from_env();
|
||||
std::env::remove_var("DB_MAX_CONNECTIONS");
|
||||
std::env::remove_var("DB_ACQUIRE_TIMEOUT_SECS");
|
||||
assert_eq!(cfg.max_connections, 50);
|
||||
assert_eq!(cfg.acquire_timeout, Duration::from_secs(3));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn db_pool_max_connections_clamps_to_at_least_one() {
|
||||
let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner());
|
||||
std::env::set_var("DB_MAX_CONNECTIONS", "0");
|
||||
let cfg = DbConfig::from_env();
|
||||
std::env::remove_var("DB_MAX_CONNECTIONS");
|
||||
assert_eq!(cfg.max_connections, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn analysis_config_defaults_when_unset() {
|
||||
let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner());
|
||||
@@ -1000,6 +1096,13 @@ mod tests {
|
||||
"BACKEND_URL",
|
||||
"BACKEND_PROXY_TIMEOUT_MS",
|
||||
"VISION_MANAGER_DATABASE_URL",
|
||||
// Compose-level deployment knobs (port publish interface + per-container
|
||||
// memory ceilings) — consumed by docker-compose.yml itself, never read
|
||||
// by the backend process, so they don't belong in its environment block.
|
||||
"FRONTEND_PUBLISH_ADDR",
|
||||
"BACKEND_MEM_LIMIT",
|
||||
"FRONTEND_MEM_LIMIT",
|
||||
"POSTGRES_MEM_LIMIT",
|
||||
];
|
||||
|
||||
/// Keys whose compose RHS is intentionally NOT a `${KEY...}`
|
||||
|
||||
@@ -71,6 +71,13 @@ pub enum SyncOutcome {
|
||||
/// Session probe failed mid-sync (avatar selector missing on the
|
||||
/// chapter page). Caller should abort the whole crawler run.
|
||||
SessionExpired,
|
||||
/// The headless browser could not be acquired (down or mid-restart).
|
||||
/// Produced only by the queue dispatcher (which calls `acquire()`); it is
|
||||
/// an *infrastructure* outage, not a job failure, so the daemon returns the
|
||||
/// job to `pending` WITHOUT burning a retry attempt. `sync_chapter_content`
|
||||
/// itself never returns this — callers that already hold a lease can treat
|
||||
/// it as unreachable.
|
||||
BrowserUnavailable,
|
||||
}
|
||||
|
||||
/// Per-chapter max fetch attempts when TOR is configured. `N = 3` means
|
||||
@@ -118,26 +125,34 @@ async fn fetch_chapter_html_once(
|
||||
) -> anyhow::Result<String> {
|
||||
guard_nav_url(source_url)?;
|
||||
rate.wait_for(source_url).await?;
|
||||
let page = browser
|
||||
.new_page(source_url)
|
||||
let page = crate::crawler::intercept::open_page(browser, source_url)
|
||||
.await
|
||||
.with_context(|| format!("open chapter page {source_url}"))?;
|
||||
crate::crawler::nav::wait_for_nav(&page)
|
||||
.await
|
||||
.context("wait for chapter nav")?;
|
||||
// Best-effort wait for the reader marker — same partial-render
|
||||
// race that bit the chapter-list parser can hit here. Timeout is
|
||||
// not an error; the chapter probe + parser sentinels still catch
|
||||
// real failures.
|
||||
let _ = crate::crawler::nav::wait_for_selector(
|
||||
&page,
|
||||
"a#pic_container",
|
||||
crate::crawler::nav::SELECTOR_TIMEOUT,
|
||||
// Close the tab on every exit path — a `?` on wait_for_nav / content()
|
||||
// would otherwise leak it (chromiumoxide doesn't close on drop).
|
||||
let closer = page.clone();
|
||||
crate::crawler::nav::close_after(
|
||||
async move {
|
||||
closer.close().await.ok();
|
||||
},
|
||||
async {
|
||||
crate::crawler::nav::wait_for_nav(&page)
|
||||
.await
|
||||
.context("wait for chapter nav")?;
|
||||
// Best-effort wait for the reader marker — same partial-render
|
||||
// race that bit the chapter-list parser can hit here. Timeout is
|
||||
// not an error; the chapter probe + parser sentinels still catch
|
||||
// real failures.
|
||||
let _ = crate::crawler::nav::wait_for_selector(
|
||||
&page,
|
||||
"a#pic_container",
|
||||
crate::crawler::nav::SELECTOR_TIMEOUT,
|
||||
)
|
||||
.await;
|
||||
page.content().await.context("read chapter html")
|
||||
},
|
||||
)
|
||||
.await;
|
||||
let html = page.content().await.context("read chapter html")?;
|
||||
page.close().await.ok();
|
||||
Ok(html)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Pure-over-IO loop: fetch + classify, up to `max_attempts` total
|
||||
@@ -580,15 +595,22 @@ pub(crate) async fn persist_pages(
|
||||
let mut tx = db.begin().await.context("open chapter sync tx")?;
|
||||
let mut page_ids: Vec<Uuid> = Vec::with_capacity(stored.len());
|
||||
let mut page_numbers: Vec<i32> = Vec::with_capacity(stored.len());
|
||||
// Pages whose row already existed and was overwritten by this re-crawl.
|
||||
// Their image changed but pages.id (and any analysis keyed to it) survived,
|
||||
// so the derived analysis is now stale and must be invalidated below.
|
||||
let mut updated_page_ids: Vec<Uuid> = Vec::new();
|
||||
for page in stored {
|
||||
let (id,): (Uuid,) = sqlx::query_as(
|
||||
// `xmax <> 0` on the RETURNING row distinguishes a conflict UPDATE
|
||||
// (existing row overwritten) from a fresh INSERT — the standard upsert
|
||||
// idiom. Cast through text/bigint since there is no direct xid<>int op.
|
||||
let (id, was_update): (Uuid, bool) = sqlx::query_as(
|
||||
"INSERT INTO pages (chapter_id, page_number, storage_key, content_type, size_bytes)
|
||||
VALUES ($1, $2, $3, $4, $5)
|
||||
ON CONFLICT (chapter_id, page_number) DO UPDATE
|
||||
SET storage_key = EXCLUDED.storage_key,
|
||||
content_type = EXCLUDED.content_type,
|
||||
size_bytes = EXCLUDED.size_bytes
|
||||
RETURNING id",
|
||||
RETURNING id, (xmax::text::bigint <> 0)",
|
||||
)
|
||||
.bind(chapter_id)
|
||||
.bind(page.page_number)
|
||||
@@ -600,6 +622,32 @@ pub(crate) async fn persist_pages(
|
||||
.with_context(|| format!("insert page row {}", page.page_number))?;
|
||||
page_ids.push(id);
|
||||
page_numbers.push(page.page_number);
|
||||
if was_update {
|
||||
updated_page_ids.push(id);
|
||||
}
|
||||
}
|
||||
// Invalidate stale analysis for re-crawled (overwritten) pages. Storage keys
|
||||
// are deterministic per page number, so the row survives the upsert and its
|
||||
// page_analysis / OCR / auto-tags / content-warnings would otherwise reflect
|
||||
// the OLD image — poisoning search results and the derived
|
||||
// manga_content_warnings. Clearing them (the DELETE on page_content_warnings
|
||||
// fires the mcw_* triggers so manga_content_warnings recomputes) drops the
|
||||
// pages back to "unanalyzed" so the enqueue below — and the admin
|
||||
// only-unanalyzed backfill — re-derive them. Newly inserted pages have no
|
||||
// prior analysis, so they're untouched.
|
||||
if !updated_page_ids.is_empty() {
|
||||
for table in [
|
||||
"page_ocr_text",
|
||||
"page_auto_tags",
|
||||
"page_content_warnings",
|
||||
"page_analysis",
|
||||
] {
|
||||
sqlx::query(&format!("DELETE FROM {table} WHERE page_id = ANY($1::uuid[])"))
|
||||
.bind(&updated_page_ids)
|
||||
.execute(&mut *tx)
|
||||
.await
|
||||
.with_context(|| format!("invalidate {table} for re-crawled pages"))?;
|
||||
}
|
||||
}
|
||||
// Drop any rows left over from a prior, larger crawl of this chapter
|
||||
// (re-crawl that now yields fewer pages). Without this the stale rows —
|
||||
@@ -1185,4 +1233,91 @@ mod tests {
|
||||
assert_eq!(fetch_n, 1);
|
||||
assert!(format!("{err:#}").contains("nav timeout"));
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn persist_pages_invalidates_stale_analysis_on_recrawl_update(pool: PgPool) {
|
||||
// Page storage keys are deterministic per page number, so a re-crawl
|
||||
// overwrites the image but keeps pages.id. Analysis keyed to page_id
|
||||
// would otherwise stay stale (old OCR/search_doc/warnings). persist_pages
|
||||
// must clear the derived analysis for updated pages so it re-derives.
|
||||
let manga_id = Uuid::new_v4();
|
||||
let chapter_id = Uuid::new_v4();
|
||||
sqlx::query("INSERT INTO mangas (id, title) VALUES ($1, 'T')")
|
||||
.bind(manga_id)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("INSERT INTO chapters (id, manga_id, number) VALUES ($1, $2, 1)")
|
||||
.bind(chapter_id)
|
||||
.bind(manga_id)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let v1 = vec![StoredPage {
|
||||
page_number: 1,
|
||||
storage_key: "k/0001.jpg".into(),
|
||||
content_type: "image/jpeg".into(),
|
||||
size_bytes: 10,
|
||||
}];
|
||||
persist_pages(&pool, chapter_id, &v1, false).await.unwrap();
|
||||
let page_id: Uuid = sqlx::query_scalar(
|
||||
"SELECT id FROM pages WHERE chapter_id = $1 AND page_number = 1",
|
||||
)
|
||||
.bind(chapter_id)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Simulate a completed prior analysis pass for that page.
|
||||
sqlx::query(
|
||||
"INSERT INTO page_analysis (page_id, status, is_nsfw, analyzed_at) \
|
||||
VALUES ($1, 'done', true, now())",
|
||||
)
|
||||
.bind(page_id)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("INSERT INTO page_ocr_text (page_id, kind, text) VALUES ($1, 'speech', 'stale')")
|
||||
.bind(page_id)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("INSERT INTO page_content_warnings (page_id, warning) VALUES ($1, 'gore')")
|
||||
.bind(page_id)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Re-crawl page 1 with a new image (same deterministic key, new size).
|
||||
let v2 = vec![StoredPage {
|
||||
page_number: 1,
|
||||
storage_key: "k/0001.jpg".into(),
|
||||
content_type: "image/jpeg".into(),
|
||||
size_bytes: 999,
|
||||
}];
|
||||
persist_pages(&pool, chapter_id, &v2, false).await.unwrap();
|
||||
|
||||
// The page row survived (deterministic key).
|
||||
let same_id: Uuid = sqlx::query_scalar(
|
||||
"SELECT id FROM pages WHERE chapter_id = $1 AND page_number = 1",
|
||||
)
|
||||
.bind(chapter_id)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(same_id, page_id, "re-crawl keeps the page id");
|
||||
|
||||
// ...but its stale analysis is cleared so it gets re-derived.
|
||||
for table in ["page_analysis", "page_ocr_text", "page_content_warnings"] {
|
||||
let n: i64 = sqlx::query_scalar(&format!(
|
||||
"SELECT COUNT(*) FROM {table} WHERE page_id = $1"
|
||||
))
|
||||
.bind(page_id)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(n, 0, "{table} must be invalidated when the re-crawl updates the page");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -66,6 +66,43 @@ const LEASE_DURATION: Duration = Duration::from_secs(60);
|
||||
/// the lease window leaves two missed-beat's slack before expiry.
|
||||
const LEASE_HEARTBEAT: Duration = Duration::from_secs(20);
|
||||
|
||||
/// Consecutive failed lease renews the heartbeat tolerates before it gives up
|
||||
/// and signals the worker to abandon the in-flight dispatch. At
|
||||
/// [`LEASE_HEARTBEAT`] spacing this is ~1 lease window of DB flakiness — past
|
||||
/// that the lease has very likely lapsed and another worker may re-lease the
|
||||
/// job, so continuing to crawl it is wasted (and duplicated) work.
|
||||
const MAX_HEARTBEAT_RENEW_FAILURES: u32 = 3;
|
||||
|
||||
/// Whether the heartbeat should abandon the job after `consecutive_failures`
|
||||
/// failed renews. Split out so the escalation threshold is unit-testable.
|
||||
fn should_abort_after_renew_failures(consecutive_failures: u32) -> bool {
|
||||
consecutive_failures >= MAX_HEARTBEAT_RENEW_FAILURES
|
||||
}
|
||||
|
||||
/// How long a worker waits after a `BrowserUnavailable` outcome before looping
|
||||
/// back to lease again. The job was released (not failed) so it stays pending;
|
||||
/// this backoff keeps the worker from hot-looping lease→acquire→release while
|
||||
/// the browser is down or mid-restart.
|
||||
const BROWSER_UNAVAILABLE_BACKOFF: Duration = Duration::from_secs(5);
|
||||
|
||||
/// Longest an idle worker waits between lease polls. Bounds the exponential
|
||||
/// [`idle_backoff`] so a worker still notices freshly-enqueued work reasonably
|
||||
/// soon after a quiet spell.
|
||||
const IDLE_BACKOFF_CAP: Duration = Duration::from_secs(30);
|
||||
|
||||
/// Backoff for a worker that keeps finding no work: 1s, 2s, 4s, … capped at
|
||||
/// [`IDLE_BACKOFF_CAP`]. `consecutive_empty` is the number of empty polls seen
|
||||
/// so far (0 on the first miss); reset to 0 the moment a job is leased. Replaces
|
||||
/// the old flat 1s sleep so an idle daemon isn't firing a row-locking `SELECT …
|
||||
/// FOR UPDATE SKIP LOCKED` lease query every second per worker.
|
||||
fn idle_backoff(consecutive_empty: u32) -> Duration {
|
||||
let cap = IDLE_BACKOFF_CAP.as_secs();
|
||||
// 1 << n grows the interval; saturate to the cap once the shift overflows
|
||||
// or the value exceeds the cap.
|
||||
let secs = 1u64.checked_shl(consecutive_empty).unwrap_or(cap).min(cap);
|
||||
Duration::from_secs(secs)
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
pub trait MetadataPass: Send + Sync {
|
||||
async fn run(&self) -> anyhow::Result<pipeline::MetadataStats>;
|
||||
@@ -356,6 +393,9 @@ struct WorkerContext {
|
||||
|
||||
impl WorkerContext {
|
||||
async fn run(self) {
|
||||
// Consecutive empty lease polls, driving the idle backoff. Reset to 0
|
||||
// the moment any job is leased.
|
||||
let mut idle_streak: u32 = 0;
|
||||
loop {
|
||||
if self.cancel.is_cancelled() {
|
||||
tracing::info!(worker = self.id, "worker: shutdown");
|
||||
@@ -385,11 +425,14 @@ impl WorkerContext {
|
||||
}
|
||||
};
|
||||
let Some(lease) = leases.into_iter().next() else {
|
||||
let backoff = idle_backoff(idle_streak);
|
||||
idle_streak = idle_streak.saturating_add(1);
|
||||
tokio::select! {
|
||||
_ = tokio::time::sleep(Duration::from_secs(1)) => continue,
|
||||
_ = tokio::time::sleep(backoff) => continue,
|
||||
_ = self.cancel.cancelled() => return,
|
||||
}
|
||||
};
|
||||
idle_streak = 0;
|
||||
self.process_lease(lease).await;
|
||||
}
|
||||
}
|
||||
@@ -413,18 +456,34 @@ impl WorkerContext {
|
||||
// dispatch runs, so a slow-but-healthy job is never re-leased and
|
||||
// never inflates `attempts` toward `max_attempts`. Stops itself
|
||||
// once the job is no longer ours (renew returns false).
|
||||
// Signalled by the heartbeat if it gives up after too many consecutive
|
||||
// renew failures, so the worker can abandon a dispatch whose lease has
|
||||
// very likely lapsed (rather than crawl a job another worker may now own).
|
||||
let hb_lost = CancellationToken::new();
|
||||
let heartbeat = {
|
||||
let hb_pool = self.pool.clone();
|
||||
let hb_id = lease.id;
|
||||
let hb_gen = lease.lease_generation;
|
||||
let hb_lost = hb_lost.clone();
|
||||
tokio::spawn(async move {
|
||||
let mut failures: u32 = 0;
|
||||
loop {
|
||||
tokio::time::sleep(LEASE_HEARTBEAT).await;
|
||||
match jobs::renew(&hb_pool, hb_id, hb_gen, LEASE_DURATION).await {
|
||||
Ok(true) => {}
|
||||
Ok(true) => failures = 0,
|
||||
Ok(false) => break,
|
||||
Err(e) => {
|
||||
tracing::warn!(lease_id = %hb_id, ?e, "heartbeat renew failed");
|
||||
failures += 1;
|
||||
tracing::warn!(lease_id = %hb_id, failures, ?e, "heartbeat renew failed");
|
||||
if should_abort_after_renew_failures(failures) {
|
||||
tracing::error!(
|
||||
lease_id = %hb_id,
|
||||
failures,
|
||||
"heartbeat lost the lease after repeated renew failures — signalling abandon"
|
||||
);
|
||||
hb_lost.cancel();
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -462,6 +521,20 @@ impl WorkerContext {
|
||||
);
|
||||
return;
|
||||
}
|
||||
_ = hb_lost.cancelled() => {
|
||||
// The heartbeat gave up renewing: the lease has very likely
|
||||
// expired and may already be re-leased elsewhere. Abandon the
|
||||
// dispatch rather than keep crawling a job we no longer own. Do
|
||||
// NOT ack/release — a generation-guarded write would no-op, and
|
||||
// another worker may now hold this lease.
|
||||
heartbeat.abort();
|
||||
tracing::error!(
|
||||
worker = self.id,
|
||||
lease_id = %lease.id,
|
||||
"worker: abandoning dispatch — lease lost (heartbeat renew failures)"
|
||||
);
|
||||
return;
|
||||
}
|
||||
o = tokio::time::timeout(self.job_timeout, dispatch) => o,
|
||||
};
|
||||
heartbeat.abort();
|
||||
@@ -502,6 +575,24 @@ impl WorkerContext {
|
||||
self.status.poke();
|
||||
let _ = jobs::release(&self.pool, lease.id, lease.lease_generation).await;
|
||||
}
|
||||
Ok(Ok(SyncOutcome::BrowserUnavailable)) => {
|
||||
// Infrastructure outage, not a job failure: the browser was
|
||||
// down or mid-restart when the dispatcher tried to acquire it.
|
||||
// Return the job to `pending` WITHOUT burning an attempt (like
|
||||
// the cancel/session paths) so an outage doesn't chew the whole
|
||||
// backlog to `dead`, then back off to avoid hot-looping while
|
||||
// the browser recovers.
|
||||
tracing::warn!(
|
||||
worker = self.id,
|
||||
lease_id = %lease.id,
|
||||
"worker: browser unavailable — released lease without burning an attempt"
|
||||
);
|
||||
let _ = jobs::release(&self.pool, lease.id, lease.lease_generation).await;
|
||||
tokio::select! {
|
||||
_ = tokio::time::sleep(BROWSER_UNAVAILABLE_BACKOFF) => {}
|
||||
_ = self.cancel.cancelled() => {}
|
||||
}
|
||||
}
|
||||
Ok(Err(e)) => {
|
||||
tracing::warn!(
|
||||
worker = self.id,
|
||||
@@ -750,6 +841,37 @@ mod tests {
|
||||
Utc.with_ymd_and_hms(y, mo, d, h, mi, 0).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn heartbeat_aborts_only_after_threshold_consecutive_failures() {
|
||||
// A blip or two is tolerated; sustained failures escalate to abandon.
|
||||
assert!(!should_abort_after_renew_failures(0));
|
||||
assert!(!should_abort_after_renew_failures(1));
|
||||
assert!(!should_abort_after_renew_failures(MAX_HEARTBEAT_RENEW_FAILURES - 1));
|
||||
assert!(should_abort_after_renew_failures(MAX_HEARTBEAT_RENEW_FAILURES));
|
||||
assert!(should_abort_after_renew_failures(MAX_HEARTBEAT_RENEW_FAILURES + 1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn idle_backoff_grows_then_caps() {
|
||||
// First miss is a short 1s poll; the interval doubles each empty poll…
|
||||
assert_eq!(idle_backoff(0), Duration::from_secs(1));
|
||||
assert_eq!(idle_backoff(1), Duration::from_secs(2));
|
||||
assert_eq!(idle_backoff(2), Duration::from_secs(4));
|
||||
assert_eq!(idle_backoff(3), Duration::from_secs(8));
|
||||
assert_eq!(idle_backoff(4), Duration::from_secs(16));
|
||||
// …and saturates at the cap rather than growing unbounded.
|
||||
assert_eq!(idle_backoff(5), IDLE_BACKOFF_CAP);
|
||||
assert_eq!(idle_backoff(100), IDLE_BACKOFF_CAP);
|
||||
// Never exceeds the cap and is monotonic non-decreasing.
|
||||
let mut prev = Duration::ZERO;
|
||||
for n in 0..40 {
|
||||
let b = idle_backoff(n);
|
||||
assert!(b >= prev, "backoff must be non-decreasing at n={n}");
|
||||
assert!(b <= IDLE_BACKOFF_CAP, "backoff must never exceed the cap");
|
||||
prev = b;
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn next_fire_in_utc_at_midnight_advances_one_day() {
|
||||
let now = dt_utc(2026, 5, 25, 12, 0); // noon UTC
|
||||
|
||||
301
backend/src/crawler/intercept.rs
Normal file
301
backend/src/crawler/intercept.rs
Normal file
@@ -0,0 +1,301 @@
|
||||
//! Optional CDP-level SSRF guard for headless-browser navigations.
|
||||
//!
|
||||
//! The reqwest clients get a DNS-filtering resolver (see
|
||||
//! [`crate::crawler::safety::SafeResolver`]) so an image/API host that
|
||||
//! resolves to an internal IP is refused at connect time. Chromium, however,
|
||||
//! does its **own** DNS and connection handling, so that resolver can't see
|
||||
//! browser navigations. Without a second guard, a scraped chapter page that
|
||||
//! `302`-redirects the browser to `http://127.0.0.1:5432/` (or an
|
||||
//! attacker-owned hostname that resolves to `169.254.169.254`) would be
|
||||
//! loaded and parsed as if it were catalog content.
|
||||
//!
|
||||
//! This module installs CDP `Fetch` interception on a page so **every** request
|
||||
//! it issues — the main-frame Document, its redirects, *and every subresource*
|
||||
//! (`<img>`, `fetch()`/XHR, media, …) — is re-validated through the same
|
||||
//! [`ensure_public_target`] + resolved-IP check the reqwest paths use, failing
|
||||
//! any that target an internal address. Intercepting only the Document would
|
||||
//! leave a scraped page free to pull `<img src="http://169.254.169.254/…">` or
|
||||
//! `fetch('http://postgres:5432')` straight past the guard, so no resource type
|
||||
//! is exempt. (WebSocket handshakes are not surfaced by CDP `Fetch`, so `ws://`
|
||||
//! internal targets remain out of this hook's reach — the reqwest-layer
|
||||
//! resolver does not see them either; documented as a known gap.)
|
||||
//!
|
||||
//! **Opt-in / default-off.** Enabling `Fetch` means every intercepted request
|
||||
//! *must* be resolved by a live handler or the navigation hangs, so this is a
|
||||
//! fragile hook in the crawler's critical path. It ships behind
|
||||
//! `CRAWLER_SSRF_INTERCEPT` (default `false`) and the wiring has **not** been
|
||||
//! exercised against a real Chromium in CI — validate with a manual crawl
|
||||
//! before enabling in production. When disabled, [`open_page`] is byte-for-byte
|
||||
//! the previous `browser.new_page(url)` behavior. When enabled, the guard is
|
||||
//! **fail-closed**: if interception can't be installed, [`open_page`] closes the
|
||||
//! blank page and returns the error rather than navigating unguarded.
|
||||
|
||||
use std::net::IpAddr;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
|
||||
use chromiumoxide::browser::Browser;
|
||||
use chromiumoxide::cdp::browser_protocol::fetch::{
|
||||
ContinueRequestParams, EnableParams, EventRequestPaused, FailRequestParams, RequestPattern,
|
||||
RequestStage,
|
||||
};
|
||||
use chromiumoxide::cdp::browser_protocol::network::ErrorReason;
|
||||
use chromiumoxide::error::Result as CdpResult;
|
||||
use chromiumoxide::Page;
|
||||
use futures_util::StreamExt;
|
||||
use reqwest::Url;
|
||||
|
||||
use crate::crawler::safety::{ensure_public_target, is_private_ip};
|
||||
|
||||
/// Process-wide toggle, set once at startup from `CRAWLER_SSRF_INTERCEPT`.
|
||||
/// A single boot-time flag (rather than threading a param through every
|
||||
/// crawler navigation signature) keeps the off-path a no-op.
|
||||
static ENABLED: AtomicBool = AtomicBool::new(false);
|
||||
|
||||
pub fn set_enabled(on: bool) {
|
||||
ENABLED.store(on, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn is_enabled() -> bool {
|
||||
ENABLED.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// What to do with a navigation URL, decided without DNS where possible so the
|
||||
/// security-critical branching is unit-testable.
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub(crate) enum Verdict {
|
||||
/// Safe to continue (non-network scheme, or a public IP literal).
|
||||
Allow,
|
||||
/// Refuse — bad scheme handled elsewhere, localhost, or a private IP literal.
|
||||
Block,
|
||||
/// A hostname that must be resolved to decide (DNS-rebinding check).
|
||||
ResolveHost { host: String, port: u16 },
|
||||
}
|
||||
|
||||
/// Pure decision over a URL string. `data:` / `blob:` / `about:` and other
|
||||
/// non-http(s) schemes are allowed (not a network-SSRF vector); http(s) with a
|
||||
/// private/loopback/localhost literal is blocked; an http(s) hostname needs
|
||||
/// resolution.
|
||||
pub(crate) fn verdict(url: &str) -> Verdict {
|
||||
let Ok(parsed) = Url::parse(url) else {
|
||||
// Unparseable — let Chromium reject it; not our call to make.
|
||||
return Verdict::Allow;
|
||||
};
|
||||
match parsed.scheme() {
|
||||
"http" | "https" => {}
|
||||
// Non-http(s) subresource schemes: block the ones that can reach the
|
||||
// local filesystem or pivot to another protocol; continue the rest
|
||||
// (data:/blob:/about: and Chrome-internal schemes) which legitimate page
|
||||
// loads depend on — blocking those would wedge navigation.
|
||||
"file" | "ftp" | "gopher" => return Verdict::Block,
|
||||
_ => return Verdict::Allow,
|
||||
}
|
||||
// Literal private IP / localhost / missing host → block (string check).
|
||||
if ensure_public_target(url).is_err() {
|
||||
return Verdict::Block;
|
||||
}
|
||||
match parsed.host_str() {
|
||||
Some(host) if !is_ip_literal(host) => Verdict::ResolveHost {
|
||||
host: host.to_string(),
|
||||
port: parsed.port_or_known_default().unwrap_or(80),
|
||||
},
|
||||
// Public IP literal (ensure_public_target already passed it).
|
||||
_ => Verdict::Allow,
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether `host` (as reqwest's `host_str()` yields it) is an IP literal.
|
||||
/// IPv6 literals arrive bracketed (`[::1]`), which don't parse as `IpAddr`
|
||||
/// directly — strip the brackets first.
|
||||
fn is_ip_literal(host: &str) -> bool {
|
||||
let unbracketed = host
|
||||
.strip_prefix('[')
|
||||
.and_then(|s| s.strip_suffix(']'))
|
||||
.unwrap_or(host);
|
||||
unbracketed.parse::<IpAddr>().is_ok()
|
||||
}
|
||||
|
||||
/// Async decision: resolves hostnames and blocks if any resolved address is
|
||||
/// private (DNS rebinding). A resolution failure is *not* treated as blocked —
|
||||
/// Chromium won't be able to connect either, so there's nothing to exfiltrate.
|
||||
pub(crate) async fn is_blocked(url: &str) -> bool {
|
||||
match verdict(url) {
|
||||
Verdict::Allow => false,
|
||||
Verdict::Block => true,
|
||||
Verdict::ResolveHost { host, port } => match tokio::net::lookup_host((host.as_str(), port))
|
||||
.await
|
||||
{
|
||||
Ok(addrs) => addrs.map(|a| a.ip()).any(|ip| is_private_ip(&ip)),
|
||||
Err(_) => false,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// Open a page for navigation. With interception off, this is exactly
|
||||
/// `browser.new_page(url)`. With it on, the page is created blank (no network),
|
||||
/// the navigation guard is installed, and only then does it navigate — so the
|
||||
/// initial request and any redirects pass through the guard.
|
||||
pub async fn open_page(browser: &Browser, url: &str) -> CdpResult<Page> {
|
||||
if !is_enabled() {
|
||||
return browser.new_page(url).await;
|
||||
}
|
||||
let page = browser.new_page("about:blank").await?;
|
||||
if let Err(e) = install_navigation_guard(&page).await {
|
||||
// Fail closed: an unguarded page could be redirected (or pull a
|
||||
// subresource) to an internal target, so refuse to navigate. Close the
|
||||
// blank page and surface the error so the caller aborts this fetch.
|
||||
tracing::warn!(url = %url, error = %e, "SSRF navigation guard failed to install; aborting navigation (fail-closed)");
|
||||
let _ = page.close().await;
|
||||
return Err(e);
|
||||
}
|
||||
page.goto(url).await?;
|
||||
Ok(page)
|
||||
}
|
||||
|
||||
/// The `Fetch` interception patterns to register. A single pattern with **no**
|
||||
/// `resource_type` constraint matches every request the page makes — Document,
|
||||
/// Image, Fetch/XHR, media, everything — so subresources to internal targets are
|
||||
/// re-validated too, not only the main-frame navigation. Restricting this to
|
||||
/// `ResourceType::Document` (the previous behavior) left every `<img>`/`fetch()`
|
||||
/// subresource unguarded, which is the SSRF hole this closes. Pinned to the
|
||||
/// request stage so each request is paused once, before it leaves the browser.
|
||||
fn interception_patterns() -> Vec<RequestPattern> {
|
||||
vec![RequestPattern::builder()
|
||||
.request_stage(RequestStage::Request)
|
||||
.build()]
|
||||
}
|
||||
|
||||
/// Enable `Fetch` for all requests on `page` and spawn a task that
|
||||
/// continues/fails each paused request per [`is_blocked`].
|
||||
async fn install_navigation_guard(page: &Page) -> CdpResult<()> {
|
||||
page.execute(EnableParams {
|
||||
patterns: Some(interception_patterns()),
|
||||
handle_auth_requests: None,
|
||||
})
|
||||
.await?;
|
||||
|
||||
let mut paused = page.event_listener::<EventRequestPaused>().await?;
|
||||
let handler_page = page.clone();
|
||||
tokio::spawn(async move {
|
||||
while let Some(ev) = paused.next().await {
|
||||
let request_id = ev.request_id.clone();
|
||||
let outcome = if is_blocked(&ev.request.url).await {
|
||||
tracing::warn!(
|
||||
url = %ev.request.url,
|
||||
"SSRF guard blocked browser navigation to an internal target"
|
||||
);
|
||||
handler_page
|
||||
.execute(FailRequestParams::new(request_id, ErrorReason::BlockedByClient))
|
||||
.await
|
||||
.map(|_| ())
|
||||
} else {
|
||||
handler_page
|
||||
.execute(ContinueRequestParams::new(request_id))
|
||||
.await
|
||||
.map(|_| ())
|
||||
};
|
||||
if let Err(e) = outcome {
|
||||
// The page/session is gone (page closed) — the stream will end
|
||||
// too; stop handling so the task exits rather than spins.
|
||||
tracing::debug!(error = %e, "fetch interceptor: continue/fail failed, ending handler");
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn verdict_allows_non_network_schemes() {
|
||||
for url in ["about:blank", "data:text/html,hi", "blob:abc", "chrome://version"] {
|
||||
assert_eq!(verdict(url), Verdict::Allow, "{url} should be allowed");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn verdict_blocks_dangerous_non_http_schemes() {
|
||||
// file:/ftp:/gopher: can reach the local filesystem or pivot protocols;
|
||||
// block them explicitly rather than falling through to Allow.
|
||||
for url in ["file:///etc/passwd", "ftp://internal/x", "gopher://internal:70/1"] {
|
||||
assert_eq!(verdict(url), Verdict::Block, "{url} should be blocked");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn verdict_blocks_private_ip_literals_and_localhost() {
|
||||
for url in [
|
||||
"http://127.0.0.1/",
|
||||
"http://10.0.0.1/",
|
||||
"http://192.168.1.1:5432/",
|
||||
"http://169.254.169.254/latest/meta-data/",
|
||||
"http://[::1]/",
|
||||
"http://[::ffff:127.0.0.1]/",
|
||||
"http://localhost/",
|
||||
"https://[fd00::1]/",
|
||||
] {
|
||||
assert_eq!(verdict(url), Verdict::Block, "{url} should be blocked");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn verdict_allows_public_ip_literals() {
|
||||
for url in ["http://93.184.216.34/", "https://[2606:4700:4700::1111]/"] {
|
||||
assert_eq!(verdict(url), Verdict::Allow, "{url} should be allowed");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn verdict_defers_hostnames_to_resolution() {
|
||||
assert_eq!(
|
||||
verdict("https://cdn.example.com/img.jpg"),
|
||||
Verdict::ResolveHost {
|
||||
host: "cdn.example.com".to_string(),
|
||||
port: 443
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
verdict("http://catalog.test:8080/list"),
|
||||
Verdict::ResolveHost {
|
||||
host: "catalog.test".to_string(),
|
||||
port: 8080
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn is_blocked_true_for_private_literal_no_dns() {
|
||||
assert!(is_blocked("http://169.254.169.254/").await);
|
||||
assert!(!is_blocked("http://93.184.216.34/").await);
|
||||
// Non-network scheme is always allowed through.
|
||||
assert!(!is_blocked("about:blank").await);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn interception_covers_all_resource_types() {
|
||||
// Regression guard for the SSRF subresource hole: the CDP Fetch pattern
|
||||
// must NOT be constrained to Document, or `<img>`/`fetch()`/XHR
|
||||
// subresources to internal targets slip past `is_blocked`. A pattern
|
||||
// with no `resource_type` set intercepts every request type.
|
||||
let patterns = interception_patterns();
|
||||
assert_eq!(patterns.len(), 1);
|
||||
assert!(
|
||||
patterns[0].resource_type.is_none(),
|
||||
"interception must cover all resource types (subresources included), not just Document"
|
||||
);
|
||||
assert_eq!(patterns[0].request_stage, Some(RequestStage::Request));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn enabled_toggle_roundtrips() {
|
||||
// Global; restore afterwards so other tests see the default.
|
||||
let prev = is_enabled();
|
||||
set_enabled(true);
|
||||
assert!(is_enabled());
|
||||
set_enabled(false);
|
||||
assert!(!is_enabled());
|
||||
set_enabled(prev);
|
||||
}
|
||||
}
|
||||
@@ -495,15 +495,33 @@ pub async fn reap_terminal(pool: &PgPool, retention_days: u32) -> sqlx::Result<u
|
||||
if retention_days == 0 {
|
||||
return Ok(0);
|
||||
}
|
||||
let result = sqlx::query(
|
||||
"DELETE FROM crawler_jobs \
|
||||
WHERE state IN ('done', 'dead') \
|
||||
AND updated_at < now() - ($1::bigint || ' days')::interval",
|
||||
)
|
||||
.bind(retention_days as i64)
|
||||
.execute(pool)
|
||||
.await?;
|
||||
Ok(result.rows_affected())
|
||||
// Delete in bounded batches rather than one statement. A single unbatched
|
||||
// DELETE over an unbounded backlog takes a long-held lock and one huge
|
||||
// transaction; batching keeps each delete short (index-backed by
|
||||
// crawler_jobs_terminal_reap_idx, 0040) so the reaper never pins the table
|
||||
// (audit M5). Loop until a batch comes back short.
|
||||
const BATCH: i64 = 5_000;
|
||||
let mut total: u64 = 0;
|
||||
loop {
|
||||
let result = sqlx::query(
|
||||
"DELETE FROM crawler_jobs \
|
||||
WHERE ctid IN ( \
|
||||
SELECT ctid FROM crawler_jobs \
|
||||
WHERE state IN ('done', 'dead') \
|
||||
AND updated_at < now() - ($1::bigint || ' days')::interval \
|
||||
LIMIT $2 )",
|
||||
)
|
||||
.bind(retention_days as i64)
|
||||
.bind(BATCH)
|
||||
.execute(pool)
|
||||
.await?;
|
||||
let n = result.rows_affected();
|
||||
total += n;
|
||||
if n < BATCH as u64 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
Ok(total)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -19,6 +19,7 @@ pub mod content;
|
||||
pub mod daemon;
|
||||
pub mod detect;
|
||||
pub mod diff;
|
||||
pub mod intercept;
|
||||
pub mod jobs;
|
||||
pub mod nav;
|
||||
pub mod pipeline;
|
||||
|
||||
@@ -85,6 +85,28 @@ pub async fn wait_for_selector(
|
||||
/// under a second.
|
||||
pub const SELECTOR_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
|
||||
/// Run `body` to completion, then run `close` — on *every* exit path,
|
||||
/// including an early `?` / `return` inside `body`. chromiumoxide's `Page`
|
||||
/// does **not** close its CDP target on drop, so a fetch helper that opens a
|
||||
/// page with `browser.new_page(...)` and then bails via `?` on a nav /
|
||||
/// content-read error would leak a browser tab for the process's lifetime.
|
||||
/// Wrapping the fallible work in `close_after` — with `close` built from a
|
||||
/// clone of the page — guarantees the tab is closed regardless of how `body`
|
||||
/// returns.
|
||||
///
|
||||
/// Both arguments are pre-built futures, so this stays generic over the
|
||||
/// body's return type (the anyhow and `PageError` fetch paths both use it)
|
||||
/// and references no browser types — which also lets it be unit-tested
|
||||
/// without standing up a real `Page`.
|
||||
pub(crate) async fn close_after<R>(
|
||||
close: impl std::future::Future<Output = ()>,
|
||||
body: impl std::future::Future<Output = R>,
|
||||
) -> R {
|
||||
let result = body.await;
|
||||
close.await;
|
||||
result
|
||||
}
|
||||
|
||||
impl NavError {
|
||||
/// Does this navigation error indicate the underlying Chromium
|
||||
/// process has died or its CDP connection has dropped? Used by the
|
||||
@@ -150,6 +172,41 @@ mod tests {
|
||||
assert!(result.is_err(), "expected Elapsed on a hung future");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn close_after_runs_close_and_returns_value_on_ok() {
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::Arc;
|
||||
let closed = Arc::new(AtomicBool::new(false));
|
||||
let c = closed.clone();
|
||||
let out: anyhow::Result<i32> = close_after(
|
||||
async move { c.store(true, Ordering::SeqCst) },
|
||||
async { Ok(7) },
|
||||
)
|
||||
.await;
|
||||
assert_eq!(out.unwrap(), 7);
|
||||
assert!(closed.load(Ordering::SeqCst), "close must run on the happy path");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn close_after_runs_close_even_when_body_errs() {
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::Arc;
|
||||
let closed = Arc::new(AtomicBool::new(false));
|
||||
let c = closed.clone();
|
||||
// This is the leak the fix targets: the body bails before it could
|
||||
// close the page itself, and `close_after` must still close it.
|
||||
let out: anyhow::Result<()> =
|
||||
close_after(async move { c.store(true, Ordering::SeqCst) }, async {
|
||||
anyhow::bail!("nav failed")
|
||||
})
|
||||
.await;
|
||||
assert!(out.is_err());
|
||||
assert!(
|
||||
closed.load(Ordering::SeqCst),
|
||||
"the page must be closed even when the body returns Err"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nav_error_timeout_message_includes_duration() {
|
||||
let e = NavError::Timeout(Duration::from_secs(30));
|
||||
|
||||
@@ -135,28 +135,7 @@ impl ResyncService for RealResyncService {
|
||||
.await
|
||||
.with_context(|| format!("fetch_manga during resync of {manga_id}"))?;
|
||||
|
||||
// Partial-render guard: same logic as run_metadata_pass.
|
||||
let source_id = source.id();
|
||||
if !manga.chapters.is_empty() || {
|
||||
let prior = repo::crawler::live_chapter_count_for_source_manga(
|
||||
&self.db,
|
||||
source_id,
|
||||
&source_manga_key,
|
||||
)
|
||||
.await
|
||||
.unwrap_or(0);
|
||||
prior == 0
|
||||
} {
|
||||
// Either the new fetch surfaced chapters, or there were
|
||||
// none before either — chapter sync is safe to run.
|
||||
} else {
|
||||
tracing::warn!(
|
||||
%manga_id,
|
||||
source_url = %source_url,
|
||||
"resync_manga: fetch returned empty chapters but prior count > 0; skipping chapter sync to avoid soft-drop"
|
||||
);
|
||||
}
|
||||
|
||||
let upsert = repo::crawler::upsert_manga_from_source(
|
||||
&self.db,
|
||||
source_id,
|
||||
@@ -201,6 +180,11 @@ impl ResyncService for RealResyncService {
|
||||
)
|
||||
.await
|
||||
.unwrap_or(0);
|
||||
// Partial-render guard (same logic as run_metadata_pass): only sync
|
||||
// chapters when the fetch surfaced some, or the manga never had any.
|
||||
// A fetch that returned empty while a prior count exists is almost
|
||||
// certainly a partial render — skip the sync so we don't soft-drop the
|
||||
// real chapters.
|
||||
if !manga.chapters.is_empty() || prior_chapter_count == 0 {
|
||||
match repo::crawler::sync_manga_chapters(
|
||||
&self.db,
|
||||
@@ -223,6 +207,12 @@ impl ResyncService for RealResyncService {
|
||||
"resync_manga: chapter sync failed"
|
||||
),
|
||||
}
|
||||
} else {
|
||||
tracing::warn!(
|
||||
%manga_id,
|
||||
source_url = %source_url,
|
||||
"resync_manga: fetch returned empty chapters but prior count > 0; skipping chapter sync to avoid soft-drop"
|
||||
);
|
||||
}
|
||||
|
||||
drop(lease);
|
||||
@@ -280,6 +270,12 @@ impl ResyncService for RealResyncService {
|
||||
SyncOutcome::SessionExpired => {
|
||||
anyhow::bail!("source session expired — operator must refresh PHPSESSID")
|
||||
}
|
||||
// Unreachable here: resync already holds its own browser lease and
|
||||
// `sync_chapter_content` never acquires one, so it can't report the
|
||||
// browser unavailable. Handled defensively for exhaustiveness.
|
||||
SyncOutcome::BrowserUnavailable => {
|
||||
anyhow::bail!("crawler browser unavailable")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -31,11 +31,13 @@
|
||||
//! URL string, and the byte accumulator is keyed off a generic stream.
|
||||
//! Easy to unit-test without a live network or browser.
|
||||
|
||||
use std::net::IpAddr;
|
||||
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr};
|
||||
use std::sync::Arc;
|
||||
|
||||
use anyhow::{bail, Context};
|
||||
use bytes::BytesMut;
|
||||
use futures_util::StreamExt;
|
||||
use reqwest::dns::{Addrs, Name, Resolve, Resolving};
|
||||
use reqwest::Url;
|
||||
|
||||
/// Default per-image download cap. A page image is generally <2 MiB;
|
||||
@@ -179,7 +181,7 @@ fn ensure_public_target_inner(raw_url: &str) -> Result<Url, UrlSafetyError> {
|
||||
Ok(url)
|
||||
}
|
||||
|
||||
fn is_private_ip(ip: &IpAddr) -> bool {
|
||||
pub(crate) fn is_private_ip(ip: &IpAddr) -> bool {
|
||||
match ip {
|
||||
IpAddr::V4(v4) => {
|
||||
v4.is_loopback()
|
||||
@@ -193,11 +195,14 @@ fn is_private_ip(ip: &IpAddr) -> bool {
|
||||
|| v4.octets()[0] == 0
|
||||
}
|
||||
IpAddr::V6(v6) => {
|
||||
// IPv4-mapped IPv6 (::ffff:0:0/96): unwrap to the embedded
|
||||
// IPv4 and recurse so `::ffff:127.0.0.1` is caught by the
|
||||
// IPv4 loopback check rather than passing through.
|
||||
// `Ipv6Addr::is_loopback()` only matches `::1` exactly.
|
||||
if let Some(v4) = v6.to_ipv4_mapped() {
|
||||
// Any IPv6 form that *embeds* an IPv4 address (mapped
|
||||
// `::ffff:0:0/96`, compatible `::/96`, NAT64 `64:ff9b::/96`,
|
||||
// 6to4 `2002::/16`) is unwrapped and re-checked as its IPv4 —
|
||||
// otherwise `::127.0.0.1` / `2002:7f00:1::` / `64:ff9b::7f00:1`
|
||||
// would smuggle an internal IPv4 past the check (the audit's
|
||||
// IPv6-embedding gap). `Ipv6Addr::is_loopback()` only matches
|
||||
// `::1` exactly, so these embeddings need explicit handling.
|
||||
if let Some(v4) = embedded_ipv4(v6) {
|
||||
return is_private_ip(&IpAddr::V4(v4));
|
||||
}
|
||||
v6.is_loopback()
|
||||
@@ -210,6 +215,114 @@ fn is_private_ip(ip: &IpAddr) -> bool {
|
||||
}
|
||||
}
|
||||
|
||||
/// Extract the IPv4 address embedded in an IPv6 literal, for every
|
||||
/// transitional encoding that can carry one: IPv4-mapped (`::ffff:0:0/96`),
|
||||
/// IPv4-compatible (`::/96`, deprecated but still routable via some stacks),
|
||||
/// NAT64 (`64:ff9b::/96`), and 6to4 (`2002::/16`). Returns `None` for a
|
||||
/// native IPv6 address. Callers recurse into [`is_private_ip`] on the result
|
||||
/// so a private IPv4 can't hide inside an IPv6 literal.
|
||||
fn embedded_ipv4(v6: &Ipv6Addr) -> Option<Ipv4Addr> {
|
||||
let seg = v6.segments();
|
||||
let low32 = |a: u16, b: u16| Ipv4Addr::new((a >> 8) as u8, (a & 0xff) as u8, (b >> 8) as u8, (b & 0xff) as u8);
|
||||
// ::ffff:0:0/96 (mapped) and ::/96 (compatible) — top 96 bits zero,
|
||||
// except mapped which has 0xffff at seg[5]. to_ipv4() covers both.
|
||||
if seg[0..5] == [0, 0, 0, 0, 0] && (seg[5] == 0 || seg[5] == 0xffff) {
|
||||
return Some(low32(seg[6], seg[7]));
|
||||
}
|
||||
// NAT64 64:ff9b::/96
|
||||
if seg[0] == 0x0064 && seg[1] == 0xff9b && seg[2..6] == [0, 0, 0, 0] {
|
||||
return Some(low32(seg[6], seg[7]));
|
||||
}
|
||||
// 6to4 2002::/16 — embedded IPv4 is bits 16..48 (seg[1], seg[2]).
|
||||
if seg[0] == 0x2002 {
|
||||
return Some(low32(seg[1], seg[2]));
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// A `reqwest::dns::Resolve` that performs normal system resolution, then
|
||||
/// drops any resolved address in a private / loopback / link-local / metadata
|
||||
/// range. Installed on every crawler + analysis reqwest client so a hostname
|
||||
/// that resolves to an internal IP (DNS rebinding: an attacker-owned domain
|
||||
/// with an `A` record for `169.254.169.254` or `10.x`) can never be connected
|
||||
/// to — closing the TOCTOU gap that the string-only `ensure_public_target`
|
||||
/// check leaves open. Fires per connection, so it also guards redirect hops.
|
||||
#[derive(Debug, Default)]
|
||||
pub struct SafeResolver;
|
||||
|
||||
/// Partition resolved addresses into the public ones, rejecting when nothing
|
||||
/// survives. Split out from the async `resolve` so the security-critical
|
||||
/// filter is unit-testable without real DNS.
|
||||
fn retain_public_addrs(
|
||||
host: &str,
|
||||
addrs: impl Iterator<Item = SocketAddr>,
|
||||
) -> Result<Vec<SocketAddr>, BlockedResolution> {
|
||||
let public: Vec<SocketAddr> = addrs.filter(|a| !is_private_ip(&a.ip())).collect();
|
||||
if public.is_empty() {
|
||||
return Err(BlockedResolution(host.to_string()));
|
||||
}
|
||||
Ok(public)
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
#[error("host {0} resolved only to private/blocked addresses")]
|
||||
struct BlockedResolution(String);
|
||||
|
||||
impl Resolve for SafeResolver {
|
||||
fn resolve(&self, name: Name) -> Resolving {
|
||||
Box::pin(async move {
|
||||
let host = name.as_str().to_string();
|
||||
// Port 0: reqwest overrides it with the URL's port after
|
||||
// resolution (same convention as reqwest's default GaiResolver).
|
||||
let resolved = tokio::net::lookup_host((host.as_str(), 0))
|
||||
.await
|
||||
.map_err(|e| Box::new(e) as Box<dyn std::error::Error + Send + Sync>)?;
|
||||
let public = retain_public_addrs(&host, resolved)
|
||||
.map_err(|e| Box::new(e) as Box<dyn std::error::Error + Send + Sync>)?;
|
||||
Ok(Box::new(public.into_iter()) as Addrs)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Shared [`SafeResolver`] for wiring into `ClientBuilder::dns_resolver`.
|
||||
pub fn safe_dns_resolver() -> Arc<SafeResolver> {
|
||||
Arc::new(SafeResolver)
|
||||
}
|
||||
|
||||
/// Whether the DNS-rebinding [`SafeResolver`] should be attached to a crawler
|
||||
/// reqwest client given the configured proxy (if any).
|
||||
///
|
||||
/// The resolver only guards targets that *reqwest itself* resolves:
|
||||
/// - **No proxy** (direct): reqwest resolves the target — attach.
|
||||
/// - **`http(s)://` proxy**: reqwest still resolves the target host locally and
|
||||
/// issues a `CONNECT`, so a hostname resolving to a private IP must be
|
||||
/// refused — attach.
|
||||
/// - **`socks5://` / `socks5h://` / `socks4://` (Tor)**: the *proxy* resolves
|
||||
/// the target; reqwest only ever resolves the proxy's own host, which
|
||||
/// legitimately lives on a private Docker IP (e.g. `tor` → 172.x). Attaching
|
||||
/// the resolver there rejects every fetch for zero security gain (a SOCKS
|
||||
/// proxy can't route into internal ranges anyway) — do **not** attach.
|
||||
///
|
||||
/// This narrows commit 134ab54, which dropped the resolver for *any* proxy, back
|
||||
/// to SOCKS-only so the http(s)-proxy path keeps its DNS-rebinding guard.
|
||||
pub fn should_attach_safe_resolver(proxy: Option<&str>) -> bool {
|
||||
match proxy {
|
||||
None => true,
|
||||
Some(p) => !is_socks_proxy(p),
|
||||
}
|
||||
}
|
||||
|
||||
/// True when `proxy`'s scheme is a SOCKS variant (`socks4`, `socks5`,
|
||||
/// `socks5h`). A scheme-less value (reqwest treats it as HTTP) is not SOCKS.
|
||||
fn is_socks_proxy(proxy: &str) -> bool {
|
||||
proxy
|
||||
.split_once("://")
|
||||
.map(|(scheme, _)| scheme)
|
||||
.unwrap_or("")
|
||||
.to_ascii_lowercase()
|
||||
.starts_with("socks")
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error, PartialEq, Eq)]
|
||||
pub enum UrlSafetyError {
|
||||
#[error("URL is not parseable")]
|
||||
@@ -394,6 +507,26 @@ mod tests {
|
||||
DownloadAllowlist::new().allow(host)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_alternate_ipv4_encodings_of_loopback() {
|
||||
// The `url` crate normalizes special-scheme IPv4 hosts per WHATWG, so
|
||||
// decimal/hex/octal/short forms all collapse to 127.0.0.1 and must be
|
||||
// rejected. This normalization is load-bearing (it's what stops these
|
||||
// encodings smuggling past the literal-IP check) but was untested — pin
|
||||
// it so a future URL-parser swap can't silently reopen the hole.
|
||||
for url in [
|
||||
"http://2130706433/", // decimal 127.0.0.1
|
||||
"http://0x7f000001/", // hex
|
||||
"http://0177.0.0.1/", // octal first octet
|
||||
"http://127.1/", // short form
|
||||
] {
|
||||
assert!(
|
||||
ensure_public_target(url).is_err(),
|
||||
"{url} normalizes to loopback and must be rejected"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn allow_any_admits_arbitrary_public_host() {
|
||||
// Operators who can't pre-enumerate a numbered-CDN fleet
|
||||
@@ -609,6 +742,79 @@ mod tests {
|
||||
assert!(matches!(err, UrlSafetyError::PrivateIp(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn is_private_ip_unwraps_embedded_ipv4_encodings() {
|
||||
// Every IPv6 encoding that can smuggle an internal IPv4 must be
|
||||
// caught. The audit flagged compatible ::/96, NAT64, and 6to4 as
|
||||
// gaps past the original mapped-only handling.
|
||||
for s in [
|
||||
"::ffff:127.0.0.1", // IPv4-mapped loopback
|
||||
"::127.0.0.1", // IPv4-compatible loopback (was a gap)
|
||||
"::ffff:10.1.2.3", // mapped RFC1918
|
||||
"::10.1.2.3", // compatible RFC1918 (was a gap)
|
||||
"64:ff9b::7f00:1", // NAT64 of 127.0.0.1 (was a gap)
|
||||
"64:ff9b::a01:203", // NAT64 of 10.1.2.3
|
||||
"2002:7f00:1::", // 6to4 of 127.0.0.1 (was a gap)
|
||||
"2002:a01:203::", // 6to4 of 10.1.2.3
|
||||
"2002:a9fe:a9fe::", // 6to4 of 169.254.169.254 (metadata)
|
||||
] {
|
||||
let ip: IpAddr = s.parse().unwrap();
|
||||
assert!(is_private_ip(&ip), "{s} must be flagged private");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn is_private_ip_allows_public_embedded_and_native_ipv6() {
|
||||
// A public IPv4 embedded in IPv6, and a native public IPv6, must
|
||||
// NOT be flagged — the unwrap only blocks when the embedded v4 is
|
||||
// itself private.
|
||||
for s in [
|
||||
"::ffff:8.8.8.8", // mapped public
|
||||
"2002:808:808::", // 6to4 of 8.8.8.8 (public)
|
||||
"2606:4700:4700::1111", // native public (Cloudflare)
|
||||
] {
|
||||
let ip: IpAddr = s.parse().unwrap();
|
||||
assert!(!is_private_ip(&ip), "{s} must be allowed");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retain_public_addrs_drops_private_and_errors_when_all_private() {
|
||||
use std::net::{Ipv4Addr, SocketAddr};
|
||||
let pub_addr = SocketAddr::from((Ipv4Addr::new(93, 184, 216, 34), 0));
|
||||
let loopback = SocketAddr::from((Ipv4Addr::new(127, 0, 0, 1), 0));
|
||||
let metadata = SocketAddr::from((Ipv4Addr::new(169, 254, 169, 254), 0));
|
||||
|
||||
// Mixed result keeps only the public address.
|
||||
let kept =
|
||||
retain_public_addrs("mixed.example", [pub_addr, loopback, metadata].into_iter())
|
||||
.expect("public address survives");
|
||||
assert_eq!(kept, vec![pub_addr]);
|
||||
|
||||
// All-private (DNS rebinding to internal) is rejected outright.
|
||||
let err =
|
||||
retain_public_addrs("rebind.attacker", [loopback, metadata].into_iter()).unwrap_err();
|
||||
assert!(err.to_string().contains("rebind.attacker"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn safe_resolver_attaches_except_for_socks_proxies() {
|
||||
// Direct + http(s) proxies: reqwest resolves the target, so the
|
||||
// DNS-rebinding guard must be attached.
|
||||
assert!(should_attach_safe_resolver(None));
|
||||
assert!(should_attach_safe_resolver(Some("http://proxy.internal:8080")));
|
||||
assert!(should_attach_safe_resolver(Some("https://proxy.internal:8080")));
|
||||
assert!(should_attach_safe_resolver(Some("HTTP://Proxy:8080")));
|
||||
// A scheme-less proxy is treated as HTTP by reqwest — keep the guard.
|
||||
assert!(should_attach_safe_resolver(Some("proxy.internal:8080")));
|
||||
// SOCKS variants (incl. Tor): the proxy resolves, so attaching the
|
||||
// resolver would reject every fetch for zero gain.
|
||||
assert!(!should_attach_safe_resolver(Some("socks5://tor:9050")));
|
||||
assert!(!should_attach_safe_resolver(Some("socks5h://tor:9050")));
|
||||
assert!(!should_attach_safe_resolver(Some("socks4://x:1080")));
|
||||
assert!(!should_attach_safe_resolver(Some("SOCKS5://Tor:9050")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn safe_url_blocks_non_http_schemes() {
|
||||
let allow = allow_just("anywhere");
|
||||
|
||||
@@ -291,25 +291,33 @@ async fn fetch_probe_html(browser: &Browser, probe_url: &str) -> anyhow::Result<
|
||||
// the URL ever becomes attacker-influenced.
|
||||
crate::crawler::safety::ensure_public_target(probe_url)
|
||||
.with_context(|| format!("refuse to navigate unsafe probe URL {probe_url}"))?;
|
||||
let page = browser
|
||||
.new_page(probe_url)
|
||||
let page = crate::crawler::intercept::open_page(browser, probe_url)
|
||||
.await
|
||||
.with_context(|| format!("open probe page {probe_url}"))?;
|
||||
crate::crawler::nav::wait_for_nav(&page)
|
||||
.await
|
||||
.context("wait for nav on probe")?;
|
||||
// Best-effort wait for the layout marker. Timeout is fine — the
|
||||
// probe classifier handles a missing `#logo` as Transient anyway,
|
||||
// and the verify loop retries on Transient.
|
||||
let _ = crate::crawler::nav::wait_for_selector(
|
||||
&page,
|
||||
"#logo",
|
||||
crate::crawler::nav::SELECTOR_TIMEOUT,
|
||||
// Close the tab on every exit path — a `?` on wait_for_nav / content()
|
||||
// would otherwise leak it (chromiumoxide doesn't close on drop).
|
||||
let closer = page.clone();
|
||||
crate::crawler::nav::close_after(
|
||||
async move {
|
||||
closer.close().await.ok();
|
||||
},
|
||||
async {
|
||||
crate::crawler::nav::wait_for_nav(&page)
|
||||
.await
|
||||
.context("wait for nav on probe")?;
|
||||
// Best-effort wait for the layout marker. Timeout is fine — the
|
||||
// probe classifier handles a missing `#logo` as Transient anyway,
|
||||
// and the verify loop retries on Transient.
|
||||
let _ = crate::crawler::nav::wait_for_selector(
|
||||
&page,
|
||||
"#logo",
|
||||
crate::crawler::nav::SELECTOR_TIMEOUT,
|
||||
)
|
||||
.await;
|
||||
page.content().await.context("read probe html")
|
||||
},
|
||||
)
|
||||
.await;
|
||||
let html = page.content().await.context("read probe html")?;
|
||||
page.close().await.ok();
|
||||
Ok(html)
|
||||
.await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -20,7 +20,7 @@ use super::{
|
||||
use crate::crawler::detect::{
|
||||
has_logo_sentinel, is_broken_page_body, retry_on_transient_with_hook, PageError,
|
||||
};
|
||||
use crate::crawler::nav::{wait_for_nav, wait_for_selector, NavError, SELECTOR_TIMEOUT};
|
||||
use crate::crawler::nav::{close_after, wait_for_nav, wait_for_selector, NavError, SELECTOR_TIMEOUT};
|
||||
use crate::crawler::safety::ensure_public_target;
|
||||
|
||||
/// `sources.id` value for this Source impl. Exposed as a const so the
|
||||
@@ -253,31 +253,37 @@ async fn navigate(
|
||||
) -> Result<String, PageError> {
|
||||
guard_navigate_url(url)?;
|
||||
ctx.rate.wait_for(url).await?;
|
||||
let page = ctx
|
||||
.browser
|
||||
.new_page(url)
|
||||
let page = crate::crawler::intercept::open_page(ctx.browser, url)
|
||||
.await
|
||||
.map_err(|e| PageError::Other(anyhow::Error::from(e)))?;
|
||||
match wait_for_nav(&page).await {
|
||||
Ok(()) => {}
|
||||
Err(NavError::Timeout(_)) => {
|
||||
page.close().await.ok();
|
||||
return Err(PageError::transient("nav timeout"));
|
||||
}
|
||||
Err(NavError::Cdp(e)) => {
|
||||
page.close().await.ok();
|
||||
return Err(PageError::Other(anyhow::Error::from(e)));
|
||||
}
|
||||
}
|
||||
// Best-effort wait for the page-type marker. We deliberately
|
||||
// discard a timeout here — see fn-level doc.
|
||||
let _ = wait_for_selector(&page, marker, SELECTOR_TIMEOUT).await;
|
||||
let html = page
|
||||
.content()
|
||||
.await
|
||||
.map_err(|e| PageError::Other(anyhow::Error::from(e)))?;
|
||||
page.close().await.ok();
|
||||
classify_navigate_html(html)
|
||||
// Close the tab on every exit path — the previous code closed on the two
|
||||
// nav-error branches but leaked it on a content() read error.
|
||||
let closer = page.clone();
|
||||
close_after(
|
||||
async move {
|
||||
closer.close().await.ok();
|
||||
},
|
||||
async {
|
||||
match wait_for_nav(&page).await {
|
||||
Ok(()) => {}
|
||||
Err(NavError::Timeout(_)) => {
|
||||
return Err(PageError::transient("nav timeout"));
|
||||
}
|
||||
Err(NavError::Cdp(e)) => {
|
||||
return Err(PageError::Other(anyhow::Error::from(e)));
|
||||
}
|
||||
}
|
||||
// Best-effort wait for the page-type marker. We deliberately
|
||||
// discard a timeout here — see fn-level doc.
|
||||
let _ = wait_for_selector(&page, marker, SELECTOR_TIMEOUT).await;
|
||||
let html = page
|
||||
.content()
|
||||
.await
|
||||
.map_err(|e| PageError::Other(anyhow::Error::from(e)))?;
|
||||
classify_navigate_html(html)
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Classify a fetched body. The broken-page template is universal across
|
||||
|
||||
@@ -207,4 +207,68 @@ mod tests {
|
||||
"text_search_not_yet_supported"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn status_mapping_is_stable() {
|
||||
// Pin the code -> HTTP status mapping so a refactor of into_response
|
||||
// can't silently change a status (e.g. 501 -> 500). The 501 NotImplemented
|
||||
// path and a deterministic 429 Retry-After were previously unexercised
|
||||
// (audit GAP B).
|
||||
let cases: Vec<(AppError, StatusCode)> = vec![
|
||||
(AppError::NotFound, StatusCode::NOT_FOUND),
|
||||
(AppError::InvalidInput("x".into()), StatusCode::BAD_REQUEST),
|
||||
(AppError::Unauthenticated, StatusCode::UNAUTHORIZED),
|
||||
(AppError::Forbidden, StatusCode::FORBIDDEN),
|
||||
(AppError::Conflict("x".into()), StatusCode::CONFLICT),
|
||||
(AppError::PayloadTooLarge("x".into()), StatusCode::PAYLOAD_TOO_LARGE),
|
||||
(
|
||||
AppError::UnsupportedMediaType("x".into()),
|
||||
StatusCode::UNSUPPORTED_MEDIA_TYPE,
|
||||
),
|
||||
(
|
||||
AppError::ServiceUnavailable("x".into()),
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
),
|
||||
(
|
||||
AppError::ValidationFailed {
|
||||
message: "x".into(),
|
||||
details: json!({}),
|
||||
},
|
||||
StatusCode::UNPROCESSABLE_ENTITY,
|
||||
),
|
||||
(
|
||||
AppError::NotImplemented {
|
||||
code: "c",
|
||||
message: "m",
|
||||
},
|
||||
StatusCode::NOT_IMPLEMENTED,
|
||||
),
|
||||
(
|
||||
AppError::TooManyRequests {
|
||||
retry_after_secs: None,
|
||||
},
|
||||
StatusCode::TOO_MANY_REQUESTS,
|
||||
),
|
||||
(
|
||||
AppError::Other(anyhow::anyhow!("boom")),
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
),
|
||||
];
|
||||
for (err, want) in cases {
|
||||
let code = err.code();
|
||||
assert_eq!(err.into_response().status(), want, "status for code {code}");
|
||||
}
|
||||
|
||||
// A 429 with retry_after carries the Retry-After header deterministically
|
||||
// (without needing a live rate limiter).
|
||||
let resp = AppError::TooManyRequests {
|
||||
retry_after_secs: Some(7),
|
||||
}
|
||||
.into_response();
|
||||
assert_eq!(resp.status(), StatusCode::TOO_MANY_REQUESTS);
|
||||
assert_eq!(
|
||||
resp.headers().get(axum::http::header::RETRY_AFTER).unwrap(),
|
||||
"7"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -214,7 +214,7 @@ pub async fn list_mangas_with_sync_state(
|
||||
let search_pat = q
|
||||
.search
|
||||
.as_ref()
|
||||
.map(|s| format!("%{}%", s.trim()))
|
||||
.map(|s| format!("%{}%", crate::repo::escape_like(s.trim())))
|
||||
.filter(|p| p.len() > 2);
|
||||
// sqlx::Type → text: bind the snake_case representation manually so
|
||||
// the SQL can compare it as text without an explicit cast.
|
||||
@@ -235,7 +235,7 @@ pub async fn list_mangas_with_sync_state(
|
||||
(SELECT MAX(last_seen_at) FROM manga_sources ms
|
||||
WHERE ms.manga_id = m.id AND ms.dropped_at IS NULL) AS latest_seen_at
|
||||
FROM mangas m
|
||||
WHERE ($1::text IS NULL OR m.title ILIKE $1)
|
||||
WHERE ($1::text IS NULL OR m.title ILIKE $1 ESCAPE '\')
|
||||
)
|
||||
SELECT * FROM classified
|
||||
WHERE ($2::text IS NULL OR sync_state = $2)
|
||||
@@ -261,7 +261,7 @@ pub async fn list_mangas_with_sync_state(
|
||||
WITH classified AS (
|
||||
SELECT {case} AS sync_state
|
||||
FROM mangas m
|
||||
WHERE ($1::text IS NULL OR m.title ILIKE $1)
|
||||
WHERE ($1::text IS NULL OR m.title ILIKE $1 ESCAPE '\')
|
||||
)
|
||||
SELECT COUNT(*) FROM classified
|
||||
WHERE ($2::text IS NULL OR sync_state = $2)
|
||||
|
||||
@@ -32,6 +32,23 @@ pub async fn create(
|
||||
Ok(row)
|
||||
}
|
||||
|
||||
/// The caller's tokens, newest first. `token_hash` is `#[serde(skip)]` on the
|
||||
/// domain type, so returning the full row never leaks the secret.
|
||||
pub async fn list_for_user(pool: &PgPool, user_id: Uuid) -> AppResult<Vec<ApiToken>> {
|
||||
let rows = sqlx::query_as::<_, ApiToken>(
|
||||
r#"
|
||||
SELECT id, user_id, name, token_hash, created_at, last_used_at, expires_at
|
||||
FROM api_tokens
|
||||
WHERE user_id = $1
|
||||
ORDER BY created_at DESC
|
||||
"#,
|
||||
)
|
||||
.bind(user_id)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
Ok(rows)
|
||||
}
|
||||
|
||||
pub async fn find_active(pool: &PgPool, token_hash: &[u8]) -> AppResult<Option<ApiToken>> {
|
||||
let row = sqlx::query_as::<_, ApiToken>(
|
||||
r#"
|
||||
|
||||
@@ -81,7 +81,7 @@ pub async fn list(
|
||||
SELECT id, name, created_at
|
||||
FROM authors
|
||||
WHERE $1::text IS NULL
|
||||
OR name ILIKE '%' || $1 || '%'
|
||||
OR name ILIKE '%' || $4 || '%' ESCAPE '\'
|
||||
OR name % $1
|
||||
ORDER BY CASE WHEN $1::text IS NULL THEN 0 ELSE similarity(name, $1) END DESC,
|
||||
lower(name) ASC
|
||||
@@ -91,6 +91,9 @@ pub async fn list(
|
||||
.bind(search)
|
||||
.bind(limit)
|
||||
.bind(offset)
|
||||
// $4: LIKE-escaped term for the ILIKE branch so `%`/`_` match literally; the
|
||||
// trigram/similarity branches keep the raw $1.
|
||||
.bind(search.map(crate::repo::escape_like))
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
Ok(rows)
|
||||
|
||||
@@ -6,13 +6,23 @@ use uuid::Uuid;
|
||||
use crate::domain::{Bookmark, BookmarkSummary};
|
||||
use crate::error::{AppError, AppResult};
|
||||
|
||||
/// Add a bookmark, idempotently. Returns the bookmark plus whether it was
|
||||
/// newly created (`true`) or already existed (`false`), so the handler can
|
||||
/// answer 201 vs 200. Re-adding an existing bookmark is a no-op success rather
|
||||
/// than a 409 — matching the idempotent collection semantics, so the UI doesn't
|
||||
/// show a false "Could not add bookmark" toast when the manga is already saved.
|
||||
///
|
||||
/// Uniqueness is per `(user_id, manga_id, chapter_id)` — enforced by the 0001
|
||||
/// constraint for chapter-level rows and the 0004 partial index for manga-level
|
||||
/// (NULL chapter) rows. `page` is not part of the key, so the existing row is
|
||||
/// returned unchanged (its page is not overwritten).
|
||||
pub async fn create(
|
||||
pool: &PgPool,
|
||||
user_id: Uuid,
|
||||
manga_id: Uuid,
|
||||
chapter_id: Option<Uuid>,
|
||||
page: Option<i32>,
|
||||
) -> AppResult<Bookmark> {
|
||||
) -> AppResult<(Bookmark, bool)> {
|
||||
let result = sqlx::query_as::<_, Bookmark>(
|
||||
r#"
|
||||
INSERT INTO bookmarks (user_id, manga_id, chapter_id, page)
|
||||
@@ -28,10 +38,27 @@ pub async fn create(
|
||||
.await;
|
||||
|
||||
match result {
|
||||
Ok(b) => Ok(b),
|
||||
Err(sqlx::Error::Database(ref db_err)) if db_err.is_unique_violation() => Err(
|
||||
AppError::Conflict("bookmark already exists for this manga/chapter".into()),
|
||||
),
|
||||
Ok(b) => Ok((b, true)),
|
||||
Err(sqlx::Error::Database(ref db_err)) if db_err.is_unique_violation() => {
|
||||
// A bookmark for this (user, manga, chapter) already exists — fetch
|
||||
// and return it. `IS NOT DISTINCT FROM` matches a NULL chapter_id
|
||||
// (manga-level bookmark) as well as a concrete one, covering both
|
||||
// uniqueness paths with a single lookup.
|
||||
let existing = sqlx::query_as::<_, Bookmark>(
|
||||
r#"
|
||||
SELECT id, user_id, manga_id, chapter_id, page, created_at
|
||||
FROM bookmarks
|
||||
WHERE user_id = $1 AND manga_id = $2
|
||||
AND chapter_id IS NOT DISTINCT FROM $3
|
||||
"#,
|
||||
)
|
||||
.bind(user_id)
|
||||
.bind(manga_id)
|
||||
.bind(chapter_id)
|
||||
.fetch_one(pool)
|
||||
.await?;
|
||||
Ok((existing, false))
|
||||
}
|
||||
Err(e) => Err(AppError::Database(e)),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,14 +12,19 @@ pub async fn list_for_manga(
|
||||
limit: i64,
|
||||
offset: i64,
|
||||
) -> AppResult<Vec<Chapter>> {
|
||||
// Display order = source-site order reversed. The crawler stamps
|
||||
// `source_index` = position in the source DOM (0 = first = newest
|
||||
// on this site, see migration 0021), so DESC puts the oldest
|
||||
// chapter first and keeps the site's variant grouping and the
|
||||
// placement of non-numeric entries (e.g. "notice. : Officials")
|
||||
// intact. NULLS LAST keeps user-uploaded chapters (no source row)
|
||||
// and rows that pre-date the migration below crawled rows; the
|
||||
// (number, created_at) tail then orders them deterministically.
|
||||
// Display order. Crawled chapters carry `source_index` = position in the
|
||||
// source DOM (0 = newest on this site, migration 0021); we display them
|
||||
// reversed (oldest first) via the `-source_index` key, which keeps the
|
||||
// site's variant grouping and non-numeric entries (e.g. "notice.") in the
|
||||
// spot the site placed them — NOT clustered at number 0.
|
||||
//
|
||||
// User-uploaded chapters have no `source_index`. Instead of dumping them
|
||||
// after every crawled chapter (the old `NULLS LAST` bug, which put an
|
||||
// uploaded chapter 5 after crawled chapter 100), each is slotted by NUMBER:
|
||||
// just before the crawled chapter with the smallest number greater than it,
|
||||
// so it interleaves. An upload newer than every crawled chapter goes last;
|
||||
// when there are no crawled chapters at all, uploads fall back to the
|
||||
// `number, created_at` tail.
|
||||
let rows = sqlx::query_as::<_, Chapter>(
|
||||
r#"
|
||||
SELECT id, manga_id, number, title, page_count, created_at,
|
||||
@@ -31,7 +36,22 @@ pub async fn list_for_manga(
|
||||
FROM pages p WHERE p.chapter_id = chapters.id)::bigint AS size_bytes
|
||||
FROM chapters
|
||||
WHERE manga_id = $1
|
||||
ORDER BY source_index DESC NULLS LAST, number ASC, created_at ASC
|
||||
ORDER BY
|
||||
CASE
|
||||
WHEN source_index IS NOT NULL THEN (-source_index)::float8
|
||||
ELSE COALESCE(
|
||||
(SELECT MIN(-c2.source_index)::float8 - 0.5
|
||||
FROM chapters c2
|
||||
WHERE c2.manga_id = chapters.manga_id
|
||||
AND c2.source_index IS NOT NULL
|
||||
AND c2.number > chapters.number),
|
||||
(SELECT COALESCE(MAX(-c3.source_index), 0)::float8 + 0.5
|
||||
FROM chapters c3
|
||||
WHERE c3.manga_id = chapters.manga_id
|
||||
AND c3.source_index IS NOT NULL)
|
||||
)
|
||||
END,
|
||||
number ASC, created_at ASC
|
||||
LIMIT $2 OFFSET $3
|
||||
"#,
|
||||
)
|
||||
|
||||
@@ -247,10 +247,13 @@ async fn sync_genres(
|
||||
let genre_id = match existing {
|
||||
Some((id,)) => id,
|
||||
None => {
|
||||
// Conflict on lower(name) (0038) not name, so a racing insert of
|
||||
// a differently-cased variant resolves to the existing row
|
||||
// instead of raising a unique violation.
|
||||
let (id,): (Uuid,) = sqlx::query_as(
|
||||
r#"
|
||||
INSERT INTO genres (name) VALUES ($1)
|
||||
ON CONFLICT (name) DO UPDATE SET name = genres.name
|
||||
ON CONFLICT (lower(name)) DO UPDATE SET name = genres.name
|
||||
RETURNING id
|
||||
"#,
|
||||
)
|
||||
@@ -720,7 +723,7 @@ pub async fn list_dead_jobs(
|
||||
offset: i64,
|
||||
) -> sqlx::Result<(Vec<DeadJob>, i64)> {
|
||||
let search_pat = search
|
||||
.map(|s| format!("%{}%", s.trim()))
|
||||
.map(|s| format!("%{}%", crate::repo::escape_like(s.trim())))
|
||||
.filter(|p| p.len() > 2);
|
||||
|
||||
let items: Vec<DeadJob> = sqlx::query_as(
|
||||
@@ -743,7 +746,7 @@ pub async fn list_dead_jobs(
|
||||
LEFT JOIN chapters c ON c.id = (cj.payload->>'chapter_id')::uuid
|
||||
LEFT JOIN mangas m ON m.id = c.manga_id
|
||||
WHERE cj.state = 'dead'
|
||||
AND ($1::text IS NULL OR m.title ILIKE $1 OR cj.payload->>'title' ILIKE $1)
|
||||
AND ($1::text IS NULL OR m.title ILIKE $1 ESCAPE '\' OR cj.payload->>'title' ILIKE $1 ESCAPE '\')
|
||||
ORDER BY cj.updated_at DESC
|
||||
LIMIT $2 OFFSET $3
|
||||
"#,
|
||||
@@ -761,7 +764,7 @@ pub async fn list_dead_jobs(
|
||||
LEFT JOIN chapters c ON c.id = (cj.payload->>'chapter_id')::uuid
|
||||
LEFT JOIN mangas m ON m.id = c.manga_id
|
||||
WHERE cj.state = 'dead'
|
||||
AND ($1::text IS NULL OR m.title ILIKE $1 OR cj.payload->>'title' ILIKE $1)
|
||||
AND ($1::text IS NULL OR m.title ILIKE $1 ESCAPE '\' OR cj.payload->>'title' ILIKE $1 ESCAPE '\')
|
||||
"#,
|
||||
)
|
||||
.bind(&search_pat)
|
||||
@@ -797,7 +800,7 @@ pub async fn list_active_jobs(
|
||||
offset: i64,
|
||||
) -> sqlx::Result<(Vec<ActiveJob>, i64)> {
|
||||
let search_pat = search
|
||||
.map(|s| format!("%{}%", s.trim()))
|
||||
.map(|s| format!("%{}%", crate::repo::escape_like(s.trim())))
|
||||
.filter(|p| p.len() > 2);
|
||||
|
||||
let items: Vec<ActiveJob> = sqlx::query_as(
|
||||
@@ -817,7 +820,7 @@ pub async fn list_active_jobs(
|
||||
LEFT JOIN mangas m ON m.id = c.manga_id
|
||||
WHERE cj.state IN ('pending','running')
|
||||
AND cj.payload->>'kind' = 'sync_chapter_content'
|
||||
AND ($1::text IS NULL OR m.title ILIKE $1)
|
||||
AND ($1::text IS NULL OR m.title ILIKE $1 ESCAPE '\')
|
||||
ORDER BY (cj.state = 'running') DESC, cj.scheduled_at, cj.created_at
|
||||
LIMIT $2 OFFSET $3
|
||||
"#,
|
||||
@@ -836,7 +839,7 @@ pub async fn list_active_jobs(
|
||||
LEFT JOIN mangas m ON m.id = c.manga_id
|
||||
WHERE cj.state IN ('pending','running')
|
||||
AND cj.payload->>'kind' = 'sync_chapter_content'
|
||||
AND ($1::text IS NULL OR m.title ILIKE $1)
|
||||
AND ($1::text IS NULL OR m.title ILIKE $1 ESCAPE '\')
|
||||
"#,
|
||||
)
|
||||
.bind(&search_pat)
|
||||
@@ -913,7 +916,7 @@ pub async fn list_job_history(
|
||||
) -> sqlx::Result<(Vec<JobHistoryRow>, i64)> {
|
||||
let search_pat = filter
|
||||
.search
|
||||
.map(|s| format!("%{}%", s.trim()))
|
||||
.map(|s| format!("%{}%", crate::repo::escape_like(s.trim())))
|
||||
.filter(|p| p.len() > 2);
|
||||
|
||||
// The same FROM/JOIN/WHERE drives both the page slice and the count, so
|
||||
@@ -951,7 +954,7 @@ pub async fn list_job_history(
|
||||
) cm ON true
|
||||
WHERE ($1::text IS NULL OR cj.state = $1)
|
||||
AND ($2::text IS NULL OR cj.payload->>'kind' = $2)
|
||||
AND ($3::text IS NULL OR m.title ILIKE $3 OR cj.payload->>'title' ILIKE $3)
|
||||
AND ($3::text IS NULL OR m.title ILIKE $3 ESCAPE '\' OR cj.payload->>'title' ILIKE $3 ESCAPE '\')
|
||||
ORDER BY cj.updated_at DESC
|
||||
LIMIT $4 OFFSET $5
|
||||
"#,
|
||||
@@ -973,7 +976,7 @@ pub async fn list_job_history(
|
||||
LEFT JOIN mangas m ON m.id = COALESCE(c.manga_id, (cj.payload->>'manga_id')::uuid)
|
||||
WHERE ($1::text IS NULL OR cj.state = $1)
|
||||
AND ($2::text IS NULL OR cj.payload->>'kind' = $2)
|
||||
AND ($3::text IS NULL OR m.title ILIKE $3 OR cj.payload->>'title' ILIKE $3)
|
||||
AND ($3::text IS NULL OR m.title ILIKE $3 ESCAPE '\' OR cj.payload->>'title' ILIKE $3 ESCAPE '\')
|
||||
"#,
|
||||
)
|
||||
.bind(filter.state)
|
||||
@@ -1018,7 +1021,7 @@ pub async fn list_missing_cover_mangas(
|
||||
offset: i64,
|
||||
) -> sqlx::Result<(Vec<MissingCoverRow>, i64)> {
|
||||
let search_pat = search
|
||||
.map(|s| format!("%{}%", s.trim()))
|
||||
.map(|s| format!("%{}%", crate::repo::escape_like(s.trim())))
|
||||
.filter(|p| p.len() > 2);
|
||||
|
||||
let items: Vec<MissingCoverRow> = sqlx::query_as(
|
||||
@@ -1030,7 +1033,7 @@ pub async fn list_missing_cover_mangas(
|
||||
SELECT 1 FROM manga_sources ms
|
||||
WHERE ms.manga_id = m.id AND ms.dropped_at IS NULL
|
||||
)
|
||||
AND ($1::text IS NULL OR m.title ILIKE $1)
|
||||
AND ($1::text IS NULL OR m.title ILIKE $1 ESCAPE '\')
|
||||
ORDER BY m.updated_at DESC
|
||||
LIMIT $2 OFFSET $3
|
||||
"#,
|
||||
@@ -1049,7 +1052,7 @@ pub async fn list_missing_cover_mangas(
|
||||
SELECT 1 FROM manga_sources ms
|
||||
WHERE ms.manga_id = m.id AND ms.dropped_at IS NULL
|
||||
)
|
||||
AND ($1::text IS NULL OR m.title ILIKE $1)
|
||||
AND ($1::text IS NULL OR m.title ILIKE $1 ESCAPE '\')
|
||||
"#,
|
||||
)
|
||||
.bind(&search_pat)
|
||||
|
||||
@@ -10,7 +10,7 @@ use uuid::Uuid;
|
||||
|
||||
use crate::domain::manga::{Manga, MangaCard, MangaDetail};
|
||||
use crate::error::{AppError, AppResult};
|
||||
use crate::repo;
|
||||
use crate::repo::{self, escape_like};
|
||||
|
||||
/// Status values mirror the CHECK constraint in 0009. Centralized so
|
||||
/// the API layer can validate uploads against the same vocabulary.
|
||||
@@ -110,13 +110,13 @@ fn manga_cols(alias: &str) -> String {
|
||||
/// true.
|
||||
const FILTER_WHERE: &str = r#"
|
||||
($1::text IS NULL
|
||||
OR title ILIKE '%' || $1 || '%'
|
||||
OR title ILIKE '%' || $8 || '%' ESCAPE '\'
|
||||
OR title % $1
|
||||
OR EXISTS (
|
||||
SELECT 1 FROM manga_authors ma
|
||||
JOIN authors a ON a.id = ma.author_id
|
||||
WHERE ma.manga_id = mangas.id
|
||||
AND (a.name ILIKE '%' || $1 || '%' OR a.name % $1)
|
||||
AND (a.name ILIKE '%' || $8 || '%' ESCAPE '\' OR a.name % $1)
|
||||
)
|
||||
)
|
||||
AND ($2::text IS NULL OR status = $2)
|
||||
@@ -144,17 +144,13 @@ const FILTER_WHERE: &str = r#"
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM unnest($6::text[]) AS req(w)
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM page_content_warnings pw
|
||||
JOIN pages p ON p.id = pw.page_id
|
||||
JOIN chapters c ON c.id = p.chapter_id
|
||||
WHERE c.manga_id = mangas.id AND pw.warning = req.w
|
||||
SELECT 1 FROM manga_content_warnings mcw
|
||||
WHERE mcw.manga_id = mangas.id AND mcw.warning = req.w
|
||||
)
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM page_content_warnings pw
|
||||
JOIN pages p ON p.id = pw.page_id
|
||||
JOIN chapters c ON c.id = p.chapter_id
|
||||
WHERE c.manga_id = mangas.id AND pw.warning = ANY($7::text[])
|
||||
SELECT 1 FROM manga_content_warnings mcw
|
||||
WHERE mcw.manga_id = mangas.id AND mcw.warning = ANY($7::text[])
|
||||
)
|
||||
"#;
|
||||
|
||||
@@ -163,7 +159,7 @@ const FILTER_WHERE: &str = r#"
|
||||
/// (`mangas_created_at_idx`, `mangas_updated_at_idx`, `mangas_title_lower_idx`)
|
||||
/// and the trigram GIN indexes (0005_search.sql, 0009_manga_metadata.sql) keep
|
||||
/// the search filter cheap as the library grows.
|
||||
pub async fn list(pool: &PgPool, query: &ListQuery) -> AppResult<(Vec<Manga>, i64)> {
|
||||
pub async fn list(pool: &PgPool, query: &ListQuery) -> AppResult<(Vec<Manga>, Option<i64>)> {
|
||||
// Both `col` and `dir` are interpolated from hard-coded enums, never from
|
||||
// request input, so this is not a SQL injection seam. The trailing `id` is
|
||||
// a stable tie-break that keeps pagination deterministic across rows with
|
||||
@@ -172,16 +168,11 @@ pub async fn list(pool: &PgPool, query: &ListQuery) -> AppResult<(Vec<Manga>, i6
|
||||
SortField::Created => "created_at",
|
||||
SortField::Updated => "updated_at",
|
||||
SortField::Title => "lower(title)",
|
||||
// Sorts on the alphabetically-first attached author. As an ORDER BY
|
||||
// key this correlated subquery is evaluated per filter-matching row
|
||||
// before LIMIT applies, so it scales worse than the indexed date/title
|
||||
// sorts — revisit with a LATERAL join or precomputed sort-name column
|
||||
// if the library grows large.
|
||||
SortField::Author => {
|
||||
"(SELECT min(lower(a.name)) \
|
||||
FROM manga_authors ma JOIN authors a ON a.id = ma.author_id \
|
||||
WHERE ma.manga_id = mangas.id)"
|
||||
}
|
||||
// Sorts on the alphabetically-first attached author. Precomputed into
|
||||
// `mangas.sort_author` (migration 0037, maintained by triggers on
|
||||
// manga_authors) and index-backed by `mangas_sort_author_idx`, so this
|
||||
// is a plain column read rather than a per-row correlated subquery.
|
||||
SortField::Author => "sort_author",
|
||||
};
|
||||
let dir = match query.order {
|
||||
SortOrder::Asc => "ASC",
|
||||
@@ -205,11 +196,15 @@ pub async fn list(pool: &PgPool, query: &ListQuery) -> AppResult<(Vec<Manga>, i6
|
||||
FROM mangas
|
||||
WHERE {FILTER_WHERE}
|
||||
ORDER BY {order_by}
|
||||
LIMIT $8 OFFSET $9
|
||||
LIMIT $9 OFFSET $10
|
||||
"#,
|
||||
cols = manga_cols(""),
|
||||
);
|
||||
|
||||
// $8 is the LIKE-escaped search term used by the ILIKE branches so `%`/`_`
|
||||
// in the term match literally; the trigram `%` branches keep the raw $1.
|
||||
let search_escaped = search.map(escape_like);
|
||||
|
||||
let rows = sqlx::query_as::<_, Manga>(&list_sql)
|
||||
.bind(search)
|
||||
.bind(status)
|
||||
@@ -218,27 +213,40 @@ pub async fn list(pool: &PgPool, query: &ListQuery) -> AppResult<(Vec<Manga>, i6
|
||||
.bind(&query.tag_ids)
|
||||
.bind(&query.cw_include)
|
||||
.bind(&query.cw_exclude)
|
||||
.bind(&search_escaped)
|
||||
.bind(query.limit)
|
||||
.bind(query.offset)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
|
||||
let count_sql = format!(
|
||||
r#"
|
||||
SELECT count(*) FROM mangas
|
||||
WHERE {FILTER_WHERE}
|
||||
"#
|
||||
);
|
||||
let (total,): (i64,) = sqlx::query_as(&count_sql)
|
||||
.bind(search)
|
||||
.bind(status)
|
||||
.bind(&query.author_ids)
|
||||
.bind(&query.genre_ids)
|
||||
.bind(&query.tag_ids)
|
||||
.bind(&query.cw_include)
|
||||
.bind(&query.cw_exclude)
|
||||
.fetch_one(pool)
|
||||
.await?;
|
||||
// The count reuses FILTER_WHERE — up to five correlated NOT EXISTS/unnest
|
||||
// subqueries (plus a page_content_warnings join under CW filters) over the
|
||||
// whole filtered set. Recomputing it on every page made pagination scale
|
||||
// with catalog size. It doesn't change as the caller walks pages, so
|
||||
// compute it once on the first page (offset 0) and return None thereafter;
|
||||
// the pagination envelope serialises that as `total: null`.
|
||||
let total = if query.offset == 0 {
|
||||
let count_sql = format!(
|
||||
r#"
|
||||
SELECT count(*) FROM mangas
|
||||
WHERE {FILTER_WHERE}
|
||||
"#
|
||||
);
|
||||
let (total,): (i64,) = sqlx::query_as(&count_sql)
|
||||
.bind(search)
|
||||
.bind(status)
|
||||
.bind(&query.author_ids)
|
||||
.bind(&query.genre_ids)
|
||||
.bind(&query.tag_ids)
|
||||
.bind(&query.cw_include)
|
||||
.bind(&query.cw_exclude)
|
||||
.bind(&search_escaped)
|
||||
.fetch_one(pool)
|
||||
.await?;
|
||||
Some(total)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
Ok((rows, total))
|
||||
}
|
||||
@@ -249,7 +257,7 @@ pub async fn list(pool: &PgPool, query: &ListQuery) -> AppResult<(Vec<Manga>, i6
|
||||
pub async fn list_cards(
|
||||
pool: &PgPool,
|
||||
query: &ListQuery,
|
||||
) -> AppResult<(Vec<MangaCard>, i64)> {
|
||||
) -> AppResult<(Vec<MangaCard>, Option<i64>)> {
|
||||
let (rows, total) = list(pool, query).await?;
|
||||
let cards = cards_from_rows(pool, rows).await?;
|
||||
Ok((cards, total))
|
||||
|
||||
@@ -21,3 +21,45 @@ pub mod tag;
|
||||
pub mod upload_history;
|
||||
pub mod user;
|
||||
pub mod user_preferences;
|
||||
|
||||
/// Escape the LIKE/ILIKE metacharacters (`%`, `_`, and the escape char `\`
|
||||
/// itself) in a user-supplied search term so they match literally rather than
|
||||
/// as wildcards. Pair the resulting value with `ESCAPE '\'` in the SQL — a
|
||||
/// single backslash, which under `standard_conforming_strings` (Postgres
|
||||
/// default) is one backslash in a single-quoted literal.
|
||||
///
|
||||
/// This is a search-correctness fix, not an injection fix: every term is
|
||||
/// already a bound parameter, so `%`/`_` can never break out of the string —
|
||||
/// they were just silently acting as wildcards (`50%` matching everything,
|
||||
/// `a_b` matching `axb`). Callers that build a substring pattern wrap the
|
||||
/// escaped term themselves, e.g. `format!("%{}%", escape_like(term))`.
|
||||
pub(crate) fn escape_like(s: &str) -> String {
|
||||
let mut out = String::with_capacity(s.len());
|
||||
for ch in s.chars() {
|
||||
if matches!(ch, '\\' | '%' | '_') {
|
||||
out.push('\\');
|
||||
}
|
||||
out.push(ch);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::escape_like;
|
||||
|
||||
#[test]
|
||||
fn escapes_wildcards_and_the_escape_char() {
|
||||
assert_eq!(escape_like("50%"), r"50\%");
|
||||
assert_eq!(escape_like("a_b"), r"a\_b");
|
||||
assert_eq!(escape_like(r"back\slash"), r"back\\slash");
|
||||
// A already-escaped-looking input is double-escaped so it stays literal.
|
||||
assert_eq!(escape_like(r"\%"), r"\\\%");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn leaves_ordinary_text_untouched() {
|
||||
assert_eq!(escape_like("naruto"), "naruto");
|
||||
assert_eq!(escape_like(""), "");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -241,6 +241,10 @@ pub async fn manga_coverage(
|
||||
limit: i64,
|
||||
offset: i64,
|
||||
) -> AppResult<(Vec<MangaCoverage>, i64)> {
|
||||
// LIKE-escape so `%`/`_` in the title filter match literally. No trigram
|
||||
// branch here, so binding the escaped term directly (rather than appending a
|
||||
// second param) is safe — $1 feeds only the ILIKE.
|
||||
let search = search.map(crate::repo::escape_like);
|
||||
let rows = sqlx::query_as::<_, MangaCoverage>(
|
||||
r#"
|
||||
SELECT m.id AS manga_id, m.title,
|
||||
@@ -250,13 +254,13 @@ pub async fn manga_coverage(
|
||||
JOIN chapters c ON c.manga_id = m.id
|
||||
JOIN pages p ON p.chapter_id = c.id
|
||||
LEFT JOIN page_analysis pa ON pa.page_id = p.id
|
||||
WHERE ($1::text IS NULL OR m.title ILIKE '%' || $1 || '%')
|
||||
WHERE ($1::text IS NULL OR m.title ILIKE '%' || $1 || '%' ESCAPE '\')
|
||||
GROUP BY m.id, m.title
|
||||
ORDER BY lower(m.title), m.id
|
||||
LIMIT $2 OFFSET $3
|
||||
"#,
|
||||
)
|
||||
.bind(search)
|
||||
.bind(&search)
|
||||
.bind(limit)
|
||||
.bind(offset)
|
||||
.fetch_all(pool)
|
||||
@@ -269,12 +273,12 @@ pub async fn manga_coverage(
|
||||
FROM mangas m
|
||||
JOIN chapters c ON c.manga_id = m.id
|
||||
JOIN pages p ON p.chapter_id = c.id
|
||||
WHERE ($1::text IS NULL OR m.title ILIKE '%' || $1 || '%')
|
||||
WHERE ($1::text IS NULL OR m.title ILIKE '%' || $1 || '%' ESCAPE '\')
|
||||
GROUP BY m.id
|
||||
) x
|
||||
"#,
|
||||
)
|
||||
.bind(search)
|
||||
.bind(&search)
|
||||
.fetch_one(pool)
|
||||
.await?;
|
||||
Ok((rows, total))
|
||||
@@ -377,7 +381,7 @@ pub async fn list_history(
|
||||
) -> AppResult<(Vec<AnalysisHistoryRow>, i64)> {
|
||||
let search_pat = filter
|
||||
.search
|
||||
.map(|s| format!("%{}%", s.trim()))
|
||||
.map(|s| format!("%{}%", crate::repo::escape_like(s.trim())))
|
||||
.filter(|p| p.len() > 2);
|
||||
|
||||
let items = sqlx::query_as::<_, AnalysisHistoryRow>(
|
||||
@@ -402,7 +406,7 @@ pub async fn list_history(
|
||||
WHERE pa.status <> 'pending'
|
||||
AND ($1::text IS NULL OR pa.status = $1)
|
||||
AND ($2::bool IS FALSE OR pa.is_nsfw)
|
||||
AND ($3::text IS NULL OR m.title ILIKE $3)
|
||||
AND ($3::text IS NULL OR m.title ILIKE $3 ESCAPE '\')
|
||||
ORDER BY pa.analyzed_at DESC NULLS LAST, pa.page_id
|
||||
LIMIT $4 OFFSET $5
|
||||
"#,
|
||||
@@ -425,7 +429,7 @@ pub async fn list_history(
|
||||
WHERE pa.status <> 'pending'
|
||||
AND ($1::text IS NULL OR pa.status = $1)
|
||||
AND ($2::bool IS FALSE OR pa.is_nsfw)
|
||||
AND ($3::text IS NULL OR m.title ILIKE $3)
|
||||
AND ($3::text IS NULL OR m.title ILIKE $3 ESCAPE '\')
|
||||
"#,
|
||||
)
|
||||
.bind(filter.status)
|
||||
|
||||
@@ -120,27 +120,11 @@ pub async fn list_for_page(
|
||||
Ok(rows.into_iter().map(|(t,)| t).collect())
|
||||
}
|
||||
|
||||
/// Escape a string for use as a LIKE pattern fragment: `%`, `_`, and
|
||||
/// `\` get a leading backslash so they're matched literally rather
|
||||
/// than as wildcards / escapes. The matching queries below pair this
|
||||
/// with `ESCAPE '\'` for explicitness — a single backslash, since the
|
||||
/// SQL lives in a raw string and Postgres treats `\\` in a single-
|
||||
/// quoted literal as one backslash under `standard_conforming_strings`.
|
||||
///
|
||||
/// The public API rejects `%`/`_`/`\` in `normalize_tag` before
|
||||
/// they reach this repo, so this is defence-in-depth — a future
|
||||
/// internal caller (worker, CLI) that bypasses the normalizer can't
|
||||
/// turn a prefix filter into a wildcard search by accident.
|
||||
fn escape_like_prefix(s: &str) -> String {
|
||||
let mut out = String::with_capacity(s.len());
|
||||
for ch in s.chars() {
|
||||
if matches!(ch, '\\' | '%' | '_') {
|
||||
out.push('\\');
|
||||
}
|
||||
out.push(ch);
|
||||
}
|
||||
out
|
||||
}
|
||||
// LIKE-escaping the autocomplete prefix is defence-in-depth: the public API
|
||||
// already rejects `%`/`_`/`\` in `normalize_tag` before they reach this repo, so
|
||||
// a stray wildcard can only arrive from a future internal caller (worker, CLI)
|
||||
// that bypasses the normalizer. Shared with the other search sites.
|
||||
use crate::repo::escape_like as escape_like_prefix;
|
||||
|
||||
/// Paged list of `user_id`'s tagged pages, with breadcrumb. When
|
||||
/// `tag_filter` is `Some(_)`, restrict to that exact tag (used by the
|
||||
|
||||
@@ -65,3 +65,15 @@ pub async fn delete_by_token_hash(pool: &PgPool, token_hash: &[u8]) -> AppResult
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Delete every session whose `expires_at` has passed, returning the number
|
||||
/// reaped. `find_active` already refuses expired sessions, so this only
|
||||
/// reclaims storage — running it on any schedule (or not at all) is safe.
|
||||
/// Backed by `sessions_expires_idx` (0002). Called by the periodic reaper in
|
||||
/// `app::build`.
|
||||
pub async fn delete_expired(pool: &PgPool) -> AppResult<u64> {
|
||||
let result = sqlx::query("DELETE FROM sessions WHERE expires_at <= now()")
|
||||
.execute(pool)
|
||||
.await?;
|
||||
Ok(result.rows_affected())
|
||||
}
|
||||
|
||||
@@ -124,7 +124,7 @@ pub async fn list(
|
||||
SELECT id, name, created_at
|
||||
FROM tags
|
||||
WHERE $1::text IS NULL
|
||||
OR name ILIKE '%' || $1 || '%'
|
||||
OR name ILIKE '%' || $3 || '%' ESCAPE '\'
|
||||
OR name % $1
|
||||
ORDER BY CASE WHEN $1::text IS NULL THEN 0 ELSE similarity(name, $1) END DESC,
|
||||
lower(name) ASC
|
||||
@@ -133,6 +133,9 @@ pub async fn list(
|
||||
)
|
||||
.bind(search)
|
||||
.bind(limit)
|
||||
// $3: LIKE-escaped term for the ILIKE branch so `%`/`_` match literally; the
|
||||
// trigram/similarity branches keep the raw $1.
|
||||
.bind(search.map(crate::repo::escape_like))
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
Ok(rows)
|
||||
|
||||
@@ -106,14 +106,14 @@ pub async fn list_with_total(
|
||||
let pat = q
|
||||
.search
|
||||
.as_ref()
|
||||
.map(|s| format!("%{}%", s.trim()))
|
||||
.map(|s| format!("%{}%", crate::repo::escape_like(s.trim())))
|
||||
.filter(|p| p.len() > 2);
|
||||
|
||||
let items = sqlx::query_as::<_, User>(
|
||||
r#"
|
||||
SELECT id, username, password_hash, created_at, is_admin
|
||||
FROM users
|
||||
WHERE ($1::text IS NULL OR username ILIKE $1)
|
||||
WHERE ($1::text IS NULL OR username ILIKE $1 ESCAPE '\')
|
||||
ORDER BY username
|
||||
LIMIT $2 OFFSET $3
|
||||
"#,
|
||||
@@ -125,7 +125,7 @@ pub async fn list_with_total(
|
||||
.await?;
|
||||
|
||||
let total: i64 = sqlx::query_scalar(
|
||||
"SELECT COUNT(*) FROM users WHERE ($1::text IS NULL OR username ILIKE $1)",
|
||||
"SELECT COUNT(*) FROM users WHERE ($1::text IS NULL OR username ILIKE $1 ESCAPE '\\')",
|
||||
)
|
||||
.bind(&pat)
|
||||
.fetch_one(pool)
|
||||
@@ -381,7 +381,7 @@ pub async fn bootstrap_admin(
|
||||
.await?;
|
||||
}
|
||||
None => {
|
||||
let hash = crate::auth::password::hash_password(password)?;
|
||||
let hash = crate::auth::password::hash_password_async(password.to_string()).await?;
|
||||
sqlx::query("INSERT INTO users (username, password_hash, is_admin) VALUES ($1, $2, true)")
|
||||
.bind(username)
|
||||
.bind(&hash)
|
||||
|
||||
@@ -269,6 +269,7 @@ impl CrawlerSettings {
|
||||
tor_control_cookie_path: base.tor_control_cookie_path.clone(),
|
||||
tor_recircuit_max_attempts: base.tor_recircuit_max_attempts,
|
||||
browser: base.browser.clone(),
|
||||
ssrf_intercept: base.ssrf_intercept,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -95,6 +95,11 @@ impl Storage for LocalStorage {
|
||||
match fs::read(&path).await {
|
||||
Ok(b) => Ok(b),
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Err(StorageError::NotFound),
|
||||
// A key resolving to a directory isn't a stored blob; `fs::read`
|
||||
// fails with EISDIR. Treat it as absent, mirroring `size`.
|
||||
Err(e) if e.kind() == std::io::ErrorKind::IsADirectory => {
|
||||
Err(StorageError::NotFound)
|
||||
}
|
||||
Err(e) => Err(e.into()),
|
||||
}
|
||||
}
|
||||
@@ -108,7 +113,14 @@ impl Storage for LocalStorage {
|
||||
}
|
||||
Err(e) => return Err(e.into()),
|
||||
};
|
||||
let size_bytes = file.metadata().await?.len();
|
||||
let meta = file.metadata().await?;
|
||||
// Opening a directory succeeds on Unix, but it isn't a stored blob:
|
||||
// its inode "size" is meaningless and ReaderStream would fail mid-read.
|
||||
// Treat it as absent, mirroring `size` and `get`.
|
||||
if !meta.is_file() {
|
||||
return Err(StorageError::NotFound);
|
||||
}
|
||||
let size_bytes = meta.len();
|
||||
// 64 KiB chunks: small enough that a few-MB page emits many frames
|
||||
// (so streaming is observable), large enough to keep syscalls cheap.
|
||||
let stream = ReaderStream::with_capacity(file, 64 * 1024);
|
||||
@@ -244,6 +256,21 @@ mod tests {
|
||||
assert!(matches!(s.size("adir").await, Err(StorageError::NotFound)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn get_on_directory_is_not_found() {
|
||||
// A key resolving to a directory isn't a stored blob. `fs::read` on a
|
||||
// dir errors with EISDIR (not NotFound), and `File::open` on a dir
|
||||
// succeeds on Unix then streams garbage — both must surface NotFound.
|
||||
let dir = tempdir().unwrap();
|
||||
let s = LocalStorage::new(dir.path());
|
||||
std::fs::create_dir(dir.path().join("adir")).unwrap();
|
||||
assert!(matches!(s.get("adir").await, Err(StorageError::NotFound)));
|
||||
assert!(matches!(
|
||||
s.get_stream("adir").await.err(),
|
||||
Some(StorageError::NotFound)
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn put_stream_writes_full_body_and_removes_temp_on_error() {
|
||||
use bytes::Bytes;
|
||||
|
||||
@@ -37,6 +37,15 @@ pub struct StagedImage {
|
||||
/// staged page out of this prefix.
|
||||
pub const STAGING_PREFIX: &str = "staging";
|
||||
|
||||
/// Upper bound on a multipart `metadata` JSON part, enforced as bytes arrive
|
||||
/// (via [`read_capped`]). Manga/chapter metadata is title + a few short lists +
|
||||
/// a description — kilobytes at most. Without this, `metadata` was the one
|
||||
/// remaining part read with the unbounded `Field::bytes()`, letting a client
|
||||
/// buffer up to the whole 200 MiB request body in memory as a single JSON blob
|
||||
/// before any validation ran. 64 KiB is generous headroom over any legitimate
|
||||
/// payload while keeping the worst case tiny.
|
||||
pub const MAX_METADATA_BYTES: usize = 64 * 1024;
|
||||
|
||||
/// Read one multipart image part and write it straight to a staging key,
|
||||
/// returning only its metadata. The per-file byte cap is enforced as bytes
|
||||
/// arrive, so an oversized part is rejected (413) without being fully
|
||||
@@ -45,21 +54,13 @@ pub const STAGING_PREFIX: &str = "staging";
|
||||
/// read-all-then-persist path.
|
||||
pub async fn stage_image_part(
|
||||
storage: &dyn Storage,
|
||||
mut field: Field<'_>,
|
||||
field: Field<'_>,
|
||||
upload_id: Uuid,
|
||||
seq: usize,
|
||||
max_size: usize,
|
||||
field_name: &str,
|
||||
) -> AppResult<StagedImage> {
|
||||
let mut bytes: Vec<u8> = Vec::new();
|
||||
while let Some(chunk) = field.chunk().await.map_err(map_multipart_error)? {
|
||||
if bytes.len().saturating_add(chunk.len()) > max_size {
|
||||
return Err(AppError::PayloadTooLarge(format!(
|
||||
"{field_name} exceeds {max_size}-byte cap"
|
||||
)));
|
||||
}
|
||||
bytes.extend_from_slice(&chunk);
|
||||
}
|
||||
let bytes = read_capped(field, max_size, field_name).await?;
|
||||
// Reuse the shared sniff + whitelist check; it re-verifies the size cap
|
||||
// (already enforced above) and derives mime/ext from magic bytes.
|
||||
let img = parse_image(bytes, max_size, field_name)?;
|
||||
@@ -73,6 +74,46 @@ pub async fn stage_image_part(
|
||||
})
|
||||
}
|
||||
|
||||
/// Read one multipart part fully into memory, enforcing the per-file byte cap
|
||||
/// as chunks arrive so an oversized part is rejected (413) **without being fully
|
||||
/// buffered**. Use for parts whose bytes the caller needs in hand — e.g. the
|
||||
/// cover image, which is written to a manga-scoped key. Page parts should prefer
|
||||
/// [`stage_image_part`], which streams straight to storage.
|
||||
///
|
||||
/// Replaces the `Field::bytes()` path, which buffered the entire field before
|
||||
/// any size check ran, letting an attacker allocate an arbitrarily large body
|
||||
/// before the cap kicked in.
|
||||
pub async fn read_capped(
|
||||
mut field: Field<'_>,
|
||||
max_size: usize,
|
||||
field_name: &str,
|
||||
) -> AppResult<Vec<u8>> {
|
||||
let mut bytes: Vec<u8> = Vec::new();
|
||||
while let Some(chunk) = field.chunk().await.map_err(map_multipart_error)? {
|
||||
push_capped(&mut bytes, &chunk, max_size, field_name)?;
|
||||
}
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
/// Append `chunk` to `buf`, rejecting with 413 as soon as the running total
|
||||
/// would exceed `max_size` — so the oversized chunk is never copied in. Split
|
||||
/// out from the read loop so the cap logic is unit-testable without a live
|
||||
/// multipart field.
|
||||
fn push_capped(
|
||||
buf: &mut Vec<u8>,
|
||||
chunk: &[u8],
|
||||
max_size: usize,
|
||||
field_name: &str,
|
||||
) -> AppResult<()> {
|
||||
if buf.len().saturating_add(chunk.len()) > max_size {
|
||||
return Err(AppError::PayloadTooLarge(format!(
|
||||
"{field_name} exceeds {max_size}-byte cap"
|
||||
)));
|
||||
}
|
||||
buf.extend_from_slice(chunk);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn parse_image(bytes: Vec<u8>, max_size: usize, field_name: &str) -> AppResult<UploadedImage> {
|
||||
if bytes.len() > max_size {
|
||||
return Err(AppError::PayloadTooLarge(format!(
|
||||
@@ -172,4 +213,30 @@ mod tests {
|
||||
assert!(matches!(err, AppError::PayloadTooLarge(_)));
|
||||
assert_eq!(err.code(), "payload_too_large");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn push_capped_accumulates_under_cap() {
|
||||
let mut buf = Vec::new();
|
||||
push_capped(&mut buf, b"hello ", 100, "cover").unwrap();
|
||||
push_capped(&mut buf, b"world", 100, "cover").unwrap();
|
||||
assert_eq!(buf, b"hello world");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn push_capped_rejects_before_copying_oversized_chunk() {
|
||||
// The whole point: the offending chunk must NOT be appended — an
|
||||
// oversized part is rejected without buffering it.
|
||||
let mut buf = vec![0u8; 90];
|
||||
let err = push_capped(&mut buf, &[0u8; 20], 100, "cover").unwrap_err();
|
||||
assert!(matches!(err, AppError::PayloadTooLarge(_)));
|
||||
assert_eq!(err.code(), "payload_too_large");
|
||||
assert_eq!(buf.len(), 90, "buffer must not grow past the cap");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn push_capped_allows_exactly_at_cap() {
|
||||
let mut buf = vec![0u8; 90];
|
||||
push_capped(&mut buf, &[0u8; 10], 100, "cover").unwrap();
|
||||
assert_eq!(buf.len(), 100);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -99,6 +99,36 @@ async fn dispatch_persists_ocr_lines_and_search_doc(pool: PgPool) {
|
||||
assert!(has_doc, "search_doc must be populated from OCR text");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn dispatch_rejects_page_image_over_the_byte_cap(pool: PgPool) {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let storage: Arc<dyn Storage> = Arc::new(LocalStorage::new(dir.path()));
|
||||
let key = "mangas/x/big.png";
|
||||
// 4 KiB on disk, 1 KiB cap — the streamed read must bail on the cap
|
||||
// rather than buffering the whole blob and OCR-ing it.
|
||||
storage.put(key, &vec![0u8; 4096]).await.unwrap();
|
||||
let page_id = seed_page(&pool, key).await;
|
||||
|
||||
let dispatcher = OcrAnalyzeDispatcher {
|
||||
db: pool.clone(),
|
||||
storage: Arc::clone(&storage),
|
||||
engine: StubOcrEngine::new(&["should not run"]),
|
||||
max_image_bytes: 1024,
|
||||
ocr_permits: Arc::new(tokio::sync::Semaphore::new(1)),
|
||||
};
|
||||
let err = dispatcher.dispatch(page_id).await.unwrap_err();
|
||||
assert!(
|
||||
err.chain().any(|c| c.to_string().contains("cap")),
|
||||
"expected an over-cap error, got: {err:#}"
|
||||
);
|
||||
|
||||
// A rejected page must not land an analysis row.
|
||||
assert!(repo::page_analysis::load(&pool, page_id)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none());
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn dispatch_missing_page_is_noop(pool: PgPool) {
|
||||
let dir = TempDir::new().unwrap();
|
||||
|
||||
@@ -189,6 +189,51 @@ async fn reenqueue_returns_503_when_analysis_disabled(pool: PgPool) {
|
||||
assert_eq!(resp.status(), StatusCode::SERVICE_UNAVAILABLE);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn reenqueue_rejects_a_malformed_body(pool: PgPool) {
|
||||
// A present-but-malformed JSON body must be a 422 — NOT silently coerced to
|
||||
// an absent body, which would run the default full "All" scope the caller
|
||||
// never asked for and enqueue jobs across the whole library.
|
||||
let h = common::harness_with_analysis(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
seed_pages(&pool, 2).await;
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::post_raw_with_cookie(
|
||||
"/api/v1/admin/analysis/reenqueue",
|
||||
"{ not valid json",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::UNPROCESSABLE_ENTITY);
|
||||
assert_eq!(analyze_job_count(&pool).await, 0, "a rejected body enqueues nothing");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn reenqueue_treats_empty_body_as_default_all_scope(pool: PgPool) {
|
||||
// An absent/empty body is still valid: it means the default "All" scope.
|
||||
let h = common::harness_with_analysis(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
seed_pages(&pool, 2).await;
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::post_raw_with_cookie(
|
||||
"/api/v1/admin/analysis/reenqueue",
|
||||
"",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = common::body_json(resp).await;
|
||||
assert_eq!(body["enqueued"], 2, "empty body backfills the whole library");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn reenqueue_backfills_existing_pages(pool: PgPool) {
|
||||
let h = common::harness_with_analysis(pool.clone());
|
||||
|
||||
@@ -147,6 +147,46 @@ async fn list_filters_by_substring_search(pool: PgPool) {
|
||||
assert_eq!(body["page"]["total"], 1);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn search_treats_like_wildcards_literally(pool: PgPool) {
|
||||
let h = common::harness(pool.clone());
|
||||
let (_admin_name, cookie, _) = seed_admin(&pool, &h.app).await;
|
||||
// Two usernames differing only at one position (underscores are legal in
|
||||
// usernames). `_` is a LIKE single-char wildcard: unescaped, `%a_b%` matches
|
||||
// BOTH; escaped, only the literal "a_b" username. Admin user search has no
|
||||
// trigram OR, so length/similarity don't matter here.
|
||||
for username in ["axbfindme", "a_bfindme"] {
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::post_json(
|
||||
"/api/v1/auth/register",
|
||||
json!({ "username": username, "password": "hunter2hunter2" }),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::CREATED);
|
||||
}
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::get_with_cookie(
|
||||
"/api/v1/admin/users?search=a_b",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = common::body_json(resp).await;
|
||||
let items = body["items"].as_array().unwrap();
|
||||
assert_eq!(
|
||||
items.len(),
|
||||
1,
|
||||
"the `_` must match literally: only a_bfindme, not axbfindme"
|
||||
);
|
||||
assert_eq!(items[0]["username"], "a_bfindme");
|
||||
}
|
||||
|
||||
// ---- self-protection -------------------------------------------------------
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
|
||||
@@ -9,6 +9,73 @@ fn creds(username: &str) -> serde_json::Value {
|
||||
json!({ "username": username, "password": "hunter2hunter2" })
|
||||
}
|
||||
|
||||
/// A login request carrying a chosen `X-Forwarded-For`. Empty password so the
|
||||
/// handler consumes a rate-limit token then short-circuits with 400 before any
|
||||
/// argon2/DB work — the burst drains near-instantly regardless of CI load.
|
||||
fn login_from(xff: &str) -> axum::http::Request<axum::body::Body> {
|
||||
axum::http::Request::builder()
|
||||
.method("POST")
|
||||
.uri("/api/v1/auth/login")
|
||||
.header(header::CONTENT_TYPE, "application/json")
|
||||
.header("x-forwarded-for", xff)
|
||||
.body(axum::body::Body::from(
|
||||
json!({ "username": "victim", "password": "" }).to_string(),
|
||||
))
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
// The per-IP rate limiter's soundness hinges on one gate: X-Forwarded-For is
|
||||
// honored ONLY when AUTH_TRUSTED_PROXY is set. These two tests pin both sides of
|
||||
// that gate at the request level (the pure parser is unit-tested separately).
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn trusted_proxy_gives_each_forwarded_ip_its_own_bucket(pool: PgPool) {
|
||||
let h = common::harness_with_auth_rate_limit_proxy(pool, 1, 2, true);
|
||||
// Drain IP A's bucket (per_sec=1, burst=2) until it 429s.
|
||||
let mut a_saw_429 = false;
|
||||
for _ in 0..8 {
|
||||
let resp = h.app.clone().oneshot(login_from("203.0.113.10")).await.unwrap();
|
||||
if resp.status() == StatusCode::TOO_MANY_REQUESTS {
|
||||
a_saw_429 = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
assert!(a_saw_429, "IP A must be rate-limited after draining its own bucket");
|
||||
// A different forwarded IP has an independent bucket — its first hit is not 429.
|
||||
let resp_b = h.app.clone().oneshot(login_from("198.51.100.20")).await.unwrap();
|
||||
assert_ne!(
|
||||
resp_b.status(),
|
||||
StatusCode::TOO_MANY_REQUESTS,
|
||||
"a distinct X-Forwarded-For hop must get its own bucket, not A's drained one"
|
||||
);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn untrusted_proxy_ignores_forwarded_ip_and_shares_one_bucket(pool: PgPool) {
|
||||
let h = common::harness_with_auth_rate_limit_proxy(pool, 1, 2, false);
|
||||
// Every request carries a DISTINCT (spoofed) X-Forwarded-For, but the backend
|
||||
// doesn't trust it — so they all fall back to the single global bucket, which
|
||||
// drains and starts returning 429. If XFF were wrongly honored here, each
|
||||
// unique IP would get a fresh bucket and none of these would ever 429.
|
||||
let mut saw_429 = false;
|
||||
for i in 0..12 {
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(login_from(&format!("10.0.0.{i}")))
|
||||
.await
|
||||
.unwrap();
|
||||
if resp.status() == StatusCode::TOO_MANY_REQUESTS {
|
||||
saw_429 = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
saw_429,
|
||||
"with trusted_proxy off, spoofed X-Forwarded-For must NOT dodge the shared limiter"
|
||||
);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn register_creates_user_and_sets_session_cookie(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
@@ -541,6 +608,62 @@ async fn create_and_use_bot_token(pool: PgPool) {
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn list_tokens_returns_callers_tokens_scoped_and_without_hash(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
let (_, cookie) = common::register_user(&h.app).await;
|
||||
|
||||
// Mint two tokens for this user, one with an expiry.
|
||||
for body in [
|
||||
json!({ "name": "no-expiry" }),
|
||||
json!({ "name": "expiring", "expires_in_days": 30 }),
|
||||
] {
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::post_json_with_cookie(
|
||||
"/api/v1/auth/tokens",
|
||||
body,
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::CREATED);
|
||||
}
|
||||
|
||||
// A second user's token must NOT appear in the first user's list.
|
||||
let (_, other) = common::register_user(&h.app).await;
|
||||
let _ = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::post_json_with_cookie(
|
||||
"/api/v1/auth/tokens",
|
||||
json!({ "name": "someone-elses" }),
|
||||
&other,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::get_with_cookie("/api/v1/auth/tokens", &cookie))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = common::body_json(resp).await;
|
||||
let items = body["items"].as_array().unwrap();
|
||||
assert_eq!(items.len(), 2, "only the caller's two tokens");
|
||||
|
||||
let names: Vec<&str> = items.iter().map(|t| t["name"].as_str().unwrap()).collect();
|
||||
assert!(names.contains(&"no-expiry") && names.contains(&"expiring"));
|
||||
// Raw secret / hash must never appear, but expiry metadata must.
|
||||
for t in items {
|
||||
assert!(t.get("token_hash").is_none(), "token_hash must be absent");
|
||||
assert!(t.get("bearer").is_none(), "raw bearer only shown at creation");
|
||||
assert!(t.get("expires_at").is_some(), "expiry metadata present");
|
||||
}
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn bot_token_with_future_expiry_authenticates(pool: PgPool) {
|
||||
// A token minted with expires_in_days is still active before its
|
||||
|
||||
@@ -30,6 +30,47 @@ fn first_author_id(manga: &Value) -> String {
|
||||
manga["authors"][0]["id"].as_str().unwrap().to_string()
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn search_treats_like_wildcards_literally(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
let (_, cookie) = common::register_user(&h.app).await;
|
||||
// Long author names differing only at one position, short search term — so
|
||||
// the trigram OR (which keeps the raw term by design) stays under threshold
|
||||
// and the ILIKE branch is what decides. Unescaped `%a_b%` matches both the
|
||||
// "axb" and "a_b" names; escaped, only the literal "a_b" name.
|
||||
create_manga(
|
||||
&h.app,
|
||||
&cookie,
|
||||
json!({ "title": "M1", "authors": ["The Quick Brown Fox axb Jumps Over"] }),
|
||||
)
|
||||
.await;
|
||||
create_manga(
|
||||
&h.app,
|
||||
&cookie,
|
||||
json!({ "title": "M2", "authors": ["The Quick Brown Fox a_b Jumps Over"] }),
|
||||
)
|
||||
.await;
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::get("/api/v1/authors?search=a_b"))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = common::body_json(resp).await;
|
||||
let names: Vec<&str> = body
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|a| a["name"].as_str().unwrap())
|
||||
.collect();
|
||||
assert_eq!(
|
||||
names,
|
||||
vec!["The Quick Brown Fox a_b Jumps Over"],
|
||||
"the `_` in the search term must match literally, not as a wildcard"
|
||||
);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn get_returns_name_and_manga_count(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
|
||||
@@ -52,7 +52,7 @@ async fn create_then_list_returns_only_own(pool: PgPool) {
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn create_returns_409_on_duplicate_manga_level(pool: PgPool) {
|
||||
async fn create_is_idempotent_on_duplicate_manga_level(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
let (_, cookie) = common::register_user(&h.app).await;
|
||||
let manga_id = common::seed_manga_via_api(&h.app, &cookie, "Berserk").await;
|
||||
@@ -64,12 +64,26 @@ async fn create_returns_409_on_duplicate_manga_level(pool: PgPool) {
|
||||
&cookie,
|
||||
)
|
||||
};
|
||||
// First add creates (201); re-adding is an idempotent no-op (200) that
|
||||
// returns the SAME bookmark rather than a 409 — collections behave this way
|
||||
// too, so the UI shouldn't surface a false "Could not add bookmark" error.
|
||||
let first = h.app.clone().oneshot(make()).await.unwrap();
|
||||
assert_eq!(first.status(), StatusCode::CREATED);
|
||||
let second = h.app.oneshot(make()).await.unwrap();
|
||||
assert_eq!(second.status(), StatusCode::CONFLICT);
|
||||
let body = common::body_json(second).await;
|
||||
assert_eq!(body["error"]["code"], "conflict");
|
||||
let first_body = common::body_json(first).await;
|
||||
|
||||
let second = h.app.clone().oneshot(make()).await.unwrap();
|
||||
assert_eq!(second.status(), StatusCode::OK);
|
||||
let second_body = common::body_json(second).await;
|
||||
assert_eq!(second_body["id"], first_body["id"], "same bookmark returned");
|
||||
|
||||
// Exactly one row exists.
|
||||
let list = h
|
||||
.app
|
||||
.oneshot(common::get_with_cookie("/api/v1/me/bookmarks", &cookie))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = common::body_json(list).await;
|
||||
assert_eq!(body["items"].as_array().unwrap().len(), 1);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
@@ -225,13 +239,16 @@ async fn concurrent_manga_bookmarks_serialised_by_unique_index(pool: PgPool) {
|
||||
|
||||
let (s1, s2) = tokio::join!(f1, f2);
|
||||
let statuses = [s1.unwrap(), s2.unwrap()];
|
||||
// The unique index serialises the two inserts: one wins with 201, the other
|
||||
// hits the violation and — now idempotent — returns the existing bookmark
|
||||
// with 200 rather than a 409.
|
||||
assert!(
|
||||
statuses.contains(&StatusCode::CREATED),
|
||||
"expected one winner with 201, got {statuses:?}"
|
||||
);
|
||||
assert!(
|
||||
statuses.contains(&StatusCode::CONFLICT),
|
||||
"expected one loser with 409 (the partial unique index), got {statuses:?}"
|
||||
statuses.contains(&StatusCode::OK),
|
||||
"expected the loser to return 200 (idempotent), got {statuses:?}"
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -148,6 +148,49 @@ async fn list_chapters_returned_in_number_order(pool: PgPool) {
|
||||
assert_eq!(body["items"][1]["title"], serde_json::Value::Null);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn list_interleaves_uploaded_and_crawled_chapters_by_number(pool: PgPool) {
|
||||
// Mixed source: crawled chapters carry a `source_index`; an uploaded
|
||||
// chapter has none. The uploaded chapter must interleave by NUMBER, not sort
|
||||
// after every crawled chapter (the old `source_index DESC NULLS LAST` bug
|
||||
// dumped uploaded chapter 2 to the end, giving [1, 3, 2]).
|
||||
let h = common::harness(pool.clone());
|
||||
let (_, cookie) = common::register_user(&h.app).await;
|
||||
let manga_id = seed_manga(&h, &cookie, "Berserk").await;
|
||||
|
||||
// Two crawled chapters (1 and 3) with DOM positions — newest-first, so
|
||||
// chapter 3 is at source_index 0 and chapter 1 at source_index 1.
|
||||
let c1 = seed_chapter(&pool, manga_id, 1, Some("Crawled One")).await;
|
||||
let c3 = seed_chapter(&pool, manga_id, 3, Some("Crawled Three")).await;
|
||||
sqlx::query("UPDATE chapters SET source_index = 1 WHERE id = $1")
|
||||
.bind(c1)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("UPDATE chapters SET source_index = 0 WHERE id = $1")
|
||||
.bind(c3)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
// Uploaded chapter 2 — no source_index.
|
||||
seed_chapter(&pool, manga_id, 2, Some("Uploaded Two")).await;
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::get(&format!("/api/v1/mangas/{manga_id}/chapters")))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = common::body_json(resp).await;
|
||||
let numbers: Vec<i64> = body["items"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|c| c["number"].as_i64().unwrap())
|
||||
.collect();
|
||||
assert_eq!(numbers, vec![1, 2, 3], "uploaded chapter 2 must slot between 1 and 3");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn list_chapters_returns_404_for_unknown_manga(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
@@ -315,3 +358,27 @@ async fn non_owner_can_upload_chapter(pool: PgPool) {
|
||||
assert_eq!(body["title"], "Contributed");
|
||||
assert_eq!(body["page_count"], 1);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn chapter_upload_rejects_oversized_metadata_part(pool: PgPool) {
|
||||
// The chapter `metadata` JSON part is capped as bytes arrive, just like the
|
||||
// manga one, so a client can't buffer a huge JSON blob in memory. A ~200 KiB
|
||||
// title blows the metadata cap before any page is staged.
|
||||
let h = common::harness(pool);
|
||||
let (_, cookie) = common::register_user(&h.app).await;
|
||||
let manga_id = seed_manga(&h, &cookie, "Berserk").await;
|
||||
|
||||
let huge = "A".repeat(200 * 1024);
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::post_multipart_with_cookie(
|
||||
&format!("/api/v1/mangas/{manga_id}/chapters"),
|
||||
MultipartBuilder::new()
|
||||
.add_json("metadata", json!({ "number": 1, "title": huge }))
|
||||
.add_file("page", "1.png", "image/png", &common::fake_png_bytes()),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::PAYLOAD_TOO_LARGE);
|
||||
}
|
||||
|
||||
74
backend/tests/api_files.rs
Normal file
74
backend/tests/api_files.rs
Normal file
@@ -0,0 +1,74 @@
|
||||
mod common;
|
||||
|
||||
use axum::http::StatusCode;
|
||||
use tower::ServiceExt;
|
||||
|
||||
/// Write a blob straight into the harness storage root at `key`, bypassing the
|
||||
/// upload handlers — the only way to land a key whose extension resolves to the
|
||||
/// `application/octet-stream` fallback (uploads always mint image extensions).
|
||||
fn write_blob(h: &common::Harness, key: &str, bytes: &[u8]) {
|
||||
let path = h._storage_dir.path().join(key);
|
||||
std::fs::create_dir_all(path.parent().unwrap()).unwrap();
|
||||
std::fs::write(path, bytes).unwrap();
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn octet_stream_blobs_are_served_as_attachment(pool: sqlx::PgPool) {
|
||||
// A blob with an unknown extension serves as application/octet-stream. Such a
|
||||
// body could be crafted HTML/JS, so it must never render inline: force a
|
||||
// download with Content-Disposition: attachment (nosniff is already set).
|
||||
let h = common::harness(pool);
|
||||
write_blob(&h, "misc/blob.bin", b"\x00\x01not-an-image");
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::get("/api/v1/files/misc/blob.bin"))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
assert_eq!(
|
||||
resp.headers().get("content-type").unwrap(),
|
||||
"application/octet-stream"
|
||||
);
|
||||
assert_eq!(
|
||||
resp.headers().get("content-disposition").unwrap(),
|
||||
"attachment"
|
||||
);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn image_blobs_are_served_inline(pool: sqlx::PgPool) {
|
||||
// Regression guard: known image types keep rendering inline (no attachment
|
||||
// disposition), so covers/pages still display in the reader.
|
||||
let h = common::harness(pool);
|
||||
write_blob(&h, "misc/pic.png", &common::fake_png_bytes());
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::get("/api/v1/files/misc/pic.png"))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
assert_eq!(resp.headers().get("content-type").unwrap(), "image/png");
|
||||
assert!(
|
||||
resp.headers().get("content-disposition").is_none(),
|
||||
"images must render inline, not download"
|
||||
);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn corrupt_image_thumbnail_falls_back_to_original_not_500(pool: sqlx::PgPool) {
|
||||
// A blob with valid PNG magic bytes but an undecodable body passes upload's
|
||||
// magic-byte sniff; a ?w= thumbnail request then tries to decode it. That
|
||||
// must not 500 the reader — fall back to streaming the original bytes.
|
||||
let h = common::harness(pool);
|
||||
write_blob(&h, "misc/corrupt.png", &common::fake_png_bytes());
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::get("/api/v1/files/misc/corrupt.png?w=320"))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK, "corrupt thumbnail must not 500");
|
||||
assert_eq!(resp.headers().get("content-type").unwrap(), "image/png");
|
||||
}
|
||||
@@ -127,6 +127,46 @@ async fn list_filters_by_content_warning(pool: PgPool) {
|
||||
assert_eq!(ids, want);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn denormalized_warnings_track_chapter_deletion(pool: PgPool) {
|
||||
// The denormalized manga_content_warnings table must stay in sync when the
|
||||
// underlying pages disappear. Deleting the only chapter (which cascade-
|
||||
// deletes its pages and their page_content_warnings) must drop the manga
|
||||
// from the include filter.
|
||||
let h = common::harness(pool.clone());
|
||||
let gory = seed_manga(&pool, "Gory", &[&["gore"]]).await;
|
||||
|
||||
// Present in the denorm table and matched by the filter.
|
||||
let count: i64 =
|
||||
sqlx::query_scalar("SELECT count(*) FROM manga_content_warnings WHERE manga_id = $1")
|
||||
.bind(gory)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(count, 1, "warning denormalized on insert");
|
||||
assert_eq!(list_ids(&h.app, "cw_include=gore").await, vec![gory.to_string()]);
|
||||
|
||||
// Delete the chapter → cascade removes pages + page_content_warnings →
|
||||
// trigger recomputes the (now empty) set for this manga.
|
||||
sqlx::query("DELETE FROM chapters WHERE manga_id = $1")
|
||||
.bind(gory)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let count: i64 =
|
||||
sqlx::query_scalar("SELECT count(*) FROM manga_content_warnings WHERE manga_id = $1")
|
||||
.bind(gory)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(count, 0, "warning removed after chapter (and pages) deleted");
|
||||
assert!(
|
||||
list_ids(&h.app, "cw_include=gore").await.is_empty(),
|
||||
"manga no longer matches the include filter"
|
||||
);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn list_rejects_unknown_warning(pool: PgPool) {
|
||||
let h = common::harness(pool.clone());
|
||||
|
||||
@@ -99,6 +99,66 @@ async fn list_returns_total_count_independent_of_pagination(pool: PgPool) {
|
||||
assert_eq!(body["page"]["total"], 3);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn list_total_is_computed_only_on_the_first_page(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
let (_, cookie) = common::register_user(&h.app).await;
|
||||
for title in ["One Piece", "Berserk", "Vinland Saga"] {
|
||||
seed(&h.app, &cookie, title).await;
|
||||
}
|
||||
|
||||
// Page 1 (offset 0): total is the full population.
|
||||
let body0 = common::body_json(
|
||||
h.app
|
||||
.clone()
|
||||
.oneshot(common::get("/api/v1/mangas?limit=2&offset=0"))
|
||||
.await
|
||||
.unwrap(),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(body0["page"]["total"], 3);
|
||||
|
||||
// Page 2 (offset 2): total is omitted (null) — the correlated count is
|
||||
// not recomputed on every page. Items still paginate correctly.
|
||||
let body1 = common::body_json(
|
||||
h.app
|
||||
.oneshot(common::get("/api/v1/mangas?limit=2&offset=2"))
|
||||
.await
|
||||
.unwrap(),
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
body1["page"]["total"].is_null(),
|
||||
"total should be null past the first page, got {}",
|
||||
body1["page"]["total"]
|
||||
);
|
||||
assert_eq!(body1["items"].as_array().unwrap().len(), 1);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn search_treats_like_wildcards_literally(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
let (_, cookie) = common::register_user(&h.app).await;
|
||||
// Two long titles differing only at one position. `_` is a LIKE single-char
|
||||
// wildcard: unescaped, `%a_b%` matches BOTH ("axb" and "a_b"); escaped, only
|
||||
// the literal "a_b" title matches. The titles are long and the term short so
|
||||
// trigram similarity stays under threshold — the ILIKE branch decides.
|
||||
seed(&h.app, &cookie, "The Quick Brown Fox Jumps axb Over The Lazy Dog").await;
|
||||
seed(&h.app, &cookie, "The Quick Brown Fox Jumps a_b Over The Lazy Dog").await;
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::get("/api/v1/mangas?search=a_b"))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = common::body_json(resp).await;
|
||||
assert_eq!(
|
||||
title_list(&body),
|
||||
vec!["The Quick Brown Fox Jumps a_b Over The Lazy Dog"],
|
||||
"the `_` in the search term must match literally, not as a wildcard"
|
||||
);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn search_via_trigram_tolerates_typos(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
mod common;
|
||||
|
||||
use axum::http::StatusCode;
|
||||
use http_body_util::BodyExt;
|
||||
use serde_json::{json, Value};
|
||||
use sqlx::PgPool;
|
||||
use tower::ServiceExt;
|
||||
@@ -45,6 +46,65 @@ fn cover_form(bytes: &[u8]) -> MultipartBuilder {
|
||||
MultipartBuilder::new().add_file("cover", "cover.bin", "application/octet-stream", bytes)
|
||||
}
|
||||
|
||||
/// A real, decodable PNG (unlike `fake_png_bytes`, which is only magic bytes)
|
||||
/// so the thumbnail endpoint has something to resize.
|
||||
fn real_png(width: u32, height: u32) -> Vec<u8> {
|
||||
let mut buf = std::io::Cursor::new(Vec::new());
|
||||
image::RgbImage::from_pixel(width, height, image::Rgb([10, 120, 200]))
|
||||
.write_to(&mut buf, image::ImageFormat::Png)
|
||||
.unwrap();
|
||||
buf.into_inner()
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn files_serves_downscaled_thumbnail_variant(pool: PgPool) {
|
||||
let h = harness(pool);
|
||||
let (_, cookie) = register_user(&h.app).await;
|
||||
let manga = create_manga_with_cover(&h.app, &cookie, "Thumb", None).await;
|
||||
let id = id_of(&manga);
|
||||
|
||||
// Upload a real 800x400 PNG cover.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(put_multipart_with_cookie(
|
||||
&format!("/api/v1/mangas/{id}/cover"),
|
||||
cover_form(&real_png(800, 400)),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let key = format!("mangas/{id}/cover.png");
|
||||
|
||||
// ?w=320 serves a 320px-wide variant with the source content-type.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get(&format!("/api/v1/files/{key}?w=320")))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
assert_eq!(
|
||||
resp.headers().get("content-type").unwrap().to_str().unwrap(),
|
||||
"image/png"
|
||||
);
|
||||
let body = resp.into_body().collect().await.unwrap().to_bytes();
|
||||
let thumb = image::load_from_memory(&body).unwrap();
|
||||
assert_eq!(thumb.width(), 320, "served a 320px-wide variant");
|
||||
assert_eq!(thumb.height(), 160, "aspect ratio preserved");
|
||||
|
||||
// The un-parametrised request still serves the full-resolution original.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get(&format!("/api/v1/files/{key}")))
|
||||
.await
|
||||
.unwrap();
|
||||
let full = resp.into_body().collect().await.unwrap().to_bytes();
|
||||
assert_eq!(image::load_from_memory(&full).unwrap().width(), 800);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn put_cover_sets_path_when_none_existed(pool: PgPool) {
|
||||
let h = harness(pool);
|
||||
|
||||
@@ -326,6 +326,54 @@ async fn patch_updates_status_authors_and_genres(pool: PgPool) {
|
||||
assert_eq!(body["genres"][0]["name"], "Drama");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn patch_authors_updates_precomputed_sort_author(pool: PgPool) {
|
||||
// The precomputed mangas.sort_author (used by ?sort=author) must track
|
||||
// author edits: changing the alphabetically-first author reorders the
|
||||
// author sort. Two mangas whose relative author order flips after a PATCH.
|
||||
let h = common::harness(pool.clone());
|
||||
let (_, cookie) = common::register_user(&h.app).await;
|
||||
|
||||
let a = id_of(&create_manga(&h.app, &cookie, json!({ "title": "A", "authors": ["Zeta"] })).await);
|
||||
let _b = id_of(&create_manga(&h.app, &cookie, json!({ "title": "B", "authors": ["Mid"] })).await);
|
||||
|
||||
// Initially: Mid < Zeta, so B before A.
|
||||
let ids = list_titles(&h.app, "sort=author&order=asc").await;
|
||||
assert_eq!(ids, vec!["B", "A"]);
|
||||
|
||||
// Re-author A to "Alpha", which now sorts first.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::patch_json_with_cookie(
|
||||
&format!("/api/v1/mangas/{a}"),
|
||||
json!({ "authors": ["Alpha"] }),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
|
||||
// Now A (Alpha) sorts before B (Mid) — proving sort_author was recomputed.
|
||||
let ids = list_titles(&h.app, "sort=author&order=asc").await;
|
||||
assert_eq!(ids, vec!["A", "B"]);
|
||||
}
|
||||
|
||||
async fn list_titles(app: &axum::Router, query: &str) -> Vec<String> {
|
||||
let resp = app
|
||||
.clone()
|
||||
.oneshot(common::get(&format!("/api/v1/mangas?{query}")))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = common::body_json(resp).await;
|
||||
body["items"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|m| m["title"].as_str().unwrap().to_string())
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn patch_404_on_unknown_id(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
@@ -879,3 +927,25 @@ async fn bearer_authed_admin_cannot_edit_null_uploader(pool: PgPool) {
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn create_rejects_oversized_metadata_part(pool: PgPool) {
|
||||
// The metadata JSON part is capped well below the 200 MiB request limit so a
|
||||
// client can't force the server to buffer a huge JSON blob in memory before
|
||||
// any validation runs. A ~200 KiB description blows the metadata cap.
|
||||
let h = common::harness(pool);
|
||||
let (_, cookie) = common::register_user(&h.app).await;
|
||||
|
||||
let huge = "A".repeat(200 * 1024);
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::post_multipart_with_cookie(
|
||||
"/api/v1/mangas",
|
||||
MultipartBuilder::new()
|
||||
.add_json("metadata", json!({ "title": "Big", "description": huge })),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::PAYLOAD_TOO_LARGE);
|
||||
}
|
||||
|
||||
@@ -149,6 +149,56 @@ async fn private_mode_blocks_register_even_when_self_register_enabled(pool: PgPo
|
||||
assert_eq!(body["error"]["code"], "forbidden");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn private_mode_serves_files_as_private_not_public_cacheable(pool: PgPool) {
|
||||
// In private mode /files is auth-gated, so a served blob must NOT carry a
|
||||
// `public` Cache-Control: a shared cache / CDN would otherwise store it and
|
||||
// hand it to anonymous clients, defeating the gate. It should be `private`.
|
||||
//
|
||||
// Register via a public harness on the shared pool so the session exists,
|
||||
// then upload + fetch a cover through the private harness (same storage).
|
||||
let public = common::harness(pool.clone());
|
||||
let (_, cookie) = common::register_user(&public.app).await;
|
||||
|
||||
let private = common::harness_with_private_mode(pool);
|
||||
let form = common::MultipartBuilder::new()
|
||||
.add_json("metadata", json!({ "title": "Secret Library" }))
|
||||
.add_file("cover", "cover.png", "image/png", &common::fake_png_bytes());
|
||||
let created = private
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::post_multipart_with_cookie(
|
||||
"/api/v1/mangas",
|
||||
form,
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(created.status(), StatusCode::CREATED);
|
||||
let body = common::body_json(created).await;
|
||||
let key = body["cover_image_path"].as_str().unwrap().to_string();
|
||||
|
||||
let resp = private
|
||||
.app
|
||||
.oneshot(common::get_with_cookie(
|
||||
&format!("/api/v1/files/{key}"),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let cc = resp
|
||||
.headers()
|
||||
.get(axum::http::header::CACHE_CONTROL)
|
||||
.unwrap()
|
||||
.to_str()
|
||||
.unwrap();
|
||||
assert!(
|
||||
cc.starts_with("private") && !cc.contains("public"),
|
||||
"private-mode blobs must be `private`, not `public`; got: {cc}"
|
||||
);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn auth_config_reports_private_mode_and_effective_self_register(pool: PgPool) {
|
||||
let h = common::harness_with_private_mode(pool);
|
||||
|
||||
@@ -172,6 +172,13 @@ async fn files_endpoint_streams_in_multiple_frames(pool: PgPool) {
|
||||
resp.headers().get("x-content-type-options").unwrap(),
|
||||
"nosniff"
|
||||
);
|
||||
// Blobs are content-addressed by unguessable, immutable keys, so they're
|
||||
// safe to cache forever — this is what makes the reader's page + next-
|
||||
// chapter preloading actually hit cache instead of re-downloading.
|
||||
assert_eq!(
|
||||
resp.headers().get(header::CACHE_CONTROL).unwrap(),
|
||||
"public, max-age=31536000, immutable"
|
||||
);
|
||||
|
||||
let mut body = resp.into_body();
|
||||
let mut frames = 0usize;
|
||||
|
||||
@@ -148,6 +148,27 @@ pub fn harness_with_auth_rate_limit(
|
||||
harness_with_auth_config(pool, storage, storage_dir, auth)
|
||||
}
|
||||
|
||||
/// Like [`harness_with_auth_rate_limit`] but also sets whether the backend
|
||||
/// trusts `X-Forwarded-For` (`AUTH_TRUSTED_PROXY`). Used to prove the per-IP
|
||||
/// rate-limit gate: with `trusted_proxy` on, distinct XFF hops get independent
|
||||
/// buckets; with it off, XFF is ignored and everything shares the global bucket.
|
||||
pub fn harness_with_auth_rate_limit_proxy(
|
||||
pool: PgPool,
|
||||
per_sec: u32,
|
||||
burst: u32,
|
||||
trusted_proxy: bool,
|
||||
) -> Harness {
|
||||
let storage_dir = tempfile::tempdir().expect("tempdir");
|
||||
let storage = Arc::new(LocalStorage::new(storage_dir.path()));
|
||||
let auth = AuthConfig {
|
||||
cookie_secure: false,
|
||||
trusted_proxy,
|
||||
rate_limit: mangalord::auth::rate_limit::RateLimitConfig { per_sec, burst },
|
||||
..AuthConfig::default()
|
||||
};
|
||||
harness_with_auth_config(pool, storage, storage_dir, auth)
|
||||
}
|
||||
|
||||
/// Like [`harness`] but slots a caller-supplied [`ResyncService`] stub
|
||||
/// into `AppState.resync`. Used by the admin resync tests so the
|
||||
/// endpoint path is exercised without standing up a real Chromium.
|
||||
@@ -475,6 +496,20 @@ pub fn post_json_with_cookie(
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
/// Like [`post_json_with_cookie`] but sends a raw (possibly malformed) body,
|
||||
/// for exercising body-parse error handling. Content-Type stays JSON and the
|
||||
/// CSRF Origin is attached so the request reaches the handler.
|
||||
pub fn post_raw_with_cookie(uri: &str, body: &str, cookie: &str) -> Request<Body> {
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri(uri)
|
||||
.header(header::CONTENT_TYPE, "application/json")
|
||||
.header(header::COOKIE, cookie)
|
||||
.header(header::ORIGIN, TEST_ORIGIN)
|
||||
.body(Body::from(body.to_string()))
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
/// Same as [`post_json_with_cookie`] but also attaches `Origin` (and
|
||||
/// optionally `Referer`) headers. Used by the admin CSRF tests to drive
|
||||
/// the cross-origin reject + allowed-origin accept paths.
|
||||
|
||||
@@ -62,6 +62,43 @@ async fn headless_browser_can_navigate_and_read_title() {
|
||||
handle.close().await.expect("close cleanly");
|
||||
}
|
||||
|
||||
/// Smoke-test the opt-in SSRF navigation guard (`CRAWLER_SSRF_INTERCEPT`).
|
||||
/// With interception ON, a normal (allowed) navigation must still complete —
|
||||
/// i.e. enabling CDP `Fetch` and installing the request handler must NOT wedge
|
||||
/// page loads. This is the regression the interceptor's fragility risks; it
|
||||
/// can only be exercised with a real Chromium, hence `#[ignore]`.
|
||||
///
|
||||
/// (The private-target *blocking* path is covered by the pure-logic unit tests
|
||||
/// in `crawler::intercept` — `verdict` / `is_blocked` — which don't need a
|
||||
/// browser.)
|
||||
#[tokio::test]
|
||||
#[ignore = "downloads Chromium; run with --ignored"]
|
||||
async fn ssrf_interception_does_not_wedge_allowed_navigation() {
|
||||
use mangalord::crawler::intercept;
|
||||
|
||||
const PAGE: &str =
|
||||
"data:text/html,<html><head><title>Guarded%20OK</title></head><body></body></html>";
|
||||
|
||||
intercept::set_enabled(true);
|
||||
let handle = browser::launch(LaunchOptions::headless())
|
||||
.await
|
||||
.expect("launch headless chromium");
|
||||
|
||||
// Route through the guarded opener (blank page -> Fetch.enable -> handler
|
||||
// -> goto). If the handler failed to continue the navigation, this would
|
||||
// hang until the test harness times out.
|
||||
let page = intercept::open_page(handle.browser(), PAGE)
|
||||
.await
|
||||
.expect("guarded open_page");
|
||||
page.wait_for_navigation().await.expect("wait for navigation");
|
||||
|
||||
let title = page.get_title().await.expect("get title");
|
||||
assert_eq!(title.as_deref(), Some("Guarded OK"));
|
||||
|
||||
handle.close().await.expect("close cleanly");
|
||||
intercept::set_enabled(false);
|
||||
}
|
||||
|
||||
/// Live end-to-end: navigate to a real page, get the rendered HTML, and
|
||||
/// parse it with `scraper`. ipify.org renders the visitor's public IP
|
||||
/// into the page DOM, so a successful run proves browser → render →
|
||||
|
||||
@@ -104,6 +104,20 @@ impl ChapterDispatcher for FailingDispatcher {
|
||||
}
|
||||
}
|
||||
|
||||
/// Always reports the browser unavailable — models a Chromium outage where
|
||||
/// `acquire()` fails. The job must be returned to `pending` WITHOUT burning an
|
||||
/// attempt, so an outage doesn't chew the backlog to `dead`.
|
||||
struct BrowserUnavailableDispatcher {
|
||||
seen: AtomicUsize,
|
||||
}
|
||||
#[async_trait::async_trait]
|
||||
impl ChapterDispatcher for BrowserUnavailableDispatcher {
|
||||
async fn dispatch(&self, _payload: JobPayload) -> anyhow::Result<SyncOutcome> {
|
||||
self.seen.fetch_add(1, Ordering::AcqRel);
|
||||
Ok(SyncOutcome::BrowserUnavailable)
|
||||
}
|
||||
}
|
||||
|
||||
/// Never completes — used to verify the worker's outer dispatch timeout.
|
||||
struct HangingDispatcher {
|
||||
seen: AtomicUsize,
|
||||
@@ -204,6 +218,58 @@ async fn shutdown_mid_dispatch_releases_lease_without_burning_attempt(pool: PgPo
|
||||
);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn browser_unavailable_releases_lease_without_burning_attempt(pool: PgPool) {
|
||||
// During a browser outage the dispatcher reports BrowserUnavailable. The
|
||||
// worker must return the job to `pending` with the attempt refunded
|
||||
// (attempts stays 0) instead of ack-failing it toward `dead`, so the whole
|
||||
// pending backlog survives the outage.
|
||||
enqueue_chapter_job(&pool).await;
|
||||
let dispatcher = Arc::new(BrowserUnavailableDispatcher {
|
||||
seen: AtomicUsize::new(0),
|
||||
});
|
||||
let session_expired = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
let cancel = CancellationToken::new();
|
||||
let handle = daemon::spawn(
|
||||
pool.clone(),
|
||||
cancel.clone(),
|
||||
make_cfg(None, dispatcher.clone(), session_expired, 1),
|
||||
);
|
||||
|
||||
// Wait until the dispatcher has been invoked at least once (the job was
|
||||
// leased and deferred).
|
||||
let mut dispatched = false;
|
||||
for _ in 0..40 {
|
||||
if dispatcher.seen.load(Ordering::Acquire) >= 1 {
|
||||
dispatched = true;
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
assert!(dispatched, "dispatcher must have been invoked");
|
||||
|
||||
handle.shutdown().await;
|
||||
|
||||
assert_eq!(
|
||||
count_state(&pool, "dead").await,
|
||||
0,
|
||||
"browser outage must not dead-letter the job"
|
||||
);
|
||||
assert_eq!(
|
||||
count_state(&pool, "pending").await,
|
||||
1,
|
||||
"job returns to pending after a browser-unavailable outcome"
|
||||
);
|
||||
let attempts: i32 = sqlx::query_scalar("SELECT attempts FROM crawler_jobs")
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
attempts, 0,
|
||||
"browser-unavailable must refund the lease attempt (no burn)"
|
||||
);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn workers_drain_jobs_through_dispatcher(pool: PgPool) {
|
||||
enqueue_chapter_job(&pool).await;
|
||||
|
||||
@@ -674,6 +674,110 @@ async fn arbitrary_genres_from_source_get_inserted(pool: PgPool) {
|
||||
assert_eq!(webtoons_count.0, 1, "case-insensitive lookup reuses the existing row");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn genre_dedup_survives_a_manga_linked_to_two_variants(pool: PgPool) {
|
||||
// Regression for a migration-0038 collision: a manga linked to TWO
|
||||
// non-canonical case-variants of one genre. A repoint-UPDATE would set both
|
||||
// rows to the canonical id in one statement -> manga_genres PK violation ->
|
||||
// migration rollback -> boot failure. #[sqlx::test] migrates a clean DB, so
|
||||
// recreate the dirty pre-index state and replay 0038's healing statements;
|
||||
// they must complete and leave exactly the one canonical link. (Mirrors the
|
||||
// healing SQL in migrations/0038_genres_name_lower_unique.sql.)
|
||||
let manga = Uuid::new_v4();
|
||||
sqlx::query("INSERT INTO mangas (id, title) VALUES ($1, 'T')")
|
||||
.bind(manga)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Drop the live guard so we can plant case-variant genres.
|
||||
sqlx::query("DROP INDEX genres_name_lower_uniq").execute(&pool).await.unwrap();
|
||||
let keep = Uuid::parse_str("00000000-0000-0000-0000-000000000001").unwrap();
|
||||
let dup2 = Uuid::parse_str("00000000-0000-0000-0000-000000000002").unwrap();
|
||||
let dup3 = Uuid::parse_str("00000000-0000-0000-0000-000000000003").unwrap();
|
||||
for (id, name) in [(keep, "Zzz"), (dup2, "zzz"), (dup3, "ZZZ")] {
|
||||
sqlx::query("INSERT INTO genres (id, name) VALUES ($1, $2)")
|
||||
.bind(id)
|
||||
.bind(name)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
// The manga links to the two NON-canonical variants, not the canonical one.
|
||||
for gid in [dup2, dup3] {
|
||||
sqlx::query("INSERT INTO manga_genres (manga_id, genre_id) VALUES ($1, $2)")
|
||||
.bind(manga)
|
||||
.bind(gid)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
// Step 1 — collision-proof canonical-link backfill (the crux of the fix).
|
||||
sqlx::query(
|
||||
"INSERT INTO manga_genres (manga_id, genre_id) \
|
||||
SELECT DISTINCT mg.manga_id, k.keep_id \
|
||||
FROM manga_genres mg JOIN genres g ON mg.genre_id = g.id \
|
||||
JOIN (SELECT lower(name) AS lname, (array_agg(id ORDER BY id))[1] AS keep_id \
|
||||
FROM genres GROUP BY lower(name)) k ON lower(g.name) = k.lname \
|
||||
WHERE g.id <> k.keep_id \
|
||||
ON CONFLICT (manga_id, genre_id) DO NOTHING",
|
||||
)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.expect("collision-proof INSERT must not raise a manga_genres PK violation");
|
||||
// Step 2 — drop the non-canonical links.
|
||||
sqlx::query(
|
||||
"DELETE FROM manga_genres mg USING genres g, \
|
||||
(SELECT lower(name) AS lname, (array_agg(id ORDER BY id))[1] AS keep_id \
|
||||
FROM genres GROUP BY lower(name)) k \
|
||||
WHERE mg.genre_id = g.id AND lower(g.name) = k.lname AND g.id <> k.keep_id",
|
||||
)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
// Step 3 — remove orphaned duplicate genres.
|
||||
sqlx::query(
|
||||
"DELETE FROM genres g USING \
|
||||
(SELECT lower(name) AS lname, (array_agg(id ORDER BY id))[1] AS keep_id \
|
||||
FROM genres GROUP BY lower(name)) k \
|
||||
WHERE lower(g.name) = k.lname AND g.id <> k.keep_id",
|
||||
)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Exactly the canonical link remains, and the guard re-creates cleanly.
|
||||
let links: Vec<Uuid> =
|
||||
sqlx::query_scalar("SELECT genre_id FROM manga_genres WHERE manga_id = $1")
|
||||
.bind(manga)
|
||||
.fetch_all(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(links, vec![keep], "manga keeps exactly the canonical genre link");
|
||||
sqlx::query("CREATE UNIQUE INDEX genres_name_lower_uniq ON genres (lower(name))")
|
||||
.execute(&pool)
|
||||
.await
|
||||
.expect("no residual case-variant duplicates remain");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn genres_reject_case_variant_duplicates_at_the_db(pool: PgPool) {
|
||||
// The sequential pre-check in sync_genres dedups the common case, but two
|
||||
// concurrent inserts could each miss it. The lower(name) unique index (0038)
|
||||
// is the race backstop: a case variant of an existing genre must be rejected
|
||||
// by the DB itself. "Action" is seeded by 0009.
|
||||
let dup = sqlx::query("INSERT INTO genres (name) VALUES ('action')")
|
||||
.execute(&pool)
|
||||
.await;
|
||||
let err = dup.expect_err("a case-variant of a seeded genre must violate the unique index");
|
||||
let msg = err.to_string();
|
||||
assert!(
|
||||
msg.contains("genres_name_lower_uniq") || msg.contains("unique") || msg.contains("duplicate"),
|
||||
"expected a unique-violation, got: {msg}"
|
||||
);
|
||||
}
|
||||
|
||||
/// User-attached tags (rows with non-NULL `added_by` in `manga_tags`)
|
||||
/// must survive a crawler upsert. The crawler owns source-attached tags
|
||||
/// (added_by IS NULL); user attachments are owned by the user who made
|
||||
@@ -1175,7 +1279,7 @@ async fn list_for_manga_returns_source_order_reversed(pool: PgPool) {
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn list_for_manga_places_null_source_index_last(pool: PgPool) {
|
||||
async fn list_for_manga_interleaves_null_source_index_by_number(pool: PgPool) {
|
||||
crawler::ensure_source(&pool, "target", "T", "https://x.example")
|
||||
.await
|
||||
.unwrap();
|
||||
@@ -1184,23 +1288,24 @@ async fn list_for_manga_places_null_source_index_last(pool: PgPool) {
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Crawled chapters get source_index 0 and 1; the upload path leaves
|
||||
// it NULL. NULLS LAST plus the (number, created_at) tail means the
|
||||
// upload sits after both crawled rows even though its number is in
|
||||
// the middle.
|
||||
// Crawled chapters, newest-first in the source DOM (so chapter 3 is at
|
||||
// source_index 0 and chapter 1 at source_index 1). Reversed for display that
|
||||
// is [Ch.1, Ch.3]. The uploaded chapter 2 must slot BETWEEN them by number,
|
||||
// not fall to the end — that was the bug (NULLS LAST dumped it last
|
||||
// regardless of its number).
|
||||
let crawled = vec![
|
||||
SourceChapterRef {
|
||||
source_chapter_key: "a".into(),
|
||||
number: 1,
|
||||
title: Some("Ch.1".into()),
|
||||
url: "https://x.example/foo/a".into(),
|
||||
},
|
||||
SourceChapterRef {
|
||||
source_chapter_key: "b".into(),
|
||||
number: 3,
|
||||
title: Some("Ch.3".into()),
|
||||
url: "https://x.example/foo/b".into(),
|
||||
},
|
||||
SourceChapterRef {
|
||||
source_chapter_key: "a".into(),
|
||||
number: 1,
|
||||
title: Some("Ch.1".into()),
|
||||
url: "https://x.example/foo/a".into(),
|
||||
},
|
||||
];
|
||||
crawler::sync_manga_chapters(&pool, "target", up.manga_id, &crawled)
|
||||
.await
|
||||
@@ -1220,11 +1325,11 @@ async fn list_for_manga_places_null_source_index_last(pool: PgPool) {
|
||||
assert_eq!(
|
||||
titles,
|
||||
vec![
|
||||
"Ch.3".to_string(),
|
||||
"Ch.1".to_string(),
|
||||
"User upload Ch.2".to_string(),
|
||||
"Ch.3".to_string(),
|
||||
],
|
||||
"crawled rows ordered by reversed source_index; user upload \
|
||||
(NULL source_index) falls through to the end",
|
||||
"uploaded chapter (NULL source_index) interleaves by number between the \
|
||||
crawled rows instead of falling to the end",
|
||||
);
|
||||
}
|
||||
|
||||
52
backend/tests/repo_session.rs
Normal file
52
backend/tests/repo_session.rs
Normal file
@@ -0,0 +1,52 @@
|
||||
mod common;
|
||||
|
||||
use chrono::{Duration, Utc};
|
||||
use sqlx::PgPool;
|
||||
|
||||
use mangalord::repo;
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn delete_expired_removes_only_lapsed_sessions(pool: PgPool) {
|
||||
let user = repo::user::create(&pool, "reaper-subject", "x").await.unwrap();
|
||||
|
||||
// One session already lapsed, one still valid.
|
||||
let expired = repo::session::create(
|
||||
&pool,
|
||||
user.id,
|
||||
b"expired-token-hash-00000000000000",
|
||||
Utc::now() - Duration::hours(1),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let active = repo::session::create(
|
||||
&pool,
|
||||
user.id,
|
||||
b"active-token-hash-000000000000000",
|
||||
Utc::now() + Duration::hours(1),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let removed = repo::session::delete_expired(&pool).await.unwrap();
|
||||
assert_eq!(removed, 1, "exactly the one lapsed session is reaped");
|
||||
|
||||
// The active session is untouched and still resolvable; the expired one is
|
||||
// gone from the table entirely (find_active already ignored it, but now the
|
||||
// row is reclaimed too).
|
||||
assert!(
|
||||
repo::session::find_active(&pool, b"active-token-hash-000000000000000")
|
||||
.await
|
||||
.unwrap()
|
||||
.is_some(),
|
||||
"active session survives the sweep"
|
||||
);
|
||||
let remaining: i64 = sqlx::query_scalar("SELECT count(*) FROM sessions")
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(remaining, 1);
|
||||
|
||||
// Sweeping again is a no-op now that nothing is expired.
|
||||
assert_eq!(repo::session::delete_expired(&pool).await.unwrap(), 0);
|
||||
let _ = (expired, active);
|
||||
}
|
||||
@@ -23,6 +23,12 @@ services:
|
||||
# takes the whole stack offline. Match the policy already set on
|
||||
# tor / docker-socket-proxy / vision-manager.
|
||||
restart: unless-stopped
|
||||
# Bound Postgres so a runaway query/backlog can't consume all host RAM.
|
||||
# Tune to your host via .env (should comfortably exceed shared_buffers +
|
||||
# work_mem * max_connections).
|
||||
mem_limit: ${POSTGRES_MEM_LIMIT:-1g}
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
|
||||
tor:
|
||||
# SOCKS5 proxy for the crawler, plus a control port so the backend
|
||||
@@ -71,12 +77,19 @@ services:
|
||||
BIND_ADDRESS: 0.0.0.0:8080
|
||||
STORAGE_DIR: /var/lib/mangalord/storage
|
||||
RUST_LOG: ${RUST_LOG:-info,mangalord=debug}
|
||||
# Postgres connection-pool sizing — see .env.example for context.
|
||||
DB_MAX_CONNECTIONS: ${DB_MAX_CONNECTIONS:-20}
|
||||
DB_ACQUIRE_TIMEOUT_SECS: ${DB_ACQUIRE_TIMEOUT_SECS:-10}
|
||||
# Auth / cookies — see .env.example for context.
|
||||
COOKIE_SECURE: ${COOKIE_SECURE:-true}
|
||||
COOKIE_DOMAIN: ${COOKIE_DOMAIN:-}
|
||||
SESSION_TTL_DAYS: ${SESSION_TTL_DAYS:-30}
|
||||
AUTH_RATE_PER_SEC: ${AUTH_RATE_PER_SEC:-5}
|
||||
AUTH_RATE_BURST: ${AUTH_RATE_BURST:-10}
|
||||
# The SvelteKit container is the single trusted hop in front of the
|
||||
# backend and stamps the real client IP, so per-IP rate limiting is on
|
||||
# by default here (unlike the backend's safe-by-default `false`).
|
||||
AUTH_TRUSTED_PROXY: ${AUTH_TRUSTED_PROXY:-true}
|
||||
# CORS — same-origin by default; populate when serving the API on
|
||||
# a different host than the frontend.
|
||||
CORS_ALLOWED_ORIGINS: ${CORS_ALLOWED_ORIGINS:-}
|
||||
@@ -105,6 +118,7 @@ services:
|
||||
CRAWLER_JOB_TIMEOUT_SECS: ${CRAWLER_JOB_TIMEOUT_SECS:-600}
|
||||
CRAWLER_METADATA_MAX_CONSECUTIVE_FAILURES: ${CRAWLER_METADATA_MAX_CONSECUTIVE_FAILURES:-10}
|
||||
CRAWLER_BROWSER_RESTART_THRESHOLD: ${CRAWLER_BROWSER_RESTART_THRESHOLD:-3}
|
||||
CRAWLER_SSRF_INTERCEPT: ${CRAWLER_SSRF_INTERCEPT:-true}
|
||||
# Crawler daemon schedule + retention. CRAWLER_DAEMON=false keeps
|
||||
# the in-process scheduler off; the dashboard force-resync still works.
|
||||
CRAWLER_DAEMON: ${CRAWLER_DAEMON:-true}
|
||||
@@ -189,6 +203,13 @@ services:
|
||||
expose:
|
||||
- "8080"
|
||||
restart: unless-stopped
|
||||
# Bound the heaviest consumer: the backend hosts the OCR/analysis worker and
|
||||
# drives a headless Chromium crawler, either of which can balloon and
|
||||
# OOM-kill the host (taking Postgres with it) with no limit. Tune to your
|
||||
# host via .env — this default is a generous ceiling, not a target.
|
||||
mem_limit: ${BACKEND_MEM_LIMIT:-4g}
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
|
||||
frontend:
|
||||
build: ./frontend
|
||||
@@ -205,8 +226,17 @@ services:
|
||||
# to 300000 (5 min) in hooks.server.ts; raise/lower via .env.
|
||||
BACKEND_PROXY_TIMEOUT_MS: ${BACKEND_PROXY_TIMEOUT_MS:-300000}
|
||||
ports:
|
||||
- "3000:3000"
|
||||
# Bind to loopback by default so SvelteKit (plain HTTP; COOKIE_SECURE=true
|
||||
# expects TLS in front) is reachable ONLY by a host-local reverse proxy,
|
||||
# not cleartext over the LAN/internet. A host-external TLS terminator can
|
||||
# set FRONTEND_PUBLISH_ADDR=0.0.0.0 (or a specific interface) in .env.
|
||||
- "${FRONTEND_PUBLISH_ADDR:-127.0.0.1}:3000:3000"
|
||||
restart: unless-stopped
|
||||
# Cap runaway memory (SvelteKit is light, but bound it anyway) and forbid
|
||||
# privilege escalation. Tune the limit to your host via .env.
|
||||
mem_limit: ${FRONTEND_MEM_LIMIT:-512m}
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
|
||||
# ----- Vision autoscaling (profile: ai) -----------------------------------
|
||||
# Two extra containers that idle-stop the mangalord-vision (llama.cpp)
|
||||
|
||||
10
frontend/csp-config.js
Normal file
10
frontend/csp-config.js
Normal file
@@ -0,0 +1,10 @@
|
||||
// Shared CSP constants, kept dependency-free so both svelte.config.js (loaded by
|
||||
// Node when Vite starts) and the vitest drift-guard test (jsdom env) can import
|
||||
// it without pulling in adapter-node / esbuild.
|
||||
|
||||
// sha256 of the inline theme <script> in src/app.html, base64-encoded, wrapped
|
||||
// for the CSP `script-src` allowlist. SvelteKit's kit.csp hash mode does not
|
||||
// cover app.html template scripts, so this is pinned by hand. src/csp-theme-hash.test.ts
|
||||
// recomputes it from app.html and fails if it drifts.
|
||||
export const THEME_SCRIPT_HASH =
|
||||
"'sha256-qj6Oim9siqow/Su+v47sZHJeJci+M/RQwGXn53eSULQ='";
|
||||
132
frontend/e2e/admin-users.spec.ts
Normal file
132
frontend/e2e/admin-users.spec.ts
Normal file
@@ -0,0 +1,132 @@
|
||||
import { test, expect, type Page } from './fixtures';
|
||||
|
||||
// E2E for the admin Users page role toggle: flipping the Admin checkbox is a
|
||||
// one-click privilege change, so it must confirm first, and the checkbox must
|
||||
// never show a state the server didn't actually apply (cancel or failure keeps
|
||||
// it on the true value).
|
||||
|
||||
const DESKTOP = { width: 1280, height: 720 } as const;
|
||||
|
||||
const adminUser = {
|
||||
id: 'u11111111-1111-1111-1111-111111111111',
|
||||
username: 'admin',
|
||||
created_at: '2026-01-01T00:00:00Z',
|
||||
is_admin: true
|
||||
};
|
||||
|
||||
const bob = {
|
||||
id: 'b22222222-2222-2222-2222-222222222222',
|
||||
username: 'bob',
|
||||
created_at: '2026-02-01T00:00:00Z',
|
||||
is_admin: false
|
||||
};
|
||||
|
||||
type Captured = { patched: boolean };
|
||||
|
||||
async function mockAdmin(page: Page, cap: Captured, opts: { patchStatus: number }) {
|
||||
await page.route('**/api/v1/auth/config', (r) =>
|
||||
r.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ self_register_enabled: true, private_mode: false })
|
||||
})
|
||||
);
|
||||
await page.route('**/api/v1/auth/me', (r) =>
|
||||
r.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ user: adminUser })
|
||||
})
|
||||
);
|
||||
await page.route('**/api/v1/auth/me/preferences', (r) =>
|
||||
r.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ reader_mode: 'single', reader_page_gap: 'small' })
|
||||
})
|
||||
);
|
||||
await page.route('**/api/v1/me/bookmarks*', (r) =>
|
||||
r.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ items: [], page: { limit: 50, offset: 0, total: 0 } })
|
||||
})
|
||||
);
|
||||
await page.route('**/api/v1/admin/system', (r) =>
|
||||
r.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({
|
||||
disk: null,
|
||||
memory: { total_bytes: 1, used_bytes: 0, percent_used: 0 },
|
||||
cpu: { percent_used: 0 },
|
||||
alerts: []
|
||||
})
|
||||
})
|
||||
);
|
||||
// The PATCH toggle — record whether it was ever sent.
|
||||
await page.route('**/api/v1/admin/users/*', async (r) => {
|
||||
cap.patched = true;
|
||||
await r.fulfill({
|
||||
status: opts.patchStatus,
|
||||
contentType: 'application/json',
|
||||
body:
|
||||
opts.patchStatus < 400
|
||||
? JSON.stringify({ ...bob, is_admin: true })
|
||||
: JSON.stringify({ error: { code: 'internal_error', message: 'boom' } })
|
||||
});
|
||||
});
|
||||
// The user list (registered after the specific PATCH glob; list is GET).
|
||||
await page.route('**/api/v1/admin/users**', async (r) => {
|
||||
if (r.request().method() !== 'GET') return r.fallback();
|
||||
await r.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({
|
||||
items: [adminUser, bob],
|
||||
page: { limit: 100, offset: 0, total: 2 }
|
||||
})
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
test.describe('/admin/users role toggle', () => {
|
||||
test('cancelling the confirm leaves the checkbox unchanged and sends no request', async ({
|
||||
page
|
||||
}) => {
|
||||
const cap: Captured = { patched: false };
|
||||
await mockAdmin(page, cap, { patchStatus: 200 });
|
||||
page.on('dialog', (d) => d.dismiss());
|
||||
await page.setViewportSize(DESKTOP);
|
||||
await page.goto('/admin/users');
|
||||
|
||||
const bobRow = page.locator('tr', { hasText: 'bob' });
|
||||
const checkbox = bobRow.getByLabel('admin');
|
||||
await expect(checkbox).not.toBeChecked();
|
||||
|
||||
await checkbox.click();
|
||||
|
||||
// Cancelled → no PATCH, and the box still reflects the server state.
|
||||
await expect(checkbox).not.toBeChecked();
|
||||
expect(cap.patched).toBe(false);
|
||||
});
|
||||
|
||||
test('a failed toggle reverts the checkbox to the server state', async ({ page }) => {
|
||||
const cap: Captured = { patched: false };
|
||||
await mockAdmin(page, cap, { patchStatus: 500 });
|
||||
page.on('dialog', (d) => d.accept());
|
||||
await page.setViewportSize(DESKTOP);
|
||||
await page.goto('/admin/users');
|
||||
|
||||
const bobRow = page.locator('tr', { hasText: 'bob' });
|
||||
const checkbox = bobRow.getByLabel('admin');
|
||||
await expect(checkbox).not.toBeChecked();
|
||||
|
||||
await checkbox.click();
|
||||
|
||||
// The PATCH was attempted and failed → the checkbox must snap back to
|
||||
// the actual (still non-admin) server state rather than stay flipped.
|
||||
await expect.poll(() => cap.patched).toBe(true);
|
||||
await expect(checkbox).not.toBeChecked();
|
||||
});
|
||||
});
|
||||
73
frontend/e2e/bookmarks-load-more.spec.ts
Normal file
73
frontend/e2e/bookmarks-load-more.spec.ts
Normal file
@@ -0,0 +1,73 @@
|
||||
import { test, expect, type Page } from './fixtures';
|
||||
|
||||
// The bookmarks list pages the results: the first 50 load with the page, and a
|
||||
// "Load more" button fetches the next page (offset-based) and appends it, so
|
||||
// bookmarks past the first page are reachable rather than silently truncated.
|
||||
|
||||
function bookmark(i: number) {
|
||||
return {
|
||||
id: `bm-${i}`,
|
||||
manga_id: `m-${i}`,
|
||||
manga_title: `Manga ${i}`,
|
||||
manga_cover_image_path: null,
|
||||
chapter_id: null,
|
||||
chapter_number: null,
|
||||
page: null,
|
||||
created_at: '2026-01-01T00:00:00Z'
|
||||
};
|
||||
}
|
||||
|
||||
async function authed(page: Page) {
|
||||
await page.route('**/api/v1/auth/config', (r) =>
|
||||
r.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ self_register_enabled: true, private_mode: false })
|
||||
})
|
||||
);
|
||||
await page.route('**/api/v1/auth/me', (r) =>
|
||||
r.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({
|
||||
user: { id: 'u1', username: 'reader', created_at: '2026-01-01T00:00:00Z', is_admin: false }
|
||||
})
|
||||
})
|
||||
);
|
||||
await page.route('**/api/v1/auth/me/preferences', (r) =>
|
||||
r.fulfill({ status: 200, contentType: 'application/json', body: '{}' })
|
||||
);
|
||||
await page.route('**/api/v1/files/**', (r) => r.fulfill({ status: 200, body: '' }));
|
||||
}
|
||||
|
||||
test('bookmarks list loads more pages on demand', async ({ page }) => {
|
||||
await authed(page);
|
||||
// total = 60: first page 50 (offset 0), second page 10 (offset 50).
|
||||
await page.route('**/api/v1/me/bookmarks*', (r) => {
|
||||
const url = new URL(r.request().url());
|
||||
const offset = Number(url.searchParams.get('offset') ?? '0');
|
||||
const items =
|
||||
offset === 0
|
||||
? Array.from({ length: 50 }, (_, i) => bookmark(i))
|
||||
: Array.from({ length: 10 }, (_, i) => bookmark(50 + i));
|
||||
return r.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ items, page: { limit: 50, offset, total: 60 } })
|
||||
});
|
||||
});
|
||||
|
||||
await page.goto('/bookmarks');
|
||||
|
||||
// First page rendered; more remain so the button shows.
|
||||
await expect(page.getByText('Manga 0')).toBeVisible();
|
||||
await expect(page.getByText('Manga 49')).toBeVisible();
|
||||
await expect(page.getByText('Manga 50')).toHaveCount(0);
|
||||
const loadMore = page.getByTestId('load-more');
|
||||
await expect(loadMore).toBeVisible();
|
||||
|
||||
// Load the rest; the button disappears once everything is shown.
|
||||
await loadMore.click();
|
||||
await expect(page.getByText('Manga 59')).toBeVisible();
|
||||
await expect(loadMore).toHaveCount(0);
|
||||
});
|
||||
74
frontend/e2e/collections-load-more.spec.ts
Normal file
74
frontend/e2e/collections-load-more.spec.ts
Normal file
@@ -0,0 +1,74 @@
|
||||
import { test, expect, type Page } from './fixtures';
|
||||
|
||||
// The collections grid pages results: a "Load more" button fetches the next
|
||||
// page (offset-based) and appends it, so collections past the first page are
|
||||
// reachable rather than silently truncated at the old 200 cap.
|
||||
|
||||
function collection(i: number) {
|
||||
return {
|
||||
id: `col-${i}`,
|
||||
user_id: 'u1',
|
||||
name: `Collection ${i}`,
|
||||
description: null,
|
||||
created_at: '2026-01-01T00:00:00Z',
|
||||
updated_at: '2026-01-01T00:00:00Z',
|
||||
manga_count: 0,
|
||||
sample_covers: []
|
||||
};
|
||||
}
|
||||
|
||||
async function authed(page: Page) {
|
||||
await page.route('**/api/v1/auth/config', (r) =>
|
||||
r.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ self_register_enabled: true, private_mode: false })
|
||||
})
|
||||
);
|
||||
await page.route('**/api/v1/auth/me', (r) =>
|
||||
r.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({
|
||||
user: { id: 'u1', username: 'reader', created_at: '2026-01-01T00:00:00Z', is_admin: false }
|
||||
})
|
||||
})
|
||||
);
|
||||
await page.route('**/api/v1/auth/me/preferences', (r) =>
|
||||
r.fulfill({ status: 200, contentType: 'application/json', body: '{}' })
|
||||
);
|
||||
await page.route('**/api/v1/me/bookmarks*', (r) =>
|
||||
r.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ items: [], page: { limit: 50, offset: 0, total: 0 } })
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
test('collections grid loads more pages on demand', async ({ page }) => {
|
||||
await authed(page);
|
||||
await page.route('**/api/v1/me/collections*', (r) => {
|
||||
const offset = Number(new URL(r.request().url()).searchParams.get('offset') ?? '0');
|
||||
const items =
|
||||
offset === 0
|
||||
? Array.from({ length: 60 }, (_, i) => collection(i))
|
||||
: Array.from({ length: 5 }, (_, i) => collection(60 + i));
|
||||
return r.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ items, page: { limit: 60, offset, total: 65 } })
|
||||
});
|
||||
});
|
||||
|
||||
await page.goto('/collections');
|
||||
|
||||
await expect(page.getByText('Collection 0')).toBeVisible();
|
||||
await expect(page.getByText('Collection 64')).toHaveCount(0);
|
||||
const loadMore = page.getByTestId('load-more');
|
||||
await expect(loadMore).toBeVisible();
|
||||
|
||||
await loadMore.click();
|
||||
await expect(page.getByText('Collection 64')).toBeVisible();
|
||||
await expect(loadMore).toHaveCount(0);
|
||||
});
|
||||
42
frontend/e2e/error-boundary.spec.ts
Normal file
42
frontend/e2e/error-boundary.spec.ts
Normal file
@@ -0,0 +1,42 @@
|
||||
import { test, expect, type Page } from './fixtures';
|
||||
|
||||
// The root +error.svelte catches load() errors that a page rethrows rather than
|
||||
// capturing in-band (e.g. the profile overview). Without it these fell through
|
||||
// to SvelteKit's bare default error page — or blanked entirely on a raw
|
||||
// TypeError. Drive the profile loader into a failure and assert the boundary.
|
||||
|
||||
async function authed(page: Page) {
|
||||
await page.route('**/api/v1/auth/config', (r) =>
|
||||
r.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ self_register_enabled: true, private_mode: false })
|
||||
})
|
||||
);
|
||||
await page.route('**/api/v1/auth/me', (r) =>
|
||||
r.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({
|
||||
user: { id: 'u1', username: 'reader', created_at: '2026-01-01T00:00:00Z', is_admin: false }
|
||||
})
|
||||
})
|
||||
);
|
||||
await page.route('**/api/v1/auth/me/preferences', (r) =>
|
||||
r.fulfill({ status: 200, contentType: 'application/json', body: '{}' })
|
||||
);
|
||||
}
|
||||
|
||||
test('profile: a network failure renders the error boundary, not a blank page', async ({
|
||||
page
|
||||
}) => {
|
||||
await authed(page);
|
||||
// Profile's load rethrows non-401 errors; a raw abort is the TypeError case
|
||||
// the client now normalises to an ApiError so it reaches the boundary.
|
||||
await page.route('**/api/v1/me/bookmarks*', (r) => r.abort());
|
||||
await page.route('**/api/v1/me/collections*', (r) => r.abort());
|
||||
|
||||
await page.goto('/profile');
|
||||
await expect(page.getByTestId('error-boundary')).toBeVisible();
|
||||
await expect(page.getByRole('button', { name: 'Try again' })).toBeVisible();
|
||||
});
|
||||
94
frontend/e2e/profile-tokens.spec.ts
Normal file
94
frontend/e2e/profile-tokens.spec.ts
Normal file
@@ -0,0 +1,94 @@
|
||||
import { test, expect, type Page } from './fixtures';
|
||||
|
||||
// E2E for the bot API token management page: list existing tokens, create one
|
||||
// (the raw bearer is shown once), and revoke one.
|
||||
|
||||
const existing = {
|
||||
id: 't1111111-1111-1111-1111-111111111111',
|
||||
user_id: 'u1',
|
||||
name: 'existing-bot',
|
||||
created_at: '2026-01-01T00:00:00Z',
|
||||
last_used_at: null,
|
||||
expires_at: null
|
||||
};
|
||||
|
||||
type Captured = { created: Record<string, unknown> | null; deleted: string | null };
|
||||
|
||||
async function mockTokens(page: Page, cap: Captured) {
|
||||
await page.route('**/api/v1/auth/config', (r) =>
|
||||
r.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ self_register_enabled: true, private_mode: false })
|
||||
})
|
||||
);
|
||||
await page.route('**/api/v1/auth/me', (r) =>
|
||||
r.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({
|
||||
user: { id: 'u1', username: 'reader', created_at: '2026-01-01T00:00:00Z', is_admin: false }
|
||||
})
|
||||
})
|
||||
);
|
||||
await page.route('**/api/v1/auth/me/preferences', (r) =>
|
||||
r.fulfill({ status: 200, contentType: 'application/json', body: '{}' })
|
||||
);
|
||||
await page.route('**/api/v1/me/bookmarks*', (r) =>
|
||||
r.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ items: [], page: { limit: 50, offset: 0, total: 0 } })
|
||||
})
|
||||
);
|
||||
|
||||
// Specific id-scoped DELETE first (last-match-wins ordering).
|
||||
await page.route('**/api/v1/auth/tokens/*', async (r) => {
|
||||
cap.deleted = r.request().url().split('/').pop() ?? null;
|
||||
await r.fulfill({ status: 204, body: '' });
|
||||
});
|
||||
await page.route('**/api/v1/auth/tokens', async (r) => {
|
||||
if (r.request().method() === 'POST') {
|
||||
cap.created = r.request().postDataJSON();
|
||||
await r.fulfill({
|
||||
status: 201,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({
|
||||
id: 't2',
|
||||
user_id: 'u1',
|
||||
name: cap.created?.name,
|
||||
created_at: '2026-03-01T00:00:00Z',
|
||||
last_used_at: null,
|
||||
expires_at: null,
|
||||
bearer: 'secret-raw-bearer-xyz'
|
||||
})
|
||||
});
|
||||
return;
|
||||
}
|
||||
await r.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ items: [existing] })
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
test('lists, creates (shows the bearer once), and revokes tokens', async ({ page }) => {
|
||||
const cap: Captured = { created: null, deleted: null };
|
||||
await mockTokens(page, cap);
|
||||
await page.goto('/profile/tokens');
|
||||
|
||||
// Existing token is listed.
|
||||
await expect(page.getByTestId(`token-row-${existing.id}`)).toContainText('existing-bot');
|
||||
|
||||
// Create a token — the raw bearer is revealed exactly once.
|
||||
await page.getByTestId('token-name').fill('new-bot');
|
||||
await page.getByRole('button', { name: 'Create token' }).click();
|
||||
await expect(page.getByTestId('token-fresh')).toContainText('secret-raw-bearer-xyz');
|
||||
await expect.poll(() => cap.created).toEqual({ name: 'new-bot' });
|
||||
|
||||
// Revoke the existing token (confirm dialog accepted).
|
||||
page.on('dialog', (d) => d.accept());
|
||||
await page.getByTestId(`token-revoke-${existing.id}`).click();
|
||||
await expect.poll(() => cap.deleted).toBe(existing.id);
|
||||
});
|
||||
@@ -170,45 +170,33 @@ test.describe('reader ?page=N deep link', () => {
|
||||
|
||||
await expect(page.getByTestId('reader-continuous')).toBeVisible();
|
||||
|
||||
// Pages 1..=4 (1-indexed in the testid, 0-indexed in the
|
||||
// template; initialIndex = 3 means we eager-load 0..=3, i.e.
|
||||
// testids 1..4). Without this guard, page 2+ would be lazy
|
||||
// and their 0×0 placeholders would let the scroll target
|
||||
// appear far above its final position.
|
||||
for (const n of [1, 2, 3, 4]) {
|
||||
// The whole chapter is eager, so the scroll target (page 4) and
|
||||
// every page before it have settled heights before the
|
||||
// scroll-to-`?page=N` effect runs — the target can't be pushed
|
||||
// past the viewport by later pages loading in.
|
||||
for (const n of [1, 2, 3, 4, 5, 6]) {
|
||||
await expect(page.getByTestId(`reader-page-${n}`)).toHaveAttribute(
|
||||
'loading',
|
||||
'eager'
|
||||
);
|
||||
}
|
||||
// Pages beyond the target stay lazy.
|
||||
for (const n of [5, 6]) {
|
||||
await expect(page.getByTestId(`reader-page-${n}`)).toHaveAttribute(
|
||||
'loading',
|
||||
'lazy'
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test('continuous mode: no ?page= eager-loads only the first two', async ({
|
||||
test('continuous mode: eager-loads the whole chapter up front', async ({
|
||||
page
|
||||
}) => {
|
||||
await mockReader(page, 'continuous');
|
||||
await page.goto(`/manga/${mangaId}/chapter/${chapterId}`);
|
||||
|
||||
await expect(page.getByTestId('reader-continuous')).toBeVisible();
|
||||
await expect(page.getByTestId('reader-page-1')).toHaveAttribute(
|
||||
'loading',
|
||||
'eager'
|
||||
);
|
||||
await expect(page.getByTestId('reader-page-2')).toHaveAttribute(
|
||||
'loading',
|
||||
'eager'
|
||||
);
|
||||
await expect(page.getByTestId('reader-page-3')).toHaveAttribute(
|
||||
'loading',
|
||||
'lazy'
|
||||
);
|
||||
// Every page is eager — the whole current chapter is preloaded so
|
||||
// pages don't pop in / reflow as the reader scrolls onto them.
|
||||
for (const n of [1, 2, 3, 4, 5, 6]) {
|
||||
await expect(page.getByTestId(`reader-page-${n}`)).toHaveAttribute(
|
||||
'loading',
|
||||
'eager'
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test('single mode: ?page=N opens at the requested page', async ({ page }) => {
|
||||
|
||||
119
frontend/e2e/reader-preload-window.spec.ts
Normal file
119
frontend/e2e/reader-preload-window.spec.ts
Normal file
@@ -0,0 +1,119 @@
|
||||
import { test, expect, type Page } from './fixtures';
|
||||
|
||||
// Continuous mode eager-loads a window of pages ahead of the reading position
|
||||
// (the deep-linked page + a lead, min 6 pages), so those are fetched and sized
|
||||
// up front. Pages beyond the window are `loading="lazy"` — the browser fetches
|
||||
// them natively as the reader scrolls near, so opening a long chapter (up to
|
||||
// 2000 pages) doesn't fire every request or decode every image at once. Unloaded
|
||||
// lazy pages reserve height so they stay distinct scroll targets for the
|
||||
// progress IntersectionObserver.
|
||||
|
||||
const MANGA_ID = 'm1';
|
||||
const CH1 = 'c1';
|
||||
// Enough pages that the last one sits tens of thousands of px below the fold —
|
||||
// well beyond any browser's lazy-load distance threshold — so "far pages don't
|
||||
// prefetch at the top" is unambiguous rather than threshold-brittle.
|
||||
const PAGE_COUNT = 40;
|
||||
|
||||
function pages(prefix: string, n: number) {
|
||||
return Array.from({ length: n }, (_, i) => ({
|
||||
id: `${prefix}p${i + 1}`,
|
||||
chapter_id: prefix,
|
||||
page_number: i + 1,
|
||||
storage_key: `${prefix}/${i + 1}`,
|
||||
content_type: 'image/svg+xml'
|
||||
}));
|
||||
}
|
||||
|
||||
function chapter(id: string, number: number, count: number) {
|
||||
return { id, manga_id: MANGA_ID, number, title: null, page_count: count, created_at: '2026-01-01T00:00:00Z', size_bytes: 0 };
|
||||
}
|
||||
|
||||
async function mockReader(page: Page) {
|
||||
await page.route('**/api/v1/**', async (route) => {
|
||||
const { pathname } = new URL(route.request().url());
|
||||
const json = (status: number, body: unknown) =>
|
||||
route.fulfill({ status, contentType: 'application/json', body: JSON.stringify(body) });
|
||||
|
||||
if (pathname.includes('/files/')) {
|
||||
// A viewBox + rect gives the <img> a real intrinsic size so pages
|
||||
// get real (tall) heights — offscreen pages then stay below the
|
||||
// fold instead of collapsing to 0 and piling into the viewport.
|
||||
return route.fulfill({ status: 200, contentType: 'image/svg+xml', body: '<svg xmlns="http://www.w3.org/2000/svg" width="800" height="1200" viewBox="0 0 800 1200"><rect width="800" height="1200" fill="#888"/></svg>' });
|
||||
}
|
||||
if (pathname.endsWith('/auth/config')) return json(200, { self_register_enabled: true, private_mode: false });
|
||||
if (pathname.endsWith('/auth/me')) return json(401, { error: { code: 'unauthenticated', message: 'no' } });
|
||||
if (pathname.endsWith('/auth/me/preferences')) return json(401, { error: { code: 'unauthenticated', message: 'no' } });
|
||||
if (pathname.endsWith(`/chapters/${CH1}/pages`)) return json(200, { pages: pages(CH1, PAGE_COUNT) });
|
||||
if (pathname.endsWith(`/chapters/${CH1}`)) return json(200, chapter(CH1, 1, PAGE_COUNT));
|
||||
if (pathname.includes(`/mangas/${MANGA_ID}/chapters`)) {
|
||||
return json(200, { items: [chapter(CH1, 1, PAGE_COUNT)], page: { limit: 200, offset: 0, total: 1 } });
|
||||
}
|
||||
if (pathname.endsWith(`/mangas/${MANGA_ID}`)) return json(200, { id: MANGA_ID, title: 'Berserk', status: 'ongoing', alt_titles: [], description: null, cover_image_path: null, created_at: '2026-01-01T00:00:00Z', updated_at: '2026-01-01T00:00:00Z', authors: [], genres: [], tags: [], content_warnings: [], chapter_storage_bytes: 0 });
|
||||
if (pathname.includes('/me/read-progress')) return json(401, { error: { code: 'unauthenticated', message: 'no' } });
|
||||
return json(503, { error: { code: 'e2e_unmocked', message: pathname } });
|
||||
});
|
||||
}
|
||||
|
||||
async function goContinuous(page: Page, path: string) {
|
||||
await page.addInitScript(() => localStorage.setItem('mangalord-reader-mode', 'continuous'));
|
||||
await page.goto(path);
|
||||
await expect(page.getByTestId('reader-continuous')).toBeVisible();
|
||||
}
|
||||
|
||||
// EAGER_MIN in the reader: opening at page 1 (index 0) eager-loads pages 1..6.
|
||||
const EAGER_THROUGH = 6;
|
||||
|
||||
test('eager-loads a leading window and leaves far pages lazy', async ({ page }) => {
|
||||
await mockReader(page);
|
||||
await goContinuous(page, `/manga/${MANGA_ID}/chapter/${CH1}`);
|
||||
|
||||
// The leading window is eager — warmed up front so the reader doesn't
|
||||
// pop-in / reflow as it scrolls onto the next few pages.
|
||||
for (let n = 1; n <= EAGER_THROUGH; n++) {
|
||||
await expect(page.getByTestId(`reader-page-${n}`)).toHaveAttribute('loading', 'eager');
|
||||
}
|
||||
// Pages beyond the window stay lazy — not fetched until scrolled near, so
|
||||
// a 2000-page chapter doesn't fire 2000 requests / decodes at once.
|
||||
for (let n = EAGER_THROUGH + 1; n <= PAGE_COUNT; n++) {
|
||||
await expect(page.getByTestId(`reader-page-${n}`)).toHaveAttribute('loading', 'lazy');
|
||||
}
|
||||
});
|
||||
|
||||
test('a windowed page is warmed up front without scrolling', async ({ page }) => {
|
||||
await mockReader(page);
|
||||
await goContinuous(page, `/manga/${MANGA_ID}/chapter/${CH1}`);
|
||||
|
||||
// A page inside the eager window finishes loading without any scroll — the
|
||||
// preload payoff (no pop-in as the reader advances a few pages) is kept.
|
||||
// (We can't assert the far lazy pages stay *unfetched* here: headless
|
||||
// Chromium prefetches `loading="lazy"` images regardless of distance. The
|
||||
// attribute partition above is the mechanism that defers them in the real
|
||||
// OOM target — mobile Safari. Playwright can only pin the attributes.)
|
||||
await expect(async () => {
|
||||
const loaded = await page
|
||||
.getByTestId(`reader-page-${EAGER_THROUGH}`)
|
||||
.evaluate((img: HTMLImageElement) => img.complete && img.naturalWidth > 0);
|
||||
expect(loaded).toBe(true);
|
||||
}).toPass();
|
||||
expect(await page.evaluate(() => window.scrollY)).toBe(0);
|
||||
});
|
||||
|
||||
test('deep-link ?page=N keeps the target inside the eager window and lands the scroll', async ({
|
||||
page
|
||||
}) => {
|
||||
await mockReader(page);
|
||||
// initialIndex = 19 → eagerThrough = 22, so pages 1..23 are eager (the
|
||||
// target and everything above it settle height before the scroll fires).
|
||||
await goContinuous(page, `/manga/${MANGA_ID}/chapter/${CH1}?page=20`);
|
||||
|
||||
await expect(page.getByTestId('reader-page-20')).toHaveAttribute('loading', 'eager');
|
||||
await expect(page.getByTestId('reader-page-22')).toHaveAttribute('loading', 'eager');
|
||||
// A page far past the widened window is still lazy.
|
||||
await expect(page.getByTestId(`reader-page-${PAGE_COUNT}`)).toHaveAttribute('loading', 'lazy');
|
||||
|
||||
// The scroll-to-`?page=N` effect lands the target in view (proves the
|
||||
// eager window settled the heights above it — a short-landing scroll would
|
||||
// leave page 20 out of the viewport).
|
||||
await expect(page.getByTestId('reader-page-20')).toBeInViewport();
|
||||
});
|
||||
56
frontend/e2e/scroll-lock.spec.ts
Normal file
56
frontend/e2e/scroll-lock.spec.ts
Normal file
@@ -0,0 +1,56 @@
|
||||
import { test, expect, type Page } from './fixtures';
|
||||
|
||||
// Opening an overlay (Sheet/Modal) locks background scroll so the page behind
|
||||
// it can't scroll under the overlay; closing restores it.
|
||||
|
||||
const MOBILE = { width: 390, height: 780 } as const;
|
||||
|
||||
async function authed(page: Page) {
|
||||
await page.route('**/api/v1/auth/config', (r) =>
|
||||
r.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ self_register_enabled: true, private_mode: false })
|
||||
})
|
||||
);
|
||||
await page.route('**/api/v1/auth/me', (r) =>
|
||||
r.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({
|
||||
user: { id: 'u1', username: 'reader', created_at: '2026-01-01T00:00:00Z', is_admin: false }
|
||||
})
|
||||
})
|
||||
);
|
||||
await page.route('**/api/v1/auth/me/preferences', (r) =>
|
||||
r.fulfill({ status: 200, contentType: 'application/json', body: '{}' })
|
||||
);
|
||||
await page.route('**/api/v1/me/bookmarks*', (r) =>
|
||||
r.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ items: [], page: { limit: 50, offset: 0, total: 0 } })
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
const bodyOverflow = (page: Page) => page.evaluate(() => document.body.style.overflow);
|
||||
|
||||
test('opening the account password sheet locks body scroll, closing restores it', async ({
|
||||
page
|
||||
}) => {
|
||||
await authed(page);
|
||||
await page.setViewportSize(MOBILE);
|
||||
await page.goto('/profile/account');
|
||||
|
||||
expect(await bodyOverflow(page)).not.toBe('hidden');
|
||||
|
||||
await page.getByTestId('account-row-change-password').click();
|
||||
await expect(page.getByTestId('password-sheet')).toBeVisible();
|
||||
expect(await bodyOverflow(page)).toBe('hidden');
|
||||
|
||||
// Close via Escape; scroll is released.
|
||||
await page.keyboard.press('Escape');
|
||||
await expect(page.getByTestId('password-sheet')).toBeHidden();
|
||||
expect(await bodyOverflow(page)).not.toBe('hidden');
|
||||
});
|
||||
37
frontend/e2e/security-headers.spec.ts
Normal file
37
frontend/e2e/security-headers.spec.ts
Normal file
@@ -0,0 +1,37 @@
|
||||
import { test, expect } from './fixtures';
|
||||
|
||||
// Defense-in-depth response headers on document navigations (T4 in the security
|
||||
// audit). The CSP is emitted by SvelteKit's kit.csp config; the clickjacking /
|
||||
// referrer / permissions headers by hooks.server.ts. We assert on the raw
|
||||
// response of a document navigation, and separately confirm the inline theme
|
||||
// script still executes under the CSP (its sha256 is allowlisted) by checking
|
||||
// the data-theme attribute it sets — a CSP block would leave it unset.
|
||||
|
||||
test('document responses carry the security headers', async ({ page }) => {
|
||||
const response = await page.goto('/');
|
||||
expect(response, 'navigation returned a response').not.toBeNull();
|
||||
const headers = response!.headers();
|
||||
|
||||
// Clickjacking: both the legacy header and the CSP directive.
|
||||
expect(headers['x-frame-options']).toBe('DENY');
|
||||
expect(headers['content-security-policy']).toContain("frame-ancestors 'none'");
|
||||
// Script-injection surface reduction.
|
||||
expect(headers['content-security-policy']).toContain("object-src 'none'");
|
||||
expect(headers['content-security-policy']).toContain('script-src');
|
||||
// The non-CSP hardening headers.
|
||||
expect(headers['referrer-policy']).toBe('strict-origin-when-cross-origin');
|
||||
expect(headers['x-content-type-options']).toBe('nosniff');
|
||||
expect(headers['permissions-policy']).toContain('geolocation=()');
|
||||
});
|
||||
|
||||
test('the inline theme script executes under the CSP (hash is allowlisted)', async ({
|
||||
page
|
||||
}) => {
|
||||
// If the theme script were CSP-blocked, data-theme would never be set.
|
||||
// Its presence proves the allowlisted sha256 matches the served script.
|
||||
await page.goto('/');
|
||||
const theme = await page.evaluate(() =>
|
||||
document.documentElement.getAttribute('data-theme')
|
||||
);
|
||||
expect(theme === 'light' || theme === 'dark').toBe(true);
|
||||
});
|
||||
59
frontend/e2e/unsaved-guard.spec.ts
Normal file
59
frontend/e2e/unsaved-guard.spec.ts
Normal file
@@ -0,0 +1,59 @@
|
||||
import { test, expect, type Page } from './fixtures';
|
||||
|
||||
// Leaving a form with unsaved work (e.g. a half-filled upload) must prompt so a
|
||||
// misclicked nav link doesn't silently discard everything.
|
||||
|
||||
const DESKTOP = { width: 1280, height: 720 } as const;
|
||||
|
||||
async function authed(page: Page) {
|
||||
await page.route('**/api/v1/auth/config', (r) =>
|
||||
r.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ self_register_enabled: true, private_mode: false })
|
||||
})
|
||||
);
|
||||
await page.route('**/api/v1/auth/me', (r) =>
|
||||
r.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({
|
||||
user: { id: 'u1', username: 'uploader', created_at: '2026-01-01T00:00:00Z', is_admin: false }
|
||||
})
|
||||
})
|
||||
);
|
||||
await page.route('**/api/v1/auth/me/preferences', (r) =>
|
||||
r.fulfill({ status: 200, contentType: 'application/json', body: '{}' })
|
||||
);
|
||||
await page.route('**/api/v1/genres', (r) =>
|
||||
r.fulfill({ status: 200, contentType: 'application/json', body: '[]' })
|
||||
);
|
||||
await page.route('**/api/v1/me/bookmarks*', (r) =>
|
||||
r.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ items: [], page: { limit: 50, offset: 0, total: 0 } })
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
test('upload form prompts before navigating away with unsaved changes', async ({ page }) => {
|
||||
await authed(page);
|
||||
await page.setViewportSize(DESKTOP);
|
||||
await page.goto('/upload');
|
||||
|
||||
// Make the form dirty.
|
||||
await page.getByLabel(/Title/).fill('WIP Manga');
|
||||
|
||||
// Cancel the confirm → navigation is blocked, we stay put with our input.
|
||||
let dialogSeen = false;
|
||||
page.once('dialog', (d) => {
|
||||
dialogSeen = true;
|
||||
d.dismiss();
|
||||
});
|
||||
await page.getByRole('link', { name: 'Bookmarks' }).click();
|
||||
|
||||
await expect.poll(() => dialogSeen).toBe(true);
|
||||
await expect(page).toHaveURL(/\/upload/);
|
||||
await expect(page.getByLabel(/Title/)).toHaveValue('WIP Manga');
|
||||
});
|
||||
104
frontend/e2e/upload-retry.spec.ts
Normal file
104
frontend/e2e/upload-retry.spec.ts
Normal file
@@ -0,0 +1,104 @@
|
||||
import { test, expect, type Page } from './fixtures';
|
||||
|
||||
// A partial upload failure (manga created, a chapter fails) must NOT create a
|
||||
// duplicate manga when the user retries — the second submit reuses the created
|
||||
// manga and only re-sends the failed chapter.
|
||||
|
||||
const userFixture = {
|
||||
id: 'u1',
|
||||
username: 'uploader',
|
||||
created_at: '2026-01-01T00:00:00Z',
|
||||
is_admin: false
|
||||
};
|
||||
|
||||
const manga = {
|
||||
id: 'm9',
|
||||
title: 'Retry Saga',
|
||||
status: 'ongoing',
|
||||
description: null,
|
||||
authors: [],
|
||||
alt_titles: [],
|
||||
genres: [],
|
||||
cover_image_path: null,
|
||||
created_at: '2026-01-01T00:00:00Z',
|
||||
updated_at: '2026-01-01T00:00:00Z'
|
||||
};
|
||||
|
||||
async function stub(page: Page, counters: { mangaPosts: number; chapterPosts: number }) {
|
||||
await page.route('**/api/v1/auth/config', (r) =>
|
||||
r.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify({ self_register_enabled: true, private_mode: false }) })
|
||||
);
|
||||
await page.route('**/api/v1/auth/me', (r) =>
|
||||
r.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify({ user: userFixture }) })
|
||||
);
|
||||
await page.route('**/api/v1/auth/me/preferences', (r) =>
|
||||
r.fulfill({ status: 200, contentType: 'application/json', body: '{}' })
|
||||
);
|
||||
await page.route('**/api/v1/genres', (r) => r.fulfill({ status: 200, contentType: 'application/json', body: '[]' }));
|
||||
|
||||
await page.route('**/api/v1/mangas', (r) => {
|
||||
if (r.request().method() === 'POST') {
|
||||
counters.mangaPosts += 1;
|
||||
return r.fulfill({ status: 201, contentType: 'application/json', body: JSON.stringify(manga) });
|
||||
}
|
||||
return r.fallback();
|
||||
});
|
||||
// Chapter POST: fail the first attempt, succeed on the retry.
|
||||
await page.route('**/api/v1/mangas/m9/chapters', (r) => {
|
||||
if (r.request().method() === 'POST') {
|
||||
counters.chapterPosts += 1;
|
||||
if (counters.chapterPosts === 1) {
|
||||
return r.fulfill({
|
||||
status: 500,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ error: { code: 'internal_error', message: 'boom' } })
|
||||
});
|
||||
}
|
||||
return r.fulfill({
|
||||
status: 201,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ id: 'c1', manga_id: 'm9', number: 1, title: null, page_count: 1, created_at: '2026-01-01T00:00:00Z' })
|
||||
});
|
||||
}
|
||||
return r.fallback();
|
||||
});
|
||||
// Post-success destination + its data.
|
||||
await page.route('**/api/v1/mangas/m9', (r) =>
|
||||
r.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify(manga) })
|
||||
);
|
||||
await page.route('**/api/v1/mangas/m9/chapters?*', (r) =>
|
||||
r.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify({ items: [], page: { limit: 200, offset: 0, total: 0 } }) })
|
||||
);
|
||||
await page.route('**/api/v1/me/bookmarks*', (r) =>
|
||||
r.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify({ items: [], page: { limit: 50, offset: 0, total: 0 } }) })
|
||||
);
|
||||
await page.route('**/api/v1/me/read-progress/m9', (r) =>
|
||||
r.fulfill({ status: 404, contentType: 'application/json', body: JSON.stringify({ error: { code: 'not_found', message: 'x' } }) })
|
||||
);
|
||||
}
|
||||
|
||||
test('retrying a partially-failed upload does not create a duplicate manga', async ({ page }) => {
|
||||
const counters = { mangaPosts: 0, chapterPosts: 0 };
|
||||
await stub(page, counters);
|
||||
|
||||
await page.goto('/upload');
|
||||
await page.getByTestId('manga-title').fill('Retry Saga');
|
||||
await page.getByTestId('add-chapter').click();
|
||||
const png = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
|
||||
await page
|
||||
.getByTestId('staged-chapter-pages-input')
|
||||
.setInputFiles([{ name: 'p.png', mimeType: 'image/png', buffer: png }]);
|
||||
|
||||
// First submit: manga created, chapter fails → the retry note + relabeled button.
|
||||
await page.getByTestId('manga-submit').click();
|
||||
await expect(page.getByTestId('manga-success')).toBeVisible();
|
||||
await expect(page.getByTestId('manga-created-note')).toBeVisible();
|
||||
await expect(page.getByTestId('manga-submit')).toHaveText('Retry failed chapters');
|
||||
expect(counters.mangaPosts).toBe(1);
|
||||
|
||||
// Retry: no second manga POST, chapter re-sent, navigate to the manga.
|
||||
await page.getByTestId('manga-submit').click();
|
||||
await expect(page).toHaveURL(/\/manga\/m9$/);
|
||||
expect(counters.mangaPosts).toBe(1);
|
||||
expect(counters.chapterPosts).toBe(2);
|
||||
});
|
||||
16
frontend/package-lock.json
generated
16
frontend/package-lock.json
generated
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "mangalord-frontend",
|
||||
"version": "0.109.1",
|
||||
"version": "0.128.24",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "mangalord-frontend",
|
||||
"version": "0.109.1",
|
||||
"version": "0.128.24",
|
||||
"devDependencies": {
|
||||
"@lucide/svelte": "^1.16.0",
|
||||
"@playwright/test": "^1.48.0",
|
||||
@@ -169,7 +169,6 @@
|
||||
}
|
||||
],
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
},
|
||||
@@ -193,7 +192,6 @@
|
||||
}
|
||||
],
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
@@ -1157,7 +1155,6 @@
|
||||
"integrity": "sha512-mQjlkNo+rJvpln7V2IGY2j99BqhcFbS4UN0AQNKNYfhBAFZTuCDAdW3a1sgf330mvtNvsBXn3HpAhcmvdJTcIQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"@standard-schema/spec": "^1.0.0",
|
||||
"@sveltejs/acorn-typescript": "^1.0.5",
|
||||
@@ -1200,7 +1197,6 @@
|
||||
"integrity": "sha512-0ba1RQ/PHen5FGpdSrW7Y3fAMQjrXantECALeOiOdBdzR5+5vPP6HVZRLmZaQL+W8m++o+haIAKq5qT+MiZ7VA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"@sveltejs/vite-plugin-svelte-inspector": "^3.0.0-next.0||^3.0.0",
|
||||
"debug": "^4.3.7",
|
||||
@@ -1359,7 +1355,6 @@
|
||||
"integrity": "sha512-dyh/xO2Fh5bYrfWaaqGrRQQGkNdmYw6AmaAUvYeUMNTWQtvb796ikLdmTchRmOlOiIJ1TDXfWgVx1QkUlQ6Hew==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"undici-types": "~6.21.0"
|
||||
}
|
||||
@@ -1507,7 +1502,6 @@
|
||||
"integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"bin": {
|
||||
"acorn": "bin/acorn"
|
||||
},
|
||||
@@ -2249,7 +2243,6 @@
|
||||
"integrity": "sha512-8i7LzZj7BF8uplX+ZyOlIz86V6TAsSs+np6m1kpW9u0JWi4z/1t+FzcK1aek+ybTnAC4KhBL4uXCNT0wcUIeCw==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"cssstyle": "^4.1.0",
|
||||
"data-urls": "^5.0.0",
|
||||
@@ -2638,7 +2631,6 @@
|
||||
"integrity": "sha512-WHeFSbZYsPu3+bLoNRUuAO+wavNlocOPf3wSHTP7hcFKVnJeWsYlCDbr3mTS14FCizf9ccIxXA8sGL8zKeQN3g==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"@types/estree": "1.0.8"
|
||||
},
|
||||
@@ -2810,7 +2802,6 @@
|
||||
"integrity": "sha512-ymI5ykLPwIHW839E053FQbI1G+jnRFJEw3Kv5Y4njixVWywQBx+NUFpkkKyk5LIb36Fg9DVXSYpqiGekLD0hyw==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"@jridgewell/remapping": "^2.3.4",
|
||||
"@jridgewell/sourcemap-codec": "^1.5.0",
|
||||
@@ -2997,7 +2988,6 @@
|
||||
"integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0",
|
||||
"peer": true,
|
||||
"bin": {
|
||||
"tsc": "bin/tsc",
|
||||
"tsserver": "bin/tsserver"
|
||||
@@ -3019,7 +3009,6 @@
|
||||
"integrity": "sha512-o5a9xKjbtuhY6Bi5S3+HvbRERmouabWbyUcpXXUA1u+GNUKoROi9byOJ8M0nHbHYHkYICiMlqxkg1KkYmm25Sw==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"esbuild": "^0.21.3",
|
||||
"postcss": "^8.4.43",
|
||||
@@ -3138,7 +3127,6 @@
|
||||
"integrity": "sha512-MSmPM9REYqDGBI8439mA4mWhV5sKmDlBKWIYbA3lRb2PTHACE0mgKwA8yQ2xq9vxDTuk4iPrECBAEW2aoFXY0Q==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"@vitest/expect": "2.1.9",
|
||||
"@vitest/mocker": "2.1.9",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "mangalord-frontend",
|
||||
"version": "0.122.1",
|
||||
"version": "0.128.25",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
|
||||
25
frontend/src/csp-theme-hash.test.ts
Normal file
25
frontend/src/csp-theme-hash.test.ts
Normal file
@@ -0,0 +1,25 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { join } from 'node:path';
|
||||
import { THEME_SCRIPT_HASH } from '../csp-config.js';
|
||||
|
||||
// The theme-flash-prevention <script> in app.html runs inline, so the CSP
|
||||
// script-src must allow it by hash. SvelteKit's kit.csp hash mode does NOT
|
||||
// cover app.html template scripts (only what it injects into %sveltekit.head%),
|
||||
// so we pin the hash by hand in svelte.config.js. This test recomputes the hash
|
||||
// from the actual app.html bytes and fails if THEME_SCRIPT_HASH drifted — i.e.
|
||||
// someone edited the theme script without updating the CSP, which would silently
|
||||
// CSP-block it (theme flash returns, only a console error to show for it).
|
||||
describe('CSP theme-script hash', () => {
|
||||
it('matches the current inline theme script in app.html', () => {
|
||||
// vitest runs with cwd at the frontend package root.
|
||||
const html = readFileSync(join(process.cwd(), 'src/app.html'), 'utf-8');
|
||||
const match = html.match(/<script>(.*?)<\/script>/s);
|
||||
expect(match, 'app.html must contain an inline <script>').not.toBeNull();
|
||||
const digest = createHash('sha256')
|
||||
.update(match![1], 'utf-8')
|
||||
.digest('base64');
|
||||
expect(THEME_SCRIPT_HASH).toBe(`'sha256-${digest}'`);
|
||||
});
|
||||
});
|
||||
@@ -8,7 +8,9 @@ import {
|
||||
type MockInstance
|
||||
} from 'vitest';
|
||||
import {
|
||||
applySecurityHeaders,
|
||||
handle,
|
||||
setForwardedFor,
|
||||
shouldBypassProxyTimeout,
|
||||
stripHopByHopHeaders
|
||||
} from './hooks.server';
|
||||
@@ -351,3 +353,69 @@ describe('stripHopByHopHeaders', () => {
|
||||
expect(src.get('connection')).toBe('close');
|
||||
});
|
||||
});
|
||||
|
||||
describe('applySecurityHeaders', () => {
|
||||
it('sets the clickjacking and defense-in-depth headers', () => {
|
||||
const h = applySecurityHeaders(new Headers());
|
||||
// Clickjacking: legacy + modern. CSP frame-ancestors is emitted by
|
||||
// SvelteKit's kit.csp config; X-Frame-Options is the belt-and-braces
|
||||
// fallback for older UAs.
|
||||
expect(h.get('x-frame-options')).toBe('DENY');
|
||||
expect(h.get('referrer-policy')).toBe('strict-origin-when-cross-origin');
|
||||
expect(h.get('x-content-type-options')).toBe('nosniff');
|
||||
// Lock down powerful features we never use.
|
||||
const pp = h.get('permissions-policy') ?? '';
|
||||
expect(pp).toContain('camera=()');
|
||||
expect(pp).toContain('microphone=()');
|
||||
expect(pp).toContain('geolocation=()');
|
||||
});
|
||||
|
||||
it('emits HSTS only when the request was secure', () => {
|
||||
const insecure = applySecurityHeaders(new Headers());
|
||||
expect(insecure.get('strict-transport-security')).toBeNull();
|
||||
|
||||
const secure = applySecurityHeaders(new Headers(), { hsts: true });
|
||||
expect(secure.get('strict-transport-security')).toBe(
|
||||
'max-age=63072000; includeSubDomains'
|
||||
);
|
||||
});
|
||||
|
||||
it('does not clobber a header the response already set', () => {
|
||||
// If a downstream route deliberately set a stricter Referrer-Policy,
|
||||
// the helper must not override it.
|
||||
const h = new Headers({ 'referrer-policy': 'no-referrer' });
|
||||
applySecurityHeaders(h);
|
||||
expect(h.get('referrer-policy')).toBe('no-referrer');
|
||||
});
|
||||
|
||||
it('returns the same Headers instance it was given (mutates in place)', () => {
|
||||
const h = new Headers();
|
||||
expect(applySecurityHeaders(h)).toBe(h);
|
||||
});
|
||||
});
|
||||
|
||||
describe('setForwardedFor', () => {
|
||||
it('overrides any client-supplied x-forwarded-for with the real address', () => {
|
||||
const headers = new Headers({ 'x-forwarded-for': '1.2.3.4', 'x-real-ip': '1.2.3.4' });
|
||||
setForwardedFor(headers, '203.0.113.9');
|
||||
expect(headers.get('x-forwarded-for')).toBe('203.0.113.9');
|
||||
// x-real-ip is dropped so it can't be used to spoof either.
|
||||
expect(headers.get('x-real-ip')).toBeNull();
|
||||
});
|
||||
|
||||
it('strips the header entirely when no client address is available', () => {
|
||||
const headers = new Headers({ 'x-forwarded-for': '9.9.9.9' });
|
||||
setForwardedFor(headers, undefined);
|
||||
expect(headers.get('x-forwarded-for')).toBeNull();
|
||||
});
|
||||
|
||||
it('trims whitespace and treats blank as absent', () => {
|
||||
const a = new Headers();
|
||||
setForwardedFor(a, ' 198.51.100.2 ');
|
||||
expect(a.get('x-forwarded-for')).toBe('198.51.100.2');
|
||||
|
||||
const b = new Headers({ 'x-forwarded-for': 'spoofed' });
|
||||
setForwardedFor(b, ' ');
|
||||
expect(b.get('x-forwarded-for')).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -47,6 +47,26 @@ export function stripHopByHopHeaders(src: Headers): Headers {
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Stamp the real client address onto `x-forwarded-for` for the upstream
|
||||
* request so axum can key its per-IP auth rate limiter on the actual client
|
||||
* (the backend only honours this when `AUTH_TRUSTED_PROXY=true`). We
|
||||
* **override** rather than append, and drop any incoming `x-forwarded-for` /
|
||||
* `x-real-ip`, so a browser can't spoof its own IP to dodge the limit — this
|
||||
* proxy is the single trusted hop. A missing/blank address leaves the headers
|
||||
* untouched (backend then falls back to its shared bucket). Exported for
|
||||
* unit-test coverage.
|
||||
*/
|
||||
export function setForwardedFor(headers: Headers, clientAddress: string | undefined): Headers {
|
||||
headers.delete('x-real-ip');
|
||||
if (clientAddress && clientAddress.trim() !== '') {
|
||||
headers.set('x-forwarded-for', clientAddress.trim());
|
||||
} else {
|
||||
headers.delete('x-forwarded-for');
|
||||
}
|
||||
return headers;
|
||||
}
|
||||
|
||||
/**
|
||||
* Cap each proxied request at 5 minutes. The bound exists to surface
|
||||
* a wedged backend (stuck on a slow DB query, deadlocked, etc.) as a
|
||||
@@ -85,11 +105,64 @@ export function shouldBypassProxyTimeout(headers: Headers): boolean {
|
||||
return accept.toLowerCase().includes('text/event-stream');
|
||||
}
|
||||
|
||||
/**
|
||||
* Defense-in-depth response headers for HTML/document responses.
|
||||
*
|
||||
* The Content-Security-Policy itself is emitted by SvelteKit's `kit.csp`
|
||||
* config (see svelte.config.js) so that `script-src` hashes cover both the
|
||||
* inline theme script in app.html AND SvelteKit's own hydration inline
|
||||
* scripts — a hand-rolled `script-src` here would break hydration on the next
|
||||
* build. What CSP can't (or shouldn't) express, we add here:
|
||||
*
|
||||
* - `X-Frame-Options: DENY` — clickjacking guard for UAs predating CSP
|
||||
* `frame-ancestors` (which the kit.csp policy also sets).
|
||||
* - `Referrer-Policy` — don't leak full URLs to cross-origin destinations.
|
||||
* - `X-Content-Type-Options: nosniff` — no MIME sniffing on documents.
|
||||
* - `Permissions-Policy` — deny powerful features the app never uses.
|
||||
*
|
||||
* Each header is only set when absent, so a route that deliberately picked a
|
||||
* stricter value keeps it. Mutates and returns the passed `Headers`. Exported
|
||||
* for unit-test coverage.
|
||||
*/
|
||||
export function applySecurityHeaders(
|
||||
headers: Headers,
|
||||
opts: { hsts?: boolean } = {}
|
||||
): Headers {
|
||||
const defaults: Record<string, string> = {
|
||||
'x-frame-options': 'DENY',
|
||||
'referrer-policy': 'strict-origin-when-cross-origin',
|
||||
'x-content-type-options': 'nosniff',
|
||||
'permissions-policy': 'camera=(), microphone=(), geolocation=(), interest-cohort=()'
|
||||
};
|
||||
// HSTS ONLY when the browser reached us over HTTPS (the caller derives this
|
||||
// from x-forwarded-proto). Sending it over plain HTTP — a dev run or a
|
||||
// misconfigured deploy — would pin the browser to HTTPS for this host for
|
||||
// two years and could lock users out, so it's opt-in per request.
|
||||
if (opts.hsts) {
|
||||
defaults['strict-transport-security'] = 'max-age=63072000; includeSubDomains';
|
||||
}
|
||||
for (const [name, value] of Object.entries(defaults)) {
|
||||
if (!headers.has(name)) headers.set(name, value);
|
||||
}
|
||||
return headers;
|
||||
}
|
||||
|
||||
export const handle: Handle = async ({ event, resolve }) => {
|
||||
if (event.url.pathname.startsWith('/api/')) {
|
||||
const target = `${BACKEND_URL}${event.url.pathname}${event.url.search}`;
|
||||
|
||||
const headers = stripHopByHopHeaders(event.request.headers);
|
||||
// Forward the real client IP for the backend's per-IP auth rate
|
||||
// limiter, overriding any client-supplied value (anti-spoof).
|
||||
// `getClientAddress()` throws if the adapter can't determine it — fall
|
||||
// back to stripping the header so no spoofed value survives.
|
||||
let clientAddress: string | undefined;
|
||||
try {
|
||||
clientAddress = event.getClientAddress();
|
||||
} catch {
|
||||
clientAddress = undefined;
|
||||
}
|
||||
setForwardedFor(headers, clientAddress);
|
||||
|
||||
// AbortController times the upstream fetch out so a backend
|
||||
// wedged on a slow DB query doesn't keep the browser request
|
||||
@@ -168,5 +241,10 @@ export const handle: Handle = async ({ event, resolve }) => {
|
||||
headers: stripHopByHopHeaders(upstream.headers)
|
||||
});
|
||||
}
|
||||
return resolve(event);
|
||||
const response = await resolve(event);
|
||||
// The TLS terminator (reverse proxy) sets x-forwarded-proto; only emit HSTS
|
||||
// when the browser's hop to it was HTTPS.
|
||||
const secure = event.request.headers.get('x-forwarded-proto') === 'https';
|
||||
applySecurityHeaders(response.headers, { hsts: secure });
|
||||
return response;
|
||||
};
|
||||
|
||||
@@ -15,6 +15,7 @@ import {
|
||||
changePassword,
|
||||
createToken,
|
||||
deleteToken,
|
||||
listTokens,
|
||||
getAuthConfig
|
||||
} from './auth';
|
||||
|
||||
@@ -170,6 +171,50 @@ describe('auth api client', () => {
|
||||
expect(url).toMatch(/\/v1\/auth\/tokens$/);
|
||||
});
|
||||
|
||||
it('createToken forwards expires_in_days when provided', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(
|
||||
ok(
|
||||
{
|
||||
id: 't2',
|
||||
user_id: 'user-1',
|
||||
name: 'expiring',
|
||||
created_at: '2026-01-01T00:00:00Z',
|
||||
last_used_at: null,
|
||||
expires_at: '2026-02-01T00:00:00Z',
|
||||
bearer: 'raw'
|
||||
},
|
||||
201
|
||||
)
|
||||
);
|
||||
await createToken('expiring', 30);
|
||||
const init = fetchSpy.mock.calls[0][1] as RequestInit;
|
||||
expect(JSON.parse(init.body as string)).toEqual({ name: 'expiring', expires_in_days: 30 });
|
||||
});
|
||||
|
||||
it('listTokens GETs /v1/auth/tokens and unwraps items', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(
|
||||
ok({
|
||||
items: [
|
||||
{
|
||||
id: 't1',
|
||||
user_id: 'user-1',
|
||||
name: 'ci-bot',
|
||||
created_at: '2026-01-01T00:00:00Z',
|
||||
last_used_at: null,
|
||||
expires_at: null
|
||||
}
|
||||
]
|
||||
})
|
||||
);
|
||||
const tokens = await listTokens();
|
||||
expect(tokens).toHaveLength(1);
|
||||
expect(tokens[0].name).toBe('ci-bot');
|
||||
const url = fetchSpy.mock.calls[0][0] as string;
|
||||
expect(url).toMatch(/\/v1\/auth\/tokens$/);
|
||||
const init = fetchSpy.mock.calls[0][1] as RequestInit;
|
||||
expect(init?.method ?? 'GET').toBe('GET');
|
||||
});
|
||||
|
||||
it('getAuthConfig GETs /v1/auth/config and parses the flag', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(ok({ self_register_enabled: false }));
|
||||
const cfg = await getAuthConfig();
|
||||
|
||||
@@ -92,15 +92,29 @@ export type ApiToken = {
|
||||
name: string;
|
||||
created_at: string;
|
||||
last_used_at: string | null;
|
||||
/** When the token stops authenticating; `null` = never expires. */
|
||||
expires_at: string | null;
|
||||
};
|
||||
|
||||
export type CreatedToken = ApiToken & { bearer: string };
|
||||
|
||||
export async function createToken(name: string): Promise<CreatedToken> {
|
||||
/** The caller's bot tokens, newest first (metadata only — the raw bearer is
|
||||
* shown once at creation and never returned again). */
|
||||
export async function listTokens(): Promise<ApiToken[]> {
|
||||
const res = await request<{ items: ApiToken[] }>('/v1/auth/tokens');
|
||||
return res.items;
|
||||
}
|
||||
|
||||
export async function createToken(
|
||||
name: string,
|
||||
expiresInDays?: number
|
||||
): Promise<CreatedToken> {
|
||||
const payload: { name: string; expires_in_days?: number } = { name };
|
||||
if (expiresInDays !== undefined) payload.expires_in_days = expiresInDays;
|
||||
return request<CreatedToken>('/v1/auth/tokens', {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify({ name })
|
||||
body: JSON.stringify(payload)
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
} from 'vitest';
|
||||
import {
|
||||
listChapters,
|
||||
listAllChapters,
|
||||
getChapter,
|
||||
getChapterPages,
|
||||
createChapter,
|
||||
@@ -142,6 +143,47 @@ describe('chapters api client', () => {
|
||||
});
|
||||
});
|
||||
|
||||
it('listAllChapters pages through the 200-row cap and returns every chapter', async () => {
|
||||
const mk = (n: number) => ({ ...chapterFixture, id: `c${n}`, number: n });
|
||||
const first = Array.from({ length: 200 }, (_, i) => mk(i + 1));
|
||||
const second = Array.from({ length: 50 }, (_, i) => mk(i + 201));
|
||||
fetchSpy
|
||||
.mockResolvedValueOnce(ok({ items: first, page: { limit: 200, offset: 0, total: 250 } }))
|
||||
.mockResolvedValueOnce(ok({ items: second, page: { limit: 200, offset: 200, total: 250 } }));
|
||||
|
||||
const all = await listAllChapters('m1');
|
||||
expect(all).toHaveLength(250);
|
||||
expect(all[249].number).toBe(250);
|
||||
expect(fetchSpy.mock.calls[0][0]).toContain('offset=0');
|
||||
expect(fetchSpy.mock.calls[1][0]).toContain('offset=200');
|
||||
});
|
||||
|
||||
it('listAllChapters keeps paging when the API omits total (total: null)', async () => {
|
||||
// The real /chapters endpoint returns `total: null` on every page
|
||||
// (PagedResponse::new). A full 200-row first page must NOT be mistaken
|
||||
// for the end — otherwise the reader dead-ends past chapter 200.
|
||||
const mk = (n: number) => ({ ...chapterFixture, id: `c${n}`, number: n });
|
||||
const first = Array.from({ length: 200 }, (_, i) => mk(i + 1));
|
||||
const second = Array.from({ length: 50 }, (_, i) => mk(i + 201));
|
||||
fetchSpy
|
||||
.mockResolvedValueOnce(ok({ items: first, page: { limit: 200, offset: 0, total: null } }))
|
||||
.mockResolvedValueOnce(ok({ items: second, page: { limit: 200, offset: 200, total: null } }));
|
||||
|
||||
const all = await listAllChapters('m1');
|
||||
expect(all).toHaveLength(250);
|
||||
expect(all[249].number).toBe(250);
|
||||
expect(fetchSpy).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('listAllChapters stops after a short final page (single request)', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(
|
||||
ok({ items: [chapterFixture], page: { limit: 200, offset: 0, total: 1 } })
|
||||
);
|
||||
const all = await listAllChapters('m1');
|
||||
expect(all).toHaveLength(1);
|
||||
expect(fetchSpy).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('getChapterPages unwraps the {pages} envelope into the array', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(
|
||||
ok({
|
||||
|
||||
@@ -40,6 +40,31 @@ export async function listChapters(
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Every chapter of a manga, in display order, paging through the API (which
|
||||
* clamps `limit` to 200 per request). The reader needs the complete list so
|
||||
* prev/next navigation and the chapter dropdown work for a deep chapter — a
|
||||
* single 200-row window dead-ended chapter #250 with null neighbours.
|
||||
*/
|
||||
export async function listAllChapters(mangaId: string): Promise<Chapter[]> {
|
||||
const PAGE = 200;
|
||||
const all: Chapter[] = [];
|
||||
let offset = 0;
|
||||
// Hard iteration cap (200 * 100 = 20k chapters) so a bad `total` can never
|
||||
// spin forever.
|
||||
for (let i = 0; i < 100; i++) {
|
||||
const { items, page } = await listChapters(mangaId, { limit: PAGE, offset });
|
||||
all.push(...items);
|
||||
offset += items.length;
|
||||
// A short page is the only reliable end-of-list signal: the /chapters
|
||||
// endpoint returns `total: null`, so we can't infer completion from it.
|
||||
// Only stop early on `total` when the API actually reports one.
|
||||
if (items.length < PAGE) break;
|
||||
if (page.total != null && all.length >= page.total) break;
|
||||
}
|
||||
return all;
|
||||
}
|
||||
|
||||
export async function getChapter(mangaId: string, chapterId: string): Promise<Chapter> {
|
||||
return request<Chapter>(
|
||||
`/v1/mangas/${encodeURIComponent(mangaId)}/chapters/${encodeURIComponent(chapterId)}`
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { describe, it, expect, vi, beforeEach, afterEach, type MockInstance } from 'vitest';
|
||||
import { ApiError, request, setOn401Hook, fileUrl } from './client';
|
||||
import { ApiError, request, setOn401Hook, fileUrl, thumbUrl, thumbSrcset } from './client';
|
||||
import { getManga } from './mangas';
|
||||
|
||||
describe('fileUrl', () => {
|
||||
@@ -19,6 +19,21 @@ describe('fileUrl', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('thumbUrl / thumbSrcset', () => {
|
||||
it('appends the width query to the file URL', () => {
|
||||
expect(thumbUrl('mangas/abc/cover.png', 320)).toBe(
|
||||
'/api/v1/files/mangas/abc/cover.png?w=320'
|
||||
);
|
||||
});
|
||||
|
||||
it('builds a srcset over the given candidate widths', () => {
|
||||
expect(thumbSrcset('mangas/abc/cover.png', [160, 320])).toBe(
|
||||
'/api/v1/files/mangas/abc/cover.png?w=160 160w, ' +
|
||||
'/api/v1/files/mangas/abc/cover.png?w=320 320w'
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('request error envelope parsing', () => {
|
||||
let fetchSpy: MockInstance<typeof globalThis.fetch>;
|
||||
|
||||
@@ -65,6 +80,17 @@ describe('request error envelope parsing', () => {
|
||||
expect(err.code).toBe('http_error');
|
||||
});
|
||||
|
||||
it('wraps a network-level fetch rejection in ApiError (status 0)', async () => {
|
||||
// Connection refused / offline / DNS: fetch rejects with a TypeError.
|
||||
// It must surface as an ApiError, not a bare TypeError that crashes a
|
||||
// load function into SvelteKit's default error page.
|
||||
fetchSpy.mockRejectedValueOnce(new TypeError('Failed to fetch'));
|
||||
const err = (await getManga('x').catch((e) => e)) as ApiError;
|
||||
expect(err).toBeInstanceOf(ApiError);
|
||||
expect(err.status).toBe(0);
|
||||
expect(err.code).toBe('network_error');
|
||||
});
|
||||
|
||||
it('treats empty 200/201 bodies as undefined (no JSON.parse crash)', async () => {
|
||||
// Regression: addMangaToCollection is typed `void` and the
|
||||
// backend returns 201 (created) / 200 (already there) with
|
||||
|
||||
@@ -19,6 +19,25 @@ export function fileUrl(key: string): string {
|
||||
return `${BASE}/v1/files/${encoded}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* URL to a width-bounded thumbnail variant of a stored image (the backend
|
||||
* `/files/{key}?w=` endpoint). Grids use this so a cover downloads ~KB instead
|
||||
* of the 1–5 MB original; the backend snaps `width` to a small allow-list and
|
||||
* caches the result.
|
||||
*/
|
||||
export function thumbUrl(key: string, width: number): string {
|
||||
return `${fileUrl(key)}?w=${width}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* A `srcset` string over the candidate `widths` for `key`, e.g.
|
||||
* `.../files/k?w=320 320w, .../files/k?w=480 480w`. Pair with a `sizes`
|
||||
* attribute so the browser picks the right variant for the rendered size.
|
||||
*/
|
||||
export function thumbSrcset(key: string, widths: number[]): string {
|
||||
return widths.map((w) => `${thumbUrl(key, w)} ${w}w`).join(', ');
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds an API URL for non-`fetch` consumers (e.g. `EventSource` for SSE),
|
||||
* applying the same `VITE_API_BASE` prefix as `request()`. `path` is the
|
||||
@@ -84,7 +103,25 @@ export async function request<T>(
|
||||
// working. For same-origin requests this is a no-op compared to the
|
||||
// default 'same-origin', so the same-origin happy path is
|
||||
// unchanged.
|
||||
const res = await fetch(`${BASE}${path}`, { credentials: 'include', ...init });
|
||||
let res: Response;
|
||||
try {
|
||||
res = await fetch(`${BASE}${path}`, { credentials: 'include', ...init });
|
||||
} catch (e) {
|
||||
// A deliberate cancellation (AbortController) must propagate unchanged so
|
||||
// callers can tell "cancelled" from "network failure" (e.g. the
|
||||
// cancellable admin fetches treat AbortError as a no-op).
|
||||
if (e instanceof DOMException && e.name === 'AbortError') throw e;
|
||||
// Any other rejection is a network-level failure (connection refused,
|
||||
// DNS, offline, blocked by CORS): `fetch` rejects with a bare TypeError
|
||||
// instead of returning a response. Normalise it to an ApiError (status 0
|
||||
// = "no response reached us") so callers and SvelteKit load functions
|
||||
// handle it uniformly instead of crashing on an unexpected TypeError.
|
||||
throw new ApiError(
|
||||
0,
|
||||
'network_error',
|
||||
'Could not reach the server. Check your connection and try again.'
|
||||
);
|
||||
}
|
||||
if (!res.ok) {
|
||||
let code = 'http_error';
|
||||
let message = `${res.status} ${res.statusText}`;
|
||||
|
||||
@@ -165,6 +165,16 @@ describe('page_tags api client', () => {
|
||||
expect(url).toMatch(/\/v1\/me\/page-tags\/mangas\?tag=funny$/);
|
||||
});
|
||||
|
||||
it('aggregation calls forward the OCR text filter when provided', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(
|
||||
ok({ items: [], page: { limit: 50, offset: 0, total: 0 } })
|
||||
);
|
||||
await listTaggedChapters({ tag: 'funny', text: 'hello world' });
|
||||
const url = fetchSpy.mock.calls[0][0] as string;
|
||||
expect(url).toContain('tag=funny');
|
||||
expect(url).toContain('text=hello+world');
|
||||
});
|
||||
|
||||
it('searchPages serializes tags (CSV), text and content-warning filters', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(
|
||||
ok({ items: [], page: { limit: 50, offset: 0, total: 0 } })
|
||||
|
||||
@@ -179,6 +179,9 @@ export type AggregateOptions = {
|
||||
order?: 'desc' | 'asc';
|
||||
limit?: number;
|
||||
offset?: number;
|
||||
/** OCR full-text filter: when set, only pages whose analysis search_doc
|
||||
* matches are aggregated (and `match_count` reflects that). */
|
||||
text?: string;
|
||||
};
|
||||
|
||||
function aggregateQs(opts: AggregateOptions): string {
|
||||
@@ -187,6 +190,7 @@ function aggregateQs(opts: AggregateOptions): string {
|
||||
if (opts.order != null) params.set('order', opts.order);
|
||||
if (opts.limit != null) params.set('limit', String(opts.limit));
|
||||
if (opts.offset != null) params.set('offset', String(opts.offset));
|
||||
if (opts.text != null && opts.text !== '') params.set('text', opts.text);
|
||||
return params.toString();
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
<script lang="ts">
|
||||
import AdaptiveDialog from './AdaptiveDialog.svelte';
|
||||
import { CancellableLoader } from '$lib/util/cancellable';
|
||||
import {
|
||||
addMangaToCollection,
|
||||
createCollection,
|
||||
@@ -74,19 +75,30 @@
|
||||
: removePageFromCollection(collectionId, t.id);
|
||||
}
|
||||
|
||||
// Reopening for a different target (or an in-flight retarget) fires load()
|
||||
// again; without this guard the two responses can resolve out of order and
|
||||
// the first target's membership overwrites the second's (audit FE-2). The
|
||||
// CancellableLoader drops a superseded call's result — the newer load then
|
||||
// owns both the state and the spinner.
|
||||
const loader = new CancellableLoader();
|
||||
|
||||
async function load() {
|
||||
loading = true;
|
||||
error = null;
|
||||
try {
|
||||
const [page, ids] = await Promise.all([
|
||||
listMyCollections({ limit: 200 }),
|
||||
loadContaining(target)
|
||||
]);
|
||||
collections = page.items;
|
||||
containingIds = new Set(ids);
|
||||
const result = await loader.run(async () => {
|
||||
const [page, ids] = await Promise.all([
|
||||
listMyCollections({ limit: 200 }),
|
||||
loadContaining(target)
|
||||
]);
|
||||
return { items: page.items, ids };
|
||||
});
|
||||
if (result === null) return; // superseded — a newer load owns the state
|
||||
collections = result.items;
|
||||
containingIds = new Set(result.ids);
|
||||
loading = false;
|
||||
} catch (e) {
|
||||
error = (e as Error).message;
|
||||
} finally {
|
||||
loading = false;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -84,7 +84,11 @@ describe('AddToCollectionModal', () => {
|
||||
|
||||
expect(pageCollectionsApi.getMyCollectionsContainingPage).toHaveBeenCalledWith('p1');
|
||||
expect(collectionsApi.getMyCollectionsContaining).not.toHaveBeenCalled();
|
||||
expect(screen.getByText('Favorite panels')).toBeTruthy();
|
||||
// Await the async render: load() resolves the collections over a few
|
||||
// microtasks (through the CancellableLoader), so assert with findByText
|
||||
// (retries until it appears) rather than a synchronous getByText tied to
|
||||
// a fixed tick budget.
|
||||
expect(await screen.findByText('Favorite panels')).toBeTruthy();
|
||||
});
|
||||
|
||||
it('does not load when closed', () => {
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user