Compare commits
27 Commits
a44511983d
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
33cc41bacd | ||
|
|
08a9819d76 | ||
|
|
70a6598924 | ||
|
|
e6aefaa804 | ||
|
|
6354a7a8a3 | ||
|
|
60d5a2efea | ||
|
|
90f2398e56 | ||
|
|
e960aae163 | ||
|
|
adce950049 | ||
|
|
bde72e47f4 | ||
|
|
cb757e7b69 | ||
|
|
042e7e9047 | ||
|
|
cc8ae2566f | ||
|
|
4f085f228a | ||
|
|
ac5641bcd3 | ||
|
|
cd5358dc9b | ||
|
|
c31830468c | ||
|
|
75f4481bbd | ||
|
|
958d5bd713 | ||
|
|
f2c9cfe162 | ||
|
|
b5f7467c47 | ||
|
|
9148e23da8 | ||
|
|
f39307232c | ||
|
|
1b7b8a3038 | ||
|
|
253d46c7e5 | ||
|
|
c570e0cc37 | ||
|
|
5784483a57 |
36
.env.example
36
.env.example
@@ -118,8 +118,10 @@ MAX_REQUEST_BYTES=209715200
|
||||
MAX_FILE_BYTES=20971520
|
||||
# Max page images accepted in one chapter upload. Bounds how many parts
|
||||
# the handler will stage before rejecting the request with 413, so a
|
||||
# client can't pin a worker with an unbounded page count. 0 disables the
|
||||
# cap. Default 2000.
|
||||
# client can't pin a worker with an unbounded page count. Default 2000.
|
||||
# Setting 0 disables THIS cap — the total is then bounded only by
|
||||
# MAX_REQUEST_BYTES (the whole-request body limit above), so leave 0 only
|
||||
# if you intend that body limit to be the sole backstop.
|
||||
MAX_PAGES_PER_CHAPTER=2000
|
||||
|
||||
# ----- Crawler download safety -----
|
||||
@@ -191,12 +193,19 @@ CRAWLER_METADATA_MAX_CONSECUTIVE_FAILURES=10
|
||||
# exhausted) that trigger an automatic coordinated browser restart.
|
||||
# Default 3.
|
||||
CRAWLER_BROWSER_RESTART_THRESHOLD=3
|
||||
# Opt-in CDP Fetch interception that re-validates every headless-browser
|
||||
# navigation/redirect against the SSRF check (blocks a scraped page that
|
||||
# redirects the browser to an internal target). Default false — enabling
|
||||
# Fetch is a fragile hook in the crawler's critical path and is not
|
||||
# CI-verified; validate with a manual crawl before turning on.
|
||||
CRAWLER_SSRF_INTERCEPT=false
|
||||
# CDP Fetch interception that re-validates every headless-browser
|
||||
# navigation/redirect/subresource against the SSRF check (blocks a scraped page
|
||||
# that drives the browser — via a redirect OR page JS/subresource — to an
|
||||
# internal target such as the cloud metadata service or postgres). Default TRUE:
|
||||
# with it off, only the top-level URL string is checked and page JS/subresources
|
||||
# reach internal IPs (the reqwest SafeResolver does not cover Chromium's own
|
||||
# stack). The CDP Fetch hook can't be exercised in CI (no Chromium); before
|
||||
# relying on a fresh deploy, validate it doesn't wedge navigation:
|
||||
# CRAWLER_CHROMIUM_BINARY=/usr/bin/chromium \
|
||||
# cargo test -p mangalord --test crawler_browser_smoke -- --ignored \
|
||||
# ssrf_interception_does_not_wedge_allowed_navigation
|
||||
# Set to false only as a break-glass if the hook destabilizes a deployment.
|
||||
CRAWLER_SSRF_INTERCEPT=true
|
||||
# Path to a system Chromium binary. When set, the crawler skips the
|
||||
# bundled-fetcher download. Required on platforms without a usable
|
||||
# upstream Chromium build (notably Linux_arm64 / Raspberry Pi). On
|
||||
@@ -418,3 +427,14 @@ VISION_MANAGER_DATABASE_URL=
|
||||
# VISION_MEM_LOW_WATERMARK_PCT used% >= this → inhibit starts (keep HIGH - LOW >= ~10 so the band beats jitter). Default 80.
|
||||
# VISION_MEM_YIELD_COOLDOWN Seconds after a pressure-stop during which restart is refused regardless of backlog. Default 300.
|
||||
# VISION_MEM_POLL_INTERVAL Mem sub-poll cadence, seconds (<= VISION_POLL_INTERVAL); catches spikes between backlog ticks. Default 5.
|
||||
|
||||
# ---- Deployment resource bounds & exposure (docker-compose.yml) ----
|
||||
# Host interface the frontend port publishes on. Default 127.0.0.1 so SvelteKit
|
||||
# (plain HTTP) is reachable only by a host-local reverse proxy. Set 0.0.0.0 only
|
||||
# if your TLS terminator runs on a different host.
|
||||
FRONTEND_PUBLISH_ADDR=127.0.0.1
|
||||
# Per-container memory ceilings (docker mem_limit). Generous defaults; tune to
|
||||
# your host. The backend runs OCR + headless Chromium and is the heavy one.
|
||||
BACKEND_MEM_LIMIT=4g
|
||||
FRONTEND_MEM_LIMIT=512m
|
||||
POSTGRES_MEM_LIMIT=1g
|
||||
|
||||
3
backend/Cargo.lock
generated
3
backend/Cargo.lock
generated
@@ -1558,7 +1558,7 @@ checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
|
||||
|
||||
[[package]]
|
||||
name = "mangalord"
|
||||
version = "0.128.4"
|
||||
version = "0.128.25"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"argon2",
|
||||
@@ -1587,7 +1587,6 @@ dependencies = [
|
||||
"serde_json",
|
||||
"sha2",
|
||||
"sqlx",
|
||||
"subtle",
|
||||
"sysinfo",
|
||||
"tempfile",
|
||||
"thiserror 1.0.69",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "mangalord"
|
||||
version = "0.128.4"
|
||||
version = "0.128.25"
|
||||
edition = "2021"
|
||||
default-run = "mangalord"
|
||||
|
||||
@@ -35,7 +35,6 @@ dotenvy = "0.15"
|
||||
argon2 = "0.5"
|
||||
rand = "0.8"
|
||||
sha2 = "0.10"
|
||||
subtle = "2"
|
||||
base64 = "0.22"
|
||||
# Image decode + downscale for the analysis worker (keep the page image
|
||||
# under the local vision model's token budget). Only the manga page formats.
|
||||
|
||||
44
backend/migrations/0038_genres_name_lower_unique.sql
Normal file
44
backend/migrations/0038_genres_name_lower_unique.sql
Normal file
@@ -0,0 +1,44 @@
|
||||
-- Enforce case-insensitive genre uniqueness, matching `authors` and `tags`
|
||||
-- (both got `UNIQUE (lower(name))` in 0009). `genres` only had a case-SENSITIVE
|
||||
-- `name UNIQUE`, but the crawler's `sync_genres` treats genres as
|
||||
-- case-insensitive (`WHERE lower(name) = lower($1)`) and can INSERT a new row
|
||||
-- from a source string. Under a race (or across ticks) two different-cased
|
||||
-- strings — "Isekai" vs "isekai" — could both miss the pre-check and both
|
||||
-- insert, since the exact-name unique doesn't collide. That leaves duplicate
|
||||
-- genre rows for one logical genre.
|
||||
|
||||
-- Heal any pre-existing case-variant duplicates before adding the index, so
|
||||
-- `sqlx::migrate!` can't crash on a dirty table (mirrors 0031's pre-dedup).
|
||||
-- Canonical = lowest id per lower(name).
|
||||
|
||||
-- 1. Ensure every manga linked to any case-variant also has the canonical link.
|
||||
-- INSERT ... ON CONFLICT DO NOTHING is collision-proof: SELECT DISTINCT
|
||||
-- collapses multiple variants of one manga to a single canonical row, and the
|
||||
-- ON CONFLICT absorbs a canonical link that already exists. (A prior
|
||||
-- UPDATE-repoint could set two non-canonical rows of the SAME manga to the
|
||||
-- same canonical id in a single statement and violate the manga_genres PK,
|
||||
-- rolling the migration back and wedging startup.)
|
||||
INSERT INTO manga_genres (manga_id, genre_id)
|
||||
SELECT DISTINCT mg.manga_id, k.keep_id
|
||||
FROM manga_genres mg
|
||||
JOIN genres g ON mg.genre_id = g.id
|
||||
JOIN (SELECT lower(name) AS lname, (array_agg(id ORDER BY id))[1] AS keep_id
|
||||
FROM genres GROUP BY lower(name)) k
|
||||
ON lower(g.name) = k.lname
|
||||
WHERE g.id <> k.keep_id
|
||||
ON CONFLICT (manga_id, genre_id) DO NOTHING;
|
||||
|
||||
-- 2. Every non-canonical link now has a canonical sibling — drop the variants.
|
||||
DELETE FROM manga_genres mg
|
||||
USING genres g,
|
||||
(SELECT lower(name) AS lname, (array_agg(id ORDER BY id))[1] AS keep_id
|
||||
FROM genres GROUP BY lower(name)) k
|
||||
WHERE mg.genre_id = g.id AND lower(g.name) = k.lname AND g.id <> k.keep_id;
|
||||
|
||||
-- Remove the now-orphaned duplicate genre rows.
|
||||
DELETE FROM genres g
|
||||
USING (SELECT lower(name) AS lname, (array_agg(id ORDER BY id))[1] AS keep_id
|
||||
FROM genres GROUP BY lower(name)) k
|
||||
WHERE lower(g.name) = k.lname AND g.id <> k.keep_id;
|
||||
|
||||
CREATE UNIQUE INDEX genres_name_lower_uniq ON genres (lower(name));
|
||||
18
backend/migrations/0039_crawler_jobs_running_lease_idx.sql
Normal file
18
backend/migrations/0039_crawler_jobs_running_lease_idx.sql
Normal file
@@ -0,0 +1,18 @@
|
||||
-- Index the crashed-worker-recovery arm of the job lease predicate.
|
||||
--
|
||||
-- lease/lease_kinds match:
|
||||
-- WHERE (state = 'pending' OR (state = 'running' AND leased_until < now()))
|
||||
-- crawler_jobs_ready_idx (0016) is `ON (scheduled_at) WHERE state = 'pending'`,
|
||||
-- so it covers only the pending arm. The `state = 'running' AND leased_until`
|
||||
-- arm had no usable index, so Postgres could not BitmapOr the two arms and
|
||||
-- degraded to a sequential scan of the ENTIRE crawler_jobs table — including all
|
||||
-- done/dead rows not yet reaped — on every lease poll, by every worker,
|
||||
-- continuously (audit H2, the headline performance finding).
|
||||
--
|
||||
-- A partial index on leased_until over just the running rows makes the recovery
|
||||
-- arm index-backed. `state = 'running'` is an immutable predicate (now() stays
|
||||
-- in the query, not the index). The running set is tiny (in-flight jobs only),
|
||||
-- so the index is cheap to maintain.
|
||||
CREATE INDEX crawler_jobs_running_lease_idx
|
||||
ON crawler_jobs (leased_until)
|
||||
WHERE state = 'running';
|
||||
@@ -0,0 +1,9 @@
|
||||
-- Index the retention reaper's scan. reap_terminal deletes
|
||||
-- WHERE state IN ('done','dead') AND updated_at < now() - interval
|
||||
-- which had no supporting index, so each daily sweep sequential-scanned the whole
|
||||
-- crawler_jobs table to find expired terminal rows (audit M5). A partial index on
|
||||
-- updated_at over just the terminal states makes the batched reaper's per-batch
|
||||
-- SELECT index-backed.
|
||||
CREATE INDEX crawler_jobs_terminal_reap_idx
|
||||
ON crawler_jobs (updated_at)
|
||||
WHERE state IN ('done', 'dead');
|
||||
@@ -37,6 +37,22 @@ const LEASE_HEARTBEAT: Duration = Duration::from_secs(20);
|
||||
/// long enough not to hammer `/health` while vision is down.
|
||||
const READINESS_POLL: Duration = Duration::from_secs(2);
|
||||
|
||||
/// Longest an idle analysis worker waits between lease polls, bounding the
|
||||
/// exponential [`idle_backoff`].
|
||||
const IDLE_BACKOFF_CAP: Duration = Duration::from_secs(30);
|
||||
|
||||
/// Exponential idle backoff (1s, 2s, 4s … capped at [`IDLE_BACKOFF_CAP`]).
|
||||
/// `consecutive_empty` is the number of empty lease polls so far (0 on the first
|
||||
/// miss); reset to 0 the moment a job is leased. Replaces the old flat 1s sleep
|
||||
/// so an idle worker isn't firing a `SELECT … FOR UPDATE SKIP LOCKED` lease
|
||||
/// query every second (audit: analysis daemon idle poll). Mirrors the crawler
|
||||
/// daemon's backoff.
|
||||
fn idle_backoff(consecutive_empty: u32) -> Duration {
|
||||
let cap = IDLE_BACKOFF_CAP.as_secs();
|
||||
let secs = 1u64.checked_shl(consecutive_empty).unwrap_or(cap).min(cap);
|
||||
Duration::from_secs(secs)
|
||||
}
|
||||
|
||||
/// The unit of work: analyze one page. Implemented by
|
||||
/// [`RealAnalyzeDispatcher`] in production and stubbed in tests.
|
||||
#[async_trait]
|
||||
@@ -135,6 +151,9 @@ impl WorkerContext {
|
||||
// Last observed readiness, so we log only on transitions (not every
|
||||
// poll). `None` until the first probe.
|
||||
let mut was_ready: Option<bool> = None;
|
||||
// Empty lease polls seen in a row, driving the idle backoff. Reset to 0
|
||||
// the moment a job is leased.
|
||||
let mut consecutive_empty: u32 = 0;
|
||||
loop {
|
||||
if self.cancel.is_cancelled() {
|
||||
tracing::info!(worker = self.id, "analysis worker: shutdown");
|
||||
@@ -178,11 +197,15 @@ impl WorkerContext {
|
||||
}
|
||||
};
|
||||
let Some(lease) = leases.into_iter().next() else {
|
||||
if self.sleep_or_cancel(Duration::from_secs(1)).await {
|
||||
let wait = idle_backoff(consecutive_empty);
|
||||
consecutive_empty = consecutive_empty.saturating_add(1);
|
||||
if self.sleep_or_cancel(wait).await {
|
||||
return;
|
||||
}
|
||||
continue;
|
||||
};
|
||||
// Leased work — drop back to a tight poll cadence.
|
||||
consecutive_empty = 0;
|
||||
self.process_lease(lease).await;
|
||||
}
|
||||
}
|
||||
@@ -510,6 +533,17 @@ mod tests {
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::net::TcpListener;
|
||||
|
||||
#[test]
|
||||
fn idle_backoff_grows_exponentially_and_caps() {
|
||||
assert_eq!(idle_backoff(0), Duration::from_secs(1));
|
||||
assert_eq!(idle_backoff(1), Duration::from_secs(2));
|
||||
assert_eq!(idle_backoff(2), Duration::from_secs(4));
|
||||
assert_eq!(idle_backoff(4), Duration::from_secs(16));
|
||||
// Caps at IDLE_BACKOFF_CAP (30s) and never overflows on large counts.
|
||||
assert_eq!(idle_backoff(5), IDLE_BACKOFF_CAP);
|
||||
assert_eq!(idle_backoff(100), IDLE_BACKOFF_CAP);
|
||||
}
|
||||
|
||||
/// Bind an ephemeral port that answers every request with `status_line`
|
||||
/// (e.g. `"200 OK"`), and return its `/health` URL. The probe under test
|
||||
/// only inspects the status code, so a zero-length body is enough.
|
||||
|
||||
@@ -328,10 +328,21 @@ fn ensure_enabled(state: &AppState) -> AppResult<()> {
|
||||
async fn reenqueue(
|
||||
State(state): State<AppState>,
|
||||
admin: RequireAdmin,
|
||||
body: Option<Json<ReenqueueBody>>,
|
||||
raw: axum::body::Bytes,
|
||||
) -> AppResult<Json<ReenqueueResponse>> {
|
||||
ensure_enabled(&state)?;
|
||||
let body = body.map(|b| b.0).unwrap_or_default();
|
||||
// An ABSENT/empty body means the default "All" scope. A PRESENT body must be
|
||||
// valid JSON — `Option<Json<T>>` would silently collapse a malformed body to
|
||||
// None and run the full-library default the caller never intended, so parse
|
||||
// the raw bytes ourselves and 422 on a real parse error.
|
||||
let body: ReenqueueBody = if raw.iter().all(u8::is_ascii_whitespace) {
|
||||
ReenqueueBody::default()
|
||||
} else {
|
||||
serde_json::from_slice(&raw).map_err(|e| AppError::ValidationFailed {
|
||||
message: "reenqueue body is not valid JSON".into(),
|
||||
details: json!({ "body": e.to_string() }),
|
||||
})?
|
||||
};
|
||||
|
||||
// Resolve the scope. manga_id and chapter_id are mutually exclusive;
|
||||
// an unknown target is a 404 rather than a silent zero-enqueue.
|
||||
|
||||
@@ -13,7 +13,7 @@ use serde::Deserialize;
|
||||
use serde_json::json;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::api::mangas::{next_field, read_field_bytes};
|
||||
use crate::api::mangas::next_field;
|
||||
use crate::api::pagination::PagedResponse;
|
||||
use crate::app::AppState;
|
||||
use crate::auth::extractor::CurrentUser;
|
||||
@@ -107,7 +107,12 @@ async fn create(
|
||||
while let Some(field) = next_field(&mut multipart).await? {
|
||||
match field.name() {
|
||||
Some("metadata") => {
|
||||
let bytes = read_field_bytes(field).await?;
|
||||
let bytes = crate::upload::read_capped(
|
||||
field,
|
||||
crate::upload::MAX_METADATA_BYTES,
|
||||
"metadata",
|
||||
)
|
||||
.await?;
|
||||
metadata = Some(serde_json::from_slice(&bytes).map_err(|e| {
|
||||
AppError::ValidationFailed {
|
||||
message: "metadata is not valid JSON".into(),
|
||||
|
||||
@@ -118,9 +118,20 @@ async fn serve_thumbnail(
|
||||
Err(e) => return Err(e.into()),
|
||||
};
|
||||
// Resizing is CPU-bound; keep it off the async worker threads.
|
||||
let thumb = tokio::task::spawn_blocking(move || make_thumbnail(&original, width, fmt))
|
||||
let thumb = match tokio::task::spawn_blocking(move || make_thumbnail(&original, width, fmt))
|
||||
.await
|
||||
.map_err(|e| AppError::Other(anyhow::anyhow!("thumbnail task join: {e}")))??;
|
||||
.map_err(|e| AppError::Other(anyhow::anyhow!("thumbnail task join: {e}")))?
|
||||
{
|
||||
Ok(t) => t,
|
||||
Err(e) => {
|
||||
// The blob's magic bytes passed upload's sniff but the body won't
|
||||
// decode (corrupt/truncated, or an encoder feature we don't support).
|
||||
// Don't 500 the reader — fall back to streaming the original, which
|
||||
// serves without decoding.
|
||||
tracing::warn!(key, error = %format!("{e:#}"), "thumbnail decode failed; serving original");
|
||||
return serve_original(state, key).await;
|
||||
}
|
||||
};
|
||||
|
||||
// Best-effort cache; a write failure just means we regenerate next time.
|
||||
let _ = state.storage.put(&derived, &thumb).await;
|
||||
@@ -141,7 +152,7 @@ fn image_response(
|
||||
content_length: String,
|
||||
body: Body,
|
||||
) -> Response {
|
||||
let headers = [
|
||||
let mut headers = vec![
|
||||
(header::CONTENT_TYPE, content_type.to_string()),
|
||||
(header::CONTENT_LENGTH, content_length),
|
||||
// `nosniff` makes the contract explicit: the browser must trust the
|
||||
@@ -169,7 +180,17 @@ fn image_response(
|
||||
},
|
||||
),
|
||||
];
|
||||
(headers, body).into_response()
|
||||
// Known image types render inline (covers/pages display in the reader). The
|
||||
// `application/octet-stream` fallback is a blob we couldn't type — it could
|
||||
// be crafted HTML/JS, so force a download rather than let the browser render
|
||||
// it inline. Belt to `nosniff`'s braces.
|
||||
if content_type == "application/octet-stream" {
|
||||
headers.push((
|
||||
header::CONTENT_DISPOSITION,
|
||||
"attachment".to_string(),
|
||||
));
|
||||
}
|
||||
(axum::response::AppendHeaders(headers), body).into_response()
|
||||
}
|
||||
|
||||
/// Parse and clamp a requested thumbnail width. Returns `None` for absent /
|
||||
|
||||
@@ -235,7 +235,12 @@ async fn create(
|
||||
while let Some(field) = next_field(&mut multipart).await? {
|
||||
match field.name() {
|
||||
Some("metadata") => {
|
||||
let bytes = read_field_bytes(field).await?;
|
||||
let bytes = crate::upload::read_capped(
|
||||
field,
|
||||
crate::upload::MAX_METADATA_BYTES,
|
||||
"metadata",
|
||||
)
|
||||
.await?;
|
||||
metadata = Some(parse_metadata_json(&bytes)?);
|
||||
}
|
||||
Some("cover") => {
|
||||
@@ -404,6 +409,17 @@ async fn put_cover(
|
||||
let old_key = repo::manga::get(&state.db, id).await?.cover_image_path;
|
||||
let new_key = format!("mangas/{}/cover.{}", id, img.ext);
|
||||
state.storage.put(&new_key, &img.bytes).await?;
|
||||
// Cover keys are reused (mangas/{id}/cover.{ext}), so a same-extension
|
||||
// replacement overwrites the blob at an existing key — drop any thumbnails
|
||||
// cached for it so we don't serve a stale variant of the old cover.
|
||||
crate::api::files::purge_thumbnails(state.storage.as_ref(), &new_key).await;
|
||||
|
||||
// Commit the DB pointer to the new blob BEFORE removing the old one. If we
|
||||
// deleted the old blob first and this write then failed, the row would point
|
||||
// at a deleted blob (a cover that 404s) and the new blob would be orphaned.
|
||||
// Ordering it first means a failure here leaves the row pointing at the
|
||||
// still-present old blob; only a harmless orphan (new blob) can result.
|
||||
repo::manga::set_cover_image_path(&state.db, id, &new_key, img.bytes.len() as i64).await?;
|
||||
|
||||
if let Some(prev) = old_key.as_deref() {
|
||||
if prev != new_key {
|
||||
@@ -418,12 +434,6 @@ async fn put_cover(
|
||||
crate::api::files::purge_thumbnails(state.storage.as_ref(), prev).await;
|
||||
}
|
||||
}
|
||||
// Cover keys are reused (mangas/{id}/cover.{ext}), so a same-extension
|
||||
// replacement overwrites the blob at an existing key — drop any thumbnails
|
||||
// cached for it so we don't serve a stale variant of the old cover.
|
||||
crate::api::files::purge_thumbnails(state.storage.as_ref(), &new_key).await;
|
||||
|
||||
repo::manga::set_cover_image_path(&state.db, id, &new_key, img.bytes.len() as i64).await?;
|
||||
Ok(Json(repo::manga::get_detail(&state.db, id).await?))
|
||||
}
|
||||
|
||||
@@ -441,12 +451,15 @@ async fn delete_cover(
|
||||
}
|
||||
require_can_edit(&state, id, user.id, admin_via_session(&session)).await?;
|
||||
if let Some(key) = repo::manga::get(&state.db, id).await?.cover_image_path {
|
||||
// Clear the DB pointer BEFORE deleting the blob, so a storage-delete
|
||||
// failure can't leave the row pointing at a removed blob. A failure
|
||||
// after the clear only orphans the blob (harmless).
|
||||
repo::manga::clear_cover_image_path(&state.db, id).await?;
|
||||
match state.storage.delete(&key).await {
|
||||
Ok(()) | Err(StorageError::NotFound) => {}
|
||||
Err(e) => return Err(e.into()),
|
||||
}
|
||||
crate::api::files::purge_thumbnails(state.storage.as_ref(), &key).await;
|
||||
repo::manga::clear_cover_image_path(&state.db, id).await?;
|
||||
}
|
||||
Ok(Json(repo::manga::get_detail(&state.db, id).await?))
|
||||
}
|
||||
@@ -677,12 +690,6 @@ pub(crate) async fn next_field(
|
||||
.map_err(map_multipart_error)
|
||||
}
|
||||
|
||||
pub(crate) async fn read_field_bytes(
|
||||
field: axum::extract::multipart::Field<'_>,
|
||||
) -> AppResult<axum::body::Bytes> {
|
||||
field.bytes().await.map_err(map_multipart_error)
|
||||
}
|
||||
|
||||
pub(crate) fn map_multipart_error(e: axum::extract::multipart::MultipartError) -> AppError {
|
||||
let status = e.status();
|
||||
if status == StatusCode::PAYLOAD_TOO_LARGE {
|
||||
|
||||
@@ -277,6 +277,11 @@ impl DaemonReloader for Supervisors {
|
||||
}
|
||||
}
|
||||
|
||||
/// How often the background reaper sweeps expired sessions. Hourly is ample:
|
||||
/// the sweep is a single indexed DELETE and expired rows are already invisible
|
||||
/// to auth, so this is purely storage hygiene.
|
||||
const SESSION_GC_INTERVAL: std::time::Duration = std::time::Duration::from_secs(3600);
|
||||
|
||||
pub async fn build(config: Config) -> anyhow::Result<AppHandle> {
|
||||
let db = PgPoolOptions::new()
|
||||
.max_connections(config.db.max_connections)
|
||||
@@ -337,6 +342,27 @@ pub async fn build(config: Config) -> anyhow::Result<AppHandle> {
|
||||
tracing::info!("analysis worker disabled");
|
||||
}
|
||||
|
||||
// Periodic reaper for lapsed sessions. `find_active` already ignores
|
||||
// expired rows, so this only reclaims storage — without it the table grows
|
||||
// unbounded as sessions lapse. Detached and best-effort: a failed sweep is
|
||||
// logged and retried next tick. Runs regardless of crawler/analysis config
|
||||
// since sessions exist in every deployment.
|
||||
{
|
||||
let db = db.clone();
|
||||
tokio::spawn(async move {
|
||||
let mut ticker = tokio::time::interval(SESSION_GC_INTERVAL);
|
||||
ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
|
||||
loop {
|
||||
ticker.tick().await;
|
||||
match repo::session::delete_expired(&db).await {
|
||||
Ok(0) => {}
|
||||
Ok(n) => tracing::info!(reaped = n, "session gc: removed expired sessions"),
|
||||
Err(e) => tracing::warn!(?e, "session gc sweep failed; retrying next tick"),
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
let auth_limiter = Arc::new(AuthRateLimiter::new(config.auth.rate_limit));
|
||||
let state = AppState {
|
||||
db,
|
||||
|
||||
@@ -3,15 +3,16 @@
|
||||
//! `generate_token` draws 32 bytes from the OS CSPRNG, encodes them as
|
||||
//! URL-safe base64 (no padding), and returns the raw string alongside its
|
||||
//! SHA-256 hash. Storage holds only the hash; the raw value lives in the
|
||||
//! cookie or `Authorization` header. Comparison goes through
|
||||
//! `constant_time_eq` to keep timing side channels off the table.
|
||||
//! cookie or `Authorization` header. Token lookup is an indexed equality on
|
||||
//! that 256-bit hash in the database (`WHERE token_hash = $1`), so there's no
|
||||
//! in-process secret comparison to time-attack: a guess has to match a full
|
||||
//! SHA-256 digest, and the DB index reveals nothing about how close it came.
|
||||
|
||||
use base64::engine::general_purpose::URL_SAFE_NO_PAD;
|
||||
use base64::Engine as _;
|
||||
use rand::rngs::OsRng;
|
||||
use rand::RngCore;
|
||||
use sha2::{Digest, Sha256};
|
||||
use subtle::ConstantTimeEq;
|
||||
|
||||
pub const TOKEN_BYTES: usize = 32;
|
||||
pub const HASH_BYTES: usize = 32;
|
||||
@@ -30,10 +31,6 @@ pub fn hash_token(raw: &str) -> [u8; HASH_BYTES] {
|
||||
hasher.finalize().into()
|
||||
}
|
||||
|
||||
pub fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
|
||||
a.ct_eq(b).into()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -58,11 +55,4 @@ mod tests {
|
||||
assert_eq!(hash_token("abc"), hash_token("abc"));
|
||||
assert_ne!(hash_token("abc"), hash_token("abd"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn constant_time_eq_compares_correctly() {
|
||||
assert!(constant_time_eq(b"abc", b"abc"));
|
||||
assert!(!constant_time_eq(b"abc", b"abd"));
|
||||
assert!(!constant_time_eq(b"abc", b"abcd"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -133,13 +133,16 @@ async fn main() -> anyhow::Result<()> {
|
||||
if let Some(proxy) = &proxy_url {
|
||||
http_builder = http_builder
|
||||
.proxy(reqwest::Proxy::all(proxy).with_context(|| format!("parse proxy URL: {proxy}"))?);
|
||||
} else {
|
||||
// DNS-rebinding guard, direct (unproxied) path only — see the matching
|
||||
// note in `spawn_crawler_daemon` (app.rs). With a `socks5h://` proxy the
|
||||
// target is resolved by the proxy, so this resolver would only ever
|
||||
// reject the proxy's own private-IP host (e.g. `tor`), breaking every
|
||||
// fetch for no security gain.
|
||||
http_builder = http_builder.dns_resolver(mangalord::crawler::safety::safe_dns_resolver());
|
||||
}
|
||||
// DNS-rebinding guard: attached on the direct path AND on http(s) proxies
|
||||
// (reqwest resolves the target itself there), skipped only for SOCKS proxies
|
||||
// where the proxy resolves the target. Use the shared predicate so this CLI
|
||||
// and the daemon (app.rs) stay in lockstep — previously the CLI attached the
|
||||
// resolver only on the fully-direct path, so an http(s) proxy silently lost
|
||||
// the rebinding guard.
|
||||
if mangalord::crawler::safety::should_attach_safe_resolver(proxy_url.as_deref()) {
|
||||
http_builder =
|
||||
http_builder.dns_resolver(mangalord::crawler::safety::safe_dns_resolver());
|
||||
}
|
||||
let http = http_builder.build().context("build http client")?;
|
||||
|
||||
|
||||
@@ -66,8 +66,10 @@ pub struct UploadConfig {
|
||||
pub max_file_bytes: usize,
|
||||
/// Max page images accepted in one chapter upload. Bounds how many
|
||||
/// parts the handler will stage before giving up, so a client can't
|
||||
/// pin a worker streaming an unbounded page count. `0` disables the
|
||||
/// cap. Defaults to 2000. `MAX_PAGES_PER_CHAPTER`.
|
||||
/// pin a worker streaming an unbounded page count. `0` disables THIS
|
||||
/// cap — the total is then bounded only by `max_request_bytes` (the
|
||||
/// whole-request body limit), which stays the backstop either way.
|
||||
/// Defaults to 2000. `MAX_PAGES_PER_CHAPTER`.
|
||||
pub max_pages_per_chapter: usize,
|
||||
}
|
||||
|
||||
@@ -520,12 +522,15 @@ pub struct CrawlerConfig {
|
||||
/// exhausted) that trigger an automatic coordinated browser restart.
|
||||
/// Defaults to 3. `CRAWLER_BROWSER_RESTART_THRESHOLD`.
|
||||
pub browser_restart_threshold: u32,
|
||||
/// Opt-in CDP `Fetch` interception that re-validates every headless-browser
|
||||
/// navigation/redirect against the SSRF check (blocks a scraped page that
|
||||
/// redirects the browser to an internal target). Default `false`: enabling
|
||||
/// `Fetch` is a fragile hook in the crawler's critical path and the wiring
|
||||
/// is not CI-verifiable (no Chromium in CI) — validate with a manual crawl
|
||||
/// before turning on. `CRAWLER_SSRF_INTERCEPT`.
|
||||
/// CDP `Fetch` interception that re-validates every headless-browser
|
||||
/// navigation/redirect/subresource against the SSRF check. Default `true`:
|
||||
/// with it off, only the top-level URL string is validated, so a scraped
|
||||
/// page's JS/subresources (which use Chromium's own network stack, not the
|
||||
/// reqwest `SafeResolver`) can reach internal targets like the cloud
|
||||
/// metadata service or postgres. The Fetch hook can't be exercised in CI (no
|
||||
/// Chromium) — the `#[ignore]`d `ssrf_interception_does_not_wedge_allowed_navigation`
|
||||
/// smoke test validates it against a real binary. `CRAWLER_SSRF_INTERCEPT`;
|
||||
/// set `false` only as a break-glass if the hook destabilizes a deployment.
|
||||
pub ssrf_intercept: bool,
|
||||
}
|
||||
|
||||
@@ -559,7 +564,7 @@ impl Default for CrawlerConfig {
|
||||
job_timeout: Duration::from_secs(600),
|
||||
metadata_max_consecutive_failures: 10,
|
||||
browser_restart_threshold: 3,
|
||||
ssrf_intercept: false,
|
||||
ssrf_intercept: true,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -725,7 +730,7 @@ impl CrawlerConfig {
|
||||
) as u32,
|
||||
browser_restart_threshold: env_u64("CRAWLER_BROWSER_RESTART_THRESHOLD", 3).max(1)
|
||||
as u32,
|
||||
ssrf_intercept: env_bool("CRAWLER_SSRF_INTERCEPT", false),
|
||||
ssrf_intercept: env_bool("CRAWLER_SSRF_INTERCEPT", true),
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1091,6 +1096,13 @@ mod tests {
|
||||
"BACKEND_URL",
|
||||
"BACKEND_PROXY_TIMEOUT_MS",
|
||||
"VISION_MANAGER_DATABASE_URL",
|
||||
// Compose-level deployment knobs (port publish interface + per-container
|
||||
// memory ceilings) — consumed by docker-compose.yml itself, never read
|
||||
// by the backend process, so they don't belong in its environment block.
|
||||
"FRONTEND_PUBLISH_ADDR",
|
||||
"BACKEND_MEM_LIMIT",
|
||||
"FRONTEND_MEM_LIMIT",
|
||||
"POSTGRES_MEM_LIMIT",
|
||||
];
|
||||
|
||||
/// Keys whose compose RHS is intentionally NOT a `${KEY...}`
|
||||
|
||||
@@ -595,15 +595,22 @@ pub(crate) async fn persist_pages(
|
||||
let mut tx = db.begin().await.context("open chapter sync tx")?;
|
||||
let mut page_ids: Vec<Uuid> = Vec::with_capacity(stored.len());
|
||||
let mut page_numbers: Vec<i32> = Vec::with_capacity(stored.len());
|
||||
// Pages whose row already existed and was overwritten by this re-crawl.
|
||||
// Their image changed but pages.id (and any analysis keyed to it) survived,
|
||||
// so the derived analysis is now stale and must be invalidated below.
|
||||
let mut updated_page_ids: Vec<Uuid> = Vec::new();
|
||||
for page in stored {
|
||||
let (id,): (Uuid,) = sqlx::query_as(
|
||||
// `xmax <> 0` on the RETURNING row distinguishes a conflict UPDATE
|
||||
// (existing row overwritten) from a fresh INSERT — the standard upsert
|
||||
// idiom. Cast through text/bigint since there is no direct xid<>int op.
|
||||
let (id, was_update): (Uuid, bool) = sqlx::query_as(
|
||||
"INSERT INTO pages (chapter_id, page_number, storage_key, content_type, size_bytes)
|
||||
VALUES ($1, $2, $3, $4, $5)
|
||||
ON CONFLICT (chapter_id, page_number) DO UPDATE
|
||||
SET storage_key = EXCLUDED.storage_key,
|
||||
content_type = EXCLUDED.content_type,
|
||||
size_bytes = EXCLUDED.size_bytes
|
||||
RETURNING id",
|
||||
RETURNING id, (xmax::text::bigint <> 0)",
|
||||
)
|
||||
.bind(chapter_id)
|
||||
.bind(page.page_number)
|
||||
@@ -615,6 +622,32 @@ pub(crate) async fn persist_pages(
|
||||
.with_context(|| format!("insert page row {}", page.page_number))?;
|
||||
page_ids.push(id);
|
||||
page_numbers.push(page.page_number);
|
||||
if was_update {
|
||||
updated_page_ids.push(id);
|
||||
}
|
||||
}
|
||||
// Invalidate stale analysis for re-crawled (overwritten) pages. Storage keys
|
||||
// are deterministic per page number, so the row survives the upsert and its
|
||||
// page_analysis / OCR / auto-tags / content-warnings would otherwise reflect
|
||||
// the OLD image — poisoning search results and the derived
|
||||
// manga_content_warnings. Clearing them (the DELETE on page_content_warnings
|
||||
// fires the mcw_* triggers so manga_content_warnings recomputes) drops the
|
||||
// pages back to "unanalyzed" so the enqueue below — and the admin
|
||||
// only-unanalyzed backfill — re-derive them. Newly inserted pages have no
|
||||
// prior analysis, so they're untouched.
|
||||
if !updated_page_ids.is_empty() {
|
||||
for table in [
|
||||
"page_ocr_text",
|
||||
"page_auto_tags",
|
||||
"page_content_warnings",
|
||||
"page_analysis",
|
||||
] {
|
||||
sqlx::query(&format!("DELETE FROM {table} WHERE page_id = ANY($1::uuid[])"))
|
||||
.bind(&updated_page_ids)
|
||||
.execute(&mut *tx)
|
||||
.await
|
||||
.with_context(|| format!("invalidate {table} for re-crawled pages"))?;
|
||||
}
|
||||
}
|
||||
// Drop any rows left over from a prior, larger crawl of this chapter
|
||||
// (re-crawl that now yields fewer pages). Without this the stale rows —
|
||||
@@ -1200,4 +1233,91 @@ mod tests {
|
||||
assert_eq!(fetch_n, 1);
|
||||
assert!(format!("{err:#}").contains("nav timeout"));
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn persist_pages_invalidates_stale_analysis_on_recrawl_update(pool: PgPool) {
|
||||
// Page storage keys are deterministic per page number, so a re-crawl
|
||||
// overwrites the image but keeps pages.id. Analysis keyed to page_id
|
||||
// would otherwise stay stale (old OCR/search_doc/warnings). persist_pages
|
||||
// must clear the derived analysis for updated pages so it re-derives.
|
||||
let manga_id = Uuid::new_v4();
|
||||
let chapter_id = Uuid::new_v4();
|
||||
sqlx::query("INSERT INTO mangas (id, title) VALUES ($1, 'T')")
|
||||
.bind(manga_id)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("INSERT INTO chapters (id, manga_id, number) VALUES ($1, $2, 1)")
|
||||
.bind(chapter_id)
|
||||
.bind(manga_id)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let v1 = vec![StoredPage {
|
||||
page_number: 1,
|
||||
storage_key: "k/0001.jpg".into(),
|
||||
content_type: "image/jpeg".into(),
|
||||
size_bytes: 10,
|
||||
}];
|
||||
persist_pages(&pool, chapter_id, &v1, false).await.unwrap();
|
||||
let page_id: Uuid = sqlx::query_scalar(
|
||||
"SELECT id FROM pages WHERE chapter_id = $1 AND page_number = 1",
|
||||
)
|
||||
.bind(chapter_id)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Simulate a completed prior analysis pass for that page.
|
||||
sqlx::query(
|
||||
"INSERT INTO page_analysis (page_id, status, is_nsfw, analyzed_at) \
|
||||
VALUES ($1, 'done', true, now())",
|
||||
)
|
||||
.bind(page_id)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("INSERT INTO page_ocr_text (page_id, kind, text) VALUES ($1, 'speech', 'stale')")
|
||||
.bind(page_id)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("INSERT INTO page_content_warnings (page_id, warning) VALUES ($1, 'gore')")
|
||||
.bind(page_id)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Re-crawl page 1 with a new image (same deterministic key, new size).
|
||||
let v2 = vec![StoredPage {
|
||||
page_number: 1,
|
||||
storage_key: "k/0001.jpg".into(),
|
||||
content_type: "image/jpeg".into(),
|
||||
size_bytes: 999,
|
||||
}];
|
||||
persist_pages(&pool, chapter_id, &v2, false).await.unwrap();
|
||||
|
||||
// The page row survived (deterministic key).
|
||||
let same_id: Uuid = sqlx::query_scalar(
|
||||
"SELECT id FROM pages WHERE chapter_id = $1 AND page_number = 1",
|
||||
)
|
||||
.bind(chapter_id)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(same_id, page_id, "re-crawl keeps the page id");
|
||||
|
||||
// ...but its stale analysis is cleared so it gets re-derived.
|
||||
for table in ["page_analysis", "page_ocr_text", "page_content_warnings"] {
|
||||
let n: i64 = sqlx::query_scalar(&format!(
|
||||
"SELECT COUNT(*) FROM {table} WHERE page_id = $1"
|
||||
))
|
||||
.bind(page_id)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(n, 0, "{table} must be invalidated when the re-crawl updates the page");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -82,6 +82,11 @@ pub(crate) fn verdict(url: &str) -> Verdict {
|
||||
};
|
||||
match parsed.scheme() {
|
||||
"http" | "https" => {}
|
||||
// Non-http(s) subresource schemes: block the ones that can reach the
|
||||
// local filesystem or pivot to another protocol; continue the rest
|
||||
// (data:/blob:/about: and Chrome-internal schemes) which legitimate page
|
||||
// loads depend on — blocking those would wedge navigation.
|
||||
"file" | "ftp" | "gopher" => return Verdict::Block,
|
||||
_ => return Verdict::Allow,
|
||||
}
|
||||
// Literal private IP / localhost / missing host → block (string check).
|
||||
@@ -210,6 +215,15 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn verdict_blocks_dangerous_non_http_schemes() {
|
||||
// file:/ftp:/gopher: can reach the local filesystem or pivot protocols;
|
||||
// block them explicitly rather than falling through to Allow.
|
||||
for url in ["file:///etc/passwd", "ftp://internal/x", "gopher://internal:70/1"] {
|
||||
assert_eq!(verdict(url), Verdict::Block, "{url} should be blocked");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn verdict_blocks_private_ip_literals_and_localhost() {
|
||||
for url in [
|
||||
|
||||
@@ -495,15 +495,33 @@ pub async fn reap_terminal(pool: &PgPool, retention_days: u32) -> sqlx::Result<u
|
||||
if retention_days == 0 {
|
||||
return Ok(0);
|
||||
}
|
||||
let result = sqlx::query(
|
||||
"DELETE FROM crawler_jobs \
|
||||
WHERE state IN ('done', 'dead') \
|
||||
AND updated_at < now() - ($1::bigint || ' days')::interval",
|
||||
)
|
||||
.bind(retention_days as i64)
|
||||
.execute(pool)
|
||||
.await?;
|
||||
Ok(result.rows_affected())
|
||||
// Delete in bounded batches rather than one statement. A single unbatched
|
||||
// DELETE over an unbounded backlog takes a long-held lock and one huge
|
||||
// transaction; batching keeps each delete short (index-backed by
|
||||
// crawler_jobs_terminal_reap_idx, 0040) so the reaper never pins the table
|
||||
// (audit M5). Loop until a batch comes back short.
|
||||
const BATCH: i64 = 5_000;
|
||||
let mut total: u64 = 0;
|
||||
loop {
|
||||
let result = sqlx::query(
|
||||
"DELETE FROM crawler_jobs \
|
||||
WHERE ctid IN ( \
|
||||
SELECT ctid FROM crawler_jobs \
|
||||
WHERE state IN ('done', 'dead') \
|
||||
AND updated_at < now() - ($1::bigint || ' days')::interval \
|
||||
LIMIT $2 )",
|
||||
)
|
||||
.bind(retention_days as i64)
|
||||
.bind(BATCH)
|
||||
.execute(pool)
|
||||
.await?;
|
||||
let n = result.rows_affected();
|
||||
total += n;
|
||||
if n < BATCH as u64 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
Ok(total)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -507,6 +507,26 @@ mod tests {
|
||||
DownloadAllowlist::new().allow(host)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_alternate_ipv4_encodings_of_loopback() {
|
||||
// The `url` crate normalizes special-scheme IPv4 hosts per WHATWG, so
|
||||
// decimal/hex/octal/short forms all collapse to 127.0.0.1 and must be
|
||||
// rejected. This normalization is load-bearing (it's what stops these
|
||||
// encodings smuggling past the literal-IP check) but was untested — pin
|
||||
// it so a future URL-parser swap can't silently reopen the hole.
|
||||
for url in [
|
||||
"http://2130706433/", // decimal 127.0.0.1
|
||||
"http://0x7f000001/", // hex
|
||||
"http://0177.0.0.1/", // octal first octet
|
||||
"http://127.1/", // short form
|
||||
] {
|
||||
assert!(
|
||||
ensure_public_target(url).is_err(),
|
||||
"{url} normalizes to loopback and must be rejected"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn allow_any_admits_arbitrary_public_host() {
|
||||
// Operators who can't pre-enumerate a numbered-CDN fleet
|
||||
|
||||
@@ -207,4 +207,68 @@ mod tests {
|
||||
"text_search_not_yet_supported"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn status_mapping_is_stable() {
|
||||
// Pin the code -> HTTP status mapping so a refactor of into_response
|
||||
// can't silently change a status (e.g. 501 -> 500). The 501 NotImplemented
|
||||
// path and a deterministic 429 Retry-After were previously unexercised
|
||||
// (audit GAP B).
|
||||
let cases: Vec<(AppError, StatusCode)> = vec![
|
||||
(AppError::NotFound, StatusCode::NOT_FOUND),
|
||||
(AppError::InvalidInput("x".into()), StatusCode::BAD_REQUEST),
|
||||
(AppError::Unauthenticated, StatusCode::UNAUTHORIZED),
|
||||
(AppError::Forbidden, StatusCode::FORBIDDEN),
|
||||
(AppError::Conflict("x".into()), StatusCode::CONFLICT),
|
||||
(AppError::PayloadTooLarge("x".into()), StatusCode::PAYLOAD_TOO_LARGE),
|
||||
(
|
||||
AppError::UnsupportedMediaType("x".into()),
|
||||
StatusCode::UNSUPPORTED_MEDIA_TYPE,
|
||||
),
|
||||
(
|
||||
AppError::ServiceUnavailable("x".into()),
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
),
|
||||
(
|
||||
AppError::ValidationFailed {
|
||||
message: "x".into(),
|
||||
details: json!({}),
|
||||
},
|
||||
StatusCode::UNPROCESSABLE_ENTITY,
|
||||
),
|
||||
(
|
||||
AppError::NotImplemented {
|
||||
code: "c",
|
||||
message: "m",
|
||||
},
|
||||
StatusCode::NOT_IMPLEMENTED,
|
||||
),
|
||||
(
|
||||
AppError::TooManyRequests {
|
||||
retry_after_secs: None,
|
||||
},
|
||||
StatusCode::TOO_MANY_REQUESTS,
|
||||
),
|
||||
(
|
||||
AppError::Other(anyhow::anyhow!("boom")),
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
),
|
||||
];
|
||||
for (err, want) in cases {
|
||||
let code = err.code();
|
||||
assert_eq!(err.into_response().status(), want, "status for code {code}");
|
||||
}
|
||||
|
||||
// A 429 with retry_after carries the Retry-After header deterministically
|
||||
// (without needing a live rate limiter).
|
||||
let resp = AppError::TooManyRequests {
|
||||
retry_after_secs: Some(7),
|
||||
}
|
||||
.into_response();
|
||||
assert_eq!(resp.status(), StatusCode::TOO_MANY_REQUESTS);
|
||||
assert_eq!(
|
||||
resp.headers().get(axum::http::header::RETRY_AFTER).unwrap(),
|
||||
"7"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -214,7 +214,7 @@ pub async fn list_mangas_with_sync_state(
|
||||
let search_pat = q
|
||||
.search
|
||||
.as_ref()
|
||||
.map(|s| format!("%{}%", s.trim()))
|
||||
.map(|s| format!("%{}%", crate::repo::escape_like(s.trim())))
|
||||
.filter(|p| p.len() > 2);
|
||||
// sqlx::Type → text: bind the snake_case representation manually so
|
||||
// the SQL can compare it as text without an explicit cast.
|
||||
@@ -235,7 +235,7 @@ pub async fn list_mangas_with_sync_state(
|
||||
(SELECT MAX(last_seen_at) FROM manga_sources ms
|
||||
WHERE ms.manga_id = m.id AND ms.dropped_at IS NULL) AS latest_seen_at
|
||||
FROM mangas m
|
||||
WHERE ($1::text IS NULL OR m.title ILIKE $1)
|
||||
WHERE ($1::text IS NULL OR m.title ILIKE $1 ESCAPE '\')
|
||||
)
|
||||
SELECT * FROM classified
|
||||
WHERE ($2::text IS NULL OR sync_state = $2)
|
||||
@@ -261,7 +261,7 @@ pub async fn list_mangas_with_sync_state(
|
||||
WITH classified AS (
|
||||
SELECT {case} AS sync_state
|
||||
FROM mangas m
|
||||
WHERE ($1::text IS NULL OR m.title ILIKE $1)
|
||||
WHERE ($1::text IS NULL OR m.title ILIKE $1 ESCAPE '\')
|
||||
)
|
||||
SELECT COUNT(*) FROM classified
|
||||
WHERE ($2::text IS NULL OR sync_state = $2)
|
||||
|
||||
@@ -81,7 +81,7 @@ pub async fn list(
|
||||
SELECT id, name, created_at
|
||||
FROM authors
|
||||
WHERE $1::text IS NULL
|
||||
OR name ILIKE '%' || $1 || '%'
|
||||
OR name ILIKE '%' || $4 || '%' ESCAPE '\'
|
||||
OR name % $1
|
||||
ORDER BY CASE WHEN $1::text IS NULL THEN 0 ELSE similarity(name, $1) END DESC,
|
||||
lower(name) ASC
|
||||
@@ -91,6 +91,9 @@ pub async fn list(
|
||||
.bind(search)
|
||||
.bind(limit)
|
||||
.bind(offset)
|
||||
// $4: LIKE-escaped term for the ILIKE branch so `%`/`_` match literally; the
|
||||
// trigram/similarity branches keep the raw $1.
|
||||
.bind(search.map(crate::repo::escape_like))
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
Ok(rows)
|
||||
|
||||
@@ -247,10 +247,13 @@ async fn sync_genres(
|
||||
let genre_id = match existing {
|
||||
Some((id,)) => id,
|
||||
None => {
|
||||
// Conflict on lower(name) (0038) not name, so a racing insert of
|
||||
// a differently-cased variant resolves to the existing row
|
||||
// instead of raising a unique violation.
|
||||
let (id,): (Uuid,) = sqlx::query_as(
|
||||
r#"
|
||||
INSERT INTO genres (name) VALUES ($1)
|
||||
ON CONFLICT (name) DO UPDATE SET name = genres.name
|
||||
ON CONFLICT (lower(name)) DO UPDATE SET name = genres.name
|
||||
RETURNING id
|
||||
"#,
|
||||
)
|
||||
@@ -720,7 +723,7 @@ pub async fn list_dead_jobs(
|
||||
offset: i64,
|
||||
) -> sqlx::Result<(Vec<DeadJob>, i64)> {
|
||||
let search_pat = search
|
||||
.map(|s| format!("%{}%", s.trim()))
|
||||
.map(|s| format!("%{}%", crate::repo::escape_like(s.trim())))
|
||||
.filter(|p| p.len() > 2);
|
||||
|
||||
let items: Vec<DeadJob> = sqlx::query_as(
|
||||
@@ -743,7 +746,7 @@ pub async fn list_dead_jobs(
|
||||
LEFT JOIN chapters c ON c.id = (cj.payload->>'chapter_id')::uuid
|
||||
LEFT JOIN mangas m ON m.id = c.manga_id
|
||||
WHERE cj.state = 'dead'
|
||||
AND ($1::text IS NULL OR m.title ILIKE $1 OR cj.payload->>'title' ILIKE $1)
|
||||
AND ($1::text IS NULL OR m.title ILIKE $1 ESCAPE '\' OR cj.payload->>'title' ILIKE $1 ESCAPE '\')
|
||||
ORDER BY cj.updated_at DESC
|
||||
LIMIT $2 OFFSET $3
|
||||
"#,
|
||||
@@ -761,7 +764,7 @@ pub async fn list_dead_jobs(
|
||||
LEFT JOIN chapters c ON c.id = (cj.payload->>'chapter_id')::uuid
|
||||
LEFT JOIN mangas m ON m.id = c.manga_id
|
||||
WHERE cj.state = 'dead'
|
||||
AND ($1::text IS NULL OR m.title ILIKE $1 OR cj.payload->>'title' ILIKE $1)
|
||||
AND ($1::text IS NULL OR m.title ILIKE $1 ESCAPE '\' OR cj.payload->>'title' ILIKE $1 ESCAPE '\')
|
||||
"#,
|
||||
)
|
||||
.bind(&search_pat)
|
||||
@@ -797,7 +800,7 @@ pub async fn list_active_jobs(
|
||||
offset: i64,
|
||||
) -> sqlx::Result<(Vec<ActiveJob>, i64)> {
|
||||
let search_pat = search
|
||||
.map(|s| format!("%{}%", s.trim()))
|
||||
.map(|s| format!("%{}%", crate::repo::escape_like(s.trim())))
|
||||
.filter(|p| p.len() > 2);
|
||||
|
||||
let items: Vec<ActiveJob> = sqlx::query_as(
|
||||
@@ -817,7 +820,7 @@ pub async fn list_active_jobs(
|
||||
LEFT JOIN mangas m ON m.id = c.manga_id
|
||||
WHERE cj.state IN ('pending','running')
|
||||
AND cj.payload->>'kind' = 'sync_chapter_content'
|
||||
AND ($1::text IS NULL OR m.title ILIKE $1)
|
||||
AND ($1::text IS NULL OR m.title ILIKE $1 ESCAPE '\')
|
||||
ORDER BY (cj.state = 'running') DESC, cj.scheduled_at, cj.created_at
|
||||
LIMIT $2 OFFSET $3
|
||||
"#,
|
||||
@@ -836,7 +839,7 @@ pub async fn list_active_jobs(
|
||||
LEFT JOIN mangas m ON m.id = c.manga_id
|
||||
WHERE cj.state IN ('pending','running')
|
||||
AND cj.payload->>'kind' = 'sync_chapter_content'
|
||||
AND ($1::text IS NULL OR m.title ILIKE $1)
|
||||
AND ($1::text IS NULL OR m.title ILIKE $1 ESCAPE '\')
|
||||
"#,
|
||||
)
|
||||
.bind(&search_pat)
|
||||
@@ -913,7 +916,7 @@ pub async fn list_job_history(
|
||||
) -> sqlx::Result<(Vec<JobHistoryRow>, i64)> {
|
||||
let search_pat = filter
|
||||
.search
|
||||
.map(|s| format!("%{}%", s.trim()))
|
||||
.map(|s| format!("%{}%", crate::repo::escape_like(s.trim())))
|
||||
.filter(|p| p.len() > 2);
|
||||
|
||||
// The same FROM/JOIN/WHERE drives both the page slice and the count, so
|
||||
@@ -951,7 +954,7 @@ pub async fn list_job_history(
|
||||
) cm ON true
|
||||
WHERE ($1::text IS NULL OR cj.state = $1)
|
||||
AND ($2::text IS NULL OR cj.payload->>'kind' = $2)
|
||||
AND ($3::text IS NULL OR m.title ILIKE $3 OR cj.payload->>'title' ILIKE $3)
|
||||
AND ($3::text IS NULL OR m.title ILIKE $3 ESCAPE '\' OR cj.payload->>'title' ILIKE $3 ESCAPE '\')
|
||||
ORDER BY cj.updated_at DESC
|
||||
LIMIT $4 OFFSET $5
|
||||
"#,
|
||||
@@ -973,7 +976,7 @@ pub async fn list_job_history(
|
||||
LEFT JOIN mangas m ON m.id = COALESCE(c.manga_id, (cj.payload->>'manga_id')::uuid)
|
||||
WHERE ($1::text IS NULL OR cj.state = $1)
|
||||
AND ($2::text IS NULL OR cj.payload->>'kind' = $2)
|
||||
AND ($3::text IS NULL OR m.title ILIKE $3 OR cj.payload->>'title' ILIKE $3)
|
||||
AND ($3::text IS NULL OR m.title ILIKE $3 ESCAPE '\' OR cj.payload->>'title' ILIKE $3 ESCAPE '\')
|
||||
"#,
|
||||
)
|
||||
.bind(filter.state)
|
||||
@@ -1018,7 +1021,7 @@ pub async fn list_missing_cover_mangas(
|
||||
offset: i64,
|
||||
) -> sqlx::Result<(Vec<MissingCoverRow>, i64)> {
|
||||
let search_pat = search
|
||||
.map(|s| format!("%{}%", s.trim()))
|
||||
.map(|s| format!("%{}%", crate::repo::escape_like(s.trim())))
|
||||
.filter(|p| p.len() > 2);
|
||||
|
||||
let items: Vec<MissingCoverRow> = sqlx::query_as(
|
||||
@@ -1030,7 +1033,7 @@ pub async fn list_missing_cover_mangas(
|
||||
SELECT 1 FROM manga_sources ms
|
||||
WHERE ms.manga_id = m.id AND ms.dropped_at IS NULL
|
||||
)
|
||||
AND ($1::text IS NULL OR m.title ILIKE $1)
|
||||
AND ($1::text IS NULL OR m.title ILIKE $1 ESCAPE '\')
|
||||
ORDER BY m.updated_at DESC
|
||||
LIMIT $2 OFFSET $3
|
||||
"#,
|
||||
@@ -1049,7 +1052,7 @@ pub async fn list_missing_cover_mangas(
|
||||
SELECT 1 FROM manga_sources ms
|
||||
WHERE ms.manga_id = m.id AND ms.dropped_at IS NULL
|
||||
)
|
||||
AND ($1::text IS NULL OR m.title ILIKE $1)
|
||||
AND ($1::text IS NULL OR m.title ILIKE $1 ESCAPE '\')
|
||||
"#,
|
||||
)
|
||||
.bind(&search_pat)
|
||||
|
||||
@@ -10,7 +10,7 @@ use uuid::Uuid;
|
||||
|
||||
use crate::domain::manga::{Manga, MangaCard, MangaDetail};
|
||||
use crate::error::{AppError, AppResult};
|
||||
use crate::repo;
|
||||
use crate::repo::{self, escape_like};
|
||||
|
||||
/// Status values mirror the CHECK constraint in 0009. Centralized so
|
||||
/// the API layer can validate uploads against the same vocabulary.
|
||||
@@ -110,13 +110,13 @@ fn manga_cols(alias: &str) -> String {
|
||||
/// true.
|
||||
const FILTER_WHERE: &str = r#"
|
||||
($1::text IS NULL
|
||||
OR title ILIKE '%' || $1 || '%'
|
||||
OR title ILIKE '%' || $8 || '%' ESCAPE '\'
|
||||
OR title % $1
|
||||
OR EXISTS (
|
||||
SELECT 1 FROM manga_authors ma
|
||||
JOIN authors a ON a.id = ma.author_id
|
||||
WHERE ma.manga_id = mangas.id
|
||||
AND (a.name ILIKE '%' || $1 || '%' OR a.name % $1)
|
||||
AND (a.name ILIKE '%' || $8 || '%' ESCAPE '\' OR a.name % $1)
|
||||
)
|
||||
)
|
||||
AND ($2::text IS NULL OR status = $2)
|
||||
@@ -196,11 +196,15 @@ pub async fn list(pool: &PgPool, query: &ListQuery) -> AppResult<(Vec<Manga>, Op
|
||||
FROM mangas
|
||||
WHERE {FILTER_WHERE}
|
||||
ORDER BY {order_by}
|
||||
LIMIT $8 OFFSET $9
|
||||
LIMIT $9 OFFSET $10
|
||||
"#,
|
||||
cols = manga_cols(""),
|
||||
);
|
||||
|
||||
// $8 is the LIKE-escaped search term used by the ILIKE branches so `%`/`_`
|
||||
// in the term match literally; the trigram `%` branches keep the raw $1.
|
||||
let search_escaped = search.map(escape_like);
|
||||
|
||||
let rows = sqlx::query_as::<_, Manga>(&list_sql)
|
||||
.bind(search)
|
||||
.bind(status)
|
||||
@@ -209,6 +213,7 @@ pub async fn list(pool: &PgPool, query: &ListQuery) -> AppResult<(Vec<Manga>, Op
|
||||
.bind(&query.tag_ids)
|
||||
.bind(&query.cw_include)
|
||||
.bind(&query.cw_exclude)
|
||||
.bind(&search_escaped)
|
||||
.bind(query.limit)
|
||||
.bind(query.offset)
|
||||
.fetch_all(pool)
|
||||
@@ -235,6 +240,7 @@ pub async fn list(pool: &PgPool, query: &ListQuery) -> AppResult<(Vec<Manga>, Op
|
||||
.bind(&query.tag_ids)
|
||||
.bind(&query.cw_include)
|
||||
.bind(&query.cw_exclude)
|
||||
.bind(&search_escaped)
|
||||
.fetch_one(pool)
|
||||
.await?;
|
||||
Some(total)
|
||||
|
||||
@@ -21,3 +21,45 @@ pub mod tag;
|
||||
pub mod upload_history;
|
||||
pub mod user;
|
||||
pub mod user_preferences;
|
||||
|
||||
/// Escape the LIKE/ILIKE metacharacters (`%`, `_`, and the escape char `\`
|
||||
/// itself) in a user-supplied search term so they match literally rather than
|
||||
/// as wildcards. Pair the resulting value with `ESCAPE '\'` in the SQL — a
|
||||
/// single backslash, which under `standard_conforming_strings` (Postgres
|
||||
/// default) is one backslash in a single-quoted literal.
|
||||
///
|
||||
/// This is a search-correctness fix, not an injection fix: every term is
|
||||
/// already a bound parameter, so `%`/`_` can never break out of the string —
|
||||
/// they were just silently acting as wildcards (`50%` matching everything,
|
||||
/// `a_b` matching `axb`). Callers that build a substring pattern wrap the
|
||||
/// escaped term themselves, e.g. `format!("%{}%", escape_like(term))`.
|
||||
pub(crate) fn escape_like(s: &str) -> String {
|
||||
let mut out = String::with_capacity(s.len());
|
||||
for ch in s.chars() {
|
||||
if matches!(ch, '\\' | '%' | '_') {
|
||||
out.push('\\');
|
||||
}
|
||||
out.push(ch);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::escape_like;
|
||||
|
||||
#[test]
|
||||
fn escapes_wildcards_and_the_escape_char() {
|
||||
assert_eq!(escape_like("50%"), r"50\%");
|
||||
assert_eq!(escape_like("a_b"), r"a\_b");
|
||||
assert_eq!(escape_like(r"back\slash"), r"back\\slash");
|
||||
// A already-escaped-looking input is double-escaped so it stays literal.
|
||||
assert_eq!(escape_like(r"\%"), r"\\\%");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn leaves_ordinary_text_untouched() {
|
||||
assert_eq!(escape_like("naruto"), "naruto");
|
||||
assert_eq!(escape_like(""), "");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -241,6 +241,10 @@ pub async fn manga_coverage(
|
||||
limit: i64,
|
||||
offset: i64,
|
||||
) -> AppResult<(Vec<MangaCoverage>, i64)> {
|
||||
// LIKE-escape so `%`/`_` in the title filter match literally. No trigram
|
||||
// branch here, so binding the escaped term directly (rather than appending a
|
||||
// second param) is safe — $1 feeds only the ILIKE.
|
||||
let search = search.map(crate::repo::escape_like);
|
||||
let rows = sqlx::query_as::<_, MangaCoverage>(
|
||||
r#"
|
||||
SELECT m.id AS manga_id, m.title,
|
||||
@@ -250,13 +254,13 @@ pub async fn manga_coverage(
|
||||
JOIN chapters c ON c.manga_id = m.id
|
||||
JOIN pages p ON p.chapter_id = c.id
|
||||
LEFT JOIN page_analysis pa ON pa.page_id = p.id
|
||||
WHERE ($1::text IS NULL OR m.title ILIKE '%' || $1 || '%')
|
||||
WHERE ($1::text IS NULL OR m.title ILIKE '%' || $1 || '%' ESCAPE '\')
|
||||
GROUP BY m.id, m.title
|
||||
ORDER BY lower(m.title), m.id
|
||||
LIMIT $2 OFFSET $3
|
||||
"#,
|
||||
)
|
||||
.bind(search)
|
||||
.bind(&search)
|
||||
.bind(limit)
|
||||
.bind(offset)
|
||||
.fetch_all(pool)
|
||||
@@ -269,12 +273,12 @@ pub async fn manga_coverage(
|
||||
FROM mangas m
|
||||
JOIN chapters c ON c.manga_id = m.id
|
||||
JOIN pages p ON p.chapter_id = c.id
|
||||
WHERE ($1::text IS NULL OR m.title ILIKE '%' || $1 || '%')
|
||||
WHERE ($1::text IS NULL OR m.title ILIKE '%' || $1 || '%' ESCAPE '\')
|
||||
GROUP BY m.id
|
||||
) x
|
||||
"#,
|
||||
)
|
||||
.bind(search)
|
||||
.bind(&search)
|
||||
.fetch_one(pool)
|
||||
.await?;
|
||||
Ok((rows, total))
|
||||
@@ -377,7 +381,7 @@ pub async fn list_history(
|
||||
) -> AppResult<(Vec<AnalysisHistoryRow>, i64)> {
|
||||
let search_pat = filter
|
||||
.search
|
||||
.map(|s| format!("%{}%", s.trim()))
|
||||
.map(|s| format!("%{}%", crate::repo::escape_like(s.trim())))
|
||||
.filter(|p| p.len() > 2);
|
||||
|
||||
let items = sqlx::query_as::<_, AnalysisHistoryRow>(
|
||||
@@ -402,7 +406,7 @@ pub async fn list_history(
|
||||
WHERE pa.status <> 'pending'
|
||||
AND ($1::text IS NULL OR pa.status = $1)
|
||||
AND ($2::bool IS FALSE OR pa.is_nsfw)
|
||||
AND ($3::text IS NULL OR m.title ILIKE $3)
|
||||
AND ($3::text IS NULL OR m.title ILIKE $3 ESCAPE '\')
|
||||
ORDER BY pa.analyzed_at DESC NULLS LAST, pa.page_id
|
||||
LIMIT $4 OFFSET $5
|
||||
"#,
|
||||
@@ -425,7 +429,7 @@ pub async fn list_history(
|
||||
WHERE pa.status <> 'pending'
|
||||
AND ($1::text IS NULL OR pa.status = $1)
|
||||
AND ($2::bool IS FALSE OR pa.is_nsfw)
|
||||
AND ($3::text IS NULL OR m.title ILIKE $3)
|
||||
AND ($3::text IS NULL OR m.title ILIKE $3 ESCAPE '\')
|
||||
"#,
|
||||
)
|
||||
.bind(filter.status)
|
||||
|
||||
@@ -120,27 +120,11 @@ pub async fn list_for_page(
|
||||
Ok(rows.into_iter().map(|(t,)| t).collect())
|
||||
}
|
||||
|
||||
/// Escape a string for use as a LIKE pattern fragment: `%`, `_`, and
|
||||
/// `\` get a leading backslash so they're matched literally rather
|
||||
/// than as wildcards / escapes. The matching queries below pair this
|
||||
/// with `ESCAPE '\'` for explicitness — a single backslash, since the
|
||||
/// SQL lives in a raw string and Postgres treats `\\` in a single-
|
||||
/// quoted literal as one backslash under `standard_conforming_strings`.
|
||||
///
|
||||
/// The public API rejects `%`/`_`/`\` in `normalize_tag` before
|
||||
/// they reach this repo, so this is defence-in-depth — a future
|
||||
/// internal caller (worker, CLI) that bypasses the normalizer can't
|
||||
/// turn a prefix filter into a wildcard search by accident.
|
||||
fn escape_like_prefix(s: &str) -> String {
|
||||
let mut out = String::with_capacity(s.len());
|
||||
for ch in s.chars() {
|
||||
if matches!(ch, '\\' | '%' | '_') {
|
||||
out.push('\\');
|
||||
}
|
||||
out.push(ch);
|
||||
}
|
||||
out
|
||||
}
|
||||
// LIKE-escaping the autocomplete prefix is defence-in-depth: the public API
|
||||
// already rejects `%`/`_`/`\` in `normalize_tag` before they reach this repo, so
|
||||
// a stray wildcard can only arrive from a future internal caller (worker, CLI)
|
||||
// that bypasses the normalizer. Shared with the other search sites.
|
||||
use crate::repo::escape_like as escape_like_prefix;
|
||||
|
||||
/// Paged list of `user_id`'s tagged pages, with breadcrumb. When
|
||||
/// `tag_filter` is `Some(_)`, restrict to that exact tag (used by the
|
||||
|
||||
@@ -65,3 +65,15 @@ pub async fn delete_by_token_hash(pool: &PgPool, token_hash: &[u8]) -> AppResult
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Delete every session whose `expires_at` has passed, returning the number
|
||||
/// reaped. `find_active` already refuses expired sessions, so this only
|
||||
/// reclaims storage — running it on any schedule (or not at all) is safe.
|
||||
/// Backed by `sessions_expires_idx` (0002). Called by the periodic reaper in
|
||||
/// `app::build`.
|
||||
pub async fn delete_expired(pool: &PgPool) -> AppResult<u64> {
|
||||
let result = sqlx::query("DELETE FROM sessions WHERE expires_at <= now()")
|
||||
.execute(pool)
|
||||
.await?;
|
||||
Ok(result.rows_affected())
|
||||
}
|
||||
|
||||
@@ -124,7 +124,7 @@ pub async fn list(
|
||||
SELECT id, name, created_at
|
||||
FROM tags
|
||||
WHERE $1::text IS NULL
|
||||
OR name ILIKE '%' || $1 || '%'
|
||||
OR name ILIKE '%' || $3 || '%' ESCAPE '\'
|
||||
OR name % $1
|
||||
ORDER BY CASE WHEN $1::text IS NULL THEN 0 ELSE similarity(name, $1) END DESC,
|
||||
lower(name) ASC
|
||||
@@ -133,6 +133,9 @@ pub async fn list(
|
||||
)
|
||||
.bind(search)
|
||||
.bind(limit)
|
||||
// $3: LIKE-escaped term for the ILIKE branch so `%`/`_` match literally; the
|
||||
// trigram/similarity branches keep the raw $1.
|
||||
.bind(search.map(crate::repo::escape_like))
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
Ok(rows)
|
||||
|
||||
@@ -106,14 +106,14 @@ pub async fn list_with_total(
|
||||
let pat = q
|
||||
.search
|
||||
.as_ref()
|
||||
.map(|s| format!("%{}%", s.trim()))
|
||||
.map(|s| format!("%{}%", crate::repo::escape_like(s.trim())))
|
||||
.filter(|p| p.len() > 2);
|
||||
|
||||
let items = sqlx::query_as::<_, User>(
|
||||
r#"
|
||||
SELECT id, username, password_hash, created_at, is_admin
|
||||
FROM users
|
||||
WHERE ($1::text IS NULL OR username ILIKE $1)
|
||||
WHERE ($1::text IS NULL OR username ILIKE $1 ESCAPE '\')
|
||||
ORDER BY username
|
||||
LIMIT $2 OFFSET $3
|
||||
"#,
|
||||
@@ -125,7 +125,7 @@ pub async fn list_with_total(
|
||||
.await?;
|
||||
|
||||
let total: i64 = sqlx::query_scalar(
|
||||
"SELECT COUNT(*) FROM users WHERE ($1::text IS NULL OR username ILIKE $1)",
|
||||
"SELECT COUNT(*) FROM users WHERE ($1::text IS NULL OR username ILIKE $1 ESCAPE '\\')",
|
||||
)
|
||||
.bind(&pat)
|
||||
.fetch_one(pool)
|
||||
|
||||
@@ -95,6 +95,11 @@ impl Storage for LocalStorage {
|
||||
match fs::read(&path).await {
|
||||
Ok(b) => Ok(b),
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Err(StorageError::NotFound),
|
||||
// A key resolving to a directory isn't a stored blob; `fs::read`
|
||||
// fails with EISDIR. Treat it as absent, mirroring `size`.
|
||||
Err(e) if e.kind() == std::io::ErrorKind::IsADirectory => {
|
||||
Err(StorageError::NotFound)
|
||||
}
|
||||
Err(e) => Err(e.into()),
|
||||
}
|
||||
}
|
||||
@@ -108,7 +113,14 @@ impl Storage for LocalStorage {
|
||||
}
|
||||
Err(e) => return Err(e.into()),
|
||||
};
|
||||
let size_bytes = file.metadata().await?.len();
|
||||
let meta = file.metadata().await?;
|
||||
// Opening a directory succeeds on Unix, but it isn't a stored blob:
|
||||
// its inode "size" is meaningless and ReaderStream would fail mid-read.
|
||||
// Treat it as absent, mirroring `size` and `get`.
|
||||
if !meta.is_file() {
|
||||
return Err(StorageError::NotFound);
|
||||
}
|
||||
let size_bytes = meta.len();
|
||||
// 64 KiB chunks: small enough that a few-MB page emits many frames
|
||||
// (so streaming is observable), large enough to keep syscalls cheap.
|
||||
let stream = ReaderStream::with_capacity(file, 64 * 1024);
|
||||
@@ -244,6 +256,21 @@ mod tests {
|
||||
assert!(matches!(s.size("adir").await, Err(StorageError::NotFound)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn get_on_directory_is_not_found() {
|
||||
// A key resolving to a directory isn't a stored blob. `fs::read` on a
|
||||
// dir errors with EISDIR (not NotFound), and `File::open` on a dir
|
||||
// succeeds on Unix then streams garbage — both must surface NotFound.
|
||||
let dir = tempdir().unwrap();
|
||||
let s = LocalStorage::new(dir.path());
|
||||
std::fs::create_dir(dir.path().join("adir")).unwrap();
|
||||
assert!(matches!(s.get("adir").await, Err(StorageError::NotFound)));
|
||||
assert!(matches!(
|
||||
s.get_stream("adir").await.err(),
|
||||
Some(StorageError::NotFound)
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn put_stream_writes_full_body_and_removes_temp_on_error() {
|
||||
use bytes::Bytes;
|
||||
|
||||
@@ -37,6 +37,15 @@ pub struct StagedImage {
|
||||
/// staged page out of this prefix.
|
||||
pub const STAGING_PREFIX: &str = "staging";
|
||||
|
||||
/// Upper bound on a multipart `metadata` JSON part, enforced as bytes arrive
|
||||
/// (via [`read_capped`]). Manga/chapter metadata is title + a few short lists +
|
||||
/// a description — kilobytes at most. Without this, `metadata` was the one
|
||||
/// remaining part read with the unbounded `Field::bytes()`, letting a client
|
||||
/// buffer up to the whole 200 MiB request body in memory as a single JSON blob
|
||||
/// before any validation ran. 64 KiB is generous headroom over any legitimate
|
||||
/// payload while keeping the worst case tiny.
|
||||
pub const MAX_METADATA_BYTES: usize = 64 * 1024;
|
||||
|
||||
/// Read one multipart image part and write it straight to a staging key,
|
||||
/// returning only its metadata. The per-file byte cap is enforced as bytes
|
||||
/// arrive, so an oversized part is rejected (413) without being fully
|
||||
|
||||
@@ -189,6 +189,51 @@ async fn reenqueue_returns_503_when_analysis_disabled(pool: PgPool) {
|
||||
assert_eq!(resp.status(), StatusCode::SERVICE_UNAVAILABLE);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn reenqueue_rejects_a_malformed_body(pool: PgPool) {
|
||||
// A present-but-malformed JSON body must be a 422 — NOT silently coerced to
|
||||
// an absent body, which would run the default full "All" scope the caller
|
||||
// never asked for and enqueue jobs across the whole library.
|
||||
let h = common::harness_with_analysis(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
seed_pages(&pool, 2).await;
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::post_raw_with_cookie(
|
||||
"/api/v1/admin/analysis/reenqueue",
|
||||
"{ not valid json",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::UNPROCESSABLE_ENTITY);
|
||||
assert_eq!(analyze_job_count(&pool).await, 0, "a rejected body enqueues nothing");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn reenqueue_treats_empty_body_as_default_all_scope(pool: PgPool) {
|
||||
// An absent/empty body is still valid: it means the default "All" scope.
|
||||
let h = common::harness_with_analysis(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
seed_pages(&pool, 2).await;
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::post_raw_with_cookie(
|
||||
"/api/v1/admin/analysis/reenqueue",
|
||||
"",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = common::body_json(resp).await;
|
||||
assert_eq!(body["enqueued"], 2, "empty body backfills the whole library");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn reenqueue_backfills_existing_pages(pool: PgPool) {
|
||||
let h = common::harness_with_analysis(pool.clone());
|
||||
|
||||
@@ -147,6 +147,46 @@ async fn list_filters_by_substring_search(pool: PgPool) {
|
||||
assert_eq!(body["page"]["total"], 1);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn search_treats_like_wildcards_literally(pool: PgPool) {
|
||||
let h = common::harness(pool.clone());
|
||||
let (_admin_name, cookie, _) = seed_admin(&pool, &h.app).await;
|
||||
// Two usernames differing only at one position (underscores are legal in
|
||||
// usernames). `_` is a LIKE single-char wildcard: unescaped, `%a_b%` matches
|
||||
// BOTH; escaped, only the literal "a_b" username. Admin user search has no
|
||||
// trigram OR, so length/similarity don't matter here.
|
||||
for username in ["axbfindme", "a_bfindme"] {
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::post_json(
|
||||
"/api/v1/auth/register",
|
||||
json!({ "username": username, "password": "hunter2hunter2" }),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::CREATED);
|
||||
}
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::get_with_cookie(
|
||||
"/api/v1/admin/users?search=a_b",
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = common::body_json(resp).await;
|
||||
let items = body["items"].as_array().unwrap();
|
||||
assert_eq!(
|
||||
items.len(),
|
||||
1,
|
||||
"the `_` must match literally: only a_bfindme, not axbfindme"
|
||||
);
|
||||
assert_eq!(items[0]["username"], "a_bfindme");
|
||||
}
|
||||
|
||||
// ---- self-protection -------------------------------------------------------
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
|
||||
@@ -9,6 +9,73 @@ fn creds(username: &str) -> serde_json::Value {
|
||||
json!({ "username": username, "password": "hunter2hunter2" })
|
||||
}
|
||||
|
||||
/// A login request carrying a chosen `X-Forwarded-For`. Empty password so the
|
||||
/// handler consumes a rate-limit token then short-circuits with 400 before any
|
||||
/// argon2/DB work — the burst drains near-instantly regardless of CI load.
|
||||
fn login_from(xff: &str) -> axum::http::Request<axum::body::Body> {
|
||||
axum::http::Request::builder()
|
||||
.method("POST")
|
||||
.uri("/api/v1/auth/login")
|
||||
.header(header::CONTENT_TYPE, "application/json")
|
||||
.header("x-forwarded-for", xff)
|
||||
.body(axum::body::Body::from(
|
||||
json!({ "username": "victim", "password": "" }).to_string(),
|
||||
))
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
// The per-IP rate limiter's soundness hinges on one gate: X-Forwarded-For is
|
||||
// honored ONLY when AUTH_TRUSTED_PROXY is set. These two tests pin both sides of
|
||||
// that gate at the request level (the pure parser is unit-tested separately).
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn trusted_proxy_gives_each_forwarded_ip_its_own_bucket(pool: PgPool) {
|
||||
let h = common::harness_with_auth_rate_limit_proxy(pool, 1, 2, true);
|
||||
// Drain IP A's bucket (per_sec=1, burst=2) until it 429s.
|
||||
let mut a_saw_429 = false;
|
||||
for _ in 0..8 {
|
||||
let resp = h.app.clone().oneshot(login_from("203.0.113.10")).await.unwrap();
|
||||
if resp.status() == StatusCode::TOO_MANY_REQUESTS {
|
||||
a_saw_429 = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
assert!(a_saw_429, "IP A must be rate-limited after draining its own bucket");
|
||||
// A different forwarded IP has an independent bucket — its first hit is not 429.
|
||||
let resp_b = h.app.clone().oneshot(login_from("198.51.100.20")).await.unwrap();
|
||||
assert_ne!(
|
||||
resp_b.status(),
|
||||
StatusCode::TOO_MANY_REQUESTS,
|
||||
"a distinct X-Forwarded-For hop must get its own bucket, not A's drained one"
|
||||
);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn untrusted_proxy_ignores_forwarded_ip_and_shares_one_bucket(pool: PgPool) {
|
||||
let h = common::harness_with_auth_rate_limit_proxy(pool, 1, 2, false);
|
||||
// Every request carries a DISTINCT (spoofed) X-Forwarded-For, but the backend
|
||||
// doesn't trust it — so they all fall back to the single global bucket, which
|
||||
// drains and starts returning 429. If XFF were wrongly honored here, each
|
||||
// unique IP would get a fresh bucket and none of these would ever 429.
|
||||
let mut saw_429 = false;
|
||||
for i in 0..12 {
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(login_from(&format!("10.0.0.{i}")))
|
||||
.await
|
||||
.unwrap();
|
||||
if resp.status() == StatusCode::TOO_MANY_REQUESTS {
|
||||
saw_429 = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
saw_429,
|
||||
"with trusted_proxy off, spoofed X-Forwarded-For must NOT dodge the shared limiter"
|
||||
);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn register_creates_user_and_sets_session_cookie(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
|
||||
@@ -30,6 +30,47 @@ fn first_author_id(manga: &Value) -> String {
|
||||
manga["authors"][0]["id"].as_str().unwrap().to_string()
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn search_treats_like_wildcards_literally(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
let (_, cookie) = common::register_user(&h.app).await;
|
||||
// Long author names differing only at one position, short search term — so
|
||||
// the trigram OR (which keeps the raw term by design) stays under threshold
|
||||
// and the ILIKE branch is what decides. Unescaped `%a_b%` matches both the
|
||||
// "axb" and "a_b" names; escaped, only the literal "a_b" name.
|
||||
create_manga(
|
||||
&h.app,
|
||||
&cookie,
|
||||
json!({ "title": "M1", "authors": ["The Quick Brown Fox axb Jumps Over"] }),
|
||||
)
|
||||
.await;
|
||||
create_manga(
|
||||
&h.app,
|
||||
&cookie,
|
||||
json!({ "title": "M2", "authors": ["The Quick Brown Fox a_b Jumps Over"] }),
|
||||
)
|
||||
.await;
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::get("/api/v1/authors?search=a_b"))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = common::body_json(resp).await;
|
||||
let names: Vec<&str> = body
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|a| a["name"].as_str().unwrap())
|
||||
.collect();
|
||||
assert_eq!(
|
||||
names,
|
||||
vec!["The Quick Brown Fox a_b Jumps Over"],
|
||||
"the `_` in the search term must match literally, not as a wildcard"
|
||||
);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn get_returns_name_and_manga_count(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
|
||||
@@ -358,3 +358,27 @@ async fn non_owner_can_upload_chapter(pool: PgPool) {
|
||||
assert_eq!(body["title"], "Contributed");
|
||||
assert_eq!(body["page_count"], 1);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn chapter_upload_rejects_oversized_metadata_part(pool: PgPool) {
|
||||
// The chapter `metadata` JSON part is capped as bytes arrive, just like the
|
||||
// manga one, so a client can't buffer a huge JSON blob in memory. A ~200 KiB
|
||||
// title blows the metadata cap before any page is staged.
|
||||
let h = common::harness(pool);
|
||||
let (_, cookie) = common::register_user(&h.app).await;
|
||||
let manga_id = seed_manga(&h, &cookie, "Berserk").await;
|
||||
|
||||
let huge = "A".repeat(200 * 1024);
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::post_multipart_with_cookie(
|
||||
&format!("/api/v1/mangas/{manga_id}/chapters"),
|
||||
MultipartBuilder::new()
|
||||
.add_json("metadata", json!({ "number": 1, "title": huge }))
|
||||
.add_file("page", "1.png", "image/png", &common::fake_png_bytes()),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::PAYLOAD_TOO_LARGE);
|
||||
}
|
||||
|
||||
74
backend/tests/api_files.rs
Normal file
74
backend/tests/api_files.rs
Normal file
@@ -0,0 +1,74 @@
|
||||
mod common;
|
||||
|
||||
use axum::http::StatusCode;
|
||||
use tower::ServiceExt;
|
||||
|
||||
/// Write a blob straight into the harness storage root at `key`, bypassing the
|
||||
/// upload handlers — the only way to land a key whose extension resolves to the
|
||||
/// `application/octet-stream` fallback (uploads always mint image extensions).
|
||||
fn write_blob(h: &common::Harness, key: &str, bytes: &[u8]) {
|
||||
let path = h._storage_dir.path().join(key);
|
||||
std::fs::create_dir_all(path.parent().unwrap()).unwrap();
|
||||
std::fs::write(path, bytes).unwrap();
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn octet_stream_blobs_are_served_as_attachment(pool: sqlx::PgPool) {
|
||||
// A blob with an unknown extension serves as application/octet-stream. Such a
|
||||
// body could be crafted HTML/JS, so it must never render inline: force a
|
||||
// download with Content-Disposition: attachment (nosniff is already set).
|
||||
let h = common::harness(pool);
|
||||
write_blob(&h, "misc/blob.bin", b"\x00\x01not-an-image");
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::get("/api/v1/files/misc/blob.bin"))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
assert_eq!(
|
||||
resp.headers().get("content-type").unwrap(),
|
||||
"application/octet-stream"
|
||||
);
|
||||
assert_eq!(
|
||||
resp.headers().get("content-disposition").unwrap(),
|
||||
"attachment"
|
||||
);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn image_blobs_are_served_inline(pool: sqlx::PgPool) {
|
||||
// Regression guard: known image types keep rendering inline (no attachment
|
||||
// disposition), so covers/pages still display in the reader.
|
||||
let h = common::harness(pool);
|
||||
write_blob(&h, "misc/pic.png", &common::fake_png_bytes());
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::get("/api/v1/files/misc/pic.png"))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
assert_eq!(resp.headers().get("content-type").unwrap(), "image/png");
|
||||
assert!(
|
||||
resp.headers().get("content-disposition").is_none(),
|
||||
"images must render inline, not download"
|
||||
);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn corrupt_image_thumbnail_falls_back_to_original_not_500(pool: sqlx::PgPool) {
|
||||
// A blob with valid PNG magic bytes but an undecodable body passes upload's
|
||||
// magic-byte sniff; a ?w= thumbnail request then tries to decode it. That
|
||||
// must not 500 the reader — fall back to streaming the original bytes.
|
||||
let h = common::harness(pool);
|
||||
write_blob(&h, "misc/corrupt.png", &common::fake_png_bytes());
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::get("/api/v1/files/misc/corrupt.png?w=320"))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK, "corrupt thumbnail must not 500");
|
||||
assert_eq!(resp.headers().get("content-type").unwrap(), "image/png");
|
||||
}
|
||||
@@ -135,6 +135,30 @@ async fn list_total_is_computed_only_on_the_first_page(pool: PgPool) {
|
||||
assert_eq!(body1["items"].as_array().unwrap().len(), 1);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn search_treats_like_wildcards_literally(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
let (_, cookie) = common::register_user(&h.app).await;
|
||||
// Two long titles differing only at one position. `_` is a LIKE single-char
|
||||
// wildcard: unescaped, `%a_b%` matches BOTH ("axb" and "a_b"); escaped, only
|
||||
// the literal "a_b" title matches. The titles are long and the term short so
|
||||
// trigram similarity stays under threshold — the ILIKE branch decides.
|
||||
seed(&h.app, &cookie, "The Quick Brown Fox Jumps axb Over The Lazy Dog").await;
|
||||
seed(&h.app, &cookie, "The Quick Brown Fox Jumps a_b Over The Lazy Dog").await;
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::get("/api/v1/mangas?search=a_b"))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = common::body_json(resp).await;
|
||||
assert_eq!(
|
||||
title_list(&body),
|
||||
vec!["The Quick Brown Fox Jumps a_b Over The Lazy Dog"],
|
||||
"the `_` in the search term must match literally, not as a wildcard"
|
||||
);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn search_via_trigram_tolerates_typos(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
|
||||
@@ -927,3 +927,25 @@ async fn bearer_authed_admin_cannot_edit_null_uploader(pool: PgPool) {
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn create_rejects_oversized_metadata_part(pool: PgPool) {
|
||||
// The metadata JSON part is capped well below the 200 MiB request limit so a
|
||||
// client can't force the server to buffer a huge JSON blob in memory before
|
||||
// any validation runs. A ~200 KiB description blows the metadata cap.
|
||||
let h = common::harness(pool);
|
||||
let (_, cookie) = common::register_user(&h.app).await;
|
||||
|
||||
let huge = "A".repeat(200 * 1024);
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::post_multipart_with_cookie(
|
||||
"/api/v1/mangas",
|
||||
MultipartBuilder::new()
|
||||
.add_json("metadata", json!({ "title": "Big", "description": huge })),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::PAYLOAD_TOO_LARGE);
|
||||
}
|
||||
|
||||
@@ -148,6 +148,27 @@ pub fn harness_with_auth_rate_limit(
|
||||
harness_with_auth_config(pool, storage, storage_dir, auth)
|
||||
}
|
||||
|
||||
/// Like [`harness_with_auth_rate_limit`] but also sets whether the backend
|
||||
/// trusts `X-Forwarded-For` (`AUTH_TRUSTED_PROXY`). Used to prove the per-IP
|
||||
/// rate-limit gate: with `trusted_proxy` on, distinct XFF hops get independent
|
||||
/// buckets; with it off, XFF is ignored and everything shares the global bucket.
|
||||
pub fn harness_with_auth_rate_limit_proxy(
|
||||
pool: PgPool,
|
||||
per_sec: u32,
|
||||
burst: u32,
|
||||
trusted_proxy: bool,
|
||||
) -> Harness {
|
||||
let storage_dir = tempfile::tempdir().expect("tempdir");
|
||||
let storage = Arc::new(LocalStorage::new(storage_dir.path()));
|
||||
let auth = AuthConfig {
|
||||
cookie_secure: false,
|
||||
trusted_proxy,
|
||||
rate_limit: mangalord::auth::rate_limit::RateLimitConfig { per_sec, burst },
|
||||
..AuthConfig::default()
|
||||
};
|
||||
harness_with_auth_config(pool, storage, storage_dir, auth)
|
||||
}
|
||||
|
||||
/// Like [`harness`] but slots a caller-supplied [`ResyncService`] stub
|
||||
/// into `AppState.resync`. Used by the admin resync tests so the
|
||||
/// endpoint path is exercised without standing up a real Chromium.
|
||||
@@ -475,6 +496,20 @@ pub fn post_json_with_cookie(
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
/// Like [`post_json_with_cookie`] but sends a raw (possibly malformed) body,
|
||||
/// for exercising body-parse error handling. Content-Type stays JSON and the
|
||||
/// CSRF Origin is attached so the request reaches the handler.
|
||||
pub fn post_raw_with_cookie(uri: &str, body: &str, cookie: &str) -> Request<Body> {
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri(uri)
|
||||
.header(header::CONTENT_TYPE, "application/json")
|
||||
.header(header::COOKIE, cookie)
|
||||
.header(header::ORIGIN, TEST_ORIGIN)
|
||||
.body(Body::from(body.to_string()))
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
/// Same as [`post_json_with_cookie`] but also attaches `Origin` (and
|
||||
/// optionally `Referer`) headers. Used by the admin CSRF tests to drive
|
||||
/// the cross-origin reject + allowed-origin accept paths.
|
||||
|
||||
@@ -674,6 +674,110 @@ async fn arbitrary_genres_from_source_get_inserted(pool: PgPool) {
|
||||
assert_eq!(webtoons_count.0, 1, "case-insensitive lookup reuses the existing row");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn genre_dedup_survives_a_manga_linked_to_two_variants(pool: PgPool) {
|
||||
// Regression for a migration-0038 collision: a manga linked to TWO
|
||||
// non-canonical case-variants of one genre. A repoint-UPDATE would set both
|
||||
// rows to the canonical id in one statement -> manga_genres PK violation ->
|
||||
// migration rollback -> boot failure. #[sqlx::test] migrates a clean DB, so
|
||||
// recreate the dirty pre-index state and replay 0038's healing statements;
|
||||
// they must complete and leave exactly the one canonical link. (Mirrors the
|
||||
// healing SQL in migrations/0038_genres_name_lower_unique.sql.)
|
||||
let manga = Uuid::new_v4();
|
||||
sqlx::query("INSERT INTO mangas (id, title) VALUES ($1, 'T')")
|
||||
.bind(manga)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Drop the live guard so we can plant case-variant genres.
|
||||
sqlx::query("DROP INDEX genres_name_lower_uniq").execute(&pool).await.unwrap();
|
||||
let keep = Uuid::parse_str("00000000-0000-0000-0000-000000000001").unwrap();
|
||||
let dup2 = Uuid::parse_str("00000000-0000-0000-0000-000000000002").unwrap();
|
||||
let dup3 = Uuid::parse_str("00000000-0000-0000-0000-000000000003").unwrap();
|
||||
for (id, name) in [(keep, "Zzz"), (dup2, "zzz"), (dup3, "ZZZ")] {
|
||||
sqlx::query("INSERT INTO genres (id, name) VALUES ($1, $2)")
|
||||
.bind(id)
|
||||
.bind(name)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
// The manga links to the two NON-canonical variants, not the canonical one.
|
||||
for gid in [dup2, dup3] {
|
||||
sqlx::query("INSERT INTO manga_genres (manga_id, genre_id) VALUES ($1, $2)")
|
||||
.bind(manga)
|
||||
.bind(gid)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
// Step 1 — collision-proof canonical-link backfill (the crux of the fix).
|
||||
sqlx::query(
|
||||
"INSERT INTO manga_genres (manga_id, genre_id) \
|
||||
SELECT DISTINCT mg.manga_id, k.keep_id \
|
||||
FROM manga_genres mg JOIN genres g ON mg.genre_id = g.id \
|
||||
JOIN (SELECT lower(name) AS lname, (array_agg(id ORDER BY id))[1] AS keep_id \
|
||||
FROM genres GROUP BY lower(name)) k ON lower(g.name) = k.lname \
|
||||
WHERE g.id <> k.keep_id \
|
||||
ON CONFLICT (manga_id, genre_id) DO NOTHING",
|
||||
)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.expect("collision-proof INSERT must not raise a manga_genres PK violation");
|
||||
// Step 2 — drop the non-canonical links.
|
||||
sqlx::query(
|
||||
"DELETE FROM manga_genres mg USING genres g, \
|
||||
(SELECT lower(name) AS lname, (array_agg(id ORDER BY id))[1] AS keep_id \
|
||||
FROM genres GROUP BY lower(name)) k \
|
||||
WHERE mg.genre_id = g.id AND lower(g.name) = k.lname AND g.id <> k.keep_id",
|
||||
)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
// Step 3 — remove orphaned duplicate genres.
|
||||
sqlx::query(
|
||||
"DELETE FROM genres g USING \
|
||||
(SELECT lower(name) AS lname, (array_agg(id ORDER BY id))[1] AS keep_id \
|
||||
FROM genres GROUP BY lower(name)) k \
|
||||
WHERE lower(g.name) = k.lname AND g.id <> k.keep_id",
|
||||
)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Exactly the canonical link remains, and the guard re-creates cleanly.
|
||||
let links: Vec<Uuid> =
|
||||
sqlx::query_scalar("SELECT genre_id FROM manga_genres WHERE manga_id = $1")
|
||||
.bind(manga)
|
||||
.fetch_all(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(links, vec![keep], "manga keeps exactly the canonical genre link");
|
||||
sqlx::query("CREATE UNIQUE INDEX genres_name_lower_uniq ON genres (lower(name))")
|
||||
.execute(&pool)
|
||||
.await
|
||||
.expect("no residual case-variant duplicates remain");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn genres_reject_case_variant_duplicates_at_the_db(pool: PgPool) {
|
||||
// The sequential pre-check in sync_genres dedups the common case, but two
|
||||
// concurrent inserts could each miss it. The lower(name) unique index (0038)
|
||||
// is the race backstop: a case variant of an existing genre must be rejected
|
||||
// by the DB itself. "Action" is seeded by 0009.
|
||||
let dup = sqlx::query("INSERT INTO genres (name) VALUES ('action')")
|
||||
.execute(&pool)
|
||||
.await;
|
||||
let err = dup.expect_err("a case-variant of a seeded genre must violate the unique index");
|
||||
let msg = err.to_string();
|
||||
assert!(
|
||||
msg.contains("genres_name_lower_uniq") || msg.contains("unique") || msg.contains("duplicate"),
|
||||
"expected a unique-violation, got: {msg}"
|
||||
);
|
||||
}
|
||||
|
||||
/// User-attached tags (rows with non-NULL `added_by` in `manga_tags`)
|
||||
/// must survive a crawler upsert. The crawler owns source-attached tags
|
||||
/// (added_by IS NULL); user attachments are owned by the user who made
|
||||
|
||||
52
backend/tests/repo_session.rs
Normal file
52
backend/tests/repo_session.rs
Normal file
@@ -0,0 +1,52 @@
|
||||
mod common;
|
||||
|
||||
use chrono::{Duration, Utc};
|
||||
use sqlx::PgPool;
|
||||
|
||||
use mangalord::repo;
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn delete_expired_removes_only_lapsed_sessions(pool: PgPool) {
|
||||
let user = repo::user::create(&pool, "reaper-subject", "x").await.unwrap();
|
||||
|
||||
// One session already lapsed, one still valid.
|
||||
let expired = repo::session::create(
|
||||
&pool,
|
||||
user.id,
|
||||
b"expired-token-hash-00000000000000",
|
||||
Utc::now() - Duration::hours(1),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let active = repo::session::create(
|
||||
&pool,
|
||||
user.id,
|
||||
b"active-token-hash-000000000000000",
|
||||
Utc::now() + Duration::hours(1),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let removed = repo::session::delete_expired(&pool).await.unwrap();
|
||||
assert_eq!(removed, 1, "exactly the one lapsed session is reaped");
|
||||
|
||||
// The active session is untouched and still resolvable; the expired one is
|
||||
// gone from the table entirely (find_active already ignored it, but now the
|
||||
// row is reclaimed too).
|
||||
assert!(
|
||||
repo::session::find_active(&pool, b"active-token-hash-000000000000000")
|
||||
.await
|
||||
.unwrap()
|
||||
.is_some(),
|
||||
"active session survives the sweep"
|
||||
);
|
||||
let remaining: i64 = sqlx::query_scalar("SELECT count(*) FROM sessions")
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(remaining, 1);
|
||||
|
||||
// Sweeping again is a no-op now that nothing is expired.
|
||||
assert_eq!(repo::session::delete_expired(&pool).await.unwrap(), 0);
|
||||
let _ = (expired, active);
|
||||
}
|
||||
@@ -23,6 +23,12 @@ services:
|
||||
# takes the whole stack offline. Match the policy already set on
|
||||
# tor / docker-socket-proxy / vision-manager.
|
||||
restart: unless-stopped
|
||||
# Bound Postgres so a runaway query/backlog can't consume all host RAM.
|
||||
# Tune to your host via .env (should comfortably exceed shared_buffers +
|
||||
# work_mem * max_connections).
|
||||
mem_limit: ${POSTGRES_MEM_LIMIT:-1g}
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
|
||||
tor:
|
||||
# SOCKS5 proxy for the crawler, plus a control port so the backend
|
||||
@@ -112,7 +118,7 @@ services:
|
||||
CRAWLER_JOB_TIMEOUT_SECS: ${CRAWLER_JOB_TIMEOUT_SECS:-600}
|
||||
CRAWLER_METADATA_MAX_CONSECUTIVE_FAILURES: ${CRAWLER_METADATA_MAX_CONSECUTIVE_FAILURES:-10}
|
||||
CRAWLER_BROWSER_RESTART_THRESHOLD: ${CRAWLER_BROWSER_RESTART_THRESHOLD:-3}
|
||||
CRAWLER_SSRF_INTERCEPT: ${CRAWLER_SSRF_INTERCEPT:-false}
|
||||
CRAWLER_SSRF_INTERCEPT: ${CRAWLER_SSRF_INTERCEPT:-true}
|
||||
# Crawler daemon schedule + retention. CRAWLER_DAEMON=false keeps
|
||||
# the in-process scheduler off; the dashboard force-resync still works.
|
||||
CRAWLER_DAEMON: ${CRAWLER_DAEMON:-true}
|
||||
@@ -197,6 +203,13 @@ services:
|
||||
expose:
|
||||
- "8080"
|
||||
restart: unless-stopped
|
||||
# Bound the heaviest consumer: the backend hosts the OCR/analysis worker and
|
||||
# drives a headless Chromium crawler, either of which can balloon and
|
||||
# OOM-kill the host (taking Postgres with it) with no limit. Tune to your
|
||||
# host via .env — this default is a generous ceiling, not a target.
|
||||
mem_limit: ${BACKEND_MEM_LIMIT:-4g}
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
|
||||
frontend:
|
||||
build: ./frontend
|
||||
@@ -213,8 +226,17 @@ services:
|
||||
# to 300000 (5 min) in hooks.server.ts; raise/lower via .env.
|
||||
BACKEND_PROXY_TIMEOUT_MS: ${BACKEND_PROXY_TIMEOUT_MS:-300000}
|
||||
ports:
|
||||
- "3000:3000"
|
||||
# Bind to loopback by default so SvelteKit (plain HTTP; COOKIE_SECURE=true
|
||||
# expects TLS in front) is reachable ONLY by a host-local reverse proxy,
|
||||
# not cleartext over the LAN/internet. A host-external TLS terminator can
|
||||
# set FRONTEND_PUBLISH_ADDR=0.0.0.0 (or a specific interface) in .env.
|
||||
- "${FRONTEND_PUBLISH_ADDR:-127.0.0.1}:3000:3000"
|
||||
restart: unless-stopped
|
||||
# Cap runaway memory (SvelteKit is light, but bound it anyway) and forbid
|
||||
# privilege escalation. Tune the limit to your host via .env.
|
||||
mem_limit: ${FRONTEND_MEM_LIMIT:-512m}
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
|
||||
# ----- Vision autoscaling (profile: ai) -----------------------------------
|
||||
# Two extra containers that idle-stop the mangalord-vision (llama.cpp)
|
||||
|
||||
10
frontend/csp-config.js
Normal file
10
frontend/csp-config.js
Normal file
@@ -0,0 +1,10 @@
|
||||
// Shared CSP constants, kept dependency-free so both svelte.config.js (loaded by
|
||||
// Node when Vite starts) and the vitest drift-guard test (jsdom env) can import
|
||||
// it without pulling in adapter-node / esbuild.
|
||||
|
||||
// sha256 of the inline theme <script> in src/app.html, base64-encoded, wrapped
|
||||
// for the CSP `script-src` allowlist. SvelteKit's kit.csp hash mode does not
|
||||
// cover app.html template scripts, so this is pinned by hand. src/csp-theme-hash.test.ts
|
||||
// recomputes it from app.html and fails if it drifts.
|
||||
export const THEME_SCRIPT_HASH =
|
||||
"'sha256-qj6Oim9siqow/Su+v47sZHJeJci+M/RQwGXn53eSULQ='";
|
||||
37
frontend/e2e/security-headers.spec.ts
Normal file
37
frontend/e2e/security-headers.spec.ts
Normal file
@@ -0,0 +1,37 @@
|
||||
import { test, expect } from './fixtures';
|
||||
|
||||
// Defense-in-depth response headers on document navigations (T4 in the security
|
||||
// audit). The CSP is emitted by SvelteKit's kit.csp config; the clickjacking /
|
||||
// referrer / permissions headers by hooks.server.ts. We assert on the raw
|
||||
// response of a document navigation, and separately confirm the inline theme
|
||||
// script still executes under the CSP (its sha256 is allowlisted) by checking
|
||||
// the data-theme attribute it sets — a CSP block would leave it unset.
|
||||
|
||||
test('document responses carry the security headers', async ({ page }) => {
|
||||
const response = await page.goto('/');
|
||||
expect(response, 'navigation returned a response').not.toBeNull();
|
||||
const headers = response!.headers();
|
||||
|
||||
// Clickjacking: both the legacy header and the CSP directive.
|
||||
expect(headers['x-frame-options']).toBe('DENY');
|
||||
expect(headers['content-security-policy']).toContain("frame-ancestors 'none'");
|
||||
// Script-injection surface reduction.
|
||||
expect(headers['content-security-policy']).toContain("object-src 'none'");
|
||||
expect(headers['content-security-policy']).toContain('script-src');
|
||||
// The non-CSP hardening headers.
|
||||
expect(headers['referrer-policy']).toBe('strict-origin-when-cross-origin');
|
||||
expect(headers['x-content-type-options']).toBe('nosniff');
|
||||
expect(headers['permissions-policy']).toContain('geolocation=()');
|
||||
});
|
||||
|
||||
test('the inline theme script executes under the CSP (hash is allowlisted)', async ({
|
||||
page
|
||||
}) => {
|
||||
// If the theme script were CSP-blocked, data-theme would never be set.
|
||||
// Its presence proves the allowlisted sha256 matches the served script.
|
||||
await page.goto('/');
|
||||
const theme = await page.evaluate(() =>
|
||||
document.documentElement.getAttribute('data-theme')
|
||||
);
|
||||
expect(theme === 'light' || theme === 'dark').toBe(true);
|
||||
});
|
||||
16
frontend/package-lock.json
generated
16
frontend/package-lock.json
generated
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "mangalord-frontend",
|
||||
"version": "0.109.1",
|
||||
"version": "0.128.24",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "mangalord-frontend",
|
||||
"version": "0.109.1",
|
||||
"version": "0.128.24",
|
||||
"devDependencies": {
|
||||
"@lucide/svelte": "^1.16.0",
|
||||
"@playwright/test": "^1.48.0",
|
||||
@@ -169,7 +169,6 @@
|
||||
}
|
||||
],
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
},
|
||||
@@ -193,7 +192,6 @@
|
||||
}
|
||||
],
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
@@ -1157,7 +1155,6 @@
|
||||
"integrity": "sha512-mQjlkNo+rJvpln7V2IGY2j99BqhcFbS4UN0AQNKNYfhBAFZTuCDAdW3a1sgf330mvtNvsBXn3HpAhcmvdJTcIQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"@standard-schema/spec": "^1.0.0",
|
||||
"@sveltejs/acorn-typescript": "^1.0.5",
|
||||
@@ -1200,7 +1197,6 @@
|
||||
"integrity": "sha512-0ba1RQ/PHen5FGpdSrW7Y3fAMQjrXantECALeOiOdBdzR5+5vPP6HVZRLmZaQL+W8m++o+haIAKq5qT+MiZ7VA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"@sveltejs/vite-plugin-svelte-inspector": "^3.0.0-next.0||^3.0.0",
|
||||
"debug": "^4.3.7",
|
||||
@@ -1359,7 +1355,6 @@
|
||||
"integrity": "sha512-dyh/xO2Fh5bYrfWaaqGrRQQGkNdmYw6AmaAUvYeUMNTWQtvb796ikLdmTchRmOlOiIJ1TDXfWgVx1QkUlQ6Hew==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"undici-types": "~6.21.0"
|
||||
}
|
||||
@@ -1507,7 +1502,6 @@
|
||||
"integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"bin": {
|
||||
"acorn": "bin/acorn"
|
||||
},
|
||||
@@ -2249,7 +2243,6 @@
|
||||
"integrity": "sha512-8i7LzZj7BF8uplX+ZyOlIz86V6TAsSs+np6m1kpW9u0JWi4z/1t+FzcK1aek+ybTnAC4KhBL4uXCNT0wcUIeCw==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"cssstyle": "^4.1.0",
|
||||
"data-urls": "^5.0.0",
|
||||
@@ -2638,7 +2631,6 @@
|
||||
"integrity": "sha512-WHeFSbZYsPu3+bLoNRUuAO+wavNlocOPf3wSHTP7hcFKVnJeWsYlCDbr3mTS14FCizf9ccIxXA8sGL8zKeQN3g==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"@types/estree": "1.0.8"
|
||||
},
|
||||
@@ -2810,7 +2802,6 @@
|
||||
"integrity": "sha512-ymI5ykLPwIHW839E053FQbI1G+jnRFJEw3Kv5Y4njixVWywQBx+NUFpkkKyk5LIb36Fg9DVXSYpqiGekLD0hyw==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"@jridgewell/remapping": "^2.3.4",
|
||||
"@jridgewell/sourcemap-codec": "^1.5.0",
|
||||
@@ -2997,7 +2988,6 @@
|
||||
"integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0",
|
||||
"peer": true,
|
||||
"bin": {
|
||||
"tsc": "bin/tsc",
|
||||
"tsserver": "bin/tsserver"
|
||||
@@ -3019,7 +3009,6 @@
|
||||
"integrity": "sha512-o5a9xKjbtuhY6Bi5S3+HvbRERmouabWbyUcpXXUA1u+GNUKoROi9byOJ8M0nHbHYHkYICiMlqxkg1KkYmm25Sw==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"esbuild": "^0.21.3",
|
||||
"postcss": "^8.4.43",
|
||||
@@ -3138,7 +3127,6 @@
|
||||
"integrity": "sha512-MSmPM9REYqDGBI8439mA4mWhV5sKmDlBKWIYbA3lRb2PTHACE0mgKwA8yQ2xq9vxDTuk4iPrECBAEW2aoFXY0Q==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"@vitest/expect": "2.1.9",
|
||||
"@vitest/mocker": "2.1.9",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "mangalord-frontend",
|
||||
"version": "0.128.4",
|
||||
"version": "0.128.25",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
|
||||
25
frontend/src/csp-theme-hash.test.ts
Normal file
25
frontend/src/csp-theme-hash.test.ts
Normal file
@@ -0,0 +1,25 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { join } from 'node:path';
|
||||
import { THEME_SCRIPT_HASH } from '../csp-config.js';
|
||||
|
||||
// The theme-flash-prevention <script> in app.html runs inline, so the CSP
|
||||
// script-src must allow it by hash. SvelteKit's kit.csp hash mode does NOT
|
||||
// cover app.html template scripts (only what it injects into %sveltekit.head%),
|
||||
// so we pin the hash by hand in svelte.config.js. This test recomputes the hash
|
||||
// from the actual app.html bytes and fails if THEME_SCRIPT_HASH drifted — i.e.
|
||||
// someone edited the theme script without updating the CSP, which would silently
|
||||
// CSP-block it (theme flash returns, only a console error to show for it).
|
||||
describe('CSP theme-script hash', () => {
|
||||
it('matches the current inline theme script in app.html', () => {
|
||||
// vitest runs with cwd at the frontend package root.
|
||||
const html = readFileSync(join(process.cwd(), 'src/app.html'), 'utf-8');
|
||||
const match = html.match(/<script>(.*?)<\/script>/s);
|
||||
expect(match, 'app.html must contain an inline <script>').not.toBeNull();
|
||||
const digest = createHash('sha256')
|
||||
.update(match![1], 'utf-8')
|
||||
.digest('base64');
|
||||
expect(THEME_SCRIPT_HASH).toBe(`'sha256-${digest}'`);
|
||||
});
|
||||
});
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
type MockInstance
|
||||
} from 'vitest';
|
||||
import {
|
||||
applySecurityHeaders,
|
||||
handle,
|
||||
setForwardedFor,
|
||||
shouldBypassProxyTimeout,
|
||||
@@ -353,6 +354,46 @@ describe('stripHopByHopHeaders', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('applySecurityHeaders', () => {
|
||||
it('sets the clickjacking and defense-in-depth headers', () => {
|
||||
const h = applySecurityHeaders(new Headers());
|
||||
// Clickjacking: legacy + modern. CSP frame-ancestors is emitted by
|
||||
// SvelteKit's kit.csp config; X-Frame-Options is the belt-and-braces
|
||||
// fallback for older UAs.
|
||||
expect(h.get('x-frame-options')).toBe('DENY');
|
||||
expect(h.get('referrer-policy')).toBe('strict-origin-when-cross-origin');
|
||||
expect(h.get('x-content-type-options')).toBe('nosniff');
|
||||
// Lock down powerful features we never use.
|
||||
const pp = h.get('permissions-policy') ?? '';
|
||||
expect(pp).toContain('camera=()');
|
||||
expect(pp).toContain('microphone=()');
|
||||
expect(pp).toContain('geolocation=()');
|
||||
});
|
||||
|
||||
it('emits HSTS only when the request was secure', () => {
|
||||
const insecure = applySecurityHeaders(new Headers());
|
||||
expect(insecure.get('strict-transport-security')).toBeNull();
|
||||
|
||||
const secure = applySecurityHeaders(new Headers(), { hsts: true });
|
||||
expect(secure.get('strict-transport-security')).toBe(
|
||||
'max-age=63072000; includeSubDomains'
|
||||
);
|
||||
});
|
||||
|
||||
it('does not clobber a header the response already set', () => {
|
||||
// If a downstream route deliberately set a stricter Referrer-Policy,
|
||||
// the helper must not override it.
|
||||
const h = new Headers({ 'referrer-policy': 'no-referrer' });
|
||||
applySecurityHeaders(h);
|
||||
expect(h.get('referrer-policy')).toBe('no-referrer');
|
||||
});
|
||||
|
||||
it('returns the same Headers instance it was given (mutates in place)', () => {
|
||||
const h = new Headers();
|
||||
expect(applySecurityHeaders(h)).toBe(h);
|
||||
});
|
||||
});
|
||||
|
||||
describe('setForwardedFor', () => {
|
||||
it('overrides any client-supplied x-forwarded-for with the real address', () => {
|
||||
const headers = new Headers({ 'x-forwarded-for': '1.2.3.4', 'x-real-ip': '1.2.3.4' });
|
||||
|
||||
@@ -105,6 +105,48 @@ export function shouldBypassProxyTimeout(headers: Headers): boolean {
|
||||
return accept.toLowerCase().includes('text/event-stream');
|
||||
}
|
||||
|
||||
/**
|
||||
* Defense-in-depth response headers for HTML/document responses.
|
||||
*
|
||||
* The Content-Security-Policy itself is emitted by SvelteKit's `kit.csp`
|
||||
* config (see svelte.config.js) so that `script-src` hashes cover both the
|
||||
* inline theme script in app.html AND SvelteKit's own hydration inline
|
||||
* scripts — a hand-rolled `script-src` here would break hydration on the next
|
||||
* build. What CSP can't (or shouldn't) express, we add here:
|
||||
*
|
||||
* - `X-Frame-Options: DENY` — clickjacking guard for UAs predating CSP
|
||||
* `frame-ancestors` (which the kit.csp policy also sets).
|
||||
* - `Referrer-Policy` — don't leak full URLs to cross-origin destinations.
|
||||
* - `X-Content-Type-Options: nosniff` — no MIME sniffing on documents.
|
||||
* - `Permissions-Policy` — deny powerful features the app never uses.
|
||||
*
|
||||
* Each header is only set when absent, so a route that deliberately picked a
|
||||
* stricter value keeps it. Mutates and returns the passed `Headers`. Exported
|
||||
* for unit-test coverage.
|
||||
*/
|
||||
export function applySecurityHeaders(
|
||||
headers: Headers,
|
||||
opts: { hsts?: boolean } = {}
|
||||
): Headers {
|
||||
const defaults: Record<string, string> = {
|
||||
'x-frame-options': 'DENY',
|
||||
'referrer-policy': 'strict-origin-when-cross-origin',
|
||||
'x-content-type-options': 'nosniff',
|
||||
'permissions-policy': 'camera=(), microphone=(), geolocation=(), interest-cohort=()'
|
||||
};
|
||||
// HSTS ONLY when the browser reached us over HTTPS (the caller derives this
|
||||
// from x-forwarded-proto). Sending it over plain HTTP — a dev run or a
|
||||
// misconfigured deploy — would pin the browser to HTTPS for this host for
|
||||
// two years and could lock users out, so it's opt-in per request.
|
||||
if (opts.hsts) {
|
||||
defaults['strict-transport-security'] = 'max-age=63072000; includeSubDomains';
|
||||
}
|
||||
for (const [name, value] of Object.entries(defaults)) {
|
||||
if (!headers.has(name)) headers.set(name, value);
|
||||
}
|
||||
return headers;
|
||||
}
|
||||
|
||||
export const handle: Handle = async ({ event, resolve }) => {
|
||||
if (event.url.pathname.startsWith('/api/')) {
|
||||
const target = `${BACKEND_URL}${event.url.pathname}${event.url.search}`;
|
||||
@@ -199,5 +241,10 @@ export const handle: Handle = async ({ event, resolve }) => {
|
||||
headers: stripHopByHopHeaders(upstream.headers)
|
||||
});
|
||||
}
|
||||
return resolve(event);
|
||||
const response = await resolve(event);
|
||||
// The TLS terminator (reverse proxy) sets x-forwarded-proto; only emit HSTS
|
||||
// when the browser's hop to it was HTTPS.
|
||||
const secure = event.request.headers.get('x-forwarded-proto') === 'https';
|
||||
applySecurityHeaders(response.headers, { hsts: secure });
|
||||
return response;
|
||||
};
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
<script lang="ts">
|
||||
import AdaptiveDialog from './AdaptiveDialog.svelte';
|
||||
import { CancellableLoader } from '$lib/util/cancellable';
|
||||
import {
|
||||
addMangaToCollection,
|
||||
createCollection,
|
||||
@@ -74,19 +75,30 @@
|
||||
: removePageFromCollection(collectionId, t.id);
|
||||
}
|
||||
|
||||
// Reopening for a different target (or an in-flight retarget) fires load()
|
||||
// again; without this guard the two responses can resolve out of order and
|
||||
// the first target's membership overwrites the second's (audit FE-2). The
|
||||
// CancellableLoader drops a superseded call's result — the newer load then
|
||||
// owns both the state and the spinner.
|
||||
const loader = new CancellableLoader();
|
||||
|
||||
async function load() {
|
||||
loading = true;
|
||||
error = null;
|
||||
try {
|
||||
const [page, ids] = await Promise.all([
|
||||
listMyCollections({ limit: 200 }),
|
||||
loadContaining(target)
|
||||
]);
|
||||
collections = page.items;
|
||||
containingIds = new Set(ids);
|
||||
const result = await loader.run(async () => {
|
||||
const [page, ids] = await Promise.all([
|
||||
listMyCollections({ limit: 200 }),
|
||||
loadContaining(target)
|
||||
]);
|
||||
return { items: page.items, ids };
|
||||
});
|
||||
if (result === null) return; // superseded — a newer load owns the state
|
||||
collections = result.items;
|
||||
containingIds = new Set(result.ids);
|
||||
loading = false;
|
||||
} catch (e) {
|
||||
error = (e as Error).message;
|
||||
} finally {
|
||||
loading = false;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -84,7 +84,11 @@ describe('AddToCollectionModal', () => {
|
||||
|
||||
expect(pageCollectionsApi.getMyCollectionsContainingPage).toHaveBeenCalledWith('p1');
|
||||
expect(collectionsApi.getMyCollectionsContaining).not.toHaveBeenCalled();
|
||||
expect(screen.getByText('Favorite panels')).toBeTruthy();
|
||||
// Await the async render: load() resolves the collections over a few
|
||||
// microtasks (through the CancellableLoader), so assert with findByText
|
||||
// (retries until it appears) rather than a synchronous getByText tied to
|
||||
// a fixed tick budget.
|
||||
expect(await screen.findByText('Favorite panels')).toBeTruthy();
|
||||
});
|
||||
|
||||
it('does not load when closed', () => {
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
import { onMount } from 'svelte';
|
||||
import Pager from '$lib/components/Pager.svelte';
|
||||
import { fmtDuration } from '$lib/format';
|
||||
import { safeExternalHref } from '$lib/safeHref';
|
||||
import SearchBar from './SearchBar.svelte';
|
||||
import {
|
||||
listCrawlerJobHistory,
|
||||
@@ -175,9 +176,14 @@
|
||||
<td class="kind">{r.kind ?? '—'}</td>
|
||||
<td>
|
||||
{#if !r.manga_title && r.payload_title && r.source_url}
|
||||
<a href={r.source_url} target="_blank" rel="noreferrer noopener"
|
||||
>{target(r)}</a
|
||||
>
|
||||
{@const href = safeExternalHref(r.source_url)}
|
||||
{#if href}
|
||||
<a {href} target="_blank" rel="noreferrer noopener"
|
||||
>{target(r)}</a
|
||||
>
|
||||
{:else}
|
||||
{target(r)}
|
||||
{/if}
|
||||
{:else}
|
||||
{target(r)}
|
||||
{/if}
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
import Pager from '$lib/components/Pager.svelte';
|
||||
import type { DeadJob, RequeueScope } from '$lib/api/admin';
|
||||
import SearchBar from './SearchBar.svelte';
|
||||
import { safeExternalHref } from '$lib/safeHref';
|
||||
|
||||
let {
|
||||
jobs,
|
||||
@@ -60,9 +61,14 @@
|
||||
{j.manga_title}
|
||||
{:else if j.payload_title}
|
||||
{#if j.source_url}
|
||||
<a href={j.source_url} target="_blank" rel="noreferrer noopener"
|
||||
>{j.payload_title}</a
|
||||
>
|
||||
{@const href = safeExternalHref(j.source_url)}
|
||||
{#if href}
|
||||
<a {href} target="_blank" rel="noreferrer noopener"
|
||||
>{j.payload_title}</a
|
||||
>
|
||||
{:else}
|
||||
{j.payload_title}
|
||||
{/if}
|
||||
{:else}
|
||||
{j.payload_title}
|
||||
{/if}
|
||||
|
||||
23
frontend/src/lib/readerKeys.test.ts
Normal file
23
frontend/src/lib/readerKeys.test.ts
Normal file
@@ -0,0 +1,23 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { isTypingTarget } from './readerKeys';
|
||||
|
||||
function el(tag: string, contentEditable = false): HTMLElement {
|
||||
const node = document.createElement(tag);
|
||||
if (contentEditable) node.setAttribute('contenteditable', 'true');
|
||||
return node;
|
||||
}
|
||||
|
||||
describe('isTypingTarget', () => {
|
||||
it('is true for text-entry and native-key elements', () => {
|
||||
expect(isTypingTarget(el('input'))).toBe(true);
|
||||
expect(isTypingTarget(el('textarea'))).toBe(true);
|
||||
expect(isTypingTarget(el('select'))).toBe(true);
|
||||
expect(isTypingTarget(el('div', true))).toBe(true);
|
||||
});
|
||||
|
||||
it('is false for ordinary elements and null', () => {
|
||||
expect(isTypingTarget(el('div'))).toBe(false);
|
||||
expect(isTypingTarget(el('button'))).toBe(false);
|
||||
expect(isTypingTarget(null)).toBe(false);
|
||||
});
|
||||
});
|
||||
19
frontend/src/lib/readerKeys.ts
Normal file
19
frontend/src/lib/readerKeys.ts
Normal file
@@ -0,0 +1,19 @@
|
||||
/**
|
||||
* Whether a keyboard event's target is a place the user is typing (or otherwise
|
||||
* expects native key handling), so the reader must NOT hijack arrows/j/k/Home/End
|
||||
* for page navigation. Covers text inputs, textareas, native `<select>` dropdowns
|
||||
* (whose Home/End/arrows are their own), and any contenteditable region.
|
||||
*/
|
||||
export function isTypingTarget(target: EventTarget | null): boolean {
|
||||
const el = target as HTMLElement | null;
|
||||
if (!el) return false;
|
||||
const tag = el.tagName;
|
||||
// `isContentEditable` is the correct runtime signal (accounts for inherited
|
||||
// editability); the attribute check is a robust fallback for the explicit
|
||||
// case and for environments that don't compute the property (jsdom).
|
||||
const editable =
|
||||
el.isContentEditable === true ||
|
||||
el.getAttribute?.('contenteditable') === 'true' ||
|
||||
el.getAttribute?.('contenteditable') === '';
|
||||
return tag === 'INPUT' || tag === 'TEXTAREA' || tag === 'SELECT' || editable;
|
||||
}
|
||||
29
frontend/src/lib/safeHref.test.ts
Normal file
29
frontend/src/lib/safeHref.test.ts
Normal file
@@ -0,0 +1,29 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { safeExternalHref } from './safeHref';
|
||||
|
||||
describe('safeExternalHref', () => {
|
||||
it('passes through http and https URLs', () => {
|
||||
expect(safeExternalHref('http://example.com/manga/1')).toBe(
|
||||
'http://example.com/manga/1'
|
||||
);
|
||||
expect(safeExternalHref('https://example.com/x?y=1#z')).toBe(
|
||||
'https://example.com/x?y=1#z'
|
||||
);
|
||||
});
|
||||
|
||||
it('rejects dangerous schemes', () => {
|
||||
expect(safeExternalHref('javascript:alert(document.cookie)')).toBeNull();
|
||||
expect(safeExternalHref('JavaScript:alert(1)')).toBeNull();
|
||||
expect(safeExternalHref('data:text/html,<script>alert(1)</script>')).toBeNull();
|
||||
expect(safeExternalHref('vbscript:msgbox(1)')).toBeNull();
|
||||
expect(safeExternalHref('file:///etc/passwd')).toBeNull();
|
||||
});
|
||||
|
||||
it('rejects relative, malformed, empty, and nullish input', () => {
|
||||
expect(safeExternalHref('/relative/path')).toBeNull();
|
||||
expect(safeExternalHref('not a url')).toBeNull();
|
||||
expect(safeExternalHref('')).toBeNull();
|
||||
expect(safeExternalHref(null)).toBeNull();
|
||||
expect(safeExternalHref(undefined)).toBeNull();
|
||||
});
|
||||
});
|
||||
21
frontend/src/lib/safeHref.ts
Normal file
21
frontend/src/lib/safeHref.ts
Normal file
@@ -0,0 +1,21 @@
|
||||
/**
|
||||
* Return `url` only if it is a well-formed `http(s)` URL that is safe to place
|
||||
* in an `href`, otherwise `null`.
|
||||
*
|
||||
* Crawler `source_url` values are external/untrusted content harvested from
|
||||
* scraped source sites. Svelte does NOT sanitize URL schemes in `href`
|
||||
* bindings, so a harvested `javascript:`/`data:`/`vbscript:` link would execute
|
||||
* (or phish) in the admin origin on click. Callers render the title as plain
|
||||
* text when this returns `null`.
|
||||
*/
|
||||
export function safeExternalHref(url: string | null | undefined): string | null {
|
||||
if (!url) return null;
|
||||
let parsed: URL;
|
||||
try {
|
||||
parsed = new URL(url);
|
||||
} catch {
|
||||
// Relative or malformed — not a safe external link.
|
||||
return null;
|
||||
}
|
||||
return parsed.protocol === 'http:' || parsed.protocol === 'https:' ? url : null;
|
||||
}
|
||||
@@ -3,6 +3,7 @@
|
||||
import { browser } from '$app/environment';
|
||||
import { afterNavigate, goto, invalidateAll } from '$app/navigation';
|
||||
import { fileUrl, ApiError } from '$lib/api/client';
|
||||
import { isTypingTarget } from '$lib/readerKeys';
|
||||
import { GAP_PX, type ReaderPageGap } from '$lib/api/preferences';
|
||||
import { preferences } from '$lib/preferences.svelte';
|
||||
import { updateReadProgress } from '$lib/api/read_progress';
|
||||
@@ -675,9 +676,9 @@
|
||||
);
|
||||
|
||||
function onKeydown(e: KeyboardEvent) {
|
||||
// Don't hijack keys while the user is typing in an input.
|
||||
const target = e.target as HTMLElement | null;
|
||||
if (target && (target.tagName === 'INPUT' || target.tagName === 'TEXTAREA')) return;
|
||||
// Don't hijack keys while the user is typing or focused on a native
|
||||
// control (input/textarea/select/contenteditable own their own keys).
|
||||
if (isTypingTarget(e.target)) return;
|
||||
// `?` opens the keyboard-shortcut help overlay. While it's open,
|
||||
// swallow every key here so paging can't happen behind it — the
|
||||
// Sheet owns its own Escape/close.
|
||||
@@ -687,6 +688,20 @@
|
||||
return;
|
||||
}
|
||||
if (shortcutsOpen) return;
|
||||
// Any other overlay (jump/settings/action/collections/tags/context menu)
|
||||
// owns its own keys; swallow reader paging so arrows/j/k/Home/End can't
|
||||
// act on the chapter *behind* the overlay. Those overlays only
|
||||
// stopPropagation() on Escape, so navigation keys otherwise bubble here.
|
||||
if (
|
||||
chapterJumpOpen ||
|
||||
settingsOpen ||
|
||||
contextMenuOpen ||
|
||||
actionSheetOpen ||
|
||||
collectionsModalOpen ||
|
||||
tagsModalOpen
|
||||
) {
|
||||
return;
|
||||
}
|
||||
// Esc always exits fullscreen if active — applies in both
|
||||
// modes, including when the bars are hidden. Handled before
|
||||
// the per-mode switches so it doesn't get shadowed.
|
||||
|
||||
@@ -1,11 +1,34 @@
|
||||
import adapter from '@sveltejs/adapter-node';
|
||||
import { vitePreprocess } from '@sveltejs/vite-plugin-svelte';
|
||||
import { THEME_SCRIPT_HASH } from './csp-config.js';
|
||||
|
||||
/** @type {import('@sveltejs/kit').Config} */
|
||||
const config = {
|
||||
preprocess: vitePreprocess(),
|
||||
kit: {
|
||||
adapter: adapter({ out: 'build' })
|
||||
adapter: adapter({ out: 'build' }),
|
||||
// Content-Security-Policy. `mode: 'hash'` makes SvelteKit hash the
|
||||
// inline scripts IT injects (the hydration bootstrap) and append those
|
||||
// hashes to `script-src`. It does NOT hash the theme <script> in
|
||||
// app.html — that template script isn't part of `%sveltekit.head%`, so
|
||||
// we pin its sha256 here by hand (THEME_SCRIPT_HASH). If that script is
|
||||
// edited, the hash drifts and the theme-flash guard would be silently
|
||||
// CSP-blocked; `svelte.config.test.js` recomputes the hash from
|
||||
// app.html and fails if it no longer matches this constant.
|
||||
// Styles are left unconstrained (Svelte emits dynamic inline `style=`
|
||||
// attributes); this policy is clickjacking + script-injection defense
|
||||
// in depth, not a full lockdown. The non-CSP defense-in-depth headers
|
||||
// (X-Frame-Options, Referrer-Policy, Permissions-Policy) live in
|
||||
// hooks.server.ts.
|
||||
csp: {
|
||||
mode: 'hash',
|
||||
directives: {
|
||||
'script-src': ['self', THEME_SCRIPT_HASH],
|
||||
'object-src': ['none'],
|
||||
'base-uri': ['self'],
|
||||
'frame-ancestors': ['none']
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
Reference in New Issue
Block a user