Compare commits
12 Commits
b845d88766
...
f57ca8e45c
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f57ca8e45c | ||
|
|
8d34132883 | ||
|
|
c5c1179e9d | ||
|
|
c320eda7cd | ||
|
|
bd9a6bd257 | ||
|
|
ebc1966103 | ||
|
|
e4333631e1 | ||
|
|
e7662d18d6 | ||
|
|
45ce0d8f12 | ||
|
|
51f42b03e9 | ||
|
|
fa0a7da311 | ||
|
|
9ff49166a5 |
32
.env.example
32
.env.example
@@ -1,13 +1,23 @@
|
||||
# Copy to .env for `docker compose up --build`. Local-dev runs (cargo run
|
||||
# / npm run dev) read backend/.env if present, or pick up the variables
|
||||
# from your shell.
|
||||
#
|
||||
# Production note: COOKIE_SECURE=true (the default below) makes browsers
|
||||
# refuse to send the session cookie over plain HTTP. Run with a TLS-
|
||||
# terminating reverse proxy (Caddy, Traefik, nginx) in front — the
|
||||
# compose file here doesn't ship one. Local/dev runs without HTTPS
|
||||
# should set COOKIE_SECURE=false.
|
||||
|
||||
# ----- Postgres -----
|
||||
# These are read by the Postgres container *and* by DATABASE_URL below;
|
||||
# changing them after the first boot won't migrate existing data, so set
|
||||
# them up front for any new deployment.
|
||||
#
|
||||
# POSTGRES_PASSWORD is REQUIRED — docker-compose.yml fails fast if it
|
||||
# isn't set in this file, to prevent a deploy without an .env booting
|
||||
# Postgres with a publicly-known credential.
|
||||
POSTGRES_USER=mangalord
|
||||
POSTGRES_PASSWORD=mangalord
|
||||
POSTGRES_PASSWORD=change-me-to-a-strong-random-string
|
||||
POSTGRES_DB=mangalord
|
||||
|
||||
# ----- Backend -----
|
||||
@@ -29,6 +39,13 @@ COOKIE_DOMAIN=
|
||||
# get reaped lazily.
|
||||
SESSION_TTL_DAYS=30
|
||||
|
||||
# ----- Auth brute-force rate limits -----
|
||||
# Token-bucket budget shared across /auth/login, /auth/register, and
|
||||
# /auth/me/password. Set per_sec=0 to disable (e.g. behind a
|
||||
# rate-limiting reverse proxy that already enforces a budget).
|
||||
AUTH_RATE_PER_SEC=5
|
||||
AUTH_RATE_BURST=10
|
||||
|
||||
# ----- CORS -----
|
||||
# Comma-separated origins allowed to call the API with credentials.
|
||||
# Default is empty: same-origin only. Set when frontend and backend live
|
||||
@@ -44,6 +61,14 @@ MAX_REQUEST_BYTES=209715200
|
||||
# Default 20 MiB.
|
||||
MAX_FILE_BYTES=20971520
|
||||
|
||||
# ----- Crawler download safety -----
|
||||
# Hosts the crawler is allowed to fetch images/covers from, in addition
|
||||
# to CRAWLER_START_URL's host and CRAWLER_CDN_HOST. Comma-separated.
|
||||
# Defends against SSRF via scraped <img src="http://10.0.0.1/...">.
|
||||
CRAWLER_DOWNLOAD_ALLOWLIST=
|
||||
# Hard cap on a single image body. Default 32 MiB.
|
||||
CRAWLER_MAX_IMAGE_BYTES=33554432
|
||||
|
||||
# ----- Frontend -----
|
||||
# The frontend container runs SvelteKit's Node adapter on :3000 and
|
||||
# proxies /api/* to BACKEND_URL via src/hooks.server.ts. In compose the
|
||||
@@ -51,3 +76,8 @@ MAX_FILE_BYTES=20971520
|
||||
# internal docker network. Override only if you're running the
|
||||
# frontend container against a backend somewhere else.
|
||||
BACKEND_URL=http://backend:8080
|
||||
# Per-request wall-clock cap for the /api/* reverse proxy (milliseconds).
|
||||
# Default 300000 (5 min) covers a typical 200 MiB chapter upload over
|
||||
# 25 Mbps; raise for users on slower upstream links or lower if a
|
||||
# tighter front proxy already bounds the request lifetime.
|
||||
BACKEND_PROXY_TIMEOUT_MS=300000
|
||||
|
||||
71
.gitea/README.md
Normal file
71
.gitea/README.md
Normal file
@@ -0,0 +1,71 @@
|
||||
# Gitea Actions
|
||||
|
||||
The [`deploy`](workflows/deploy.yml) workflow runs on every push to `main`
|
||||
(and via manual `workflow_dispatch`). It tests, builds, pushes the images
|
||||
to a private registry, and rolls the stack over by SSH on the target host.
|
||||
|
||||
## Required secrets
|
||||
|
||||
Set under *Repo Settings → Actions → Secrets*:
|
||||
|
||||
| Name | Example | Purpose |
|
||||
| -------------------- | ------------------------ | ---------------------------------------------------------------- |
|
||||
| `REGISTRY_URL` | `registry.example.com` | Registry host. No scheme, no trailing slash. |
|
||||
| `REGISTRY_USERNAME` | `mangalord-ci` | `docker login` user. |
|
||||
| `REGISTRY_PASSWORD` | `<token>` | `docker login` token/password. |
|
||||
| `SSH_HOST` | `mangalord.example.com` | Deploy target hostname/IP. |
|
||||
| `SSH_USER` | `deploy` | SSH user on the target (must be in the `docker` group). |
|
||||
| `SSH_PRIVATE_KEY` | `-----BEGIN OPENSSH...` | Private key authorised in the target user's `authorized_keys`. |
|
||||
| `SSH_PORT` | `22` | Optional. Defaults to `22` if unset. |
|
||||
|
||||
## Required variables
|
||||
|
||||
Set under *Repo Settings → Actions → Variables* (not secrets — they appear
|
||||
in logs):
|
||||
|
||||
| Name | Example | Purpose |
|
||||
| ------------- | ------------------------ | ---------------------------------------------------------------------- |
|
||||
| `DEPLOY_PATH` | `/srv/mangalord` | Directory on target holding `docker-compose.yml`, `.env`, and the prod overlay. |
|
||||
|
||||
## One-time host setup
|
||||
|
||||
The workflow assumes the deploy target already has:
|
||||
|
||||
1. Docker + Docker Compose v2 installed and the `SSH_USER` in the `docker` group.
|
||||
2. `$DEPLOY_PATH/docker-compose.yml` (copy of the repo's [docker-compose.yml](../docker-compose.yml)).
|
||||
3. `$DEPLOY_PATH/docker-compose.prod.yml` (copy of the repo's [docker-compose.prod.yml](../docker-compose.prod.yml)).
|
||||
4. `$DEPLOY_PATH/.env` populated from [.env.example](../.env.example) with production values (real `POSTGRES_PASSWORD`, `COOKIE_SECURE=true`, etc.).
|
||||
|
||||
Bootstrap once:
|
||||
|
||||
```bash
|
||||
ssh deploy@mangalord.example.com
|
||||
sudo mkdir -p /srv/mangalord && sudo chown deploy:deploy /srv/mangalord
|
||||
cd /srv/mangalord
|
||||
# place docker-compose.yml, docker-compose.prod.yml, and .env here
|
||||
```
|
||||
|
||||
The first workflow run will pull the images, bring the stack up, and run
|
||||
the embedded migrations on startup.
|
||||
|
||||
## Image tags
|
||||
|
||||
Every push produces three tags per image:
|
||||
|
||||
- `mangalord-{backend,frontend}:latest`
|
||||
- `mangalord-{backend,frontend}:<git-sha>` — used by the deploy job; lets
|
||||
you pin a deploy to a specific commit
|
||||
- `mangalord-{backend,frontend}:<version>` — the version from
|
||||
[backend/Cargo.toml](../backend/Cargo.toml) (verified in lockstep with
|
||||
[frontend/package.json](../frontend/package.json))
|
||||
|
||||
## Rollback
|
||||
|
||||
SSH to the target, set `IMAGE_TAG` to a previous commit SHA, and re-up:
|
||||
|
||||
```bash
|
||||
cd /srv/mangalord
|
||||
export REGISTRY_URL=registry.example.com
|
||||
export IMAGE_TAG=<previous-sha>
|
||||
docker compose -f docker-compose.yml -f docker-compose.prod.yml up -d
|
||||
```
|
||||
151
.gitea/workflows/deploy.yml
Normal file
151
.gitea/workflows/deploy.yml
Normal file
@@ -0,0 +1,151 @@
|
||||
name: deploy
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
branches: [main]
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
test-backend:
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: rust:1-slim
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16-alpine
|
||||
env:
|
||||
POSTGRES_USER: mangalord
|
||||
POSTGRES_PASSWORD: mangalord
|
||||
POSTGRES_DB: mangalord
|
||||
options: >-
|
||||
--health-cmd "pg_isready -U mangalord"
|
||||
--health-interval 5s
|
||||
--health-timeout 5s
|
||||
--health-retries 10
|
||||
env:
|
||||
DATABASE_URL: postgres://mangalord:mangalord@postgres:5432/mangalord
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Install build deps
|
||||
run: |
|
||||
apt-get update
|
||||
apt-get install -y --no-install-recommends pkg-config libssl-dev ca-certificates
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
backend/target
|
||||
key: cargo-${{ runner.os }}-${{ hashFiles('backend/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
cargo-${{ runner.os }}-
|
||||
- name: cargo test
|
||||
working-directory: backend
|
||||
run: cargo test --locked
|
||||
|
||||
test-frontend:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '22'
|
||||
cache: npm
|
||||
cache-dependency-path: frontend/package-lock.json
|
||||
- name: npm ci
|
||||
working-directory: frontend
|
||||
run: npm ci
|
||||
- name: vitest
|
||||
working-directory: frontend
|
||||
run: npm test
|
||||
|
||||
build-and-push:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [test-backend, test-frontend]
|
||||
# PRs only run the test jobs; build + deploy are reserved for
|
||||
# post-merge pushes to main. Without this gate every PR would push
|
||||
# a tagged image to the registry and SSH-deploy to prod.
|
||||
if: github.event_name != 'pull_request'
|
||||
outputs:
|
||||
image_tag: ${{ steps.meta.outputs.image_tag }}
|
||||
version: ${{ steps.meta.outputs.version }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Resolve image tags
|
||||
id: meta
|
||||
run: |
|
||||
version="$(grep -m1 '^version' backend/Cargo.toml | cut -d'"' -f2)"
|
||||
frontend_version="$(grep -m1 '"version"' frontend/package.json | cut -d'"' -f4)"
|
||||
if [ "$version" != "$frontend_version" ]; then
|
||||
echo "Version mismatch: backend=$version frontend=$frontend_version" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "image_tag=${GITHUB_SHA}" >> "$GITHUB_OUTPUT"
|
||||
echo "version=${version}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: docker login
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ secrets.REGISTRY_URL }}
|
||||
username: ${{ secrets.REGISTRY_USERNAME }}
|
||||
password: ${{ secrets.REGISTRY_PASSWORD }}
|
||||
|
||||
- name: Build & push backend
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: ./backend
|
||||
push: true
|
||||
tags: |
|
||||
${{ secrets.REGISTRY_URL }}/mangalord-backend:latest
|
||||
${{ secrets.REGISTRY_URL }}/mangalord-backend:${{ steps.meta.outputs.image_tag }}
|
||||
${{ secrets.REGISTRY_URL }}/mangalord-backend:${{ steps.meta.outputs.version }}
|
||||
cache-from: type=gha,scope=backend
|
||||
cache-to: type=gha,mode=max,scope=backend
|
||||
|
||||
- name: Build & push frontend
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: ./frontend
|
||||
push: true
|
||||
tags: |
|
||||
${{ secrets.REGISTRY_URL }}/mangalord-frontend:latest
|
||||
${{ secrets.REGISTRY_URL }}/mangalord-frontend:${{ steps.meta.outputs.image_tag }}
|
||||
${{ secrets.REGISTRY_URL }}/mangalord-frontend:${{ steps.meta.outputs.version }}
|
||||
cache-from: type=gha,scope=frontend
|
||||
cache-to: type=gha,mode=max,scope=frontend
|
||||
|
||||
deploy:
|
||||
runs-on: ubuntu-latest
|
||||
needs: build-and-push
|
||||
if: github.event_name != 'pull_request'
|
||||
steps:
|
||||
- name: SSH deploy
|
||||
uses: appleboy/ssh-action@v1.0.3
|
||||
with:
|
||||
host: ${{ secrets.SSH_HOST }}
|
||||
username: ${{ secrets.SSH_USER }}
|
||||
key: ${{ secrets.SSH_PRIVATE_KEY }}
|
||||
port: ${{ secrets.SSH_PORT || 22 }}
|
||||
envs: REGISTRY_URL,REGISTRY_USERNAME,REGISTRY_PASSWORD,IMAGE_TAG,DEPLOY_PATH
|
||||
script_stop: true
|
||||
script: |
|
||||
set -euo pipefail
|
||||
cd "$DEPLOY_PATH"
|
||||
echo "$REGISTRY_PASSWORD" | docker login "$REGISTRY_URL" -u "$REGISTRY_USERNAME" --password-stdin
|
||||
export REGISTRY_URL IMAGE_TAG
|
||||
docker compose -f docker-compose.yml -f docker-compose.prod.yml pull
|
||||
docker compose -f docker-compose.yml -f docker-compose.prod.yml up -d
|
||||
docker image prune -f
|
||||
docker logout "$REGISTRY_URL"
|
||||
env:
|
||||
REGISTRY_URL: ${{ secrets.REGISTRY_URL }}
|
||||
REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }}
|
||||
REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
|
||||
IMAGE_TAG: ${{ needs.build-and-push.outputs.image_tag }}
|
||||
DEPLOY_PATH: ${{ vars.DEPLOY_PATH }}
|
||||
18
backend/Cargo.lock
generated
18
backend/Cargo.lock
generated
@@ -1470,7 +1470,7 @@ checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
|
||||
|
||||
[[package]]
|
||||
name = "mangalord"
|
||||
version = "0.29.0"
|
||||
version = "0.35.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"argon2",
|
||||
@@ -2324,6 +2324,7 @@ dependencies = [
|
||||
"cookie",
|
||||
"cookie_store",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"http",
|
||||
"http-body",
|
||||
"http-body-util",
|
||||
@@ -2343,12 +2344,14 @@ dependencies = [
|
||||
"sync_wrapper",
|
||||
"tokio",
|
||||
"tokio-rustls",
|
||||
"tokio-util",
|
||||
"tower",
|
||||
"tower-http",
|
||||
"tower-service",
|
||||
"url",
|
||||
"wasm-bindgen",
|
||||
"wasm-bindgen-futures",
|
||||
"wasm-streams",
|
||||
"web-sys",
|
||||
"webpki-roots",
|
||||
]
|
||||
@@ -3527,6 +3530,19 @@ dependencies = [
|
||||
"wasmparser",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasm-streams"
|
||||
version = "0.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65"
|
||||
dependencies = [
|
||||
"futures-util",
|
||||
"js-sys",
|
||||
"wasm-bindgen",
|
||||
"wasm-bindgen-futures",
|
||||
"web-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasmparser"
|
||||
version = "0.244.0"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "mangalord"
|
||||
version = "0.29.0"
|
||||
version = "0.35.0"
|
||||
edition = "2021"
|
||||
default-run = "mangalord"
|
||||
|
||||
@@ -46,7 +46,7 @@ futures-util = "0.3"
|
||||
bytes = "1"
|
||||
chromiumoxide = { version = "0.7", features = ["tokio-runtime", "_fetcher-rusttls-tokio"], default-features = false }
|
||||
scraper = "0.20"
|
||||
reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "socks", "cookies"] }
|
||||
reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "socks", "cookies", "stream"] }
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3"
|
||||
|
||||
@@ -19,12 +19,49 @@ COPY migrations ./migrations
|
||||
RUN touch src/main.rs src/lib.rs && cargo build --locked --release
|
||||
|
||||
FROM debian:bookworm-slim
|
||||
# `curl` is for the container HEALTHCHECK; `ca-certificates` is for
|
||||
# outbound HTTPS (crawler covers/pages).
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends ca-certificates \
|
||||
&& apt-get install -y --no-install-recommends ca-certificates curl \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Non-root runtime user. The API binary doesn't need any root
|
||||
# privilege; the crawler daemon's Chromium launcher uses --no-sandbox
|
||||
# precisely because user-namespace sandboxing is fragile, so dropping
|
||||
# privileges costs nothing operationally and shrinks the blast radius
|
||||
# of any RCE.
|
||||
ARG APP_UID=10001
|
||||
ARG APP_GID=10001
|
||||
RUN groupadd --system --gid ${APP_GID} app \
|
||||
&& useradd --system --uid ${APP_UID} --gid app --home-dir /home/app --create-home --shell /usr/sbin/nologin app
|
||||
|
||||
WORKDIR /app
|
||||
COPY --from=builder /app/target/release/mangalord /usr/local/bin/mangalord
|
||||
COPY --from=builder /app/migrations /app/migrations
|
||||
|
||||
ENV STORAGE_DIR=/var/lib/mangalord/storage
|
||||
# Pre-create the storage dir so the entrypoint doesn't need to
|
||||
# mkdir-as-root and so the named volume mount inherits the right
|
||||
# ownership.
|
||||
#
|
||||
# UPGRADE NOTE for operators: if you're moving from an older image
|
||||
# that ran as root, the existing `storage-data` volume has files owned
|
||||
# by UID 0 and the new UID-10001 user can't write them. Run once
|
||||
# before the upgrade:
|
||||
# docker compose run --rm --user 0 backend \
|
||||
# chown -R 10001:10001 /var/lib/mangalord/storage
|
||||
# (Postgres is unaffected — that image's `postgres` user UID hasn't
|
||||
# changed.)
|
||||
RUN mkdir -p ${STORAGE_DIR} \
|
||||
&& chown -R app:app ${STORAGE_DIR} /app /home/app
|
||||
|
||||
USER app
|
||||
EXPOSE 8080
|
||||
|
||||
# `--start-period` is generous because first boot runs sqlx::migrate
|
||||
# against postgres which can take a few seconds; subsequent restarts
|
||||
# are sub-second.
|
||||
HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 \
|
||||
CMD curl -fsS http://localhost:8080/api/v1/health > /dev/null || exit 1
|
||||
|
||||
CMD ["mangalord"]
|
||||
|
||||
15
backend/migrations/0016_crawler_jobs_drop_failed_state.sql
Normal file
15
backend/migrations/0016_crawler_jobs_drop_failed_state.sql
Normal file
@@ -0,0 +1,15 @@
|
||||
-- The original 0012 partial index covers `state IN ('pending','failed')`,
|
||||
-- but `ack_failed` in src/crawler/jobs.rs only writes `dead` or
|
||||
-- `pending` — `failed` is never set. The index branch on `failed`
|
||||
-- never matches any row, so it's dead weight on every write.
|
||||
--
|
||||
-- Drop and recreate the index without the dead branch. The CHECK
|
||||
-- constraint on `state` still allows `'failed'` so a future migration
|
||||
-- can adopt that terminal-but-retryable state without a second
|
||||
-- schema change.
|
||||
|
||||
DROP INDEX IF EXISTS crawler_jobs_ready_idx;
|
||||
|
||||
CREATE INDEX crawler_jobs_ready_idx
|
||||
ON crawler_jobs (scheduled_at)
|
||||
WHERE state = 'pending';
|
||||
@@ -4,6 +4,8 @@
|
||||
//! expire naturally rather than being explicitly invalidated, so other
|
||||
//! devices keep their existing logins).
|
||||
|
||||
use std::sync::OnceLock;
|
||||
|
||||
use axum::extract::{Path, State};
|
||||
use axum::http::StatusCode;
|
||||
use axum::response::IntoResponse;
|
||||
@@ -80,6 +82,7 @@ async fn register(
|
||||
jar: CookieJar,
|
||||
Json(input): Json<Credentials>,
|
||||
) -> AppResult<impl IntoResponse> {
|
||||
check_auth_rate_limit(&state, "register")?;
|
||||
let username = input.username.trim();
|
||||
validate_username(username)?;
|
||||
validate_password(&input.password)?;
|
||||
@@ -95,6 +98,7 @@ async fn login(
|
||||
jar: CookieJar,
|
||||
Json(input): Json<Credentials>,
|
||||
) -> AppResult<impl IntoResponse> {
|
||||
check_auth_rate_limit(&state, "login")?;
|
||||
let username = input.username.trim();
|
||||
if username.is_empty() || input.password.is_empty() {
|
||||
return Err(AppError::InvalidInput(
|
||||
@@ -102,9 +106,15 @@ async fn login(
|
||||
));
|
||||
}
|
||||
|
||||
let user = repo::user::find_by_username(&state.db, username)
|
||||
.await?
|
||||
.ok_or(AppError::Unauthenticated)?;
|
||||
let user = repo::user::find_by_username(&state.db, username).await?;
|
||||
let Some(user) = user else {
|
||||
// No such user. Run argon2 against a stable dummy hash so the
|
||||
// response time matches the wrong-password branch — otherwise
|
||||
// an attacker can enumerate usernames by timing the no-user
|
||||
// 401 against the wrong-password 401.
|
||||
let _ = verify_password(&input.password, dummy_password_hash());
|
||||
return Err(AppError::Unauthenticated);
|
||||
};
|
||||
if !verify_password(&input.password, &user.password_hash) {
|
||||
return Err(AppError::Unauthenticated);
|
||||
}
|
||||
@@ -113,6 +123,21 @@ async fn login(
|
||||
Ok((StatusCode::OK, jar, Json(AuthResponse { user })))
|
||||
}
|
||||
|
||||
/// Lazily-computed argon2 hash used to equalise login response time
|
||||
/// across the "no such user" and "wrong password" branches. Computing
|
||||
/// it once (on the first login of the process) is enough — the hash is
|
||||
/// never compared against a real password, only used to force argon2
|
||||
/// to do the same amount of work it would for a real verify.
|
||||
fn dummy_password_hash() -> &'static str {
|
||||
static DUMMY: OnceLock<String> = OnceLock::new();
|
||||
DUMMY
|
||||
.get_or_init(|| {
|
||||
crate::auth::password::hash_password("login-timing-equaliser")
|
||||
.expect("hash_password on a fixed input cannot fail")
|
||||
})
|
||||
.as_str()
|
||||
}
|
||||
|
||||
async fn logout(
|
||||
State(state): State<AppState>,
|
||||
jar: CookieJar,
|
||||
@@ -149,6 +174,7 @@ async fn change_password(
|
||||
jar: CookieJar,
|
||||
Json(input): Json<ChangePassword>,
|
||||
) -> AppResult<impl IntoResponse> {
|
||||
check_auth_rate_limit(&state, "change_password")?;
|
||||
if !verify_password(&input.current_password, &user.password_hash) {
|
||||
return Err(AppError::Unauthenticated);
|
||||
}
|
||||
@@ -230,8 +256,24 @@ async fn create_token(
|
||||
Json(input): Json<CreateTokenInput>,
|
||||
) -> AppResult<impl IntoResponse> {
|
||||
let name = input.name.trim();
|
||||
// Both arms use `ValidationFailed` (422 with field details) to
|
||||
// match the structured-error shape `attach_tag` returns for the
|
||||
// same kind of free-form-identifier validation. The other
|
||||
// /auth/* handlers in this file use `InvalidInput` (400); the
|
||||
// divergence is pre-existing and would warrant a project-wide
|
||||
// pass to flip them all if the client side wants uniform per-
|
||||
// field error rendering.
|
||||
if name.is_empty() {
|
||||
return Err(AppError::InvalidInput("token name is required".into()));
|
||||
return Err(AppError::ValidationFailed {
|
||||
message: "token name is required".into(),
|
||||
details: serde_json::json!({ "name": "required" }),
|
||||
});
|
||||
}
|
||||
if name.chars().count() > 64 {
|
||||
return Err(AppError::ValidationFailed {
|
||||
message: "token name too long".into(),
|
||||
details: serde_json::json!({ "name": "max 64 characters" }),
|
||||
});
|
||||
}
|
||||
let (raw, hash) = generate_token();
|
||||
let token = repo::api_token::create(&state.db, user.id, name, &hash).await?;
|
||||
@@ -293,6 +335,33 @@ fn build_expired_cookie(cfg: &AuthConfig) -> Cookie<'static> {
|
||||
builder.build()
|
||||
}
|
||||
|
||||
/// Consume one token from the shared auth rate limiter. Called at the
|
||||
/// start of `register`, `login`, and `change_password` so credential
|
||||
/// stuffing / spraying / username-probe loops are throttled by the
|
||||
/// configured budget (default 5/sec with a 10-request burst).
|
||||
///
|
||||
/// All three endpoints share one bucket — they all expose the same
|
||||
/// argon2-verify-or-create work and the same enumeration channels, so
|
||||
/// any one of them in a tight loop should trip the limit. `endpoint`
|
||||
/// is included in the rate-limit-hit log line so operators can tell
|
||||
/// which endpoint is being probed.
|
||||
fn check_auth_rate_limit(state: &AppState, endpoint: &'static str) -> AppResult<()> {
|
||||
use crate::auth::rate_limit::AcquireResult;
|
||||
match state.auth_limiter.try_acquire() {
|
||||
AcquireResult::Allowed => Ok(()),
|
||||
AcquireResult::Denied { retry_after_secs } => {
|
||||
tracing::warn!(
|
||||
endpoint,
|
||||
retry_after_secs,
|
||||
"auth rate limit hit; returning 429"
|
||||
);
|
||||
Err(AppError::TooManyRequests {
|
||||
retry_after_secs: Some(retry_after_secs),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_username(u: &str) -> AppResult<()> {
|
||||
if u.is_empty() {
|
||||
return Err(AppError::InvalidInput("username is required".into()));
|
||||
|
||||
@@ -67,14 +67,7 @@ async fn create(
|
||||
// the foreign-key violation collapse into a generic 500.
|
||||
repo::manga::get(&state.db, input.manga_id).await?;
|
||||
if let Some(chapter_id) = input.chapter_id {
|
||||
let exists: Option<(Uuid,)> = sqlx::query_as(
|
||||
"SELECT id FROM chapters WHERE id = $1 AND manga_id = $2",
|
||||
)
|
||||
.bind(chapter_id)
|
||||
.bind(input.manga_id)
|
||||
.fetch_optional(&state.db)
|
||||
.await?;
|
||||
if exists.is_none() {
|
||||
if !repo::chapter::belongs_to_manga(&state.db, chapter_id, input.manga_id).await? {
|
||||
return Err(AppError::NotFound);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use axum::extract::{Multipart, Path, Query, State};
|
||||
use axum::http::StatusCode;
|
||||
use axum::routing::{delete, get, post};
|
||||
use axum::routing::{delete, get, post, put};
|
||||
use axum::{Json, Router};
|
||||
use serde::Deserialize;
|
||||
use serde_json::json;
|
||||
@@ -14,12 +14,14 @@ use crate::domain::patch::Patch;
|
||||
use crate::domain::tag::TagRef;
|
||||
use crate::error::{AppError, AppResult};
|
||||
use crate::repo;
|
||||
use crate::storage::StorageError;
|
||||
use crate::upload::{parse_image, UploadedImage};
|
||||
|
||||
pub fn routes() -> Router<AppState> {
|
||||
Router::new()
|
||||
.route("/mangas", get(list).post(create))
|
||||
.route("/mangas/:id", get(get_one).patch(update))
|
||||
.route("/mangas/:id/cover", put(put_cover).delete(delete_cover))
|
||||
.route("/mangas/:id/tags", post(attach_tag))
|
||||
.route("/mangas/:id/tags/:tag_id", delete(detach_tag))
|
||||
}
|
||||
@@ -194,16 +196,14 @@ async fn create(
|
||||
|
||||
async fn update(
|
||||
State(state): State<AppState>,
|
||||
CurrentUser(_user): CurrentUser,
|
||||
CurrentUser(user): CurrentUser,
|
||||
Path(id): Path<Uuid>,
|
||||
Json(patch): Json<MangaPatch>,
|
||||
) -> AppResult<Json<MangaDetail>> {
|
||||
// TODO(auth): until uploaders are tracked (Phase 5), any signed-in
|
||||
// user can edit any manga. Restrict to uploader + admin once that
|
||||
// column lands.
|
||||
if !repo::manga::exists(&state.db, id).await? {
|
||||
return Err(AppError::NotFound);
|
||||
}
|
||||
require_can_edit(&state, id, user.id).await?;
|
||||
|
||||
if let Some(ref status) = patch.status {
|
||||
let trimmed = status.trim();
|
||||
@@ -259,6 +259,80 @@ async fn update(
|
||||
Ok(Json(repo::manga::get_detail(&state.db, id).await?))
|
||||
}
|
||||
|
||||
/// `PUT /api/v1/mangas/:id/cover` is multipart/form-data with a single
|
||||
/// required `cover` part containing image bytes. MIME is sniffed by
|
||||
/// magic bytes (jpeg/png/webp/gif/avif); filename and Content-Type from
|
||||
/// the client are ignored. Replaces any existing cover, deleting the
|
||||
/// previous blob if its extension differs. Returns the refreshed
|
||||
/// `MangaDetail`.
|
||||
async fn put_cover(
|
||||
State(state): State<AppState>,
|
||||
CurrentUser(user): CurrentUser,
|
||||
Path(id): Path<Uuid>,
|
||||
mut multipart: Multipart,
|
||||
) -> AppResult<Json<MangaDetail>> {
|
||||
if !repo::manga::exists(&state.db, id).await? {
|
||||
return Err(AppError::NotFound);
|
||||
}
|
||||
require_can_edit(&state, id, user.id).await?;
|
||||
|
||||
let mut cover: Option<UploadedImage> = None;
|
||||
while let Some(field) = next_field(&mut multipart).await? {
|
||||
if field.name() == Some("cover") {
|
||||
let bytes = read_field_bytes(field).await?.to_vec();
|
||||
cover = Some(parse_image(bytes, state.upload.max_file_bytes, "cover")?);
|
||||
}
|
||||
}
|
||||
let img = cover.ok_or_else(|| AppError::ValidationFailed {
|
||||
message: "cover part is required".into(),
|
||||
details: json!({ "cover": "required" }),
|
||||
})?;
|
||||
|
||||
// Read the old key BEFORE writing so we can clean up an orphan if
|
||||
// the extension changed (e.g., .png → .jpg). Same-extension is a
|
||||
// `put` overwrite — no delete needed.
|
||||
let old_key = repo::manga::get(&state.db, id).await?.cover_image_path;
|
||||
let new_key = format!("mangas/{}/cover.{}", id, img.ext);
|
||||
state.storage.put(&new_key, &img.bytes).await?;
|
||||
|
||||
if let Some(prev) = old_key.as_deref() {
|
||||
if prev != new_key {
|
||||
// Swallow NotFound — AppError maps it to a client 404,
|
||||
// which would be wrong here. The DB row can outlive a
|
||||
// manually-deleted blob.
|
||||
match state.storage.delete(prev).await {
|
||||
Ok(()) | Err(StorageError::NotFound) => {}
|
||||
Err(e) => return Err(e.into()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
repo::manga::set_cover_image_path(&state.db, id, &new_key).await?;
|
||||
Ok(Json(repo::manga::get_detail(&state.db, id).await?))
|
||||
}
|
||||
|
||||
/// `DELETE /api/v1/mangas/:id/cover` clears `cover_image_path` and
|
||||
/// removes the blob. Idempotent: removing a non-existent cover succeeds
|
||||
/// with the unchanged detail.
|
||||
async fn delete_cover(
|
||||
State(state): State<AppState>,
|
||||
CurrentUser(user): CurrentUser,
|
||||
Path(id): Path<Uuid>,
|
||||
) -> AppResult<Json<MangaDetail>> {
|
||||
if !repo::manga::exists(&state.db, id).await? {
|
||||
return Err(AppError::NotFound);
|
||||
}
|
||||
require_can_edit(&state, id, user.id).await?;
|
||||
if let Some(key) = repo::manga::get(&state.db, id).await?.cover_image_path {
|
||||
match state.storage.delete(&key).await {
|
||||
Ok(()) | Err(StorageError::NotFound) => {}
|
||||
Err(e) => return Err(e.into()),
|
||||
}
|
||||
repo::manga::clear_cover_image_path(&state.db, id).await?;
|
||||
}
|
||||
Ok(Json(repo::manga::get_detail(&state.db, id).await?))
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct AttachTagBody {
|
||||
pub name: String,
|
||||
@@ -270,6 +344,7 @@ async fn attach_tag(
|
||||
Path(id): Path<Uuid>,
|
||||
Json(body): Json<AttachTagBody>,
|
||||
) -> AppResult<(StatusCode, Json<TagRef>)> {
|
||||
validate_tag_name(&body.name)?;
|
||||
if !repo::manga::exists(&state.db, id).await? {
|
||||
return Err(AppError::NotFound);
|
||||
}
|
||||
@@ -316,6 +391,27 @@ async fn detach_tag(
|
||||
}
|
||||
}
|
||||
|
||||
/// Request-side validation for `POST /mangas/:id/tags` body. Mirrors
|
||||
/// the repo-level cap in `repo::tag::upsert_by_name` (max 64 chars
|
||||
/// after trim) but surfaces the failure at the handler boundary with
|
||||
/// the same envelope shape other validations use.
|
||||
fn validate_tag_name(name: &str) -> AppResult<()> {
|
||||
let trimmed = name.trim();
|
||||
if trimmed.is_empty() {
|
||||
return Err(AppError::ValidationFailed {
|
||||
message: "tag name cannot be empty".into(),
|
||||
details: json!({ "name": "required" }),
|
||||
});
|
||||
}
|
||||
if trimmed.chars().count() > 64 {
|
||||
return Err(AppError::ValidationFailed {
|
||||
message: "tag name too long".into(),
|
||||
details: json!({ "name": "max 64 characters" }),
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_new_manga(input: &NewManga) -> AppResult<()> {
|
||||
if input.title.trim().is_empty() {
|
||||
return Err(AppError::ValidationFailed {
|
||||
@@ -335,6 +431,30 @@ fn validate_new_manga(input: &NewManga) -> AppResult<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Authorisation gate for manga mutations. The manga is assumed to
|
||||
/// exist (the caller runs [`repo::manga::exists`] first so a missing id
|
||||
/// surfaces as `NotFound`, not `Forbidden`).
|
||||
///
|
||||
/// Rule: a non-NULL `uploaded_by` must match the current user. Legacy
|
||||
/// rows with `uploaded_by IS NULL` (pre-migration-0011) are still
|
||||
/// editable by any signed-in user — there's nobody to gate on yet, and
|
||||
/// the historical-data note in 0011 acknowledges the gap. Once an
|
||||
/// admin role lands the NULL case can flip to admin-only.
|
||||
///
|
||||
/// Returns `Forbidden` (not `NotFound`) on owner mismatch — mangas
|
||||
/// are listable via `GET /mangas`, so existence isn't a secret and
|
||||
/// the more accurate 403 is fine. This deliberately differs from
|
||||
/// `repo::collection::require_owner`, which collapses both states to
|
||||
/// `NotFound` because collections are private to a user and existence
|
||||
/// itself is information worth hiding from non-owners.
|
||||
async fn require_can_edit(state: &AppState, manga_id: Uuid, user_id: Uuid) -> AppResult<()> {
|
||||
match repo::manga::uploaded_by(&state.db, manga_id).await? {
|
||||
Some(owner) if owner != user_id => Err(AppError::Forbidden),
|
||||
// Some(owner) == user_id (good) or None (legacy row, no owner).
|
||||
_ => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
async fn validate_genre_ids(state: &AppState, ids: &[Uuid]) -> AppResult<()> {
|
||||
if ids.is_empty() {
|
||||
return Ok(());
|
||||
|
||||
@@ -12,14 +12,18 @@ use tokio_util::sync::CancellationToken;
|
||||
use tower_http::cors::{AllowOrigin, CorsLayer};
|
||||
use tower_http::trace::TraceLayer;
|
||||
|
||||
use crate::config::{AuthConfig, Config, CrawlerConfig, UploadConfig};
|
||||
use crate::auth::rate_limit::AuthRateLimiter;
|
||||
use crate::config::{AuthConfig, Config, CrawlerConfig, CrawlerModePref, UploadConfig};
|
||||
use crate::crawler::browser_manager::{self, BrowserManager};
|
||||
use crate::crawler::content::{self, SyncOutcome};
|
||||
use crate::crawler::daemon::{self, ChapterDispatcher, DaemonConfig, MetadataPass};
|
||||
use crate::crawler::jobs::JobPayload;
|
||||
use crate::crawler::pipeline::{self, MetadataStats};
|
||||
use crate::crawler::rate_limit::HostRateLimiters;
|
||||
use crate::crawler::safety::DownloadAllowlist;
|
||||
use crate::crawler::session;
|
||||
use crate::crawler::source::{target as target_source, DiscoverMode};
|
||||
use crate::repo;
|
||||
use crate::storage::{LocalStorage, Storage};
|
||||
|
||||
#[derive(Clone)]
|
||||
@@ -28,6 +32,10 @@ pub struct AppState {
|
||||
pub storage: Arc<dyn Storage>,
|
||||
pub auth: AuthConfig,
|
||||
pub upload: UploadConfig,
|
||||
/// Shared rate limiter guarding the `/auth/*` mutation endpoints.
|
||||
/// One instance per AppState so tests stay isolated across the
|
||||
/// same process.
|
||||
pub auth_limiter: Arc<AuthRateLimiter>,
|
||||
}
|
||||
|
||||
/// Bundle returned by [`build`]. The router is what `axum::serve` consumes;
|
||||
@@ -62,11 +70,13 @@ pub async fn build(config: Config) -> anyhow::Result<AppHandle> {
|
||||
None
|
||||
};
|
||||
|
||||
let auth_limiter = Arc::new(AuthRateLimiter::new(config.auth.rate_limit));
|
||||
let state = AppState {
|
||||
db,
|
||||
storage,
|
||||
auth: config.auth.clone(),
|
||||
upload: config.upload.clone(),
|
||||
auth_limiter,
|
||||
};
|
||||
let router = router(state).layer(cors_layer(&config.cors_allowed_origins));
|
||||
Ok(AppHandle { router, daemon })
|
||||
@@ -149,6 +159,10 @@ async fn spawn_crawler_daemon(
|
||||
http: http.clone(),
|
||||
rate: Arc::clone(&rate),
|
||||
start_url: url.clone(),
|
||||
mode_pref: cfg.mode,
|
||||
incremental_stop_after: cfg.incremental_stop_after,
|
||||
download_allowlist: cfg.download_allowlist.clone(),
|
||||
max_image_bytes: cfg.max_image_bytes,
|
||||
});
|
||||
m
|
||||
});
|
||||
@@ -159,6 +173,8 @@ async fn spawn_crawler_daemon(
|
||||
storage: Arc::clone(&storage),
|
||||
http,
|
||||
rate: Arc::clone(&rate),
|
||||
download_allowlist: cfg.download_allowlist.clone(),
|
||||
max_image_bytes: cfg.max_image_bytes,
|
||||
});
|
||||
|
||||
// Shared cancellation: daemon shutdown cancels the BrowserManager's
|
||||
@@ -210,11 +226,22 @@ struct RealMetadataPass {
|
||||
http: reqwest::Client,
|
||||
rate: Arc<HostRateLimiters>,
|
||||
start_url: String,
|
||||
mode_pref: CrawlerModePref,
|
||||
incremental_stop_after: usize,
|
||||
download_allowlist: DownloadAllowlist,
|
||||
max_image_bytes: usize,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl MetadataPass for RealMetadataPass {
|
||||
async fn run(&self) -> anyhow::Result<MetadataStats> {
|
||||
let mode = resolve_mode(
|
||||
&self.db,
|
||||
target_source::SOURCE_ID,
|
||||
self.mode_pref,
|
||||
self.incremental_stop_after,
|
||||
)
|
||||
.await?;
|
||||
pipeline::run_metadata_pass(
|
||||
&self.browser_manager,
|
||||
&self.db,
|
||||
@@ -224,17 +251,66 @@ impl MetadataPass for RealMetadataPass {
|
||||
&self.start_url,
|
||||
0,
|
||||
false,
|
||||
mode,
|
||||
&self.download_allowlist,
|
||||
self.max_image_bytes,
|
||||
)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
/// Pick the active mode for this tick. `Explicit` short-circuits the
|
||||
/// DB lookup. `Auto` reads `seed_completed_at`: missing → Backfill
|
||||
/// (initial seed for this source), present → Incremental with the
|
||||
/// configured threshold.
|
||||
///
|
||||
/// A DB error during the Auto lookup propagates as `Err` rather than
|
||||
/// silently degrading to Backfill — the daemon's `run_tick` catches
|
||||
/// the error, logs, and skips the tick. That's safer than running a
|
||||
/// full re-backfill (including a drop pass against stale-looking rows)
|
||||
/// when the DB is flaky.
|
||||
async fn resolve_mode(
|
||||
db: &PgPool,
|
||||
source_id: &str,
|
||||
pref: CrawlerModePref,
|
||||
incremental_stop_after: usize,
|
||||
) -> anyhow::Result<DiscoverMode> {
|
||||
match pref {
|
||||
CrawlerModePref::Explicit(m) => {
|
||||
tracing::info!(?m, "crawler mode: explicit (CRAWLER_MODE override)");
|
||||
Ok(m)
|
||||
}
|
||||
CrawlerModePref::Auto => {
|
||||
let seeded = repo::crawler::seed_completed_at(db, source_id)
|
||||
.await
|
||||
.context("seed_completed_at lookup for mode auto-detection")?;
|
||||
match seeded {
|
||||
Some(at) => {
|
||||
tracing::info!(
|
||||
seed_completed_at = %at.to_rfc3339(),
|
||||
"crawler mode: auto → incremental (seed previously completed)"
|
||||
);
|
||||
Ok(DiscoverMode::Incremental {
|
||||
stop_after_unchanged: incremental_stop_after,
|
||||
})
|
||||
}
|
||||
None => {
|
||||
tracing::info!("crawler mode: auto → backfill (no seed marker for source)");
|
||||
Ok(DiscoverMode::Backfill)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct RealChapterDispatcher {
|
||||
browser_manager: Arc<BrowserManager>,
|
||||
db: PgPool,
|
||||
storage: Arc<dyn Storage>,
|
||||
http: reqwest::Client,
|
||||
rate: Arc<HostRateLimiters>,
|
||||
download_allowlist: DownloadAllowlist,
|
||||
max_image_bytes: usize,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
@@ -246,18 +322,9 @@ impl ChapterDispatcher for RealChapterDispatcher {
|
||||
chapter_id,
|
||||
source_chapter_key: _,
|
||||
} => {
|
||||
// Look up manga_id + source_url for this chapter.
|
||||
let row: Option<(uuid::Uuid, String)> = sqlx::query_as(
|
||||
"SELECT c.manga_id, cs.source_url \
|
||||
FROM chapters c \
|
||||
JOIN chapter_sources cs ON cs.chapter_id = c.id \
|
||||
WHERE c.id = $1 \
|
||||
LIMIT 1",
|
||||
)
|
||||
.bind(chapter_id)
|
||||
.fetch_optional(&self.db)
|
||||
.await
|
||||
.context("look up chapter for dispatch")?;
|
||||
let row = repo::chapter::dispatch_target(&self.db, chapter_id)
|
||||
.await
|
||||
.context("look up chapter for dispatch")?;
|
||||
let Some((manga_id, source_url)) = row else {
|
||||
// Chapter (or its source row) is gone — ack done.
|
||||
return Ok(SyncOutcome::Skipped);
|
||||
@@ -273,6 +340,8 @@ impl ChapterDispatcher for RealChapterDispatcher {
|
||||
manga_id,
|
||||
&source_url,
|
||||
false,
|
||||
&self.download_allowlist,
|
||||
self.max_image_bytes,
|
||||
)
|
||||
.await?;
|
||||
drop(lease);
|
||||
|
||||
@@ -7,4 +7,5 @@
|
||||
|
||||
pub mod extractor;
|
||||
pub mod password;
|
||||
pub mod rate_limit;
|
||||
pub mod token;
|
||||
|
||||
179
backend/src/auth/rate_limit.rs
Normal file
179
backend/src/auth/rate_limit.rs
Normal file
@@ -0,0 +1,179 @@
|
||||
//! Per-process token-bucket rate limiter for the auth endpoints.
|
||||
//!
|
||||
//! Protects `/auth/login`, `/auth/register`, and `/auth/me/password`
|
||||
//! from credential stuffing / password spraying / username probing.
|
||||
//!
|
||||
//! The current deploy puts SvelteKit's hooks.server.ts proxy in front
|
||||
//! of axum without forwarding the original client IP (no
|
||||
//! `X-Forwarded-For`), so per-IP buckets would all collapse to the
|
||||
//! proxy container's address. Until the proxy learns to forward the
|
||||
//! peer address, a single global bucket gives equivalent protection
|
||||
//! against mass-attack patterns and trades a small DoS surface
|
||||
//! (legitimate users sharing the limit) for simplicity.
|
||||
//!
|
||||
//! Each `AppState` carries its own [`AuthRateLimiter`] instance, so
|
||||
//! tests run in isolated buckets and won't bleed across `#[sqlx::test]`
|
||||
//! cases that share a process.
|
||||
|
||||
use std::sync::Mutex;
|
||||
use std::time::Instant;
|
||||
|
||||
/// Tunable limits. `per_sec == 0` disables the limiter — used by the
|
||||
/// test harness and by anyone who wants to opt out via env config.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct RateLimitConfig {
|
||||
pub per_sec: u32,
|
||||
pub burst: u32,
|
||||
}
|
||||
|
||||
impl Default for RateLimitConfig {
|
||||
/// Disabled by default. The production `AuthConfig::from_env`
|
||||
/// overrides to a real limit; the test harness keeps the default
|
||||
/// so existing tests don't flake against shared buckets.
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
per_sec: 0,
|
||||
burst: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Production defaults: 5 requests/sec sustained, 10-request burst.
|
||||
/// Tight enough to make brute force impractical, loose enough that a
|
||||
/// real user mistyping their password three times in a row doesn't
|
||||
/// hit it.
|
||||
pub const PRODUCTION_PER_SEC: u32 = 5;
|
||||
pub const PRODUCTION_BURST: u32 = 10;
|
||||
|
||||
struct Bucket {
|
||||
tokens: f64,
|
||||
last_refill: Instant,
|
||||
}
|
||||
|
||||
/// Outcome of [`AuthRateLimiter::try_acquire`]. When `Denied`, the
|
||||
/// caller can use `retry_after_secs` for a `Retry-After: N` header
|
||||
/// (RFC 6585 §4) so well-behaved clients back off correctly rather
|
||||
/// than retrying in a tight loop.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum AcquireResult {
|
||||
Allowed,
|
||||
Denied { retry_after_secs: u64 },
|
||||
}
|
||||
|
||||
/// Single-bucket token-bucket limiter. `try_acquire` is cheap (one
|
||||
/// mutex acquire, no allocations) so the auth path doesn't pay a real
|
||||
/// cost for the check.
|
||||
pub struct AuthRateLimiter {
|
||||
cfg: RateLimitConfig,
|
||||
bucket: Mutex<Bucket>,
|
||||
}
|
||||
|
||||
impl AuthRateLimiter {
|
||||
pub fn new(cfg: RateLimitConfig) -> Self {
|
||||
Self {
|
||||
cfg,
|
||||
bucket: Mutex::new(Bucket {
|
||||
tokens: cfg.burst as f64,
|
||||
last_refill: Instant::now(),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
/// Consume one token if available. Returns `Denied` with a
|
||||
/// rounded-up seconds-until-refill so the caller can emit a
|
||||
/// `Retry-After` header.
|
||||
pub fn try_acquire(&self) -> AcquireResult {
|
||||
if self.cfg.per_sec == 0 {
|
||||
return AcquireResult::Allowed;
|
||||
}
|
||||
let now = Instant::now();
|
||||
let mut bucket = self.bucket.lock().expect("rate limiter mutex poisoned");
|
||||
let elapsed = now.duration_since(bucket.last_refill).as_secs_f64();
|
||||
bucket.tokens =
|
||||
(bucket.tokens + elapsed * f64::from(self.cfg.per_sec)).min(f64::from(self.cfg.burst));
|
||||
bucket.last_refill = now;
|
||||
if bucket.tokens >= 1.0 {
|
||||
bucket.tokens -= 1.0;
|
||||
AcquireResult::Allowed
|
||||
} else {
|
||||
// ceil((1 - tokens) / per_sec), minimum 1 — a `Retry-After: 0`
|
||||
// would tell clients to retry immediately, which is what we're
|
||||
// actively trying to discourage.
|
||||
let deficit = 1.0 - bucket.tokens;
|
||||
let wait_secs = (deficit / f64::from(self.cfg.per_sec)).ceil() as u64;
|
||||
AcquireResult::Denied {
|
||||
retry_after_secs: wait_secs.max(1),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn disabled_limiter_always_allows() {
|
||||
let rl = AuthRateLimiter::new(RateLimitConfig {
|
||||
per_sec: 0,
|
||||
burst: 0,
|
||||
});
|
||||
for _ in 0..1000 {
|
||||
assert_eq!(rl.try_acquire(), AcquireResult::Allowed);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn burst_lets_through_initial_window_then_blocks() {
|
||||
// 0 refill, burst 3 → first three pass, fourth blocks.
|
||||
let rl = AuthRateLimiter::new(RateLimitConfig {
|
||||
per_sec: 1,
|
||||
burst: 3,
|
||||
});
|
||||
assert_eq!(rl.try_acquire(), AcquireResult::Allowed);
|
||||
assert_eq!(rl.try_acquire(), AcquireResult::Allowed);
|
||||
assert_eq!(rl.try_acquire(), AcquireResult::Allowed);
|
||||
match rl.try_acquire() {
|
||||
AcquireResult::Denied { retry_after_secs } => {
|
||||
// Bucket is at ~0 tokens, refill rate 1/sec → ~1s wait.
|
||||
assert!(
|
||||
retry_after_secs >= 1,
|
||||
"retry_after must be at least 1s, got {retry_after_secs}"
|
||||
);
|
||||
}
|
||||
AcquireResult::Allowed => panic!("fourth request must be denied"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tokens_refill_over_time() {
|
||||
// 10/sec → after ~120ms we should have at least one token back.
|
||||
let rl = AuthRateLimiter::new(RateLimitConfig {
|
||||
per_sec: 10,
|
||||
burst: 1,
|
||||
});
|
||||
assert_eq!(rl.try_acquire(), AcquireResult::Allowed);
|
||||
assert!(matches!(rl.try_acquire(), AcquireResult::Denied { .. }));
|
||||
std::thread::sleep(std::time::Duration::from_millis(150));
|
||||
assert_eq!(
|
||||
rl.try_acquire(),
|
||||
AcquireResult::Allowed,
|
||||
"token should have refilled"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retry_after_scales_inversely_with_refill_rate() {
|
||||
// 1/sec → wait ~1s after burst exhausted.
|
||||
// 10/sec → wait <1s, but we clamp to a minimum of 1s.
|
||||
let slow = AuthRateLimiter::new(RateLimitConfig {
|
||||
per_sec: 1,
|
||||
burst: 1,
|
||||
});
|
||||
slow.try_acquire();
|
||||
match slow.try_acquire() {
|
||||
AcquireResult::Denied { retry_after_secs } => assert_eq!(retry_after_secs, 1),
|
||||
_ => panic!("expected Denied"),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -31,6 +31,7 @@ use mangalord::crawler::content::{self, SyncOutcome};
|
||||
use mangalord::crawler::pipeline;
|
||||
use mangalord::crawler::rate_limit::HostRateLimiters;
|
||||
use mangalord::crawler::session;
|
||||
use mangalord::crawler::source::DiscoverMode;
|
||||
use mangalord::storage::{LocalStorage, Storage};
|
||||
use sqlx::postgres::PgPoolOptions;
|
||||
use sqlx::PgPool;
|
||||
@@ -62,6 +63,8 @@ async fn main() -> anyhow::Result<()> {
|
||||
let cdn_rate_ms = env_u64("CRAWLER_CDN_RATE_MS", rate_ms);
|
||||
let limit = env_u64("CRAWLER_LIMIT", 0) as usize;
|
||||
let skip_chapters = env_bool("CRAWLER_SKIP_CHAPTERS", false);
|
||||
let incremental_stop_after = env_u64("CRAWLER_INCREMENTAL_STOP_AFTER", 20).max(1) as usize;
|
||||
let mode = parse_crawler_mode(incremental_stop_after)?;
|
||||
let skip_chapter_content = env_bool("CRAWLER_SKIP_CHAPTER_CONTENT", false);
|
||||
let chapter_workers = env_u64("CRAWLER_CHAPTER_WORKERS", 1).max(1) as usize;
|
||||
let force_refetch_chapters = env_bool("CRAWLER_FORCE_REFETCH_CHAPTERS", false);
|
||||
@@ -140,6 +143,7 @@ async fn main() -> anyhow::Result<()> {
|
||||
user_agent = ?user_agent,
|
||||
proxy = ?proxy_url,
|
||||
keep_open,
|
||||
?mode,
|
||||
storage_dir = %storage_dir.display(),
|
||||
"starting crawler"
|
||||
);
|
||||
@@ -187,6 +191,7 @@ async fn main() -> anyhow::Result<()> {
|
||||
skip_chapter_content || !session_ready,
|
||||
chapter_workers,
|
||||
force_refetch_chapters,
|
||||
mode,
|
||||
)
|
||||
.await;
|
||||
|
||||
@@ -216,6 +221,7 @@ async fn run(
|
||||
skip_chapter_content: bool,
|
||||
chapter_workers: usize,
|
||||
force_refetch_chapters: bool,
|
||||
mode: DiscoverMode,
|
||||
) -> anyhow::Result<()> {
|
||||
let mut rate = HostRateLimiters::new(Duration::from_millis(rate_ms));
|
||||
if let Some(host) = cdn_host {
|
||||
@@ -223,6 +229,33 @@ async fn run(
|
||||
}
|
||||
let rate = Arc::new(rate);
|
||||
|
||||
// SSRF defence: only download from the catalog host + CDN host
|
||||
// (plus optional CRAWLER_DOWNLOAD_ALLOWLIST extras), and cap
|
||||
// single-image downloads at CRAWLER_MAX_IMAGE_BYTES bytes.
|
||||
let mut allowlist =
|
||||
mangalord::crawler::safety::DownloadAllowlist::new();
|
||||
if let Ok(parsed) = reqwest::Url::parse(start_url) {
|
||||
if let Some(h) = parsed.host_str() {
|
||||
allowlist = allowlist.allow(h);
|
||||
}
|
||||
}
|
||||
if let Some(host) = cdn_host {
|
||||
allowlist = allowlist.allow(host);
|
||||
}
|
||||
if let Ok(extras) = std::env::var("CRAWLER_DOWNLOAD_ALLOWLIST") {
|
||||
for piece in extras.split(',') {
|
||||
let trimmed = piece.trim();
|
||||
if !trimmed.is_empty() {
|
||||
allowlist = allowlist.allow(trimmed);
|
||||
}
|
||||
}
|
||||
}
|
||||
let max_image_bytes: usize = std::env::var("CRAWLER_MAX_IMAGE_BYTES")
|
||||
.ok()
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(mangalord::crawler::safety::DEFAULT_MAX_IMAGE_BYTES);
|
||||
let allowlist = Arc::new(allowlist);
|
||||
|
||||
let stats = pipeline::run_metadata_pass(
|
||||
manager.as_ref(),
|
||||
db,
|
||||
@@ -232,6 +265,9 @@ async fn run(
|
||||
start_url,
|
||||
limit,
|
||||
skip_chapters,
|
||||
mode,
|
||||
allowlist.as_ref(),
|
||||
max_image_bytes,
|
||||
)
|
||||
.await?;
|
||||
tracing::info!(?stats, "metadata pass complete");
|
||||
@@ -246,6 +282,8 @@ async fn run(
|
||||
"target",
|
||||
chapter_workers,
|
||||
force_refetch_chapters,
|
||||
Arc::clone(&allowlist),
|
||||
max_image_bytes,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
@@ -269,6 +307,8 @@ async fn sync_bookmarked_chapter_content(
|
||||
source_id: &str,
|
||||
workers: usize,
|
||||
force_refetch: bool,
|
||||
allowlist: Arc<mangalord::crawler::safety::DownloadAllowlist>,
|
||||
max_image_bytes: usize,
|
||||
) -> anyhow::Result<()> {
|
||||
let pending: Vec<(Uuid, Uuid, String)> = sqlx::query_as(
|
||||
r#"
|
||||
@@ -305,6 +345,7 @@ async fn sync_bookmarked_chapter_content(
|
||||
let storage = Arc::clone(&storage);
|
||||
let rate = Arc::clone(&rate);
|
||||
let manager = Arc::clone(&manager);
|
||||
let allowlist = Arc::clone(&allowlist);
|
||||
let stats = &stats;
|
||||
async move {
|
||||
if session_expired.load(std::sync::atomic::Ordering::Relaxed) {
|
||||
@@ -329,6 +370,8 @@ async fn sync_bookmarked_chapter_content(
|
||||
manga_id,
|
||||
&source_url,
|
||||
force_refetch,
|
||||
allowlist.as_ref(),
|
||||
max_image_bytes,
|
||||
)
|
||||
.await;
|
||||
drop(lease);
|
||||
@@ -390,6 +433,38 @@ fn resolve_start_url() -> anyhow::Result<String> {
|
||||
})
|
||||
}
|
||||
|
||||
/// Parse the CLI's `CRAWLER_MODE`. Defaults to `backfill` because the
|
||||
/// binary is operator-driven (manual reseeds, force-refetches) — the
|
||||
/// auto-detect logic lives in the daemon. `auto` is rejected because
|
||||
/// the CLI has no DB state to consult before the run.
|
||||
fn parse_crawler_mode(incremental_stop_after: usize) -> anyhow::Result<DiscoverMode> {
|
||||
parse_crawler_mode_str(
|
||||
std::env::var("CRAWLER_MODE").ok().as_deref(),
|
||||
incremental_stop_after,
|
||||
)
|
||||
}
|
||||
|
||||
/// Pure variant of [`parse_crawler_mode`] — testable without env-var
|
||||
/// mutation.
|
||||
fn parse_crawler_mode_str(
|
||||
raw: Option<&str>,
|
||||
incremental_stop_after: usize,
|
||||
) -> anyhow::Result<DiscoverMode> {
|
||||
match raw.map(|s| s.trim().to_ascii_lowercase()).as_deref() {
|
||||
None | Some("") | Some("backfill") => Ok(DiscoverMode::Backfill),
|
||||
Some("incremental") => Ok(DiscoverMode::Incremental {
|
||||
stop_after_unchanged: incremental_stop_after,
|
||||
}),
|
||||
Some("auto") => Err(anyhow!(
|
||||
"CRAWLER_MODE=auto isn't supported by the CLI (use backfill or incremental); \
|
||||
the daemon does auto-detection"
|
||||
)),
|
||||
Some(other) => Err(anyhow!(
|
||||
"CRAWLER_MODE must be one of: backfill, incremental (got {other:?})"
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
fn env_u64(name: &str, default: u64) -> u64 {
|
||||
std::env::var(name)
|
||||
.ok()
|
||||
@@ -405,3 +480,55 @@ fn env_bool(name: &str, default: bool) -> bool {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn cli_mode_defaults_to_backfill_when_unset_or_blank() {
|
||||
let none = parse_crawler_mode_str(None, 20).unwrap();
|
||||
assert!(matches!(none, DiscoverMode::Backfill));
|
||||
let blank = parse_crawler_mode_str(Some(""), 20).unwrap();
|
||||
assert!(matches!(blank, DiscoverMode::Backfill));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cli_mode_recognizes_backfill_and_incremental() {
|
||||
let backfill = parse_crawler_mode_str(Some("backfill"), 20).unwrap();
|
||||
assert!(matches!(backfill, DiscoverMode::Backfill));
|
||||
|
||||
let incremental = parse_crawler_mode_str(Some("incremental"), 9).unwrap();
|
||||
assert!(matches!(
|
||||
incremental,
|
||||
DiscoverMode::Incremental { stop_after_unchanged: 9 }
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cli_mode_rejects_auto_explicitly() {
|
||||
let err = parse_crawler_mode_str(Some("auto"), 20).unwrap_err();
|
||||
let msg = format!("{err}");
|
||||
assert!(
|
||||
msg.contains("daemon"),
|
||||
"rejection should point operator at the daemon: {msg}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cli_mode_rejects_unknown_value() {
|
||||
let err = parse_crawler_mode_str(Some("garbage"), 20).unwrap_err();
|
||||
let msg = format!("{err}");
|
||||
assert!(msg.contains("backfill"));
|
||||
assert!(msg.contains("incremental"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cli_mode_is_case_insensitive_and_trims() {
|
||||
let mixed = parse_crawler_mode_str(Some(" Incremental "), 4).unwrap();
|
||||
assert!(matches!(
|
||||
mixed,
|
||||
DiscoverMode::Incremental { stop_after_unchanged: 4 }
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -5,12 +5,24 @@ use chrono::NaiveTime;
|
||||
use chrono_tz::Tz;
|
||||
|
||||
use crate::crawler::browser::LaunchOptions;
|
||||
use crate::crawler::safety::{DownloadAllowlist, DEFAULT_MAX_IMAGE_BYTES};
|
||||
use crate::crawler::source::DiscoverMode;
|
||||
|
||||
/// What `CRAWLER_MODE` was set to. `Auto` is the daemon's default —
|
||||
/// pick Backfill until `seed_completed_at` is written, then flip to
|
||||
/// Incremental. `Explicit` forces a single mode regardless.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub enum CrawlerModePref {
|
||||
Auto,
|
||||
Explicit(DiscoverMode),
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct AuthConfig {
|
||||
pub cookie_secure: bool,
|
||||
pub cookie_domain: Option<String>,
|
||||
pub session_ttl_days: i64,
|
||||
pub rate_limit: crate::auth::rate_limit::RateLimitConfig,
|
||||
}
|
||||
|
||||
impl Default for AuthConfig {
|
||||
@@ -19,6 +31,11 @@ impl Default for AuthConfig {
|
||||
cookie_secure: true,
|
||||
cookie_domain: None,
|
||||
session_ttl_days: 30,
|
||||
// Disabled by default so the test harness inherits a
|
||||
// non-throttling limiter. Production `from_env` overrides
|
||||
// to the [`PRODUCTION_PER_SEC`]/[`PRODUCTION_BURST`]
|
||||
// defaults.
|
||||
rate_limit: crate::auth::rate_limit::RateLimitConfig::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -77,6 +94,19 @@ pub struct CrawlerConfig {
|
||||
pub user_agent: Option<String>,
|
||||
pub proxy: Option<String>,
|
||||
pub browser: LaunchOptions,
|
||||
/// Mode preference for the metadata pass. Daemon default is `Auto`
|
||||
/// (Backfill until `seed_completed_at` is written, then Incremental).
|
||||
pub mode: CrawlerModePref,
|
||||
/// `stop_after_unchanged` threshold supplied to Incremental in both
|
||||
/// `Auto` (post-seed) and `Explicit(Incremental)` modes.
|
||||
pub incremental_stop_after: usize,
|
||||
/// Hosts the crawler is allowed to download images / covers from.
|
||||
/// Always seeded with the host of `start_url` and (when set) the
|
||||
/// configured `cdn_host`. Additional hosts can be added via
|
||||
/// `CRAWLER_DOWNLOAD_ALLOWLIST` (comma-separated).
|
||||
pub download_allowlist: DownloadAllowlist,
|
||||
/// Hard upper bound on a single image download. Defaults to 32 MiB.
|
||||
pub max_image_bytes: usize,
|
||||
}
|
||||
|
||||
impl Default for CrawlerConfig {
|
||||
@@ -97,6 +127,10 @@ impl Default for CrawlerConfig {
|
||||
user_agent: None,
|
||||
proxy: None,
|
||||
browser: LaunchOptions::headless(),
|
||||
mode: CrawlerModePref::Auto,
|
||||
incremental_stop_after: 20,
|
||||
download_allowlist: DownloadAllowlist::new(),
|
||||
max_image_bytes: DEFAULT_MAX_IMAGE_BYTES,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -117,6 +151,16 @@ impl Config {
|
||||
.ok()
|
||||
.filter(|s| !s.is_empty()),
|
||||
session_ttl_days: env_i64("SESSION_TTL_DAYS", 30),
|
||||
rate_limit: crate::auth::rate_limit::RateLimitConfig {
|
||||
per_sec: env_u64(
|
||||
"AUTH_RATE_PER_SEC",
|
||||
crate::auth::rate_limit::PRODUCTION_PER_SEC.into(),
|
||||
) as u32,
|
||||
burst: env_u64(
|
||||
"AUTH_RATE_BURST",
|
||||
crate::auth::rate_limit::PRODUCTION_BURST.into(),
|
||||
) as u32,
|
||||
},
|
||||
},
|
||||
upload: UploadConfig {
|
||||
max_request_bytes: env_usize("MAX_REQUEST_BYTES", 200 * 1024 * 1024),
|
||||
@@ -151,6 +195,17 @@ impl CrawlerConfig {
|
||||
.parse()
|
||||
.map_err(|e| anyhow::anyhow!("CRAWLER_TZ must be a valid IANA TZ (got {raw:?}): {e}"))?,
|
||||
};
|
||||
let incremental_stop_after =
|
||||
env_u64("CRAWLER_INCREMENTAL_STOP_AFTER", 20).max(1) as usize;
|
||||
let mode = parse_mode_env(incremental_stop_after)?;
|
||||
let start_url = std::env::var("CRAWLER_START_URL")
|
||||
.ok()
|
||||
.filter(|s| !s.trim().is_empty());
|
||||
let cdn_host = std::env::var("CRAWLER_CDN_HOST")
|
||||
.ok()
|
||||
.filter(|s| !s.trim().is_empty());
|
||||
let download_allowlist =
|
||||
build_download_allowlist(start_url.as_deref(), cdn_host.as_deref());
|
||||
Ok(Self {
|
||||
daemon_enabled: env_bool("CRAWLER_DAEMON", true),
|
||||
daily_at,
|
||||
@@ -158,13 +213,9 @@ impl CrawlerConfig {
|
||||
idle_timeout: Duration::from_secs(env_u64("CRAWLER_IDLE_TIMEOUT_S", 600)),
|
||||
chapter_workers: env_u64("CRAWLER_CHAPTER_WORKERS", 1).max(1) as usize,
|
||||
retention_days: env_u64("CRAWLER_JOB_RETENTION_DAYS", 7) as u32,
|
||||
start_url: std::env::var("CRAWLER_START_URL")
|
||||
.ok()
|
||||
.filter(|s| !s.trim().is_empty()),
|
||||
start_url,
|
||||
rate_ms: env_u64("CRAWLER_RATE_MS", 1000),
|
||||
cdn_host: std::env::var("CRAWLER_CDN_HOST")
|
||||
.ok()
|
||||
.filter(|s| !s.trim().is_empty()),
|
||||
cdn_host,
|
||||
cdn_rate_ms: env_u64("CRAWLER_CDN_RATE_MS", env_u64("CRAWLER_RATE_MS", 1000)),
|
||||
phpsessid: std::env::var("CRAWLER_PHPSESSID")
|
||||
.ok()
|
||||
@@ -179,10 +230,73 @@ impl CrawlerConfig {
|
||||
.ok()
|
||||
.filter(|s| !s.trim().is_empty()),
|
||||
browser: LaunchOptions::from_env(),
|
||||
mode,
|
||||
incremental_stop_after,
|
||||
download_allowlist,
|
||||
max_image_bytes: env_usize("CRAWLER_MAX_IMAGE_BYTES", DEFAULT_MAX_IMAGE_BYTES),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Build the download allowlist from env. Always includes
|
||||
/// `CRAWLER_START_URL`'s host (so the crawler can fetch covers from
|
||||
/// the catalog itself) and `CRAWLER_CDN_HOST` when set. Additional
|
||||
/// hosts can be supplied via `CRAWLER_DOWNLOAD_ALLOWLIST` (comma-
|
||||
/// separated). Empty by default — meaning the crawler refuses to
|
||||
/// download anything when no source is configured, which is the safe
|
||||
/// fail-closed posture.
|
||||
fn build_download_allowlist(
|
||||
start_url: Option<&str>,
|
||||
cdn_host: Option<&str>,
|
||||
) -> DownloadAllowlist {
|
||||
let mut allow = DownloadAllowlist::new();
|
||||
if let Some(url) = start_url {
|
||||
if let Ok(parsed) = reqwest::Url::parse(url) {
|
||||
if let Some(h) = parsed.host_str() {
|
||||
allow = allow.allow(h);
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(host) = cdn_host {
|
||||
allow = allow.allow(host);
|
||||
}
|
||||
if let Ok(extras) = std::env::var("CRAWLER_DOWNLOAD_ALLOWLIST") {
|
||||
for piece in extras.split(',') {
|
||||
let trimmed = piece.trim();
|
||||
if !trimmed.is_empty() {
|
||||
allow = allow.allow(trimmed);
|
||||
}
|
||||
}
|
||||
}
|
||||
allow
|
||||
}
|
||||
|
||||
/// Parse `CRAWLER_MODE`. Empty/unset → `Auto`. Recognized values are
|
||||
/// `auto`, `backfill`, and `incremental` (case-insensitive). Anything
|
||||
/// else is a hard error so a typo can't silently fall through to the
|
||||
/// default and mask itself.
|
||||
fn parse_mode_env(incremental_stop_after: usize) -> anyhow::Result<CrawlerModePref> {
|
||||
parse_mode_str(std::env::var("CRAWLER_MODE").ok().as_deref(), incremental_stop_after)
|
||||
}
|
||||
|
||||
/// Pure variant of [`parse_mode_env`] — testable without env-var
|
||||
/// mutation. Takes the raw value (or `None` if unset).
|
||||
pub(crate) fn parse_mode_str(
|
||||
raw: Option<&str>,
|
||||
incremental_stop_after: usize,
|
||||
) -> anyhow::Result<CrawlerModePref> {
|
||||
match raw.map(|s| s.trim().to_ascii_lowercase()).as_deref() {
|
||||
None | Some("") | Some("auto") => Ok(CrawlerModePref::Auto),
|
||||
Some("backfill") => Ok(CrawlerModePref::Explicit(DiscoverMode::Backfill)),
|
||||
Some("incremental") => Ok(CrawlerModePref::Explicit(DiscoverMode::Incremental {
|
||||
stop_after_unchanged: incremental_stop_after,
|
||||
})),
|
||||
Some(other) => Err(anyhow::anyhow!(
|
||||
"CRAWLER_MODE must be one of: auto, backfill, incremental (got {other:?})"
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
fn env_u64(name: &str, default: u64) -> u64 {
|
||||
std::env::var(name)
|
||||
.ok()
|
||||
@@ -211,3 +325,63 @@ fn env_usize(name: &str, default: usize) -> usize {
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(default)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn parse_mode_str_defaults_to_auto_when_unset_or_blank() {
|
||||
let none = parse_mode_str(None, 20).unwrap();
|
||||
assert!(matches!(none, CrawlerModePref::Auto));
|
||||
let blank = parse_mode_str(Some(""), 20).unwrap();
|
||||
assert!(matches!(blank, CrawlerModePref::Auto));
|
||||
let whitespace = parse_mode_str(Some(" "), 20).unwrap();
|
||||
assert!(matches!(whitespace, CrawlerModePref::Auto));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_mode_str_recognizes_each_keyword() {
|
||||
let auto = parse_mode_str(Some("auto"), 20).unwrap();
|
||||
assert!(matches!(auto, CrawlerModePref::Auto));
|
||||
|
||||
let backfill = parse_mode_str(Some("backfill"), 20).unwrap();
|
||||
assert!(matches!(
|
||||
backfill,
|
||||
CrawlerModePref::Explicit(DiscoverMode::Backfill)
|
||||
));
|
||||
|
||||
let incremental = parse_mode_str(Some("incremental"), 7).unwrap();
|
||||
assert!(matches!(
|
||||
incremental,
|
||||
CrawlerModePref::Explicit(DiscoverMode::Incremental {
|
||||
stop_after_unchanged: 7
|
||||
})
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_mode_str_is_case_insensitive_and_trims_whitespace() {
|
||||
let mixed = parse_mode_str(Some(" Incremental "), 5).unwrap();
|
||||
assert!(matches!(
|
||||
mixed,
|
||||
CrawlerModePref::Explicit(DiscoverMode::Incremental {
|
||||
stop_after_unchanged: 5
|
||||
})
|
||||
));
|
||||
let upper = parse_mode_str(Some("BACKFILL"), 5).unwrap();
|
||||
assert!(matches!(
|
||||
upper,
|
||||
CrawlerModePref::Explicit(DiscoverMode::Backfill)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_mode_str_hard_errors_on_unknown_value() {
|
||||
let err = parse_mode_str(Some("backfil"), 20).unwrap_err();
|
||||
let msg = format!("{err}");
|
||||
assert!(msg.contains("backfill"), "error should list valid values: {msg}");
|
||||
assert!(msg.contains("auto"));
|
||||
assert!(msg.contains("incremental"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,12 +27,12 @@ use tokio::task::JoinHandle;
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum BrowserMode {
|
||||
/// Real window. Needs `$DISPLAY` (or `xvfb-run` wrapping the
|
||||
/// binary). This is the default the old Puppeteer crawler used and
|
||||
/// the assumed mode for the target site until we prove headless
|
||||
/// works against it.
|
||||
/// binary). Opt-in via `CRAWLER_BROWSER_MODE=headed` — useful for
|
||||
/// debugging a flow visually or for sites that fingerprint
|
||||
/// headless Chrome. Not used in production.
|
||||
Headed,
|
||||
/// No window. Faster, lower resource use, but more likely to trip
|
||||
/// fingerprinting on hostile sites.
|
||||
/// No window. Faster, lower resource use, runs without a display.
|
||||
/// This is the default for both `from_env()` and `Default`.
|
||||
Headless,
|
||||
}
|
||||
|
||||
@@ -65,13 +65,13 @@ impl LaunchOptions {
|
||||
}
|
||||
|
||||
/// Reads `CRAWLER_BROWSER_MODE` (`headless`|`headed`, default
|
||||
/// `headed`) and `CRAWLER_BROWSER_ARGS` (whitespace-separated
|
||||
/// `headless`) and `CRAWLER_BROWSER_ARGS` (whitespace-separated
|
||||
/// Chromium flags). Flags containing whitespace aren't supported
|
||||
/// through the env var — use the programmatic API for those.
|
||||
pub fn from_env() -> Self {
|
||||
let mode = match std::env::var("CRAWLER_BROWSER_MODE").as_deref() {
|
||||
Ok("headless") => BrowserMode::Headless,
|
||||
_ => BrowserMode::Headed,
|
||||
Ok("headed") => BrowserMode::Headed,
|
||||
_ => BrowserMode::Headless,
|
||||
};
|
||||
let extra_args = std::env::var("CRAWLER_BROWSER_ARGS")
|
||||
.map(|s| parse_args(&s))
|
||||
@@ -82,7 +82,7 @@ impl LaunchOptions {
|
||||
|
||||
impl Default for LaunchOptions {
|
||||
fn default() -> Self {
|
||||
Self::headed()
|
||||
Self::headless()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -251,4 +251,14 @@ mod tests {
|
||||
assert!(parse_args("").is_empty());
|
||||
assert!(parse_args(" \t\n").is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn default_launch_options_are_headless() {
|
||||
// Headless is the production-safe default — no display required,
|
||||
// smaller resource footprint. `Headed` stays available as an
|
||||
// opt-in for debugging via CRAWLER_BROWSER_MODE=headed.
|
||||
assert_eq!(LaunchOptions::default().mode, BrowserMode::Headless);
|
||||
assert_eq!(LaunchOptions::headless().mode, BrowserMode::Headless);
|
||||
assert_eq!(LaunchOptions::headed().mode, BrowserMode::Headed);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,15 +16,27 @@ use anyhow::Context;
|
||||
use sqlx::PgPool;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::crawler::detect::PageError;
|
||||
use crate::crawler::rate_limit::HostRateLimiters;
|
||||
use crate::crawler::session;
|
||||
use crate::crawler::safety::{fetch_bytes_capped, looks_like_image, DownloadAllowlist};
|
||||
use crate::crawler::session::{self, ChapterProbe};
|
||||
use crate::storage::Storage;
|
||||
|
||||
/// Parse the chapter page DOM and return the page images in `pageN`
|
||||
/// order. Filters out the loader `<img class="loading">` and any
|
||||
/// `<img>` without a numeric `id="pageN"`.
|
||||
pub fn parse_chapter_pages(html: &str) -> Vec<ChapterImage> {
|
||||
///
|
||||
/// Reader pages don't render the site's `#logo` element, so the
|
||||
/// universal logo-sentinel can't apply here — instead we assert
|
||||
/// `a#pic_container` is present. Its absence means the response is the
|
||||
/// transient broken-page response (or a redirect to some other layout)
|
||||
/// and the caller should retry.
|
||||
pub fn parse_chapter_pages(html: &str) -> Result<Vec<ChapterImage>, PageError> {
|
||||
let doc = scraper::Html::parse_document(html);
|
||||
let container_sel = scraper::Selector::parse("a#pic_container").unwrap();
|
||||
if doc.select(&container_sel).next().is_none() {
|
||||
return Err(PageError::transient("reader: a#pic_container missing"));
|
||||
}
|
||||
let sel = scraper::Selector::parse("a#pic_container img:not(.loading)").unwrap();
|
||||
let mut pages: Vec<ChapterImage> = doc
|
||||
.select(&sel)
|
||||
@@ -39,7 +51,7 @@ pub fn parse_chapter_pages(html: &str) -> Vec<ChapterImage> {
|
||||
})
|
||||
.collect();
|
||||
pages.sort_by_key(|p| p.page_number);
|
||||
pages
|
||||
Ok(pages)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
@@ -77,6 +89,8 @@ pub async fn sync_chapter_content(
|
||||
manga_id: Uuid,
|
||||
source_url: &str,
|
||||
force_refetch: bool,
|
||||
allowlist: &DownloadAllowlist,
|
||||
max_image_bytes: usize,
|
||||
) -> anyhow::Result<SyncOutcome> {
|
||||
// Skip if already fetched, unless caller explicitly forces.
|
||||
if !force_refetch {
|
||||
@@ -99,17 +113,30 @@ pub async fn sync_chapter_content(
|
||||
.with_context(|| format!("open chapter page {source_url}"))?;
|
||||
page.wait_for_navigation().await.context("wait for chapter nav")?;
|
||||
|
||||
// Session probe: avatar present == still logged in. Missing means
|
||||
// PHPSESSID expired; bail the entire crawler run.
|
||||
if page.find_element("#avatar_menu").await.is_err() {
|
||||
page.close().await.ok();
|
||||
return Ok(SyncOutcome::SessionExpired);
|
||||
}
|
||||
|
||||
let html = page.content().await.context("read chapter html")?;
|
||||
page.close().await.ok();
|
||||
|
||||
let images = parse_chapter_pages(&html);
|
||||
// Three-way session classification: distinguishes a transient
|
||||
// hiccup (broken-page body or logged-in-but-no-reader) from a
|
||||
// genuine PHPSESSID expiry (no reader and no avatar widget). The
|
||||
// earlier binary `#avatar_menu` check conflated both and froze
|
||||
// every worker on a layout shift.
|
||||
match session::classify_chapter_probe(&html) {
|
||||
ChapterProbe::Unauthenticated => return Ok(SyncOutcome::SessionExpired),
|
||||
ChapterProbe::Transient => {
|
||||
// Surface as a typed Err so the dispatcher path runs
|
||||
// ack_failed with exponential backoff (rather than the
|
||||
// session-expired sticky flag).
|
||||
anyhow::bail!(
|
||||
"chapter page at {source_url} returned a transient response \
|
||||
(broken-page body or reader didn't render); will retry"
|
||||
);
|
||||
}
|
||||
ChapterProbe::Ok => {}
|
||||
}
|
||||
|
||||
let images = parse_chapter_pages(&html)
|
||||
.with_context(|| format!("parse chapter pages at {source_url}"))?;
|
||||
if images.is_empty() {
|
||||
anyhow::bail!("no page images parsed from {source_url}");
|
||||
}
|
||||
@@ -126,18 +153,29 @@ pub async fn sync_chapter_content(
|
||||
format!("join image URL {} onto {source_url}", img.url)
|
||||
})?;
|
||||
rate.wait_for(url.as_str()).await?;
|
||||
let resp = http
|
||||
.get(url.clone())
|
||||
// Source CDNs commonly check Referer. Set it to the
|
||||
// chapter page — matches what the browser would send.
|
||||
.header(reqwest::header::REFERER, source_url)
|
||||
.send()
|
||||
.await
|
||||
.with_context(|| format!("GET {url}"))?
|
||||
.error_for_status()
|
||||
.with_context(|| format!("non-2xx for {url}"))?;
|
||||
let bytes = resp.bytes().await.context("read image body")?.to_vec();
|
||||
let ext = infer::get(&bytes).map(|k| k.extension()).unwrap_or("bin");
|
||||
let bytes = fetch_bytes_capped(
|
||||
http,
|
||||
url.as_str(),
|
||||
Some(source_url),
|
||||
allowlist,
|
||||
max_image_bytes,
|
||||
)
|
||||
.await?
|
||||
.to_vec();
|
||||
// Reject any non-image response: the only valid output of an
|
||||
// image URL is an image. `infer` returns None on truncated
|
||||
// bytes too, which also wants to be a failure not a silent
|
||||
// `.bin` extension.
|
||||
if !looks_like_image(&bytes) {
|
||||
anyhow::bail!(
|
||||
"image URL {url} returned non-image bytes \
|
||||
(first 16: {:?}); refusing to store as binary blob",
|
||||
&bytes.get(..16.min(bytes.len()))
|
||||
);
|
||||
}
|
||||
let ext = infer::get(&bytes)
|
||||
.map(|k| k.extension())
|
||||
.expect("looks_like_image asserted infer succeeded");
|
||||
fetched.push((img.page_number, bytes, ext));
|
||||
}
|
||||
|
||||
@@ -182,8 +220,9 @@ pub async fn sync_chapter_content(
|
||||
Ok(SyncOutcome::Fetched { pages: fetched.len() })
|
||||
}
|
||||
|
||||
// Suppress unused-import warning for `session` until the bin/crawler
|
||||
// wiring lands in this branch and uses it through this module.
|
||||
// Suppress unused-import warning for `session::registrable_domain`
|
||||
// until the bin/crawler wiring lands in this branch and uses it
|
||||
// through this module.
|
||||
#[allow(dead_code)]
|
||||
fn _keep_session_in_scope() {
|
||||
let _ = session::registrable_domain;
|
||||
@@ -205,7 +244,7 @@ mod tests {
|
||||
<img id="not-a-page" src="https://cdn/not-a-page.jpg">
|
||||
</a></body></html>
|
||||
"#;
|
||||
let pages = parse_chapter_pages(html);
|
||||
let pages = parse_chapter_pages(html).expect("parse");
|
||||
assert_eq!(pages.len(), 2);
|
||||
assert_eq!(pages[0].page_number, 1);
|
||||
assert_eq!(pages[0].url, "https://cdn/1.jpg");
|
||||
@@ -221,7 +260,7 @@ mod tests {
|
||||
<img id="page2" src="https://cdn/2.jpg">
|
||||
</a>
|
||||
"#;
|
||||
let pages = parse_chapter_pages(html);
|
||||
let pages = parse_chapter_pages(html).expect("parse");
|
||||
assert_eq!(pages.len(), 1);
|
||||
assert_eq!(pages[0].page_number, 2);
|
||||
}
|
||||
@@ -235,10 +274,22 @@ mod tests {
|
||||
<img id="page50" src="https://cdn/50.jpg">
|
||||
</a>
|
||||
"#;
|
||||
let pages = parse_chapter_pages(html);
|
||||
let pages = parse_chapter_pages(html).expect("parse");
|
||||
assert_eq!(
|
||||
pages.iter().map(|p| p.page_number).collect::<Vec<_>>(),
|
||||
vec![9, 50, 126]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_chapter_pages_returns_transient_when_container_missing() {
|
||||
// Reader doesn't render #logo, so the universal logo sentinel
|
||||
// can't be used here — a#pic_container is the reader-specific
|
||||
// marker. Broken-page response trips this.
|
||||
let html = "<html><body>\
|
||||
<p>we're sorry, the request file are not found.</p>\
|
||||
</body></html>";
|
||||
let err = parse_chapter_pages(html).expect_err("expected Transient");
|
||||
assert!(err.is_transient(), "got non-transient: {err}");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -317,14 +317,10 @@ impl WorkerContext {
|
||||
// (because a force-refetch race or a job that was re-enqueued
|
||||
// after a previous one finished), ack done without re-fetching.
|
||||
if let JobPayload::SyncChapterContent { chapter_id, .. } = &lease.payload {
|
||||
let page_count: Option<i32> = sqlx::query_scalar(
|
||||
"SELECT page_count FROM chapters WHERE id = $1",
|
||||
)
|
||||
.bind(chapter_id)
|
||||
.fetch_optional(&self.pool)
|
||||
.await
|
||||
.ok()
|
||||
.flatten();
|
||||
let page_count = crate::repo::chapter::page_count(&self.pool, *chapter_id)
|
||||
.await
|
||||
.ok()
|
||||
.flatten();
|
||||
if matches!(page_count, Some(n) if n > 0) {
|
||||
let _ = jobs::ack_done(&self.pool, lease.id).await;
|
||||
return;
|
||||
|
||||
250
backend/src/crawler/detect.rs
Normal file
250
backend/src/crawler/detect.rs
Normal file
@@ -0,0 +1,250 @@
|
||||
//! Transient-page detection.
|
||||
//!
|
||||
//! The target site occasionally responds with a 403 + tiny "we're sorry,
|
||||
//! the request file are not found" body on pages that actually exist.
|
||||
//! Selectors on that body match nothing, which is indistinguishable from
|
||||
//! a genuinely empty page unless we look for the broken-page markers
|
||||
//! explicitly. The same shape covers full-site outages: 5xx pages,
|
||||
//! Cloudflare interstitials, and "site is down" placeholders all share
|
||||
//! the trait that the normal layout (`#logo` in the header) is absent.
|
||||
//!
|
||||
//! Helpers here are split into two signals so callers can compose them:
|
||||
//! - [`is_broken_page_body`]: pattern-match on the known broken-page
|
||||
//! string. Works for *any* page on the site, including the reader,
|
||||
//! which doesn't render `#logo`.
|
||||
//! - [`has_logo_sentinel`]: assert `#logo` is in the parsed DOM. Site-
|
||||
//! structural marker — present on the manga list, manga detail,
|
||||
//! chapter-list, and login probe pages. **Not** present on the reader,
|
||||
//! so callers in the reader path must rely on the body signature only.
|
||||
//!
|
||||
//! [`PageError::Transient`] is the typed signal returned by parser and
|
||||
//! navigate wrappers. Job handlers map it to "reschedule with backoff"
|
||||
//! rather than the per-page silent skip the parsers used to do.
|
||||
|
||||
use std::future::Future;
|
||||
use std::time::Duration;
|
||||
|
||||
use thiserror::Error;
|
||||
|
||||
/// Universal substring of the broken-page body. The site renders the
|
||||
/// exact string verbatim in a single `<p>`, so a case-insensitive
|
||||
/// substring match is enough — we deliberately do *not* anchor to the
|
||||
/// kaomoji because that part is more likely to change than the prose.
|
||||
const BROKEN_PAGE_MARKER: &str = "we're sorry, the request file are not found";
|
||||
|
||||
/// Outcome of a page fetch or parse when the caller wants to
|
||||
/// distinguish "site/page is transiently broken — retry later" from
|
||||
/// other errors. `Transient` is the only retry-friendly variant; every
|
||||
/// other failure mode stays as `anyhow::Error` and is treated as today.
|
||||
#[derive(Debug, Error)]
|
||||
pub enum PageError {
|
||||
/// Page came back but the site signaled trouble — broken-page body
|
||||
/// signature, structural sentinel missing, etc. Caller should
|
||||
/// reschedule this fetch rather than treat it as data.
|
||||
#[error("transient page error: {reason}")]
|
||||
Transient { reason: String },
|
||||
#[error(transparent)]
|
||||
Other(#[from] anyhow::Error),
|
||||
}
|
||||
|
||||
impl PageError {
|
||||
pub fn transient(reason: impl Into<String>) -> Self {
|
||||
Self::Transient { reason: reason.into() }
|
||||
}
|
||||
|
||||
pub fn is_transient(&self) -> bool {
|
||||
matches!(self, Self::Transient { .. })
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns true when the response body matches the known broken-page
|
||||
/// template. Case-insensitive substring match — small bodies (~150B)
|
||||
/// make the scan trivially fast, and the broken page is always tiny so
|
||||
/// false positives on a real catalog page are not a concern.
|
||||
pub fn is_broken_page_body(html: &str) -> bool {
|
||||
html.to_ascii_lowercase().contains(BROKEN_PAGE_MARKER)
|
||||
}
|
||||
|
||||
/// Returns true when the parsed document contains `#logo` — the site's
|
||||
/// header logo element, present on every full-layout page and absent on
|
||||
/// the broken-page response and on the reader.
|
||||
pub fn has_logo_sentinel(doc: &scraper::Html) -> bool {
|
||||
let sel = scraper::Selector::parse("#logo").expect("#logo is a valid selector");
|
||||
doc.select(&sel).next().is_some()
|
||||
}
|
||||
|
||||
/// Retry `op` up to `max_attempts` times whenever it returns
|
||||
/// [`PageError::Transient`], sleeping `delay` between attempts.
|
||||
/// Non-transient errors short-circuit immediately. Used by discover-loop
|
||||
/// callers so a single broken page doesn't drop the whole walk — the
|
||||
/// caller can fall back on the job system's retry/backoff once the
|
||||
/// inline budget is exhausted.
|
||||
pub async fn retry_on_transient<F, Fut, T>(
|
||||
mut op: F,
|
||||
max_attempts: u32,
|
||||
delay: Duration,
|
||||
) -> Result<T, PageError>
|
||||
where
|
||||
F: FnMut() -> Fut,
|
||||
Fut: Future<Output = Result<T, PageError>>,
|
||||
{
|
||||
debug_assert!(max_attempts >= 1, "max_attempts must be at least 1");
|
||||
let mut attempt = 0u32;
|
||||
loop {
|
||||
attempt += 1;
|
||||
match op().await {
|
||||
Ok(v) => return Ok(v),
|
||||
Err(e) if !e.is_transient() => return Err(e),
|
||||
Err(e) if attempt >= max_attempts => return Err(e),
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
attempt,
|
||||
max_attempts,
|
||||
error = %e,
|
||||
"transient error; sleeping before retry"
|
||||
);
|
||||
tokio::time::sleep(delay).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn broken_page_body_matches_exact_template() {
|
||||
let html = "<html><head></head><body>\
|
||||
<p>we're sorry, the request file are not found. Σ(っ°Д °;)っ</p>\
|
||||
</body></html>";
|
||||
assert!(is_broken_page_body(html));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn broken_page_body_is_case_insensitive() {
|
||||
let html = "<p>WE'RE SORRY, THE REQUEST FILE ARE NOT FOUND.</p>";
|
||||
assert!(is_broken_page_body(html));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn broken_page_body_does_not_match_normal_listing() {
|
||||
let html = "<html><body><div id='logo'></div>\
|
||||
<ul><li>Manga A</li><li>Manga B</li></ul></body></html>";
|
||||
assert!(!is_broken_page_body(html));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn broken_page_body_does_not_match_empty_string() {
|
||||
assert!(!is_broken_page_body(""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn logo_sentinel_present_on_normal_page() {
|
||||
let doc = scraper::Html::parse_document(
|
||||
"<html><body><div id='logo'>Site</div><main>...</main></body></html>",
|
||||
);
|
||||
assert!(has_logo_sentinel(&doc));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn logo_sentinel_absent_on_broken_page() {
|
||||
let doc = scraper::Html::parse_document(
|
||||
"<html><head></head><body>\
|
||||
<p>we're sorry, the request file are not found.</p></body></html>",
|
||||
);
|
||||
assert!(!has_logo_sentinel(&doc));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn logo_sentinel_absent_on_empty_document() {
|
||||
let doc = scraper::Html::parse_document("");
|
||||
assert!(!has_logo_sentinel(&doc));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn page_error_transient_constructor_sets_reason() {
|
||||
let e = PageError::transient("logo missing");
|
||||
assert!(e.is_transient());
|
||||
assert_eq!(e.to_string(), "transient page error: logo missing");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn page_error_other_is_not_transient() {
|
||||
let e: PageError = anyhow::anyhow!("something else").into();
|
||||
assert!(!e.is_transient());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn retry_returns_ok_after_a_transient_streak() {
|
||||
let mut attempt = 0u32;
|
||||
let result: Result<i32, PageError> = retry_on_transient(
|
||||
|| {
|
||||
attempt += 1;
|
||||
let n = attempt;
|
||||
async move {
|
||||
if n < 3 {
|
||||
Err(PageError::transient("not yet"))
|
||||
} else {
|
||||
Ok(42)
|
||||
}
|
||||
}
|
||||
},
|
||||
5,
|
||||
Duration::from_millis(0),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(result.unwrap(), 42);
|
||||
assert_eq!(attempt, 3);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn retry_gives_up_after_max_attempts_on_persistent_transient() {
|
||||
let mut attempt = 0u32;
|
||||
let result: Result<i32, PageError> = retry_on_transient(
|
||||
|| {
|
||||
attempt += 1;
|
||||
async { Err(PageError::transient("always")) }
|
||||
},
|
||||
3,
|
||||
Duration::from_millis(0),
|
||||
)
|
||||
.await;
|
||||
let err = result.expect_err("expected Transient");
|
||||
assert!(err.is_transient());
|
||||
assert_eq!(attempt, 3, "retried max_attempts times, no more");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn retry_does_not_retry_non_transient_errors() {
|
||||
let mut attempt = 0u32;
|
||||
let result: Result<i32, PageError> = retry_on_transient(
|
||||
|| {
|
||||
attempt += 1;
|
||||
async { Err(PageError::Other(anyhow::anyhow!("permanent"))) }
|
||||
},
|
||||
5,
|
||||
Duration::from_millis(0),
|
||||
)
|
||||
.await;
|
||||
assert!(result.is_err());
|
||||
assert!(!result.unwrap_err().is_transient());
|
||||
assert_eq!(attempt, 1, "non-transient must fail immediately");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn retry_returns_ok_on_first_attempt_without_sleeping() {
|
||||
let mut attempt = 0u32;
|
||||
let result: Result<i32, PageError> = retry_on_transient(
|
||||
|| {
|
||||
attempt += 1;
|
||||
async { Ok(7) }
|
||||
},
|
||||
5,
|
||||
Duration::from_secs(60),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(result.unwrap(), 7);
|
||||
assert_eq!(attempt, 1);
|
||||
}
|
||||
}
|
||||
@@ -17,9 +17,12 @@ pub mod browser;
|
||||
pub mod browser_manager;
|
||||
pub mod content;
|
||||
pub mod daemon;
|
||||
pub mod detect;
|
||||
pub mod diff;
|
||||
pub mod jobs;
|
||||
pub mod pipeline;
|
||||
pub mod rate_limit;
|
||||
pub mod safety;
|
||||
pub mod session;
|
||||
pub mod source;
|
||||
pub mod url_utils;
|
||||
|
||||
@@ -9,6 +9,7 @@ use uuid::Uuid;
|
||||
use crate::crawler::browser_manager::BrowserManager;
|
||||
use crate::crawler::jobs::{self, EnqueueResult, JobPayload};
|
||||
use crate::crawler::rate_limit::HostRateLimiters;
|
||||
use crate::crawler::safety::{fetch_bytes_capped, looks_like_image, DownloadAllowlist};
|
||||
use crate::crawler::source::target::TargetSource;
|
||||
use crate::crawler::source::{DiscoverMode, FetchContext, Source};
|
||||
use crate::repo;
|
||||
@@ -23,14 +24,34 @@ pub struct MetadataStats {
|
||||
pub mangas_failed: usize,
|
||||
}
|
||||
|
||||
/// Decide whether the per-ref loop should stop based on the Incremental
|
||||
/// streak counter. Pulled out as a pure function so the rule is unit-
|
||||
/// testable without standing up the walker or DB.
|
||||
pub(crate) fn should_stop(mode: DiscoverMode, consecutive_unchanged: usize) -> bool {
|
||||
match mode {
|
||||
DiscoverMode::Backfill => false,
|
||||
DiscoverMode::Incremental { stop_after_unchanged } => {
|
||||
consecutive_unchanged >= stop_after_unchanged
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Runs the discover → fetch → upsert → cover → chapter-list-diff pipeline
|
||||
/// for the target source. Pure metadata; chapter content is enqueued as
|
||||
/// separate `SyncChapterContent` jobs by the caller after this returns.
|
||||
///
|
||||
/// `limit == 0` means no cap (full backfill). `skip_chapters == true` is
|
||||
/// the "metadata-only" mode (parser doesn't extract chapters, and
|
||||
/// `sync_manga_chapters` is skipped — otherwise an empty chapter list
|
||||
/// would soft-drop existing rows).
|
||||
/// `limit == 0` means no cap (full sweep up to the source's own bound).
|
||||
/// `skip_chapters == true` is the "metadata-only" mode (parser doesn't
|
||||
/// extract chapters, and `sync_manga_chapters` is skipped — otherwise an
|
||||
/// empty chapter list would soft-drop existing rows).
|
||||
///
|
||||
/// `mode` controls the walk:
|
||||
/// - `Backfill` — oldest-first, no early exit. The only mode that runs
|
||||
/// the end-of-walk drop pass + writes `seed_completed_at`.
|
||||
/// - `Incremental { stop_after_unchanged }` — newest-first, breaks out
|
||||
/// after N consecutive Unchanged upserts. Drop pass is skipped (the
|
||||
/// tail of the index is never visited, so its `last_seen_at` is
|
||||
/// stale and using it to soft-drop would be unsafe).
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub async fn run_metadata_pass(
|
||||
browser_manager: &BrowserManager,
|
||||
@@ -41,6 +62,9 @@ pub async fn run_metadata_pass(
|
||||
start_url: &str,
|
||||
limit: usize,
|
||||
skip_chapters: bool,
|
||||
mode: DiscoverMode,
|
||||
allowlist: &DownloadAllowlist,
|
||||
max_image_bytes: usize,
|
||||
) -> anyhow::Result<MetadataStats> {
|
||||
let lease = browser_manager
|
||||
.acquire()
|
||||
@@ -74,123 +98,191 @@ pub async fn run_metadata_pass(
|
||||
let run_started_at = chrono::Utc::now();
|
||||
let max_refs = (limit > 0).then_some(limit);
|
||||
|
||||
tracing::info!(?max_refs, "discovering manga list");
|
||||
let refs = source
|
||||
.discover(&ctx, DiscoverMode::Backfill, max_refs)
|
||||
tracing::info!(?mode, ?max_refs, "starting metadata pass");
|
||||
let mut walker = source
|
||||
.discover(&ctx, mode)
|
||||
.await
|
||||
.context("discover failed")?;
|
||||
tracing::info!(count = refs.len(), "discovered manga list");
|
||||
|
||||
let mut stats = MetadataStats {
|
||||
discovered: refs.len(),
|
||||
..MetadataStats::default()
|
||||
};
|
||||
let mut stats = MetadataStats::default();
|
||||
let mut consecutive_unchanged: usize = 0;
|
||||
let mut walked_to_completion = false;
|
||||
let mut hit_limit = false;
|
||||
let mut hit_incremental_stop = false;
|
||||
|
||||
for (i, r) in refs.iter().enumerate() {
|
||||
tracing::info!(
|
||||
idx = i + 1,
|
||||
total = stats.discovered,
|
||||
key = %r.source_manga_key,
|
||||
"fetching metadata"
|
||||
);
|
||||
let manga = match source.fetch_manga(&ctx, r).await {
|
||||
Ok(m) => m,
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
key = %r.source_manga_key,
|
||||
url = %r.url,
|
||||
error = ?e,
|
||||
"fetch_manga failed"
|
||||
);
|
||||
stats.mangas_failed += 1;
|
||||
continue;
|
||||
'outer: loop {
|
||||
let batch = match walker.next_batch(&ctx).await? {
|
||||
Some(b) => b,
|
||||
None => {
|
||||
walked_to_completion = true;
|
||||
break;
|
||||
}
|
||||
};
|
||||
|
||||
let upsert = match repo::crawler::upsert_manga_from_source(db, source_id, &r.url, &manga)
|
||||
.await
|
||||
{
|
||||
Ok(u) => u,
|
||||
Err(e) => {
|
||||
tracing::error!(
|
||||
key = %r.source_manga_key,
|
||||
error = ?e,
|
||||
"upsert_manga_from_source failed"
|
||||
);
|
||||
stats.mangas_failed += 1;
|
||||
continue;
|
||||
for r in batch {
|
||||
if max_refs.map(|m| stats.discovered >= m).unwrap_or(false) {
|
||||
hit_limit = true;
|
||||
tracing::info!(cap = ?max_refs, "max_results reached; halting walk");
|
||||
break 'outer;
|
||||
}
|
||||
};
|
||||
stats.upserted += 1;
|
||||
tracing::info!(
|
||||
key = %manga.source_manga_key,
|
||||
manga_id = %upsert.manga_id,
|
||||
status = ?upsert.status,
|
||||
title = %manga.title,
|
||||
"manga upserted"
|
||||
);
|
||||
|
||||
// Cover image: download when missing in storage or when metadata
|
||||
// signaled an update (cover URL is part of metadata_hash, so
|
||||
// Updated implies the URL may have moved). Failures are non-fatal.
|
||||
let needs_cover = upsert.cover_image_path.is_none()
|
||||
|| matches!(upsert.status, repo::crawler::UpsertStatus::Updated);
|
||||
if needs_cover {
|
||||
if let Some(cover_url) = manga.cover_url.as_deref() {
|
||||
match download_and_store_cover(
|
||||
db,
|
||||
storage,
|
||||
http,
|
||||
rate,
|
||||
&r.url,
|
||||
upsert.manga_id,
|
||||
cover_url,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(()) => stats.covers_fetched += 1,
|
||||
Err(e) => tracing::warn!(
|
||||
manga_id = %upsert.manga_id,
|
||||
stats.discovered += 1;
|
||||
tracing::info!(
|
||||
idx = stats.discovered,
|
||||
key = %r.source_manga_key,
|
||||
"fetching metadata"
|
||||
);
|
||||
let manga = match source.fetch_manga(&ctx, &r).await {
|
||||
Ok(m) => m,
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
key = %r.source_manga_key,
|
||||
url = %r.url,
|
||||
error = ?e,
|
||||
"cover download failed"
|
||||
),
|
||||
"fetch_manga failed"
|
||||
);
|
||||
stats.mangas_failed += 1;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
if !skip_chapters {
|
||||
match repo::crawler::sync_manga_chapters(
|
||||
db,
|
||||
source_id,
|
||||
upsert.manga_id,
|
||||
&manga.chapters,
|
||||
let upsert = match repo::crawler::upsert_manga_from_source(
|
||||
db, source_id, &r.url, &manga,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(diff) => tracing::info!(
|
||||
manga_id = %upsert.manga_id,
|
||||
new = diff.new,
|
||||
refreshed = diff.refreshed,
|
||||
dropped = diff.dropped,
|
||||
"chapters synced"
|
||||
),
|
||||
Err(e) => tracing::warn!(
|
||||
manga_id = %upsert.manga_id,
|
||||
error = ?e,
|
||||
"chapter sync failed"
|
||||
),
|
||||
Ok(u) => u,
|
||||
Err(e) => {
|
||||
tracing::error!(
|
||||
key = %r.source_manga_key,
|
||||
error = ?e,
|
||||
"upsert_manga_from_source failed"
|
||||
);
|
||||
stats.mangas_failed += 1;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
stats.upserted += 1;
|
||||
tracing::info!(
|
||||
key = %manga.source_manga_key,
|
||||
manga_id = %upsert.manga_id,
|
||||
status = ?upsert.status,
|
||||
title = %manga.title,
|
||||
"manga upserted"
|
||||
);
|
||||
|
||||
// Cover image: download when missing in storage or when metadata
|
||||
// signaled an update (cover URL is part of metadata_hash, so
|
||||
// Updated implies the URL may have moved). Failures are non-fatal.
|
||||
let needs_cover = upsert.cover_image_path.is_none()
|
||||
|| matches!(upsert.status, repo::crawler::UpsertStatus::Updated);
|
||||
if needs_cover {
|
||||
if let Some(cover_url) = manga.cover_url.as_deref() {
|
||||
match download_and_store_cover(
|
||||
db,
|
||||
storage,
|
||||
http,
|
||||
rate,
|
||||
&r.url,
|
||||
upsert.manga_id,
|
||||
cover_url,
|
||||
allowlist,
|
||||
max_image_bytes,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(()) => stats.covers_fetched += 1,
|
||||
Err(e) => tracing::warn!(
|
||||
manga_id = %upsert.manga_id,
|
||||
error = ?e,
|
||||
"cover download failed"
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !skip_chapters {
|
||||
match repo::crawler::sync_manga_chapters(
|
||||
db,
|
||||
source_id,
|
||||
upsert.manga_id,
|
||||
&manga.chapters,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(diff) => tracing::info!(
|
||||
manga_id = %upsert.manga_id,
|
||||
new = diff.new,
|
||||
refreshed = diff.refreshed,
|
||||
dropped = diff.dropped,
|
||||
"chapters synced"
|
||||
),
|
||||
Err(e) => tracing::warn!(
|
||||
manga_id = %upsert.manga_id,
|
||||
error = ?e,
|
||||
"chapter sync failed"
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
// Incremental stop: count consecutive Unchanged upserts and
|
||||
// bail once the threshold is reached. New/Updated resets the
|
||||
// streak so a fresh entry mid-batch doesn't accidentally trip
|
||||
// the cutoff.
|
||||
match upsert.status {
|
||||
repo::crawler::UpsertStatus::Unchanged => {
|
||||
consecutive_unchanged += 1;
|
||||
}
|
||||
repo::crawler::UpsertStatus::New | repo::crawler::UpsertStatus::Updated => {
|
||||
consecutive_unchanged = 0;
|
||||
}
|
||||
}
|
||||
if should_stop(mode, consecutive_unchanged) {
|
||||
hit_incremental_stop = true;
|
||||
tracing::info!(
|
||||
consecutive_unchanged,
|
||||
"incremental stop threshold reached; halting walk"
|
||||
);
|
||||
break 'outer;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if limit == 0 {
|
||||
// Drop pass: only when the walk truly covered everything the source
|
||||
// surfaces. `last_seen_at` on un-visited rows is stale, so running
|
||||
// the drop on a partial walk would soft-drop the tail of the index.
|
||||
let full_walk = walked_to_completion && !hit_limit && !hit_incremental_stop;
|
||||
let backfill_complete = full_walk && matches!(mode, DiscoverMode::Backfill);
|
||||
if full_walk {
|
||||
match repo::crawler::mark_dropped_mangas(db, source_id, run_started_at).await {
|
||||
Ok(n) => tracing::info!(dropped = n, "marked unseen manga as dropped"),
|
||||
Err(e) => tracing::warn!(error = ?e, "drop-pass failed"),
|
||||
}
|
||||
} else {
|
||||
tracing::info!(limit, "partial sync — skipping drop pass");
|
||||
tracing::info!(
|
||||
?mode,
|
||||
hit_limit,
|
||||
hit_incremental_stop,
|
||||
"partial sync — skipping drop pass"
|
||||
);
|
||||
}
|
||||
if backfill_complete {
|
||||
if let Err(e) = repo::crawler::mark_seed_completed(db, source_id, run_started_at).await {
|
||||
tracing::warn!(error = ?e, "mark_seed_completed failed");
|
||||
} else {
|
||||
tracing::info!(source_id, "seed marked complete");
|
||||
}
|
||||
}
|
||||
|
||||
tracing::info!(
|
||||
?mode,
|
||||
discovered = stats.discovered,
|
||||
upserted = stats.upserted,
|
||||
covers_fetched = stats.covers_fetched,
|
||||
mangas_failed = stats.mangas_failed,
|
||||
walked_to_completion,
|
||||
hit_limit,
|
||||
hit_incremental_stop,
|
||||
"metadata pass complete"
|
||||
);
|
||||
|
||||
drop(lease);
|
||||
Ok(stats)
|
||||
@@ -295,6 +387,7 @@ pub struct EnqueueSummary {
|
||||
/// pipeline because the CLI still calls it from its inline chapter-content
|
||||
/// loop; once the worker pool fully replaces that path we can fold this
|
||||
/// into `pipeline` proper.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn download_and_store_cover(
|
||||
db: &PgPool,
|
||||
storage: &dyn Storage,
|
||||
@@ -303,6 +396,8 @@ async fn download_and_store_cover(
|
||||
manga_url: &str,
|
||||
manga_id: Uuid,
|
||||
cover_url: &str,
|
||||
allowlist: &DownloadAllowlist,
|
||||
max_image_bytes: usize,
|
||||
) -> anyhow::Result<()> {
|
||||
let absolute = reqwest::Url::parse(manga_url)
|
||||
.context("parse manga URL")?
|
||||
@@ -310,17 +405,22 @@ async fn download_and_store_cover(
|
||||
.context("join cover URL onto manga URL")?;
|
||||
|
||||
rate.wait_for(absolute.as_str()).await?;
|
||||
let resp = http
|
||||
.get(absolute.clone())
|
||||
.header(reqwest::header::REFERER, manga_url)
|
||||
.send()
|
||||
.await
|
||||
.with_context(|| format!("GET {absolute}"))?
|
||||
.error_for_status()
|
||||
.with_context(|| format!("non-2xx for {absolute}"))?;
|
||||
let bytes = resp.bytes().await.context("read cover body")?;
|
||||
let kind = infer::get(&bytes);
|
||||
let ext = kind.map(|k| k.extension()).unwrap_or("bin");
|
||||
let bytes = fetch_bytes_capped(
|
||||
http,
|
||||
absolute.as_str(),
|
||||
Some(manga_url),
|
||||
allowlist,
|
||||
max_image_bytes,
|
||||
)
|
||||
.await?;
|
||||
if !looks_like_image(&bytes) {
|
||||
anyhow::bail!(
|
||||
"cover URL {absolute} returned non-image bytes; refusing to store as binary blob"
|
||||
);
|
||||
}
|
||||
let ext = infer::get(&bytes)
|
||||
.map(|k| k.extension())
|
||||
.expect("looks_like_image asserted infer succeeded");
|
||||
let key = format!("mangas/{manga_id}/cover.{ext}");
|
||||
|
||||
storage
|
||||
@@ -340,8 +440,37 @@ async fn download_and_store_cover(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn origin_of(url: &str) -> Option<String> {
|
||||
let (scheme, rest) = url.split_once("://")?;
|
||||
let host = rest.split('/').next()?;
|
||||
Some(format!("{scheme}://{host}"))
|
||||
use crate::crawler::url_utils::origin_of;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn backfill_never_stops_regardless_of_streak() {
|
||||
assert!(!should_stop(DiscoverMode::Backfill, 0));
|
||||
assert!(!should_stop(DiscoverMode::Backfill, 100));
|
||||
assert!(!should_stop(DiscoverMode::Backfill, usize::MAX));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn incremental_stops_when_streak_meets_threshold() {
|
||||
let mode = DiscoverMode::Incremental {
|
||||
stop_after_unchanged: 3,
|
||||
};
|
||||
assert!(!should_stop(mode, 0));
|
||||
assert!(!should_stop(mode, 2));
|
||||
assert!(should_stop(mode, 3), "stops at exactly the threshold");
|
||||
assert!(should_stop(mode, 100), "stops at anything past threshold");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn incremental_with_zero_threshold_stops_immediately() {
|
||||
// A nonsensical config (no Unchanged needed to stop) shouldn't
|
||||
// panic — it just means the very first ref triggers the bail.
|
||||
let mode = DiscoverMode::Incremental {
|
||||
stop_after_unchanged: 0,
|
||||
};
|
||||
assert!(should_stop(mode, 0));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -98,15 +98,9 @@ impl HostRateLimiters {
|
||||
}
|
||||
}
|
||||
|
||||
/// Extract the host (no port) from a URL string. Returns `None` for
|
||||
/// inputs without a `scheme://host` shape — those would never have
|
||||
/// reached the network layer anyway.
|
||||
fn host_of(url: &str) -> Option<String> {
|
||||
let after_scheme = url.split_once("://")?.1;
|
||||
let host_with_port = after_scheme.split('/').next()?;
|
||||
let host = host_with_port.rsplit_once(':').map_or(host_with_port, |(h, _)| h);
|
||||
(!host.is_empty()).then(|| host.to_ascii_lowercase())
|
||||
}
|
||||
// `host_of` was duplicated across session/rate_limit/pipeline; the
|
||||
// canonical version now lives in `crawler::url_utils`.
|
||||
use crate::crawler::url_utils::host_of;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
|
||||
486
backend/src/crawler/safety.rs
Normal file
486
backend/src/crawler/safety.rs
Normal file
@@ -0,0 +1,486 @@
|
||||
//! Defensive helpers for the image-download paths.
|
||||
//!
|
||||
//! Two threats this module addresses:
|
||||
//!
|
||||
//! - **SSRF**: a scraped chapter or manga page can embed an absolute
|
||||
//! `<img src="http://10.0.0.1/...">`. The crawler runs inside the
|
||||
//! backend container with intra-compose access to `postgres:5432`
|
||||
//! and possibly other internal services; without a host check the
|
||||
//! crawler would happily probe them. [`is_safe_url`] rejects
|
||||
//! anything whose host isn't on the operator-configured allowlist,
|
||||
//! plus any IP literal in RFC1918 / loopback / link-local / unique-
|
||||
//! local space (including IPv4-mapped IPv6 like `::ffff:127.0.0.1`)
|
||||
//! as a second defence for the case where an allowlisted hostname's
|
||||
//! DNS happens to resolve to a literal private address.
|
||||
//!
|
||||
//! **DNS rebinding is not covered.** A hostname like `cdn.allowed.com`
|
||||
//! that *resolves* to `127.0.0.1` via hostile DNS bypasses the IP
|
||||
//! check entirely — `is_safe_url` only inspects URL strings, not
|
||||
//! resolved IPs. Mitigating that requires a custom reqwest resolver
|
||||
//! that filters IPs after DNS, which would mean rebuilding reqwest's
|
||||
//! connector. The allowlist + good operator DNS hygiene is the
|
||||
//! realistic mitigation today.
|
||||
//!
|
||||
//! - **Unbounded download**: `Response::bytes().await` reads the full
|
||||
//! body before returning. A malicious source serving a 10 GiB image
|
||||
//! would fill memory and then disk. [`accumulate_capped`] streams
|
||||
//! the body chunk-by-chunk into a [`bytes::BytesMut`] and bails as
|
||||
//! soon as the running total exceeds the cap.
|
||||
//!
|
||||
//! Both helpers are pure-data: the SSRF check is keyed off a parsed
|
||||
//! URL string, and the byte accumulator is keyed off a generic stream.
|
||||
//! Easy to unit-test without a live network or browser.
|
||||
|
||||
use std::net::IpAddr;
|
||||
|
||||
use anyhow::{bail, Context};
|
||||
use bytes::BytesMut;
|
||||
use futures_util::StreamExt;
|
||||
use reqwest::Url;
|
||||
|
||||
/// Default per-image download cap. A page image is generally <2 MiB;
|
||||
/// 32 MiB leaves headroom for high-resolution covers while still
|
||||
/// stopping a misbehaving CDN dead. Override via `CRAWLER_MAX_IMAGE_BYTES`.
|
||||
pub const DEFAULT_MAX_IMAGE_BYTES: usize = 32 * 1024 * 1024;
|
||||
|
||||
/// Hosts that are always allowed in addition to the operator's
|
||||
/// configured allowlist. None by default — keeping the surface area
|
||||
/// minimal so the only way a URL gets through is if it matches an
|
||||
/// explicit catalog/CDN entry.
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub struct DownloadAllowlist {
|
||||
hosts: Vec<String>,
|
||||
}
|
||||
|
||||
impl DownloadAllowlist {
|
||||
pub fn new() -> Self {
|
||||
Self { hosts: Vec::new() }
|
||||
}
|
||||
|
||||
/// Add a host (case-insensitive match). Sub-domains are *not*
|
||||
/// implied: pass `cdn.example.com` and `example.com` separately
|
||||
/// if both should be reachable.
|
||||
pub fn allow(mut self, host: impl Into<String>) -> Self {
|
||||
let h = host.into().to_ascii_lowercase();
|
||||
if !h.is_empty() && !self.hosts.iter().any(|existing| existing == &h) {
|
||||
self.hosts.push(h);
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.hosts.is_empty()
|
||||
}
|
||||
|
||||
pub fn contains(&self, host: &str) -> bool {
|
||||
let lower = host.to_ascii_lowercase();
|
||||
self.hosts.iter().any(|h| h == &lower)
|
||||
}
|
||||
}
|
||||
|
||||
/// Verify a URL is safe for the crawler to fetch.
|
||||
///
|
||||
/// Rejects:
|
||||
/// - non-http(s) schemes (file://, gopher://, …),
|
||||
/// - any IP literal in private / loopback / link-local / unique-local
|
||||
/// space (defense in depth — a DNS allowlist alone wouldn't cover an
|
||||
/// attacker that places an entry like `cdn.evil` pointing at
|
||||
/// `192.168.1.1`),
|
||||
/// - the literal hostname `localhost`,
|
||||
/// - hosts that aren't on the supplied allowlist.
|
||||
///
|
||||
/// An empty allowlist rejects everything (the conservative default —
|
||||
/// callers must explicitly allow the catalog and CDN hosts).
|
||||
pub fn is_safe_url(raw_url: &str, allow: &DownloadAllowlist) -> Result<(), UrlSafetyError> {
|
||||
let url = Url::parse(raw_url).map_err(|_| UrlSafetyError::Unparseable)?;
|
||||
let scheme = url.scheme();
|
||||
if scheme != "http" && scheme != "https" {
|
||||
return Err(UrlSafetyError::BadScheme(scheme.to_string()));
|
||||
}
|
||||
let host = url.host_str().ok_or(UrlSafetyError::NoHost)?;
|
||||
let lower_host = host.to_ascii_lowercase();
|
||||
if lower_host == "localhost" {
|
||||
return Err(UrlSafetyError::Loopback);
|
||||
}
|
||||
// Reject IP literals in private/loopback ranges regardless of the
|
||||
// allowlist — if someone puts an IP literal on the allowlist they
|
||||
// almost certainly didn't mean a private range.
|
||||
// reqwest::Url normalises IPv6 literals as `[::1]` (brackets
|
||||
// included) in `host_str()`. Strip the brackets before parsing.
|
||||
let ip_candidate = lower_host
|
||||
.strip_prefix('[')
|
||||
.and_then(|s| s.strip_suffix(']'))
|
||||
.unwrap_or(&lower_host);
|
||||
if let Ok(ip) = ip_candidate.parse::<IpAddr>() {
|
||||
if is_private_ip(&ip) {
|
||||
return Err(UrlSafetyError::PrivateIp(ip));
|
||||
}
|
||||
}
|
||||
if !allow.contains(&lower_host) {
|
||||
return Err(UrlSafetyError::HostNotAllowed(lower_host));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn is_private_ip(ip: &IpAddr) -> bool {
|
||||
match ip {
|
||||
IpAddr::V4(v4) => {
|
||||
v4.is_loopback()
|
||||
|| v4.is_private()
|
||||
|| v4.is_link_local()
|
||||
|| v4.is_unspecified()
|
||||
|| v4.is_broadcast()
|
||||
// CGNAT 100.64.0.0/10
|
||||
|| (v4.octets()[0] == 100 && (v4.octets()[1] & 0xC0) == 64)
|
||||
// 169.254/16 link-local already covered, but 0.0.0.0/8 is special-use
|
||||
|| v4.octets()[0] == 0
|
||||
}
|
||||
IpAddr::V6(v6) => {
|
||||
// IPv4-mapped IPv6 (::ffff:0:0/96): unwrap to the embedded
|
||||
// IPv4 and recurse so `::ffff:127.0.0.1` is caught by the
|
||||
// IPv4 loopback check rather than passing through.
|
||||
// `Ipv6Addr::is_loopback()` only matches `::1` exactly.
|
||||
if let Some(v4) = v6.to_ipv4_mapped() {
|
||||
return is_private_ip(&IpAddr::V4(v4));
|
||||
}
|
||||
v6.is_loopback()
|
||||
|| v6.is_unspecified()
|
||||
// fc00::/7 unique-local
|
||||
|| (v6.segments()[0] & 0xfe00) == 0xfc00
|
||||
// fe80::/10 link-local
|
||||
|| (v6.segments()[0] & 0xffc0) == 0xfe80
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error, PartialEq, Eq)]
|
||||
pub enum UrlSafetyError {
|
||||
#[error("URL is not parseable")]
|
||||
Unparseable,
|
||||
#[error("scheme {0:?} is not http or https")]
|
||||
BadScheme(String),
|
||||
#[error("URL is missing a host")]
|
||||
NoHost,
|
||||
#[error("host points at the loopback interface")]
|
||||
Loopback,
|
||||
#[error("host is a private/internal IP: {0}")]
|
||||
PrivateIp(IpAddr),
|
||||
#[error("host {0:?} is not on the crawler download allowlist")]
|
||||
HostNotAllowed(String),
|
||||
}
|
||||
|
||||
/// Drain a byte stream into a single buffer, bailing out as soon as
|
||||
/// the running total exceeds `max_bytes`. Generic over the stream so
|
||||
/// it's testable without a live HTTP response.
|
||||
pub async fn accumulate_capped<S, E>(stream: S, max_bytes: usize) -> anyhow::Result<bytes::Bytes>
|
||||
where
|
||||
S: futures_core::Stream<Item = Result<bytes::Bytes, E>>,
|
||||
E: std::error::Error + Send + Sync + 'static,
|
||||
{
|
||||
let mut buf = BytesMut::new();
|
||||
let mut stream = std::pin::pin!(stream);
|
||||
while let Some(chunk) = stream.next().await {
|
||||
let chunk = chunk.map_err(|e| anyhow::anyhow!("stream chunk: {e}"))?;
|
||||
if buf.len().saturating_add(chunk.len()) > max_bytes {
|
||||
bail!(
|
||||
"response exceeds {max_bytes}-byte cap (received >{}+{})",
|
||||
buf.len(),
|
||||
chunk.len()
|
||||
);
|
||||
}
|
||||
buf.extend_from_slice(&chunk);
|
||||
}
|
||||
Ok(buf.freeze())
|
||||
}
|
||||
|
||||
/// Send `req` and stream the response into a length-limited buffer.
|
||||
/// Combines [`is_safe_url`] check + [`accumulate_capped`] so each
|
||||
/// call-site is one line.
|
||||
pub async fn fetch_bytes_capped(
|
||||
http: &reqwest::Client,
|
||||
url: &str,
|
||||
referer: Option<&str>,
|
||||
allow: &DownloadAllowlist,
|
||||
max_bytes: usize,
|
||||
) -> anyhow::Result<bytes::Bytes> {
|
||||
is_safe_url(url, allow).with_context(|| format!("reject unsafe URL {url}"))?;
|
||||
let mut req = http.get(url);
|
||||
if let Some(r) = referer {
|
||||
req = req.header(reqwest::header::REFERER, r);
|
||||
}
|
||||
let resp = req
|
||||
.send()
|
||||
.await
|
||||
.with_context(|| format!("GET {url}"))?
|
||||
.error_for_status()
|
||||
.with_context(|| format!("non-2xx for {url}"))?;
|
||||
accumulate_capped(resp.bytes_stream(), max_bytes)
|
||||
.await
|
||||
.with_context(|| format!("download body for {url}"))
|
||||
}
|
||||
|
||||
/// True when `bytes` sniffs as one of the *renderable* image formats
|
||||
/// the `/files/*key` endpoint can serve with a correct Content-Type:
|
||||
/// JPEG, PNG, WebP, GIF, AVIF. Matches the upload pipeline's
|
||||
/// whitelist in `upload::parse_image`.
|
||||
///
|
||||
/// `infer::MatcherType::Image` is intentionally NOT used — it also
|
||||
/// matches BMP, TIFF, HEIF, ICO, PSD, and JP2. Those would sniff as
|
||||
/// "image" here but [`api::files::content_type_for`] would fall back
|
||||
/// to `application/octet-stream`, prompting browsers to download
|
||||
/// instead of render. Keep the two layers aligned.
|
||||
pub fn looks_like_image(bytes: &[u8]) -> bool {
|
||||
matches!(
|
||||
infer::get(bytes).map(|k| k.mime_type()),
|
||||
Some("image/jpeg" | "image/png" | "image/webp" | "image/gif" | "image/avif")
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use futures_util::stream;
|
||||
|
||||
fn allow_just(host: &str) -> DownloadAllowlist {
|
||||
DownloadAllowlist::new().allow(host)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn safe_url_allows_listed_host() {
|
||||
let allow = allow_just("cdn.example.com");
|
||||
assert!(is_safe_url("https://cdn.example.com/img.jpg", &allow).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn safe_url_blocks_unlisted_host() {
|
||||
let allow = allow_just("cdn.example.com");
|
||||
let err = is_safe_url("https://evil.example.org/img.jpg", &allow).unwrap_err();
|
||||
assert!(matches!(err, UrlSafetyError::HostNotAllowed(h) if h == "evil.example.org"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn safe_url_blocks_localhost_even_if_allowlisted() {
|
||||
let allow = allow_just("localhost");
|
||||
assert!(matches!(
|
||||
is_safe_url("http://localhost:8080/", &allow).unwrap_err(),
|
||||
UrlSafetyError::Loopback
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn safe_url_blocks_loopback_ipv4() {
|
||||
let allow = allow_just("127.0.0.1");
|
||||
assert!(matches!(
|
||||
is_safe_url("http://127.0.0.1/", &allow).unwrap_err(),
|
||||
UrlSafetyError::PrivateIp(_)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn safe_url_blocks_rfc1918() {
|
||||
let allow = allow_just("10.0.0.1");
|
||||
for url in [
|
||||
"http://10.0.0.1/",
|
||||
"http://192.168.1.1/",
|
||||
"http://172.16.0.5/",
|
||||
"http://172.31.255.255/",
|
||||
] {
|
||||
assert!(
|
||||
matches!(
|
||||
is_safe_url(url, &allow).unwrap_err(),
|
||||
UrlSafetyError::PrivateIp(_)
|
||||
),
|
||||
"should reject {url}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn safe_url_blocks_link_local() {
|
||||
let allow = allow_just("169.254.169.254");
|
||||
// 169.254.169.254 is the AWS/GCP metadata service — the most
|
||||
// dangerous SSRF target on a default cloud VM.
|
||||
assert!(matches!(
|
||||
is_safe_url("http://169.254.169.254/", &allow).unwrap_err(),
|
||||
UrlSafetyError::PrivateIp(_)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn safe_url_blocks_ipv6_loopback_and_ula() {
|
||||
// Debug what host_str returns first — reqwest::Url normalises
|
||||
// IPv6 literals as `[::1]` with brackets, which doesn't parse
|
||||
// as `IpAddr` directly. The implementation strips them.
|
||||
let allow = allow_just("[::1]");
|
||||
let err = is_safe_url("http://[::1]/", &allow).unwrap_err();
|
||||
assert!(
|
||||
matches!(err, UrlSafetyError::PrivateIp(_)),
|
||||
"expected PrivateIp, got {err:?}"
|
||||
);
|
||||
let allow = allow_just("[fd00::1]");
|
||||
let err = is_safe_url("http://[fd00::1]/", &allow).unwrap_err();
|
||||
assert!(
|
||||
matches!(err, UrlSafetyError::PrivateIp(_)),
|
||||
"expected PrivateIp, got {err:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn safe_url_blocks_ipv4_mapped_ipv6_loopback() {
|
||||
// `Ipv6Addr::is_loopback()` only matches `::1` exactly, so
|
||||
// `::ffff:127.0.0.1` would slip through without the
|
||||
// to_ipv4_mapped() unwrap in is_private_ip.
|
||||
let allow = allow_just("[::ffff:127.0.0.1]");
|
||||
let err = is_safe_url("http://[::ffff:127.0.0.1]/", &allow).unwrap_err();
|
||||
assert!(
|
||||
matches!(err, UrlSafetyError::PrivateIp(_)),
|
||||
"expected PrivateIp, got {err:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn safe_url_blocks_ipv4_mapped_ipv6_rfc1918() {
|
||||
let allow = allow_just("[::ffff:10.0.0.1]");
|
||||
let err = is_safe_url("http://[::ffff:10.0.0.1]/", &allow).unwrap_err();
|
||||
assert!(matches!(err, UrlSafetyError::PrivateIp(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn safe_url_blocks_non_http_schemes() {
|
||||
let allow = allow_just("anywhere");
|
||||
assert!(matches!(
|
||||
is_safe_url("file:///etc/passwd", &allow).unwrap_err(),
|
||||
UrlSafetyError::BadScheme(_)
|
||||
));
|
||||
assert!(matches!(
|
||||
is_safe_url("gopher://anywhere:70/", &allow).unwrap_err(),
|
||||
UrlSafetyError::BadScheme(_)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn safe_url_rejects_unparseable() {
|
||||
let allow = allow_just("anywhere");
|
||||
assert!(matches!(
|
||||
is_safe_url("not a url", &allow).unwrap_err(),
|
||||
UrlSafetyError::Unparseable
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn safe_url_empty_allowlist_rejects_everything() {
|
||||
let allow = DownloadAllowlist::new();
|
||||
let err = is_safe_url("https://cdn.example.com/img.jpg", &allow).unwrap_err();
|
||||
assert!(matches!(err, UrlSafetyError::HostNotAllowed(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn allowlist_matches_case_insensitively() {
|
||||
let allow = DownloadAllowlist::new().allow("CDN.Example.COM");
|
||||
assert!(is_safe_url("https://cdn.example.com/x.jpg", &allow).is_ok());
|
||||
assert!(is_safe_url("https://CDN.EXAMPLE.com/x.jpg", &allow).is_ok());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accumulate_capped_returns_full_body_under_cap() {
|
||||
let chunks: Vec<Result<bytes::Bytes, std::io::Error>> = vec![
|
||||
Ok(bytes::Bytes::from_static(b"hello ")),
|
||||
Ok(bytes::Bytes::from_static(b"world")),
|
||||
];
|
||||
let s = stream::iter(chunks);
|
||||
let out = accumulate_capped(s, 100).await.unwrap();
|
||||
assert_eq!(out.as_ref(), b"hello world");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accumulate_capped_bails_past_cap() {
|
||||
let chunks: Vec<Result<bytes::Bytes, std::io::Error>> = vec![
|
||||
Ok(bytes::Bytes::from(vec![0u8; 50])),
|
||||
Ok(bytes::Bytes::from(vec![0u8; 60])),
|
||||
];
|
||||
let s = stream::iter(chunks);
|
||||
let err = accumulate_capped(s, 100).await.unwrap_err();
|
||||
assert!(err.to_string().contains("100-byte cap"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accumulate_capped_surfaces_stream_errors() {
|
||||
let chunks: Vec<Result<bytes::Bytes, std::io::Error>> = vec![
|
||||
Ok(bytes::Bytes::from_static(b"ok")),
|
||||
Err(std::io::Error::other("network blip")),
|
||||
];
|
||||
let s = stream::iter(chunks);
|
||||
let err = accumulate_capped(s, 100).await.unwrap_err();
|
||||
assert!(err.to_string().contains("network blip"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn looks_like_image_accepts_jpeg() {
|
||||
// JPEG SOI + APP0 segment.
|
||||
let jpeg = [0xff, 0xd8, 0xff, 0xe0, 0, 0x10, b'J', b'F', b'I', b'F'];
|
||||
assert!(looks_like_image(&jpeg));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn looks_like_image_accepts_png() {
|
||||
let png = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0, 0, 0, 0];
|
||||
assert!(looks_like_image(&png));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn looks_like_image_rejects_html_disguised_as_image() {
|
||||
let html = b"<html><body>not an image</body></html>";
|
||||
assert!(!looks_like_image(html));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn looks_like_image_rejects_empty() {
|
||||
assert!(!looks_like_image(&[]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn looks_like_image_rejects_renderable_but_unsupported_formats() {
|
||||
// BMP, TIFF, ICO, PSD are `infer::MatcherType::Image` but the
|
||||
// /files/*key handler doesn't have Content-Type mappings for
|
||||
// them, so they'd be served as application/octet-stream and
|
||||
// download instead of render. Reject at the crawler so we
|
||||
// never land them in storage.
|
||||
// BMP magic: "BM" + 4-byte size.
|
||||
let bmp = [b'B', b'M', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0];
|
||||
assert!(!looks_like_image(&bmp), "BMP must be rejected (not renderable by /files)");
|
||||
|
||||
// TIFF little-endian magic: "II" + 42.
|
||||
let tiff = [0x49, 0x49, 0x2a, 0x00, 0, 0, 0, 0];
|
||||
assert!(!looks_like_image(&tiff), "TIFF must be rejected");
|
||||
|
||||
// ICO magic: 0x00,0x00,0x01,0x00.
|
||||
let ico = [0x00, 0x00, 0x01, 0x00, 1, 0, 16, 16, 0, 0, 1, 0, 0x18, 0, 0x40, 0, 0, 0, 0x16, 0, 0, 0];
|
||||
assert!(!looks_like_image(&ico), "ICO must be rejected");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn looks_like_image_accepts_webp_gif_avif() {
|
||||
// Cover the three remaining whitelisted formats so a future
|
||||
// tightening that drops one would fail noisily.
|
||||
let webp = [
|
||||
b'R', b'I', b'F', b'F',
|
||||
0, 0, 0, 0,
|
||||
b'W', b'E', b'B', b'P',
|
||||
b'V', b'P', b'8', b' ',
|
||||
];
|
||||
assert!(looks_like_image(&webp));
|
||||
|
||||
let gif = [b'G', b'I', b'F', b'8', b'7', b'a', 0, 0, 0, 0];
|
||||
assert!(looks_like_image(&gif));
|
||||
|
||||
let avif = [
|
||||
0x00, 0x00, 0x00, 0x18,
|
||||
b'f', b't', b'y', b'p',
|
||||
b'a', b'v', b'i', b'f',
|
||||
0x00, 0x00, 0x00, 0x00,
|
||||
b'm', b'i', b'f', b'1',
|
||||
b'a', b'v', b'i', b'f',
|
||||
];
|
||||
assert!(looks_like_image(&avif));
|
||||
}
|
||||
}
|
||||
@@ -9,50 +9,43 @@
|
||||
//! Two things the cookie alone doesn't give us:
|
||||
//! 1. The cookie value is only meaningful to the *server* — we have
|
||||
//! no way to predict from the value alone whether it's still valid.
|
||||
//! `verify_session` does a navigation and checks for `#avatar_menu`,
|
||||
//! which only renders for authenticated visitors. Bail clean at
|
||||
//! startup if it's missing rather than discovering it 30 minutes
|
||||
//! into a backfill.
|
||||
//! `verify_session` does a navigation and inspects the probe page
|
||||
//! for three outcomes: broken-page response (transient — retry the
|
||||
//! probe), `#logo` present but `#avatar_menu` absent (genuine logout
|
||||
//! — bail loudly), or both present (authenticated). The earlier
|
||||
//! avatar-only check conflated "site is hiccuping" with "session is
|
||||
//! dead" and refused to start the crawler when the site had a brief
|
||||
//! 503.
|
||||
//! 2. The reqwest client (used for cover and chapter-image downloads)
|
||||
//! has its own cookie store; we seed it for the catalog host only.
|
||||
//! CDN hosts are deliberately *not* given the cookie — they serve
|
||||
//! image bytes by signed URLs and don't need it.
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::{anyhow, Context};
|
||||
use chromiumoxide::browser::Browser;
|
||||
use chromiumoxide::cdp::browser_protocol::network::CookieParam;
|
||||
|
||||
/// Compute the cookie domain (e.g. `.example.com`) from a start URL.
|
||||
/// The leading dot makes the cookie cover every subdomain — the source
|
||||
/// often redirects between `www.` and other prefixes mid-crawl, and a
|
||||
/// host-only cookie would silently drop on the cross-subdomain hop.
|
||||
///
|
||||
/// Caveat: this takes the last two dot-labels, which is wrong for
|
||||
/// multi-part TLDs (`.co.uk`, `.com.br` would resolve to `.co.uk` and
|
||||
/// attach to every site on `.co.uk`). For those, the operator should
|
||||
/// override via `CRAWLER_COOKIE_DOMAIN` rather than relying on this
|
||||
/// function — pulling in the Public Suffix List for one knob isn't
|
||||
/// worth it yet.
|
||||
pub fn registrable_domain(url: &str) -> Option<String> {
|
||||
let after_scheme = url.split_once("://")?.1;
|
||||
let host_with_port = after_scheme.split('/').next()?;
|
||||
let host = host_with_port
|
||||
.rsplit_once(':')
|
||||
.map_or(host_with_port, |(h, _)| h)
|
||||
.to_ascii_lowercase();
|
||||
if host.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let labels: Vec<&str> = host.split('.').filter(|l| !l.is_empty()).collect();
|
||||
if labels.len() < 2 {
|
||||
// Bare hostname (e.g. `localhost`) — return as-is, no leading
|
||||
// dot. Setting `.localhost` as cookie domain is invalid.
|
||||
return Some(host);
|
||||
}
|
||||
let registrable = &labels[labels.len() - 2..];
|
||||
Some(format!(".{}", registrable.join(".")))
|
||||
use crate::crawler::detect::{has_logo_sentinel, is_broken_page_body};
|
||||
|
||||
/// Outcome of inspecting a probe-page response.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum SessionProbe {
|
||||
/// `#logo` present and `#avatar_menu` present — session valid.
|
||||
Ok,
|
||||
/// `#logo` present but `#avatar_menu` absent — site rendered the
|
||||
/// normal layout for an unauthenticated visitor; refresh PHPSESSID.
|
||||
Unauthenticated,
|
||||
/// Broken-page body signature or `#logo` missing — site is hiccuping.
|
||||
/// Caller retries the probe rather than blaming the session.
|
||||
Transient,
|
||||
}
|
||||
|
||||
/// Re-export so existing callers keep working after the helper moved
|
||||
/// to `crawler::url_utils`. The body lives there.
|
||||
pub use crate::crawler::url_utils::registrable_domain;
|
||||
|
||||
/// Inject the PHPSESSID cookie into the browser's cookie store for the
|
||||
/// catalog domain. Must be called before any navigation that depends on
|
||||
/// authentication; subsequent navigations include the cookie
|
||||
@@ -86,76 +79,262 @@ pub async fn inject_phpsessid(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Navigate to `probe_url` and confirm the logged-in `#avatar_menu`
|
||||
/// element is present. The selector only renders for authenticated
|
||||
/// visitors, so its absence is the unambiguous signal that PHPSESSID
|
||||
/// is missing, expired, or revoked.
|
||||
/// Three-way classification of a probe-page response. Pure over HTML so
|
||||
/// it's unit-testable without a real browser. Order matters: a body
|
||||
/// matching the broken-page template is `Transient` even if the page
|
||||
/// happens to contain `#avatar_menu` HTML somewhere — trust the universal
|
||||
/// site signal over a stray selector match.
|
||||
pub fn classify_probe(html: &str) -> SessionProbe {
|
||||
if is_broken_page_body(html) {
|
||||
return SessionProbe::Transient;
|
||||
}
|
||||
let doc = scraper::Html::parse_document(html);
|
||||
if !has_logo_sentinel(&doc) {
|
||||
return SessionProbe::Transient;
|
||||
}
|
||||
let avatar_sel = scraper::Selector::parse("#avatar_menu").unwrap();
|
||||
if doc.select(&avatar_sel).next().is_some() {
|
||||
SessionProbe::Ok
|
||||
} else {
|
||||
SessionProbe::Unauthenticated
|
||||
}
|
||||
}
|
||||
|
||||
/// Three-way classification of a chapter page response.
|
||||
///
|
||||
/// This burns one navigation against the catalog's rate limiter. The
|
||||
/// trade is worth it — failing here costs ~1s; failing 30 minutes into
|
||||
/// a backfill costs 30 minutes.
|
||||
/// Reader pages don't render `#logo`, so [`classify_probe`] can't be
|
||||
/// reused as-is. The chapter-specific marker is `a#pic_container`
|
||||
/// (asserted by the reader-page parser at `parse_chapter_pages`).
|
||||
///
|
||||
/// Order matters: broken-page body wins over selector matches, so a
|
||||
/// transient site-wide 5xx that happens to render the avatar widget
|
||||
/// elsewhere doesn't falsely reach `Ok`.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ChapterProbe {
|
||||
/// `a#pic_container` present — reader rendered. Whether
|
||||
/// `#avatar_menu` is also there is informational; if the reader
|
||||
/// loaded the session is by definition still good.
|
||||
Ok,
|
||||
/// Site rendered a "logged out" or "please log in" page (no
|
||||
/// reader, no broken-page body, and no avatar widget either).
|
||||
/// Distinguishes the genuine expired-session case from a
|
||||
/// transient site hiccup.
|
||||
Unauthenticated,
|
||||
/// Broken-page body, or reader didn't render but the user is
|
||||
/// still logged in (avatar widget present). Caller should retry
|
||||
/// rather than blame the session.
|
||||
Transient,
|
||||
}
|
||||
|
||||
pub fn classify_chapter_probe(html: &str) -> ChapterProbe {
|
||||
if is_broken_page_body(html) {
|
||||
return ChapterProbe::Transient;
|
||||
}
|
||||
let doc = scraper::Html::parse_document(html);
|
||||
let container = scraper::Selector::parse("a#pic_container").unwrap();
|
||||
if doc.select(&container).next().is_some() {
|
||||
return ChapterProbe::Ok;
|
||||
}
|
||||
let avatar = scraper::Selector::parse("#avatar_menu").unwrap();
|
||||
if doc.select(&avatar).next().is_some() {
|
||||
// Logged-in user, but the reader didn't render — most likely
|
||||
// the layout shifted or the site is serving an interstitial.
|
||||
ChapterProbe::Transient
|
||||
} else {
|
||||
// No reader, no avatar, no broken-body marker — site rendered
|
||||
// the "please log in" page, which is the genuine session-
|
||||
// expired signal on this route.
|
||||
ChapterProbe::Unauthenticated
|
||||
}
|
||||
}
|
||||
|
||||
/// In-startup retry budget for the session probe. Small but non-zero —
|
||||
/// startup hitting a 5-second site hiccup shouldn't fail the operator
|
||||
/// with "PHPSESSID expired" when the session is actually fine.
|
||||
const PROBE_MAX_ATTEMPTS: u32 = 3;
|
||||
const PROBE_RETRY_DELAY: Duration = Duration::from_secs(2);
|
||||
|
||||
/// Navigate to `probe_url` and classify the response. Retries the probe
|
||||
/// on `Transient` outcomes (broken-page body, missing `#logo`); fails
|
||||
/// fast on `Unauthenticated`; returns `Ok(())` on success.
|
||||
///
|
||||
/// This burns one navigation per attempt against the catalog's rate
|
||||
/// limiter. The trade is worth it — failing here costs ~1s; failing 30
|
||||
/// minutes into a backfill costs 30 minutes.
|
||||
pub async fn verify_session(browser: &Browser, probe_url: &str) -> anyhow::Result<()> {
|
||||
let mut attempt = 0u32;
|
||||
loop {
|
||||
attempt += 1;
|
||||
let html = fetch_probe_html(browser, probe_url).await?;
|
||||
match classify_probe(&html) {
|
||||
SessionProbe::Ok => {
|
||||
tracing::info!(attempt, "session probe ok — #logo + #avatar_menu present");
|
||||
return Ok(());
|
||||
}
|
||||
SessionProbe::Unauthenticated => {
|
||||
return Err(anyhow!(
|
||||
"session probe failed — #avatar_menu not present at {probe_url} \
|
||||
(page rendered the normal layout); PHPSESSID is missing, expired, \
|
||||
or revoked. Refresh CRAWLER_PHPSESSID and re-run."
|
||||
));
|
||||
}
|
||||
SessionProbe::Transient if attempt < PROBE_MAX_ATTEMPTS => {
|
||||
tracing::warn!(
|
||||
attempt,
|
||||
max_attempts = PROBE_MAX_ATTEMPTS,
|
||||
"session probe got a transient page; retrying"
|
||||
);
|
||||
tokio::time::sleep(PROBE_RETRY_DELAY).await;
|
||||
}
|
||||
SessionProbe::Transient => {
|
||||
return Err(anyhow!(
|
||||
"session probe failed — probe page at {probe_url} returned a \
|
||||
broken-page response after {PROBE_MAX_ATTEMPTS} attempts. \
|
||||
The site appears to be down or rate-limiting us; try again \
|
||||
later before refreshing CRAWLER_PHPSESSID."
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn fetch_probe_html(browser: &Browser, probe_url: &str) -> anyhow::Result<String> {
|
||||
let page = browser
|
||||
.new_page(probe_url)
|
||||
.await
|
||||
.with_context(|| format!("open probe page {probe_url}"))?;
|
||||
page.wait_for_navigation().await.context("wait for nav on probe")?;
|
||||
// The avatar menu is rendered server-side as part of the header
|
||||
// when a valid session cookie is present; absent JS is fine.
|
||||
let found = page.find_element("#avatar_menu").await.is_ok();
|
||||
let html = page.content().await.context("read probe html")?;
|
||||
page.close().await.ok();
|
||||
if found {
|
||||
tracing::info!("session probe ok — #avatar_menu present");
|
||||
Ok(())
|
||||
} else {
|
||||
Err(anyhow!(
|
||||
"session probe failed — #avatar_menu not present at {probe_url}; \
|
||||
PHPSESSID is missing, expired, or revoked. Refresh CRAWLER_PHPSESSID \
|
||||
and re-run."
|
||||
))
|
||||
}
|
||||
Ok(html)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
// registrable_domain tests live in crawler::url_utils now —
|
||||
// it's the canonical home for that helper.
|
||||
|
||||
#[test]
|
||||
fn registrable_domain_strips_subdomain() {
|
||||
fn classify_probe_ok_when_logo_and_avatar_present() {
|
||||
let html = r#"<html><body>
|
||||
<header><div id="logo">Target</div><div id="avatar_menu"></div></header>
|
||||
</body></html>"#;
|
||||
assert_eq!(classify_probe(html), SessionProbe::Ok);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classify_probe_unauth_when_logo_present_but_avatar_absent() {
|
||||
// Real "logged out" response: site layout renders fine, just no
|
||||
// avatar widget. This is the only state that should blame the
|
||||
// session cookie.
|
||||
let html = r#"<html><body>
|
||||
<header><div id="logo">Target</div></header>
|
||||
<main>Please log in.</main>
|
||||
</body></html>"#;
|
||||
assert_eq!(classify_probe(html), SessionProbe::Unauthenticated);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classify_probe_transient_on_broken_page_body() {
|
||||
let html = "<html><body>\
|
||||
<p>we're sorry, the request file are not found.</p>\
|
||||
</body></html>";
|
||||
assert_eq!(classify_probe(html), SessionProbe::Transient);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classify_probe_transient_when_logo_missing() {
|
||||
// No broken-body marker, but no site layout either — treat as
|
||||
// transient (could be a Cloudflare interstitial, a 5xx page,
|
||||
// etc.) rather than blaming the session.
|
||||
let html = "<html><body><h1>Service Unavailable</h1></body></html>";
|
||||
assert_eq!(classify_probe(html), SessionProbe::Transient);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classify_probe_transient_on_empty_response() {
|
||||
assert_eq!(classify_probe(""), SessionProbe::Transient);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classify_chapter_probe_ok_when_reader_rendered() {
|
||||
let html = r#"
|
||||
<html><body>
|
||||
<a id="pic_container">
|
||||
<img id="page1" src="https://cdn/1.jpg">
|
||||
</a>
|
||||
</body></html>
|
||||
"#;
|
||||
assert_eq!(classify_chapter_probe(html), ChapterProbe::Ok);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classify_chapter_probe_unauthenticated_when_no_reader_and_no_avatar() {
|
||||
// What a logged-out hit on a chapter URL renders: a normal
|
||||
// site layout (header etc.) with a "please log in" body, but
|
||||
// no reader and no avatar widget.
|
||||
let html = r#"
|
||||
<html><body>
|
||||
<header><div id="logo">Catalog</div></header>
|
||||
<main>Please log in to read this chapter.</main>
|
||||
</body></html>
|
||||
"#;
|
||||
assert_eq!(
|
||||
registrable_domain("https://www.target-site.com/manga/foo/").as_deref(),
|
||||
Some(".target-site.com")
|
||||
);
|
||||
assert_eq!(
|
||||
registrable_domain("https://m.example.org").as_deref(),
|
||||
Some(".example.org")
|
||||
classify_chapter_probe(html),
|
||||
ChapterProbe::Unauthenticated
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn registrable_domain_keeps_two_label_host() {
|
||||
assert_eq!(
|
||||
registrable_domain("https://example.com/").as_deref(),
|
||||
Some(".example.com")
|
||||
);
|
||||
fn classify_chapter_probe_transient_when_logged_in_but_reader_missing() {
|
||||
// Avatar shows the session is still valid; reader didn't
|
||||
// render — site is serving an interstitial or the layout
|
||||
// momentarily shifted. Retry, don't blame the session.
|
||||
let html = r#"
|
||||
<html><body>
|
||||
<header><div id="logo">Catalog</div><div id="avatar_menu"></div></header>
|
||||
<main>Site maintenance — back in 5 minutes.</main>
|
||||
</body></html>
|
||||
"#;
|
||||
assert_eq!(classify_chapter_probe(html), ChapterProbe::Transient);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn registrable_domain_handles_port() {
|
||||
assert_eq!(
|
||||
registrable_domain("http://www.foo.bar:8080/x").as_deref(),
|
||||
Some(".foo.bar")
|
||||
);
|
||||
fn classify_chapter_probe_transient_on_broken_page_body() {
|
||||
let html =
|
||||
"<html><body><p>we're sorry, the request file are not found.</p></body></html>";
|
||||
assert_eq!(classify_chapter_probe(html), ChapterProbe::Transient);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn registrable_domain_bare_hostname_no_leading_dot() {
|
||||
// .localhost would be invalid as a cookie Domain.
|
||||
assert_eq!(registrable_domain("http://localhost:5173").as_deref(), Some("localhost"));
|
||||
fn classify_chapter_probe_does_not_misfire_on_avatar_alone_without_reader() {
|
||||
// Regression for the original bug: the binary
|
||||
// find_element("#avatar_menu") check treated "no avatar" as
|
||||
// session-expired even when a transient hiccup was the real
|
||||
// cause. classify_chapter_probe must NOT trip on that pattern
|
||||
// when pic_container *is* present.
|
||||
let html = r#"
|
||||
<html><body>
|
||||
<a id="pic_container">
|
||||
<img id="page1" src="https://cdn/1.jpg">
|
||||
</a>
|
||||
</body></html>
|
||||
"#;
|
||||
assert_eq!(classify_chapter_probe(html), ChapterProbe::Ok);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn registrable_domain_returns_none_for_garbage() {
|
||||
assert!(registrable_domain("not a url").is_none());
|
||||
fn classify_probe_trusts_broken_body_over_stray_avatar_match() {
|
||||
// Defensive: if a broken-page body somehow contains an
|
||||
// #avatar_menu element (e.g. an unrelated debug page on the
|
||||
// same template), the body signature still wins.
|
||||
let html = r#"<html><body>
|
||||
<p>we're sorry, the request file are not found.</p>
|
||||
<div id="logo"></div>
|
||||
<div id="avatar_menu"></div>
|
||||
</body></html>"#;
|
||||
assert_eq!(classify_probe(html), SessionProbe::Transient);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -82,21 +82,42 @@ pub struct FetchContext<'a> {
|
||||
pub rate: &'a crate::crawler::rate_limit::HostRateLimiters,
|
||||
}
|
||||
|
||||
/// Lazy iterator over discovered manga refs. The caller drives the
|
||||
/// walk one batch at a time, so it can break out as soon as a
|
||||
/// downstream stop condition is met (e.g. N consecutive Unchanged
|
||||
/// upserts in Incremental mode) without paying for pages it won't use.
|
||||
///
|
||||
/// Batches are typically one source-index page each. Within a batch
|
||||
/// refs are already in the right per-page order for the active mode
|
||||
/// (Backfill reverses each page to oldest-first; Incremental leaves
|
||||
/// the source's natural newest-first ordering).
|
||||
#[async_trait]
|
||||
pub trait DiscoverWalk: Send {
|
||||
/// Return the next batch of refs, or `Ok(None)` when the source has
|
||||
/// no more pages. The walker is single-use; calling `next_batch`
|
||||
/// after `None` is allowed and continues to return `None`.
|
||||
async fn next_batch(
|
||||
&mut self,
|
||||
ctx: &FetchContext<'_>,
|
||||
) -> anyhow::Result<Option<Vec<SourceMangaRef>>>;
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
pub trait Source: Send + Sync {
|
||||
/// Stable identifier — also the row key in the `sources` table.
|
||||
fn id(&self) -> &'static str;
|
||||
|
||||
/// Returns up to `max_results` manga refs in source order. Pass
|
||||
/// `None` for an uncapped walk (full backfill / incremental sweep).
|
||||
/// Implementations should stop paginating as soon as the cap is
|
||||
/// reached so partial runs don't pay for pages they won't use.
|
||||
/// Begin discovery in `mode`. Returns a walker the caller drives
|
||||
/// page-by-page via `next_batch`. The initial page-1 probe (used
|
||||
/// to determine `last_page` and warm the cache for sites that
|
||||
/// can't be paged without knowing the bound) happens inside this
|
||||
/// call, so a fresh walker is ready to yield its first batch
|
||||
/// without further setup.
|
||||
async fn discover(
|
||||
&self,
|
||||
ctx: &FetchContext<'_>,
|
||||
mode: DiscoverMode,
|
||||
max_results: Option<usize>,
|
||||
) -> anyhow::Result<Vec<SourceMangaRef>>;
|
||||
) -> anyhow::Result<Box<dyn DiscoverWalk + Send>>;
|
||||
|
||||
async fn fetch_manga(
|
||||
&self,
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
//! (`td:has(label:contains("Author:"))`) are implemented by walking
|
||||
//! the parsed tree.
|
||||
|
||||
use std::collections::VecDeque;
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::Context;
|
||||
@@ -14,9 +15,24 @@ use async_trait::async_trait;
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use super::{
|
||||
DiscoverMode, FetchContext, Source, SourceChapter, SourceChapterRef, SourceManga,
|
||||
SourceMangaRef,
|
||||
DiscoverMode, DiscoverWalk, FetchContext, Source, SourceChapter, SourceChapterRef,
|
||||
SourceManga, SourceMangaRef,
|
||||
};
|
||||
use crate::crawler::detect::{
|
||||
has_logo_sentinel, is_broken_page_body, retry_on_transient, PageError,
|
||||
};
|
||||
|
||||
/// `sources.id` value for this Source impl. Exposed as a const so the
|
||||
/// daemon can look up per-source state (e.g. `seed_completed_at`)
|
||||
/// before constructing the Source itself.
|
||||
pub const SOURCE_ID: &str = "target";
|
||||
|
||||
/// In-loop retry budget for transient pages encountered during a single
|
||||
/// `discover` walk. Bounded small because the job system itself retries
|
||||
/// the whole `Discover` job on failure — these inline retries only need
|
||||
/// to absorb a brief site hiccup mid-walk.
|
||||
const PAGE_TRANSIENT_RETRY_ATTEMPTS: u32 = 3;
|
||||
const PAGE_TRANSIENT_RETRY_DELAY: Duration = Duration::from_secs(2);
|
||||
|
||||
pub struct TargetSource {
|
||||
base_url: String,
|
||||
@@ -50,34 +66,31 @@ impl TargetSource {
|
||||
#[async_trait]
|
||||
impl Source for TargetSource {
|
||||
fn id(&self) -> &'static str {
|
||||
"target"
|
||||
SOURCE_ID
|
||||
}
|
||||
|
||||
async fn discover(
|
||||
&self,
|
||||
ctx: &FetchContext<'_>,
|
||||
mode: DiscoverMode,
|
||||
max_results: Option<usize>,
|
||||
) -> anyhow::Result<Vec<SourceMangaRef>> {
|
||||
) -> anyhow::Result<Box<dyn DiscoverWalk + Send>> {
|
||||
// Always visit page 1 first because that's the only way to
|
||||
// discover `last_page`. We cache the HTML so we don't have to
|
||||
// re-navigate when the iteration reaches page 1 again.
|
||||
let first_html = navigate(ctx, self.base_url.as_str()).await?;
|
||||
// discover `last_page`. Retry it on transient — a broken first
|
||||
// page would otherwise abort the whole walk before we've even
|
||||
// started.
|
||||
let first_html = retry_on_transient(
|
||||
|| async { navigate(ctx, self.base_url.as_str()).await },
|
||||
PAGE_TRANSIENT_RETRY_ATTEMPTS,
|
||||
PAGE_TRANSIENT_RETRY_DELAY,
|
||||
)
|
||||
.await?;
|
||||
let last_page = {
|
||||
let doc = scraper::Html::parse_document(&first_html);
|
||||
parse_last_page(&doc)
|
||||
};
|
||||
|
||||
let backfill = matches!(mode, DiscoverMode::Backfill);
|
||||
let order: Vec<i32> = match (last_page, backfill) {
|
||||
(None, _) => vec![1],
|
||||
// Backfill = oldest-first: walk pages last → 1, then
|
||||
// reverse within each page (the listing is update_date
|
||||
// DESC, so the bottom of the last page is the oldest
|
||||
// entry the source still surfaces).
|
||||
(Some(last), true) => (1..=last).rev().collect(),
|
||||
(Some(last), false) => (1..=last).collect(),
|
||||
};
|
||||
let order = build_page_order(last_page, backfill);
|
||||
tracing::info!(
|
||||
?mode,
|
||||
last_page = ?last_page,
|
||||
@@ -85,29 +98,12 @@ impl Source for TargetSource {
|
||||
"walking pagination"
|
||||
);
|
||||
|
||||
let mut all = Vec::new();
|
||||
for page_num in order {
|
||||
let html = if page_num == 1 {
|
||||
first_html.clone()
|
||||
} else {
|
||||
navigate(ctx, &page_url(&self.base_url, page_num)).await?
|
||||
};
|
||||
let mut page_refs = {
|
||||
let doc = scraper::Html::parse_document(&html);
|
||||
parse_manga_list_from(&doc)
|
||||
};
|
||||
if backfill {
|
||||
page_refs.reverse();
|
||||
}
|
||||
tracing::info!(page_num, count = page_refs.len(), "page walked");
|
||||
all.extend(page_refs);
|
||||
if cap_reached(&all, max_results) {
|
||||
tracing::info!(cap = ?max_results, "max_results reached; halting pagination");
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(truncate_to_cap(all, max_results))
|
||||
Ok(Box::new(TargetSourceWalker {
|
||||
base_url: self.base_url.clone(),
|
||||
backfill,
|
||||
pages_remaining: order,
|
||||
first_page_html: Some(first_html),
|
||||
}))
|
||||
}
|
||||
|
||||
async fn fetch_manga(
|
||||
@@ -116,8 +112,12 @@ impl Source for TargetSource {
|
||||
r: &SourceMangaRef,
|
||||
) -> anyhow::Result<SourceManga> {
|
||||
let html = navigate(ctx, r.url.as_str()).await?;
|
||||
parse_manga_detail(&html, &r.source_manga_key, self.parse_chapters)
|
||||
.with_context(|| format!("parse manga detail at {}", r.url))
|
||||
// Convert PageError → anyhow::Error via `?`. PageError stays
|
||||
// downcastable from the wrapped anyhow::Error so the pipeline
|
||||
// can still recognize Transient via `error.downcast_ref::<PageError>()`.
|
||||
let manga = parse_manga_detail(&html, &r.source_manga_key, self.parse_chapters)
|
||||
.with_context(|| format!("parse manga detail at {}", r.url))?;
|
||||
Ok(manga)
|
||||
}
|
||||
|
||||
async fn fetch_chapter_list(
|
||||
@@ -137,29 +137,118 @@ impl Source for TargetSource {
|
||||
}
|
||||
}
|
||||
|
||||
fn cap_reached<T>(buf: &[T], max: Option<usize>) -> bool {
|
||||
matches!(max, Some(m) if buf.len() >= m)
|
||||
/// Build the queue of page numbers `TargetSource::discover` will walk.
|
||||
/// Backfill is oldest-first: pages `last..=1` (within each page the
|
||||
/// walker reverses entries, since the source orders by update_date
|
||||
/// DESC). Incremental is newest-first: pages `1..=last` in natural
|
||||
/// order. If `last_page` is unknown (source surfaces no pagination)
|
||||
/// only page 1 is visited.
|
||||
fn build_page_order(last_page: Option<i32>, backfill: bool) -> VecDeque<i32> {
|
||||
match (last_page, backfill) {
|
||||
(None, _) => VecDeque::from([1]),
|
||||
(Some(last), true) => (1..=last).rev().collect(),
|
||||
(Some(last), false) => (1..=last).collect(),
|
||||
}
|
||||
}
|
||||
|
||||
fn truncate_to_cap<T>(mut buf: Vec<T>, max: Option<usize>) -> Vec<T> {
|
||||
if let Some(m) = max {
|
||||
buf.truncate(m);
|
||||
/// Walker returned by [`TargetSource::discover`]. Pops one source-index
|
||||
/// page per `next_batch` call. Page 1's HTML is cached at construction
|
||||
/// time (the discover call needed it to read `last_page` anyway) so the
|
||||
/// batch covering page 1 doesn't re-fetch.
|
||||
struct TargetSourceWalker {
|
||||
base_url: String,
|
||||
backfill: bool,
|
||||
pages_remaining: VecDeque<i32>,
|
||||
first_page_html: Option<String>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl DiscoverWalk for TargetSourceWalker {
|
||||
async fn next_batch(
|
||||
&mut self,
|
||||
ctx: &FetchContext<'_>,
|
||||
) -> anyhow::Result<Option<Vec<SourceMangaRef>>> {
|
||||
let Some(page_num) = self.pages_remaining.pop_front() else {
|
||||
return Ok(None);
|
||||
};
|
||||
let mut page_refs = if page_num == 1 {
|
||||
// Reuse the cached page-1 HTML from the initial probe. Take
|
||||
// it (rather than clone) so a malformed page-order queue
|
||||
// that re-visits page 1 still falls back to a real fetch.
|
||||
match self.first_page_html.take() {
|
||||
Some(html) => {
|
||||
let doc = scraper::Html::parse_document(&html);
|
||||
parse_manga_list_from(&doc)?
|
||||
}
|
||||
None => {
|
||||
retry_on_transient(
|
||||
|| async {
|
||||
let html = navigate(ctx, self.base_url.as_str()).await?;
|
||||
let doc = scraper::Html::parse_document(&html);
|
||||
parse_manga_list_from(&doc)
|
||||
},
|
||||
PAGE_TRANSIENT_RETRY_ATTEMPTS,
|
||||
PAGE_TRANSIENT_RETRY_DELAY,
|
||||
)
|
||||
.await?
|
||||
}
|
||||
}
|
||||
} else {
|
||||
retry_on_transient(
|
||||
|| async {
|
||||
let url = page_url(&self.base_url, page_num);
|
||||
let html = navigate(ctx, &url).await?;
|
||||
let doc = scraper::Html::parse_document(&html);
|
||||
parse_manga_list_from(&doc)
|
||||
},
|
||||
PAGE_TRANSIENT_RETRY_ATTEMPTS,
|
||||
PAGE_TRANSIENT_RETRY_DELAY,
|
||||
)
|
||||
.await?
|
||||
};
|
||||
if self.backfill {
|
||||
page_refs.reverse();
|
||||
}
|
||||
tracing::info!(page_num, count = page_refs.len(), "page walked");
|
||||
Ok(Some(page_refs))
|
||||
}
|
||||
buf
|
||||
}
|
||||
|
||||
/// Single point of rate-limited navigation. Every Source request goes
|
||||
/// through here, so the per-host limiter map is the only knob that
|
||||
/// controls per-origin RPS.
|
||||
async fn navigate(ctx: &FetchContext<'_>, url: &str) -> anyhow::Result<String> {
|
||||
/// controls per-origin RPS. Also the choke point for transient-page
|
||||
/// detection — every fetched body is screened by
|
||||
/// [`classify_navigate_html`] before being handed to a selector.
|
||||
async fn navigate(ctx: &FetchContext<'_>, url: &str) -> Result<String, PageError> {
|
||||
ctx.rate.wait_for(url).await?;
|
||||
let page = ctx.browser.new_page(url).await?;
|
||||
page.wait_for_navigation().await?;
|
||||
let page = ctx
|
||||
.browser
|
||||
.new_page(url)
|
||||
.await
|
||||
.map_err(|e| PageError::Other(anyhow::Error::from(e)))?;
|
||||
page.wait_for_navigation()
|
||||
.await
|
||||
.map_err(|e| PageError::Other(anyhow::Error::from(e)))?;
|
||||
// Stopgap until we wait on a specific selector per page type —
|
||||
// gives any post-load JS a beat to finish injecting content.
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
let html = page.content().await?;
|
||||
page.close().await?;
|
||||
let html = page
|
||||
.content()
|
||||
.await
|
||||
.map_err(|e| PageError::Other(anyhow::Error::from(e)))?;
|
||||
page.close().await.ok();
|
||||
classify_navigate_html(html)
|
||||
}
|
||||
|
||||
/// Classify a fetched body. The broken-page template is universal across
|
||||
/// the site — every page type (list, detail, chapter list, reader) gets
|
||||
/// the same `we're sorry, the request file are not found` body when the
|
||||
/// server is hiccuping. Catching it here means individual parsers
|
||||
/// downstream don't have to repeat the check.
|
||||
fn classify_navigate_html(html: String) -> Result<String, PageError> {
|
||||
if is_broken_page_body(&html) {
|
||||
return Err(PageError::transient("broken-page body signature"));
|
||||
}
|
||||
Ok(html)
|
||||
}
|
||||
|
||||
@@ -204,14 +293,23 @@ fn page_url(template_url: &str, page: i32) -> String {
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn parse_manga_list(html: &str) -> Vec<SourceMangaRef> {
|
||||
fn parse_manga_list(html: &str) -> Result<Vec<SourceMangaRef>, PageError> {
|
||||
let doc = scraper::Html::parse_document(html);
|
||||
parse_manga_list_from(&doc)
|
||||
}
|
||||
|
||||
fn parse_manga_list_from(doc: &scraper::Html) -> Vec<SourceMangaRef> {
|
||||
/// Parse a manga listing page. `#logo` is present on every well-formed
|
||||
/// listing page on the source; its absence means the response is a
|
||||
/// broken-page placeholder (transient) rather than a genuinely empty
|
||||
/// listing. Empty listings (last-page tail, search with no hits) remain
|
||||
/// `Ok(vec![])`.
|
||||
fn parse_manga_list_from(doc: &scraper::Html) -> Result<Vec<SourceMangaRef>, PageError> {
|
||||
if !has_logo_sentinel(doc) {
|
||||
return Err(PageError::transient("manga list: #logo sentinel missing"));
|
||||
}
|
||||
let sel = scraper::Selector::parse("#left_side .pic_list .updatesli span a").unwrap();
|
||||
doc.select(&sel)
|
||||
Ok(doc
|
||||
.select(&sel)
|
||||
.filter_map(|a| {
|
||||
let url = a.value().attr("href")?.trim().to_string();
|
||||
if url.is_empty() {
|
||||
@@ -227,16 +325,22 @@ fn parse_manga_list_from(doc: &scraper::Html) -> Vec<SourceMangaRef> {
|
||||
url,
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn parse_manga_detail(
|
||||
html: &str,
|
||||
key: &str,
|
||||
include_chapters: bool,
|
||||
) -> anyhow::Result<SourceManga> {
|
||||
) -> Result<SourceManga, PageError> {
|
||||
let doc = scraper::Html::parse_document(html);
|
||||
|
||||
// Sentinel first: a broken-page response will trip this before any
|
||||
// anyhow context is added for missing required fields.
|
||||
if !has_logo_sentinel(&doc) {
|
||||
return Err(PageError::transient("manga detail: #logo sentinel missing"));
|
||||
}
|
||||
|
||||
let title = first_text(&doc, ".w-title h1").context("missing .w-title h1")?;
|
||||
let summary = first_text(&doc, ".manga_summary");
|
||||
let cover_url = first_attr(&doc, ".cover > img:nth-child(1)", "src");
|
||||
@@ -494,6 +598,7 @@ mod tests {
|
||||
|
||||
const LISTING_HTML: &str = r#"
|
||||
<html><body>
|
||||
<header><div id="logo">Target</div></header>
|
||||
<div id="left_side">
|
||||
<div class="pic_list">
|
||||
<div class="updatesli">
|
||||
@@ -512,6 +617,7 @@ mod tests {
|
||||
|
||||
const DETAIL_HTML: &str = r#"
|
||||
<html><body>
|
||||
<header><div id="logo">Target</div></header>
|
||||
<div class="w-title"><h1>Test Manga Title</h1></div>
|
||||
<div class="cover"><img src="/cover.jpg"><img src="/extra-not-cover.jpg"></div>
|
||||
<div class="manga_summary">A summary of the manga.</div>
|
||||
@@ -537,7 +643,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn parse_manga_list_extracts_title_url_and_derives_key() {
|
||||
let refs = parse_manga_list(LISTING_HTML);
|
||||
let refs = parse_manga_list(LISTING_HTML).expect("parse");
|
||||
assert_eq!(refs.len(), 2, "third entry has empty href and is skipped");
|
||||
assert_eq!(refs[0].title, "Foo Manga");
|
||||
assert_eq!(refs[0].url, "https://target.example/manga/foo");
|
||||
@@ -546,6 +652,30 @@ mod tests {
|
||||
assert_eq!(refs[1].source_manga_key, "bar-baz");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_manga_list_returns_transient_when_logo_missing() {
|
||||
// Broken-page response: no #logo, no listing. Empty Vec would
|
||||
// hide this as "page has no mangas"; Transient is the signal
|
||||
// upstream code retries on.
|
||||
let html = r#"<html><body>\
|
||||
<p>we're sorry, the request file are not found.</p>\
|
||||
</body></html>"#;
|
||||
let err = parse_manga_list(html).expect_err("expected Transient");
|
||||
assert!(err.is_transient(), "got non-transient: {err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_manga_list_ok_empty_when_logo_present_but_no_items() {
|
||||
// Last page of pagination, "no results" search, etc. Legitimately
|
||||
// empty must stay distinguishable from "page is broken".
|
||||
let html = r#"<html><body>\
|
||||
<header><div id="logo">Target</div></header>\
|
||||
<div id="left_side"><div class="pic_list"></div></div>\
|
||||
</body></html>"#;
|
||||
let refs = parse_manga_list(html).expect("logo present == not transient");
|
||||
assert!(refs.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_manga_detail_pulls_all_fields() {
|
||||
let m = parse_manga_detail(DETAIL_HTML, "test-key", true).expect("parse");
|
||||
@@ -761,7 +891,9 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn missing_optional_fields_parse_to_none() {
|
||||
let html = r#"<html><body><div class="w-title"><h1>Minimal</h1></div></body></html>"#;
|
||||
let html = r#"<html><body>\
|
||||
<header><div id="logo">Target</div></header>\
|
||||
<div class="w-title"><h1>Minimal</h1></div></body></html>"#;
|
||||
let m = parse_manga_detail(html, "min", true).unwrap();
|
||||
assert_eq!(m.title, "Minimal");
|
||||
assert!(m.summary.is_none());
|
||||
@@ -785,8 +917,77 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn parse_manga_detail_errors_on_missing_title() {
|
||||
let html = "<html><body><p>nothing</p></body></html>";
|
||||
// Logo present (page is alive) — failure here is a real parse
|
||||
// miss (Other), not Transient.
|
||||
let html = r#"<html><body>\
|
||||
<header><div id="logo">Target</div></header>\
|
||||
<p>nothing</p></body></html>"#;
|
||||
let err = parse_manga_detail(html, "x", true).unwrap_err();
|
||||
assert!(!err.is_transient(), "expected Other, got Transient: {err}");
|
||||
assert!(err.to_string().contains("missing .w-title h1"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classify_navigate_html_passes_normal_body_through() {
|
||||
let body = "<html><body><header><div id='logo'>Target</div></header>\
|
||||
<p>content</p></body></html>"
|
||||
.to_string();
|
||||
let out = classify_navigate_html(body.clone()).expect("ok");
|
||||
assert_eq!(out, body);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classify_navigate_html_returns_transient_for_broken_template() {
|
||||
let body = "<html><head></head><body>\
|
||||
<p>we're sorry, the request file are not found.</p>\
|
||||
</body></html>"
|
||||
.to_string();
|
||||
let err = classify_navigate_html(body).expect_err("expected Transient");
|
||||
assert!(err.is_transient(), "got non-transient: {err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_manga_detail_returns_transient_when_logo_missing() {
|
||||
// Broken-page response on a detail URL — must be reported as
|
||||
// Transient so the job is retried rather than logging "missing
|
||||
// .w-title h1" against a permanently-skipped manga.
|
||||
let html = "<html><body>\
|
||||
<p>we're sorry, the request file are not found.</p>\
|
||||
</body></html>";
|
||||
let err = parse_manga_detail(html, "x", true).expect_err("expected Transient");
|
||||
assert!(err.is_transient(), "got non-transient: {err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_page_order_backfill_is_last_to_one() {
|
||||
// Backfill walks pages oldest-first: queue is [last, last-1, ..., 1]
|
||||
// so popping from the front yields the last page first.
|
||||
let order = build_page_order(Some(3), true);
|
||||
assert_eq!(Vec::from(order), vec![3, 2, 1]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_page_order_incremental_is_one_to_last() {
|
||||
// Incremental walks newest-first in natural source order.
|
||||
let order = build_page_order(Some(3), false);
|
||||
assert_eq!(Vec::from(order), vec![1, 2, 3]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_page_order_falls_back_to_page_one_only_without_pagination() {
|
||||
let backfill = build_page_order(None, true);
|
||||
assert_eq!(Vec::from(backfill), vec![1]);
|
||||
let incremental = build_page_order(None, false);
|
||||
assert_eq!(Vec::from(incremental), vec![1]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_page_order_single_page_index_yields_one_entry() {
|
||||
// Sources with exactly one page should not yield duplicates
|
||||
// regardless of mode.
|
||||
let backfill = build_page_order(Some(1), true);
|
||||
assert_eq!(Vec::from(backfill), vec![1]);
|
||||
let incremental = build_page_order(Some(1), false);
|
||||
assert_eq!(Vec::from(incremental), vec![1]);
|
||||
}
|
||||
}
|
||||
|
||||
194
backend/src/crawler/url_utils.rs
Normal file
194
backend/src/crawler/url_utils.rs
Normal file
@@ -0,0 +1,194 @@
|
||||
//! Centralised URL helpers for the crawler subsystem.
|
||||
//!
|
||||
//! Three near-identical hand-rolled URL parsers used to live in
|
||||
//! `crawler::session`, `crawler::rate_limit`, and `crawler::pipeline`
|
||||
//! respectively, each with subtly different edge-case behaviour
|
||||
//! around port handling and IPv6 literals. They're consolidated here
|
||||
//! so the divergence can't drift again.
|
||||
//!
|
||||
//! The hand-rolled implementations are kept intentionally — they
|
||||
//! preserve the exact semantics every existing test pins. A future
|
||||
//! refactor can switch to `reqwest::Url` if it can be done without
|
||||
//! changing those semantics.
|
||||
|
||||
/// Lowercased host (no port). Returns `None` for inputs without a
|
||||
/// `scheme://host` shape — those would never have reached the network
|
||||
/// layer anyway. Used by the per-host rate limiter as its bucket key.
|
||||
///
|
||||
/// IPv6 literals are kept in their `[::1]` bracketed form so the
|
||||
/// `rsplit_once(':')` port-stripping logic doesn't split inside the
|
||||
/// address (e.g. `https://[::1]/foo` used to return `"[:"` because
|
||||
/// the rightmost `:` is inside the literal). Buckets keyed by
|
||||
/// `[::1]` vs `::1` are still uniquely-per-host; the brackets are
|
||||
/// cosmetic.
|
||||
pub fn host_of(url: &str) -> Option<String> {
|
||||
let after_scheme = url.split_once("://")?.1;
|
||||
let host_with_port = after_scheme.split('/').next()?;
|
||||
let host = if host_with_port.starts_with('[') {
|
||||
// IPv6 literal: keep through the closing bracket. There may
|
||||
// be a trailing `:port` after `]`; strip only that.
|
||||
match host_with_port.rfind(']') {
|
||||
Some(end) => &host_with_port[..=end],
|
||||
None => host_with_port,
|
||||
}
|
||||
} else {
|
||||
// Hostnames and IPv4 literals: trailing `:port` (if any) is
|
||||
// after the last `:`.
|
||||
host_with_port
|
||||
.rsplit_once(':')
|
||||
.map_or(host_with_port, |(h, _)| h)
|
||||
};
|
||||
(!host.is_empty()).then(|| host.to_ascii_lowercase())
|
||||
}
|
||||
|
||||
/// `scheme://host` with no path or port stripping. Used by the metadata
|
||||
/// pass to seed `sources.base_url` from `CRAWLER_START_URL`.
|
||||
pub fn origin_of(url: &str) -> Option<String> {
|
||||
let (scheme, rest) = url.split_once("://")?;
|
||||
let host = rest.split('/').next()?;
|
||||
Some(format!("{scheme}://{host}"))
|
||||
}
|
||||
|
||||
/// Approximate registrable-domain calculation: take the last two
|
||||
/// dot-labels of the host, prefix with `.`. Used to set a parent-
|
||||
/// domain cookie so the catalog's `www.` / `m.` redirects don't drop
|
||||
/// the cookie mid-crawl.
|
||||
///
|
||||
/// Caveat: wrong for multi-part TLDs (`.co.uk`, `.com.br`). The
|
||||
/// operator can override via `CRAWLER_COOKIE_DOMAIN`; pulling in the
|
||||
/// Public Suffix List for one knob isn't worth it yet.
|
||||
///
|
||||
/// Bare hostnames (e.g. `localhost`) return the host as-is, with no
|
||||
/// leading dot — setting `.localhost` as a cookie domain is invalid.
|
||||
/// IPv6 literals (e.g. `[::1]`) are returned bracketed and unchanged;
|
||||
/// the browser will reject them as a cookie `Domain` anyway, but the
|
||||
/// representation stays sensible. Same `starts_with('[')` branch as
|
||||
/// [`host_of`] for consistent IPv6 handling across the module.
|
||||
pub fn registrable_domain(url: &str) -> Option<String> {
|
||||
let after_scheme = url.split_once("://")?.1;
|
||||
let host_with_port = after_scheme.split('/').next()?;
|
||||
let host_str = if host_with_port.starts_with('[') {
|
||||
// IPv6 literal: keep through the closing bracket; an optional
|
||||
// `:port` follows `]`.
|
||||
match host_with_port.rfind(']') {
|
||||
Some(end) => &host_with_port[..=end],
|
||||
None => host_with_port,
|
||||
}
|
||||
} else {
|
||||
host_with_port
|
||||
.rsplit_once(':')
|
||||
.map_or(host_with_port, |(h, _)| h)
|
||||
};
|
||||
let host = host_str.to_ascii_lowercase();
|
||||
if host.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let labels: Vec<&str> = host.split('.').filter(|l| !l.is_empty()).collect();
|
||||
if labels.len() < 2 {
|
||||
return Some(host);
|
||||
}
|
||||
let registrable = &labels[labels.len() - 2..];
|
||||
Some(format!(".{}", registrable.join(".")))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn host_of_strips_port_and_lowercases() {
|
||||
assert_eq!(
|
||||
host_of("https://CDN.Example.com:443/x").as_deref(),
|
||||
Some("cdn.example.com")
|
||||
);
|
||||
assert_eq!(host_of("http://localhost/").as_deref(), Some("localhost"));
|
||||
assert_eq!(host_of("not a url"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn host_of_keeps_bracketed_ipv6_literal_intact() {
|
||||
// Regression: the old impl rsplit_once(':')'d the IPv6 address,
|
||||
// returning "[:" instead of "[::1]". A real IPv6 source would
|
||||
// silently get a wrong rate-limit bucket key.
|
||||
assert_eq!(host_of("https://[::1]/").as_deref(), Some("[::1]"));
|
||||
assert_eq!(host_of("https://[::1]:8080/").as_deref(), Some("[::1]"));
|
||||
assert_eq!(
|
||||
host_of("https://[2001:db8::1]/foo").as_deref(),
|
||||
Some("[2001:db8::1]")
|
||||
);
|
||||
assert_eq!(
|
||||
host_of("https://[2001:db8::1]:443/foo").as_deref(),
|
||||
Some("[2001:db8::1]")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn origin_of_returns_scheme_and_host() {
|
||||
assert_eq!(
|
||||
origin_of("https://example.com/some/path?q=1").as_deref(),
|
||||
Some("https://example.com")
|
||||
);
|
||||
assert_eq!(origin_of("garbage"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn registrable_domain_strips_subdomain() {
|
||||
assert_eq!(
|
||||
registrable_domain("https://www.target-site.com/manga/foo/").as_deref(),
|
||||
Some(".target-site.com")
|
||||
);
|
||||
assert_eq!(
|
||||
registrable_domain("https://m.example.org").as_deref(),
|
||||
Some(".example.org")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn registrable_domain_keeps_two_label_host() {
|
||||
assert_eq!(
|
||||
registrable_domain("https://example.com/").as_deref(),
|
||||
Some(".example.com")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn registrable_domain_handles_port() {
|
||||
assert_eq!(
|
||||
registrable_domain("http://www.foo.bar:8080/x").as_deref(),
|
||||
Some(".foo.bar")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn registrable_domain_bare_hostname_no_leading_dot() {
|
||||
assert_eq!(
|
||||
registrable_domain("http://localhost:5173").as_deref(),
|
||||
Some("localhost")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn registrable_domain_returns_none_for_garbage() {
|
||||
assert!(registrable_domain("not a url").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn registrable_domain_keeps_bracketed_ipv6_literal_intact() {
|
||||
// Symmetric with host_of's IPv6 fix. The cookie-domain code
|
||||
// won't accept an IP as a `Domain` value, but the function
|
||||
// should at least return a sensible representation rather
|
||||
// than the truncated `"[:"` the old port-stripper produced.
|
||||
assert_eq!(
|
||||
registrable_domain("https://[::1]/").as_deref(),
|
||||
Some("[::1]")
|
||||
);
|
||||
assert_eq!(
|
||||
registrable_domain("https://[::1]:8080/").as_deref(),
|
||||
Some("[::1]")
|
||||
);
|
||||
assert_eq!(
|
||||
registrable_domain("https://[2001:db8::1]/foo").as_deref(),
|
||||
Some("[2001:db8::1]")
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -21,6 +21,11 @@ pub enum AppError {
|
||||
PayloadTooLarge(String),
|
||||
#[error("unsupported media type: {0}")]
|
||||
UnsupportedMediaType(String),
|
||||
/// 429 with an optional `Retry-After` header value (in seconds).
|
||||
#[error("too many requests")]
|
||||
TooManyRequests {
|
||||
retry_after_secs: Option<u64>,
|
||||
},
|
||||
/// Semantic per-field validation failure. `details` is rendered into the
|
||||
/// envelope so the client can highlight the bad field(s).
|
||||
#[error("validation failed")]
|
||||
@@ -51,6 +56,7 @@ impl AppError {
|
||||
AppError::Conflict(_) => "conflict",
|
||||
AppError::PayloadTooLarge(_) => "payload_too_large",
|
||||
AppError::UnsupportedMediaType(_) => "unsupported_media_type",
|
||||
AppError::TooManyRequests { .. } => "too_many_requests",
|
||||
AppError::ValidationFailed { .. } => "validation_failed",
|
||||
AppError::Database(sqlx::Error::RowNotFound) => "not_found",
|
||||
AppError::Database(_) => "internal_error",
|
||||
@@ -79,6 +85,31 @@ impl IntoResponse for AppError {
|
||||
AppError::UnsupportedMediaType(msg) => {
|
||||
(StatusCode::UNSUPPORTED_MEDIA_TYPE, msg.clone(), None)
|
||||
}
|
||||
AppError::TooManyRequests { retry_after_secs } => {
|
||||
// Emit `Retry-After: N` (RFC 6585 §4) so a well-behaved
|
||||
// client can back off correctly. Done by building the
|
||||
// response by hand below — the `(status, headers,
|
||||
// body)` tuple shape doesn't fit the standard
|
||||
// `(status, body)` IntoResponse path for the other
|
||||
// variants.
|
||||
let body = json!({
|
||||
"error": {
|
||||
"code": code,
|
||||
"message": "too many requests; slow down",
|
||||
}
|
||||
});
|
||||
let mut resp = (StatusCode::TOO_MANY_REQUESTS, Json(body)).into_response();
|
||||
if let Some(secs) = retry_after_secs {
|
||||
// `HeaderValue: From<u64>` skips both the
|
||||
// intermediate `String` allocation and the
|
||||
// fallible-by-shape `from_str` path.
|
||||
resp.headers_mut().insert(
|
||||
axum::http::header::RETRY_AFTER,
|
||||
axum::http::HeaderValue::from(*secs),
|
||||
);
|
||||
}
|
||||
return resp;
|
||||
}
|
||||
AppError::ValidationFailed { message, details } => (
|
||||
StatusCode::UNPROCESSABLE_ENTITY,
|
||||
message.clone(),
|
||||
|
||||
@@ -17,10 +17,7 @@ async fn main() -> anyhow::Result<()> {
|
||||
tracing::info!(%addr, "mangalord listening");
|
||||
let listener = tokio::net::TcpListener::bind(addr).await?;
|
||||
axum::serve(listener, router)
|
||||
.with_graceful_shutdown(async {
|
||||
let _ = tokio::signal::ctrl_c().await;
|
||||
tracing::info!("ctrl-c received; shutting down");
|
||||
})
|
||||
.with_graceful_shutdown(shutdown_signal())
|
||||
.await?;
|
||||
|
||||
// Drain background tasks (crawler daemon) before exiting so Chromium
|
||||
@@ -30,3 +27,33 @@ async fn main() -> anyhow::Result<()> {
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Wait for either Ctrl-C (interactive shell) or SIGTERM (Docker /
|
||||
/// Kubernetes / Podman / systemd stop) and log which arrived. Without
|
||||
/// the SIGTERM branch, `docker compose stop` runs out its grace period
|
||||
/// and skips straight to SIGKILL — the daemon never gets the
|
||||
/// `daemon.shutdown().await` path, leaking Chromium.
|
||||
async fn shutdown_signal() {
|
||||
use tokio::signal::unix::{signal, SignalKind};
|
||||
let mut sigterm = match signal(SignalKind::terminate()) {
|
||||
Ok(s) => s,
|
||||
Err(e) => {
|
||||
// SignalKind::terminate() is supported on every Unix the
|
||||
// tokio runtime runs on; if registration fails we still
|
||||
// honour Ctrl-C so the process is at least
|
||||
// interactive-shutdownable.
|
||||
tracing::warn!(error = %e, "could not install SIGTERM handler; falling back to ctrl_c only");
|
||||
let _ = tokio::signal::ctrl_c().await;
|
||||
tracing::info!("ctrl-c received; shutting down");
|
||||
return;
|
||||
}
|
||||
};
|
||||
tokio::select! {
|
||||
_ = tokio::signal::ctrl_c() => {
|
||||
tracing::info!("ctrl-c received; shutting down");
|
||||
}
|
||||
_ = sigterm.recv() => {
|
||||
tracing::info!("SIGTERM received; shutting down");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -99,6 +99,11 @@ pub async fn list(
|
||||
/// Atomically replace the set of authors on a manga. Caller passes a
|
||||
/// `&mut PgConnection` (`&mut *tx` works) so the delete+upserts run in
|
||||
/// one transaction with whatever called us.
|
||||
///
|
||||
/// Note: `crawler::repo::sync_authors` does a similar replace with the
|
||||
/// same semantics on names. The duplication is intentional — handler
|
||||
/// callers want the `Vec<AuthorRef>` for the API response; the
|
||||
/// crawler doesn't need it and stays inside its own transaction.
|
||||
pub async fn set_for_manga(
|
||||
conn: &mut PgConnection,
|
||||
manga_id: Uuid,
|
||||
|
||||
@@ -29,9 +29,9 @@ pub async fn create(
|
||||
|
||||
match result {
|
||||
Ok(b) => Ok(b),
|
||||
Err(e) if is_unique_violation(&e) => Err(AppError::Conflict(
|
||||
"bookmark already exists for this manga/chapter".into(),
|
||||
)),
|
||||
Err(sqlx::Error::Database(ref db_err)) if db_err.is_unique_violation() => Err(
|
||||
AppError::Conflict("bookmark already exists for this manga/chapter".into()),
|
||||
),
|
||||
Err(e) => Err(AppError::Database(e)),
|
||||
}
|
||||
}
|
||||
@@ -97,10 +97,3 @@ pub async fn delete(pool: &PgPool, id: Uuid) -> AppResult<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn is_unique_violation(err: &sqlx::Error) -> bool {
|
||||
if let sqlx::Error::Database(db_err) = err {
|
||||
db_err.code().as_deref() == Some("23505")
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,7 +4,7 @@ use sqlx::{PgExecutor, PgPool};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::domain::Chapter;
|
||||
use crate::error::{AppError, AppResult};
|
||||
use crate::error::AppResult;
|
||||
|
||||
pub async fn list_for_manga(
|
||||
pool: &PgPool,
|
||||
@@ -62,10 +62,9 @@ pub async fn find_by_id_in_manga(
|
||||
///
|
||||
/// Chapter identity is the row UUID; the same (manga_id, number)
|
||||
/// combination can repeat (multiple translations, re-uploads). The
|
||||
/// `is_unique_violation` branch below is a defensive holdover from
|
||||
/// 0001's (manga_id, number) UNIQUE — it can no longer fire under
|
||||
/// normal operation, but we surface a clean 409 if a future migration
|
||||
/// re-adds any chapter uniqueness.
|
||||
/// 0013 migration dropped the (manga_id, number) UNIQUE, so duplicate
|
||||
/// inserts succeed by design. If a future migration re-adds any
|
||||
/// uniqueness, surface a 409 by adding a unique-violation arm here.
|
||||
pub async fn create<'e, E: PgExecutor<'e>>(
|
||||
executor: E,
|
||||
manga_id: Uuid,
|
||||
@@ -73,7 +72,7 @@ pub async fn create<'e, E: PgExecutor<'e>>(
|
||||
title: Option<&str>,
|
||||
uploaded_by: Option<Uuid>,
|
||||
) -> AppResult<Chapter> {
|
||||
let result = sqlx::query_as::<_, Chapter>(
|
||||
let row = sqlx::query_as::<_, Chapter>(
|
||||
r#"
|
||||
INSERT INTO chapters (manga_id, number, title, uploaded_by)
|
||||
VALUES ($1, $2, $3, $4)
|
||||
@@ -85,15 +84,58 @@ pub async fn create<'e, E: PgExecutor<'e>>(
|
||||
.bind(title)
|
||||
.bind(uploaded_by)
|
||||
.fetch_one(executor)
|
||||
.await;
|
||||
.await?;
|
||||
Ok(row)
|
||||
}
|
||||
|
||||
match result {
|
||||
Ok(c) => Ok(c),
|
||||
Err(e) if is_unique_violation(&e) => Err(AppError::Conflict(format!(
|
||||
"chapter {number} conflicts with an existing chapter for this manga"
|
||||
))),
|
||||
Err(e) => Err(AppError::Database(e)),
|
||||
}
|
||||
/// Cross-link guard for `POST /bookmarks`: the bookmarks FK accepts
|
||||
/// any valid chapter id, but a chapter must belong to the bookmark's
|
||||
/// manga or the bookmark would dangle on a foreign manga. Handlers
|
||||
/// call this before the insert and surface `NotFound` when it
|
||||
/// returns `false`.
|
||||
pub async fn belongs_to_manga(
|
||||
pool: &PgPool,
|
||||
chapter_id: Uuid,
|
||||
manga_id: Uuid,
|
||||
) -> AppResult<bool> {
|
||||
let (exists,): (bool,) = sqlx::query_as(
|
||||
"SELECT EXISTS(SELECT 1 FROM chapters WHERE id = $1 AND manga_id = $2)",
|
||||
)
|
||||
.bind(chapter_id)
|
||||
.bind(manga_id)
|
||||
.fetch_one(pool)
|
||||
.await?;
|
||||
Ok(exists)
|
||||
}
|
||||
|
||||
/// Read just the page_count for a chapter. Used by the crawler
|
||||
/// daemon's consumer-side dedup safety net so it can ack-done a job
|
||||
/// whose chapter has already been fetched by a racing worker.
|
||||
pub async fn page_count(pool: &PgPool, id: Uuid) -> sqlx::Result<Option<i32>> {
|
||||
sqlx::query_scalar("SELECT page_count FROM chapters WHERE id = $1")
|
||||
.bind(id)
|
||||
.fetch_optional(pool)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Look up the manga_id + most recent source_url for a chapter. Used
|
||||
/// by the daemon's chapter dispatcher to resolve the URL it needs to
|
||||
/// hand to `content::sync_chapter_content`. Returns `None` if the
|
||||
/// chapter (or its source row) is gone.
|
||||
pub async fn dispatch_target(
|
||||
pool: &PgPool,
|
||||
chapter_id: Uuid,
|
||||
) -> sqlx::Result<Option<(Uuid, String)>> {
|
||||
sqlx::query_as(
|
||||
"SELECT c.manga_id, cs.source_url \
|
||||
FROM chapters c \
|
||||
JOIN chapter_sources cs ON cs.chapter_id = c.id \
|
||||
WHERE c.id = $1 \
|
||||
LIMIT 1",
|
||||
)
|
||||
.bind(chapter_id)
|
||||
.fetch_optional(pool)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn set_page_count<'e, E: PgExecutor<'e>>(
|
||||
@@ -109,10 +151,3 @@ pub async fn set_page_count<'e, E: PgExecutor<'e>>(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn is_unique_violation(err: &sqlx::Error) -> bool {
|
||||
if let sqlx::Error::Database(db_err) = err {
|
||||
db_err.code().as_deref() == Some("23505")
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
@@ -274,7 +274,20 @@ async fn sync_tags(
|
||||
manga_id: Uuid,
|
||||
tags: &[String],
|
||||
) -> sqlx::Result<()> {
|
||||
sqlx::query("DELETE FROM manga_tags WHERE manga_id = $1")
|
||||
// Only clear crawler-owned attachments (added_by IS NULL). User-
|
||||
// attached tags are owned by the attaching user and must survive
|
||||
// the recurring metadata pass — see manga_tags.added_by in
|
||||
// migration 0009.
|
||||
//
|
||||
// Note on orphans: `manga_tags.added_by` is `ON DELETE SET NULL`,
|
||||
// so an attachment whose user was deleted becomes
|
||||
// indistinguishable from a crawler-owned row and is cleaned up
|
||||
// here. That mirrors how `api::mangas::detach_tag` already treats
|
||||
// orphans ("nobody owns it, refuse to let anyone but admin clear
|
||||
// them") — the crawler now becomes the eventual reaper. Tracked
|
||||
// by `sync_tags_garbage_collects_orphan_user_attachments` in
|
||||
// backend/tests/crawler_sync.rs.
|
||||
sqlx::query("DELETE FROM manga_tags WHERE manga_id = $1 AND added_by IS NULL")
|
||||
.bind(manga_id)
|
||||
.execute(&mut **tx)
|
||||
.await?;
|
||||
@@ -412,6 +425,53 @@ pub async fn sync_manga_chapters(
|
||||
Ok(diff)
|
||||
}
|
||||
|
||||
/// Record that a complete Backfill walk has finished for `source_id`.
|
||||
/// The presence of this row is what the daemon's mode auto-detection
|
||||
/// uses to flip from Backfill to Incremental on subsequent ticks.
|
||||
///
|
||||
/// Keyed `seed_completed:<source_id>` in `crawler_state`. JSON payload
|
||||
/// stores the timestamp so we can surface "last fully reseeded at" in
|
||||
/// future ops tooling without another migration.
|
||||
pub async fn mark_seed_completed(
|
||||
pool: &PgPool,
|
||||
source_id: &str,
|
||||
at: DateTime<Utc>,
|
||||
) -> sqlx::Result<()> {
|
||||
let key = format!("seed_completed:{source_id}");
|
||||
sqlx::query(
|
||||
"INSERT INTO crawler_state (key, value, updated_at) \
|
||||
VALUES ($1, $2, now()) \
|
||||
ON CONFLICT (key) DO UPDATE \
|
||||
SET value = EXCLUDED.value, updated_at = now()",
|
||||
)
|
||||
.bind(&key)
|
||||
.bind(serde_json::json!({ "at": at.to_rfc3339() }))
|
||||
.execute(pool)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Read the timestamp written by [`mark_seed_completed`], if any.
|
||||
/// `None` means no complete Backfill has ever finished for this
|
||||
/// source — the daemon should run Backfill on the next tick.
|
||||
pub async fn seed_completed_at(
|
||||
pool: &PgPool,
|
||||
source_id: &str,
|
||||
) -> sqlx::Result<Option<DateTime<Utc>>> {
|
||||
let key = format!("seed_completed:{source_id}");
|
||||
let row: Option<serde_json::Value> =
|
||||
sqlx::query_scalar("SELECT value FROM crawler_state WHERE key = $1")
|
||||
.bind(&key)
|
||||
.fetch_optional(pool)
|
||||
.await?;
|
||||
Ok(row.and_then(|v| {
|
||||
v.get("at")
|
||||
.and_then(|s| s.as_str())
|
||||
.and_then(|s| DateTime::parse_from_rfc3339(s).ok())
|
||||
.map(|dt| dt.with_timezone(&Utc))
|
||||
}))
|
||||
}
|
||||
|
||||
pub async fn mark_dropped_mangas(
|
||||
pool: &PgPool,
|
||||
source_id: &str,
|
||||
|
||||
@@ -61,6 +61,11 @@ pub async fn load_for_mangas(
|
||||
/// FK constraint would reject them, so we filter upstream rather than
|
||||
/// surface a 500 here. (The API layer validates the set against
|
||||
/// `list_all` first.)
|
||||
///
|
||||
/// Note: `crawler::repo::sync_genres` does a similar replace, but by
|
||||
/// *name* and with auto-create of unseen genres — the crawler can't
|
||||
/// validate against the curated vocabulary on its own. Both paths are
|
||||
/// intentional; don't merge them without preserving that semantic.
|
||||
pub async fn set_for_manga(
|
||||
conn: &mut PgConnection,
|
||||
manga_id: Uuid,
|
||||
|
||||
@@ -262,6 +262,17 @@ pub async fn set_cover_image_path<'e, E: PgExecutor<'e>>(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn clear_cover_image_path<'e, E: PgExecutor<'e>>(
|
||||
executor: E,
|
||||
id: Uuid,
|
||||
) -> AppResult<()> {
|
||||
sqlx::query("UPDATE mangas SET cover_image_path = NULL, updated_at = now() WHERE id = $1")
|
||||
.bind(id)
|
||||
.execute(executor)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn exists(pool: &PgPool, id: Uuid) -> AppResult<bool> {
|
||||
let (exists,): (bool,) =
|
||||
sqlx::query_as("SELECT EXISTS(SELECT 1 FROM mangas WHERE id = $1)")
|
||||
@@ -270,3 +281,17 @@ pub async fn exists(pool: &PgPool, id: Uuid) -> AppResult<bool> {
|
||||
.await?;
|
||||
Ok(exists)
|
||||
}
|
||||
|
||||
/// Returns the uploader's user id for a manga. `None` either when the
|
||||
/// manga doesn't exist or when the row predates the `uploaded_by`
|
||||
/// column (historical NULL — see migration 0011). Callers must
|
||||
/// distinguish "manga missing" via [`exists`] before relying on this
|
||||
/// to make an authz decision.
|
||||
pub async fn uploaded_by(pool: &PgPool, id: Uuid) -> AppResult<Option<Uuid>> {
|
||||
let row: Option<(Option<Uuid>,)> =
|
||||
sqlx::query_as("SELECT uploaded_by FROM mangas WHERE id = $1")
|
||||
.bind(id)
|
||||
.fetch_optional(pool)
|
||||
.await?;
|
||||
Ok(row.and_then(|(u,)| u))
|
||||
}
|
||||
|
||||
@@ -21,7 +21,7 @@ pub async fn create(pool: &PgPool, username: &str, password_hash: &str) -> AppRe
|
||||
|
||||
match result {
|
||||
Ok(user) => Ok(user),
|
||||
Err(e) if is_unique_violation(&e) => {
|
||||
Err(sqlx::Error::Database(ref db_err)) if db_err.is_unique_violation() => {
|
||||
Err(AppError::Conflict("username is already taken".into()))
|
||||
}
|
||||
Err(e) => Err(AppError::Database(e)),
|
||||
@@ -56,10 +56,3 @@ pub async fn find_by_id(pool: &PgPool, id: Uuid) -> AppResult<Option<User>> {
|
||||
Ok(row)
|
||||
}
|
||||
|
||||
fn is_unique_violation(err: &sqlx::Error) -> bool {
|
||||
if let sqlx::Error::Database(db_err) = err {
|
||||
db_err.code().as_deref() == Some("23505")
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,6 +16,13 @@ impl LocalStorage {
|
||||
}
|
||||
|
||||
fn resolve(&self, key: &str) -> Result<PathBuf, StorageError> {
|
||||
// NUL bytes are rejected by the Linux syscall layer, but the
|
||||
// error surfaces as an opaque IO failure rather than the
|
||||
// explicit `BadKey` the rest of the contract uses. Catch it
|
||||
// here so the error path is consistent.
|
||||
if key.contains('\0') {
|
||||
return Err(StorageError::BadKey);
|
||||
}
|
||||
let key = key.trim_start_matches('/');
|
||||
if key.is_empty() {
|
||||
return Err(StorageError::BadKey);
|
||||
@@ -114,6 +121,9 @@ mod tests {
|
||||
assert!(matches!(s.get(".").await, Err(StorageError::BadKey)));
|
||||
// Empty segment via doubled slash.
|
||||
assert!(matches!(s.get("a//b").await, Err(StorageError::BadKey)));
|
||||
// NUL byte (rejected explicitly so callers see BadKey rather
|
||||
// than an opaque IO error from the kernel).
|
||||
assert!(matches!(s.put("a\0b", b"x").await, Err(StorageError::BadKey)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -567,6 +567,166 @@ async fn user_a_cannot_delete_user_b_token(pool: PgPool) {
|
||||
assert_eq!(resp.status(), StatusCode::NO_CONTENT);
|
||||
}
|
||||
|
||||
/// Username enumeration via login response time: an attacker probes
|
||||
/// for valid usernames by measuring how long /auth/login takes. Before
|
||||
/// the equalisation fix, the no-user branch returned 401 in <1 ms
|
||||
/// while the wrong-password branch took ~50-100 ms (the argon2 verify
|
||||
/// cost). This test asserts the no-user branch now spends at least
|
||||
/// some meaningful fraction of the wrong-password branch's time.
|
||||
///
|
||||
/// Tolerance is intentionally loose so CI variance doesn't flap the
|
||||
/// test. The unequalised gap is large enough (~50x) that even a noisy
|
||||
/// CI run with a 5x slack still catches it.
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn login_no_user_branch_runs_argon2_for_timing_equalisation(pool: PgPool) {
|
||||
use std::time::Instant;
|
||||
|
||||
let h = common::harness(pool);
|
||||
|
||||
// Register the victim user so the wrong-password branch has a real
|
||||
// argon2 hash to verify against.
|
||||
let _ = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::post_json(
|
||||
"/api/v1/auth/register",
|
||||
json!({ "username": "victim", "password": "hunter2hunter2" }),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Warm-up: first login of the process initialises the dummy hash
|
||||
// lazily. Skip that cost when measuring.
|
||||
let _ = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::post_json(
|
||||
"/api/v1/auth/login",
|
||||
json!({ "username": "victim", "password": "wrong" }),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let _ = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::post_json(
|
||||
"/api/v1/auth/login",
|
||||
json!({ "username": "ghost", "password": "wrong" }),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Median-of-N is more stable than a single sample.
|
||||
async fn sample_min(
|
||||
app: &axum::Router,
|
||||
username: &str,
|
||||
n: u32,
|
||||
) -> std::time::Duration {
|
||||
let mut samples = Vec::with_capacity(n as usize);
|
||||
for _ in 0..n {
|
||||
let req = common::post_json(
|
||||
"/api/v1/auth/login",
|
||||
json!({ "username": username, "password": "wrong-guess" }),
|
||||
);
|
||||
let t = Instant::now();
|
||||
let resp = app.clone().oneshot(req).await.unwrap();
|
||||
let d = t.elapsed();
|
||||
assert_eq!(resp.status(), StatusCode::UNAUTHORIZED);
|
||||
samples.push(d);
|
||||
}
|
||||
// Use the minimum: it's the floor that argon2 takes, robust
|
||||
// against unrelated stalls (DB connection acquisition, etc.).
|
||||
*samples.iter().min().unwrap()
|
||||
}
|
||||
|
||||
let wrong_pwd = sample_min(&h.app, "victim", 3).await;
|
||||
let no_user = sample_min(&h.app, "ghost", 3).await;
|
||||
|
||||
// 5x slack: argon2 dominates both branches, so they should be
|
||||
// within an order of magnitude. Unequalised, no_user would be
|
||||
// ~50-100x faster. Asserting "no_user >= wrong_pwd / 5" catches
|
||||
// the bug without being flaky in CI.
|
||||
assert!(
|
||||
no_user * 5 >= wrong_pwd,
|
||||
"login timing leaks user existence: no_user={no_user:?}, wrong_pwd={wrong_pwd:?}"
|
||||
);
|
||||
}
|
||||
|
||||
/// Brute-force / spray protection: at default production limits, a
|
||||
/// tight loop of /auth/login attempts should burst through the bucket
|
||||
/// and then 429 every subsequent request until the bucket refills.
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn login_rate_limited_under_burst_pressure(pool: PgPool) {
|
||||
let h = common::harness_with_auth_rate_limit(pool, 1, 3);
|
||||
|
||||
// Register a victim so the wrong-password branch is real work.
|
||||
let _ = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::post_json("/api/v1/auth/register", creds("victim")))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Register consumed one token from the burst-3 bucket. Fire 30
|
||||
// wrong-password logins back-to-back; with per_sec=1 the refill
|
||||
// is too slow to keep up and at least one must come back 429.
|
||||
let mut saw_429 = false;
|
||||
for _ in 0..30 {
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::post_json(
|
||||
"/api/v1/auth/login",
|
||||
json!({ "username": "victim", "password": "wrong" }),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
if resp.status() == StatusCode::TOO_MANY_REQUESTS {
|
||||
// RFC 6585 §4: 429 SHOULD include a Retry-After header. The
|
||||
// value is in seconds; with per_sec=1 the bucket needs ~1s
|
||||
// to refill, so the header should be 1 or 2.
|
||||
let retry_after = resp
|
||||
.headers()
|
||||
.get(axum::http::header::RETRY_AFTER)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.and_then(|s| s.parse::<u32>().ok())
|
||||
.expect("Retry-After header present and numeric");
|
||||
assert!(
|
||||
retry_after >= 1,
|
||||
"Retry-After must be at least 1s, got {retry_after}"
|
||||
);
|
||||
let body = common::body_json(resp).await;
|
||||
assert_eq!(body["error"]["code"], "too_many_requests");
|
||||
saw_429 = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
saw_429,
|
||||
"expected at least one 429 within 30 rapid login attempts"
|
||||
);
|
||||
}
|
||||
|
||||
/// Default (test-harness) limits are disabled, so existing tests that
|
||||
/// fire multiple auth requests don't start failing.
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn default_test_harness_does_not_rate_limit(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
for i in 0..50 {
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::post_json(
|
||||
"/api/v1/auth/login",
|
||||
json!({ "username": format!("nobody-{i}"), "password": "x" }),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
// None of these should be 429 — only 401.
|
||||
assert_eq!(resp.status(), StatusCode::UNAUTHORIZED, "iter {i}");
|
||||
}
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn delete_unknown_token_is_404(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
@@ -581,3 +741,27 @@ async fn delete_unknown_token_is_404(pool: PgPool) {
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::NOT_FOUND);
|
||||
}
|
||||
|
||||
/// Bot token names are user-supplied free-form strings; a 10 MB name
|
||||
/// was accepted before. Cap at 64 chars to match the other free-form
|
||||
/// identifier caps (tags, collection names). The response uses
|
||||
/// `ValidationFailed` (422 with per-field details) so clients can
|
||||
/// render the same shape they already handle for `attach_tag`.
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn create_token_rejects_name_over_64_chars(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
let (_, cookie) = common::register_user(&h.app).await;
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::post_json_with_cookie(
|
||||
"/api/v1/auth/tokens",
|
||||
json!({ "name": "x".repeat(65) }),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::UNPROCESSABLE_ENTITY);
|
||||
let body = common::body_json(resp).await;
|
||||
assert_eq!(body["error"]["code"], "validation_failed");
|
||||
assert!(body["error"]["details"]["name"].is_string());
|
||||
}
|
||||
|
||||
462
backend/tests/api_mangas_cover.rs
Normal file
462
backend/tests/api_mangas_cover.rs
Normal file
@@ -0,0 +1,462 @@
|
||||
mod common;
|
||||
|
||||
use axum::http::StatusCode;
|
||||
use serde_json::{json, Value};
|
||||
use sqlx::PgPool;
|
||||
use tower::ServiceExt;
|
||||
use uuid::Uuid;
|
||||
|
||||
use common::{
|
||||
body_json, delete_with_cookie, fake_jpeg_bytes, fake_png_bytes, get, harness,
|
||||
post_multipart_with_cookie, put_multipart, put_multipart_with_cookie, register_user,
|
||||
MultipartBuilder,
|
||||
};
|
||||
|
||||
async fn create_manga_with_cover(
|
||||
app: &axum::Router,
|
||||
cookie: &str,
|
||||
title: &str,
|
||||
cover: Option<(&str, &[u8])>,
|
||||
) -> Value {
|
||||
let mut form =
|
||||
MultipartBuilder::new().add_json("metadata", json!({ "title": title }));
|
||||
if let Some((ct, bytes)) = cover {
|
||||
form = form.add_file("cover", "cover.bin", ct, bytes);
|
||||
}
|
||||
let resp = app
|
||||
.clone()
|
||||
.oneshot(post_multipart_with_cookie("/api/v1/mangas", form, cookie))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
resp.status(),
|
||||
StatusCode::CREATED,
|
||||
"seed create_manga failed: {:?}",
|
||||
resp.status()
|
||||
);
|
||||
body_json(resp).await
|
||||
}
|
||||
|
||||
fn id_of(body: &Value) -> Uuid {
|
||||
Uuid::parse_str(body["id"].as_str().unwrap()).unwrap()
|
||||
}
|
||||
|
||||
fn cover_form(bytes: &[u8]) -> MultipartBuilder {
|
||||
MultipartBuilder::new().add_file("cover", "cover.bin", "application/octet-stream", bytes)
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn put_cover_sets_path_when_none_existed(pool: PgPool) {
|
||||
let h = harness(pool);
|
||||
let (_, cookie) = register_user(&h.app).await;
|
||||
let manga = create_manga_with_cover(&h.app, &cookie, "Cover Me", None).await;
|
||||
let id = id_of(&manga);
|
||||
assert!(manga["cover_image_path"].is_null());
|
||||
|
||||
let bytes = fake_png_bytes();
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(put_multipart_with_cookie(
|
||||
&format!("/api/v1/mangas/{id}/cover"),
|
||||
cover_form(&bytes),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = body_json(resp).await;
|
||||
let expected_key = format!("mangas/{id}/cover.png");
|
||||
assert_eq!(body["cover_image_path"], expected_key);
|
||||
assert_eq!(body["title"], "Cover Me");
|
||||
|
||||
let file_resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get(&format!("/api/v1/files/{expected_key}")))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(file_resp.status(), StatusCode::OK);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn put_cover_replaces_existing_same_extension(pool: PgPool) {
|
||||
let h = harness(pool);
|
||||
let (_, cookie) = register_user(&h.app).await;
|
||||
let original = fake_png_bytes();
|
||||
let manga = create_manga_with_cover(
|
||||
&h.app,
|
||||
&cookie,
|
||||
"Replace Me",
|
||||
Some(("image/png", &original)),
|
||||
)
|
||||
.await;
|
||||
let id = id_of(&manga);
|
||||
let original_key = format!("mangas/{id}/cover.png");
|
||||
assert_eq!(manga["cover_image_path"], original_key);
|
||||
|
||||
let mut replacement = fake_png_bytes();
|
||||
replacement.extend_from_slice(b"-replacement-marker");
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(put_multipart_with_cookie(
|
||||
&format!("/api/v1/mangas/{id}/cover"),
|
||||
cover_form(&replacement),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = body_json(resp).await;
|
||||
assert_eq!(body["cover_image_path"], original_key);
|
||||
|
||||
let file_resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get(&format!("/api/v1/files/{original_key}")))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(file_resp.status(), StatusCode::OK);
|
||||
let body_bytes = http_body_util::BodyExt::collect(file_resp.into_body())
|
||||
.await
|
||||
.unwrap()
|
||||
.to_bytes();
|
||||
assert_eq!(body_bytes.as_ref(), replacement.as_slice());
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn put_cover_replaces_existing_different_extension_and_deletes_old_blob(pool: PgPool) {
|
||||
let h = harness(pool);
|
||||
let (_, cookie) = register_user(&h.app).await;
|
||||
let png = fake_png_bytes();
|
||||
let manga = create_manga_with_cover(
|
||||
&h.app,
|
||||
&cookie,
|
||||
"Switch Ext",
|
||||
Some(("image/png", &png)),
|
||||
)
|
||||
.await;
|
||||
let id = id_of(&manga);
|
||||
let old_key = format!("mangas/{id}/cover.png");
|
||||
assert_eq!(manga["cover_image_path"], old_key);
|
||||
|
||||
let jpeg = fake_jpeg_bytes();
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(put_multipart_with_cookie(
|
||||
&format!("/api/v1/mangas/{id}/cover"),
|
||||
cover_form(&jpeg),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = body_json(resp).await;
|
||||
let new_key = format!("mangas/{id}/cover.jpg");
|
||||
assert_eq!(body["cover_image_path"], new_key);
|
||||
|
||||
let new_file = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get(&format!("/api/v1/files/{new_key}")))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(new_file.status(), StatusCode::OK);
|
||||
|
||||
let old_file = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get(&format!("/api/v1/files/{old_key}")))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(old_file.status(), StatusCode::NOT_FOUND);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn put_cover_rejects_unauthenticated(pool: PgPool) {
|
||||
let h = harness(pool);
|
||||
let (_, cookie) = register_user(&h.app).await;
|
||||
let manga = create_manga_with_cover(&h.app, &cookie, "Public Read", None).await;
|
||||
let id = id_of(&manga);
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(put_multipart(
|
||||
&format!("/api/v1/mangas/{id}/cover"),
|
||||
cover_form(&fake_png_bytes()),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::UNAUTHORIZED);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn put_cover_404_on_unknown_id(pool: PgPool) {
|
||||
let h = harness(pool);
|
||||
let (_, cookie) = register_user(&h.app).await;
|
||||
let id = Uuid::new_v4();
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(put_multipart_with_cookie(
|
||||
&format!("/api/v1/mangas/{id}/cover"),
|
||||
cover_form(&fake_png_bytes()),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::NOT_FOUND);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn put_cover_rejects_non_image_with_unsupported_media_type(pool: PgPool) {
|
||||
let h = harness(pool);
|
||||
let (_, cookie) = register_user(&h.app).await;
|
||||
let manga = create_manga_with_cover(&h.app, &cookie, "Not Image", None).await;
|
||||
let id = id_of(&manga);
|
||||
|
||||
let pdf = b"%PDF-1.4\n%\xc4\xe5".to_vec();
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(put_multipart_with_cookie(
|
||||
&format!("/api/v1/mangas/{id}/cover"),
|
||||
cover_form(&pdf),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::UNSUPPORTED_MEDIA_TYPE);
|
||||
let body = body_json(resp).await;
|
||||
assert_eq!(body["error"]["code"], "unsupported_media_type");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn put_cover_rejects_oversized(pool: PgPool) {
|
||||
let h = harness(pool);
|
||||
let (_, cookie) = register_user(&h.app).await;
|
||||
let manga = create_manga_with_cover(&h.app, &cookie, "Too Big", None).await;
|
||||
let id = id_of(&manga);
|
||||
|
||||
// Harness max_file_bytes is 256 KiB; 300 KiB trips the cap.
|
||||
let mut bytes = fake_png_bytes();
|
||||
bytes.resize(300 * 1024, 0);
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(put_multipart_with_cookie(
|
||||
&format!("/api/v1/mangas/{id}/cover"),
|
||||
cover_form(&bytes),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::PAYLOAD_TOO_LARGE);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn put_cover_rejects_missing_cover_part(pool: PgPool) {
|
||||
let h = harness(pool);
|
||||
let (_, cookie) = register_user(&h.app).await;
|
||||
let manga = create_manga_with_cover(&h.app, &cookie, "Empty Form", None).await;
|
||||
let id = id_of(&manga);
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(put_multipart_with_cookie(
|
||||
&format!("/api/v1/mangas/{id}/cover"),
|
||||
MultipartBuilder::new(),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::UNPROCESSABLE_ENTITY);
|
||||
let body = body_json(resp).await;
|
||||
assert_eq!(body["error"]["code"], "validation_failed");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn put_cover_preserves_other_metadata(pool: PgPool) {
|
||||
let h = harness(pool);
|
||||
let (_, cookie) = register_user(&h.app).await;
|
||||
let manga = create_manga_with_cover(
|
||||
&h.app,
|
||||
&cookie,
|
||||
"Keep My Fields",
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
let id = id_of(&manga);
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(put_multipart_with_cookie(
|
||||
&format!("/api/v1/mangas/{id}/cover"),
|
||||
cover_form(&fake_png_bytes()),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = body_json(resp).await;
|
||||
assert_eq!(body["title"], "Keep My Fields");
|
||||
assert_eq!(body["status"], "ongoing");
|
||||
assert_eq!(body["authors"], json!([]));
|
||||
assert_eq!(body["genres"], json!([]));
|
||||
assert_eq!(body["tags"], json!([]));
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn delete_cover_clears_path_and_removes_blob(pool: PgPool) {
|
||||
let h = harness(pool);
|
||||
let (_, cookie) = register_user(&h.app).await;
|
||||
let png = fake_png_bytes();
|
||||
let manga = create_manga_with_cover(
|
||||
&h.app,
|
||||
&cookie,
|
||||
"Bye Cover",
|
||||
Some(("image/png", &png)),
|
||||
)
|
||||
.await;
|
||||
let id = id_of(&manga);
|
||||
let key = format!("mangas/{id}/cover.png");
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(delete_with_cookie(
|
||||
&format!("/api/v1/mangas/{id}/cover"),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = body_json(resp).await;
|
||||
assert!(body["cover_image_path"].is_null());
|
||||
assert_eq!(body["title"], "Bye Cover");
|
||||
|
||||
let file_resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(get(&format!("/api/v1/files/{key}")))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(file_resp.status(), StatusCode::NOT_FOUND);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn delete_cover_is_idempotent_when_no_cover_present(pool: PgPool) {
|
||||
let h = harness(pool);
|
||||
let (_, cookie) = register_user(&h.app).await;
|
||||
let manga = create_manga_with_cover(&h.app, &cookie, "Never Had One", None).await;
|
||||
let id = id_of(&manga);
|
||||
|
||||
for _ in 0..2 {
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(delete_with_cookie(
|
||||
&format!("/api/v1/mangas/{id}/cover"),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = body_json(resp).await;
|
||||
assert!(body["cover_image_path"].is_null());
|
||||
}
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn delete_cover_rejects_unauthenticated(pool: PgPool) {
|
||||
let h = harness(pool);
|
||||
let (_, cookie) = register_user(&h.app).await;
|
||||
let manga = create_manga_with_cover(&h.app, &cookie, "Locked", None).await;
|
||||
let id = id_of(&manga);
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(
|
||||
axum::http::Request::builder()
|
||||
.method("DELETE")
|
||||
.uri(format!("/api/v1/mangas/{id}/cover"))
|
||||
.body(axum::body::Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::UNAUTHORIZED);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn delete_cover_404_on_unknown_id(pool: PgPool) {
|
||||
let h = harness(pool);
|
||||
let (_, cookie) = register_user(&h.app).await;
|
||||
let id = Uuid::new_v4();
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(delete_with_cookie(
|
||||
&format!("/api/v1/mangas/{id}/cover"),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::NOT_FOUND);
|
||||
}
|
||||
|
||||
/// Authz: PUT /mangas/:id/cover must be uploader-only.
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn put_cover_forbidden_for_non_uploader(pool: PgPool) {
|
||||
let h = harness(pool);
|
||||
let (_, owner_cookie) = register_user(&h.app).await;
|
||||
let (_, intruder_cookie) = register_user(&h.app).await;
|
||||
|
||||
let manga =
|
||||
create_manga_with_cover(&h.app, &owner_cookie, "Mine", None).await;
|
||||
let id = id_of(&manga);
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(put_multipart_with_cookie(
|
||||
&format!("/api/v1/mangas/{id}/cover"),
|
||||
cover_form(&fake_png_bytes()),
|
||||
&intruder_cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
|
||||
}
|
||||
|
||||
/// Authz: DELETE /mangas/:id/cover must be uploader-only.
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn delete_cover_forbidden_for_non_uploader(pool: PgPool) {
|
||||
let h = harness(pool);
|
||||
let (_, owner_cookie) = register_user(&h.app).await;
|
||||
let (_, intruder_cookie) = register_user(&h.app).await;
|
||||
|
||||
let manga = create_manga_with_cover(
|
||||
&h.app,
|
||||
&owner_cookie,
|
||||
"Mine",
|
||||
Some(("image/jpeg", &fake_jpeg_bytes())),
|
||||
)
|
||||
.await;
|
||||
let id = id_of(&manga);
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(delete_with_cookie(
|
||||
&format!("/api/v1/mangas/{id}/cover"),
|
||||
&intruder_cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
|
||||
}
|
||||
@@ -566,3 +566,78 @@ async fn patch_requires_authentication(pool: PgPool) {
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::UNAUTHORIZED);
|
||||
}
|
||||
|
||||
/// A signed-in user who didn't upload the manga must not be able to
|
||||
/// PATCH it. Without the uploader-gate this returned 200 — see
|
||||
/// REVIEW.md "manga PATCH / cover endpoints don't check ownership".
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn patch_forbidden_for_non_uploader(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
let (_, owner_cookie) = common::register_user(&h.app).await;
|
||||
let (_, intruder_cookie) = common::register_user(&h.app).await;
|
||||
|
||||
let created = create_manga(&h.app, &owner_cookie, json!({ "title": "Mine" })).await;
|
||||
let id = id_of(&created);
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::patch_json_with_cookie(
|
||||
&format!("/api/v1/mangas/{id}"),
|
||||
json!({ "status": "completed" }),
|
||||
&intruder_cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
|
||||
}
|
||||
|
||||
/// Owner can still edit their own manga (regression guard for the
|
||||
/// authz fix).
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn patch_allowed_for_uploader(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
let (_, cookie) = common::register_user(&h.app).await;
|
||||
let created = create_manga(&h.app, &cookie, json!({ "title": "Owned" })).await;
|
||||
let id = id_of(&created);
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::patch_json_with_cookie(
|
||||
&format!("/api/v1/mangas/{id}"),
|
||||
json!({ "status": "completed" }),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
}
|
||||
|
||||
/// Legacy rows with `uploaded_by IS NULL` (created before migration
|
||||
/// 0011) remain editable by any signed-in user. Without this carve-out
|
||||
/// the historical-data note in 0011 would be broken.
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn patch_allowed_on_legacy_null_uploader(pool: PgPool) {
|
||||
let h = common::harness(pool.clone());
|
||||
let (_, cookie) = common::register_user(&h.app).await;
|
||||
let created = create_manga(&h.app, &cookie, json!({ "title": "Legacy" })).await;
|
||||
let id = id_of(&created);
|
||||
|
||||
// Simulate a row uploaded before the column existed: clear
|
||||
// uploaded_by directly via SQL.
|
||||
sqlx::query("UPDATE mangas SET uploaded_by = NULL WHERE id = $1")
|
||||
.bind(id)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let (_, other_cookie) = common::register_user(&h.app).await;
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::patch_json_with_cookie(
|
||||
&format!("/api/v1/mangas/{id}"),
|
||||
json!({ "status": "completed" }),
|
||||
&other_cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
}
|
||||
|
||||
@@ -59,6 +59,31 @@ async fn reattach_same_tag_is_idempotent_and_returns_200(pool: PgPool) {
|
||||
assert_eq!(second.status(), StatusCode::OK);
|
||||
}
|
||||
|
||||
/// Tag names over 64 chars are rejected at the handler boundary. The
|
||||
/// repo enforces the same cap, but doing it at the handler keeps the
|
||||
/// envelope consistent with the other validation paths
|
||||
/// (username, collection name, etc.).
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn attach_rejects_tag_name_over_64_chars(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
let (_, cookie) = common::register_user(&h.app).await;
|
||||
let manga_id = common::seed_manga_via_api(&h.app, &cookie, "Berserk").await;
|
||||
|
||||
let long_name: String = "x".repeat(65);
|
||||
let resp = h
|
||||
.app
|
||||
.oneshot(common::post_json_with_cookie(
|
||||
&format!("/api/v1/mangas/{manga_id}/tags"),
|
||||
json!({ "name": long_name }),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::UNPROCESSABLE_ENTITY);
|
||||
let body = common::body_json(resp).await;
|
||||
assert_eq!(body["error"]["code"], "validation_failed");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn tag_names_dedup_case_insensitively(pool: PgPool) {
|
||||
let h = common::harness(pool);
|
||||
|
||||
@@ -15,6 +15,7 @@ use tempfile::TempDir;
|
||||
use tower::ServiceExt;
|
||||
|
||||
use mangalord::app::{router, AppState};
|
||||
use mangalord::auth::rate_limit::AuthRateLimiter;
|
||||
use mangalord::config::{AuthConfig, UploadConfig};
|
||||
use mangalord::storage::{LocalStorage, Storage, StorageError, StreamingFile};
|
||||
|
||||
@@ -49,20 +50,51 @@ fn harness_inner(
|
||||
storage: Arc<dyn Storage>,
|
||||
storage_dir: TempDir,
|
||||
) -> Harness {
|
||||
harness_with_auth_config(pool, storage, storage_dir, AuthConfig {
|
||||
cookie_secure: false,
|
||||
..AuthConfig::default()
|
||||
})
|
||||
}
|
||||
|
||||
fn harness_with_auth_config(
|
||||
pool: PgPool,
|
||||
storage: Arc<dyn Storage>,
|
||||
storage_dir: TempDir,
|
||||
auth: AuthConfig,
|
||||
) -> Harness {
|
||||
let auth_limiter = Arc::new(AuthRateLimiter::new(auth.rate_limit));
|
||||
let state = AppState {
|
||||
db: pool,
|
||||
storage,
|
||||
auth: AuthConfig { cookie_secure: false, ..AuthConfig::default() },
|
||||
auth,
|
||||
upload: UploadConfig {
|
||||
// Keep file caps small in tests so the size-cap path is cheap to
|
||||
// exercise without producing tens of MBs of bytes.
|
||||
max_request_bytes: 4 * 1024 * 1024,
|
||||
max_file_bytes: 256 * 1024,
|
||||
},
|
||||
auth_limiter,
|
||||
};
|
||||
Harness { app: router(state), _storage_dir: storage_dir }
|
||||
}
|
||||
|
||||
/// Like [`harness`] but configures a tight auth rate limit. Used by
|
||||
/// the brute-force-rate-limiting test.
|
||||
pub fn harness_with_auth_rate_limit(
|
||||
pool: PgPool,
|
||||
per_sec: u32,
|
||||
burst: u32,
|
||||
) -> Harness {
|
||||
let storage_dir = tempfile::tempdir().expect("tempdir");
|
||||
let storage = Arc::new(LocalStorage::new(storage_dir.path()));
|
||||
let auth = AuthConfig {
|
||||
cookie_secure: false,
|
||||
rate_limit: mangalord::auth::rate_limit::RateLimitConfig { per_sec, burst },
|
||||
..AuthConfig::default()
|
||||
};
|
||||
harness_with_auth_config(pool, storage, storage_dir, auth)
|
||||
}
|
||||
|
||||
/// Wraps a real `Storage` and fails on the N-th `put` call so tests can
|
||||
/// assert that handlers roll their DB writes back when storage errors
|
||||
/// mid-upload. Reads and other operations delegate to `inner`.
|
||||
@@ -336,6 +368,37 @@ pub fn post_multipart_with_cookie(
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
pub fn put_multipart_with_cookie(
|
||||
uri: &str,
|
||||
builder: MultipartBuilder,
|
||||
cookie: &str,
|
||||
) -> Request<Body> {
|
||||
let (boundary, body) = builder.finalize();
|
||||
Request::builder()
|
||||
.method("PUT")
|
||||
.uri(uri)
|
||||
.header(
|
||||
header::CONTENT_TYPE,
|
||||
format!("multipart/form-data; boundary={boundary}"),
|
||||
)
|
||||
.header(header::COOKIE, cookie)
|
||||
.body(Body::from(body))
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
pub fn put_multipart(uri: &str, builder: MultipartBuilder) -> Request<Body> {
|
||||
let (boundary, body) = builder.finalize();
|
||||
Request::builder()
|
||||
.method("PUT")
|
||||
.uri(uri)
|
||||
.header(
|
||||
header::CONTENT_TYPE,
|
||||
format!("multipart/form-data; boundary={boundary}"),
|
||||
)
|
||||
.body(Body::from(body))
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
/// Realistic PNG file header bytes — enough for `infer` to identify.
|
||||
pub fn fake_png_bytes() -> Vec<u8> {
|
||||
vec![0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0, 0, 0, 0]
|
||||
|
||||
85
backend/tests/crawler_incremental.rs
Normal file
85
backend/tests/crawler_incremental.rs
Normal file
@@ -0,0 +1,85 @@
|
||||
//! Integration tests for the incremental-mode coordination state:
|
||||
//! `mark_seed_completed` / `seed_completed_at` round-trip via the
|
||||
//! `crawler_state` table.
|
||||
//!
|
||||
//! End-to-end pipeline behavior (walker + stop-on-Unchanged) requires
|
||||
//! a real `chromiumoxide::Browser` to construct a `FetchContext`, so
|
||||
//! the live integration of that path is covered by
|
||||
//! `crawler_browser_smoke.rs` instead. The pure stop logic itself is
|
||||
//! unit-tested in `crawler::pipeline::tests`.
|
||||
|
||||
use chrono::Utc;
|
||||
use mangalord::repo::crawler;
|
||||
use sqlx::PgPool;
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn seed_completed_at_none_before_any_run(pool: PgPool) {
|
||||
crawler::ensure_source(&pool, "target", "T", "https://x.example")
|
||||
.await
|
||||
.unwrap();
|
||||
let res = crawler::seed_completed_at(&pool, "target").await.unwrap();
|
||||
assert!(res.is_none(), "fresh source has no seed marker");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn mark_seed_completed_then_read_round_trips_timestamp(pool: PgPool) {
|
||||
crawler::ensure_source(&pool, "target", "T", "https://x.example")
|
||||
.await
|
||||
.unwrap();
|
||||
let at = Utc::now();
|
||||
crawler::mark_seed_completed(&pool, "target", at)
|
||||
.await
|
||||
.unwrap();
|
||||
let read = crawler::seed_completed_at(&pool, "target")
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("marker present after mark");
|
||||
// RFC3339 round-trip is millisecond-precise on chrono::Utc; allow a
|
||||
// 1ms tolerance to absorb postgres jsonb whitespace canonicalization.
|
||||
let drift = (read - at).num_milliseconds().abs();
|
||||
assert!(drift <= 1, "round-trip drift: {drift}ms (at={at}, read={read})");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn mark_seed_completed_overwrites_previous_value(pool: PgPool) {
|
||||
crawler::ensure_source(&pool, "target", "T", "https://x.example")
|
||||
.await
|
||||
.unwrap();
|
||||
let first = Utc::now() - chrono::Duration::hours(1);
|
||||
let second = Utc::now();
|
||||
crawler::mark_seed_completed(&pool, "target", first)
|
||||
.await
|
||||
.unwrap();
|
||||
crawler::mark_seed_completed(&pool, "target", second)
|
||||
.await
|
||||
.unwrap();
|
||||
let read = crawler::seed_completed_at(&pool, "target")
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("marker present");
|
||||
let drift = (read - second).num_milliseconds().abs();
|
||||
assert!(drift <= 1, "should reflect the latest mark, not the first");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn seed_completed_is_per_source(pool: PgPool) {
|
||||
// Two sources, only one is marked complete. The other must still
|
||||
// report None — the key is namespaced by source_id.
|
||||
crawler::ensure_source(&pool, "target", "T", "https://x.example")
|
||||
.await
|
||||
.unwrap();
|
||||
crawler::ensure_source(&pool, "other", "O", "https://y.example")
|
||||
.await
|
||||
.unwrap();
|
||||
crawler::mark_seed_completed(&pool, "target", Utc::now())
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(crawler::seed_completed_at(&pool, "target")
|
||||
.await
|
||||
.unwrap()
|
||||
.is_some());
|
||||
assert!(crawler::seed_completed_at(&pool, "other")
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none());
|
||||
}
|
||||
@@ -440,6 +440,170 @@ async fn arbitrary_genres_from_source_get_inserted(pool: PgPool) {
|
||||
assert_eq!(webtoons_count.0, 1, "case-insensitive lookup reuses the existing row");
|
||||
}
|
||||
|
||||
/// User-attached tags (rows with non-NULL `added_by` in `manga_tags`)
|
||||
/// must survive a crawler upsert. The crawler owns source-attached tags
|
||||
/// (added_by IS NULL); user attachments are owned by the user who made
|
||||
/// them and the recurring metadata pass must not delete them.
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn sync_tags_preserves_user_attached_tags(pool: PgPool) {
|
||||
crawler::ensure_source(&pool, "target", "T", "https://x.example")
|
||||
.await
|
||||
.unwrap();
|
||||
let m = sample_manga("foo", "Foo Manga", "hash-1");
|
||||
let up = crawler::upsert_manga_from_source(&pool, "target", "https://x.example/foo", &m)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// A real user attaches a personal tag.
|
||||
let user = mangalord::repo::user::create(&pool, "alice", "phc-stub")
|
||||
.await
|
||||
.unwrap();
|
||||
let outcome = mangalord::repo::tag::attach_to_manga(&pool, up.manga_id, "personal", user.id)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(outcome.created_attachment);
|
||||
|
||||
// Second crawler pass. Use a different metadata_hash so the upsert
|
||||
// takes the Updated branch, but the bug also fires on Unchanged
|
||||
// ticks since sync_tags runs unconditionally.
|
||||
let mut m2 = m.clone();
|
||||
m2.metadata_hash = "hash-2".into();
|
||||
m2.tags = vec!["popular".into(), "weekly".into()];
|
||||
let _ = crawler::upsert_manga_from_source(&pool, "target", "https://x.example/foo", &m2)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// The user tag must still be attached.
|
||||
let user_tag_rows: (i64,) = sqlx::query_as(
|
||||
"SELECT COUNT(*) FROM manga_tags mt \
|
||||
JOIN tags t ON t.id = mt.tag_id \
|
||||
WHERE mt.manga_id = $1 AND lower(t.name) = 'personal' \
|
||||
AND mt.added_by = $2",
|
||||
)
|
||||
.bind(up.manga_id)
|
||||
.bind(user.id)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
user_tag_rows.0, 1,
|
||||
"user-attached tag must survive a crawler upsert"
|
||||
);
|
||||
|
||||
// The source's tags should still attach as well, as crawler-owned.
|
||||
let source_tag_rows: (i64,) = sqlx::query_as(
|
||||
"SELECT COUNT(*) FROM manga_tags mt \
|
||||
JOIN tags t ON t.id = mt.tag_id \
|
||||
WHERE mt.manga_id = $1 \
|
||||
AND mt.added_by IS NULL \
|
||||
AND lower(t.name) IN ('popular', 'weekly')",
|
||||
)
|
||||
.bind(up.manga_id)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(source_tag_rows.0, 2, "source tags re-attach on each pass");
|
||||
|
||||
// A subsequent pass where the source drops a previously-seen tag
|
||||
// must clear that crawler-owned attachment (otherwise crawler-tags
|
||||
// would only ever accumulate).
|
||||
let mut m3 = m2.clone();
|
||||
m3.metadata_hash = "hash-3".into();
|
||||
m3.tags = vec!["popular".into()];
|
||||
let _ = crawler::upsert_manga_from_source(&pool, "target", "https://x.example/foo", &m3)
|
||||
.await
|
||||
.unwrap();
|
||||
let weekly_rows: (i64,) = sqlx::query_as(
|
||||
"SELECT COUNT(*) FROM manga_tags mt \
|
||||
JOIN tags t ON t.id = mt.tag_id \
|
||||
WHERE mt.manga_id = $1 AND lower(t.name) = 'weekly'",
|
||||
)
|
||||
.bind(up.manga_id)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(weekly_rows.0, 0, "source-owned tag dropped by source goes away");
|
||||
|
||||
// And the user tag still survives that third pass.
|
||||
let user_tag_rows: (i64,) = sqlx::query_as(
|
||||
"SELECT COUNT(*) FROM manga_tags mt \
|
||||
JOIN tags t ON t.id = mt.tag_id \
|
||||
WHERE mt.manga_id = $1 AND lower(t.name) = 'personal' \
|
||||
AND mt.added_by = $2",
|
||||
)
|
||||
.bind(up.manga_id)
|
||||
.bind(user.id)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(user_tag_rows.0, 1);
|
||||
}
|
||||
|
||||
/// `manga_tags.added_by` is `ON DELETE SET NULL` on the user FK. When
|
||||
/// the attaching user is deleted, their attachments become orphans
|
||||
/// indistinguishable from crawler-owned rows — and the crawler should
|
||||
/// reap them on the next pass. Pins the semantic so a future change
|
||||
/// can't quietly leave orphan rows lying around.
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn sync_tags_garbage_collects_orphan_user_attachments(pool: PgPool) {
|
||||
crawler::ensure_source(&pool, "target", "T", "https://x.example")
|
||||
.await
|
||||
.unwrap();
|
||||
let m = sample_manga("foo", "Foo", "hash-1");
|
||||
let up = crawler::upsert_manga_from_source(&pool, "target", "https://x.example/foo", &m)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// A user attaches "personal", then the user gets deleted. The
|
||||
// attachment row stays (manga_tags.manga_id FK is CASCADE on
|
||||
// mangas only; we never CASCADE-delete user attachments). The FK
|
||||
// on added_by is `ON DELETE SET NULL`, so the row's owner column
|
||||
// goes NULL — same shape as a crawler-owned row.
|
||||
let user = mangalord::repo::user::create(&pool, "bob", "phc-stub")
|
||||
.await
|
||||
.unwrap();
|
||||
let _ = mangalord::repo::tag::attach_to_manga(&pool, up.manga_id, "personal", user.id)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("DELETE FROM users WHERE id = $1")
|
||||
.bind(user.id)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Sanity: the orphan still exists post-user-delete with added_by NULL.
|
||||
let (orphan_rows,): (i64,) = sqlx::query_as(
|
||||
"SELECT COUNT(*) FROM manga_tags mt \
|
||||
JOIN tags t ON t.id = mt.tag_id \
|
||||
WHERE mt.manga_id = $1 AND lower(t.name) = 'personal' \
|
||||
AND mt.added_by IS NULL",
|
||||
)
|
||||
.bind(up.manga_id)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(orphan_rows, 1);
|
||||
|
||||
// Next crawler pass — orphan should be reaped along with any
|
||||
// other source-owned rows that aren't in the new tag list.
|
||||
let mut m2 = m.clone();
|
||||
m2.metadata_hash = "hash-2".into();
|
||||
m2.tags = vec!["popular".into()];
|
||||
let _ = crawler::upsert_manga_from_source(&pool, "target", "https://x.example/foo", &m2)
|
||||
.await
|
||||
.unwrap();
|
||||
let (orphan_rows,): (i64,) = sqlx::query_as(
|
||||
"SELECT COUNT(*) FROM manga_tags mt \
|
||||
JOIN tags t ON t.id = mt.tag_id \
|
||||
WHERE mt.manga_id = $1 AND lower(t.name) = 'personal'",
|
||||
)
|
||||
.bind(up.manga_id)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(orphan_rows, 0, "orphan user-attached tag should be reaped");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn re_appearing_manga_clears_dropped_at(pool: PgPool) {
|
||||
crawler::ensure_source(&pool, "target", "T", "https://x.example")
|
||||
|
||||
22
docker-compose.prod.yml
Normal file
22
docker-compose.prod.yml
Normal file
@@ -0,0 +1,22 @@
|
||||
# Production overlay: layer on top of docker-compose.yml on the deploy
|
||||
# host so the backend and frontend run from pre-built registry images
|
||||
# instead of building locally.
|
||||
#
|
||||
# docker compose -f docker-compose.yml -f docker-compose.prod.yml up -d
|
||||
#
|
||||
# REGISTRY_URL and IMAGE_TAG are injected by .gitea/workflows/deploy.yml
|
||||
# at deploy time. IMAGE_TAG defaults to `latest` so a manual
|
||||
# `docker compose ... up -d` on the host still works.
|
||||
|
||||
services:
|
||||
backend:
|
||||
build: !reset null
|
||||
image: ${REGISTRY_URL}/mangalord-backend:${IMAGE_TAG:-latest}
|
||||
pull_policy: always
|
||||
restart: unless-stopped
|
||||
|
||||
frontend:
|
||||
build: !reset null
|
||||
image: ${REGISTRY_URL}/mangalord-frontend:${IMAGE_TAG:-latest}
|
||||
pull_policy: always
|
||||
restart: unless-stopped
|
||||
@@ -1,9 +1,15 @@
|
||||
# Production-like compose. Requires a populated `.env` next to this
|
||||
# file: at minimum POSTGRES_PASSWORD must be set to a non-default
|
||||
# value (the `?required` form below fails fast otherwise). The
|
||||
# frontend container expects HTTPS in front (Caddy/Traefik/nginx)
|
||||
# because COOKIE_SECURE=true browsers will refuse to send the session
|
||||
# cookie over plain HTTP.
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16-alpine
|
||||
environment:
|
||||
POSTGRES_USER: ${POSTGRES_USER:-mangalord}
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-mangalord}
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD must be set in .env}
|
||||
POSTGRES_DB: ${POSTGRES_DB:-mangalord}
|
||||
volumes:
|
||||
- postgres-data:/var/lib/postgresql/data
|
||||
@@ -19,7 +25,7 @@ services:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
DATABASE_URL: postgres://${POSTGRES_USER:-mangalord}:${POSTGRES_PASSWORD:-mangalord}@postgres:5432/${POSTGRES_DB:-mangalord}
|
||||
DATABASE_URL: postgres://${POSTGRES_USER:-mangalord}:${POSTGRES_PASSWORD:?POSTGRES_PASSWORD must be set in .env}@postgres:5432/${POSTGRES_DB:-mangalord}
|
||||
BIND_ADDRESS: 0.0.0.0:8080
|
||||
STORAGE_DIR: /var/lib/mangalord/storage
|
||||
RUST_LOG: ${RUST_LOG:-info,mangalord=debug}
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
FROM node:22-alpine AS builder
|
||||
WORKDIR /app
|
||||
COPY package.json package-lock.json* ./
|
||||
RUN npm install
|
||||
# `npm ci` installs the locked versions exactly; `npm install` would
|
||||
# silently rewrite package-lock.json mid-build. CI (.gitea/workflows)
|
||||
# also uses `npm ci`, so this keeps the image build deterministic and
|
||||
# matches what the test job validated.
|
||||
RUN npm ci
|
||||
COPY . .
|
||||
RUN npm run build
|
||||
|
||||
@@ -10,8 +14,20 @@ WORKDIR /app
|
||||
ENV NODE_ENV=production
|
||||
ENV HOST=0.0.0.0
|
||||
ENV PORT=3000
|
||||
COPY --from=builder /app/build ./build
|
||||
COPY --from=builder /app/node_modules ./node_modules
|
||||
COPY --from=builder /app/package.json ./
|
||||
|
||||
# node:22-alpine ships a `node` user (UID 1000); use it instead of
|
||||
# running the SvelteKit server as root.
|
||||
COPY --from=builder --chown=node:node /app/build ./build
|
||||
COPY --from=builder --chown=node:node /app/node_modules ./node_modules
|
||||
COPY --from=builder --chown=node:node /app/package.json ./
|
||||
|
||||
USER node
|
||||
EXPOSE 3000
|
||||
|
||||
# Alpine's busybox `wget` is the canonical lightweight HTTP probe.
|
||||
# `--spider` doesn't follow redirects; `node build` serves a 200 on
|
||||
# `/` for the homepage so this works without a dedicated /health.
|
||||
HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
|
||||
CMD wget -q --spider http://localhost:3000/ || exit 1
|
||||
|
||||
CMD ["node", "build"]
|
||||
|
||||
147
frontend/e2e/manga-edit.spec.ts
Normal file
147
frontend/e2e/manga-edit.spec.ts
Normal file
@@ -0,0 +1,147 @@
|
||||
import { test, expect, type Page } from '@playwright/test';
|
||||
|
||||
const userFixture = {
|
||||
id: 'u1',
|
||||
username: 'alice',
|
||||
created_at: '2026-01-01T00:00:00Z'
|
||||
};
|
||||
|
||||
const baseManga = {
|
||||
id: 'm1',
|
||||
title: 'Berserk',
|
||||
status: 'ongoing',
|
||||
alt_titles: ['Old Alt'],
|
||||
description: 'Original description',
|
||||
cover_image_path: null,
|
||||
created_at: '2026-01-01T00:00:00Z',
|
||||
updated_at: '2026-01-01T00:00:00Z',
|
||||
authors: [{ id: 'a1', name: 'Kentaro Miura' }],
|
||||
genres: [],
|
||||
tags: []
|
||||
};
|
||||
|
||||
async function stubAuthenticatedAndGenres(page: Page) {
|
||||
await page.route('**/api/v1/auth/me', (route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ user: userFixture })
|
||||
})
|
||||
);
|
||||
await page.route('**/api/v1/genres', (route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify([
|
||||
{ id: 'g-action', name: 'Action' },
|
||||
{ id: 'g-fantasy', name: 'Fantasy' }
|
||||
])
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
test('anonymous user sees sign-in prompt on /manga/[id]/edit', async ({ page }) => {
|
||||
await page.route('**/api/v1/auth/me', (route) =>
|
||||
route.fulfill({
|
||||
status: 401,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({
|
||||
error: { code: 'unauthenticated', message: 'unauthenticated' }
|
||||
})
|
||||
})
|
||||
);
|
||||
await page.route('**/api/v1/genres', (route) =>
|
||||
route.fulfill({ status: 200, contentType: 'application/json', body: '[]' })
|
||||
);
|
||||
await page.route('**/api/v1/mangas/m1', (route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify(baseManga)
|
||||
})
|
||||
);
|
||||
|
||||
await page.goto('/manga/m1/edit');
|
||||
await expect(page.getByTestId('edit-signin')).toBeVisible();
|
||||
});
|
||||
|
||||
test('/manga/[id]/edit PATCHes the changed metadata and lands on the manga page', async ({
|
||||
page
|
||||
}) => {
|
||||
await stubAuthenticatedAndGenres(page);
|
||||
|
||||
let patchBody: Record<string, unknown> | null = null;
|
||||
let mangaAfter = { ...baseManga };
|
||||
await page.route('**/api/v1/mangas/m1', async (route) => {
|
||||
const method = route.request().method();
|
||||
if (method === 'GET') {
|
||||
await route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify(mangaAfter)
|
||||
});
|
||||
} else if (method === 'PATCH') {
|
||||
patchBody = JSON.parse(route.request().postData() ?? '{}');
|
||||
mangaAfter = {
|
||||
...mangaAfter,
|
||||
title: (patchBody.title as string) ?? mangaAfter.title,
|
||||
description:
|
||||
'description' in (patchBody as Record<string, unknown>)
|
||||
? ((patchBody.description as string | null) ?? null)
|
||||
: mangaAfter.description
|
||||
};
|
||||
await route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify(mangaAfter)
|
||||
});
|
||||
} else {
|
||||
await route.fallback();
|
||||
}
|
||||
});
|
||||
await page.route('**/api/v1/mangas/m1/chapters*', (route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({
|
||||
items: [],
|
||||
page: { limit: 50, offset: 0, total: 0 }
|
||||
})
|
||||
})
|
||||
);
|
||||
await page.route('**/api/v1/me/bookmarks*', (route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({
|
||||
items: [],
|
||||
page: { limit: 50, offset: 0, total: 0 }
|
||||
})
|
||||
})
|
||||
);
|
||||
await page.route('**/api/v1/me/read-progress/m1', (route) =>
|
||||
route.fulfill({
|
||||
status: 404,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({
|
||||
error: { code: 'not_found', message: 'no progress' }
|
||||
})
|
||||
})
|
||||
);
|
||||
|
||||
await page.goto('/manga/m1');
|
||||
// Edit link is gated on session.user — it should be visible to the
|
||||
// stubbed authenticated user.
|
||||
await page.getByTestId('edit-manga-link').click();
|
||||
await expect(page).toHaveURL(/\/manga\/m1\/edit$/);
|
||||
|
||||
const titleInput = page.getByTestId('manga-title');
|
||||
await expect(titleInput).toHaveValue('Berserk');
|
||||
await titleInput.fill('Berserk (Deluxe)');
|
||||
await page.getByTestId('manga-edit-submit').click();
|
||||
|
||||
await expect(page).toHaveURL(/\/manga\/m1$/);
|
||||
await expect(page.getByTestId('manga-title')).toHaveText('Berserk (Deluxe)');
|
||||
expect(patchBody).not.toBeNull();
|
||||
expect((patchBody as Record<string, unknown>).title).toBe('Berserk (Deluxe)');
|
||||
});
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "mangalord-frontend",
|
||||
"version": "0.29.0",
|
||||
"version": "0.35.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
|
||||
@@ -118,4 +118,77 @@ describe('hooks.server proxy', () => {
|
||||
expect(body.error.code).toBe('upstream_unavailable');
|
||||
expect(errSpy).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('strips every hop-by-hop header listed in RFC 7230 §6.1', async () => {
|
||||
// Defence in depth: axum doesn't emit these, but a future
|
||||
// middleware that did would otherwise leak per-connection
|
||||
// state across the proxy boundary.
|
||||
fetchSpy.mockResolvedValueOnce(new Response('[]', { status: 200 }));
|
||||
const resolve = vi.fn();
|
||||
await handle({
|
||||
event: makeEvent('/api/v1/health', {
|
||||
headers: {
|
||||
host: 'app.example.com',
|
||||
'content-length': '0',
|
||||
connection: 'keep-alive',
|
||||
'keep-alive': 'timeout=5',
|
||||
'proxy-authenticate': 'Basic realm=x',
|
||||
'proxy-authorization': 'Basic xyz',
|
||||
te: 'trailers',
|
||||
trailer: 'Expires',
|
||||
'transfer-encoding': 'chunked',
|
||||
upgrade: 'websocket',
|
||||
// A non-hop-by-hop header to ensure non-targets
|
||||
// aren't accidentally stripped.
|
||||
'x-custom': 'pass-through'
|
||||
}
|
||||
}),
|
||||
resolve
|
||||
});
|
||||
const init = fetchSpy.mock.calls[0][1] as RequestInit;
|
||||
const headers = init.headers as Headers;
|
||||
for (const h of [
|
||||
'host',
|
||||
'content-length',
|
||||
'connection',
|
||||
'keep-alive',
|
||||
'proxy-authenticate',
|
||||
'proxy-authorization',
|
||||
'te',
|
||||
'trailer',
|
||||
'transfer-encoding',
|
||||
'upgrade'
|
||||
]) {
|
||||
expect(headers.get(h), `${h} should be stripped`).toBeNull();
|
||||
}
|
||||
expect(headers.get('x-custom')).toBe('pass-through');
|
||||
});
|
||||
|
||||
it('aborts and returns 502 when the upstream stalls past the timeout', async () => {
|
||||
const errSpy = vi.spyOn(console, 'error').mockImplementation(() => {});
|
||||
// Simulate an aborted fetch (AbortController.abort() raises a
|
||||
// DOMException with name 'AbortError' on Node's fetch). The
|
||||
// handler should treat it as the same upstream_unavailable
|
||||
// 502 it uses for any other network failure.
|
||||
const abortErr = new DOMException('aborted', 'AbortError');
|
||||
fetchSpy.mockRejectedValueOnce(abortErr);
|
||||
|
||||
const resolve = vi.fn();
|
||||
const resp = await handle({ event: makeEvent('/api/v1/slow'), resolve });
|
||||
expect(resp.status).toBe(502);
|
||||
const body = await resp.json();
|
||||
expect(body.error.code).toBe('upstream_unavailable');
|
||||
expect(errSpy).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('attaches an AbortSignal to the upstream fetch so it can time out', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(new Response('[]', { status: 200 }));
|
||||
const resolve = vi.fn();
|
||||
await handle({ event: makeEvent('/api/v1/health'), resolve });
|
||||
const init = fetchSpy.mock.calls[0][1] as RequestInit;
|
||||
expect(init.signal).toBeInstanceOf(AbortSignal);
|
||||
// The signal hasn't fired (handler returned in time), but its
|
||||
// presence is the contract this test is pinning.
|
||||
expect(init.signal?.aborted).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -12,20 +12,66 @@ import type { Handle } from '@sveltejs/kit';
|
||||
|
||||
const BACKEND_URL = process.env.BACKEND_URL ?? 'http://localhost:8080';
|
||||
|
||||
/**
|
||||
* Hop-by-hop headers per RFC 7230 §6.1. These are scoped to a single
|
||||
* transport-level connection and must not be forwarded by a proxy.
|
||||
* Plus `host` and `content-length`: `host` would mislead the backend
|
||||
* about its origin, and `content-length` is recomputed by the upstream
|
||||
* fetch from the body stream.
|
||||
*/
|
||||
const HOP_BY_HOP_HEADERS = [
|
||||
'host',
|
||||
'content-length',
|
||||
'connection',
|
||||
'keep-alive',
|
||||
'proxy-authenticate',
|
||||
'proxy-authorization',
|
||||
'te',
|
||||
'trailer',
|
||||
'transfer-encoding',
|
||||
'upgrade'
|
||||
];
|
||||
|
||||
/**
|
||||
* Cap each proxied request at 5 minutes. The bound exists to surface
|
||||
* a wedged backend (stuck on a slow DB query, deadlocked, etc.) as a
|
||||
* 502 rather than letting the browser request hang indefinitely.
|
||||
*
|
||||
* The default leans toward the slow-upload end of the spectrum: at a
|
||||
* 1 Mbps upstream, a 200 MiB chapter upload (the default
|
||||
* `MAX_REQUEST_BYTES` cap) needs ~27 minutes; 300 s covers the more
|
||||
* realistic 25 Mbps urban-broadband case (~64 s for the same upload)
|
||||
* with comfortable headroom. Operators serving very slow clients
|
||||
* should raise `BACKEND_PROXY_TIMEOUT_MS`; operators behind a
|
||||
* tighter upstream proxy may want to lower it. A future improvement
|
||||
* is an idle-based timeout (reset per chunk) instead of this
|
||||
* wall-clock budget — that's a fair bit more code, deferred.
|
||||
*/
|
||||
const PROXY_TIMEOUT_MS = (() => {
|
||||
const raw = process.env.BACKEND_PROXY_TIMEOUT_MS;
|
||||
const n = raw ? Number(raw) : 300_000;
|
||||
return Number.isFinite(n) && n > 0 ? n : 300_000;
|
||||
})();
|
||||
|
||||
export const handle: Handle = async ({ event, resolve }) => {
|
||||
if (event.url.pathname.startsWith('/api/')) {
|
||||
const target = `${BACKEND_URL}${event.url.pathname}${event.url.search}`;
|
||||
|
||||
// Strip hop-by-hop headers — `host` would mislead the backend
|
||||
// about the origin, and `content-length` will be recomputed.
|
||||
const headers = new Headers(event.request.headers);
|
||||
headers.delete('host');
|
||||
headers.delete('content-length');
|
||||
for (const h of HOP_BY_HOP_HEADERS) headers.delete(h);
|
||||
|
||||
// AbortController times the upstream fetch out so a backend
|
||||
// wedged on a slow DB query doesn't keep the browser request
|
||||
// hanging forever. The `signal` is also wired into the
|
||||
// RequestInit so the body stream is cancelled cleanly.
|
||||
const ctrl = new AbortController();
|
||||
const timeoutHandle = setTimeout(() => ctrl.abort(), PROXY_TIMEOUT_MS);
|
||||
|
||||
const init: RequestInit & { duplex?: 'half' } = {
|
||||
method: event.request.method,
|
||||
headers,
|
||||
redirect: 'manual'
|
||||
redirect: 'manual',
|
||||
signal: ctrl.signal
|
||||
};
|
||||
if (event.request.method !== 'GET' && event.request.method !== 'HEAD') {
|
||||
init.body = event.request.body;
|
||||
@@ -39,11 +85,13 @@ export const handle: Handle = async ({ event, resolve }) => {
|
||||
upstream = await fetch(target, init);
|
||||
} catch (e) {
|
||||
// Network-layer failure (DNS / connection refused / TLS
|
||||
// handshake) — most commonly "backend container restarting".
|
||||
// SvelteKit's default 500 would be an HTML page that
|
||||
// client.ts can't .json(), which masks the real cause. Emit
|
||||
// the standard envelope with a dedicated code instead.
|
||||
// handshake / abort by timeout) — most commonly "backend
|
||||
// container restarting". SvelteKit's default 500 would be
|
||||
// an HTML page that client.ts can't .json(), which masks
|
||||
// the real cause. Emit the standard envelope with a
|
||||
// dedicated code instead.
|
||||
console.error('Proxy to backend failed:', e);
|
||||
clearTimeout(timeoutHandle);
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
error: {
|
||||
@@ -58,6 +106,7 @@ export const handle: Handle = async ({ event, resolve }) => {
|
||||
);
|
||||
}
|
||||
|
||||
clearTimeout(timeoutHandle);
|
||||
return new Response(upstream.body, {
|
||||
status: upstream.status,
|
||||
statusText: upstream.statusText,
|
||||
|
||||
@@ -94,6 +94,11 @@ describe('auth api client', () => {
|
||||
expect(url).toMatch(/\/v1\/auth\/logout$/);
|
||||
const init = fetchSpy.mock.calls[0][1] as RequestInit;
|
||||
expect(init.method).toBe('POST');
|
||||
// Consistent content-type for all mutation requests, matching
|
||||
// the rest of the module — axum doesn't require it but the
|
||||
// header keeps the request style uniform.
|
||||
const headers = new Headers(init.headers);
|
||||
expect(headers.get('content-type')).toBe('application/json');
|
||||
});
|
||||
|
||||
it('me returns the user on 200', async () => {
|
||||
|
||||
@@ -32,7 +32,14 @@ export async function login(creds: Credentials): Promise<User> {
|
||||
}
|
||||
|
||||
export async function logout(): Promise<void> {
|
||||
await request<void>('/v1/auth/logout', { method: 'POST' });
|
||||
await request<void>('/v1/auth/logout', {
|
||||
method: 'POST',
|
||||
// Consistent with the other POST/PATCH helpers in this module.
|
||||
// axum doesn't require it (no body), but keeping the header
|
||||
// on every mutation request avoids the false-flag in logs and
|
||||
// matches the project's style.
|
||||
headers: { 'content-type': 'application/json' }
|
||||
});
|
||||
}
|
||||
|
||||
export type ChangePassword = {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { describe, it, expect, vi, beforeEach, afterEach, type MockInstance } from 'vitest';
|
||||
import { ApiError, request } from './client';
|
||||
import { ApiError, request, setOn401Hook } from './client';
|
||||
import { getManga } from './mangas';
|
||||
|
||||
describe('request error envelope parsing', () => {
|
||||
@@ -73,3 +73,88 @@ describe('request error envelope parsing', () => {
|
||||
expect(err.code).toBe('http_error');
|
||||
});
|
||||
});
|
||||
|
||||
describe('on401 hook', () => {
|
||||
let fetchSpy: MockInstance<typeof globalThis.fetch>;
|
||||
|
||||
beforeEach(() => {
|
||||
fetchSpy = vi.spyOn(globalThis, 'fetch');
|
||||
});
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
// Critical: reset the module-level hook between tests so a
|
||||
// hook installed by one test doesn't leak into the next.
|
||||
setOn401Hook(null);
|
||||
});
|
||||
|
||||
it('invokes the hook exactly once on a 401 response and re-throws', async () => {
|
||||
const hook = vi.fn();
|
||||
setOn401Hook(hook);
|
||||
fetchSpy.mockResolvedValueOnce(
|
||||
new Response(
|
||||
JSON.stringify({ error: { code: 'unauthenticated', message: 'no auth' } }),
|
||||
{ status: 401, headers: { 'content-type': 'application/json' } }
|
||||
)
|
||||
);
|
||||
await expect(getManga('x')).rejects.toMatchObject({
|
||||
status: 401,
|
||||
code: 'unauthenticated'
|
||||
});
|
||||
expect(hook).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('does not invoke the hook on non-401 errors', async () => {
|
||||
const hook = vi.fn();
|
||||
setOn401Hook(hook);
|
||||
fetchSpy.mockResolvedValueOnce(
|
||||
new Response(
|
||||
JSON.stringify({ error: { code: 'not_found', message: 'no' } }),
|
||||
{ status: 404, headers: { 'content-type': 'application/json' } }
|
||||
)
|
||||
);
|
||||
await expect(getManga('x')).rejects.toMatchObject({ status: 404 });
|
||||
expect(hook).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('does not invoke the hook on successful responses', async () => {
|
||||
const hook = vi.fn();
|
||||
setOn401Hook(hook);
|
||||
fetchSpy.mockResolvedValueOnce(
|
||||
new Response(
|
||||
JSON.stringify({
|
||||
id: 'm1',
|
||||
title: 't',
|
||||
status: 'ongoing',
|
||||
alt_titles: [],
|
||||
description: null,
|
||||
cover_image_path: null,
|
||||
created_at: '2026-01-01T00:00:00Z',
|
||||
updated_at: '2026-01-01T00:00:00Z',
|
||||
authors: [],
|
||||
genres: [],
|
||||
tags: []
|
||||
}),
|
||||
{ status: 200, headers: { 'content-type': 'application/json' } }
|
||||
)
|
||||
);
|
||||
await getManga('m1');
|
||||
expect(hook).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('swallows hook exceptions so the original ApiError still propagates', async () => {
|
||||
const consoleSpy = vi.spyOn(console, 'error').mockImplementation(() => {});
|
||||
setOn401Hook(() => {
|
||||
throw new Error('hook boom');
|
||||
});
|
||||
fetchSpy.mockResolvedValueOnce(
|
||||
new Response(
|
||||
JSON.stringify({ error: { code: 'unauthenticated', message: 'x' } }),
|
||||
{ status: 401, headers: { 'content-type': 'application/json' } }
|
||||
)
|
||||
);
|
||||
await expect(getManga('x')).rejects.toMatchObject({ status: 401 });
|
||||
// The original ApiError won — the hook's panic was logged but
|
||||
// didn't replace the API error.
|
||||
expect(consoleSpy).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -25,6 +25,21 @@ export class ApiError extends Error {
|
||||
|
||||
type ErrorEnvelope = { error?: { code?: unknown; message?: unknown } };
|
||||
|
||||
/**
|
||||
* Optional hook fired the first moment `request()` observes a 401 on
|
||||
* any endpoint. Used by the session store to clear the cached user
|
||||
* when the server reports the session is no longer valid (expired
|
||||
* cookie, rotated server-side, password changed on another device).
|
||||
*
|
||||
* Set to `null` (or `undefined`) to disable. Tests that don't want
|
||||
* the side effect should leave it unset.
|
||||
*/
|
||||
let on401Hook: (() => void) | null = null;
|
||||
|
||||
export function setOn401Hook(handler: (() => void) | null): void {
|
||||
on401Hook = handler;
|
||||
}
|
||||
|
||||
export async function request<T>(path: string, init?: RequestInit): Promise<T> {
|
||||
// Forward credentials (session cookie) explicitly so cross-origin
|
||||
// deployments — those configured via CORS_ALLOWED_ORIGINS — keep
|
||||
@@ -54,6 +69,16 @@ export async function request<T>(path: string, init?: RequestInit): Promise<T> {
|
||||
} catch {
|
||||
// Body wasn't parseable; keep the http_error fallback.
|
||||
}
|
||||
if (res.status === 401 && on401Hook) {
|
||||
// Fire before throwing so the session store updates even
|
||||
// if the caller swallows the ApiError (e.g. the *OrEmpty
|
||||
// wrappers used by guest-rendering pages).
|
||||
try {
|
||||
on401Hook();
|
||||
} catch (e) {
|
||||
console.error('on401 hook threw:', e);
|
||||
}
|
||||
}
|
||||
throw new ApiError(res.status, code, message);
|
||||
}
|
||||
// Any empty body (not just 204) returns undefined — the manga-add
|
||||
|
||||
@@ -4,6 +4,8 @@ import {
|
||||
createManga,
|
||||
getManga,
|
||||
updateManga,
|
||||
updateMangaCover,
|
||||
deleteMangaCover,
|
||||
attachTag,
|
||||
detachTag
|
||||
} from './mangas';
|
||||
@@ -184,6 +186,49 @@ describe('mangas api client', () => {
|
||||
});
|
||||
});
|
||||
|
||||
it('updateMangaCover PUTs multipart with the cover blob', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(
|
||||
ok(detailFixture({ cover_image_path: 'mangas/b1/cover.png' }))
|
||||
);
|
||||
const cover = new Blob([new Uint8Array([0x89, 0x50, 0x4e, 0x47])], { type: 'image/png' });
|
||||
const updated = await updateMangaCover('b1', cover);
|
||||
expect(updated.cover_image_path).toBe('mangas/b1/cover.png');
|
||||
const url = fetchSpy.mock.calls[0][0] as string;
|
||||
expect(url).toMatch(/\/v1\/mangas\/b1\/cover$/);
|
||||
const init = fetchSpy.mock.calls[0][1] as RequestInit;
|
||||
expect(init.method).toBe('PUT');
|
||||
expect(init.body).toBeInstanceOf(FormData);
|
||||
const form = init.body as FormData;
|
||||
expect(form.get('cover')).toBeInstanceOf(Blob);
|
||||
// Boundary is filled in by the browser when body is FormData.
|
||||
expect(init.headers).toBeUndefined();
|
||||
});
|
||||
|
||||
it('updateMangaCover throws ApiError on payload_too_large', async () => {
|
||||
fetchSpy.mockResolvedValue(
|
||||
envelope(413, 'payload_too_large', 'cover exceeds size cap')
|
||||
);
|
||||
const cover = new Blob([new Uint8Array(1)]);
|
||||
await expect(updateMangaCover('b1', cover)).rejects.toMatchObject({
|
||||
name: 'ApiError',
|
||||
status: 413,
|
||||
code: 'payload_too_large'
|
||||
});
|
||||
});
|
||||
|
||||
it('deleteMangaCover DELETEs and returns the refreshed detail with null path', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(
|
||||
ok(detailFixture({ cover_image_path: null }))
|
||||
);
|
||||
const updated = await deleteMangaCover('b1');
|
||||
expect(updated.cover_image_path).toBeNull();
|
||||
const url = fetchSpy.mock.calls[0][0] as string;
|
||||
expect(url).toMatch(/\/v1\/mangas\/b1\/cover$/);
|
||||
const init = fetchSpy.mock.calls[0][1] as RequestInit;
|
||||
expect(init.method).toBe('DELETE');
|
||||
expect(init.body).toBeUndefined();
|
||||
});
|
||||
|
||||
it('attachTag POSTs the name and returns the TagRef', async () => {
|
||||
fetchSpy.mockResolvedValueOnce(
|
||||
ok({ id: 't9', name: 'Dark Fantasy', added_by: 'u1' }, 201)
|
||||
|
||||
@@ -109,6 +109,31 @@ export async function updateManga(
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* PUT /api/v1/mangas/:id/cover (multipart). Replaces the cover image and
|
||||
* returns the refreshed detail. As with createManga the browser fills in
|
||||
* the multipart boundary automatically, so we must NOT set Content-Type.
|
||||
*/
|
||||
export async function updateMangaCover(
|
||||
id: string,
|
||||
cover: Blob
|
||||
): Promise<MangaDetail> {
|
||||
const form = new FormData();
|
||||
form.append('cover', cover);
|
||||
return request<MangaDetail>(
|
||||
`/v1/mangas/${encodeURIComponent(id)}/cover`,
|
||||
{ method: 'PUT', body: form }
|
||||
);
|
||||
}
|
||||
|
||||
/** DELETE /api/v1/mangas/:id/cover. Returns the refreshed detail. */
|
||||
export async function deleteMangaCover(id: string): Promise<MangaDetail> {
|
||||
return request<MangaDetail>(
|
||||
`/v1/mangas/${encodeURIComponent(id)}/cover`,
|
||||
{ method: 'DELETE' }
|
||||
);
|
||||
}
|
||||
|
||||
export async function attachTag(
|
||||
mangaId: string,
|
||||
name: string
|
||||
|
||||
@@ -3,7 +3,17 @@
|
||||
// Only mutated client-side (onMount / form submits) so the module-level
|
||||
// instance can't leak across SSR requests — SSR always renders the
|
||||
// `loaded === false` state, and the client refreshes after hydration.
|
||||
//
|
||||
// IMPORTANT: do not call any `api/*` helper from `+page.server.ts` /
|
||||
// `+layout.server.ts`. The `setOn401Hook` below is registered at
|
||||
// module load (gated on `browser`, so it only fires in the client
|
||||
// bundle), so a 401 from a server-side fetch would mutate this
|
||||
// module-level `session.user` across SvelteKit requests — a real
|
||||
// cross-request state leak. The `if (browser)` guard makes that
|
||||
// failure mode mechanical rather than convention-based.
|
||||
|
||||
import { browser } from '$app/environment';
|
||||
import { setOn401Hook } from './api/client';
|
||||
import { me, type User } from './api/auth';
|
||||
|
||||
class SessionStore {
|
||||
@@ -31,3 +41,16 @@ class SessionStore {
|
||||
}
|
||||
|
||||
export const session = new SessionStore();
|
||||
|
||||
// When any backend call returns 401, drop the cached user. Before this
|
||||
// hook, the `*OrEmpty` wrappers silently returned empty pages on 401
|
||||
// — so a mid-session expiry left the UI rendering as "logged in but
|
||||
// no bookmarks/collections/etc." until the user manually reloaded.
|
||||
// With the hook the session.user reactive store flips to null on the
|
||||
// first 401, so the layout re-renders the login affordance.
|
||||
//
|
||||
// Gated on `browser` so it's only installed in the client bundle.
|
||||
// See the module-level comment above for the SSR rationale.
|
||||
if (browser) {
|
||||
setOn401Hook(() => session.setUser(null));
|
||||
}
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
import AddToCollectionModal from '$lib/components/AddToCollectionModal.svelte';
|
||||
import Plus from '@lucide/svelte/icons/plus';
|
||||
import FolderPlus from '@lucide/svelte/icons/folder-plus';
|
||||
import Pencil from '@lucide/svelte/icons/pencil';
|
||||
import UploadCloud from '@lucide/svelte/icons/upload-cloud';
|
||||
|
||||
let { data } = $props();
|
||||
@@ -327,6 +328,14 @@
|
||||
<FolderPlus size={16} aria-hidden="true" />
|
||||
<span>Add to collection</span>
|
||||
</button>
|
||||
<a
|
||||
class="action"
|
||||
href="/manga/{manga.id}/edit"
|
||||
data-testid="edit-manga-link"
|
||||
>
|
||||
<Pencil size={16} aria-hidden="true" />
|
||||
<span>Edit</span>
|
||||
</a>
|
||||
<a
|
||||
class="action"
|
||||
href="/manga/{manga.id}/upload-chapter"
|
||||
|
||||
@@ -350,54 +350,48 @@
|
||||
});
|
||||
|
||||
/**
|
||||
* `fetch()` initiated during `pagehide` / `beforeunload` is
|
||||
* cancelled by every browser by default. `sendBeacon` is the
|
||||
* supported way to ship a small payload during unload — it's
|
||||
* guaranteed to survive even if the tab is closing. Failure here
|
||||
* is silent because the API is fire-and-forget.
|
||||
* Flush read-progress as the tab is closing. A plain `fetch()`
|
||||
* during `pagehide` / `beforeunload` is cancelled by every
|
||||
* browser; `fetch(..., { keepalive: true })` is the supported
|
||||
* escape hatch and survives the close.
|
||||
*
|
||||
* `sendBeacon` would be the textbook alternative, but it's
|
||||
* POST-only and `/me/read-progress` takes PUT — so a beacon
|
||||
* always 405s, adds server-log noise, then falls through to this
|
||||
* same keepalive path anyway. The beacon was dropped; the
|
||||
* keepalive fetch is the only path.
|
||||
*/
|
||||
function beaconFinalProgress() {
|
||||
function flushFinalProgress() {
|
||||
if (!session.user) return;
|
||||
const body = JSON.stringify({
|
||||
manga_id: manga.id,
|
||||
chapter_id: chapter.id,
|
||||
page: progressPage
|
||||
});
|
||||
const blob = new Blob([body], { type: 'application/json' });
|
||||
// sendBeacon only supports POST — the server's PUT route is
|
||||
// strict on method. The dedicated POST alias is omitted; in
|
||||
// practice the in-app navigation path (back-link, chapter
|
||||
// links) already covers the common-case unmount via the
|
||||
// onDestroy fetch. Fall through to fetch+keepalive for browser
|
||||
// implementations that don't honor sendBeacon for this endpoint.
|
||||
try {
|
||||
const ok = navigator.sendBeacon('/api/v1/me/read-progress', blob);
|
||||
if (!ok) throw new Error('sendBeacon rejected');
|
||||
void fetch('/api/v1/me/read-progress', {
|
||||
method: 'PUT',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body,
|
||||
keepalive: true,
|
||||
credentials: 'include'
|
||||
});
|
||||
} catch {
|
||||
try {
|
||||
void fetch('/api/v1/me/read-progress', {
|
||||
method: 'PUT',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body,
|
||||
keepalive: true,
|
||||
credentials: 'include'
|
||||
});
|
||||
} catch {
|
||||
// Final fallback failed; the in-app onDestroy flush
|
||||
// below catches the SPA-navigation case.
|
||||
}
|
||||
// keepalive fetch was rejected (very old Firefox etc.);
|
||||
// the in-app onDestroy flush below catches the SPA-
|
||||
// navigation case, which is the common one anyway.
|
||||
}
|
||||
}
|
||||
|
||||
onMount(() => {
|
||||
window.addEventListener('pagehide', beaconFinalProgress);
|
||||
window.addEventListener('pagehide', flushFinalProgress);
|
||||
});
|
||||
|
||||
onDestroy(() => {
|
||||
observer?.disconnect();
|
||||
if (progressTimer) clearTimeout(progressTimer);
|
||||
if (typeof window !== 'undefined') {
|
||||
window.removeEventListener('pagehide', beaconFinalProgress);
|
||||
window.removeEventListener('pagehide', flushFinalProgress);
|
||||
}
|
||||
// Don't let the fullscreen flag leak to non-reader pages —
|
||||
// otherwise the layout header would stay slid-off on /upload
|
||||
|
||||
481
frontend/src/routes/manga/[id]/edit/+page.svelte
Normal file
481
frontend/src/routes/manga/[id]/edit/+page.svelte
Normal file
@@ -0,0 +1,481 @@
|
||||
<script lang="ts">
|
||||
import { goto } from '$app/navigation';
|
||||
import { ApiError, fileUrl } from '$lib/api/client';
|
||||
import {
|
||||
deleteMangaCover,
|
||||
updateManga,
|
||||
updateMangaCover,
|
||||
type MangaStatus
|
||||
} from '$lib/api/mangas';
|
||||
import { session } from '$lib/session.svelte';
|
||||
import { formatBytes, validateImageFile } from '$lib/upload-validation';
|
||||
import Chip from '$lib/components/Chip.svelte';
|
||||
import Plus from '@lucide/svelte/icons/plus';
|
||||
import Trash2 from '@lucide/svelte/icons/trash-2';
|
||||
|
||||
let { data } = $props();
|
||||
const manga = $derived(data.manga);
|
||||
const genres = $derived(data.genres);
|
||||
|
||||
// Snapshot data.manga into local state once. The edit form is the
|
||||
// source of truth from here on — we deliberately don't re-derive
|
||||
// from `data` after the initial paint.
|
||||
/* svelte-ignore state_referenced_locally */
|
||||
let mangaTitle = $state(data.manga.title);
|
||||
/* svelte-ignore state_referenced_locally */
|
||||
let mangaStatus = $state<MangaStatus>(data.manga.status);
|
||||
/* svelte-ignore state_referenced_locally */
|
||||
let mangaDescription = $state(data.manga.description ?? '');
|
||||
/* svelte-ignore state_referenced_locally */
|
||||
let mangaAuthors = $state<string[]>(data.manga.authors.map((a) => a.name));
|
||||
let authorDraft = $state('');
|
||||
/* svelte-ignore state_referenced_locally */
|
||||
let mangaAltTitles = $state<string[]>([...data.manga.alt_titles]);
|
||||
let altTitleDraft = $state('');
|
||||
/* svelte-ignore state_referenced_locally */
|
||||
let mangaGenreIds = $state<string[]>(data.manga.genres.map((g) => g.id));
|
||||
|
||||
let coverFile = $state<File | null>(null);
|
||||
let coverError = $state<string | null>(null);
|
||||
let pendingCoverRemoval = $state(false);
|
||||
/* svelte-ignore state_referenced_locally */
|
||||
let currentCoverPath = $state<string | null>(data.manga.cover_image_path);
|
||||
|
||||
let submitting = $state(false);
|
||||
let mangaError = $state<string | null>(null);
|
||||
|
||||
const canSubmit = $derived(
|
||||
mangaTitle.trim().length > 0 && !coverError && !submitting
|
||||
);
|
||||
|
||||
function addAuthor() {
|
||||
const name = authorDraft.trim();
|
||||
if (!name) return;
|
||||
if (!mangaAuthors.some((a) => a.toLowerCase() === name.toLowerCase())) {
|
||||
mangaAuthors = [...mangaAuthors, name];
|
||||
}
|
||||
authorDraft = '';
|
||||
}
|
||||
function removeAuthor(name: string) {
|
||||
mangaAuthors = mangaAuthors.filter((a) => a !== name);
|
||||
}
|
||||
function addAltTitle() {
|
||||
const t = altTitleDraft.trim();
|
||||
if (!t) return;
|
||||
if (!mangaAltTitles.includes(t)) {
|
||||
mangaAltTitles = [...mangaAltTitles, t];
|
||||
}
|
||||
altTitleDraft = '';
|
||||
}
|
||||
function removeAltTitle(t: string) {
|
||||
mangaAltTitles = mangaAltTitles.filter((x) => x !== t);
|
||||
}
|
||||
function toggleGenre(id: string) {
|
||||
mangaGenreIds = mangaGenreIds.includes(id)
|
||||
? mangaGenreIds.filter((g) => g !== id)
|
||||
: [...mangaGenreIds, id];
|
||||
}
|
||||
function onCoverChange(e: Event) {
|
||||
const input = e.target as HTMLInputElement;
|
||||
const file = input.files?.[0] ?? null;
|
||||
coverFile = file;
|
||||
coverError = file ? validateImageFile(file) : null;
|
||||
// Picking a replacement supersedes a pending "remove" click.
|
||||
if (file) pendingCoverRemoval = false;
|
||||
}
|
||||
function markCoverForRemoval() {
|
||||
pendingCoverRemoval = true;
|
||||
coverFile = null;
|
||||
coverError = null;
|
||||
// Clear the file input so re-picking the same file still fires
|
||||
// `change` and undoes the removal.
|
||||
const input = document.getElementById('cover-input') as HTMLInputElement | null;
|
||||
if (input) input.value = '';
|
||||
}
|
||||
function undoCoverRemoval() {
|
||||
pendingCoverRemoval = false;
|
||||
}
|
||||
|
||||
async function submit(e: SubmitEvent) {
|
||||
e.preventDefault();
|
||||
if (!canSubmit) return;
|
||||
if (authorDraft.trim()) addAuthor();
|
||||
if (altTitleDraft.trim()) addAltTitle();
|
||||
submitting = true;
|
||||
mangaError = null;
|
||||
try {
|
||||
// The textarea is the source of truth for description on
|
||||
// screen, so we always send it — trimmed-empty collapses to
|
||||
// null (explicit clear).
|
||||
await updateManga(manga.id, {
|
||||
title: mangaTitle.trim(),
|
||||
status: mangaStatus,
|
||||
authors: mangaAuthors,
|
||||
alt_titles: mangaAltTitles,
|
||||
genre_ids: mangaGenreIds,
|
||||
description: mangaDescription.trim() || null
|
||||
});
|
||||
if (pendingCoverRemoval) {
|
||||
const refreshed = await deleteMangaCover(manga.id);
|
||||
currentCoverPath = refreshed.cover_image_path;
|
||||
} else if (coverFile) {
|
||||
const refreshed = await updateMangaCover(manga.id, coverFile);
|
||||
currentCoverPath = refreshed.cover_image_path;
|
||||
}
|
||||
await goto(`/manga/${manga.id}`);
|
||||
} catch (e) {
|
||||
if (e instanceof ApiError && e.status === 401) {
|
||||
await goto(`/login?next=/manga/${manga.id}/edit`);
|
||||
return;
|
||||
}
|
||||
mangaError = e instanceof Error ? e.message : String(e);
|
||||
submitting = false;
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<svelte:head>
|
||||
<title>Edit {manga.title} — Mangalord</title>
|
||||
</svelte:head>
|
||||
|
||||
<h1>Edit manga</h1>
|
||||
|
||||
{#if !session.loaded}
|
||||
<p class="status" data-testid="edit-loading">Loading…</p>
|
||||
{:else if !session.user}
|
||||
<p class="status" data-testid="edit-signin">
|
||||
<a href="/login?next=/manga/{manga.id}/edit">Sign in</a> to edit this manga.
|
||||
</p>
|
||||
{:else}
|
||||
<form onsubmit={submit} action="javascript:void(0)" data-testid="manga-edit-form">
|
||||
<section class="card">
|
||||
<h2>Manga details</h2>
|
||||
<label class="form-field">
|
||||
<span>Title <span aria-hidden="true">*</span></span>
|
||||
<input
|
||||
type="text"
|
||||
bind:value={mangaTitle}
|
||||
required
|
||||
maxlength="200"
|
||||
data-testid="manga-title"
|
||||
/>
|
||||
</label>
|
||||
|
||||
<label class="form-field">
|
||||
<span>Status</span>
|
||||
<select bind:value={mangaStatus} data-testid="manga-status">
|
||||
<option value="ongoing">Ongoing</option>
|
||||
<option value="completed">Completed</option>
|
||||
</select>
|
||||
</label>
|
||||
|
||||
<div class="form-field">
|
||||
<span>Authors</span>
|
||||
<div class="token-row">
|
||||
{#each mangaAuthors as a (a)}
|
||||
<Chip label={a} variant="primary" onRemove={() => removeAuthor(a)} />
|
||||
{/each}
|
||||
</div>
|
||||
<div class="token-input-row">
|
||||
<input
|
||||
type="text"
|
||||
bind:value={authorDraft}
|
||||
onkeydown={(e) => {
|
||||
if (e.key === 'Enter') {
|
||||
e.preventDefault();
|
||||
addAuthor();
|
||||
}
|
||||
}}
|
||||
placeholder="Add author"
|
||||
maxlength="200"
|
||||
data-testid="manga-author-input"
|
||||
/>
|
||||
<button
|
||||
type="button"
|
||||
class="icon-btn primary"
|
||||
onclick={addAuthor}
|
||||
disabled={!authorDraft.trim()}
|
||||
aria-label="Add author"
|
||||
title="Add author"
|
||||
>
|
||||
<Plus size={16} aria-hidden="true" />
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="form-field">
|
||||
<span>Genres</span>
|
||||
<div class="genre-grid" data-testid="manga-genres">
|
||||
{#each genres as g (g.id)}
|
||||
<label class="genre-toggle">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={mangaGenreIds.includes(g.id)}
|
||||
onchange={() => toggleGenre(g.id)}
|
||||
/>
|
||||
<span>{g.name}</span>
|
||||
</label>
|
||||
{/each}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="form-field">
|
||||
<span>Alternative titles</span>
|
||||
<div class="token-row">
|
||||
{#each mangaAltTitles as t (t)}
|
||||
<Chip label={t} onRemove={() => removeAltTitle(t)} />
|
||||
{/each}
|
||||
</div>
|
||||
<div class="token-input-row">
|
||||
<input
|
||||
type="text"
|
||||
bind:value={altTitleDraft}
|
||||
onkeydown={(e) => {
|
||||
if (e.key === 'Enter') {
|
||||
e.preventDefault();
|
||||
addAltTitle();
|
||||
}
|
||||
}}
|
||||
placeholder="Add alternative title"
|
||||
maxlength="200"
|
||||
data-testid="manga-alt-input"
|
||||
/>
|
||||
<button
|
||||
type="button"
|
||||
class="icon-btn primary"
|
||||
onclick={addAltTitle}
|
||||
disabled={!altTitleDraft.trim()}
|
||||
aria-label="Add alternative title"
|
||||
title="Add alternative title"
|
||||
>
|
||||
<Plus size={16} aria-hidden="true" />
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<label class="form-field">
|
||||
<span>Description</span>
|
||||
<textarea
|
||||
bind:value={mangaDescription}
|
||||
rows="4"
|
||||
data-testid="manga-description"
|
||||
></textarea>
|
||||
</label>
|
||||
|
||||
<div class="form-field">
|
||||
<span>Cover</span>
|
||||
{#if currentCoverPath && !pendingCoverRemoval}
|
||||
<div class="cover-preview" data-testid="cover-preview">
|
||||
<img
|
||||
src={fileUrl(currentCoverPath)}
|
||||
alt="Current cover"
|
||||
/>
|
||||
<button
|
||||
type="button"
|
||||
class="icon-btn danger"
|
||||
onclick={markCoverForRemoval}
|
||||
aria-label="Remove cover"
|
||||
title="Remove cover"
|
||||
data-testid="cover-remove"
|
||||
>
|
||||
<Trash2 size={16} aria-hidden="true" />
|
||||
</button>
|
||||
</div>
|
||||
{:else if pendingCoverRemoval}
|
||||
<p class="hint" data-testid="cover-pending-removal">
|
||||
Cover will be removed on save.
|
||||
<button
|
||||
type="button"
|
||||
class="text-link"
|
||||
onclick={undoCoverRemoval}
|
||||
data-testid="cover-undo-remove"
|
||||
>
|
||||
Undo
|
||||
</button>
|
||||
</p>
|
||||
{/if}
|
||||
<input
|
||||
id="cover-input"
|
||||
type="file"
|
||||
accept="image/*"
|
||||
onchange={onCoverChange}
|
||||
data-testid="manga-cover"
|
||||
/>
|
||||
{#if coverFile}
|
||||
<span class="hint">
|
||||
Will upload: {coverFile.name} ({formatBytes(coverFile.size)})
|
||||
</span>
|
||||
{/if}
|
||||
{#if coverError}
|
||||
<span class="field-error" role="alert">{coverError}</span>
|
||||
{/if}
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<div class="actions">
|
||||
<button
|
||||
class="primary"
|
||||
type="submit"
|
||||
disabled={!canSubmit}
|
||||
data-testid="manga-edit-submit"
|
||||
>
|
||||
{submitting ? 'Saving…' : 'Save changes'}
|
||||
</button>
|
||||
<a class="cancel" href="/manga/{manga.id}" data-testid="manga-edit-cancel">
|
||||
Cancel
|
||||
</a>
|
||||
</div>
|
||||
{#if mangaError}
|
||||
<p role="alert" class="form-error" data-testid="manga-edit-error">{mangaError}</p>
|
||||
{/if}
|
||||
</form>
|
||||
{/if}
|
||||
|
||||
<style>
|
||||
.status {
|
||||
color: var(--text-muted);
|
||||
}
|
||||
|
||||
form {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: var(--space-4);
|
||||
}
|
||||
|
||||
.card {
|
||||
background: var(--surface);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius-md);
|
||||
padding: var(--space-4);
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: var(--space-3);
|
||||
}
|
||||
|
||||
.actions {
|
||||
display: flex;
|
||||
gap: var(--space-3);
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
.primary {
|
||||
background: var(--primary);
|
||||
color: var(--primary-contrast);
|
||||
border-color: var(--primary);
|
||||
}
|
||||
|
||||
.primary:hover:not(:disabled) {
|
||||
background: var(--primary-hover);
|
||||
border-color: var(--primary-hover);
|
||||
}
|
||||
|
||||
.cancel {
|
||||
color: var(--text-muted);
|
||||
}
|
||||
|
||||
.hint {
|
||||
color: var(--text-muted);
|
||||
font-size: var(--font-sm);
|
||||
}
|
||||
|
||||
.field-error {
|
||||
color: var(--danger);
|
||||
font-size: var(--font-sm);
|
||||
}
|
||||
|
||||
.form-error {
|
||||
color: var(--danger);
|
||||
}
|
||||
|
||||
.token-row {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: var(--space-2);
|
||||
margin-bottom: var(--space-1);
|
||||
}
|
||||
|
||||
.token-input-row {
|
||||
display: flex;
|
||||
gap: var(--space-2);
|
||||
}
|
||||
|
||||
.token-input-row input {
|
||||
flex: 1;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.genre-grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fill, minmax(150px, 1fr));
|
||||
gap: var(--space-2);
|
||||
}
|
||||
|
||||
.genre-toggle {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: var(--space-2);
|
||||
color: var(--text);
|
||||
font-size: var(--font-sm);
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.icon-btn {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
width: 32px;
|
||||
height: 32px;
|
||||
padding: 0;
|
||||
background: transparent;
|
||||
color: var(--text-muted);
|
||||
border: 1px solid transparent;
|
||||
border-radius: var(--radius-sm);
|
||||
}
|
||||
|
||||
.icon-btn:hover:not(:disabled) {
|
||||
background: var(--surface-elevated);
|
||||
color: var(--text);
|
||||
}
|
||||
|
||||
.icon-btn.primary {
|
||||
background: var(--primary);
|
||||
color: var(--primary-contrast);
|
||||
border-color: var(--primary);
|
||||
}
|
||||
|
||||
.icon-btn.primary:hover:not(:disabled) {
|
||||
background: var(--primary-hover);
|
||||
border-color: var(--primary-hover);
|
||||
}
|
||||
|
||||
.icon-btn.danger:hover:not(:disabled) {
|
||||
color: var(--danger);
|
||||
}
|
||||
|
||||
.cover-preview {
|
||||
display: flex;
|
||||
align-items: flex-start;
|
||||
gap: var(--space-2);
|
||||
}
|
||||
|
||||
.cover-preview img {
|
||||
max-width: 160px;
|
||||
height: auto;
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius-sm);
|
||||
display: block;
|
||||
}
|
||||
|
||||
.text-link {
|
||||
background: none;
|
||||
border: none;
|
||||
padding: 0;
|
||||
color: var(--primary);
|
||||
cursor: pointer;
|
||||
font: inherit;
|
||||
}
|
||||
|
||||
.text-link:hover {
|
||||
text-decoration: underline;
|
||||
}
|
||||
</style>
|
||||
13
frontend/src/routes/manga/[id]/edit/+page.ts
Normal file
13
frontend/src/routes/manga/[id]/edit/+page.ts
Normal file
@@ -0,0 +1,13 @@
|
||||
import { getManga } from '$lib/api/mangas';
|
||||
import { listGenres, type Genre } from '$lib/api/genres';
|
||||
import type { PageLoad } from './$types';
|
||||
|
||||
export const ssr = false;
|
||||
|
||||
export const load: PageLoad = async ({ params }) => {
|
||||
const [manga, genres] = await Promise.all([
|
||||
getManga(params.id),
|
||||
listGenres()
|
||||
]);
|
||||
return { manga, genres: genres as Genre[] };
|
||||
};
|
||||
@@ -1,20 +1,26 @@
|
||||
import { sveltekit } from '@sveltejs/kit/vite';
|
||||
import { defineConfig } from 'vite';
|
||||
import { defineConfig, loadEnv } from 'vite';
|
||||
|
||||
export default defineConfig({
|
||||
plugins: [sveltekit()],
|
||||
server: {
|
||||
port: 5173,
|
||||
proxy: {
|
||||
'/api': {
|
||||
target: process.env.BACKEND_URL ?? 'http://localhost:8080',
|
||||
changeOrigin: true
|
||||
export default defineConfig(({ mode }) => {
|
||||
// Pull in .env so VITE_PORT / BACKEND_URL pin the dev URL across runs.
|
||||
// Empty prefix loads every key, not just VITE_*.
|
||||
const env = { ...process.env, ...loadEnv(mode, process.cwd(), '') };
|
||||
return {
|
||||
plugins: [sveltekit()],
|
||||
server: {
|
||||
port: Number(env.VITE_PORT ?? 5173),
|
||||
strictPort: env.VITE_PORT != null,
|
||||
proxy: {
|
||||
'/api': {
|
||||
target: env.BACKEND_URL ?? 'http://localhost:8080',
|
||||
changeOrigin: true
|
||||
}
|
||||
}
|
||||
},
|
||||
test: {
|
||||
environment: 'jsdom',
|
||||
include: ['src/**/*.test.ts'],
|
||||
globals: false
|
||||
}
|
||||
},
|
||||
test: {
|
||||
environment: 'jsdom',
|
||||
include: ['src/**/*.test.ts'],
|
||||
globals: false
|
||||
}
|
||||
};
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user