Compare commits
1 Commits
chore/repo
...
feat/auth-
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
699c1d0d69 |
@@ -29,6 +29,13 @@ COOKIE_DOMAIN=
|
|||||||
# get reaped lazily.
|
# get reaped lazily.
|
||||||
SESSION_TTL_DAYS=30
|
SESSION_TTL_DAYS=30
|
||||||
|
|
||||||
|
# ----- Auth brute-force rate limits -----
|
||||||
|
# Token-bucket budget shared across /auth/login, /auth/register, and
|
||||||
|
# /auth/me/password. Set per_sec=0 to disable (e.g. behind a
|
||||||
|
# rate-limiting reverse proxy that already enforces a budget).
|
||||||
|
AUTH_RATE_PER_SEC=5
|
||||||
|
AUTH_RATE_BURST=10
|
||||||
|
|
||||||
# ----- CORS -----
|
# ----- CORS -----
|
||||||
# Comma-separated origins allowed to call the API with credentials.
|
# Comma-separated origins allowed to call the API with credentials.
|
||||||
# Default is empty: same-origin only. Set when frontend and backend live
|
# Default is empty: same-origin only. Set when frontend and backend live
|
||||||
|
|||||||
2
backend/Cargo.lock
generated
2
backend/Cargo.lock
generated
@@ -1470,7 +1470,7 @@ checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "mangalord"
|
name = "mangalord"
|
||||||
version = "0.34.0"
|
version = "0.35.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"argon2",
|
"argon2",
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "mangalord"
|
name = "mangalord"
|
||||||
version = "0.34.0"
|
version = "0.35.0"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
default-run = "mangalord"
|
default-run = "mangalord"
|
||||||
|
|
||||||
|
|||||||
@@ -80,6 +80,7 @@ async fn register(
|
|||||||
jar: CookieJar,
|
jar: CookieJar,
|
||||||
Json(input): Json<Credentials>,
|
Json(input): Json<Credentials>,
|
||||||
) -> AppResult<impl IntoResponse> {
|
) -> AppResult<impl IntoResponse> {
|
||||||
|
check_auth_rate_limit(&state, "register")?;
|
||||||
let username = input.username.trim();
|
let username = input.username.trim();
|
||||||
validate_username(username)?;
|
validate_username(username)?;
|
||||||
validate_password(&input.password)?;
|
validate_password(&input.password)?;
|
||||||
@@ -95,6 +96,7 @@ async fn login(
|
|||||||
jar: CookieJar,
|
jar: CookieJar,
|
||||||
Json(input): Json<Credentials>,
|
Json(input): Json<Credentials>,
|
||||||
) -> AppResult<impl IntoResponse> {
|
) -> AppResult<impl IntoResponse> {
|
||||||
|
check_auth_rate_limit(&state, "login")?;
|
||||||
let username = input.username.trim();
|
let username = input.username.trim();
|
||||||
if username.is_empty() || input.password.is_empty() {
|
if username.is_empty() || input.password.is_empty() {
|
||||||
return Err(AppError::InvalidInput(
|
return Err(AppError::InvalidInput(
|
||||||
@@ -149,6 +151,7 @@ async fn change_password(
|
|||||||
jar: CookieJar,
|
jar: CookieJar,
|
||||||
Json(input): Json<ChangePassword>,
|
Json(input): Json<ChangePassword>,
|
||||||
) -> AppResult<impl IntoResponse> {
|
) -> AppResult<impl IntoResponse> {
|
||||||
|
check_auth_rate_limit(&state, "change_password")?;
|
||||||
if !verify_password(&input.current_password, &user.password_hash) {
|
if !verify_password(&input.current_password, &user.password_hash) {
|
||||||
return Err(AppError::Unauthenticated);
|
return Err(AppError::Unauthenticated);
|
||||||
}
|
}
|
||||||
@@ -293,6 +296,33 @@ fn build_expired_cookie(cfg: &AuthConfig) -> Cookie<'static> {
|
|||||||
builder.build()
|
builder.build()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Consume one token from the shared auth rate limiter. Called at the
|
||||||
|
/// start of `register`, `login`, and `change_password` so credential
|
||||||
|
/// stuffing / spraying / username-probe loops are throttled by the
|
||||||
|
/// configured budget (default 5/sec with a 10-request burst).
|
||||||
|
///
|
||||||
|
/// All three endpoints share one bucket — they all expose the same
|
||||||
|
/// argon2-verify-or-create work and the same enumeration channels, so
|
||||||
|
/// any one of them in a tight loop should trip the limit. `endpoint`
|
||||||
|
/// is included in the rate-limit-hit log line so operators can tell
|
||||||
|
/// which endpoint is being probed.
|
||||||
|
fn check_auth_rate_limit(state: &AppState, endpoint: &'static str) -> AppResult<()> {
|
||||||
|
use crate::auth::rate_limit::AcquireResult;
|
||||||
|
match state.auth_limiter.try_acquire() {
|
||||||
|
AcquireResult::Allowed => Ok(()),
|
||||||
|
AcquireResult::Denied { retry_after_secs } => {
|
||||||
|
tracing::warn!(
|
||||||
|
endpoint,
|
||||||
|
retry_after_secs,
|
||||||
|
"auth rate limit hit; returning 429"
|
||||||
|
);
|
||||||
|
Err(AppError::TooManyRequests {
|
||||||
|
retry_after_secs: Some(retry_after_secs),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn validate_username(u: &str) -> AppResult<()> {
|
fn validate_username(u: &str) -> AppResult<()> {
|
||||||
if u.is_empty() {
|
if u.is_empty() {
|
||||||
return Err(AppError::InvalidInput("username is required".into()));
|
return Err(AppError::InvalidInput("username is required".into()));
|
||||||
|
|||||||
@@ -67,7 +67,14 @@ async fn create(
|
|||||||
// the foreign-key violation collapse into a generic 500.
|
// the foreign-key violation collapse into a generic 500.
|
||||||
repo::manga::get(&state.db, input.manga_id).await?;
|
repo::manga::get(&state.db, input.manga_id).await?;
|
||||||
if let Some(chapter_id) = input.chapter_id {
|
if let Some(chapter_id) = input.chapter_id {
|
||||||
if !repo::chapter::belongs_to_manga(&state.db, chapter_id, input.manga_id).await? {
|
let exists: Option<(Uuid,)> = sqlx::query_as(
|
||||||
|
"SELECT id FROM chapters WHERE id = $1 AND manga_id = $2",
|
||||||
|
)
|
||||||
|
.bind(chapter_id)
|
||||||
|
.bind(input.manga_id)
|
||||||
|
.fetch_optional(&state.db)
|
||||||
|
.await?;
|
||||||
|
if exists.is_none() {
|
||||||
return Err(AppError::NotFound);
|
return Err(AppError::NotFound);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ use tokio_util::sync::CancellationToken;
|
|||||||
use tower_http::cors::{AllowOrigin, CorsLayer};
|
use tower_http::cors::{AllowOrigin, CorsLayer};
|
||||||
use tower_http::trace::TraceLayer;
|
use tower_http::trace::TraceLayer;
|
||||||
|
|
||||||
|
use crate::auth::rate_limit::AuthRateLimiter;
|
||||||
use crate::config::{AuthConfig, Config, CrawlerConfig, CrawlerModePref, UploadConfig};
|
use crate::config::{AuthConfig, Config, CrawlerConfig, CrawlerModePref, UploadConfig};
|
||||||
use crate::crawler::browser_manager::{self, BrowserManager};
|
use crate::crawler::browser_manager::{self, BrowserManager};
|
||||||
use crate::crawler::content::{self, SyncOutcome};
|
use crate::crawler::content::{self, SyncOutcome};
|
||||||
@@ -30,6 +31,10 @@ pub struct AppState {
|
|||||||
pub storage: Arc<dyn Storage>,
|
pub storage: Arc<dyn Storage>,
|
||||||
pub auth: AuthConfig,
|
pub auth: AuthConfig,
|
||||||
pub upload: UploadConfig,
|
pub upload: UploadConfig,
|
||||||
|
/// Shared rate limiter guarding the `/auth/*` mutation endpoints.
|
||||||
|
/// One instance per AppState so tests stay isolated across the
|
||||||
|
/// same process.
|
||||||
|
pub auth_limiter: Arc<AuthRateLimiter>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Bundle returned by [`build`]. The router is what `axum::serve` consumes;
|
/// Bundle returned by [`build`]. The router is what `axum::serve` consumes;
|
||||||
@@ -64,11 +69,13 @@ pub async fn build(config: Config) -> anyhow::Result<AppHandle> {
|
|||||||
None
|
None
|
||||||
};
|
};
|
||||||
|
|
||||||
|
let auth_limiter = Arc::new(AuthRateLimiter::new(config.auth.rate_limit));
|
||||||
let state = AppState {
|
let state = AppState {
|
||||||
db,
|
db,
|
||||||
storage,
|
storage,
|
||||||
auth: config.auth.clone(),
|
auth: config.auth.clone(),
|
||||||
upload: config.upload.clone(),
|
upload: config.upload.clone(),
|
||||||
|
auth_limiter,
|
||||||
};
|
};
|
||||||
let router = router(state).layer(cors_layer(&config.cors_allowed_origins));
|
let router = router(state).layer(cors_layer(&config.cors_allowed_origins));
|
||||||
Ok(AppHandle { router, daemon })
|
Ok(AppHandle { router, daemon })
|
||||||
@@ -304,9 +311,18 @@ impl ChapterDispatcher for RealChapterDispatcher {
|
|||||||
chapter_id,
|
chapter_id,
|
||||||
source_chapter_key: _,
|
source_chapter_key: _,
|
||||||
} => {
|
} => {
|
||||||
let row = repo::chapter::dispatch_target(&self.db, chapter_id)
|
// Look up manga_id + source_url for this chapter.
|
||||||
.await
|
let row: Option<(uuid::Uuid, String)> = sqlx::query_as(
|
||||||
.context("look up chapter for dispatch")?;
|
"SELECT c.manga_id, cs.source_url \
|
||||||
|
FROM chapters c \
|
||||||
|
JOIN chapter_sources cs ON cs.chapter_id = c.id \
|
||||||
|
WHERE c.id = $1 \
|
||||||
|
LIMIT 1",
|
||||||
|
)
|
||||||
|
.bind(chapter_id)
|
||||||
|
.fetch_optional(&self.db)
|
||||||
|
.await
|
||||||
|
.context("look up chapter for dispatch")?;
|
||||||
let Some((manga_id, source_url)) = row else {
|
let Some((manga_id, source_url)) = row else {
|
||||||
// Chapter (or its source row) is gone — ack done.
|
// Chapter (or its source row) is gone — ack done.
|
||||||
return Ok(SyncOutcome::Skipped);
|
return Ok(SyncOutcome::Skipped);
|
||||||
|
|||||||
@@ -7,4 +7,5 @@
|
|||||||
|
|
||||||
pub mod extractor;
|
pub mod extractor;
|
||||||
pub mod password;
|
pub mod password;
|
||||||
|
pub mod rate_limit;
|
||||||
pub mod token;
|
pub mod token;
|
||||||
|
|||||||
179
backend/src/auth/rate_limit.rs
Normal file
179
backend/src/auth/rate_limit.rs
Normal file
@@ -0,0 +1,179 @@
|
|||||||
|
//! Per-process token-bucket rate limiter for the auth endpoints.
|
||||||
|
//!
|
||||||
|
//! Protects `/auth/login`, `/auth/register`, and `/auth/me/password`
|
||||||
|
//! from credential stuffing / password spraying / username probing.
|
||||||
|
//!
|
||||||
|
//! The current deploy puts SvelteKit's hooks.server.ts proxy in front
|
||||||
|
//! of axum without forwarding the original client IP (no
|
||||||
|
//! `X-Forwarded-For`), so per-IP buckets would all collapse to the
|
||||||
|
//! proxy container's address. Until the proxy learns to forward the
|
||||||
|
//! peer address, a single global bucket gives equivalent protection
|
||||||
|
//! against mass-attack patterns and trades a small DoS surface
|
||||||
|
//! (legitimate users sharing the limit) for simplicity.
|
||||||
|
//!
|
||||||
|
//! Each `AppState` carries its own [`AuthRateLimiter`] instance, so
|
||||||
|
//! tests run in isolated buckets and won't bleed across `#[sqlx::test]`
|
||||||
|
//! cases that share a process.
|
||||||
|
|
||||||
|
use std::sync::Mutex;
|
||||||
|
use std::time::Instant;
|
||||||
|
|
||||||
|
/// Tunable limits. `per_sec == 0` disables the limiter — used by the
|
||||||
|
/// test harness and by anyone who wants to opt out via env config.
|
||||||
|
#[derive(Clone, Copy, Debug)]
|
||||||
|
pub struct RateLimitConfig {
|
||||||
|
pub per_sec: u32,
|
||||||
|
pub burst: u32,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for RateLimitConfig {
|
||||||
|
/// Disabled by default. The production `AuthConfig::from_env`
|
||||||
|
/// overrides to a real limit; the test harness keeps the default
|
||||||
|
/// so existing tests don't flake against shared buckets.
|
||||||
|
fn default() -> Self {
|
||||||
|
Self {
|
||||||
|
per_sec: 0,
|
||||||
|
burst: 0,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Production defaults: 5 requests/sec sustained, 10-request burst.
|
||||||
|
/// Tight enough to make brute force impractical, loose enough that a
|
||||||
|
/// real user mistyping their password three times in a row doesn't
|
||||||
|
/// hit it.
|
||||||
|
pub const PRODUCTION_PER_SEC: u32 = 5;
|
||||||
|
pub const PRODUCTION_BURST: u32 = 10;
|
||||||
|
|
||||||
|
struct Bucket {
|
||||||
|
tokens: f64,
|
||||||
|
last_refill: Instant,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Outcome of [`AuthRateLimiter::try_acquire`]. When `Denied`, the
|
||||||
|
/// caller can use `retry_after_secs` for a `Retry-After: N` header
|
||||||
|
/// (RFC 6585 §4) so well-behaved clients back off correctly rather
|
||||||
|
/// than retrying in a tight loop.
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub enum AcquireResult {
|
||||||
|
Allowed,
|
||||||
|
Denied { retry_after_secs: u64 },
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Single-bucket token-bucket limiter. `try_acquire` is cheap (one
|
||||||
|
/// mutex acquire, no allocations) so the auth path doesn't pay a real
|
||||||
|
/// cost for the check.
|
||||||
|
pub struct AuthRateLimiter {
|
||||||
|
cfg: RateLimitConfig,
|
||||||
|
bucket: Mutex<Bucket>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl AuthRateLimiter {
|
||||||
|
pub fn new(cfg: RateLimitConfig) -> Self {
|
||||||
|
Self {
|
||||||
|
cfg,
|
||||||
|
bucket: Mutex::new(Bucket {
|
||||||
|
tokens: cfg.burst as f64,
|
||||||
|
last_refill: Instant::now(),
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Consume one token if available. Returns `Denied` with a
|
||||||
|
/// rounded-up seconds-until-refill so the caller can emit a
|
||||||
|
/// `Retry-After` header.
|
||||||
|
pub fn try_acquire(&self) -> AcquireResult {
|
||||||
|
if self.cfg.per_sec == 0 {
|
||||||
|
return AcquireResult::Allowed;
|
||||||
|
}
|
||||||
|
let now = Instant::now();
|
||||||
|
let mut bucket = self.bucket.lock().expect("rate limiter mutex poisoned");
|
||||||
|
let elapsed = now.duration_since(bucket.last_refill).as_secs_f64();
|
||||||
|
bucket.tokens =
|
||||||
|
(bucket.tokens + elapsed * f64::from(self.cfg.per_sec)).min(f64::from(self.cfg.burst));
|
||||||
|
bucket.last_refill = now;
|
||||||
|
if bucket.tokens >= 1.0 {
|
||||||
|
bucket.tokens -= 1.0;
|
||||||
|
AcquireResult::Allowed
|
||||||
|
} else {
|
||||||
|
// ceil((1 - tokens) / per_sec), minimum 1 — a `Retry-After: 0`
|
||||||
|
// would tell clients to retry immediately, which is what we're
|
||||||
|
// actively trying to discourage.
|
||||||
|
let deficit = 1.0 - bucket.tokens;
|
||||||
|
let wait_secs = (deficit / f64::from(self.cfg.per_sec)).ceil() as u64;
|
||||||
|
AcquireResult::Denied {
|
||||||
|
retry_after_secs: wait_secs.max(1),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn disabled_limiter_always_allows() {
|
||||||
|
let rl = AuthRateLimiter::new(RateLimitConfig {
|
||||||
|
per_sec: 0,
|
||||||
|
burst: 0,
|
||||||
|
});
|
||||||
|
for _ in 0..1000 {
|
||||||
|
assert_eq!(rl.try_acquire(), AcquireResult::Allowed);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn burst_lets_through_initial_window_then_blocks() {
|
||||||
|
// 0 refill, burst 3 → first three pass, fourth blocks.
|
||||||
|
let rl = AuthRateLimiter::new(RateLimitConfig {
|
||||||
|
per_sec: 1,
|
||||||
|
burst: 3,
|
||||||
|
});
|
||||||
|
assert_eq!(rl.try_acquire(), AcquireResult::Allowed);
|
||||||
|
assert_eq!(rl.try_acquire(), AcquireResult::Allowed);
|
||||||
|
assert_eq!(rl.try_acquire(), AcquireResult::Allowed);
|
||||||
|
match rl.try_acquire() {
|
||||||
|
AcquireResult::Denied { retry_after_secs } => {
|
||||||
|
// Bucket is at ~0 tokens, refill rate 1/sec → ~1s wait.
|
||||||
|
assert!(
|
||||||
|
retry_after_secs >= 1,
|
||||||
|
"retry_after must be at least 1s, got {retry_after_secs}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
AcquireResult::Allowed => panic!("fourth request must be denied"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn tokens_refill_over_time() {
|
||||||
|
// 10/sec → after ~120ms we should have at least one token back.
|
||||||
|
let rl = AuthRateLimiter::new(RateLimitConfig {
|
||||||
|
per_sec: 10,
|
||||||
|
burst: 1,
|
||||||
|
});
|
||||||
|
assert_eq!(rl.try_acquire(), AcquireResult::Allowed);
|
||||||
|
assert!(matches!(rl.try_acquire(), AcquireResult::Denied { .. }));
|
||||||
|
std::thread::sleep(std::time::Duration::from_millis(150));
|
||||||
|
assert_eq!(
|
||||||
|
rl.try_acquire(),
|
||||||
|
AcquireResult::Allowed,
|
||||||
|
"token should have refilled"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn retry_after_scales_inversely_with_refill_rate() {
|
||||||
|
// 1/sec → wait ~1s after burst exhausted.
|
||||||
|
// 10/sec → wait <1s, but we clamp to a minimum of 1s.
|
||||||
|
let slow = AuthRateLimiter::new(RateLimitConfig {
|
||||||
|
per_sec: 1,
|
||||||
|
burst: 1,
|
||||||
|
});
|
||||||
|
slow.try_acquire();
|
||||||
|
match slow.try_acquire() {
|
||||||
|
AcquireResult::Denied { retry_after_secs } => assert_eq!(retry_after_secs, 1),
|
||||||
|
_ => panic!("expected Denied"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -21,6 +21,7 @@ pub struct AuthConfig {
|
|||||||
pub cookie_secure: bool,
|
pub cookie_secure: bool,
|
||||||
pub cookie_domain: Option<String>,
|
pub cookie_domain: Option<String>,
|
||||||
pub session_ttl_days: i64,
|
pub session_ttl_days: i64,
|
||||||
|
pub rate_limit: crate::auth::rate_limit::RateLimitConfig,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Default for AuthConfig {
|
impl Default for AuthConfig {
|
||||||
@@ -29,6 +30,11 @@ impl Default for AuthConfig {
|
|||||||
cookie_secure: true,
|
cookie_secure: true,
|
||||||
cookie_domain: None,
|
cookie_domain: None,
|
||||||
session_ttl_days: 30,
|
session_ttl_days: 30,
|
||||||
|
// Disabled by default so the test harness inherits a
|
||||||
|
// non-throttling limiter. Production `from_env` overrides
|
||||||
|
// to the [`PRODUCTION_PER_SEC`]/[`PRODUCTION_BURST`]
|
||||||
|
// defaults.
|
||||||
|
rate_limit: crate::auth::rate_limit::RateLimitConfig::default(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -135,6 +141,16 @@ impl Config {
|
|||||||
.ok()
|
.ok()
|
||||||
.filter(|s| !s.is_empty()),
|
.filter(|s| !s.is_empty()),
|
||||||
session_ttl_days: env_i64("SESSION_TTL_DAYS", 30),
|
session_ttl_days: env_i64("SESSION_TTL_DAYS", 30),
|
||||||
|
rate_limit: crate::auth::rate_limit::RateLimitConfig {
|
||||||
|
per_sec: env_u64(
|
||||||
|
"AUTH_RATE_PER_SEC",
|
||||||
|
crate::auth::rate_limit::PRODUCTION_PER_SEC.into(),
|
||||||
|
) as u32,
|
||||||
|
burst: env_u64(
|
||||||
|
"AUTH_RATE_BURST",
|
||||||
|
crate::auth::rate_limit::PRODUCTION_BURST.into(),
|
||||||
|
) as u32,
|
||||||
|
},
|
||||||
},
|
},
|
||||||
upload: UploadConfig {
|
upload: UploadConfig {
|
||||||
max_request_bytes: env_usize("MAX_REQUEST_BYTES", 200 * 1024 * 1024),
|
max_request_bytes: env_usize("MAX_REQUEST_BYTES", 200 * 1024 * 1024),
|
||||||
|
|||||||
@@ -317,10 +317,14 @@ impl WorkerContext {
|
|||||||
// (because a force-refetch race or a job that was re-enqueued
|
// (because a force-refetch race or a job that was re-enqueued
|
||||||
// after a previous one finished), ack done without re-fetching.
|
// after a previous one finished), ack done without re-fetching.
|
||||||
if let JobPayload::SyncChapterContent { chapter_id, .. } = &lease.payload {
|
if let JobPayload::SyncChapterContent { chapter_id, .. } = &lease.payload {
|
||||||
let page_count = crate::repo::chapter::page_count(&self.pool, *chapter_id)
|
let page_count: Option<i32> = sqlx::query_scalar(
|
||||||
.await
|
"SELECT page_count FROM chapters WHERE id = $1",
|
||||||
.ok()
|
)
|
||||||
.flatten();
|
.bind(chapter_id)
|
||||||
|
.fetch_optional(&self.pool)
|
||||||
|
.await
|
||||||
|
.ok()
|
||||||
|
.flatten();
|
||||||
if matches!(page_count, Some(n) if n > 0) {
|
if matches!(page_count, Some(n) if n > 0) {
|
||||||
let _ = jobs::ack_done(&self.pool, lease.id).await;
|
let _ = jobs::ack_done(&self.pool, lease.id).await;
|
||||||
return;
|
return;
|
||||||
|
|||||||
@@ -24,4 +24,3 @@ pub mod pipeline;
|
|||||||
pub mod rate_limit;
|
pub mod rate_limit;
|
||||||
pub mod session;
|
pub mod session;
|
||||||
pub mod source;
|
pub mod source;
|
||||||
pub mod url_utils;
|
|
||||||
|
|||||||
@@ -427,7 +427,11 @@ async fn download_and_store_cover(
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
use crate::crawler::url_utils::origin_of;
|
fn origin_of(url: &str) -> Option<String> {
|
||||||
|
let (scheme, rest) = url.split_once("://")?;
|
||||||
|
let host = rest.split('/').next()?;
|
||||||
|
Some(format!("{scheme}://{host}"))
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
|
|||||||
@@ -98,9 +98,15 @@ impl HostRateLimiters {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// `host_of` was duplicated across session/rate_limit/pipeline; the
|
/// Extract the host (no port) from a URL string. Returns `None` for
|
||||||
// canonical version now lives in `crawler::url_utils`.
|
/// inputs without a `scheme://host` shape — those would never have
|
||||||
use crate::crawler::url_utils::host_of;
|
/// reached the network layer anyway.
|
||||||
|
fn host_of(url: &str) -> Option<String> {
|
||||||
|
let after_scheme = url.split_once("://")?.1;
|
||||||
|
let host_with_port = after_scheme.split('/').next()?;
|
||||||
|
let host = host_with_port.rsplit_once(':').map_or(host_with_port, |(h, _)| h);
|
||||||
|
(!host.is_empty()).then(|| host.to_ascii_lowercase())
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
|
|||||||
@@ -42,9 +42,36 @@ pub enum SessionProbe {
|
|||||||
Transient,
|
Transient,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Re-export so existing callers keep working after the helper moved
|
/// Compute the cookie domain (e.g. `.example.com`) from a start URL.
|
||||||
/// to `crawler::url_utils`. The body lives there.
|
/// The leading dot makes the cookie cover every subdomain — the source
|
||||||
pub use crate::crawler::url_utils::registrable_domain;
|
/// often redirects between `www.` and other prefixes mid-crawl, and a
|
||||||
|
/// host-only cookie would silently drop on the cross-subdomain hop.
|
||||||
|
///
|
||||||
|
/// Caveat: this takes the last two dot-labels, which is wrong for
|
||||||
|
/// multi-part TLDs (`.co.uk`, `.com.br` would resolve to `.co.uk` and
|
||||||
|
/// attach to every site on `.co.uk`). For those, the operator should
|
||||||
|
/// override via `CRAWLER_COOKIE_DOMAIN` rather than relying on this
|
||||||
|
/// function — pulling in the Public Suffix List for one knob isn't
|
||||||
|
/// worth it yet.
|
||||||
|
pub fn registrable_domain(url: &str) -> Option<String> {
|
||||||
|
let after_scheme = url.split_once("://")?.1;
|
||||||
|
let host_with_port = after_scheme.split('/').next()?;
|
||||||
|
let host = host_with_port
|
||||||
|
.rsplit_once(':')
|
||||||
|
.map_or(host_with_port, |(h, _)| h)
|
||||||
|
.to_ascii_lowercase();
|
||||||
|
if host.is_empty() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let labels: Vec<&str> = host.split('.').filter(|l| !l.is_empty()).collect();
|
||||||
|
if labels.len() < 2 {
|
||||||
|
// Bare hostname (e.g. `localhost`) — return as-is, no leading
|
||||||
|
// dot. Setting `.localhost` as cookie domain is invalid.
|
||||||
|
return Some(host);
|
||||||
|
}
|
||||||
|
let registrable = &labels[labels.len() - 2..];
|
||||||
|
Some(format!(".{}", registrable.join(".")))
|
||||||
|
}
|
||||||
|
|
||||||
/// Inject the PHPSESSID cookie into the browser's cookie store for the
|
/// Inject the PHPSESSID cookie into the browser's cookie store for the
|
||||||
/// catalog domain. Must be called before any navigation that depends on
|
/// catalog domain. Must be called before any navigation that depends on
|
||||||
@@ -165,8 +192,44 @@ async fn fetch_probe_html(browser: &Browser, probe_url: &str) -> anyhow::Result<
|
|||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
// registrable_domain tests live in crawler::url_utils now —
|
#[test]
|
||||||
// it's the canonical home for that helper.
|
fn registrable_domain_strips_subdomain() {
|
||||||
|
assert_eq!(
|
||||||
|
registrable_domain("https://www.target-site.com/manga/foo/").as_deref(),
|
||||||
|
Some(".target-site.com")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
registrable_domain("https://m.example.org").as_deref(),
|
||||||
|
Some(".example.org")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn registrable_domain_keeps_two_label_host() {
|
||||||
|
assert_eq!(
|
||||||
|
registrable_domain("https://example.com/").as_deref(),
|
||||||
|
Some(".example.com")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn registrable_domain_handles_port() {
|
||||||
|
assert_eq!(
|
||||||
|
registrable_domain("http://www.foo.bar:8080/x").as_deref(),
|
||||||
|
Some(".foo.bar")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn registrable_domain_bare_hostname_no_leading_dot() {
|
||||||
|
// .localhost would be invalid as a cookie Domain.
|
||||||
|
assert_eq!(registrable_domain("http://localhost:5173").as_deref(), Some("localhost"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn registrable_domain_returns_none_for_garbage() {
|
||||||
|
assert!(registrable_domain("not a url").is_none());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn classify_probe_ok_when_logo_and_avatar_present() {
|
fn classify_probe_ok_when_logo_and_avatar_present() {
|
||||||
|
|||||||
@@ -1,194 +0,0 @@
|
|||||||
//! Centralised URL helpers for the crawler subsystem.
|
|
||||||
//!
|
|
||||||
//! Three near-identical hand-rolled URL parsers used to live in
|
|
||||||
//! `crawler::session`, `crawler::rate_limit`, and `crawler::pipeline`
|
|
||||||
//! respectively, each with subtly different edge-case behaviour
|
|
||||||
//! around port handling and IPv6 literals. They're consolidated here
|
|
||||||
//! so the divergence can't drift again.
|
|
||||||
//!
|
|
||||||
//! The hand-rolled implementations are kept intentionally — they
|
|
||||||
//! preserve the exact semantics every existing test pins. A future
|
|
||||||
//! refactor can switch to `reqwest::Url` if it can be done without
|
|
||||||
//! changing those semantics.
|
|
||||||
|
|
||||||
/// Lowercased host (no port). Returns `None` for inputs without a
|
|
||||||
/// `scheme://host` shape — those would never have reached the network
|
|
||||||
/// layer anyway. Used by the per-host rate limiter as its bucket key.
|
|
||||||
///
|
|
||||||
/// IPv6 literals are kept in their `[::1]` bracketed form so the
|
|
||||||
/// `rsplit_once(':')` port-stripping logic doesn't split inside the
|
|
||||||
/// address (e.g. `https://[::1]/foo` used to return `"[:"` because
|
|
||||||
/// the rightmost `:` is inside the literal). Buckets keyed by
|
|
||||||
/// `[::1]` vs `::1` are still uniquely-per-host; the brackets are
|
|
||||||
/// cosmetic.
|
|
||||||
pub fn host_of(url: &str) -> Option<String> {
|
|
||||||
let after_scheme = url.split_once("://")?.1;
|
|
||||||
let host_with_port = after_scheme.split('/').next()?;
|
|
||||||
let host = if host_with_port.starts_with('[') {
|
|
||||||
// IPv6 literal: keep through the closing bracket. There may
|
|
||||||
// be a trailing `:port` after `]`; strip only that.
|
|
||||||
match host_with_port.rfind(']') {
|
|
||||||
Some(end) => &host_with_port[..=end],
|
|
||||||
None => host_with_port,
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
// Hostnames and IPv4 literals: trailing `:port` (if any) is
|
|
||||||
// after the last `:`.
|
|
||||||
host_with_port
|
|
||||||
.rsplit_once(':')
|
|
||||||
.map_or(host_with_port, |(h, _)| h)
|
|
||||||
};
|
|
||||||
(!host.is_empty()).then(|| host.to_ascii_lowercase())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// `scheme://host` with no path or port stripping. Used by the metadata
|
|
||||||
/// pass to seed `sources.base_url` from `CRAWLER_START_URL`.
|
|
||||||
pub fn origin_of(url: &str) -> Option<String> {
|
|
||||||
let (scheme, rest) = url.split_once("://")?;
|
|
||||||
let host = rest.split('/').next()?;
|
|
||||||
Some(format!("{scheme}://{host}"))
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Approximate registrable-domain calculation: take the last two
|
|
||||||
/// dot-labels of the host, prefix with `.`. Used to set a parent-
|
|
||||||
/// domain cookie so the catalog's `www.` / `m.` redirects don't drop
|
|
||||||
/// the cookie mid-crawl.
|
|
||||||
///
|
|
||||||
/// Caveat: wrong for multi-part TLDs (`.co.uk`, `.com.br`). The
|
|
||||||
/// operator can override via `CRAWLER_COOKIE_DOMAIN`; pulling in the
|
|
||||||
/// Public Suffix List for one knob isn't worth it yet.
|
|
||||||
///
|
|
||||||
/// Bare hostnames (e.g. `localhost`) return the host as-is, with no
|
|
||||||
/// leading dot — setting `.localhost` as a cookie domain is invalid.
|
|
||||||
/// IPv6 literals (e.g. `[::1]`) are returned bracketed and unchanged;
|
|
||||||
/// the browser will reject them as a cookie `Domain` anyway, but the
|
|
||||||
/// representation stays sensible. Same `starts_with('[')` branch as
|
|
||||||
/// [`host_of`] for consistent IPv6 handling across the module.
|
|
||||||
pub fn registrable_domain(url: &str) -> Option<String> {
|
|
||||||
let after_scheme = url.split_once("://")?.1;
|
|
||||||
let host_with_port = after_scheme.split('/').next()?;
|
|
||||||
let host_str = if host_with_port.starts_with('[') {
|
|
||||||
// IPv6 literal: keep through the closing bracket; an optional
|
|
||||||
// `:port` follows `]`.
|
|
||||||
match host_with_port.rfind(']') {
|
|
||||||
Some(end) => &host_with_port[..=end],
|
|
||||||
None => host_with_port,
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
host_with_port
|
|
||||||
.rsplit_once(':')
|
|
||||||
.map_or(host_with_port, |(h, _)| h)
|
|
||||||
};
|
|
||||||
let host = host_str.to_ascii_lowercase();
|
|
||||||
if host.is_empty() {
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
let labels: Vec<&str> = host.split('.').filter(|l| !l.is_empty()).collect();
|
|
||||||
if labels.len() < 2 {
|
|
||||||
return Some(host);
|
|
||||||
}
|
|
||||||
let registrable = &labels[labels.len() - 2..];
|
|
||||||
Some(format!(".{}", registrable.join(".")))
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod tests {
|
|
||||||
use super::*;
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn host_of_strips_port_and_lowercases() {
|
|
||||||
assert_eq!(
|
|
||||||
host_of("https://CDN.Example.com:443/x").as_deref(),
|
|
||||||
Some("cdn.example.com")
|
|
||||||
);
|
|
||||||
assert_eq!(host_of("http://localhost/").as_deref(), Some("localhost"));
|
|
||||||
assert_eq!(host_of("not a url"), None);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn host_of_keeps_bracketed_ipv6_literal_intact() {
|
|
||||||
// Regression: the old impl rsplit_once(':')'d the IPv6 address,
|
|
||||||
// returning "[:" instead of "[::1]". A real IPv6 source would
|
|
||||||
// silently get a wrong rate-limit bucket key.
|
|
||||||
assert_eq!(host_of("https://[::1]/").as_deref(), Some("[::1]"));
|
|
||||||
assert_eq!(host_of("https://[::1]:8080/").as_deref(), Some("[::1]"));
|
|
||||||
assert_eq!(
|
|
||||||
host_of("https://[2001:db8::1]/foo").as_deref(),
|
|
||||||
Some("[2001:db8::1]")
|
|
||||||
);
|
|
||||||
assert_eq!(
|
|
||||||
host_of("https://[2001:db8::1]:443/foo").as_deref(),
|
|
||||||
Some("[2001:db8::1]")
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn origin_of_returns_scheme_and_host() {
|
|
||||||
assert_eq!(
|
|
||||||
origin_of("https://example.com/some/path?q=1").as_deref(),
|
|
||||||
Some("https://example.com")
|
|
||||||
);
|
|
||||||
assert_eq!(origin_of("garbage"), None);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn registrable_domain_strips_subdomain() {
|
|
||||||
assert_eq!(
|
|
||||||
registrable_domain("https://www.target-site.com/manga/foo/").as_deref(),
|
|
||||||
Some(".target-site.com")
|
|
||||||
);
|
|
||||||
assert_eq!(
|
|
||||||
registrable_domain("https://m.example.org").as_deref(),
|
|
||||||
Some(".example.org")
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn registrable_domain_keeps_two_label_host() {
|
|
||||||
assert_eq!(
|
|
||||||
registrable_domain("https://example.com/").as_deref(),
|
|
||||||
Some(".example.com")
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn registrable_domain_handles_port() {
|
|
||||||
assert_eq!(
|
|
||||||
registrable_domain("http://www.foo.bar:8080/x").as_deref(),
|
|
||||||
Some(".foo.bar")
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn registrable_domain_bare_hostname_no_leading_dot() {
|
|
||||||
assert_eq!(
|
|
||||||
registrable_domain("http://localhost:5173").as_deref(),
|
|
||||||
Some("localhost")
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn registrable_domain_returns_none_for_garbage() {
|
|
||||||
assert!(registrable_domain("not a url").is_none());
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn registrable_domain_keeps_bracketed_ipv6_literal_intact() {
|
|
||||||
// Symmetric with host_of's IPv6 fix. The cookie-domain code
|
|
||||||
// won't accept an IP as a `Domain` value, but the function
|
|
||||||
// should at least return a sensible representation rather
|
|
||||||
// than the truncated `"[:"` the old port-stripper produced.
|
|
||||||
assert_eq!(
|
|
||||||
registrable_domain("https://[::1]/").as_deref(),
|
|
||||||
Some("[::1]")
|
|
||||||
);
|
|
||||||
assert_eq!(
|
|
||||||
registrable_domain("https://[::1]:8080/").as_deref(),
|
|
||||||
Some("[::1]")
|
|
||||||
);
|
|
||||||
assert_eq!(
|
|
||||||
registrable_domain("https://[2001:db8::1]/foo").as_deref(),
|
|
||||||
Some("[2001:db8::1]")
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -21,6 +21,11 @@ pub enum AppError {
|
|||||||
PayloadTooLarge(String),
|
PayloadTooLarge(String),
|
||||||
#[error("unsupported media type: {0}")]
|
#[error("unsupported media type: {0}")]
|
||||||
UnsupportedMediaType(String),
|
UnsupportedMediaType(String),
|
||||||
|
/// 429 with an optional `Retry-After` header value (in seconds).
|
||||||
|
#[error("too many requests")]
|
||||||
|
TooManyRequests {
|
||||||
|
retry_after_secs: Option<u64>,
|
||||||
|
},
|
||||||
/// Semantic per-field validation failure. `details` is rendered into the
|
/// Semantic per-field validation failure. `details` is rendered into the
|
||||||
/// envelope so the client can highlight the bad field(s).
|
/// envelope so the client can highlight the bad field(s).
|
||||||
#[error("validation failed")]
|
#[error("validation failed")]
|
||||||
@@ -51,6 +56,7 @@ impl AppError {
|
|||||||
AppError::Conflict(_) => "conflict",
|
AppError::Conflict(_) => "conflict",
|
||||||
AppError::PayloadTooLarge(_) => "payload_too_large",
|
AppError::PayloadTooLarge(_) => "payload_too_large",
|
||||||
AppError::UnsupportedMediaType(_) => "unsupported_media_type",
|
AppError::UnsupportedMediaType(_) => "unsupported_media_type",
|
||||||
|
AppError::TooManyRequests { .. } => "too_many_requests",
|
||||||
AppError::ValidationFailed { .. } => "validation_failed",
|
AppError::ValidationFailed { .. } => "validation_failed",
|
||||||
AppError::Database(sqlx::Error::RowNotFound) => "not_found",
|
AppError::Database(sqlx::Error::RowNotFound) => "not_found",
|
||||||
AppError::Database(_) => "internal_error",
|
AppError::Database(_) => "internal_error",
|
||||||
@@ -79,6 +85,31 @@ impl IntoResponse for AppError {
|
|||||||
AppError::UnsupportedMediaType(msg) => {
|
AppError::UnsupportedMediaType(msg) => {
|
||||||
(StatusCode::UNSUPPORTED_MEDIA_TYPE, msg.clone(), None)
|
(StatusCode::UNSUPPORTED_MEDIA_TYPE, msg.clone(), None)
|
||||||
}
|
}
|
||||||
|
AppError::TooManyRequests { retry_after_secs } => {
|
||||||
|
// Emit `Retry-After: N` (RFC 6585 §4) so a well-behaved
|
||||||
|
// client can back off correctly. Done by building the
|
||||||
|
// response by hand below — the `(status, headers,
|
||||||
|
// body)` tuple shape doesn't fit the standard
|
||||||
|
// `(status, body)` IntoResponse path for the other
|
||||||
|
// variants.
|
||||||
|
let body = json!({
|
||||||
|
"error": {
|
||||||
|
"code": code,
|
||||||
|
"message": "too many requests; slow down",
|
||||||
|
}
|
||||||
|
});
|
||||||
|
let mut resp = (StatusCode::TOO_MANY_REQUESTS, Json(body)).into_response();
|
||||||
|
if let Some(secs) = retry_after_secs {
|
||||||
|
// `HeaderValue: From<u64>` skips both the
|
||||||
|
// intermediate `String` allocation and the
|
||||||
|
// fallible-by-shape `from_str` path.
|
||||||
|
resp.headers_mut().insert(
|
||||||
|
axum::http::header::RETRY_AFTER,
|
||||||
|
axum::http::HeaderValue::from(*secs),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return resp;
|
||||||
|
}
|
||||||
AppError::ValidationFailed { message, details } => (
|
AppError::ValidationFailed { message, details } => (
|
||||||
StatusCode::UNPROCESSABLE_ENTITY,
|
StatusCode::UNPROCESSABLE_ENTITY,
|
||||||
message.clone(),
|
message.clone(),
|
||||||
|
|||||||
@@ -99,11 +99,6 @@ pub async fn list(
|
|||||||
/// Atomically replace the set of authors on a manga. Caller passes a
|
/// Atomically replace the set of authors on a manga. Caller passes a
|
||||||
/// `&mut PgConnection` (`&mut *tx` works) so the delete+upserts run in
|
/// `&mut PgConnection` (`&mut *tx` works) so the delete+upserts run in
|
||||||
/// one transaction with whatever called us.
|
/// one transaction with whatever called us.
|
||||||
///
|
|
||||||
/// Note: `crawler::repo::sync_authors` does a similar replace with the
|
|
||||||
/// same semantics on names. The duplication is intentional — handler
|
|
||||||
/// callers want the `Vec<AuthorRef>` for the API response; the
|
|
||||||
/// crawler doesn't need it and stays inside its own transaction.
|
|
||||||
pub async fn set_for_manga(
|
pub async fn set_for_manga(
|
||||||
conn: &mut PgConnection,
|
conn: &mut PgConnection,
|
||||||
manga_id: Uuid,
|
manga_id: Uuid,
|
||||||
|
|||||||
@@ -29,9 +29,9 @@ pub async fn create(
|
|||||||
|
|
||||||
match result {
|
match result {
|
||||||
Ok(b) => Ok(b),
|
Ok(b) => Ok(b),
|
||||||
Err(sqlx::Error::Database(ref db_err)) if db_err.is_unique_violation() => Err(
|
Err(e) if is_unique_violation(&e) => Err(AppError::Conflict(
|
||||||
AppError::Conflict("bookmark already exists for this manga/chapter".into()),
|
"bookmark already exists for this manga/chapter".into(),
|
||||||
),
|
)),
|
||||||
Err(e) => Err(AppError::Database(e)),
|
Err(e) => Err(AppError::Database(e)),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -97,3 +97,10 @@ pub async fn delete(pool: &PgPool, id: Uuid) -> AppResult<()> {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn is_unique_violation(err: &sqlx::Error) -> bool {
|
||||||
|
if let sqlx::Error::Database(db_err) = err {
|
||||||
|
db_err.code().as_deref() == Some("23505")
|
||||||
|
} else {
|
||||||
|
false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ use sqlx::{PgExecutor, PgPool};
|
|||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
use crate::domain::Chapter;
|
use crate::domain::Chapter;
|
||||||
use crate::error::AppResult;
|
use crate::error::{AppError, AppResult};
|
||||||
|
|
||||||
pub async fn list_for_manga(
|
pub async fn list_for_manga(
|
||||||
pool: &PgPool,
|
pool: &PgPool,
|
||||||
@@ -62,9 +62,10 @@ pub async fn find_by_id_in_manga(
|
|||||||
///
|
///
|
||||||
/// Chapter identity is the row UUID; the same (manga_id, number)
|
/// Chapter identity is the row UUID; the same (manga_id, number)
|
||||||
/// combination can repeat (multiple translations, re-uploads). The
|
/// combination can repeat (multiple translations, re-uploads). The
|
||||||
/// 0013 migration dropped the (manga_id, number) UNIQUE, so duplicate
|
/// `is_unique_violation` branch below is a defensive holdover from
|
||||||
/// inserts succeed by design. If a future migration re-adds any
|
/// 0001's (manga_id, number) UNIQUE — it can no longer fire under
|
||||||
/// uniqueness, surface a 409 by adding a unique-violation arm here.
|
/// normal operation, but we surface a clean 409 if a future migration
|
||||||
|
/// re-adds any chapter uniqueness.
|
||||||
pub async fn create<'e, E: PgExecutor<'e>>(
|
pub async fn create<'e, E: PgExecutor<'e>>(
|
||||||
executor: E,
|
executor: E,
|
||||||
manga_id: Uuid,
|
manga_id: Uuid,
|
||||||
@@ -72,7 +73,7 @@ pub async fn create<'e, E: PgExecutor<'e>>(
|
|||||||
title: Option<&str>,
|
title: Option<&str>,
|
||||||
uploaded_by: Option<Uuid>,
|
uploaded_by: Option<Uuid>,
|
||||||
) -> AppResult<Chapter> {
|
) -> AppResult<Chapter> {
|
||||||
let row = sqlx::query_as::<_, Chapter>(
|
let result = sqlx::query_as::<_, Chapter>(
|
||||||
r#"
|
r#"
|
||||||
INSERT INTO chapters (manga_id, number, title, uploaded_by)
|
INSERT INTO chapters (manga_id, number, title, uploaded_by)
|
||||||
VALUES ($1, $2, $3, $4)
|
VALUES ($1, $2, $3, $4)
|
||||||
@@ -84,58 +85,15 @@ pub async fn create<'e, E: PgExecutor<'e>>(
|
|||||||
.bind(title)
|
.bind(title)
|
||||||
.bind(uploaded_by)
|
.bind(uploaded_by)
|
||||||
.fetch_one(executor)
|
.fetch_one(executor)
|
||||||
.await?;
|
.await;
|
||||||
Ok(row)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Cross-link guard for `POST /bookmarks`: the bookmarks FK accepts
|
match result {
|
||||||
/// any valid chapter id, but a chapter must belong to the bookmark's
|
Ok(c) => Ok(c),
|
||||||
/// manga or the bookmark would dangle on a foreign manga. Handlers
|
Err(e) if is_unique_violation(&e) => Err(AppError::Conflict(format!(
|
||||||
/// call this before the insert and surface `NotFound` when it
|
"chapter {number} conflicts with an existing chapter for this manga"
|
||||||
/// returns `false`.
|
))),
|
||||||
pub async fn belongs_to_manga(
|
Err(e) => Err(AppError::Database(e)),
|
||||||
pool: &PgPool,
|
}
|
||||||
chapter_id: Uuid,
|
|
||||||
manga_id: Uuid,
|
|
||||||
) -> AppResult<bool> {
|
|
||||||
let (exists,): (bool,) = sqlx::query_as(
|
|
||||||
"SELECT EXISTS(SELECT 1 FROM chapters WHERE id = $1 AND manga_id = $2)",
|
|
||||||
)
|
|
||||||
.bind(chapter_id)
|
|
||||||
.bind(manga_id)
|
|
||||||
.fetch_one(pool)
|
|
||||||
.await?;
|
|
||||||
Ok(exists)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Read just the page_count for a chapter. Used by the crawler
|
|
||||||
/// daemon's consumer-side dedup safety net so it can ack-done a job
|
|
||||||
/// whose chapter has already been fetched by a racing worker.
|
|
||||||
pub async fn page_count(pool: &PgPool, id: Uuid) -> sqlx::Result<Option<i32>> {
|
|
||||||
sqlx::query_scalar("SELECT page_count FROM chapters WHERE id = $1")
|
|
||||||
.bind(id)
|
|
||||||
.fetch_optional(pool)
|
|
||||||
.await
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Look up the manga_id + most recent source_url for a chapter. Used
|
|
||||||
/// by the daemon's chapter dispatcher to resolve the URL it needs to
|
|
||||||
/// hand to `content::sync_chapter_content`. Returns `None` if the
|
|
||||||
/// chapter (or its source row) is gone.
|
|
||||||
pub async fn dispatch_target(
|
|
||||||
pool: &PgPool,
|
|
||||||
chapter_id: Uuid,
|
|
||||||
) -> sqlx::Result<Option<(Uuid, String)>> {
|
|
||||||
sqlx::query_as(
|
|
||||||
"SELECT c.manga_id, cs.source_url \
|
|
||||||
FROM chapters c \
|
|
||||||
JOIN chapter_sources cs ON cs.chapter_id = c.id \
|
|
||||||
WHERE c.id = $1 \
|
|
||||||
LIMIT 1",
|
|
||||||
)
|
|
||||||
.bind(chapter_id)
|
|
||||||
.fetch_optional(pool)
|
|
||||||
.await
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn set_page_count<'e, E: PgExecutor<'e>>(
|
pub async fn set_page_count<'e, E: PgExecutor<'e>>(
|
||||||
@@ -151,3 +109,10 @@ pub async fn set_page_count<'e, E: PgExecutor<'e>>(
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn is_unique_violation(err: &sqlx::Error) -> bool {
|
||||||
|
if let sqlx::Error::Database(db_err) = err {
|
||||||
|
db_err.code().as_deref() == Some("23505")
|
||||||
|
} else {
|
||||||
|
false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -61,11 +61,6 @@ pub async fn load_for_mangas(
|
|||||||
/// FK constraint would reject them, so we filter upstream rather than
|
/// FK constraint would reject them, so we filter upstream rather than
|
||||||
/// surface a 500 here. (The API layer validates the set against
|
/// surface a 500 here. (The API layer validates the set against
|
||||||
/// `list_all` first.)
|
/// `list_all` first.)
|
||||||
///
|
|
||||||
/// Note: `crawler::repo::sync_genres` does a similar replace, but by
|
|
||||||
/// *name* and with auto-create of unseen genres — the crawler can't
|
|
||||||
/// validate against the curated vocabulary on its own. Both paths are
|
|
||||||
/// intentional; don't merge them without preserving that semantic.
|
|
||||||
pub async fn set_for_manga(
|
pub async fn set_for_manga(
|
||||||
conn: &mut PgConnection,
|
conn: &mut PgConnection,
|
||||||
manga_id: Uuid,
|
manga_id: Uuid,
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ pub async fn create(pool: &PgPool, username: &str, password_hash: &str) -> AppRe
|
|||||||
|
|
||||||
match result {
|
match result {
|
||||||
Ok(user) => Ok(user),
|
Ok(user) => Ok(user),
|
||||||
Err(sqlx::Error::Database(ref db_err)) if db_err.is_unique_violation() => {
|
Err(e) if is_unique_violation(&e) => {
|
||||||
Err(AppError::Conflict("username is already taken".into()))
|
Err(AppError::Conflict("username is already taken".into()))
|
||||||
}
|
}
|
||||||
Err(e) => Err(AppError::Database(e)),
|
Err(e) => Err(AppError::Database(e)),
|
||||||
@@ -56,3 +56,10 @@ pub async fn find_by_id(pool: &PgPool, id: Uuid) -> AppResult<Option<User>> {
|
|||||||
Ok(row)
|
Ok(row)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn is_unique_violation(err: &sqlx::Error) -> bool {
|
||||||
|
if let sqlx::Error::Database(db_err) = err {
|
||||||
|
db_err.code().as_deref() == Some("23505")
|
||||||
|
} else {
|
||||||
|
false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -567,6 +567,81 @@ async fn user_a_cannot_delete_user_b_token(pool: PgPool) {
|
|||||||
assert_eq!(resp.status(), StatusCode::NO_CONTENT);
|
assert_eq!(resp.status(), StatusCode::NO_CONTENT);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Brute-force / spray protection: at default production limits, a
|
||||||
|
/// tight loop of /auth/login attempts should burst through the bucket
|
||||||
|
/// and then 429 every subsequent request until the bucket refills.
|
||||||
|
#[sqlx::test(migrations = "./migrations")]
|
||||||
|
async fn login_rate_limited_under_burst_pressure(pool: PgPool) {
|
||||||
|
let h = common::harness_with_auth_rate_limit(pool, 1, 3);
|
||||||
|
|
||||||
|
// Register a victim so the wrong-password branch is real work.
|
||||||
|
let _ = h
|
||||||
|
.app
|
||||||
|
.clone()
|
||||||
|
.oneshot(common::post_json("/api/v1/auth/register", creds("victim")))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
// Register consumed one token from the burst-3 bucket. Fire 30
|
||||||
|
// wrong-password logins back-to-back; with per_sec=1 the refill
|
||||||
|
// is too slow to keep up and at least one must come back 429.
|
||||||
|
let mut saw_429 = false;
|
||||||
|
for _ in 0..30 {
|
||||||
|
let resp = h
|
||||||
|
.app
|
||||||
|
.clone()
|
||||||
|
.oneshot(common::post_json(
|
||||||
|
"/api/v1/auth/login",
|
||||||
|
json!({ "username": "victim", "password": "wrong" }),
|
||||||
|
))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
if resp.status() == StatusCode::TOO_MANY_REQUESTS {
|
||||||
|
// RFC 6585 §4: 429 SHOULD include a Retry-After header. The
|
||||||
|
// value is in seconds; with per_sec=1 the bucket needs ~1s
|
||||||
|
// to refill, so the header should be 1 or 2.
|
||||||
|
let retry_after = resp
|
||||||
|
.headers()
|
||||||
|
.get(axum::http::header::RETRY_AFTER)
|
||||||
|
.and_then(|v| v.to_str().ok())
|
||||||
|
.and_then(|s| s.parse::<u32>().ok())
|
||||||
|
.expect("Retry-After header present and numeric");
|
||||||
|
assert!(
|
||||||
|
retry_after >= 1,
|
||||||
|
"Retry-After must be at least 1s, got {retry_after}"
|
||||||
|
);
|
||||||
|
let body = common::body_json(resp).await;
|
||||||
|
assert_eq!(body["error"]["code"], "too_many_requests");
|
||||||
|
saw_429 = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert!(
|
||||||
|
saw_429,
|
||||||
|
"expected at least one 429 within 30 rapid login attempts"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Default (test-harness) limits are disabled, so existing tests that
|
||||||
|
/// fire multiple auth requests don't start failing.
|
||||||
|
#[sqlx::test(migrations = "./migrations")]
|
||||||
|
async fn default_test_harness_does_not_rate_limit(pool: PgPool) {
|
||||||
|
let h = common::harness(pool);
|
||||||
|
for i in 0..50 {
|
||||||
|
let resp = h
|
||||||
|
.app
|
||||||
|
.clone()
|
||||||
|
.oneshot(common::post_json(
|
||||||
|
"/api/v1/auth/login",
|
||||||
|
json!({ "username": format!("nobody-{i}"), "password": "x" }),
|
||||||
|
))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
// None of these should be 429 — only 401.
|
||||||
|
assert_eq!(resp.status(), StatusCode::UNAUTHORIZED, "iter {i}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[sqlx::test(migrations = "./migrations")]
|
#[sqlx::test(migrations = "./migrations")]
|
||||||
async fn delete_unknown_token_is_404(pool: PgPool) {
|
async fn delete_unknown_token_is_404(pool: PgPool) {
|
||||||
let h = common::harness(pool);
|
let h = common::harness(pool);
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ use tempfile::TempDir;
|
|||||||
use tower::ServiceExt;
|
use tower::ServiceExt;
|
||||||
|
|
||||||
use mangalord::app::{router, AppState};
|
use mangalord::app::{router, AppState};
|
||||||
|
use mangalord::auth::rate_limit::AuthRateLimiter;
|
||||||
use mangalord::config::{AuthConfig, UploadConfig};
|
use mangalord::config::{AuthConfig, UploadConfig};
|
||||||
use mangalord::storage::{LocalStorage, Storage, StorageError, StreamingFile};
|
use mangalord::storage::{LocalStorage, Storage, StorageError, StreamingFile};
|
||||||
|
|
||||||
@@ -49,20 +50,51 @@ fn harness_inner(
|
|||||||
storage: Arc<dyn Storage>,
|
storage: Arc<dyn Storage>,
|
||||||
storage_dir: TempDir,
|
storage_dir: TempDir,
|
||||||
) -> Harness {
|
) -> Harness {
|
||||||
|
harness_with_auth_config(pool, storage, storage_dir, AuthConfig {
|
||||||
|
cookie_secure: false,
|
||||||
|
..AuthConfig::default()
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn harness_with_auth_config(
|
||||||
|
pool: PgPool,
|
||||||
|
storage: Arc<dyn Storage>,
|
||||||
|
storage_dir: TempDir,
|
||||||
|
auth: AuthConfig,
|
||||||
|
) -> Harness {
|
||||||
|
let auth_limiter = Arc::new(AuthRateLimiter::new(auth.rate_limit));
|
||||||
let state = AppState {
|
let state = AppState {
|
||||||
db: pool,
|
db: pool,
|
||||||
storage,
|
storage,
|
||||||
auth: AuthConfig { cookie_secure: false, ..AuthConfig::default() },
|
auth,
|
||||||
upload: UploadConfig {
|
upload: UploadConfig {
|
||||||
// Keep file caps small in tests so the size-cap path is cheap to
|
// Keep file caps small in tests so the size-cap path is cheap to
|
||||||
// exercise without producing tens of MBs of bytes.
|
// exercise without producing tens of MBs of bytes.
|
||||||
max_request_bytes: 4 * 1024 * 1024,
|
max_request_bytes: 4 * 1024 * 1024,
|
||||||
max_file_bytes: 256 * 1024,
|
max_file_bytes: 256 * 1024,
|
||||||
},
|
},
|
||||||
|
auth_limiter,
|
||||||
};
|
};
|
||||||
Harness { app: router(state), _storage_dir: storage_dir }
|
Harness { app: router(state), _storage_dir: storage_dir }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Like [`harness`] but configures a tight auth rate limit. Used by
|
||||||
|
/// the brute-force-rate-limiting test.
|
||||||
|
pub fn harness_with_auth_rate_limit(
|
||||||
|
pool: PgPool,
|
||||||
|
per_sec: u32,
|
||||||
|
burst: u32,
|
||||||
|
) -> Harness {
|
||||||
|
let storage_dir = tempfile::tempdir().expect("tempdir");
|
||||||
|
let storage = Arc::new(LocalStorage::new(storage_dir.path()));
|
||||||
|
let auth = AuthConfig {
|
||||||
|
cookie_secure: false,
|
||||||
|
rate_limit: mangalord::auth::rate_limit::RateLimitConfig { per_sec, burst },
|
||||||
|
..AuthConfig::default()
|
||||||
|
};
|
||||||
|
harness_with_auth_config(pool, storage, storage_dir, auth)
|
||||||
|
}
|
||||||
|
|
||||||
/// Wraps a real `Storage` and fails on the N-th `put` call so tests can
|
/// Wraps a real `Storage` and fails on the N-th `put` call so tests can
|
||||||
/// assert that handlers roll their DB writes back when storage errors
|
/// assert that handlers roll their DB writes back when storage errors
|
||||||
/// mid-upload. Reads and other operations delegate to `inner`.
|
/// mid-upload. Reads and other operations delegate to `inner`.
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "mangalord-frontend",
|
"name": "mangalord-frontend",
|
||||||
"version": "0.34.0",
|
"version": "0.35.0",
|
||||||
"private": true,
|
"private": true,
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
|
|||||||
Reference in New Issue
Block a user