Compare commits

..

1 Commits

Author SHA1 Message Date
MechaCat02
f276f0b55e chore: run containers as non-root, add HEALTHCHECK, npm ci
Backend: new `app` user (UID 10001), STORAGE_DIR pre-chowned so the
named volume inherits ownership, curl installed for the HEALTHCHECK
that pings /api/v1/health. The crawler's Chromium uses --no-sandbox
already so dropping privileges costs nothing operationally.

Frontend: switch `npm install` to `npm ci` (matches CI; deterministic
versions; refuses to silently rewrite package-lock.json mid-build).
Run as the built-in `node` user via --chown=node:node, add a busybox
wget HEALTHCHECK on port 3000.

Both images now expose container-level health so orchestrators can
take a wedged container out of rotation instead of letting it keep
serving timeouts.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-28 20:01:01 +02:00
14 changed files with 101 additions and 812 deletions

View File

@@ -44,14 +44,6 @@ MAX_REQUEST_BYTES=209715200
# Default 20 MiB. # Default 20 MiB.
MAX_FILE_BYTES=20971520 MAX_FILE_BYTES=20971520
# ----- Crawler download safety -----
# Hosts the crawler is allowed to fetch images/covers from, in addition
# to CRAWLER_START_URL's host and CRAWLER_CDN_HOST. Comma-separated.
# Defends against SSRF via scraped <img src="http://10.0.0.1/...">.
CRAWLER_DOWNLOAD_ALLOWLIST=
# Hard cap on a single image body. Default 32 MiB.
CRAWLER_MAX_IMAGE_BYTES=33554432
# ----- Frontend ----- # ----- Frontend -----
# The frontend container runs SvelteKit's Node adapter on :3000 and # The frontend container runs SvelteKit's Node adapter on :3000 and
# proxies /api/* to BACKEND_URL via src/hooks.server.ts. In compose the # proxies /api/* to BACKEND_URL via src/hooks.server.ts. In compose the

18
backend/Cargo.lock generated
View File

@@ -1470,7 +1470,7 @@ checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
[[package]] [[package]]
name = "mangalord" name = "mangalord"
version = "0.34.1" version = "0.34.0"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"argon2", "argon2",
@@ -2324,7 +2324,6 @@ dependencies = [
"cookie", "cookie",
"cookie_store", "cookie_store",
"futures-core", "futures-core",
"futures-util",
"http", "http",
"http-body", "http-body",
"http-body-util", "http-body-util",
@@ -2344,14 +2343,12 @@ dependencies = [
"sync_wrapper", "sync_wrapper",
"tokio", "tokio",
"tokio-rustls", "tokio-rustls",
"tokio-util",
"tower", "tower",
"tower-http", "tower-http",
"tower-service", "tower-service",
"url", "url",
"wasm-bindgen", "wasm-bindgen",
"wasm-bindgen-futures", "wasm-bindgen-futures",
"wasm-streams",
"web-sys", "web-sys",
"webpki-roots", "webpki-roots",
] ]
@@ -3530,19 +3527,6 @@ dependencies = [
"wasmparser", "wasmparser",
] ]
[[package]]
name = "wasm-streams"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65"
dependencies = [
"futures-util",
"js-sys",
"wasm-bindgen",
"wasm-bindgen-futures",
"web-sys",
]
[[package]] [[package]]
name = "wasmparser" name = "wasmparser"
version = "0.244.0" version = "0.244.0"

View File

@@ -1,6 +1,6 @@
[package] [package]
name = "mangalord" name = "mangalord"
version = "0.34.1" version = "0.34.0"
edition = "2021" edition = "2021"
default-run = "mangalord" default-run = "mangalord"
@@ -46,7 +46,7 @@ futures-util = "0.3"
bytes = "1" bytes = "1"
chromiumoxide = { version = "0.7", features = ["tokio-runtime", "_fetcher-rusttls-tokio"], default-features = false } chromiumoxide = { version = "0.7", features = ["tokio-runtime", "_fetcher-rusttls-tokio"], default-features = false }
scraper = "0.20" scraper = "0.20"
reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "socks", "cookies", "stream"] } reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "socks", "cookies"] }
[dev-dependencies] [dev-dependencies]
tempfile = "3" tempfile = "3"

View File

@@ -19,12 +19,49 @@ COPY migrations ./migrations
RUN touch src/main.rs src/lib.rs && cargo build --locked --release RUN touch src/main.rs src/lib.rs && cargo build --locked --release
FROM debian:bookworm-slim FROM debian:bookworm-slim
# `curl` is for the container HEALTHCHECK; `ca-certificates` is for
# outbound HTTPS (crawler covers/pages).
RUN apt-get update \ RUN apt-get update \
&& apt-get install -y --no-install-recommends ca-certificates \ && apt-get install -y --no-install-recommends ca-certificates curl \
&& rm -rf /var/lib/apt/lists/* && rm -rf /var/lib/apt/lists/*
# Non-root runtime user. The API binary doesn't need any root
# privilege; the crawler daemon's Chromium launcher uses --no-sandbox
# precisely because user-namespace sandboxing is fragile, so dropping
# privileges costs nothing operationally and shrinks the blast radius
# of any RCE.
ARG APP_UID=10001
ARG APP_GID=10001
RUN groupadd --system --gid ${APP_GID} app \
&& useradd --system --uid ${APP_UID} --gid app --home-dir /home/app --create-home --shell /usr/sbin/nologin app
WORKDIR /app WORKDIR /app
COPY --from=builder /app/target/release/mangalord /usr/local/bin/mangalord COPY --from=builder /app/target/release/mangalord /usr/local/bin/mangalord
COPY --from=builder /app/migrations /app/migrations COPY --from=builder /app/migrations /app/migrations
ENV STORAGE_DIR=/var/lib/mangalord/storage ENV STORAGE_DIR=/var/lib/mangalord/storage
# Pre-create the storage dir so the entrypoint doesn't need to
# mkdir-as-root and so the named volume mount inherits the right
# ownership.
#
# UPGRADE NOTE for operators: if you're moving from an older image
# that ran as root, the existing `storage-data` volume has files owned
# by UID 0 and the new UID-10001 user can't write them. Run once
# before the upgrade:
# docker compose run --rm --user 0 backend \
# chown -R 10001:10001 /var/lib/mangalord/storage
# (Postgres is unaffected — that image's `postgres` user UID hasn't
# changed.)
RUN mkdir -p ${STORAGE_DIR} \
&& chown -R app:app ${STORAGE_DIR} /app /home/app
USER app
EXPOSE 8080 EXPOSE 8080
# `--start-period` is generous because first boot runs sqlx::migrate
# against postgres which can take a few seconds; subsequent restarts
# are sub-second.
HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 \
CMD curl -fsS http://localhost:8080/api/v1/health > /dev/null || exit 1
CMD ["mangalord"] CMD ["mangalord"]

View File

@@ -19,7 +19,6 @@ use crate::crawler::daemon::{self, ChapterDispatcher, DaemonConfig, MetadataPass
use crate::crawler::jobs::JobPayload; use crate::crawler::jobs::JobPayload;
use crate::crawler::pipeline::{self, MetadataStats}; use crate::crawler::pipeline::{self, MetadataStats};
use crate::crawler::rate_limit::HostRateLimiters; use crate::crawler::rate_limit::HostRateLimiters;
use crate::crawler::safety::DownloadAllowlist;
use crate::crawler::session; use crate::crawler::session;
use crate::crawler::source::{target as target_source, DiscoverMode}; use crate::crawler::source::{target as target_source, DiscoverMode};
use crate::repo; use crate::repo;
@@ -154,8 +153,6 @@ async fn spawn_crawler_daemon(
start_url: url.clone(), start_url: url.clone(),
mode_pref: cfg.mode, mode_pref: cfg.mode,
incremental_stop_after: cfg.incremental_stop_after, incremental_stop_after: cfg.incremental_stop_after,
download_allowlist: cfg.download_allowlist.clone(),
max_image_bytes: cfg.max_image_bytes,
}); });
m m
}); });
@@ -166,8 +163,6 @@ async fn spawn_crawler_daemon(
storage: Arc::clone(&storage), storage: Arc::clone(&storage),
http, http,
rate: Arc::clone(&rate), rate: Arc::clone(&rate),
download_allowlist: cfg.download_allowlist.clone(),
max_image_bytes: cfg.max_image_bytes,
}); });
// Shared cancellation: daemon shutdown cancels the BrowserManager's // Shared cancellation: daemon shutdown cancels the BrowserManager's
@@ -221,8 +216,6 @@ struct RealMetadataPass {
start_url: String, start_url: String,
mode_pref: CrawlerModePref, mode_pref: CrawlerModePref,
incremental_stop_after: usize, incremental_stop_after: usize,
download_allowlist: DownloadAllowlist,
max_image_bytes: usize,
} }
#[async_trait] #[async_trait]
@@ -245,8 +238,6 @@ impl MetadataPass for RealMetadataPass {
0, 0,
false, false,
mode, mode,
&self.download_allowlist,
self.max_image_bytes,
) )
.await .await
} }
@@ -302,8 +293,6 @@ struct RealChapterDispatcher {
storage: Arc<dyn Storage>, storage: Arc<dyn Storage>,
http: reqwest::Client, http: reqwest::Client,
rate: Arc<HostRateLimiters>, rate: Arc<HostRateLimiters>,
download_allowlist: DownloadAllowlist,
max_image_bytes: usize,
} }
#[async_trait] #[async_trait]
@@ -342,8 +331,6 @@ impl ChapterDispatcher for RealChapterDispatcher {
manga_id, manga_id,
&source_url, &source_url,
false, false,
&self.download_allowlist,
self.max_image_bytes,
) )
.await?; .await?;
drop(lease); drop(lease);

View File

@@ -229,33 +229,6 @@ async fn run(
} }
let rate = Arc::new(rate); let rate = Arc::new(rate);
// SSRF defence: only download from the catalog host + CDN host
// (plus optional CRAWLER_DOWNLOAD_ALLOWLIST extras), and cap
// single-image downloads at CRAWLER_MAX_IMAGE_BYTES bytes.
let mut allowlist =
mangalord::crawler::safety::DownloadAllowlist::new();
if let Ok(parsed) = reqwest::Url::parse(start_url) {
if let Some(h) = parsed.host_str() {
allowlist = allowlist.allow(h);
}
}
if let Some(host) = cdn_host {
allowlist = allowlist.allow(host);
}
if let Ok(extras) = std::env::var("CRAWLER_DOWNLOAD_ALLOWLIST") {
for piece in extras.split(',') {
let trimmed = piece.trim();
if !trimmed.is_empty() {
allowlist = allowlist.allow(trimmed);
}
}
}
let max_image_bytes: usize = std::env::var("CRAWLER_MAX_IMAGE_BYTES")
.ok()
.and_then(|s| s.parse().ok())
.unwrap_or(mangalord::crawler::safety::DEFAULT_MAX_IMAGE_BYTES);
let allowlist = Arc::new(allowlist);
let stats = pipeline::run_metadata_pass( let stats = pipeline::run_metadata_pass(
manager.as_ref(), manager.as_ref(),
db, db,
@@ -266,8 +239,6 @@ async fn run(
limit, limit,
skip_chapters, skip_chapters,
mode, mode,
allowlist.as_ref(),
max_image_bytes,
) )
.await?; .await?;
tracing::info!(?stats, "metadata pass complete"); tracing::info!(?stats, "metadata pass complete");
@@ -282,8 +253,6 @@ async fn run(
"target", "target",
chapter_workers, chapter_workers,
force_refetch_chapters, force_refetch_chapters,
Arc::clone(&allowlist),
max_image_bytes,
) )
.await?; .await?;
} }
@@ -307,8 +276,6 @@ async fn sync_bookmarked_chapter_content(
source_id: &str, source_id: &str,
workers: usize, workers: usize,
force_refetch: bool, force_refetch: bool,
allowlist: Arc<mangalord::crawler::safety::DownloadAllowlist>,
max_image_bytes: usize,
) -> anyhow::Result<()> { ) -> anyhow::Result<()> {
let pending: Vec<(Uuid, Uuid, String)> = sqlx::query_as( let pending: Vec<(Uuid, Uuid, String)> = sqlx::query_as(
r#" r#"
@@ -345,7 +312,6 @@ async fn sync_bookmarked_chapter_content(
let storage = Arc::clone(&storage); let storage = Arc::clone(&storage);
let rate = Arc::clone(&rate); let rate = Arc::clone(&rate);
let manager = Arc::clone(&manager); let manager = Arc::clone(&manager);
let allowlist = Arc::clone(&allowlist);
let stats = &stats; let stats = &stats;
async move { async move {
if session_expired.load(std::sync::atomic::Ordering::Relaxed) { if session_expired.load(std::sync::atomic::Ordering::Relaxed) {
@@ -370,8 +336,6 @@ async fn sync_bookmarked_chapter_content(
manga_id, manga_id,
&source_url, &source_url,
force_refetch, force_refetch,
allowlist.as_ref(),
max_image_bytes,
) )
.await; .await;
drop(lease); drop(lease);

View File

@@ -5,7 +5,6 @@ use chrono::NaiveTime;
use chrono_tz::Tz; use chrono_tz::Tz;
use crate::crawler::browser::LaunchOptions; use crate::crawler::browser::LaunchOptions;
use crate::crawler::safety::{DownloadAllowlist, DEFAULT_MAX_IMAGE_BYTES};
use crate::crawler::source::DiscoverMode; use crate::crawler::source::DiscoverMode;
/// What `CRAWLER_MODE` was set to. `Auto` is the daemon's default — /// What `CRAWLER_MODE` was set to. `Auto` is the daemon's default —
@@ -94,13 +93,6 @@ pub struct CrawlerConfig {
/// `stop_after_unchanged` threshold supplied to Incremental in both /// `stop_after_unchanged` threshold supplied to Incremental in both
/// `Auto` (post-seed) and `Explicit(Incremental)` modes. /// `Auto` (post-seed) and `Explicit(Incremental)` modes.
pub incremental_stop_after: usize, pub incremental_stop_after: usize,
/// Hosts the crawler is allowed to download images / covers from.
/// Always seeded with the host of `start_url` and (when set) the
/// configured `cdn_host`. Additional hosts can be added via
/// `CRAWLER_DOWNLOAD_ALLOWLIST` (comma-separated).
pub download_allowlist: DownloadAllowlist,
/// Hard upper bound on a single image download. Defaults to 32 MiB.
pub max_image_bytes: usize,
} }
impl Default for CrawlerConfig { impl Default for CrawlerConfig {
@@ -123,8 +115,6 @@ impl Default for CrawlerConfig {
browser: LaunchOptions::headless(), browser: LaunchOptions::headless(),
mode: CrawlerModePref::Auto, mode: CrawlerModePref::Auto,
incremental_stop_after: 20, incremental_stop_after: 20,
download_allowlist: DownloadAllowlist::new(),
max_image_bytes: DEFAULT_MAX_IMAGE_BYTES,
} }
} }
} }
@@ -182,14 +172,6 @@ impl CrawlerConfig {
let incremental_stop_after = let incremental_stop_after =
env_u64("CRAWLER_INCREMENTAL_STOP_AFTER", 20).max(1) as usize; env_u64("CRAWLER_INCREMENTAL_STOP_AFTER", 20).max(1) as usize;
let mode = parse_mode_env(incremental_stop_after)?; let mode = parse_mode_env(incremental_stop_after)?;
let start_url = std::env::var("CRAWLER_START_URL")
.ok()
.filter(|s| !s.trim().is_empty());
let cdn_host = std::env::var("CRAWLER_CDN_HOST")
.ok()
.filter(|s| !s.trim().is_empty());
let download_allowlist =
build_download_allowlist(start_url.as_deref(), cdn_host.as_deref());
Ok(Self { Ok(Self {
daemon_enabled: env_bool("CRAWLER_DAEMON", true), daemon_enabled: env_bool("CRAWLER_DAEMON", true),
daily_at, daily_at,
@@ -197,9 +179,13 @@ impl CrawlerConfig {
idle_timeout: Duration::from_secs(env_u64("CRAWLER_IDLE_TIMEOUT_S", 600)), idle_timeout: Duration::from_secs(env_u64("CRAWLER_IDLE_TIMEOUT_S", 600)),
chapter_workers: env_u64("CRAWLER_CHAPTER_WORKERS", 1).max(1) as usize, chapter_workers: env_u64("CRAWLER_CHAPTER_WORKERS", 1).max(1) as usize,
retention_days: env_u64("CRAWLER_JOB_RETENTION_DAYS", 7) as u32, retention_days: env_u64("CRAWLER_JOB_RETENTION_DAYS", 7) as u32,
start_url, start_url: std::env::var("CRAWLER_START_URL")
.ok()
.filter(|s| !s.trim().is_empty()),
rate_ms: env_u64("CRAWLER_RATE_MS", 1000), rate_ms: env_u64("CRAWLER_RATE_MS", 1000),
cdn_host, cdn_host: std::env::var("CRAWLER_CDN_HOST")
.ok()
.filter(|s| !s.trim().is_empty()),
cdn_rate_ms: env_u64("CRAWLER_CDN_RATE_MS", env_u64("CRAWLER_RATE_MS", 1000)), cdn_rate_ms: env_u64("CRAWLER_CDN_RATE_MS", env_u64("CRAWLER_RATE_MS", 1000)),
phpsessid: std::env::var("CRAWLER_PHPSESSID") phpsessid: std::env::var("CRAWLER_PHPSESSID")
.ok() .ok()
@@ -216,45 +202,10 @@ impl CrawlerConfig {
browser: LaunchOptions::from_env(), browser: LaunchOptions::from_env(),
mode, mode,
incremental_stop_after, incremental_stop_after,
download_allowlist,
max_image_bytes: env_usize("CRAWLER_MAX_IMAGE_BYTES", DEFAULT_MAX_IMAGE_BYTES),
}) })
} }
} }
/// Build the download allowlist from env. Always includes
/// `CRAWLER_START_URL`'s host (so the crawler can fetch covers from
/// the catalog itself) and `CRAWLER_CDN_HOST` when set. Additional
/// hosts can be supplied via `CRAWLER_DOWNLOAD_ALLOWLIST` (comma-
/// separated). Empty by default — meaning the crawler refuses to
/// download anything when no source is configured, which is the safe
/// fail-closed posture.
fn build_download_allowlist(
start_url: Option<&str>,
cdn_host: Option<&str>,
) -> DownloadAllowlist {
let mut allow = DownloadAllowlist::new();
if let Some(url) = start_url {
if let Ok(parsed) = reqwest::Url::parse(url) {
if let Some(h) = parsed.host_str() {
allow = allow.allow(h);
}
}
}
if let Some(host) = cdn_host {
allow = allow.allow(host);
}
if let Ok(extras) = std::env::var("CRAWLER_DOWNLOAD_ALLOWLIST") {
for piece in extras.split(',') {
let trimmed = piece.trim();
if !trimmed.is_empty() {
allow = allow.allow(trimmed);
}
}
}
allow
}
/// Parse `CRAWLER_MODE`. Empty/unset → `Auto`. Recognized values are /// Parse `CRAWLER_MODE`. Empty/unset → `Auto`. Recognized values are
/// `auto`, `backfill`, and `incremental` (case-insensitive). Anything /// `auto`, `backfill`, and `incremental` (case-insensitive). Anything
/// else is a hard error so a typo can't silently fall through to the /// else is a hard error so a typo can't silently fall through to the

View File

@@ -18,8 +18,7 @@ use uuid::Uuid;
use crate::crawler::detect::PageError; use crate::crawler::detect::PageError;
use crate::crawler::rate_limit::HostRateLimiters; use crate::crawler::rate_limit::HostRateLimiters;
use crate::crawler::safety::{fetch_bytes_capped, looks_like_image, DownloadAllowlist}; use crate::crawler::session;
use crate::crawler::session::{self, ChapterProbe};
use crate::storage::Storage; use crate::storage::Storage;
/// Parse the chapter page DOM and return the page images in `pageN` /// Parse the chapter page DOM and return the page images in `pageN`
@@ -89,8 +88,6 @@ pub async fn sync_chapter_content(
manga_id: Uuid, manga_id: Uuid,
source_url: &str, source_url: &str,
force_refetch: bool, force_refetch: bool,
allowlist: &DownloadAllowlist,
max_image_bytes: usize,
) -> anyhow::Result<SyncOutcome> { ) -> anyhow::Result<SyncOutcome> {
// Skip if already fetched, unless caller explicitly forces. // Skip if already fetched, unless caller explicitly forces.
if !force_refetch { if !force_refetch {
@@ -113,28 +110,16 @@ pub async fn sync_chapter_content(
.with_context(|| format!("open chapter page {source_url}"))?; .with_context(|| format!("open chapter page {source_url}"))?;
page.wait_for_navigation().await.context("wait for chapter nav")?; page.wait_for_navigation().await.context("wait for chapter nav")?;
// Session probe: avatar present == still logged in. Missing means
// PHPSESSID expired; bail the entire crawler run.
if page.find_element("#avatar_menu").await.is_err() {
page.close().await.ok();
return Ok(SyncOutcome::SessionExpired);
}
let html = page.content().await.context("read chapter html")?; let html = page.content().await.context("read chapter html")?;
page.close().await.ok(); page.close().await.ok();
// Three-way session classification: distinguishes a transient
// hiccup (broken-page body or logged-in-but-no-reader) from a
// genuine PHPSESSID expiry (no reader and no avatar widget). The
// earlier binary `#avatar_menu` check conflated both and froze
// every worker on a layout shift.
match session::classify_chapter_probe(&html) {
ChapterProbe::Unauthenticated => return Ok(SyncOutcome::SessionExpired),
ChapterProbe::Transient => {
// Surface as a typed Err so the dispatcher path runs
// ack_failed with exponential backoff (rather than the
// session-expired sticky flag).
anyhow::bail!(
"chapter page at {source_url} returned a transient response \
(broken-page body or reader didn't render); will retry"
);
}
ChapterProbe::Ok => {}
}
let images = parse_chapter_pages(&html) let images = parse_chapter_pages(&html)
.with_context(|| format!("parse chapter pages at {source_url}"))?; .with_context(|| format!("parse chapter pages at {source_url}"))?;
if images.is_empty() { if images.is_empty() {
@@ -153,29 +138,18 @@ pub async fn sync_chapter_content(
format!("join image URL {} onto {source_url}", img.url) format!("join image URL {} onto {source_url}", img.url)
})?; })?;
rate.wait_for(url.as_str()).await?; rate.wait_for(url.as_str()).await?;
let bytes = fetch_bytes_capped( let resp = http
http, .get(url.clone())
url.as_str(), // Source CDNs commonly check Referer. Set it to the
Some(source_url), // chapter page — matches what the browser would send.
allowlist, .header(reqwest::header::REFERER, source_url)
max_image_bytes, .send()
) .await
.await? .with_context(|| format!("GET {url}"))?
.to_vec(); .error_for_status()
// Reject any non-image response: the only valid output of an .with_context(|| format!("non-2xx for {url}"))?;
// image URL is an image. `infer` returns None on truncated let bytes = resp.bytes().await.context("read image body")?.to_vec();
// bytes too, which also wants to be a failure not a silent let ext = infer::get(&bytes).map(|k| k.extension()).unwrap_or("bin");
// `.bin` extension.
if !looks_like_image(&bytes) {
anyhow::bail!(
"image URL {url} returned non-image bytes \
(first 16: {:?}); refusing to store as binary blob",
&bytes.get(..16.min(bytes.len()))
);
}
let ext = infer::get(&bytes)
.map(|k| k.extension())
.expect("looks_like_image asserted infer succeeded");
fetched.push((img.page_number, bytes, ext)); fetched.push((img.page_number, bytes, ext));
} }
@@ -220,9 +194,8 @@ pub async fn sync_chapter_content(
Ok(SyncOutcome::Fetched { pages: fetched.len() }) Ok(SyncOutcome::Fetched { pages: fetched.len() })
} }
// Suppress unused-import warning for `session::registrable_domain` // Suppress unused-import warning for `session` until the bin/crawler
// until the bin/crawler wiring lands in this branch and uses it // wiring lands in this branch and uses it through this module.
// through this module.
#[allow(dead_code)] #[allow(dead_code)]
fn _keep_session_in_scope() { fn _keep_session_in_scope() {
let _ = session::registrable_domain; let _ = session::registrable_domain;

View File

@@ -22,6 +22,5 @@ pub mod diff;
pub mod jobs; pub mod jobs;
pub mod pipeline; pub mod pipeline;
pub mod rate_limit; pub mod rate_limit;
pub mod safety;
pub mod session; pub mod session;
pub mod source; pub mod source;

View File

@@ -9,7 +9,6 @@ use uuid::Uuid;
use crate::crawler::browser_manager::BrowserManager; use crate::crawler::browser_manager::BrowserManager;
use crate::crawler::jobs::{self, EnqueueResult, JobPayload}; use crate::crawler::jobs::{self, EnqueueResult, JobPayload};
use crate::crawler::rate_limit::HostRateLimiters; use crate::crawler::rate_limit::HostRateLimiters;
use crate::crawler::safety::{fetch_bytes_capped, looks_like_image, DownloadAllowlist};
use crate::crawler::source::target::TargetSource; use crate::crawler::source::target::TargetSource;
use crate::crawler::source::{DiscoverMode, FetchContext, Source}; use crate::crawler::source::{DiscoverMode, FetchContext, Source};
use crate::repo; use crate::repo;
@@ -63,8 +62,6 @@ pub async fn run_metadata_pass(
limit: usize, limit: usize,
skip_chapters: bool, skip_chapters: bool,
mode: DiscoverMode, mode: DiscoverMode,
allowlist: &DownloadAllowlist,
max_image_bytes: usize,
) -> anyhow::Result<MetadataStats> { ) -> anyhow::Result<MetadataStats> {
let lease = browser_manager let lease = browser_manager
.acquire() .acquire()
@@ -184,8 +181,6 @@ pub async fn run_metadata_pass(
&r.url, &r.url,
upsert.manga_id, upsert.manga_id,
cover_url, cover_url,
allowlist,
max_image_bytes,
) )
.await .await
{ {
@@ -387,7 +382,6 @@ pub struct EnqueueSummary {
/// pipeline because the CLI still calls it from its inline chapter-content /// pipeline because the CLI still calls it from its inline chapter-content
/// loop; once the worker pool fully replaces that path we can fold this /// loop; once the worker pool fully replaces that path we can fold this
/// into `pipeline` proper. /// into `pipeline` proper.
#[allow(clippy::too_many_arguments)]
async fn download_and_store_cover( async fn download_and_store_cover(
db: &PgPool, db: &PgPool,
storage: &dyn Storage, storage: &dyn Storage,
@@ -396,8 +390,6 @@ async fn download_and_store_cover(
manga_url: &str, manga_url: &str,
manga_id: Uuid, manga_id: Uuid,
cover_url: &str, cover_url: &str,
allowlist: &DownloadAllowlist,
max_image_bytes: usize,
) -> anyhow::Result<()> { ) -> anyhow::Result<()> {
let absolute = reqwest::Url::parse(manga_url) let absolute = reqwest::Url::parse(manga_url)
.context("parse manga URL")? .context("parse manga URL")?
@@ -405,22 +397,17 @@ async fn download_and_store_cover(
.context("join cover URL onto manga URL")?; .context("join cover URL onto manga URL")?;
rate.wait_for(absolute.as_str()).await?; rate.wait_for(absolute.as_str()).await?;
let bytes = fetch_bytes_capped( let resp = http
http, .get(absolute.clone())
absolute.as_str(), .header(reqwest::header::REFERER, manga_url)
Some(manga_url), .send()
allowlist, .await
max_image_bytes, .with_context(|| format!("GET {absolute}"))?
) .error_for_status()
.await?; .with_context(|| format!("non-2xx for {absolute}"))?;
if !looks_like_image(&bytes) { let bytes = resp.bytes().await.context("read cover body")?;
anyhow::bail!( let kind = infer::get(&bytes);
"cover URL {absolute} returned non-image bytes; refusing to store as binary blob" let ext = kind.map(|k| k.extension()).unwrap_or("bin");
);
}
let ext = infer::get(&bytes)
.map(|k| k.extension())
.expect("looks_like_image asserted infer succeeded");
let key = format!("mangas/{manga_id}/cover.{ext}"); let key = format!("mangas/{manga_id}/cover.{ext}");
storage storage

View File

@@ -1,486 +0,0 @@
//! Defensive helpers for the image-download paths.
//!
//! Two threats this module addresses:
//!
//! - **SSRF**: a scraped chapter or manga page can embed an absolute
//! `<img src="http://10.0.0.1/...">`. The crawler runs inside the
//! backend container with intra-compose access to `postgres:5432`
//! and possibly other internal services; without a host check the
//! crawler would happily probe them. [`is_safe_url`] rejects
//! anything whose host isn't on the operator-configured allowlist,
//! plus any IP literal in RFC1918 / loopback / link-local / unique-
//! local space (including IPv4-mapped IPv6 like `::ffff:127.0.0.1`)
//! as a second defence for the case where an allowlisted hostname's
//! DNS happens to resolve to a literal private address.
//!
//! **DNS rebinding is not covered.** A hostname like `cdn.allowed.com`
//! that *resolves* to `127.0.0.1` via hostile DNS bypasses the IP
//! check entirely — `is_safe_url` only inspects URL strings, not
//! resolved IPs. Mitigating that requires a custom reqwest resolver
//! that filters IPs after DNS, which would mean rebuilding reqwest's
//! connector. The allowlist + good operator DNS hygiene is the
//! realistic mitigation today.
//!
//! - **Unbounded download**: `Response::bytes().await` reads the full
//! body before returning. A malicious source serving a 10 GiB image
//! would fill memory and then disk. [`accumulate_capped`] streams
//! the body chunk-by-chunk into a [`bytes::BytesMut`] and bails as
//! soon as the running total exceeds the cap.
//!
//! Both helpers are pure-data: the SSRF check is keyed off a parsed
//! URL string, and the byte accumulator is keyed off a generic stream.
//! Easy to unit-test without a live network or browser.
use std::net::IpAddr;
use anyhow::{bail, Context};
use bytes::BytesMut;
use futures_util::StreamExt;
use reqwest::Url;
/// Default per-image download cap. A page image is generally <2 MiB;
/// 32 MiB leaves headroom for high-resolution covers while still
/// stopping a misbehaving CDN dead. Override via `CRAWLER_MAX_IMAGE_BYTES`.
pub const DEFAULT_MAX_IMAGE_BYTES: usize = 32 * 1024 * 1024;
/// Hosts that are always allowed in addition to the operator's
/// configured allowlist. None by default — keeping the surface area
/// minimal so the only way a URL gets through is if it matches an
/// explicit catalog/CDN entry.
#[derive(Clone, Debug, Default)]
pub struct DownloadAllowlist {
hosts: Vec<String>,
}
impl DownloadAllowlist {
pub fn new() -> Self {
Self { hosts: Vec::new() }
}
/// Add a host (case-insensitive match). Sub-domains are *not*
/// implied: pass `cdn.example.com` and `example.com` separately
/// if both should be reachable.
pub fn allow(mut self, host: impl Into<String>) -> Self {
let h = host.into().to_ascii_lowercase();
if !h.is_empty() && !self.hosts.iter().any(|existing| existing == &h) {
self.hosts.push(h);
}
self
}
pub fn is_empty(&self) -> bool {
self.hosts.is_empty()
}
pub fn contains(&self, host: &str) -> bool {
let lower = host.to_ascii_lowercase();
self.hosts.iter().any(|h| h == &lower)
}
}
/// Verify a URL is safe for the crawler to fetch.
///
/// Rejects:
/// - non-http(s) schemes (file://, gopher://, …),
/// - any IP literal in private / loopback / link-local / unique-local
/// space (defense in depth — a DNS allowlist alone wouldn't cover an
/// attacker that places an entry like `cdn.evil` pointing at
/// `192.168.1.1`),
/// - the literal hostname `localhost`,
/// - hosts that aren't on the supplied allowlist.
///
/// An empty allowlist rejects everything (the conservative default —
/// callers must explicitly allow the catalog and CDN hosts).
pub fn is_safe_url(raw_url: &str, allow: &DownloadAllowlist) -> Result<(), UrlSafetyError> {
let url = Url::parse(raw_url).map_err(|_| UrlSafetyError::Unparseable)?;
let scheme = url.scheme();
if scheme != "http" && scheme != "https" {
return Err(UrlSafetyError::BadScheme(scheme.to_string()));
}
let host = url.host_str().ok_or(UrlSafetyError::NoHost)?;
let lower_host = host.to_ascii_lowercase();
if lower_host == "localhost" {
return Err(UrlSafetyError::Loopback);
}
// Reject IP literals in private/loopback ranges regardless of the
// allowlist — if someone puts an IP literal on the allowlist they
// almost certainly didn't mean a private range.
// reqwest::Url normalises IPv6 literals as `[::1]` (brackets
// included) in `host_str()`. Strip the brackets before parsing.
let ip_candidate = lower_host
.strip_prefix('[')
.and_then(|s| s.strip_suffix(']'))
.unwrap_or(&lower_host);
if let Ok(ip) = ip_candidate.parse::<IpAddr>() {
if is_private_ip(&ip) {
return Err(UrlSafetyError::PrivateIp(ip));
}
}
if !allow.contains(&lower_host) {
return Err(UrlSafetyError::HostNotAllowed(lower_host));
}
Ok(())
}
fn is_private_ip(ip: &IpAddr) -> bool {
match ip {
IpAddr::V4(v4) => {
v4.is_loopback()
|| v4.is_private()
|| v4.is_link_local()
|| v4.is_unspecified()
|| v4.is_broadcast()
// CGNAT 100.64.0.0/10
|| (v4.octets()[0] == 100 && (v4.octets()[1] & 0xC0) == 64)
// 169.254/16 link-local already covered, but 0.0.0.0/8 is special-use
|| v4.octets()[0] == 0
}
IpAddr::V6(v6) => {
// IPv4-mapped IPv6 (::ffff:0:0/96): unwrap to the embedded
// IPv4 and recurse so `::ffff:127.0.0.1` is caught by the
// IPv4 loopback check rather than passing through.
// `Ipv6Addr::is_loopback()` only matches `::1` exactly.
if let Some(v4) = v6.to_ipv4_mapped() {
return is_private_ip(&IpAddr::V4(v4));
}
v6.is_loopback()
|| v6.is_unspecified()
// fc00::/7 unique-local
|| (v6.segments()[0] & 0xfe00) == 0xfc00
// fe80::/10 link-local
|| (v6.segments()[0] & 0xffc0) == 0xfe80
}
}
}
#[derive(Debug, thiserror::Error, PartialEq, Eq)]
pub enum UrlSafetyError {
#[error("URL is not parseable")]
Unparseable,
#[error("scheme {0:?} is not http or https")]
BadScheme(String),
#[error("URL is missing a host")]
NoHost,
#[error("host points at the loopback interface")]
Loopback,
#[error("host is a private/internal IP: {0}")]
PrivateIp(IpAddr),
#[error("host {0:?} is not on the crawler download allowlist")]
HostNotAllowed(String),
}
/// Drain a byte stream into a single buffer, bailing out as soon as
/// the running total exceeds `max_bytes`. Generic over the stream so
/// it's testable without a live HTTP response.
pub async fn accumulate_capped<S, E>(stream: S, max_bytes: usize) -> anyhow::Result<bytes::Bytes>
where
S: futures_core::Stream<Item = Result<bytes::Bytes, E>>,
E: std::error::Error + Send + Sync + 'static,
{
let mut buf = BytesMut::new();
let mut stream = std::pin::pin!(stream);
while let Some(chunk) = stream.next().await {
let chunk = chunk.map_err(|e| anyhow::anyhow!("stream chunk: {e}"))?;
if buf.len().saturating_add(chunk.len()) > max_bytes {
bail!(
"response exceeds {max_bytes}-byte cap (received >{}+{})",
buf.len(),
chunk.len()
);
}
buf.extend_from_slice(&chunk);
}
Ok(buf.freeze())
}
/// Send `req` and stream the response into a length-limited buffer.
/// Combines [`is_safe_url`] check + [`accumulate_capped`] so each
/// call-site is one line.
pub async fn fetch_bytes_capped(
http: &reqwest::Client,
url: &str,
referer: Option<&str>,
allow: &DownloadAllowlist,
max_bytes: usize,
) -> anyhow::Result<bytes::Bytes> {
is_safe_url(url, allow).with_context(|| format!("reject unsafe URL {url}"))?;
let mut req = http.get(url);
if let Some(r) = referer {
req = req.header(reqwest::header::REFERER, r);
}
let resp = req
.send()
.await
.with_context(|| format!("GET {url}"))?
.error_for_status()
.with_context(|| format!("non-2xx for {url}"))?;
accumulate_capped(resp.bytes_stream(), max_bytes)
.await
.with_context(|| format!("download body for {url}"))
}
/// True when `bytes` sniffs as one of the *renderable* image formats
/// the `/files/*key` endpoint can serve with a correct Content-Type:
/// JPEG, PNG, WebP, GIF, AVIF. Matches the upload pipeline's
/// whitelist in `upload::parse_image`.
///
/// `infer::MatcherType::Image` is intentionally NOT used — it also
/// matches BMP, TIFF, HEIF, ICO, PSD, and JP2. Those would sniff as
/// "image" here but [`api::files::content_type_for`] would fall back
/// to `application/octet-stream`, prompting browsers to download
/// instead of render. Keep the two layers aligned.
pub fn looks_like_image(bytes: &[u8]) -> bool {
matches!(
infer::get(bytes).map(|k| k.mime_type()),
Some("image/jpeg" | "image/png" | "image/webp" | "image/gif" | "image/avif")
)
}
#[cfg(test)]
mod tests {
use super::*;
use futures_util::stream;
fn allow_just(host: &str) -> DownloadAllowlist {
DownloadAllowlist::new().allow(host)
}
#[test]
fn safe_url_allows_listed_host() {
let allow = allow_just("cdn.example.com");
assert!(is_safe_url("https://cdn.example.com/img.jpg", &allow).is_ok());
}
#[test]
fn safe_url_blocks_unlisted_host() {
let allow = allow_just("cdn.example.com");
let err = is_safe_url("https://evil.example.org/img.jpg", &allow).unwrap_err();
assert!(matches!(err, UrlSafetyError::HostNotAllowed(h) if h == "evil.example.org"));
}
#[test]
fn safe_url_blocks_localhost_even_if_allowlisted() {
let allow = allow_just("localhost");
assert!(matches!(
is_safe_url("http://localhost:8080/", &allow).unwrap_err(),
UrlSafetyError::Loopback
));
}
#[test]
fn safe_url_blocks_loopback_ipv4() {
let allow = allow_just("127.0.0.1");
assert!(matches!(
is_safe_url("http://127.0.0.1/", &allow).unwrap_err(),
UrlSafetyError::PrivateIp(_)
));
}
#[test]
fn safe_url_blocks_rfc1918() {
let allow = allow_just("10.0.0.1");
for url in [
"http://10.0.0.1/",
"http://192.168.1.1/",
"http://172.16.0.5/",
"http://172.31.255.255/",
] {
assert!(
matches!(
is_safe_url(url, &allow).unwrap_err(),
UrlSafetyError::PrivateIp(_)
),
"should reject {url}"
);
}
}
#[test]
fn safe_url_blocks_link_local() {
let allow = allow_just("169.254.169.254");
// 169.254.169.254 is the AWS/GCP metadata service — the most
// dangerous SSRF target on a default cloud VM.
assert!(matches!(
is_safe_url("http://169.254.169.254/", &allow).unwrap_err(),
UrlSafetyError::PrivateIp(_)
));
}
#[test]
fn safe_url_blocks_ipv6_loopback_and_ula() {
// Debug what host_str returns first — reqwest::Url normalises
// IPv6 literals as `[::1]` with brackets, which doesn't parse
// as `IpAddr` directly. The implementation strips them.
let allow = allow_just("[::1]");
let err = is_safe_url("http://[::1]/", &allow).unwrap_err();
assert!(
matches!(err, UrlSafetyError::PrivateIp(_)),
"expected PrivateIp, got {err:?}"
);
let allow = allow_just("[fd00::1]");
let err = is_safe_url("http://[fd00::1]/", &allow).unwrap_err();
assert!(
matches!(err, UrlSafetyError::PrivateIp(_)),
"expected PrivateIp, got {err:?}"
);
}
#[test]
fn safe_url_blocks_ipv4_mapped_ipv6_loopback() {
// `Ipv6Addr::is_loopback()` only matches `::1` exactly, so
// `::ffff:127.0.0.1` would slip through without the
// to_ipv4_mapped() unwrap in is_private_ip.
let allow = allow_just("[::ffff:127.0.0.1]");
let err = is_safe_url("http://[::ffff:127.0.0.1]/", &allow).unwrap_err();
assert!(
matches!(err, UrlSafetyError::PrivateIp(_)),
"expected PrivateIp, got {err:?}"
);
}
#[test]
fn safe_url_blocks_ipv4_mapped_ipv6_rfc1918() {
let allow = allow_just("[::ffff:10.0.0.1]");
let err = is_safe_url("http://[::ffff:10.0.0.1]/", &allow).unwrap_err();
assert!(matches!(err, UrlSafetyError::PrivateIp(_)));
}
#[test]
fn safe_url_blocks_non_http_schemes() {
let allow = allow_just("anywhere");
assert!(matches!(
is_safe_url("file:///etc/passwd", &allow).unwrap_err(),
UrlSafetyError::BadScheme(_)
));
assert!(matches!(
is_safe_url("gopher://anywhere:70/", &allow).unwrap_err(),
UrlSafetyError::BadScheme(_)
));
}
#[test]
fn safe_url_rejects_unparseable() {
let allow = allow_just("anywhere");
assert!(matches!(
is_safe_url("not a url", &allow).unwrap_err(),
UrlSafetyError::Unparseable
));
}
#[test]
fn safe_url_empty_allowlist_rejects_everything() {
let allow = DownloadAllowlist::new();
let err = is_safe_url("https://cdn.example.com/img.jpg", &allow).unwrap_err();
assert!(matches!(err, UrlSafetyError::HostNotAllowed(_)));
}
#[test]
fn allowlist_matches_case_insensitively() {
let allow = DownloadAllowlist::new().allow("CDN.Example.COM");
assert!(is_safe_url("https://cdn.example.com/x.jpg", &allow).is_ok());
assert!(is_safe_url("https://CDN.EXAMPLE.com/x.jpg", &allow).is_ok());
}
#[tokio::test]
async fn accumulate_capped_returns_full_body_under_cap() {
let chunks: Vec<Result<bytes::Bytes, std::io::Error>> = vec![
Ok(bytes::Bytes::from_static(b"hello ")),
Ok(bytes::Bytes::from_static(b"world")),
];
let s = stream::iter(chunks);
let out = accumulate_capped(s, 100).await.unwrap();
assert_eq!(out.as_ref(), b"hello world");
}
#[tokio::test]
async fn accumulate_capped_bails_past_cap() {
let chunks: Vec<Result<bytes::Bytes, std::io::Error>> = vec![
Ok(bytes::Bytes::from(vec![0u8; 50])),
Ok(bytes::Bytes::from(vec![0u8; 60])),
];
let s = stream::iter(chunks);
let err = accumulate_capped(s, 100).await.unwrap_err();
assert!(err.to_string().contains("100-byte cap"));
}
#[tokio::test]
async fn accumulate_capped_surfaces_stream_errors() {
let chunks: Vec<Result<bytes::Bytes, std::io::Error>> = vec![
Ok(bytes::Bytes::from_static(b"ok")),
Err(std::io::Error::other("network blip")),
];
let s = stream::iter(chunks);
let err = accumulate_capped(s, 100).await.unwrap_err();
assert!(err.to_string().contains("network blip"));
}
#[test]
fn looks_like_image_accepts_jpeg() {
// JPEG SOI + APP0 segment.
let jpeg = [0xff, 0xd8, 0xff, 0xe0, 0, 0x10, b'J', b'F', b'I', b'F'];
assert!(looks_like_image(&jpeg));
}
#[test]
fn looks_like_image_accepts_png() {
let png = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0, 0, 0, 0];
assert!(looks_like_image(&png));
}
#[test]
fn looks_like_image_rejects_html_disguised_as_image() {
let html = b"<html><body>not an image</body></html>";
assert!(!looks_like_image(html));
}
#[test]
fn looks_like_image_rejects_empty() {
assert!(!looks_like_image(&[]));
}
#[test]
fn looks_like_image_rejects_renderable_but_unsupported_formats() {
// BMP, TIFF, ICO, PSD are `infer::MatcherType::Image` but the
// /files/*key handler doesn't have Content-Type mappings for
// them, so they'd be served as application/octet-stream and
// download instead of render. Reject at the crawler so we
// never land them in storage.
// BMP magic: "BM" + 4-byte size.
let bmp = [b'B', b'M', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0];
assert!(!looks_like_image(&bmp), "BMP must be rejected (not renderable by /files)");
// TIFF little-endian magic: "II" + 42.
let tiff = [0x49, 0x49, 0x2a, 0x00, 0, 0, 0, 0];
assert!(!looks_like_image(&tiff), "TIFF must be rejected");
// ICO magic: 0x00,0x00,0x01,0x00.
let ico = [0x00, 0x00, 0x01, 0x00, 1, 0, 16, 16, 0, 0, 1, 0, 0x18, 0, 0x40, 0, 0, 0, 0x16, 0, 0, 0];
assert!(!looks_like_image(&ico), "ICO must be rejected");
}
#[test]
fn looks_like_image_accepts_webp_gif_avif() {
// Cover the three remaining whitelisted formats so a future
// tightening that drops one would fail noisily.
let webp = [
b'R', b'I', b'F', b'F',
0, 0, 0, 0,
b'W', b'E', b'B', b'P',
b'V', b'P', b'8', b' ',
];
assert!(looks_like_image(&webp));
let gif = [b'G', b'I', b'F', b'8', b'7', b'a', 0, 0, 0, 0];
assert!(looks_like_image(&gif));
let avif = [
0x00, 0x00, 0x00, 0x18,
b'f', b't', b'y', b'p',
b'a', b'v', b'i', b'f',
0x00, 0x00, 0x00, 0x00,
b'm', b'i', b'f', b'1',
b'a', b'v', b'i', b'f',
];
assert!(looks_like_image(&avif));
}
}

View File

@@ -127,54 +127,6 @@ pub fn classify_probe(html: &str) -> SessionProbe {
} }
} }
/// Three-way classification of a chapter page response.
///
/// Reader pages don't render `#logo`, so [`classify_probe`] can't be
/// reused as-is. The chapter-specific marker is `a#pic_container`
/// (asserted by the reader-page parser at `parse_chapter_pages`).
///
/// Order matters: broken-page body wins over selector matches, so a
/// transient site-wide 5xx that happens to render the avatar widget
/// elsewhere doesn't falsely reach `Ok`.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ChapterProbe {
/// `a#pic_container` present — reader rendered. Whether
/// `#avatar_menu` is also there is informational; if the reader
/// loaded the session is by definition still good.
Ok,
/// Site rendered a "logged out" or "please log in" page (no
/// reader, no broken-page body, and no avatar widget either).
/// Distinguishes the genuine expired-session case from a
/// transient site hiccup.
Unauthenticated,
/// Broken-page body, or reader didn't render but the user is
/// still logged in (avatar widget present). Caller should retry
/// rather than blame the session.
Transient,
}
pub fn classify_chapter_probe(html: &str) -> ChapterProbe {
if is_broken_page_body(html) {
return ChapterProbe::Transient;
}
let doc = scraper::Html::parse_document(html);
let container = scraper::Selector::parse("a#pic_container").unwrap();
if doc.select(&container).next().is_some() {
return ChapterProbe::Ok;
}
let avatar = scraper::Selector::parse("#avatar_menu").unwrap();
if doc.select(&avatar).next().is_some() {
// Logged-in user, but the reader didn't render — most likely
// the layout shifted or the site is serving an interstitial.
ChapterProbe::Transient
} else {
// No reader, no avatar, no broken-body marker — site rendered
// the "please log in" page, which is the genuine session-
// expired signal on this route.
ChapterProbe::Unauthenticated
}
}
/// In-startup retry budget for the session probe. Small but non-zero — /// In-startup retry budget for the session probe. Small but non-zero —
/// startup hitting a 5-second site hiccup shouldn't fail the operator /// startup hitting a 5-second site hiccup shouldn't fail the operator
/// with "PHPSESSID expired" when the session is actually fine. /// with "PHPSESSID expired" when the session is actually fine.
@@ -321,73 +273,6 @@ mod tests {
assert_eq!(classify_probe(""), SessionProbe::Transient); assert_eq!(classify_probe(""), SessionProbe::Transient);
} }
#[test]
fn classify_chapter_probe_ok_when_reader_rendered() {
let html = r#"
<html><body>
<a id="pic_container">
<img id="page1" src="https://cdn/1.jpg">
</a>
</body></html>
"#;
assert_eq!(classify_chapter_probe(html), ChapterProbe::Ok);
}
#[test]
fn classify_chapter_probe_unauthenticated_when_no_reader_and_no_avatar() {
// What a logged-out hit on a chapter URL renders: a normal
// site layout (header etc.) with a "please log in" body, but
// no reader and no avatar widget.
let html = r#"
<html><body>
<header><div id="logo">Catalog</div></header>
<main>Please log in to read this chapter.</main>
</body></html>
"#;
assert_eq!(
classify_chapter_probe(html),
ChapterProbe::Unauthenticated
);
}
#[test]
fn classify_chapter_probe_transient_when_logged_in_but_reader_missing() {
// Avatar shows the session is still valid; reader didn't
// render — site is serving an interstitial or the layout
// momentarily shifted. Retry, don't blame the session.
let html = r#"
<html><body>
<header><div id="logo">Catalog</div><div id="avatar_menu"></div></header>
<main>Site maintenance — back in 5 minutes.</main>
</body></html>
"#;
assert_eq!(classify_chapter_probe(html), ChapterProbe::Transient);
}
#[test]
fn classify_chapter_probe_transient_on_broken_page_body() {
let html =
"<html><body><p>we're sorry, the request file are not found.</p></body></html>";
assert_eq!(classify_chapter_probe(html), ChapterProbe::Transient);
}
#[test]
fn classify_chapter_probe_does_not_misfire_on_avatar_alone_without_reader() {
// Regression for the original bug: the binary
// find_element("#avatar_menu") check treated "no avatar" as
// session-expired even when a transient hiccup was the real
// cause. classify_chapter_probe must NOT trip on that pattern
// when pic_container *is* present.
let html = r#"
<html><body>
<a id="pic_container">
<img id="page1" src="https://cdn/1.jpg">
</a>
</body></html>
"#;
assert_eq!(classify_chapter_probe(html), ChapterProbe::Ok);
}
#[test] #[test]
fn classify_probe_trusts_broken_body_over_stray_avatar_match() { fn classify_probe_trusts_broken_body_over_stray_avatar_match() {
// Defensive: if a broken-page body somehow contains an // Defensive: if a broken-page body somehow contains an

View File

@@ -1,7 +1,11 @@
FROM node:22-alpine AS builder FROM node:22-alpine AS builder
WORKDIR /app WORKDIR /app
COPY package.json package-lock.json* ./ COPY package.json package-lock.json* ./
RUN npm install # `npm ci` installs the locked versions exactly; `npm install` would
# silently rewrite package-lock.json mid-build. CI (.gitea/workflows)
# also uses `npm ci`, so this keeps the image build deterministic and
# matches what the test job validated.
RUN npm ci
COPY . . COPY . .
RUN npm run build RUN npm run build
@@ -10,8 +14,20 @@ WORKDIR /app
ENV NODE_ENV=production ENV NODE_ENV=production
ENV HOST=0.0.0.0 ENV HOST=0.0.0.0
ENV PORT=3000 ENV PORT=3000
COPY --from=builder /app/build ./build
COPY --from=builder /app/node_modules ./node_modules # node:22-alpine ships a `node` user (UID 1000); use it instead of
COPY --from=builder /app/package.json ./ # running the SvelteKit server as root.
COPY --from=builder --chown=node:node /app/build ./build
COPY --from=builder --chown=node:node /app/node_modules ./node_modules
COPY --from=builder --chown=node:node /app/package.json ./
USER node
EXPOSE 3000 EXPOSE 3000
# Alpine's busybox `wget` is the canonical lightweight HTTP probe.
# `--spider` doesn't follow redirects; `node build` serves a 200 on
# `/` for the homepage so this works without a dedicated /health.
HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
CMD wget -q --spider http://localhost:3000/ || exit 1
CMD ["node", "build"] CMD ["node", "build"]

View File

@@ -1,6 +1,6 @@
{ {
"name": "mangalord-frontend", "name": "mangalord-frontend",
"version": "0.34.1", "version": "0.34.0",
"private": true, "private": true,
"type": "module", "type": "module",
"scripts": { "scripts": {