bugfix: security & correctness bundle (0.34.1)
Five fixes bundled into one release: - preserve user-attached tags across crawler upserts (repo::crawler::sync_tags now scopes to added_by IS NULL; orphaned attachments from deleted users are reaped as crawler-owned) - gate manga PATCH and cover endpoints on uploaded_by (require_can_edit in api::mangas; non-NULL uploaded_by must match the caller) - equalise login response time across user-existence branches (run argon2 against a OnceLock-cached dummy hash on the no-user branch so timing doesn't leak username existence) - crawler download defences (SSRF allowlist of host literals including IPv4-mapped IPv6 ranges, 32 MiB streamed size cap, reject non-whitelisted image types, three-way chapter-probe classifier replaces the binary #avatar_menu check) - tighten validation and clean up dead unload path (attach_tag + create_token enforce 64-char caps; LocalStorage rejects NUL bytes explicitly; reader flushFinalProgress drops the always-405 sendBeacon path) Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -18,7 +18,8 @@ use uuid::Uuid;
|
||||
|
||||
use crate::crawler::detect::PageError;
|
||||
use crate::crawler::rate_limit::HostRateLimiters;
|
||||
use crate::crawler::session;
|
||||
use crate::crawler::safety::{fetch_bytes_capped, looks_like_image, DownloadAllowlist};
|
||||
use crate::crawler::session::{self, ChapterProbe};
|
||||
use crate::storage::Storage;
|
||||
|
||||
/// Parse the chapter page DOM and return the page images in `pageN`
|
||||
@@ -88,6 +89,8 @@ pub async fn sync_chapter_content(
|
||||
manga_id: Uuid,
|
||||
source_url: &str,
|
||||
force_refetch: bool,
|
||||
allowlist: &DownloadAllowlist,
|
||||
max_image_bytes: usize,
|
||||
) -> anyhow::Result<SyncOutcome> {
|
||||
// Skip if already fetched, unless caller explicitly forces.
|
||||
if !force_refetch {
|
||||
@@ -110,16 +113,28 @@ pub async fn sync_chapter_content(
|
||||
.with_context(|| format!("open chapter page {source_url}"))?;
|
||||
page.wait_for_navigation().await.context("wait for chapter nav")?;
|
||||
|
||||
// Session probe: avatar present == still logged in. Missing means
|
||||
// PHPSESSID expired; bail the entire crawler run.
|
||||
if page.find_element("#avatar_menu").await.is_err() {
|
||||
page.close().await.ok();
|
||||
return Ok(SyncOutcome::SessionExpired);
|
||||
}
|
||||
|
||||
let html = page.content().await.context("read chapter html")?;
|
||||
page.close().await.ok();
|
||||
|
||||
// Three-way session classification: distinguishes a transient
|
||||
// hiccup (broken-page body or logged-in-but-no-reader) from a
|
||||
// genuine PHPSESSID expiry (no reader and no avatar widget). The
|
||||
// earlier binary `#avatar_menu` check conflated both and froze
|
||||
// every worker on a layout shift.
|
||||
match session::classify_chapter_probe(&html) {
|
||||
ChapterProbe::Unauthenticated => return Ok(SyncOutcome::SessionExpired),
|
||||
ChapterProbe::Transient => {
|
||||
// Surface as a typed Err so the dispatcher path runs
|
||||
// ack_failed with exponential backoff (rather than the
|
||||
// session-expired sticky flag).
|
||||
anyhow::bail!(
|
||||
"chapter page at {source_url} returned a transient response \
|
||||
(broken-page body or reader didn't render); will retry"
|
||||
);
|
||||
}
|
||||
ChapterProbe::Ok => {}
|
||||
}
|
||||
|
||||
let images = parse_chapter_pages(&html)
|
||||
.with_context(|| format!("parse chapter pages at {source_url}"))?;
|
||||
if images.is_empty() {
|
||||
@@ -138,18 +153,29 @@ pub async fn sync_chapter_content(
|
||||
format!("join image URL {} onto {source_url}", img.url)
|
||||
})?;
|
||||
rate.wait_for(url.as_str()).await?;
|
||||
let resp = http
|
||||
.get(url.clone())
|
||||
// Source CDNs commonly check Referer. Set it to the
|
||||
// chapter page — matches what the browser would send.
|
||||
.header(reqwest::header::REFERER, source_url)
|
||||
.send()
|
||||
.await
|
||||
.with_context(|| format!("GET {url}"))?
|
||||
.error_for_status()
|
||||
.with_context(|| format!("non-2xx for {url}"))?;
|
||||
let bytes = resp.bytes().await.context("read image body")?.to_vec();
|
||||
let ext = infer::get(&bytes).map(|k| k.extension()).unwrap_or("bin");
|
||||
let bytes = fetch_bytes_capped(
|
||||
http,
|
||||
url.as_str(),
|
||||
Some(source_url),
|
||||
allowlist,
|
||||
max_image_bytes,
|
||||
)
|
||||
.await?
|
||||
.to_vec();
|
||||
// Reject any non-image response: the only valid output of an
|
||||
// image URL is an image. `infer` returns None on truncated
|
||||
// bytes too, which also wants to be a failure not a silent
|
||||
// `.bin` extension.
|
||||
if !looks_like_image(&bytes) {
|
||||
anyhow::bail!(
|
||||
"image URL {url} returned non-image bytes \
|
||||
(first 16: {:?}); refusing to store as binary blob",
|
||||
&bytes.get(..16.min(bytes.len()))
|
||||
);
|
||||
}
|
||||
let ext = infer::get(&bytes)
|
||||
.map(|k| k.extension())
|
||||
.expect("looks_like_image asserted infer succeeded");
|
||||
fetched.push((img.page_number, bytes, ext));
|
||||
}
|
||||
|
||||
@@ -194,8 +220,9 @@ pub async fn sync_chapter_content(
|
||||
Ok(SyncOutcome::Fetched { pages: fetched.len() })
|
||||
}
|
||||
|
||||
// Suppress unused-import warning for `session` until the bin/crawler
|
||||
// wiring lands in this branch and uses it through this module.
|
||||
// Suppress unused-import warning for `session::registrable_domain`
|
||||
// until the bin/crawler wiring lands in this branch and uses it
|
||||
// through this module.
|
||||
#[allow(dead_code)]
|
||||
fn _keep_session_in_scope() {
|
||||
let _ = session::registrable_domain;
|
||||
|
||||
Reference in New Issue
Block a user