bugfix: security & correctness bundle (0.34.1)
Five fixes bundled into one release: - preserve user-attached tags across crawler upserts (repo::crawler::sync_tags now scopes to added_by IS NULL; orphaned attachments from deleted users are reaped as crawler-owned) - gate manga PATCH and cover endpoints on uploaded_by (require_can_edit in api::mangas; non-NULL uploaded_by must match the caller) - equalise login response time across user-existence branches (run argon2 against a OnceLock-cached dummy hash on the no-user branch so timing doesn't leak username existence) - crawler download defences (SSRF allowlist of host literals including IPv4-mapped IPv6 ranges, 32 MiB streamed size cap, reject non-whitelisted image types, three-way chapter-probe classifier replaces the binary #avatar_menu check) - tighten validation and clean up dead unload path (attach_tag + create_token enforce 64-char caps; LocalStorage rejects NUL bytes explicitly; reader flushFinalProgress drops the always-405 sendBeacon path) Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -229,6 +229,33 @@ async fn run(
|
||||
}
|
||||
let rate = Arc::new(rate);
|
||||
|
||||
// SSRF defence: only download from the catalog host + CDN host
|
||||
// (plus optional CRAWLER_DOWNLOAD_ALLOWLIST extras), and cap
|
||||
// single-image downloads at CRAWLER_MAX_IMAGE_BYTES bytes.
|
||||
let mut allowlist =
|
||||
mangalord::crawler::safety::DownloadAllowlist::new();
|
||||
if let Ok(parsed) = reqwest::Url::parse(start_url) {
|
||||
if let Some(h) = parsed.host_str() {
|
||||
allowlist = allowlist.allow(h);
|
||||
}
|
||||
}
|
||||
if let Some(host) = cdn_host {
|
||||
allowlist = allowlist.allow(host);
|
||||
}
|
||||
if let Ok(extras) = std::env::var("CRAWLER_DOWNLOAD_ALLOWLIST") {
|
||||
for piece in extras.split(',') {
|
||||
let trimmed = piece.trim();
|
||||
if !trimmed.is_empty() {
|
||||
allowlist = allowlist.allow(trimmed);
|
||||
}
|
||||
}
|
||||
}
|
||||
let max_image_bytes: usize = std::env::var("CRAWLER_MAX_IMAGE_BYTES")
|
||||
.ok()
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(mangalord::crawler::safety::DEFAULT_MAX_IMAGE_BYTES);
|
||||
let allowlist = Arc::new(allowlist);
|
||||
|
||||
let stats = pipeline::run_metadata_pass(
|
||||
manager.as_ref(),
|
||||
db,
|
||||
@@ -239,6 +266,8 @@ async fn run(
|
||||
limit,
|
||||
skip_chapters,
|
||||
mode,
|
||||
allowlist.as_ref(),
|
||||
max_image_bytes,
|
||||
)
|
||||
.await?;
|
||||
tracing::info!(?stats, "metadata pass complete");
|
||||
@@ -253,6 +282,8 @@ async fn run(
|
||||
"target",
|
||||
chapter_workers,
|
||||
force_refetch_chapters,
|
||||
Arc::clone(&allowlist),
|
||||
max_image_bytes,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
@@ -276,6 +307,8 @@ async fn sync_bookmarked_chapter_content(
|
||||
source_id: &str,
|
||||
workers: usize,
|
||||
force_refetch: bool,
|
||||
allowlist: Arc<mangalord::crawler::safety::DownloadAllowlist>,
|
||||
max_image_bytes: usize,
|
||||
) -> anyhow::Result<()> {
|
||||
let pending: Vec<(Uuid, Uuid, String)> = sqlx::query_as(
|
||||
r#"
|
||||
@@ -312,6 +345,7 @@ async fn sync_bookmarked_chapter_content(
|
||||
let storage = Arc::clone(&storage);
|
||||
let rate = Arc::clone(&rate);
|
||||
let manager = Arc::clone(&manager);
|
||||
let allowlist = Arc::clone(&allowlist);
|
||||
let stats = &stats;
|
||||
async move {
|
||||
if session_expired.load(std::sync::atomic::Ordering::Relaxed) {
|
||||
@@ -336,6 +370,8 @@ async fn sync_bookmarked_chapter_content(
|
||||
manga_id,
|
||||
&source_url,
|
||||
force_refetch,
|
||||
allowlist.as_ref(),
|
||||
max_image_bytes,
|
||||
)
|
||||
.await;
|
||||
drop(lease);
|
||||
|
||||
Reference in New Issue
Block a user