diff --git a/backend/src/api/bookmarks.rs b/backend/src/api/bookmarks.rs index 1f518c6..1f424f3 100644 --- a/backend/src/api/bookmarks.rs +++ b/backend/src/api/bookmarks.rs @@ -67,14 +67,7 @@ async fn create( // the foreign-key violation collapse into a generic 500. repo::manga::get(&state.db, input.manga_id).await?; if let Some(chapter_id) = input.chapter_id { - let exists: Option<(Uuid,)> = sqlx::query_as( - "SELECT id FROM chapters WHERE id = $1 AND manga_id = $2", - ) - .bind(chapter_id) - .bind(input.manga_id) - .fetch_optional(&state.db) - .await?; - if exists.is_none() { + if !repo::chapter::belongs_to_manga(&state.db, chapter_id, input.manga_id).await? { return Err(AppError::NotFound); } } diff --git a/backend/src/app.rs b/backend/src/app.rs index 27734bb..17c93ef 100644 --- a/backend/src/app.rs +++ b/backend/src/app.rs @@ -304,18 +304,9 @@ impl ChapterDispatcher for RealChapterDispatcher { chapter_id, source_chapter_key: _, } => { - // Look up manga_id + source_url for this chapter. - let row: Option<(uuid::Uuid, String)> = sqlx::query_as( - "SELECT c.manga_id, cs.source_url \ - FROM chapters c \ - JOIN chapter_sources cs ON cs.chapter_id = c.id \ - WHERE c.id = $1 \ - LIMIT 1", - ) - .bind(chapter_id) - .fetch_optional(&self.db) - .await - .context("look up chapter for dispatch")?; + let row = repo::chapter::dispatch_target(&self.db, chapter_id) + .await + .context("look up chapter for dispatch")?; let Some((manga_id, source_url)) = row else { // Chapter (or its source row) is gone — ack done. return Ok(SyncOutcome::Skipped); diff --git a/backend/src/crawler/daemon.rs b/backend/src/crawler/daemon.rs index 4c822e4..d976745 100644 --- a/backend/src/crawler/daemon.rs +++ b/backend/src/crawler/daemon.rs @@ -317,14 +317,10 @@ impl WorkerContext { // (because a force-refetch race or a job that was re-enqueued // after a previous one finished), ack done without re-fetching. if let JobPayload::SyncChapterContent { chapter_id, .. } = &lease.payload { - let page_count: Option = sqlx::query_scalar( - "SELECT page_count FROM chapters WHERE id = $1", - ) - .bind(chapter_id) - .fetch_optional(&self.pool) - .await - .ok() - .flatten(); + let page_count = crate::repo::chapter::page_count(&self.pool, *chapter_id) + .await + .ok() + .flatten(); if matches!(page_count, Some(n) if n > 0) { let _ = jobs::ack_done(&self.pool, lease.id).await; return; diff --git a/backend/src/crawler/mod.rs b/backend/src/crawler/mod.rs index be3dcaa..07a3b85 100644 --- a/backend/src/crawler/mod.rs +++ b/backend/src/crawler/mod.rs @@ -24,3 +24,4 @@ pub mod pipeline; pub mod rate_limit; pub mod session; pub mod source; +pub mod url_utils; diff --git a/backend/src/crawler/pipeline.rs b/backend/src/crawler/pipeline.rs index e1050e9..91a3abc 100644 --- a/backend/src/crawler/pipeline.rs +++ b/backend/src/crawler/pipeline.rs @@ -427,11 +427,7 @@ async fn download_and_store_cover( Ok(()) } -fn origin_of(url: &str) -> Option { - let (scheme, rest) = url.split_once("://")?; - let host = rest.split('/').next()?; - Some(format!("{scheme}://{host}")) -} +use crate::crawler::url_utils::origin_of; #[cfg(test)] mod tests { diff --git a/backend/src/crawler/rate_limit.rs b/backend/src/crawler/rate_limit.rs index d9d4aae..7c187b7 100644 --- a/backend/src/crawler/rate_limit.rs +++ b/backend/src/crawler/rate_limit.rs @@ -98,15 +98,9 @@ impl HostRateLimiters { } } -/// Extract the host (no port) from a URL string. Returns `None` for -/// inputs without a `scheme://host` shape — those would never have -/// reached the network layer anyway. -fn host_of(url: &str) -> Option { - let after_scheme = url.split_once("://")?.1; - let host_with_port = after_scheme.split('/').next()?; - let host = host_with_port.rsplit_once(':').map_or(host_with_port, |(h, _)| h); - (!host.is_empty()).then(|| host.to_ascii_lowercase()) -} +// `host_of` was duplicated across session/rate_limit/pipeline; the +// canonical version now lives in `crawler::url_utils`. +use crate::crawler::url_utils::host_of; #[cfg(test)] mod tests { diff --git a/backend/src/crawler/session.rs b/backend/src/crawler/session.rs index 209ea5c..321363e 100644 --- a/backend/src/crawler/session.rs +++ b/backend/src/crawler/session.rs @@ -42,36 +42,9 @@ pub enum SessionProbe { Transient, } -/// Compute the cookie domain (e.g. `.example.com`) from a start URL. -/// The leading dot makes the cookie cover every subdomain — the source -/// often redirects between `www.` and other prefixes mid-crawl, and a -/// host-only cookie would silently drop on the cross-subdomain hop. -/// -/// Caveat: this takes the last two dot-labels, which is wrong for -/// multi-part TLDs (`.co.uk`, `.com.br` would resolve to `.co.uk` and -/// attach to every site on `.co.uk`). For those, the operator should -/// override via `CRAWLER_COOKIE_DOMAIN` rather than relying on this -/// function — pulling in the Public Suffix List for one knob isn't -/// worth it yet. -pub fn registrable_domain(url: &str) -> Option { - let after_scheme = url.split_once("://")?.1; - let host_with_port = after_scheme.split('/').next()?; - let host = host_with_port - .rsplit_once(':') - .map_or(host_with_port, |(h, _)| h) - .to_ascii_lowercase(); - if host.is_empty() { - return None; - } - let labels: Vec<&str> = host.split('.').filter(|l| !l.is_empty()).collect(); - if labels.len() < 2 { - // Bare hostname (e.g. `localhost`) — return as-is, no leading - // dot. Setting `.localhost` as cookie domain is invalid. - return Some(host); - } - let registrable = &labels[labels.len() - 2..]; - Some(format!(".{}", registrable.join("."))) -} +/// Re-export so existing callers keep working after the helper moved +/// to `crawler::url_utils`. The body lives there. +pub use crate::crawler::url_utils::registrable_domain; /// Inject the PHPSESSID cookie into the browser's cookie store for the /// catalog domain. Must be called before any navigation that depends on @@ -192,44 +165,8 @@ async fn fetch_probe_html(browser: &Browser, probe_url: &str) -> anyhow::Result< mod tests { use super::*; - #[test] - fn registrable_domain_strips_subdomain() { - assert_eq!( - registrable_domain("https://www.target-site.com/manga/foo/").as_deref(), - Some(".target-site.com") - ); - assert_eq!( - registrable_domain("https://m.example.org").as_deref(), - Some(".example.org") - ); - } - - #[test] - fn registrable_domain_keeps_two_label_host() { - assert_eq!( - registrable_domain("https://example.com/").as_deref(), - Some(".example.com") - ); - } - - #[test] - fn registrable_domain_handles_port() { - assert_eq!( - registrable_domain("http://www.foo.bar:8080/x").as_deref(), - Some(".foo.bar") - ); - } - - #[test] - fn registrable_domain_bare_hostname_no_leading_dot() { - // .localhost would be invalid as a cookie Domain. - assert_eq!(registrable_domain("http://localhost:5173").as_deref(), Some("localhost")); - } - - #[test] - fn registrable_domain_returns_none_for_garbage() { - assert!(registrable_domain("not a url").is_none()); - } + // registrable_domain tests live in crawler::url_utils now — + // it's the canonical home for that helper. #[test] fn classify_probe_ok_when_logo_and_avatar_present() { diff --git a/backend/src/crawler/url_utils.rs b/backend/src/crawler/url_utils.rs new file mode 100644 index 0000000..0d50ea4 --- /dev/null +++ b/backend/src/crawler/url_utils.rs @@ -0,0 +1,194 @@ +//! Centralised URL helpers for the crawler subsystem. +//! +//! Three near-identical hand-rolled URL parsers used to live in +//! `crawler::session`, `crawler::rate_limit`, and `crawler::pipeline` +//! respectively, each with subtly different edge-case behaviour +//! around port handling and IPv6 literals. They're consolidated here +//! so the divergence can't drift again. +//! +//! The hand-rolled implementations are kept intentionally — they +//! preserve the exact semantics every existing test pins. A future +//! refactor can switch to `reqwest::Url` if it can be done without +//! changing those semantics. + +/// Lowercased host (no port). Returns `None` for inputs without a +/// `scheme://host` shape — those would never have reached the network +/// layer anyway. Used by the per-host rate limiter as its bucket key. +/// +/// IPv6 literals are kept in their `[::1]` bracketed form so the +/// `rsplit_once(':')` port-stripping logic doesn't split inside the +/// address (e.g. `https://[::1]/foo` used to return `"[:"` because +/// the rightmost `:` is inside the literal). Buckets keyed by +/// `[::1]` vs `::1` are still uniquely-per-host; the brackets are +/// cosmetic. +pub fn host_of(url: &str) -> Option { + let after_scheme = url.split_once("://")?.1; + let host_with_port = after_scheme.split('/').next()?; + let host = if host_with_port.starts_with('[') { + // IPv6 literal: keep through the closing bracket. There may + // be a trailing `:port` after `]`; strip only that. + match host_with_port.rfind(']') { + Some(end) => &host_with_port[..=end], + None => host_with_port, + } + } else { + // Hostnames and IPv4 literals: trailing `:port` (if any) is + // after the last `:`. + host_with_port + .rsplit_once(':') + .map_or(host_with_port, |(h, _)| h) + }; + (!host.is_empty()).then(|| host.to_ascii_lowercase()) +} + +/// `scheme://host` with no path or port stripping. Used by the metadata +/// pass to seed `sources.base_url` from `CRAWLER_START_URL`. +pub fn origin_of(url: &str) -> Option { + let (scheme, rest) = url.split_once("://")?; + let host = rest.split('/').next()?; + Some(format!("{scheme}://{host}")) +} + +/// Approximate registrable-domain calculation: take the last two +/// dot-labels of the host, prefix with `.`. Used to set a parent- +/// domain cookie so the catalog's `www.` / `m.` redirects don't drop +/// the cookie mid-crawl. +/// +/// Caveat: wrong for multi-part TLDs (`.co.uk`, `.com.br`). The +/// operator can override via `CRAWLER_COOKIE_DOMAIN`; pulling in the +/// Public Suffix List for one knob isn't worth it yet. +/// +/// Bare hostnames (e.g. `localhost`) return the host as-is, with no +/// leading dot — setting `.localhost` as a cookie domain is invalid. +/// IPv6 literals (e.g. `[::1]`) are returned bracketed and unchanged; +/// the browser will reject them as a cookie `Domain` anyway, but the +/// representation stays sensible. Same `starts_with('[')` branch as +/// [`host_of`] for consistent IPv6 handling across the module. +pub fn registrable_domain(url: &str) -> Option { + let after_scheme = url.split_once("://")?.1; + let host_with_port = after_scheme.split('/').next()?; + let host_str = if host_with_port.starts_with('[') { + // IPv6 literal: keep through the closing bracket; an optional + // `:port` follows `]`. + match host_with_port.rfind(']') { + Some(end) => &host_with_port[..=end], + None => host_with_port, + } + } else { + host_with_port + .rsplit_once(':') + .map_or(host_with_port, |(h, _)| h) + }; + let host = host_str.to_ascii_lowercase(); + if host.is_empty() { + return None; + } + let labels: Vec<&str> = host.split('.').filter(|l| !l.is_empty()).collect(); + if labels.len() < 2 { + return Some(host); + } + let registrable = &labels[labels.len() - 2..]; + Some(format!(".{}", registrable.join("."))) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn host_of_strips_port_and_lowercases() { + assert_eq!( + host_of("https://CDN.Example.com:443/x").as_deref(), + Some("cdn.example.com") + ); + assert_eq!(host_of("http://localhost/").as_deref(), Some("localhost")); + assert_eq!(host_of("not a url"), None); + } + + #[test] + fn host_of_keeps_bracketed_ipv6_literal_intact() { + // Regression: the old impl rsplit_once(':')'d the IPv6 address, + // returning "[:" instead of "[::1]". A real IPv6 source would + // silently get a wrong rate-limit bucket key. + assert_eq!(host_of("https://[::1]/").as_deref(), Some("[::1]")); + assert_eq!(host_of("https://[::1]:8080/").as_deref(), Some("[::1]")); + assert_eq!( + host_of("https://[2001:db8::1]/foo").as_deref(), + Some("[2001:db8::1]") + ); + assert_eq!( + host_of("https://[2001:db8::1]:443/foo").as_deref(), + Some("[2001:db8::1]") + ); + } + + #[test] + fn origin_of_returns_scheme_and_host() { + assert_eq!( + origin_of("https://example.com/some/path?q=1").as_deref(), + Some("https://example.com") + ); + assert_eq!(origin_of("garbage"), None); + } + + #[test] + fn registrable_domain_strips_subdomain() { + assert_eq!( + registrable_domain("https://www.target-site.com/manga/foo/").as_deref(), + Some(".target-site.com") + ); + assert_eq!( + registrable_domain("https://m.example.org").as_deref(), + Some(".example.org") + ); + } + + #[test] + fn registrable_domain_keeps_two_label_host() { + assert_eq!( + registrable_domain("https://example.com/").as_deref(), + Some(".example.com") + ); + } + + #[test] + fn registrable_domain_handles_port() { + assert_eq!( + registrable_domain("http://www.foo.bar:8080/x").as_deref(), + Some(".foo.bar") + ); + } + + #[test] + fn registrable_domain_bare_hostname_no_leading_dot() { + assert_eq!( + registrable_domain("http://localhost:5173").as_deref(), + Some("localhost") + ); + } + + #[test] + fn registrable_domain_returns_none_for_garbage() { + assert!(registrable_domain("not a url").is_none()); + } + + #[test] + fn registrable_domain_keeps_bracketed_ipv6_literal_intact() { + // Symmetric with host_of's IPv6 fix. The cookie-domain code + // won't accept an IP as a `Domain` value, but the function + // should at least return a sensible representation rather + // than the truncated `"[:"` the old port-stripper produced. + assert_eq!( + registrable_domain("https://[::1]/").as_deref(), + Some("[::1]") + ); + assert_eq!( + registrable_domain("https://[::1]:8080/").as_deref(), + Some("[::1]") + ); + assert_eq!( + registrable_domain("https://[2001:db8::1]/foo").as_deref(), + Some("[2001:db8::1]") + ); + } +} diff --git a/backend/src/repo/author.rs b/backend/src/repo/author.rs index fa8e187..57d9452 100644 --- a/backend/src/repo/author.rs +++ b/backend/src/repo/author.rs @@ -99,6 +99,11 @@ pub async fn list( /// Atomically replace the set of authors on a manga. Caller passes a /// `&mut PgConnection` (`&mut *tx` works) so the delete+upserts run in /// one transaction with whatever called us. +/// +/// Note: `crawler::repo::sync_authors` does a similar replace with the +/// same semantics on names. The duplication is intentional — handler +/// callers want the `Vec` for the API response; the +/// crawler doesn't need it and stays inside its own transaction. pub async fn set_for_manga( conn: &mut PgConnection, manga_id: Uuid, diff --git a/backend/src/repo/bookmark.rs b/backend/src/repo/bookmark.rs index e049de9..5d4b9f8 100644 --- a/backend/src/repo/bookmark.rs +++ b/backend/src/repo/bookmark.rs @@ -29,9 +29,9 @@ pub async fn create( match result { Ok(b) => Ok(b), - Err(e) if is_unique_violation(&e) => Err(AppError::Conflict( - "bookmark already exists for this manga/chapter".into(), - )), + Err(sqlx::Error::Database(ref db_err)) if db_err.is_unique_violation() => Err( + AppError::Conflict("bookmark already exists for this manga/chapter".into()), + ), Err(e) => Err(AppError::Database(e)), } } @@ -97,10 +97,3 @@ pub async fn delete(pool: &PgPool, id: Uuid) -> AppResult<()> { Ok(()) } -fn is_unique_violation(err: &sqlx::Error) -> bool { - if let sqlx::Error::Database(db_err) = err { - db_err.code().as_deref() == Some("23505") - } else { - false - } -} diff --git a/backend/src/repo/chapter.rs b/backend/src/repo/chapter.rs index 73f06f5..97d1732 100644 --- a/backend/src/repo/chapter.rs +++ b/backend/src/repo/chapter.rs @@ -4,7 +4,7 @@ use sqlx::{PgExecutor, PgPool}; use uuid::Uuid; use crate::domain::Chapter; -use crate::error::{AppError, AppResult}; +use crate::error::AppResult; pub async fn list_for_manga( pool: &PgPool, @@ -62,10 +62,9 @@ pub async fn find_by_id_in_manga( /// /// Chapter identity is the row UUID; the same (manga_id, number) /// combination can repeat (multiple translations, re-uploads). The -/// `is_unique_violation` branch below is a defensive holdover from -/// 0001's (manga_id, number) UNIQUE — it can no longer fire under -/// normal operation, but we surface a clean 409 if a future migration -/// re-adds any chapter uniqueness. +/// 0013 migration dropped the (manga_id, number) UNIQUE, so duplicate +/// inserts succeed by design. If a future migration re-adds any +/// uniqueness, surface a 409 by adding a unique-violation arm here. pub async fn create<'e, E: PgExecutor<'e>>( executor: E, manga_id: Uuid, @@ -73,7 +72,7 @@ pub async fn create<'e, E: PgExecutor<'e>>( title: Option<&str>, uploaded_by: Option, ) -> AppResult { - let result = sqlx::query_as::<_, Chapter>( + let row = sqlx::query_as::<_, Chapter>( r#" INSERT INTO chapters (manga_id, number, title, uploaded_by) VALUES ($1, $2, $3, $4) @@ -85,15 +84,58 @@ pub async fn create<'e, E: PgExecutor<'e>>( .bind(title) .bind(uploaded_by) .fetch_one(executor) - .await; + .await?; + Ok(row) +} - match result { - Ok(c) => Ok(c), - Err(e) if is_unique_violation(&e) => Err(AppError::Conflict(format!( - "chapter {number} conflicts with an existing chapter for this manga" - ))), - Err(e) => Err(AppError::Database(e)), - } +/// Cross-link guard for `POST /bookmarks`: the bookmarks FK accepts +/// any valid chapter id, but a chapter must belong to the bookmark's +/// manga or the bookmark would dangle on a foreign manga. Handlers +/// call this before the insert and surface `NotFound` when it +/// returns `false`. +pub async fn belongs_to_manga( + pool: &PgPool, + chapter_id: Uuid, + manga_id: Uuid, +) -> AppResult { + let (exists,): (bool,) = sqlx::query_as( + "SELECT EXISTS(SELECT 1 FROM chapters WHERE id = $1 AND manga_id = $2)", + ) + .bind(chapter_id) + .bind(manga_id) + .fetch_one(pool) + .await?; + Ok(exists) +} + +/// Read just the page_count for a chapter. Used by the crawler +/// daemon's consumer-side dedup safety net so it can ack-done a job +/// whose chapter has already been fetched by a racing worker. +pub async fn page_count(pool: &PgPool, id: Uuid) -> sqlx::Result> { + sqlx::query_scalar("SELECT page_count FROM chapters WHERE id = $1") + .bind(id) + .fetch_optional(pool) + .await +} + +/// Look up the manga_id + most recent source_url for a chapter. Used +/// by the daemon's chapter dispatcher to resolve the URL it needs to +/// hand to `content::sync_chapter_content`. Returns `None` if the +/// chapter (or its source row) is gone. +pub async fn dispatch_target( + pool: &PgPool, + chapter_id: Uuid, +) -> sqlx::Result> { + sqlx::query_as( + "SELECT c.manga_id, cs.source_url \ + FROM chapters c \ + JOIN chapter_sources cs ON cs.chapter_id = c.id \ + WHERE c.id = $1 \ + LIMIT 1", + ) + .bind(chapter_id) + .fetch_optional(pool) + .await } pub async fn set_page_count<'e, E: PgExecutor<'e>>( @@ -109,10 +151,3 @@ pub async fn set_page_count<'e, E: PgExecutor<'e>>( Ok(()) } -fn is_unique_violation(err: &sqlx::Error) -> bool { - if let sqlx::Error::Database(db_err) = err { - db_err.code().as_deref() == Some("23505") - } else { - false - } -} diff --git a/backend/src/repo/genre.rs b/backend/src/repo/genre.rs index 95ee11b..367dc1e 100644 --- a/backend/src/repo/genre.rs +++ b/backend/src/repo/genre.rs @@ -61,6 +61,11 @@ pub async fn load_for_mangas( /// FK constraint would reject them, so we filter upstream rather than /// surface a 500 here. (The API layer validates the set against /// `list_all` first.) +/// +/// Note: `crawler::repo::sync_genres` does a similar replace, but by +/// *name* and with auto-create of unseen genres — the crawler can't +/// validate against the curated vocabulary on its own. Both paths are +/// intentional; don't merge them without preserving that semantic. pub async fn set_for_manga( conn: &mut PgConnection, manga_id: Uuid, diff --git a/backend/src/repo/user.rs b/backend/src/repo/user.rs index 8fc1f1b..82d39c9 100644 --- a/backend/src/repo/user.rs +++ b/backend/src/repo/user.rs @@ -21,7 +21,7 @@ pub async fn create(pool: &PgPool, username: &str, password_hash: &str) -> AppRe match result { Ok(user) => Ok(user), - Err(e) if is_unique_violation(&e) => { + Err(sqlx::Error::Database(ref db_err)) if db_err.is_unique_violation() => { Err(AppError::Conflict("username is already taken".into())) } Err(e) => Err(AppError::Database(e)), @@ -56,10 +56,3 @@ pub async fn find_by_id(pool: &PgPool, id: Uuid) -> AppResult> { Ok(row) } -fn is_unique_violation(err: &sqlx::Error) -> bool { - if let sqlx::Error::Database(db_err) = err { - db_err.code().as_deref() == Some("23505") - } else { - false - } -}