diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 609e6cd..1ea07f2 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -1517,7 +1517,7 @@ checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" [[package]] name = "mangalord" -version = "0.68.0" +version = "0.69.0" dependencies = [ "anyhow", "argon2", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index a424043..9a040f2 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mangalord" -version = "0.68.0" +version = "0.69.0" edition = "2021" default-run = "mangalord" diff --git a/backend/src/api/page_tags.rs b/backend/src/api/page_tags.rs index e85835c..9edfc7d 100644 --- a/backend/src/api/page_tags.rs +++ b/backend/src/api/page_tags.rs @@ -14,12 +14,14 @@ use uuid::Uuid; use crate::api::pagination::PagedResponse; use crate::app::AppState; use crate::auth::extractor::CurrentUser; +use crate::domain::page_analysis::{ContentWarning, PageSearchItem}; use crate::domain::page_tag::{ NewPageTag, PageTagSummary, TaggedChapterAggregate, TaggedMangaAggregate, TaggedPageItem, }; use crate::error::{AppError, AppResult}; use crate::repo; +use crate::repo::page_analysis::PageSearchQuery; use crate::repo::page_tag::Order; pub fn routes() -> Router { @@ -31,6 +33,7 @@ pub fn routes() -> Router { // though the cookie already implies it. .route("/pages/:id/my-tags", get(list_for_page)) .route("/me/page-tags", get(list_mine)) + .route("/me/page-search", get(page_search)) .route("/me/page-tags/distinct", get(list_distinct_mine)) .route("/me/page-tags/chapters", get(list_chapters_for_tag)) .route("/me/page-tags/mangas", get(list_mangas_for_tag)) @@ -248,6 +251,109 @@ async fn list_mine( Ok(Json(PagedResponse::with_total(items, limit, offset, total))) } +#[derive(Debug, Deserialize)] +pub struct PageSearchParams { + #[serde(default = "default_limit")] + pub limit: i64, + #[serde(default)] + pub offset: i64, + /// Comma-separated tags, AND-ed. Matched against the caller's page + /// tags ∪ the global auto-tags. + #[serde(default)] + pub tags: Option, + /// Free-text query over the OCR + scene-description search document. + #[serde(default)] + pub text: Option, + /// Comma-separated content warnings the page must carry (all of them). + #[serde(default)] + pub cw_include: Option, + /// Comma-separated content warnings the page must NOT carry (any of). + #[serde(default)] + pub cw_exclude: Option, +} + +/// Split a comma-separated tag list into normalized, deduped tag names. +fn parse_tags_csv(raw: Option<&str>) -> AppResult> { + let Some(raw) = raw else { return Ok(Vec::new()) }; + let mut seen = std::collections::HashSet::new(); + let mut out = Vec::new(); + for part in raw.split(',') { + if part.trim().is_empty() { + continue; + } + let norm = normalize_tag(part)?; + if seen.insert(norm.clone()) { + out.push(norm); + } + } + Ok(out) +} + +/// Split + validate a comma-separated content-warning list against the +/// closed vocabulary, returning canonical lowercase names. Unknown values +/// are a 422 rather than a silent drop so a typo'd filter is visible. +fn parse_warnings_csv(raw: Option<&str>) -> AppResult> { + let Some(raw) = raw else { return Ok(Vec::new()) }; + let mut seen = std::collections::HashSet::new(); + let mut out = Vec::new(); + for part in raw.split(',') { + let t = part.trim(); + if t.is_empty() { + continue; + } + let w = ContentWarning::parse_strict(t).ok_or_else(|| AppError::ValidationFailed { + message: format!("unknown content warning {t:?}"), + details: json!({ "content_warning": "must be one of sexual|nudity|gore|violence|disturbing" }), + })?; + let canon = format!("{w:?}").to_lowercase(); + if seen.insert(canon.clone()) { + out.push(canon); + } + } + Ok(out) +} + +/// Content search over the caller's reachable pages: multi-tag AND (own +/// page tags ∪ global auto-tags), weighted OCR/scene text ranking, and +/// content-warning include/exclude. At least one positive filter (tags, +/// text, or cw_include) is required so the endpoint never dumps every page. +async fn page_search( + State(state): State, + CurrentUser(user): CurrentUser, + Query(params): Query, +) -> AppResult>> { + let limit = params.limit.clamp(1, 200); + let offset = params.offset.max(0); + let tags = parse_tags_csv(params.tags.as_deref())?; + let cw_include = parse_warnings_csv(params.cw_include.as_deref())?; + let cw_exclude = parse_warnings_csv(params.cw_exclude.as_deref())?; + let text = params + .text + .as_deref() + .map(str::trim) + .filter(|s| !s.is_empty()) + .map(str::to_string); + + if tags.is_empty() && text.is_none() && cw_include.is_empty() { + return Err(AppError::ValidationFailed { + message: "at least one of tags, text, or cw_include is required".into(), + details: json!({ "filter": "required" }), + }); + } + + let query = PageSearchQuery { + user_id: user.id, + tags, + text, + cw_include, + cw_exclude, + limit, + offset, + }; + let (items, total) = repo::page_analysis::page_search(&state.db, &query).await?; + Ok(Json(PagedResponse::with_total(items, limit, offset, total))) +} + async fn list_distinct_mine( State(state): State, CurrentUser(user): CurrentUser, diff --git a/backend/src/domain/mod.rs b/backend/src/domain/mod.rs index 06593e9..3f9d83e 100644 --- a/backend/src/domain/mod.rs +++ b/backend/src/domain/mod.rs @@ -28,8 +28,8 @@ pub use genre::{Genre, GenreRef}; pub use manga::{Manga, MangaCard, MangaDetail}; pub use page::Page; pub use page_analysis::{ - AnalysisStatus, ContentWarning, OcrKind, OcrResult, PageAnalysis, SafetyFlag, - VisionAnalysis, + AnalysisStatus, ContentWarning, OcrKind, OcrResult, PageAnalysis, PageSearchItem, + SafetyFlag, VisionAnalysis, }; pub use page_tag::{ NewPageTag, PageTagSummary, TaggedChapterAggregate, TaggedMangaAggregate, diff --git a/backend/src/domain/page_analysis.rs b/backend/src/domain/page_analysis.rs index 7d9b622..aad4655 100644 --- a/backend/src/domain/page_analysis.rs +++ b/backend/src/domain/page_analysis.rs @@ -108,6 +108,26 @@ impl ContentWarning { } } +/// One result row from the page content-search (`GET /v1/me/page-search`). +/// One row per matching page, carrying the breadcrumb plus the moderation +/// flags and the text-search rank so the UI can badge and order results. +#[derive(Debug, Clone, Serialize, FromRow)] +pub struct PageSearchItem { + pub page_id: Uuid, + pub chapter_id: Uuid, + pub manga_id: Uuid, + pub page_number: i32, + pub chapter_number: i32, + pub chapter_title: Option, + pub manga_title: String, + pub storage_key: String, + pub is_nsfw: bool, + /// Deduped content warnings on this page (canonical lowercase names). + pub content_warnings: Vec, + /// `ts_rank` against the text query; `0` for tag/warning-only searches. + pub rank: f32, +} + /// One persisted `page_analysis` row. #[derive(Debug, Clone, Serialize, FromRow)] pub struct PageAnalysis { diff --git a/backend/src/repo/page_analysis.rs b/backend/src/repo/page_analysis.rs index 62169aa..f989def 100644 --- a/backend/src/repo/page_analysis.rs +++ b/backend/src/repo/page_analysis.rs @@ -15,9 +15,27 @@ use sqlx::PgPool; use uuid::Uuid; use crate::crawler::jobs::{self, JobPayload}; -use crate::domain::page_analysis::{ContentWarning, OcrKind, PageAnalysis, VisionAnalysis}; +use crate::domain::page_analysis::{ + ContentWarning, OcrKind, PageAnalysis, PageSearchItem, VisionAnalysis, +}; use crate::error::AppResult; +/// Filter set for [`page_search`]. `tags` are AND-ed (a page must carry +/// every one, satisfiable by EITHER the user's page tag or a global auto +/// tag); `text` ranks via the weighted tsvector; `cw_include` requires all +/// listed warnings; `cw_exclude` rejects any. All string values arrive +/// pre-normalized (tags lowercased, warnings validated) from the handler. +#[derive(Debug, Default, Clone)] +pub struct PageSearchQuery { + pub user_id: uuid::Uuid, + pub tags: Vec, + pub text: Option, + pub cw_include: Vec, + pub cw_exclude: Vec, + pub limit: i64, + pub offset: i64, +} + /// Longest tag the shared `tags` table accepts (`upsert_by_name` enforces /// the same bound). Auto-tags over this are dropped here rather than /// aborting the whole page's analysis on one bad model output. @@ -245,6 +263,107 @@ pub async fn persist_analysis( Ok(()) } +/// Content search over pages: multi-tag AND across (user page tags ∪ +/// global auto tags), optional weighted text search over the OCR/scene +/// document, and content-warning include/exclude. Returns one row per +/// matching page plus the total for pagination. +/// +/// Empty `tags` makes the tag clause vacuously true (text- or +/// warning-only search), mirroring the manga search's `unnest` idiom. The +/// caller is responsible for requiring at least one positive filter so +/// this never degenerates into "every page". +pub async fn page_search( + pool: &PgPool, + q: &PageSearchQuery, +) -> AppResult<(Vec, i64)> { + // `text` participates in three places ($3): the rank, the match + // predicate, and the order key. Empty string disables text filtering. + let text = q.text.as_deref().unwrap_or("").trim(); + + const WHERE: &str = r#" + NOT EXISTS ( + SELECT 1 FROM unnest($2::text[]) AS req(name) + WHERE NOT EXISTS ( + SELECT 1 FROM page_tags ut + JOIN tags t ON t.id = ut.tag_id + WHERE ut.page_id = p.id AND ut.user_id = $1 AND lower(t.name) = req.name + UNION ALL + SELECT 1 FROM page_auto_tags at + JOIN tags t ON t.id = at.tag_id + WHERE at.page_id = p.id AND lower(t.name) = req.name + ) + ) + AND ($3 = '' OR pa.search_doc @@ plainto_tsquery('simple', $3)) + AND NOT EXISTS ( + SELECT 1 FROM unnest($4::text[]) AS req(w) + WHERE NOT EXISTS ( + SELECT 1 FROM page_content_warnings pw + WHERE pw.page_id = p.id AND pw.warning = req.w + ) + ) + AND NOT EXISTS ( + SELECT 1 FROM page_content_warnings pw + WHERE pw.page_id = p.id AND pw.warning = ANY($5::text[]) + ) + "#; + + let rows_sql = format!( + r#" + SELECT + p.id AS page_id, + p.chapter_id AS chapter_id, + ch.manga_id AS manga_id, + p.page_number AS page_number, + ch.number AS chapter_number, + ch.title AS chapter_title, + m.title AS manga_title, + p.storage_key AS storage_key, + COALESCE(pa.is_nsfw, false) AS is_nsfw, + COALESCE( + (SELECT array_agg(pw.warning ORDER BY pw.warning) + FROM page_content_warnings pw WHERE pw.page_id = p.id), + ARRAY[]::text[] + ) AS content_warnings, + COALESCE(ts_rank(pa.search_doc, plainto_tsquery('simple', $3)), 0)::real AS rank + FROM pages p + JOIN chapters ch ON ch.id = p.chapter_id + JOIN mangas m ON m.id = ch.manga_id + LEFT JOIN page_analysis pa ON pa.page_id = p.id + WHERE {WHERE} + ORDER BY (CASE WHEN $3 = '' THEN 0 ELSE 1 END) DESC, rank DESC, p.id + LIMIT $6 OFFSET $7 + "# + ); + let rows = sqlx::query_as::<_, PageSearchItem>(&rows_sql) + .bind(q.user_id) + .bind(&q.tags) + .bind(text) + .bind(&q.cw_include) + .bind(&q.cw_exclude) + .bind(q.limit) + .bind(q.offset) + .fetch_all(pool) + .await?; + + let count_sql = format!( + "SELECT count(*) FROM pages p \ + JOIN chapters ch ON ch.id = p.chapter_id \ + JOIN mangas m ON m.id = ch.manga_id \ + LEFT JOIN page_analysis pa ON pa.page_id = p.id \ + WHERE {WHERE}" + ); + let (total,): (i64,) = sqlx::query_as(&count_sql) + .bind(q.user_id) + .bind(&q.tags) + .bind(text) + .bind(&q.cw_include) + .bind(&q.cw_exclude) + .fetch_one(pool) + .await?; + + Ok((rows, total)) +} + /// Append a token to a tsvector text bucket with a trailing space. fn push_token(bucket: &mut String, text: &str) { bucket.push_str(text); diff --git a/backend/tests/api_page_search.rs b/backend/tests/api_page_search.rs new file mode 100644 index 0000000..91df59f --- /dev/null +++ b/backend/tests/api_page_search.rs @@ -0,0 +1,265 @@ +//! Integration tests for the page content-search endpoint +//! `GET /v1/me/page-search`: multi-tag AND across (user ∪ auto) tags, +//! weighted OCR/scene text ranking, and content-warning include/exclude. + +mod common; + +use axum::http::StatusCode; +use axum::Router; +use serde_json::json; +use sqlx::PgPool; +use tower::ServiceExt; +use uuid::Uuid; + +use mangalord::domain::page_analysis::{OcrResult, SafetyFlag, VisionAnalysis}; +use mangalord::repo; + +/// Seed a manga + chapter and return the chapter id for adding pages. +async fn seed_chapter(pool: &PgPool, title: &str) -> Uuid { + let manga_id: Uuid = + sqlx::query_scalar("INSERT INTO mangas (title) VALUES ($1) RETURNING id") + .bind(title) + .fetch_one(pool) + .await + .unwrap(); + sqlx::query_scalar("INSERT INTO chapters (manga_id, number) VALUES ($1, 1) RETURNING id") + .bind(manga_id) + .fetch_one(pool) + .await + .unwrap() +} + +async fn add_page(pool: &PgPool, chapter_id: Uuid, n: i32) -> Uuid { + sqlx::query_scalar( + "INSERT INTO pages (chapter_id, page_number, storage_key, content_type) \ + VALUES ($1, $2, $3, 'image/png') RETURNING id", + ) + .bind(chapter_id) + .bind(n) + .bind(format!("k/{n}.png")) + .fetch_one(pool) + .await + .unwrap() +} + +fn analysis( + ocr: &[(&str, &str)], + auto_tags: &[&str], + nsfw: bool, + warnings: &[&str], +) -> VisionAnalysis { + VisionAnalysis { + ocr_results: ocr + .iter() + .map(|(t, k)| OcrResult { + text: (*t).into(), + kind: (*k).into(), + }) + .collect(), + tagging_results: auto_tags.iter().map(|t| t.to_string()).collect(), + scene_description: String::new(), + safety_flag: SafetyFlag { + is_nsfw: nsfw, + content_type: warnings.iter().map(|w| w.to_string()).collect(), + }, + } +} + +async fn user_id_for(pool: &PgPool, username: &str) -> Uuid { + repo::user::find_by_username(pool, username) + .await + .unwrap() + .unwrap() + .id +} + +async fn search(app: &Router, cookie: &str, query: &str) -> serde_json::Value { + let resp = app + .clone() + .oneshot(common::get_with_cookie( + &format!("/api/v1/me/page-search?{query}"), + cookie, + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK, "search {query} failed"); + common::body_json(resp).await +} + +fn page_ids(body: &serde_json::Value) -> Vec { + body["items"] + .as_array() + .unwrap() + .iter() + .map(|i| i["page_id"].as_str().unwrap().to_string()) + .collect() +} + +#[sqlx::test(migrations = "./migrations")] +async fn multi_tag_and_across_user_and_auto_tags(pool: PgPool) { + let h = common::harness(pool.clone()); + let (username, cookie) = common::register_user(&h.app).await; + let uid = user_id_for(&pool, &username).await; + + let ch = seed_chapter(&pool, "M").await; + let p1 = add_page(&pool, ch, 1).await; + let p2 = add_page(&pool, ch, 2).await; + + // p1: auto-tag "b"; user adds tag "a" → has both a (user) and b (auto). + repo::page_analysis::persist_analysis(&pool, p1, &analysis(&[], &["b"], false, &[]), "m") + .await + .unwrap(); + repo::page_tag::upsert(&pool, uid, p1, "a").await.unwrap(); + // p2: only auto-tag "a". + repo::page_analysis::persist_analysis(&pool, p2, &analysis(&[], &["a"], false, &[]), "m") + .await + .unwrap(); + + // tags=a,b (AND) → only p1 satisfies both. + let body = search(&h.app, &cookie, "tags=a,b").await; + assert_eq!(page_ids(&body), vec![p1.to_string()]); + + // tags=a alone → both (p1 via user tag, p2 via auto tag). + let body = search(&h.app, &cookie, "tags=a").await; + let mut ids = page_ids(&body); + ids.sort(); + let mut want = vec![p1.to_string(), p2.to_string()]; + want.sort(); + assert_eq!(ids, want); +} + +#[sqlx::test(migrations = "./migrations")] +async fn text_search_ranks_speech_above_sfx(pool: PgPool) { + let h = common::harness(pool.clone()); + let (_u, cookie) = common::register_user(&h.app).await; + let ch = seed_chapter(&pool, "M").await; + let speech_page = add_page(&pool, ch, 1).await; + let sfx_page = add_page(&pool, ch, 2).await; + + repo::page_analysis::persist_analysis( + &pool, + speech_page, + &analysis(&[("alpha", "speech")], &[], false, &[]), + "m", + ) + .await + .unwrap(); + repo::page_analysis::persist_analysis( + &pool, + sfx_page, + &analysis(&[("alpha", "sfx")], &[], false, &[]), + "m", + ) + .await + .unwrap(); + + let body = search(&h.app, &cookie, "text=alpha").await; + let ids = page_ids(&body); + assert_eq!(ids.len(), 2, "both pages match the term"); + assert_eq!( + ids[0], + speech_page.to_string(), + "speech (weight A) must rank above sfx (weight D)" + ); +} + +#[sqlx::test(migrations = "./migrations")] +async fn content_warning_include_and_exclude(pool: PgPool) { + let h = common::harness(pool.clone()); + let (_u, cookie) = common::register_user(&h.app).await; + let ch = seed_chapter(&pool, "M").await; + let sexual = add_page(&pool, ch, 1).await; + let gore = add_page(&pool, ch, 2).await; + + repo::page_analysis::persist_analysis( + &pool, + sexual, + &analysis(&[("x", "speech")], &["nsfw-tag"], true, &["sexual"]), + "m", + ) + .await + .unwrap(); + repo::page_analysis::persist_analysis( + &pool, + gore, + &analysis(&[("x", "speech")], &["nsfw-tag"], true, &["gore"]), + "m", + ) + .await + .unwrap(); + + // include sexual → only the sexual page. + let body = search(&h.app, &cookie, "tags=nsfw-tag&cw_include=sexual").await; + assert_eq!(page_ids(&body), vec![sexual.to_string()]); + // and it carries the warning + nsfw flag in the row. + assert_eq!(body["items"][0]["is_nsfw"], true); + assert_eq!(body["items"][0]["content_warnings"][0], "sexual"); + + // exclude sexual → only the gore page. + let body = search(&h.app, &cookie, "tags=nsfw-tag&cw_exclude=sexual").await; + assert_eq!(page_ids(&body), vec![gore.to_string()]); +} + +#[sqlx::test(migrations = "./migrations")] +async fn text_only_search_needs_no_tags(pool: PgPool) { + let h = common::harness(pool.clone()); + let (_u, cookie) = common::register_user(&h.app).await; + let ch = seed_chapter(&pool, "M").await; + let p = add_page(&pool, ch, 1).await; + repo::page_analysis::persist_analysis( + &pool, + p, + &analysis(&[("dragon", "speech")], &[], false, &[]), + "m", + ) + .await + .unwrap(); + + // The previously-501 text param now returns results (no tags needed). + let body = search(&h.app, &cookie, "text=dragon").await; + assert_eq!(page_ids(&body), vec![p.to_string()]); +} + +#[sqlx::test(migrations = "./migrations")] +async fn search_without_any_filter_is_rejected(pool: PgPool) { + let h = common::harness(pool.clone()); + let (_u, cookie) = common::register_user(&h.app).await; + + let resp = h + .app + .clone() + .oneshot(common::get_with_cookie("/api/v1/me/page-search", &cookie)) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::UNPROCESSABLE_ENTITY); +} + +#[sqlx::test(migrations = "./migrations")] +async fn unknown_content_warning_is_rejected(pool: PgPool) { + let h = common::harness(pool.clone()); + let (_u, cookie) = common::register_user(&h.app).await; + + let resp = h + .app + .clone() + .oneshot(common::get_with_cookie( + "/api/v1/me/page-search?cw_include=spicy", + &cookie, + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::UNPROCESSABLE_ENTITY); +} + +#[sqlx::test(migrations = "./migrations")] +async fn requires_authentication(pool: PgPool) { + let h = common::harness(pool.clone()); + let resp = h + .app + .clone() + .oneshot(common::get("/api/v1/me/page-search?text=x")) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::UNAUTHORIZED); + let _ = json!({}); // silence unused import in some configs +} diff --git a/frontend/package.json b/frontend/package.json index 08833b1..f468ff3 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "mangalord-frontend", - "version": "0.68.0", + "version": "0.69.0", "private": true, "type": "module", "scripts": {