From 63e1aa54848c46307ba7f3323a1935a2008a686e Mon Sep 17 00:00:00 2001 From: MechaCat02 Date: Sat, 13 Jun 2026 18:27:08 +0200 Subject: [PATCH] feat(analysis): page-analysis schema, domain types, and analyze_page job kind Adds the persistence foundation for the AI content-analysis worker: - Migration 0025: page_analysis (status + scene + is_nsfw + weighted search_doc tsvector), page_ocr_text (kind-tagged pieces), global page_auto_tags (shared tags vocabulary, separate from per-user page_tags), and page_content_warnings (closed vocabulary). - domain::page_analysis: AnalysisStatus/OcrKind/ContentWarning enums with kind->tsvector-weight mapping and lenient model-string parsing, plus the VisionAnalysis response DTOs. - JobPayload::AnalyzePage { page_id, force } + KIND_ANALYZE_PAGE, reusing the existing crawler_jobs queue. - repo::page_analysis: enqueue_for_page, load, mark_failed, and the single transactional persist_analysis (delete+reinsert; idempotent; never touches user page_tags; computes the A/B/C/D-weighted search_doc). Tests: repo integration (persist/idempotency/user-tags-untouched/enqueue/ mark_failed/speech>sfx ranking) + unit (weights, parsing, DTO, job serde). Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/Cargo.lock | 2 +- backend/Cargo.toml | 2 +- backend/migrations/0025_page_analysis.sql | 81 +++++++ backend/src/crawler/jobs.rs | 36 ++++ backend/src/domain/mod.rs | 5 + backend/src/domain/page_analysis.rs | 212 ++++++++++++++++++ backend/src/repo/mod.rs | 1 + backend/src/repo/page_analysis.rs | 224 +++++++++++++++++++ backend/tests/api_page_analysis.rs | 252 ++++++++++++++++++++++ frontend/package.json | 2 +- 10 files changed, 814 insertions(+), 3 deletions(-) create mode 100644 backend/migrations/0025_page_analysis.sql create mode 100644 backend/src/domain/page_analysis.rs create mode 100644 backend/src/repo/page_analysis.rs create mode 100644 backend/tests/api_page_analysis.rs diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 96d5878..098e9a4 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -1470,7 +1470,7 @@ checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" [[package]] name = "mangalord" -version = "0.64.0" +version = "0.65.0" dependencies = [ "anyhow", "argon2", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index ba2a52d..e364e9f 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mangalord" -version = "0.64.0" +version = "0.65.0" edition = "2021" default-run = "mangalord" diff --git a/backend/migrations/0025_page_analysis.sql b/backend/migrations/0025_page_analysis.sql new file mode 100644 index 0000000..71b1930 --- /dev/null +++ b/backend/migrations/0025_page_analysis.sql @@ -0,0 +1,81 @@ +-- AI content-analysis / enrichment / moderation results, one analysis +-- pass per page image. A background worker calls a local vision model +-- and writes four kinds of output here: OCR text (weighted by kind), +-- global auto-tags, a scene description, and content-warning flags. +-- +-- Everything cascades with `pages` (like page_tags / collection_pages in +-- 0023): re-uploading a chapter recreates its page rows, intentionally +-- dropping the stale analysis so it gets re-enqueued and re-derived. +-- +-- The auto-tags live in their OWN table (`page_auto_tags`), NOT in the +-- per-user `page_tags`, so re-analysis is a clean delete+reinsert that +-- never touches a user's personal tags. They reference the SAME shared +-- `tags` vocabulary (0009 / 0024) that manga tags and page tags use, so +-- model-proposed tags and human tags share one global namespace. + +-- One row per analyzed page. `search_doc` is the worker-computed, +-- kind-weighted tsvector that the page text-search ranks against; it is +-- filled in the same transaction that writes the OCR rows (a generated +-- column can't aggregate the child `page_ocr_text` rows, and a trigger +-- would re-fire on every child insert during the delete+reinsert). +CREATE TABLE page_analysis ( + page_id uuid PRIMARY KEY REFERENCES pages(id) ON DELETE CASCADE, + status text NOT NULL DEFAULT 'pending' + CHECK (status IN ('pending', 'done', 'failed')), + scene_description text, + is_nsfw boolean NOT NULL DEFAULT false, + model text, + error text, + analyzed_at timestamptz, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + search_doc tsvector +); + +-- Full-text ranking over the weighted document. +CREATE INDEX page_analysis_search_idx ON page_analysis USING gin (search_doc); +-- Partial index for "is this page flagged?" lookups (content-warning +-- joins and NSFW filters touch only the flagged minority). +CREATE INDEX page_analysis_nsfw_idx ON page_analysis (page_id) WHERE is_nsfw; + +-- Extracted text pieces, each tagged with its kind. Kind drives the +-- tsvector weight (speech/title=A, narration/thought/caption=B, +-- sfx=D; scene_description is weighted C from page_analysis). `ord` +-- preserves reading order within the page. +CREATE TABLE page_ocr_text ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + page_id uuid NOT NULL REFERENCES pages(id) ON DELETE CASCADE, + kind text NOT NULL + CHECK (kind IN ('speech', 'thought', 'narration', 'sfx', 'title', 'caption')), + text text NOT NULL, + ord integer NOT NULL DEFAULT 0 +); + +CREATE INDEX page_ocr_text_page_idx ON page_ocr_text (page_id); + +-- Global, model-derived page tags — visible to every user and searchable +-- alongside personal page tags. Keyed to the shared `tags` table by id. +CREATE TABLE page_auto_tags ( + page_id uuid NOT NULL REFERENCES pages(id) ON DELETE CASCADE, + tag_id uuid NOT NULL REFERENCES tags(id) ON DELETE CASCADE, + created_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (page_id, tag_id) +); + +-- tag -> pages (search by auto-tag) and page -> tags (render a page's +-- auto-tags); the PK already covers (page_id, tag_id) but a standalone +-- tag_id index serves the reverse direction. +CREATE INDEX page_auto_tags_tag_idx ON page_auto_tags (tag_id); + +-- Per-page content warnings from the closed moderation vocabulary. The +-- manga detail page shows the deduplicated union across all its pages; +-- search filters include/exclude by these. +CREATE TABLE page_content_warnings ( + page_id uuid NOT NULL REFERENCES pages(id) ON DELETE CASCADE, + warning text NOT NULL + CHECK (warning IN ('sexual', 'nudity', 'gore', 'violence', 'disturbing')), + PRIMARY KEY (page_id, warning) +); + +-- warning -> pages, for the manga/page content-warning filters. +CREATE INDEX page_content_warnings_warning_idx ON page_content_warnings (warning); diff --git a/backend/src/crawler/jobs.rs b/backend/src/crawler/jobs.rs index 9f65d1b..f324c52 100644 --- a/backend/src/crawler/jobs.rs +++ b/backend/src/crawler/jobs.rs @@ -34,6 +34,15 @@ pub enum JobPayload { chapter_id: Uuid, source_chapter_key: String, }, + /// Run AI content-analysis (OCR, auto-tags, scene description, NSFW + /// moderation) on a single page image via the local vision model. + /// `force` re-analyzes a page that is already `done` (manual admin + /// re-trigger); otherwise the worker skips it. + AnalyzePage { + page_id: Uuid, + #[serde(default)] + force: bool, + }, } #[derive(Clone, Copy, Debug, sqlx::Type, Serialize, Deserialize)] @@ -52,6 +61,10 @@ pub enum JobState { /// without re-spelling the literal. pub const KIND_SYNC_CHAPTER_CONTENT: &str = "sync_chapter_content"; +/// Kind discriminator for AI page-analysis jobs. The analysis daemon +/// leases with this filter so it never contends with crawl jobs. +pub const KIND_ANALYZE_PAGE: &str = "analyze_page"; + #[derive(Debug)] pub enum EnqueueResult { Inserted(Uuid), @@ -323,6 +336,29 @@ pub async fn reap_done(pool: &PgPool, retention_days: u32) -> sqlx::Result mod tests { use super::*; + #[test] + fn analyze_page_payload_round_trips_through_tagged_json() { + let page_id = Uuid::new_v4(); + let payload = JobPayload::AnalyzePage { + page_id, + force: true, + }; + let json = serde_json::to_value(&payload).unwrap(); + assert_eq!(json["kind"], "analyze_page"); + assert_eq!(json["page_id"], page_id.to_string()); + assert_eq!(json["force"], true); + + // `force` defaults to false when an older enqueue omitted it. + let legacy = serde_json::json!({ "kind": "analyze_page", "page_id": page_id }); + match serde_json::from_value::(legacy).unwrap() { + JobPayload::AnalyzePage { page_id: pid, force } => { + assert_eq!(pid, page_id); + assert!(!force); + } + other => panic!("expected AnalyzePage, got {other:?}"), + } + } + #[test] fn backoff_base_grows_exponentially_and_caps_at_one_hour() { // attempts == 1 → 60s, doubling each step. diff --git a/backend/src/domain/mod.rs b/backend/src/domain/mod.rs index 6b60ca4..06593e9 100644 --- a/backend/src/domain/mod.rs +++ b/backend/src/domain/mod.rs @@ -7,6 +7,7 @@ pub mod collection; pub mod genre; pub mod manga; pub mod page; +pub mod page_analysis; pub mod page_tag; pub mod patch; pub mod read_progress; @@ -26,6 +27,10 @@ pub use collection::{Collection, CollectionPageItem, CollectionSummary}; pub use genre::{Genre, GenreRef}; pub use manga::{Manga, MangaCard, MangaDetail}; pub use page::Page; +pub use page_analysis::{ + AnalysisStatus, ContentWarning, OcrKind, OcrResult, PageAnalysis, SafetyFlag, + VisionAnalysis, +}; pub use page_tag::{ NewPageTag, PageTagSummary, TaggedChapterAggregate, TaggedMangaAggregate, TaggedPageItem, diff --git a/backend/src/domain/page_analysis.rs b/backend/src/domain/page_analysis.rs new file mode 100644 index 0000000..7d9b622 --- /dev/null +++ b/backend/src/domain/page_analysis.rs @@ -0,0 +1,212 @@ +//! AI page-analysis domain types. +//! +//! Two distinct shapes live here: +//! +//! * The **persisted** row types (`PageAnalysis`) and the closed +//! vocabularies (`OcrKind`, `ContentWarning`, `AnalysisStatus`) that the +//! `page_analysis` / `page_ocr_text` / `page_content_warnings` tables +//! constrain via CHECKs. +//! * The **vision-response** DTOs (`VisionAnalysis` and friends) that the +//! worker deserializes from the local model's JSON. These are +//! deliberately lenient — `kind` and `content_type` arrive as free +//! strings because a small local model can emit anything; mapping to the +//! closed enums (and dropping the unmappable) happens at persist time via +//! [`OcrKind::from_model_str`] / [`ContentWarning::from_model_str`]. + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use sqlx::FromRow; +use uuid::Uuid; + +/// Lifecycle of a page's analysis row. +#[derive(Debug, Clone, Copy, PartialEq, Eq, sqlx::Type, Serialize, Deserialize)] +#[sqlx(type_name = "text", rename_all = "snake_case")] +#[serde(rename_all = "snake_case")] +pub enum AnalysisStatus { + Pending, + Done, + Failed, +} + +/// Kind of an OCR'd text piece. Drives the search-ranking weight: +/// `Speech`/`Title` → A, `Narration`/`Thought`/`Caption` → B, `Sfx` → D +/// (the scene description is weighted C separately). See +/// [`OcrKind::weight`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq, sqlx::Type, Serialize, Deserialize)] +#[sqlx(type_name = "text", rename_all = "snake_case")] +#[serde(rename_all = "snake_case")] +pub enum OcrKind { + Speech, + Thought, + Narration, + Sfx, + Title, + Caption, +} + +impl OcrKind { + /// Postgres `tsvector` weight label for this kind. The default + /// `ts_rank` weights `{D,C,B,A} = {0.1,0.2,0.4,1.0}` then realize the + /// intended relevance ordering: speech/title most important, sfx least. + pub fn weight(self) -> char { + match self { + OcrKind::Speech | OcrKind::Title => 'A', + OcrKind::Narration | OcrKind::Thought | OcrKind::Caption => 'B', + OcrKind::Sfx => 'D', + } + } + + /// Map a model-supplied kind string onto the closed vocabulary. Unknown + /// or unparseable kinds fall back to `Narration` (the neutral + /// mid-weight bucket) rather than dropping the text — losing the + /// transcription entirely is worse than mis-weighting it. + pub fn from_model_str(raw: &str) -> OcrKind { + match raw.trim().to_lowercase().as_str() { + "speech" => OcrKind::Speech, + "thought" => OcrKind::Thought, + "narration" => OcrKind::Narration, + "sfx" => OcrKind::Sfx, + "title" => OcrKind::Title, + "caption" => OcrKind::Caption, + _ => OcrKind::Narration, + } + } +} + +/// A content-warning category from the closed moderation vocabulary. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, sqlx::Type, Serialize, Deserialize)] +#[sqlx(type_name = "text", rename_all = "snake_case")] +#[serde(rename_all = "snake_case")] +pub enum ContentWarning { + Sexual, + Nudity, + Gore, + Violence, + Disturbing, +} + +impl ContentWarning { + /// Map a model-supplied content-type string onto the closed + /// vocabulary, or `None` if it isn't one we recognize. Unlike OCR + /// kinds, an unknown warning is *dropped* — flagging a page with a + /// category we can't filter on is meaningless. + pub fn from_model_str(raw: &str) -> Option { + match raw.trim().to_lowercase().as_str() { + "sexual" => Some(ContentWarning::Sexual), + "nudity" => Some(ContentWarning::Nudity), + "gore" => Some(ContentWarning::Gore), + "violence" => Some(ContentWarning::Violence), + "disturbing" => Some(ContentWarning::Disturbing), + _ => None, + } + } + + /// Parse a wire/query-param value strictly (no fallback). Used by the + /// API layer to validate `cw_include` / `cw_exclude` filters. + pub fn parse_strict(raw: &str) -> Option { + ContentWarning::from_model_str(raw) + } +} + +/// One persisted `page_analysis` row. +#[derive(Debug, Clone, Serialize, FromRow)] +pub struct PageAnalysis { + pub page_id: Uuid, + pub status: AnalysisStatus, + pub scene_description: Option, + pub is_nsfw: bool, + pub model: Option, + pub error: Option, + pub analyzed_at: Option>, +} + +// --- Vision-response DTOs (deserialized from the local model) --------- + +/// The full JSON object the vision model returns for one page. Lenient by +/// design: see the module docs. +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)] +pub struct VisionAnalysis { + #[serde(default)] + pub ocr_results: Vec, + #[serde(default)] + pub tagging_results: Vec, + #[serde(default)] + pub scene_description: String, + #[serde(default)] + pub safety_flag: SafetyFlag, +} + +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)] +pub struct OcrResult { + #[serde(default)] + pub text: String, + /// Free string from the model; mapped via [`OcrKind::from_model_str`]. + #[serde(default)] + pub kind: String, +} + +#[derive(Debug, Clone, Default, Deserialize, Serialize, PartialEq)] +pub struct SafetyFlag { + #[serde(default)] + pub is_nsfw: bool, + /// Free strings; mapped via [`ContentWarning::from_model_str`]. + #[serde(default)] + pub content_type: Vec, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn ocr_kind_weights_match_spec() { + assert_eq!(OcrKind::Speech.weight(), 'A'); + assert_eq!(OcrKind::Title.weight(), 'A'); + assert_eq!(OcrKind::Narration.weight(), 'B'); + assert_eq!(OcrKind::Thought.weight(), 'B'); + assert_eq!(OcrKind::Caption.weight(), 'B'); + assert_eq!(OcrKind::Sfx.weight(), 'D'); + } + + #[test] + fn ocr_kind_from_model_str_falls_back_to_narration() { + assert_eq!(OcrKind::from_model_str("SPEECH"), OcrKind::Speech); + assert_eq!(OcrKind::from_model_str(" sfx "), OcrKind::Sfx); + assert_eq!(OcrKind::from_model_str("dialogue"), OcrKind::Narration); + assert_eq!(OcrKind::from_model_str(""), OcrKind::Narration); + } + + #[test] + fn content_warning_from_model_str_drops_unknown() { + assert_eq!( + ContentWarning::from_model_str("Sexual"), + Some(ContentWarning::Sexual) + ); + assert_eq!( + ContentWarning::from_model_str(" gore"), + Some(ContentWarning::Gore) + ); + assert_eq!(ContentWarning::from_model_str("spicy"), None); + assert_eq!(ContentWarning::from_model_str(""), None); + } + + #[test] + fn vision_analysis_deserializes_sample_and_tolerates_missing_fields() { + let json = r#"{ + "ocr_results": [{"text":"Hi","kind":"speech"}], + "tagging_results": ["action","city"], + "scene_description": "A street.", + "safety_flag": {"is_nsfw": false, "content_type": []} + }"#; + let v: VisionAnalysis = serde_json::from_str(json).unwrap(); + assert_eq!(v.ocr_results.len(), 1); + assert_eq!(v.tagging_results, vec!["action", "city"]); + assert!(!v.safety_flag.is_nsfw); + + // Missing optional fields default rather than failing the parse. + let sparse: VisionAnalysis = serde_json::from_str("{}").unwrap(); + assert!(sparse.ocr_results.is_empty()); + assert_eq!(sparse.scene_description, ""); + assert!(!sparse.safety_flag.is_nsfw); + } +} diff --git a/backend/src/repo/mod.rs b/backend/src/repo/mod.rs index 38ea155..63a02fc 100644 --- a/backend/src/repo/mod.rs +++ b/backend/src/repo/mod.rs @@ -9,6 +9,7 @@ pub mod crawler; pub mod genre; pub mod manga; pub mod page; +pub mod page_analysis; pub mod page_tag; pub mod read_progress; pub mod session; diff --git a/backend/src/repo/page_analysis.rs b/backend/src/repo/page_analysis.rs new file mode 100644 index 0000000..a406a0d --- /dev/null +++ b/backend/src/repo/page_analysis.rs @@ -0,0 +1,224 @@ +//! Persistence for AI page-analysis results. +//! +//! [`persist_analysis`] is the single transactional writer: it replaces a +//! page's OCR text, global auto-tags, and content warnings, upserts the +//! `page_analysis` row, and computes the kind-weighted `search_doc` +//! tsvector — all atomically, so re-analysis (delete+reinsert) is +//! idempotent and never leaves a half-written page. It deliberately does +//! NOT touch the per-user `page_tags` table. +//! +//! Auto-tag names resolve to the shared `tags` vocabulary via +//! [`crate::repo::tag::upsert_by_name`], the same path manga tags and +//! personal page tags use. + +use sqlx::PgPool; +use uuid::Uuid; + +use crate::crawler::jobs::{self, JobPayload}; +use crate::domain::page_analysis::{ContentWarning, OcrKind, PageAnalysis, VisionAnalysis}; +use crate::error::AppResult; + +/// Longest tag the shared `tags` table accepts (`upsert_by_name` enforces +/// the same bound). Auto-tags over this are dropped here rather than +/// aborting the whole page's analysis on one bad model output. +const MAX_TAG_CHARS: usize = 64; + +/// Enqueue an `analyze_page` job for `page_id`. `force` re-analyzes a page +/// that is already `done`. Enqueue is idempotent at the job level only in +/// that duplicate pending jobs are harmless — processing is idempotent. +pub async fn enqueue_for_page(pool: &PgPool, page_id: Uuid, force: bool) -> AppResult<()> { + jobs::enqueue(pool, &JobPayload::AnalyzePage { page_id, force }).await?; + Ok(()) +} + +/// Load a page's analysis row, if it has one. +pub async fn load(pool: &PgPool, page_id: Uuid) -> AppResult> { + let row = sqlx::query_as::<_, PageAnalysis>( + r#" + SELECT page_id, status, scene_description, is_nsfw, model, error, analyzed_at + FROM page_analysis + WHERE page_id = $1 + "#, + ) + .bind(page_id) + .fetch_optional(pool) + .await?; + Ok(row) +} + +/// Record that analysis failed terminally for a page (retries exhausted / +/// dead-lettered). Leaves a `failed` row so the page's state is +/// observable; it simply contributes nothing to search. +pub async fn mark_failed(pool: &PgPool, page_id: Uuid, error: &str) -> AppResult<()> { + sqlx::query( + r#" + INSERT INTO page_analysis (page_id, status, error) + VALUES ($1, 'failed', $2) + ON CONFLICT (page_id) DO UPDATE + SET status = 'failed', error = EXCLUDED.error, updated_at = now() + "#, + ) + .bind(page_id) + .bind(error) + .execute(pool) + .await?; + Ok(()) +} + +/// Persist a completed analysis for a page, replacing any previous result. +/// +/// The model's free-form `kind` / `content_type` strings are mapped onto +/// the closed vocabularies here ([`OcrKind::from_model_str`] / +/// [`ContentWarning::from_model_str`]); unrecognized warnings are dropped, +/// unknown OCR kinds fall back to `narration`. Empty / over-long tags and +/// empty OCR text are skipped so one bad item never aborts the page. +pub async fn persist_analysis( + pool: &PgPool, + page_id: Uuid, + analysis: &VisionAnalysis, + model: &str, +) -> AppResult<()> { + // Kind-weighted text buckets for the tsvector: A=speech/title, + // B=narration/thought/caption, C=scene, D=sfx. + let mut bucket_a = String::new(); + let mut bucket_b = String::new(); + let bucket_c = analysis.scene_description.trim().to_string(); + let mut bucket_d = String::new(); + + let mut ocr_rows: Vec<(OcrKind, String)> = Vec::new(); + for r in &analysis.ocr_results { + let text = r.text.trim(); + if text.is_empty() { + continue; + } + let kind = OcrKind::from_model_str(&r.kind); + match kind.weight() { + 'A' => push_token(&mut bucket_a, text), + 'B' => push_token(&mut bucket_b, text), + 'D' => push_token(&mut bucket_d, text), + _ => {} + } + ocr_rows.push((kind, text.to_string())); + } + + // Dedup tags case-insensitively, preserving first-seen order; drop + // empties and over-long names rather than failing the transaction. + let mut seen_tags = std::collections::HashSet::new(); + let mut tags: Vec = Vec::new(); + for raw in &analysis.tagging_results { + let t = raw.trim(); + if t.is_empty() || t.chars().count() > MAX_TAG_CHARS { + continue; + } + if seen_tags.insert(t.to_lowercase()) { + tags.push(t.to_string()); + } + } + + // Map + dedup warnings. + let mut seen_warn = std::collections::HashSet::new(); + let mut warnings: Vec = Vec::new(); + for raw in &analysis.safety_flag.content_type { + if let Some(w) = ContentWarning::from_model_str(raw) { + if seen_warn.insert(w) { + warnings.push(w); + } + } + } + + let mut tx = pool.begin().await?; + + sqlx::query("DELETE FROM page_ocr_text WHERE page_id = $1") + .bind(page_id) + .execute(&mut *tx) + .await?; + sqlx::query("DELETE FROM page_auto_tags WHERE page_id = $1") + .bind(page_id) + .execute(&mut *tx) + .await?; + sqlx::query("DELETE FROM page_content_warnings WHERE page_id = $1") + .bind(page_id) + .execute(&mut *tx) + .await?; + + for (ord, (kind, text)) in ocr_rows.iter().enumerate() { + sqlx::query( + "INSERT INTO page_ocr_text (page_id, kind, text, ord) VALUES ($1, $2, $3, $4)", + ) + .bind(page_id) + .bind(kind) + .bind(text) + .bind(ord as i32) + .execute(&mut *tx) + .await?; + } + + for tag in &tags { + let tag_row = crate::repo::tag::upsert_by_name(&mut *tx, tag).await?; + sqlx::query( + "INSERT INTO page_auto_tags (page_id, tag_id) VALUES ($1, $2) \ + ON CONFLICT (page_id, tag_id) DO NOTHING", + ) + .bind(page_id) + .bind(tag_row.id) + .execute(&mut *tx) + .await?; + } + + for w in &warnings { + sqlx::query( + "INSERT INTO page_content_warnings (page_id, warning) VALUES ($1, $2) \ + ON CONFLICT (page_id, warning) DO NOTHING", + ) + .bind(page_id) + .bind(w) + .execute(&mut *tx) + .await?; + } + + let scene = if bucket_c.is_empty() { + None + } else { + Some(bucket_c.as_str()) + }; + sqlx::query( + r#" + INSERT INTO page_analysis + (page_id, status, scene_description, is_nsfw, model, analyzed_at, search_doc) + VALUES + ($1, 'done', $2, $3, $4, now(), + setweight(to_tsvector('simple', $5), 'A') || + setweight(to_tsvector('simple', $6), 'B') || + setweight(to_tsvector('simple', $7), 'C') || + setweight(to_tsvector('simple', $8), 'D')) + ON CONFLICT (page_id) DO UPDATE SET + status = 'done', + scene_description = EXCLUDED.scene_description, + is_nsfw = EXCLUDED.is_nsfw, + model = EXCLUDED.model, + analyzed_at = now(), + error = NULL, + search_doc = EXCLUDED.search_doc, + updated_at = now() + "#, + ) + .bind(page_id) + .bind(scene) + .bind(analysis.safety_flag.is_nsfw) + .bind(model) + .bind(&bucket_a) + .bind(&bucket_b) + .bind(&bucket_c) + .bind(&bucket_d) + .execute(&mut *tx) + .await?; + + tx.commit().await?; + Ok(()) +} + +/// Append a token to a tsvector text bucket with a trailing space. +fn push_token(bucket: &mut String, text: &str) { + bucket.push_str(text); + bucket.push(' '); +} diff --git a/backend/tests/api_page_analysis.rs b/backend/tests/api_page_analysis.rs new file mode 100644 index 0000000..0647980 --- /dev/null +++ b/backend/tests/api_page_analysis.rs @@ -0,0 +1,252 @@ +//! Integration tests for `repo::page_analysis` — the transactional writer +//! that persists AI page-analysis output (OCR text, global auto-tags, +//! content warnings, the weighted search tsvector) and the `analyze_page` +//! job enqueue. Each `#[sqlx::test]` gets a fresh migrated DB. + +use mangalord::domain::page_analysis::{ + AnalysisStatus, OcrResult, SafetyFlag, VisionAnalysis, +}; +use mangalord::repo::page_analysis; +use sqlx::PgPool; +use uuid::Uuid; + +/// Seed a manga → chapter → page chain and return the page id. +async fn seed_page(pool: &PgPool) -> Uuid { + let manga_id: Uuid = + sqlx::query_scalar("INSERT INTO mangas (title) VALUES ('M') RETURNING id") + .fetch_one(pool) + .await + .unwrap(); + let chapter_id: Uuid = sqlx::query_scalar( + "INSERT INTO chapters (manga_id, number) VALUES ($1, 1) RETURNING id", + ) + .bind(manga_id) + .fetch_one(pool) + .await + .unwrap(); + sqlx::query_scalar( + "INSERT INTO pages (chapter_id, page_number, storage_key, content_type) \ + VALUES ($1, 1, 'mangas/x/p1.png', 'image/png') RETURNING id", + ) + .bind(chapter_id) + .fetch_one(pool) + .await + .unwrap() +} + +fn analysis( + ocr: &[(&str, &str)], + tags: &[&str], + scene: &str, + nsfw: bool, + warnings: &[&str], +) -> VisionAnalysis { + VisionAnalysis { + ocr_results: ocr + .iter() + .map(|(text, kind)| OcrResult { + text: (*text).to_string(), + kind: (*kind).to_string(), + }) + .collect(), + tagging_results: tags.iter().map(|t| t.to_string()).collect(), + scene_description: scene.to_string(), + safety_flag: SafetyFlag { + is_nsfw: nsfw, + content_type: warnings.iter().map(|w| w.to_string()).collect(), + }, + } +} + +async fn count(pool: &PgPool, table: &str, page_id: Uuid) -> i64 { + sqlx::query_scalar(&format!("SELECT count(*) FROM {table} WHERE page_id = $1")) + .bind(page_id) + .fetch_one(pool) + .await + .unwrap() +} + +#[sqlx::test(migrations = "./migrations")] +async fn persist_writes_ocr_tags_warnings_and_search_doc(pool: PgPool) { + let page_id = seed_page(&pool).await; + let a = analysis( + &[("Hello", "speech"), ("BOOM", "sfx")], + &["action", "city"], + "A rainy street at night.", + true, + &["violence"], + ); + + page_analysis::persist_analysis(&pool, page_id, &a, "test-model") + .await + .unwrap(); + + assert_eq!(count(&pool, "page_ocr_text", page_id).await, 2); + assert_eq!(count(&pool, "page_auto_tags", page_id).await, 2); + assert_eq!(count(&pool, "page_content_warnings", page_id).await, 1); + + let row = page_analysis::load(&pool, page_id).await.unwrap().unwrap(); + assert_eq!(row.status, AnalysisStatus::Done); + assert!(row.is_nsfw); + assert_eq!(row.scene_description.as_deref(), Some("A rainy street at night.")); + assert_eq!(row.model.as_deref(), Some("test-model")); + + // search_doc must be populated (non-null, non-empty). + let has_doc: bool = sqlx::query_scalar( + "SELECT search_doc IS NOT NULL AND search_doc != ''::tsvector \ + FROM page_analysis WHERE page_id = $1", + ) + .bind(page_id) + .fetch_one(&pool) + .await + .unwrap(); + assert!(has_doc, "search_doc should be a populated tsvector"); +} + +#[sqlx::test(migrations = "./migrations")] +async fn persist_is_idempotent_and_replaces_prior_result(pool: PgPool) { + let page_id = seed_page(&pool).await; + + page_analysis::persist_analysis( + &pool, + page_id, + &analysis(&[("one", "speech")], &["alpha", "beta"], "first", true, &["gore"]), + "m1", + ) + .await + .unwrap(); + + // Re-analysis with different output fully replaces the prior rows. + page_analysis::persist_analysis( + &pool, + page_id, + &analysis(&[("two", "narration")], &["beta", "gamma"], "second", false, &[]), + "m2", + ) + .await + .unwrap(); + + // Exactly one analysis row, reflecting the second pass. + let total: i64 = sqlx::query_scalar("SELECT count(*) FROM page_analysis WHERE page_id = $1") + .bind(page_id) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(total, 1); + + let row = page_analysis::load(&pool, page_id).await.unwrap().unwrap(); + assert!(!row.is_nsfw); + assert_eq!(row.scene_description.as_deref(), Some("second")); + assert_eq!(row.model.as_deref(), Some("m2")); + + // Auto-tags are {beta, gamma}, NOT the union with the first pass. + let tag_names: Vec = sqlx::query_scalar( + "SELECT t.name FROM page_auto_tags pat JOIN tags t ON t.id = pat.tag_id \ + WHERE pat.page_id = $1 ORDER BY t.name", + ) + .bind(page_id) + .fetch_all(&pool) + .await + .unwrap(); + assert_eq!(tag_names, vec!["beta", "gamma"]); + + // Warnings cleared (second pass had none). + assert_eq!(count(&pool, "page_content_warnings", page_id).await, 0); + assert_eq!(count(&pool, "page_ocr_text", page_id).await, 1); +} + +#[sqlx::test(migrations = "./migrations")] +async fn persist_leaves_user_page_tags_untouched(pool: PgPool) { + let page_id = seed_page(&pool).await; + let user_id: Uuid = sqlx::query_scalar( + "INSERT INTO users (username, password_hash) VALUES ('alice', 'x') RETURNING id", + ) + .fetch_one(&pool) + .await + .unwrap(); + + // A personal page tag the user added. + mangalord::repo::page_tag::upsert(&pool, user_id, page_id, "favourite") + .await + .unwrap(); + + // The auto-tagger proposes its own (overlapping + new) tags. + page_analysis::persist_analysis( + &pool, + page_id, + &analysis(&[], &["favourite", "robot"], "", false, &[]), + "m", + ) + .await + .unwrap(); + + // The user's personal tag survives untouched. + let mine = mangalord::repo::page_tag::list_for_page(&pool, user_id, page_id) + .await + .unwrap(); + assert_eq!(mine, vec!["favourite"]); + + // And the global auto-tags are separate rows. + assert_eq!(count(&pool, "page_auto_tags", page_id).await, 2); +} + +#[sqlx::test(migrations = "./migrations")] +async fn enqueue_for_page_inserts_analyze_page_job(pool: PgPool) { + let page_id = seed_page(&pool).await; + + page_analysis::enqueue_for_page(&pool, page_id, false) + .await + .unwrap(); + + let payload: serde_json::Value = sqlx::query_scalar( + "SELECT payload FROM crawler_jobs WHERE payload->>'kind' = 'analyze_page'", + ) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(payload["page_id"].as_str().unwrap(), page_id.to_string()); + assert_eq!(payload["force"].as_bool(), Some(false)); +} + +#[sqlx::test(migrations = "./migrations")] +async fn mark_failed_writes_a_failed_row(pool: PgPool) { + let page_id = seed_page(&pool).await; + + page_analysis::mark_failed(&pool, page_id, "model timed out") + .await + .unwrap(); + + let row = page_analysis::load(&pool, page_id).await.unwrap().unwrap(); + assert_eq!(row.status, AnalysisStatus::Failed); + assert_eq!(row.error.as_deref(), Some("model timed out")); +} + +#[sqlx::test(migrations = "./migrations")] +async fn search_doc_ranks_speech_above_sfx(pool: PgPool) { + let page_id = seed_page(&pool).await; + // "alpha" is speech (weight A), "beta" is sfx (weight D). + page_analysis::persist_analysis( + &pool, + page_id, + &analysis(&[("alpha", "speech"), ("beta", "sfx")], &[], "", false, &[]), + "m", + ) + .await + .unwrap(); + + let (ra, rb): (f32, f32) = sqlx::query_as( + "SELECT ts_rank(search_doc, plainto_tsquery('simple', 'alpha')), \ + ts_rank(search_doc, plainto_tsquery('simple', 'beta')) \ + FROM page_analysis WHERE page_id = $1", + ) + .bind(page_id) + .fetch_one(&pool) + .await + .unwrap(); + + assert!(ra > 0.0 && rb > 0.0, "both terms should match the doc"); + assert!( + ra > rb, + "speech (weight A) must rank higher than sfx (weight D): {ra} vs {rb}" + ); +} diff --git a/frontend/package.json b/frontend/package.json index d64fc9d..b75f678 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "mangalord-frontend", - "version": "0.64.0", + "version": "0.65.0", "private": true, "type": "module", "scripts": {