feat(analysis): page-analysis schema, domain types, and analyze_page job kind

Adds the persistence foundation for the AI content-analysis worker:

- Migration 0025: page_analysis (status + scene + is_nsfw + weighted
  search_doc tsvector), page_ocr_text (kind-tagged pieces), global
  page_auto_tags (shared tags vocabulary, separate from per-user
  page_tags), and page_content_warnings (closed vocabulary).
- domain::page_analysis: AnalysisStatus/OcrKind/ContentWarning enums with
  kind->tsvector-weight mapping and lenient model-string parsing, plus the
  VisionAnalysis response DTOs.
- JobPayload::AnalyzePage { page_id, force } + KIND_ANALYZE_PAGE, reusing
  the existing crawler_jobs queue.
- repo::page_analysis: enqueue_for_page, load, mark_failed, and the single
  transactional persist_analysis (delete+reinsert; idempotent; never
  touches user page_tags; computes the A/B/C/D-weighted search_doc).

Tests: repo integration (persist/idempotency/user-tags-untouched/enqueue/
mark_failed/speech>sfx ranking) + unit (weights, parsing, DTO, job serde).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
MechaCat02
2026-06-13 18:27:08 +02:00
parent f30600162e
commit 63e1aa5484
10 changed files with 814 additions and 3 deletions

View File

@@ -7,6 +7,7 @@ pub mod collection;
pub mod genre;
pub mod manga;
pub mod page;
pub mod page_analysis;
pub mod page_tag;
pub mod patch;
pub mod read_progress;
@@ -26,6 +27,10 @@ pub use collection::{Collection, CollectionPageItem, CollectionSummary};
pub use genre::{Genre, GenreRef};
pub use manga::{Manga, MangaCard, MangaDetail};
pub use page::Page;
pub use page_analysis::{
AnalysisStatus, ContentWarning, OcrKind, OcrResult, PageAnalysis, SafetyFlag,
VisionAnalysis,
};
pub use page_tag::{
NewPageTag, PageTagSummary, TaggedChapterAggregate, TaggedMangaAggregate,
TaggedPageItem,

View File

@@ -0,0 +1,212 @@
//! AI page-analysis domain types.
//!
//! Two distinct shapes live here:
//!
//! * The **persisted** row types (`PageAnalysis`) and the closed
//! vocabularies (`OcrKind`, `ContentWarning`, `AnalysisStatus`) that the
//! `page_analysis` / `page_ocr_text` / `page_content_warnings` tables
//! constrain via CHECKs.
//! * The **vision-response** DTOs (`VisionAnalysis` and friends) that the
//! worker deserializes from the local model's JSON. These are
//! deliberately lenient — `kind` and `content_type` arrive as free
//! strings because a small local model can emit anything; mapping to the
//! closed enums (and dropping the unmappable) happens at persist time via
//! [`OcrKind::from_model_str`] / [`ContentWarning::from_model_str`].
use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use sqlx::FromRow;
use uuid::Uuid;
/// Lifecycle of a page's analysis row.
#[derive(Debug, Clone, Copy, PartialEq, Eq, sqlx::Type, Serialize, Deserialize)]
#[sqlx(type_name = "text", rename_all = "snake_case")]
#[serde(rename_all = "snake_case")]
pub enum AnalysisStatus {
Pending,
Done,
Failed,
}
/// Kind of an OCR'd text piece. Drives the search-ranking weight:
/// `Speech`/`Title` → A, `Narration`/`Thought`/`Caption` → B, `Sfx` → D
/// (the scene description is weighted C separately). See
/// [`OcrKind::weight`].
#[derive(Debug, Clone, Copy, PartialEq, Eq, sqlx::Type, Serialize, Deserialize)]
#[sqlx(type_name = "text", rename_all = "snake_case")]
#[serde(rename_all = "snake_case")]
pub enum OcrKind {
Speech,
Thought,
Narration,
Sfx,
Title,
Caption,
}
impl OcrKind {
/// Postgres `tsvector` weight label for this kind. The default
/// `ts_rank` weights `{D,C,B,A} = {0.1,0.2,0.4,1.0}` then realize the
/// intended relevance ordering: speech/title most important, sfx least.
pub fn weight(self) -> char {
match self {
OcrKind::Speech | OcrKind::Title => 'A',
OcrKind::Narration | OcrKind::Thought | OcrKind::Caption => 'B',
OcrKind::Sfx => 'D',
}
}
/// Map a model-supplied kind string onto the closed vocabulary. Unknown
/// or unparseable kinds fall back to `Narration` (the neutral
/// mid-weight bucket) rather than dropping the text — losing the
/// transcription entirely is worse than mis-weighting it.
pub fn from_model_str(raw: &str) -> OcrKind {
match raw.trim().to_lowercase().as_str() {
"speech" => OcrKind::Speech,
"thought" => OcrKind::Thought,
"narration" => OcrKind::Narration,
"sfx" => OcrKind::Sfx,
"title" => OcrKind::Title,
"caption" => OcrKind::Caption,
_ => OcrKind::Narration,
}
}
}
/// A content-warning category from the closed moderation vocabulary.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, sqlx::Type, Serialize, Deserialize)]
#[sqlx(type_name = "text", rename_all = "snake_case")]
#[serde(rename_all = "snake_case")]
pub enum ContentWarning {
Sexual,
Nudity,
Gore,
Violence,
Disturbing,
}
impl ContentWarning {
/// Map a model-supplied content-type string onto the closed
/// vocabulary, or `None` if it isn't one we recognize. Unlike OCR
/// kinds, an unknown warning is *dropped* — flagging a page with a
/// category we can't filter on is meaningless.
pub fn from_model_str(raw: &str) -> Option<ContentWarning> {
match raw.trim().to_lowercase().as_str() {
"sexual" => Some(ContentWarning::Sexual),
"nudity" => Some(ContentWarning::Nudity),
"gore" => Some(ContentWarning::Gore),
"violence" => Some(ContentWarning::Violence),
"disturbing" => Some(ContentWarning::Disturbing),
_ => None,
}
}
/// Parse a wire/query-param value strictly (no fallback). Used by the
/// API layer to validate `cw_include` / `cw_exclude` filters.
pub fn parse_strict(raw: &str) -> Option<ContentWarning> {
ContentWarning::from_model_str(raw)
}
}
/// One persisted `page_analysis` row.
#[derive(Debug, Clone, Serialize, FromRow)]
pub struct PageAnalysis {
pub page_id: Uuid,
pub status: AnalysisStatus,
pub scene_description: Option<String>,
pub is_nsfw: bool,
pub model: Option<String>,
pub error: Option<String>,
pub analyzed_at: Option<DateTime<Utc>>,
}
// --- Vision-response DTOs (deserialized from the local model) ---------
/// The full JSON object the vision model returns for one page. Lenient by
/// design: see the module docs.
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)]
pub struct VisionAnalysis {
#[serde(default)]
pub ocr_results: Vec<OcrResult>,
#[serde(default)]
pub tagging_results: Vec<String>,
#[serde(default)]
pub scene_description: String,
#[serde(default)]
pub safety_flag: SafetyFlag,
}
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)]
pub struct OcrResult {
#[serde(default)]
pub text: String,
/// Free string from the model; mapped via [`OcrKind::from_model_str`].
#[serde(default)]
pub kind: String,
}
#[derive(Debug, Clone, Default, Deserialize, Serialize, PartialEq)]
pub struct SafetyFlag {
#[serde(default)]
pub is_nsfw: bool,
/// Free strings; mapped via [`ContentWarning::from_model_str`].
#[serde(default)]
pub content_type: Vec<String>,
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn ocr_kind_weights_match_spec() {
assert_eq!(OcrKind::Speech.weight(), 'A');
assert_eq!(OcrKind::Title.weight(), 'A');
assert_eq!(OcrKind::Narration.weight(), 'B');
assert_eq!(OcrKind::Thought.weight(), 'B');
assert_eq!(OcrKind::Caption.weight(), 'B');
assert_eq!(OcrKind::Sfx.weight(), 'D');
}
#[test]
fn ocr_kind_from_model_str_falls_back_to_narration() {
assert_eq!(OcrKind::from_model_str("SPEECH"), OcrKind::Speech);
assert_eq!(OcrKind::from_model_str(" sfx "), OcrKind::Sfx);
assert_eq!(OcrKind::from_model_str("dialogue"), OcrKind::Narration);
assert_eq!(OcrKind::from_model_str(""), OcrKind::Narration);
}
#[test]
fn content_warning_from_model_str_drops_unknown() {
assert_eq!(
ContentWarning::from_model_str("Sexual"),
Some(ContentWarning::Sexual)
);
assert_eq!(
ContentWarning::from_model_str(" gore"),
Some(ContentWarning::Gore)
);
assert_eq!(ContentWarning::from_model_str("spicy"), None);
assert_eq!(ContentWarning::from_model_str(""), None);
}
#[test]
fn vision_analysis_deserializes_sample_and_tolerates_missing_fields() {
let json = r#"{
"ocr_results": [{"text":"Hi","kind":"speech"}],
"tagging_results": ["action","city"],
"scene_description": "A street.",
"safety_flag": {"is_nsfw": false, "content_type": []}
}"#;
let v: VisionAnalysis = serde_json::from_str(json).unwrap();
assert_eq!(v.ocr_results.len(), 1);
assert_eq!(v.tagging_results, vec!["action", "city"]);
assert!(!v.safety_flag.is_nsfw);
// Missing optional fields default rather than failing the parse.
let sparse: VisionAnalysis = serde_json::from_str("{}").unwrap();
assert!(sparse.ocr_results.is_empty());
assert_eq!(sparse.scene_description, "");
assert!(!sparse.safety_flag.is_nsfw);
}
}