feat(analysis): page-analysis schema, domain types, and analyze_page job kind

Adds the persistence foundation for the AI content-analysis worker:

- Migration 0025: page_analysis (status + scene + is_nsfw + weighted
  search_doc tsvector), page_ocr_text (kind-tagged pieces), global
  page_auto_tags (shared tags vocabulary, separate from per-user
  page_tags), and page_content_warnings (closed vocabulary).
- domain::page_analysis: AnalysisStatus/OcrKind/ContentWarning enums with
  kind->tsvector-weight mapping and lenient model-string parsing, plus the
  VisionAnalysis response DTOs.
- JobPayload::AnalyzePage { page_id, force } + KIND_ANALYZE_PAGE, reusing
  the existing crawler_jobs queue.
- repo::page_analysis: enqueue_for_page, load, mark_failed, and the single
  transactional persist_analysis (delete+reinsert; idempotent; never
  touches user page_tags; computes the A/B/C/D-weighted search_doc).

Tests: repo integration (persist/idempotency/user-tags-untouched/enqueue/
mark_failed/speech>sfx ranking) + unit (weights, parsing, DTO, job serde).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
MechaCat02
2026-06-13 18:27:08 +02:00
parent f30600162e
commit 63e1aa5484
10 changed files with 814 additions and 3 deletions

View File

@@ -0,0 +1,252 @@
//! Integration tests for `repo::page_analysis` — the transactional writer
//! that persists AI page-analysis output (OCR text, global auto-tags,
//! content warnings, the weighted search tsvector) and the `analyze_page`
//! job enqueue. Each `#[sqlx::test]` gets a fresh migrated DB.
use mangalord::domain::page_analysis::{
AnalysisStatus, OcrResult, SafetyFlag, VisionAnalysis,
};
use mangalord::repo::page_analysis;
use sqlx::PgPool;
use uuid::Uuid;
/// Seed a manga → chapter → page chain and return the page id.
async fn seed_page(pool: &PgPool) -> Uuid {
let manga_id: Uuid =
sqlx::query_scalar("INSERT INTO mangas (title) VALUES ('M') RETURNING id")
.fetch_one(pool)
.await
.unwrap();
let chapter_id: Uuid = sqlx::query_scalar(
"INSERT INTO chapters (manga_id, number) VALUES ($1, 1) RETURNING id",
)
.bind(manga_id)
.fetch_one(pool)
.await
.unwrap();
sqlx::query_scalar(
"INSERT INTO pages (chapter_id, page_number, storage_key, content_type) \
VALUES ($1, 1, 'mangas/x/p1.png', 'image/png') RETURNING id",
)
.bind(chapter_id)
.fetch_one(pool)
.await
.unwrap()
}
fn analysis(
ocr: &[(&str, &str)],
tags: &[&str],
scene: &str,
nsfw: bool,
warnings: &[&str],
) -> VisionAnalysis {
VisionAnalysis {
ocr_results: ocr
.iter()
.map(|(text, kind)| OcrResult {
text: (*text).to_string(),
kind: (*kind).to_string(),
})
.collect(),
tagging_results: tags.iter().map(|t| t.to_string()).collect(),
scene_description: scene.to_string(),
safety_flag: SafetyFlag {
is_nsfw: nsfw,
content_type: warnings.iter().map(|w| w.to_string()).collect(),
},
}
}
async fn count(pool: &PgPool, table: &str, page_id: Uuid) -> i64 {
sqlx::query_scalar(&format!("SELECT count(*) FROM {table} WHERE page_id = $1"))
.bind(page_id)
.fetch_one(pool)
.await
.unwrap()
}
#[sqlx::test(migrations = "./migrations")]
async fn persist_writes_ocr_tags_warnings_and_search_doc(pool: PgPool) {
let page_id = seed_page(&pool).await;
let a = analysis(
&[("Hello", "speech"), ("BOOM", "sfx")],
&["action", "city"],
"A rainy street at night.",
true,
&["violence"],
);
page_analysis::persist_analysis(&pool, page_id, &a, "test-model")
.await
.unwrap();
assert_eq!(count(&pool, "page_ocr_text", page_id).await, 2);
assert_eq!(count(&pool, "page_auto_tags", page_id).await, 2);
assert_eq!(count(&pool, "page_content_warnings", page_id).await, 1);
let row = page_analysis::load(&pool, page_id).await.unwrap().unwrap();
assert_eq!(row.status, AnalysisStatus::Done);
assert!(row.is_nsfw);
assert_eq!(row.scene_description.as_deref(), Some("A rainy street at night."));
assert_eq!(row.model.as_deref(), Some("test-model"));
// search_doc must be populated (non-null, non-empty).
let has_doc: bool = sqlx::query_scalar(
"SELECT search_doc IS NOT NULL AND search_doc != ''::tsvector \
FROM page_analysis WHERE page_id = $1",
)
.bind(page_id)
.fetch_one(&pool)
.await
.unwrap();
assert!(has_doc, "search_doc should be a populated tsvector");
}
#[sqlx::test(migrations = "./migrations")]
async fn persist_is_idempotent_and_replaces_prior_result(pool: PgPool) {
let page_id = seed_page(&pool).await;
page_analysis::persist_analysis(
&pool,
page_id,
&analysis(&[("one", "speech")], &["alpha", "beta"], "first", true, &["gore"]),
"m1",
)
.await
.unwrap();
// Re-analysis with different output fully replaces the prior rows.
page_analysis::persist_analysis(
&pool,
page_id,
&analysis(&[("two", "narration")], &["beta", "gamma"], "second", false, &[]),
"m2",
)
.await
.unwrap();
// Exactly one analysis row, reflecting the second pass.
let total: i64 = sqlx::query_scalar("SELECT count(*) FROM page_analysis WHERE page_id = $1")
.bind(page_id)
.fetch_one(&pool)
.await
.unwrap();
assert_eq!(total, 1);
let row = page_analysis::load(&pool, page_id).await.unwrap().unwrap();
assert!(!row.is_nsfw);
assert_eq!(row.scene_description.as_deref(), Some("second"));
assert_eq!(row.model.as_deref(), Some("m2"));
// Auto-tags are {beta, gamma}, NOT the union with the first pass.
let tag_names: Vec<String> = sqlx::query_scalar(
"SELECT t.name FROM page_auto_tags pat JOIN tags t ON t.id = pat.tag_id \
WHERE pat.page_id = $1 ORDER BY t.name",
)
.bind(page_id)
.fetch_all(&pool)
.await
.unwrap();
assert_eq!(tag_names, vec!["beta", "gamma"]);
// Warnings cleared (second pass had none).
assert_eq!(count(&pool, "page_content_warnings", page_id).await, 0);
assert_eq!(count(&pool, "page_ocr_text", page_id).await, 1);
}
#[sqlx::test(migrations = "./migrations")]
async fn persist_leaves_user_page_tags_untouched(pool: PgPool) {
let page_id = seed_page(&pool).await;
let user_id: Uuid = sqlx::query_scalar(
"INSERT INTO users (username, password_hash) VALUES ('alice', 'x') RETURNING id",
)
.fetch_one(&pool)
.await
.unwrap();
// A personal page tag the user added.
mangalord::repo::page_tag::upsert(&pool, user_id, page_id, "favourite")
.await
.unwrap();
// The auto-tagger proposes its own (overlapping + new) tags.
page_analysis::persist_analysis(
&pool,
page_id,
&analysis(&[], &["favourite", "robot"], "", false, &[]),
"m",
)
.await
.unwrap();
// The user's personal tag survives untouched.
let mine = mangalord::repo::page_tag::list_for_page(&pool, user_id, page_id)
.await
.unwrap();
assert_eq!(mine, vec!["favourite"]);
// And the global auto-tags are separate rows.
assert_eq!(count(&pool, "page_auto_tags", page_id).await, 2);
}
#[sqlx::test(migrations = "./migrations")]
async fn enqueue_for_page_inserts_analyze_page_job(pool: PgPool) {
let page_id = seed_page(&pool).await;
page_analysis::enqueue_for_page(&pool, page_id, false)
.await
.unwrap();
let payload: serde_json::Value = sqlx::query_scalar(
"SELECT payload FROM crawler_jobs WHERE payload->>'kind' = 'analyze_page'",
)
.fetch_one(&pool)
.await
.unwrap();
assert_eq!(payload["page_id"].as_str().unwrap(), page_id.to_string());
assert_eq!(payload["force"].as_bool(), Some(false));
}
#[sqlx::test(migrations = "./migrations")]
async fn mark_failed_writes_a_failed_row(pool: PgPool) {
let page_id = seed_page(&pool).await;
page_analysis::mark_failed(&pool, page_id, "model timed out")
.await
.unwrap();
let row = page_analysis::load(&pool, page_id).await.unwrap().unwrap();
assert_eq!(row.status, AnalysisStatus::Failed);
assert_eq!(row.error.as_deref(), Some("model timed out"));
}
#[sqlx::test(migrations = "./migrations")]
async fn search_doc_ranks_speech_above_sfx(pool: PgPool) {
let page_id = seed_page(&pool).await;
// "alpha" is speech (weight A), "beta" is sfx (weight D).
page_analysis::persist_analysis(
&pool,
page_id,
&analysis(&[("alpha", "speech"), ("beta", "sfx")], &[], "", false, &[]),
"m",
)
.await
.unwrap();
let (ra, rb): (f32, f32) = sqlx::query_as(
"SELECT ts_rank(search_doc, plainto_tsquery('simple', 'alpha')), \
ts_rank(search_doc, plainto_tsquery('simple', 'beta')) \
FROM page_analysis WHERE page_id = $1",
)
.bind(page_id)
.fetch_one(&pool)
.await
.unwrap();
assert!(ra > 0.0 && rb > 0.0, "both terms should match the doc");
assert!(
ra > rb,
"speech (weight A) must rank higher than sfx (weight D): {ra} vs {rb}"
);
}