feat(analysis): page-analysis schema, domain types, and analyze_page job kind
Adds the persistence foundation for the AI content-analysis worker:
- Migration 0025: page_analysis (status + scene + is_nsfw + weighted
search_doc tsvector), page_ocr_text (kind-tagged pieces), global
page_auto_tags (shared tags vocabulary, separate from per-user
page_tags), and page_content_warnings (closed vocabulary).
- domain::page_analysis: AnalysisStatus/OcrKind/ContentWarning enums with
kind->tsvector-weight mapping and lenient model-string parsing, plus the
VisionAnalysis response DTOs.
- JobPayload::AnalyzePage { page_id, force } + KIND_ANALYZE_PAGE, reusing
the existing crawler_jobs queue.
- repo::page_analysis: enqueue_for_page, load, mark_failed, and the single
transactional persist_analysis (delete+reinsert; idempotent; never
touches user page_tags; computes the A/B/C/D-weighted search_doc).
Tests: repo integration (persist/idempotency/user-tags-untouched/enqueue/
mark_failed/speech>sfx ranking) + unit (weights, parsing, DTO, job serde).
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
252
backend/tests/api_page_analysis.rs
Normal file
252
backend/tests/api_page_analysis.rs
Normal file
@@ -0,0 +1,252 @@
|
||||
//! Integration tests for `repo::page_analysis` — the transactional writer
|
||||
//! that persists AI page-analysis output (OCR text, global auto-tags,
|
||||
//! content warnings, the weighted search tsvector) and the `analyze_page`
|
||||
//! job enqueue. Each `#[sqlx::test]` gets a fresh migrated DB.
|
||||
|
||||
use mangalord::domain::page_analysis::{
|
||||
AnalysisStatus, OcrResult, SafetyFlag, VisionAnalysis,
|
||||
};
|
||||
use mangalord::repo::page_analysis;
|
||||
use sqlx::PgPool;
|
||||
use uuid::Uuid;
|
||||
|
||||
/// Seed a manga → chapter → page chain and return the page id.
|
||||
async fn seed_page(pool: &PgPool) -> Uuid {
|
||||
let manga_id: Uuid =
|
||||
sqlx::query_scalar("INSERT INTO mangas (title) VALUES ('M') RETURNING id")
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let chapter_id: Uuid = sqlx::query_scalar(
|
||||
"INSERT INTO chapters (manga_id, number) VALUES ($1, 1) RETURNING id",
|
||||
)
|
||||
.bind(manga_id)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query_scalar(
|
||||
"INSERT INTO pages (chapter_id, page_number, storage_key, content_type) \
|
||||
VALUES ($1, 1, 'mangas/x/p1.png', 'image/png') RETURNING id",
|
||||
)
|
||||
.bind(chapter_id)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn analysis(
|
||||
ocr: &[(&str, &str)],
|
||||
tags: &[&str],
|
||||
scene: &str,
|
||||
nsfw: bool,
|
||||
warnings: &[&str],
|
||||
) -> VisionAnalysis {
|
||||
VisionAnalysis {
|
||||
ocr_results: ocr
|
||||
.iter()
|
||||
.map(|(text, kind)| OcrResult {
|
||||
text: (*text).to_string(),
|
||||
kind: (*kind).to_string(),
|
||||
})
|
||||
.collect(),
|
||||
tagging_results: tags.iter().map(|t| t.to_string()).collect(),
|
||||
scene_description: scene.to_string(),
|
||||
safety_flag: SafetyFlag {
|
||||
is_nsfw: nsfw,
|
||||
content_type: warnings.iter().map(|w| w.to_string()).collect(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
async fn count(pool: &PgPool, table: &str, page_id: Uuid) -> i64 {
|
||||
sqlx::query_scalar(&format!("SELECT count(*) FROM {table} WHERE page_id = $1"))
|
||||
.bind(page_id)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn persist_writes_ocr_tags_warnings_and_search_doc(pool: PgPool) {
|
||||
let page_id = seed_page(&pool).await;
|
||||
let a = analysis(
|
||||
&[("Hello", "speech"), ("BOOM", "sfx")],
|
||||
&["action", "city"],
|
||||
"A rainy street at night.",
|
||||
true,
|
||||
&["violence"],
|
||||
);
|
||||
|
||||
page_analysis::persist_analysis(&pool, page_id, &a, "test-model")
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(count(&pool, "page_ocr_text", page_id).await, 2);
|
||||
assert_eq!(count(&pool, "page_auto_tags", page_id).await, 2);
|
||||
assert_eq!(count(&pool, "page_content_warnings", page_id).await, 1);
|
||||
|
||||
let row = page_analysis::load(&pool, page_id).await.unwrap().unwrap();
|
||||
assert_eq!(row.status, AnalysisStatus::Done);
|
||||
assert!(row.is_nsfw);
|
||||
assert_eq!(row.scene_description.as_deref(), Some("A rainy street at night."));
|
||||
assert_eq!(row.model.as_deref(), Some("test-model"));
|
||||
|
||||
// search_doc must be populated (non-null, non-empty).
|
||||
let has_doc: bool = sqlx::query_scalar(
|
||||
"SELECT search_doc IS NOT NULL AND search_doc != ''::tsvector \
|
||||
FROM page_analysis WHERE page_id = $1",
|
||||
)
|
||||
.bind(page_id)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(has_doc, "search_doc should be a populated tsvector");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn persist_is_idempotent_and_replaces_prior_result(pool: PgPool) {
|
||||
let page_id = seed_page(&pool).await;
|
||||
|
||||
page_analysis::persist_analysis(
|
||||
&pool,
|
||||
page_id,
|
||||
&analysis(&[("one", "speech")], &["alpha", "beta"], "first", true, &["gore"]),
|
||||
"m1",
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Re-analysis with different output fully replaces the prior rows.
|
||||
page_analysis::persist_analysis(
|
||||
&pool,
|
||||
page_id,
|
||||
&analysis(&[("two", "narration")], &["beta", "gamma"], "second", false, &[]),
|
||||
"m2",
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Exactly one analysis row, reflecting the second pass.
|
||||
let total: i64 = sqlx::query_scalar("SELECT count(*) FROM page_analysis WHERE page_id = $1")
|
||||
.bind(page_id)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(total, 1);
|
||||
|
||||
let row = page_analysis::load(&pool, page_id).await.unwrap().unwrap();
|
||||
assert!(!row.is_nsfw);
|
||||
assert_eq!(row.scene_description.as_deref(), Some("second"));
|
||||
assert_eq!(row.model.as_deref(), Some("m2"));
|
||||
|
||||
// Auto-tags are {beta, gamma}, NOT the union with the first pass.
|
||||
let tag_names: Vec<String> = sqlx::query_scalar(
|
||||
"SELECT t.name FROM page_auto_tags pat JOIN tags t ON t.id = pat.tag_id \
|
||||
WHERE pat.page_id = $1 ORDER BY t.name",
|
||||
)
|
||||
.bind(page_id)
|
||||
.fetch_all(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(tag_names, vec!["beta", "gamma"]);
|
||||
|
||||
// Warnings cleared (second pass had none).
|
||||
assert_eq!(count(&pool, "page_content_warnings", page_id).await, 0);
|
||||
assert_eq!(count(&pool, "page_ocr_text", page_id).await, 1);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn persist_leaves_user_page_tags_untouched(pool: PgPool) {
|
||||
let page_id = seed_page(&pool).await;
|
||||
let user_id: Uuid = sqlx::query_scalar(
|
||||
"INSERT INTO users (username, password_hash) VALUES ('alice', 'x') RETURNING id",
|
||||
)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// A personal page tag the user added.
|
||||
mangalord::repo::page_tag::upsert(&pool, user_id, page_id, "favourite")
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// The auto-tagger proposes its own (overlapping + new) tags.
|
||||
page_analysis::persist_analysis(
|
||||
&pool,
|
||||
page_id,
|
||||
&analysis(&[], &["favourite", "robot"], "", false, &[]),
|
||||
"m",
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// The user's personal tag survives untouched.
|
||||
let mine = mangalord::repo::page_tag::list_for_page(&pool, user_id, page_id)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(mine, vec!["favourite"]);
|
||||
|
||||
// And the global auto-tags are separate rows.
|
||||
assert_eq!(count(&pool, "page_auto_tags", page_id).await, 2);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn enqueue_for_page_inserts_analyze_page_job(pool: PgPool) {
|
||||
let page_id = seed_page(&pool).await;
|
||||
|
||||
page_analysis::enqueue_for_page(&pool, page_id, false)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let payload: serde_json::Value = sqlx::query_scalar(
|
||||
"SELECT payload FROM crawler_jobs WHERE payload->>'kind' = 'analyze_page'",
|
||||
)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(payload["page_id"].as_str().unwrap(), page_id.to_string());
|
||||
assert_eq!(payload["force"].as_bool(), Some(false));
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn mark_failed_writes_a_failed_row(pool: PgPool) {
|
||||
let page_id = seed_page(&pool).await;
|
||||
|
||||
page_analysis::mark_failed(&pool, page_id, "model timed out")
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let row = page_analysis::load(&pool, page_id).await.unwrap().unwrap();
|
||||
assert_eq!(row.status, AnalysisStatus::Failed);
|
||||
assert_eq!(row.error.as_deref(), Some("model timed out"));
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn search_doc_ranks_speech_above_sfx(pool: PgPool) {
|
||||
let page_id = seed_page(&pool).await;
|
||||
// "alpha" is speech (weight A), "beta" is sfx (weight D).
|
||||
page_analysis::persist_analysis(
|
||||
&pool,
|
||||
page_id,
|
||||
&analysis(&[("alpha", "speech"), ("beta", "sfx")], &[], "", false, &[]),
|
||||
"m",
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let (ra, rb): (f32, f32) = sqlx::query_as(
|
||||
"SELECT ts_rank(search_doc, plainto_tsquery('simple', 'alpha')), \
|
||||
ts_rank(search_doc, plainto_tsquery('simple', 'beta')) \
|
||||
FROM page_analysis WHERE page_id = $1",
|
||||
)
|
||||
.bind(page_id)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(ra > 0.0 && rb > 0.0, "both terms should match the doc");
|
||||
assert!(
|
||||
ra > rb,
|
||||
"speech (weight A) must rank higher than sfx (weight D): {ra} vs {rb}"
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user