feat(analysis): enqueue page-analysis on upload/crawl + admin backfill endpoints
Wires the analyze_page job kind into the page-create paths and adds admin controls, all gated on a new ANALYSIS_ENABLED flag (AppState.analysis_enabled, config::AnalysisConfig): - Chapter upload (api::chapters) enqueues one analyze_page job per page after the tx commits (best-effort; failures logged, never fatal). - Crawler content sync (persist_pages → RETURNING id) enqueues per persisted page when the daemon dispatcher's analysis_enabled flag is set; threaded through sync_chapter_content (CLI/resync pass false). - repo::page_analysis::enqueue_all_pages: bulk backfill via INSERT..SELECT, skipping done pages (only_unanalyzed) and existing pending jobs. - New admin endpoints (RequireAdmin, 503 when disabled, audited): POST /admin/analysis/reenqueue and POST /admin/pages/:id/analyze (force). Tests: upload enqueues N jobs / no-op when disabled; crawler persist_pages enqueue gate; admin reenqueue (backfill, idempotent, only-unanalyzed, 503, non-admin 403) and force-analyze (force flag, 404). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
MechaCat02
269
backend/tests/api_admin_analysis.rs
Normal file
269
backend/tests/api_admin_analysis.rs
Normal file
@@ -0,0 +1,269 @@
|
||||
//! Integration tests for the AI-analysis enqueue wiring: chapter upload
|
||||
//! enqueues `analyze_page` jobs, and the admin backfill / force-reanalyze
|
||||
//! endpoints. All gated on `analysis_enabled` (the `harness_with_analysis`
|
||||
//! variant turns it on; the default harness leaves it off → 503).
|
||||
|
||||
mod common;
|
||||
|
||||
use axum::http::StatusCode;
|
||||
use axum::Router;
|
||||
use serde_json::json;
|
||||
use sqlx::PgPool;
|
||||
use tower::ServiceExt;
|
||||
|
||||
use mangalord::repo;
|
||||
|
||||
async fn seed_admin(pool: &PgPool, app: &Router) -> String {
|
||||
let (username, cookie) = common::register_user(app).await;
|
||||
let u = repo::user::find_by_username(pool, &username)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
repo::user::set_is_admin_unchecked(pool, u.id, true)
|
||||
.await
|
||||
.unwrap();
|
||||
cookie
|
||||
}
|
||||
|
||||
/// Seed a manga → chapter with `n` pages directly in the DB, returning the
|
||||
/// page ids. Used by the backfill tests (no upload round-trip needed).
|
||||
async fn seed_pages(pool: &PgPool, n: i32) -> Vec<uuid::Uuid> {
|
||||
let manga_id: uuid::Uuid =
|
||||
sqlx::query_scalar("INSERT INTO mangas (title) VALUES ('M') RETURNING id")
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let chapter_id: uuid::Uuid = sqlx::query_scalar(
|
||||
"INSERT INTO chapters (manga_id, number) VALUES ($1, 1) RETURNING id",
|
||||
)
|
||||
.bind(manga_id)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut ids = Vec::new();
|
||||
for i in 1..=n {
|
||||
let id: uuid::Uuid = sqlx::query_scalar(
|
||||
"INSERT INTO pages (chapter_id, page_number, storage_key, content_type) \
|
||||
VALUES ($1, $2, $3, 'image/png') RETURNING id",
|
||||
)
|
||||
.bind(chapter_id)
|
||||
.bind(i)
|
||||
.bind(format!("k/{i}.png"))
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
ids.push(id);
|
||||
}
|
||||
ids
|
||||
}
|
||||
|
||||
async fn analyze_job_count(pool: &PgPool) -> i64 {
|
||||
sqlx::query_scalar(
|
||||
"SELECT COUNT(*) FROM crawler_jobs WHERE payload->>'kind' = 'analyze_page'",
|
||||
)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn chapter_upload_enqueues_one_analysis_job_per_page(pool: PgPool) {
|
||||
let h = common::harness_with_analysis(pool.clone());
|
||||
let (_user, cookie) = common::register_user(&h.app).await;
|
||||
let manga_id = common::seed_manga_via_api(&h.app, &cookie, "Uploaded").await;
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::post_multipart_with_cookie(
|
||||
&format!("/api/v1/mangas/{manga_id}/chapters"),
|
||||
common::MultipartBuilder::new()
|
||||
.add_json("metadata", json!({ "number": 1, "title": "Ch 1" }))
|
||||
.add_file("page", "p1.png", "image/png", &common::fake_png_bytes())
|
||||
.add_file("page", "p2.png", "image/png", &common::fake_png_bytes()),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::CREATED);
|
||||
|
||||
assert_eq!(analyze_job_count(&pool).await, 2);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn chapter_upload_does_not_enqueue_when_analysis_disabled(pool: PgPool) {
|
||||
// Default harness has analysis_enabled = false.
|
||||
let h = common::harness(pool.clone());
|
||||
let (_user, cookie) = common::register_user(&h.app).await;
|
||||
let manga_id = common::seed_manga_via_api(&h.app, &cookie, "Uploaded").await;
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::post_multipart_with_cookie(
|
||||
&format!("/api/v1/mangas/{manga_id}/chapters"),
|
||||
common::MultipartBuilder::new()
|
||||
.add_json("metadata", json!({ "number": 1 }))
|
||||
.add_file("page", "p1.png", "image/png", &common::fake_png_bytes()),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::CREATED);
|
||||
assert_eq!(analyze_job_count(&pool).await, 0);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn reenqueue_is_forbidden_for_non_admin(pool: PgPool) {
|
||||
let h = common::harness_with_analysis(pool.clone());
|
||||
let (_user, cookie) = common::register_user(&h.app).await;
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::post_json_with_cookie(
|
||||
"/api/v1/admin/analysis/reenqueue",
|
||||
json!({}),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn reenqueue_returns_503_when_analysis_disabled(pool: PgPool) {
|
||||
let h = common::harness(pool.clone()); // analysis disabled
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::post_json_with_cookie(
|
||||
"/api/v1/admin/analysis/reenqueue",
|
||||
json!({}),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::SERVICE_UNAVAILABLE);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn reenqueue_backfills_existing_pages(pool: PgPool) {
|
||||
let h = common::harness_with_analysis(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
seed_pages(&pool, 3).await;
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::post_json_with_cookie(
|
||||
"/api/v1/admin/analysis/reenqueue",
|
||||
json!({ "only_unanalyzed": true }),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = common::body_json(resp).await;
|
||||
assert_eq!(body["enqueued"], 3);
|
||||
assert_eq!(analyze_job_count(&pool).await, 3);
|
||||
|
||||
// Idempotent: a second call enqueues nothing (pending jobs already exist).
|
||||
let resp2 = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::post_json_with_cookie(
|
||||
"/api/v1/admin/analysis/reenqueue",
|
||||
json!({ "only_unanalyzed": true }),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body2 = common::body_json(resp2).await;
|
||||
assert_eq!(body2["enqueued"], 0);
|
||||
assert_eq!(analyze_job_count(&pool).await, 3);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn reenqueue_only_unanalyzed_skips_done_pages(pool: PgPool) {
|
||||
use mangalord::domain::page_analysis::{SafetyFlag, VisionAnalysis};
|
||||
|
||||
let h = common::harness_with_analysis(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
let ids = seed_pages(&pool, 2).await;
|
||||
|
||||
// Mark the first page as already analyzed.
|
||||
repo::page_analysis::persist_analysis(
|
||||
&pool,
|
||||
ids[0],
|
||||
&VisionAnalysis {
|
||||
ocr_results: vec![],
|
||||
tagging_results: vec![],
|
||||
scene_description: String::new(),
|
||||
safety_flag: SafetyFlag::default(),
|
||||
},
|
||||
"m",
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::post_json_with_cookie(
|
||||
"/api/v1/admin/analysis/reenqueue",
|
||||
json!({ "only_unanalyzed": true }),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = common::body_json(resp).await;
|
||||
assert_eq!(body["enqueued"], 1, "only the un-analyzed page is enqueued");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn force_analyze_page_enqueues_with_force_flag(pool: PgPool) {
|
||||
let h = common::harness_with_analysis(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
let ids = seed_pages(&pool, 1).await;
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::post_json_with_cookie(
|
||||
&format!("/api/v1/admin/pages/{}/analyze", ids[0]),
|
||||
json!({}),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
|
||||
let payload: serde_json::Value = sqlx::query_scalar(
|
||||
"SELECT payload FROM crawler_jobs WHERE payload->>'kind' = 'analyze_page'",
|
||||
)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(payload["force"].as_bool(), Some(true));
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn force_analyze_unknown_page_is_404(pool: PgPool) {
|
||||
let h = common::harness_with_analysis(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::post_json_with_cookie(
|
||||
&format!("/api/v1/admin/pages/{}/analyze", uuid::Uuid::new_v4()),
|
||||
json!({}),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::NOT_FOUND);
|
||||
}
|
||||
Reference in New Issue
Block a user