feat(analysis): scoped admin re-enqueue (all/manga/chapter) + force-on-include
Generalizes the admin re-enqueue beyond global backfill:
- repo::page_analysis::enqueue_pages(scope, only_unanalyzed) with
ReenqueueScope::{All,Manga,Chapter}. Fixes include-analyzed semantics:
only_unanalyzed=false now enqueues jobs with force=true so the worker
actually re-analyzes already-done pages instead of skipping them.
- POST /v1/admin/analysis/reenqueue accepts optional manga_id / chapter_id
(mutually exclusive, 404 on unknown target) alongside only_unanalyzed.
Tests: manga/chapter scope counts, include-analyzed sets force=true on the
done page, mutual-exclusivity 422, unknown-target 404. Existing global
backfill tests still green (13 total).
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
2
backend/Cargo.lock
generated
2
backend/Cargo.lock
generated
@@ -1517,7 +1517,7 @@ checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
|
||||
|
||||
[[package]]
|
||||
name = "mangalord"
|
||||
version = "0.71.0"
|
||||
version = "0.72.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"argon2",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "mangalord"
|
||||
version = "0.71.0"
|
||||
version = "0.72.0"
|
||||
edition = "2021"
|
||||
default-run = "mangalord"
|
||||
|
||||
|
||||
@@ -27,12 +27,26 @@ pub fn routes() -> Router<AppState> {
|
||||
.route("/admin/pages/:id/analyze", post(analyze_page))
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Deserialize)]
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct ReenqueueBody {
|
||||
/// Skip pages that already have a `done` analysis row. Defaults to
|
||||
/// true so a routine backfill only touches the gaps.
|
||||
/// true so a routine backfill only touches the gaps; `false` forces a
|
||||
/// full re-analysis of in-scope pages.
|
||||
#[serde(default = "default_true")]
|
||||
pub only_unanalyzed: bool,
|
||||
/// Limit the re-enqueue to one manga (all its chapters' pages).
|
||||
#[serde(default)]
|
||||
pub manga_id: Option<Uuid>,
|
||||
/// Limit the re-enqueue to one chapter's pages. Mutually exclusive
|
||||
/// with `manga_id`.
|
||||
#[serde(default)]
|
||||
pub chapter_id: Option<Uuid>,
|
||||
}
|
||||
|
||||
impl Default for ReenqueueBody {
|
||||
fn default() -> Self {
|
||||
Self { only_unanalyzed: true, manga_id: None, chapter_id: None }
|
||||
}
|
||||
}
|
||||
|
||||
fn default_true() -> bool {
|
||||
@@ -67,17 +81,58 @@ async fn reenqueue(
|
||||
body: Option<Json<ReenqueueBody>>,
|
||||
) -> AppResult<Json<ReenqueueResponse>> {
|
||||
ensure_enabled(&state)?;
|
||||
let only_unanalyzed = body.map(|b| b.0.only_unanalyzed).unwrap_or(true);
|
||||
let body = body.map(|b| b.0).unwrap_or_default();
|
||||
|
||||
let enqueued = repo::page_analysis::enqueue_all_pages(&state.db, only_unanalyzed).await?;
|
||||
// Resolve the scope. manga_id and chapter_id are mutually exclusive;
|
||||
// an unknown target is a 404 rather than a silent zero-enqueue.
|
||||
if body.manga_id.is_some() && body.chapter_id.is_some() {
|
||||
return Err(AppError::ValidationFailed {
|
||||
message: "manga_id and chapter_id are mutually exclusive".into(),
|
||||
details: json!({ "scope": "pick at most one" }),
|
||||
});
|
||||
}
|
||||
let (scope, target_type, target_id) = if let Some(chapter_id) = body.chapter_id {
|
||||
let exists: bool =
|
||||
sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM chapters WHERE id = $1)")
|
||||
.bind(chapter_id)
|
||||
.fetch_one(&state.db)
|
||||
.await?;
|
||||
if !exists {
|
||||
return Err(AppError::NotFound);
|
||||
}
|
||||
(
|
||||
repo::page_analysis::ReenqueueScope::Chapter(chapter_id),
|
||||
"chapter",
|
||||
Some(chapter_id),
|
||||
)
|
||||
} else if let Some(manga_id) = body.manga_id {
|
||||
if !repo::manga::exists(&state.db, manga_id).await? {
|
||||
return Err(AppError::NotFound);
|
||||
}
|
||||
(
|
||||
repo::page_analysis::ReenqueueScope::Manga(manga_id),
|
||||
"manga",
|
||||
Some(manga_id),
|
||||
)
|
||||
} else {
|
||||
(repo::page_analysis::ReenqueueScope::All, "analysis", None)
|
||||
};
|
||||
|
||||
let enqueued =
|
||||
repo::page_analysis::enqueue_pages(&state.db, scope, body.only_unanalyzed).await?;
|
||||
|
||||
repo::admin_audit::insert(
|
||||
&state.db,
|
||||
admin.0.id,
|
||||
"analysis_reenqueue",
|
||||
"analysis",
|
||||
None,
|
||||
json!({ "only_unanalyzed": only_unanalyzed, "enqueued": enqueued }),
|
||||
target_type,
|
||||
target_id,
|
||||
json!({
|
||||
"only_unanalyzed": body.only_unanalyzed,
|
||||
"manga_id": body.manga_id,
|
||||
"chapter_id": body.chapter_id,
|
||||
"enqueued": enqueued,
|
||||
}),
|
||||
)
|
||||
.await?;
|
||||
|
||||
|
||||
@@ -49,32 +49,62 @@ pub async fn enqueue_for_page(pool: &PgPool, page_id: Uuid, force: bool) -> AppR
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Bulk-enqueue `analyze_page` jobs for existing pages — the admin
|
||||
/// backfill path for content uploaded/crawled before analysis was
|
||||
/// enabled. When `only_unanalyzed` is true, pages that already have a
|
||||
/// `done` analysis row are skipped. Pages with a pending/running
|
||||
/// How wide a bulk re-enqueue reaches.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub enum ReenqueueScope {
|
||||
/// Every page in the library.
|
||||
All,
|
||||
/// Every page across all chapters of one manga.
|
||||
Manga(uuid::Uuid),
|
||||
/// Every page of one chapter.
|
||||
Chapter(uuid::Uuid),
|
||||
}
|
||||
|
||||
/// Bulk-enqueue `analyze_page` jobs for existing pages within `scope` — the
|
||||
/// admin backfill / re-analyze path.
|
||||
///
|
||||
/// When `only_unanalyzed` is true, pages that already have a `done`
|
||||
/// analysis row are skipped and the enqueued jobs carry `force = false`.
|
||||
/// When false, ALL in-scope pages are enqueued with `force = true` so the
|
||||
/// worker re-analyzes even already-done pages (otherwise the worker's
|
||||
/// skip-if-done net would no-op them). Pages with a pending/running
|
||||
/// `analyze_page` job are always skipped so repeated calls don't pile up
|
||||
/// duplicates. Returns the number of jobs enqueued.
|
||||
pub async fn enqueue_all_pages(pool: &PgPool, only_unanalyzed: bool) -> AppResult<u64> {
|
||||
let result = sqlx::query(
|
||||
pub async fn enqueue_pages(
|
||||
pool: &PgPool,
|
||||
scope: ReenqueueScope,
|
||||
only_unanalyzed: bool,
|
||||
) -> AppResult<u64> {
|
||||
// Scope predicate; the bound uuid (when present) is always $2.
|
||||
let scope_clause = match scope {
|
||||
ReenqueueScope::All => "",
|
||||
ReenqueueScope::Manga(_) => {
|
||||
"AND p.chapter_id IN (SELECT id FROM chapters WHERE manga_id = $2)"
|
||||
}
|
||||
ReenqueueScope::Chapter(_) => "AND p.chapter_id = $2",
|
||||
};
|
||||
let sql = format!(
|
||||
r#"
|
||||
INSERT INTO crawler_jobs (payload)
|
||||
SELECT jsonb_build_object('kind', 'analyze_page', 'page_id', p.id, 'force', false)
|
||||
SELECT jsonb_build_object('kind', 'analyze_page', 'page_id', p.id, 'force', NOT $1)
|
||||
FROM pages p
|
||||
WHERE ($1 = false OR NOT EXISTS (
|
||||
SELECT 1 FROM page_analysis pa
|
||||
WHERE pa.page_id = p.id AND pa.status = 'done'))
|
||||
{scope_clause}
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM crawler_jobs j
|
||||
WHERE j.payload->>'kind' = 'analyze_page'
|
||||
AND j.payload->>'page_id' = p.id::text
|
||||
AND j.state IN ('pending', 'running'))
|
||||
"#,
|
||||
)
|
||||
.bind(only_unanalyzed)
|
||||
.execute(pool)
|
||||
.await?;
|
||||
Ok(result.rows_affected())
|
||||
"#
|
||||
);
|
||||
let query = sqlx::query(&sql).bind(only_unanalyzed);
|
||||
let query = match scope {
|
||||
ReenqueueScope::All => query,
|
||||
ReenqueueScope::Manga(id) | ReenqueueScope::Chapter(id) => query.bind(id),
|
||||
};
|
||||
Ok(query.execute(pool).await?.rows_affected())
|
||||
}
|
||||
|
||||
/// Deduplicated union of content warnings across all of a manga's pages,
|
||||
|
||||
@@ -25,9 +25,12 @@ async fn seed_admin(pool: &PgPool, app: &Router) -> String {
|
||||
cookie
|
||||
}
|
||||
|
||||
/// Seed a manga → chapter with `n` pages directly in the DB, returning the
|
||||
/// page ids. Used by the backfill tests (no upload round-trip needed).
|
||||
async fn seed_pages(pool: &PgPool, n: i32) -> Vec<uuid::Uuid> {
|
||||
/// Seed a manga → chapter with `n` pages, returning (manga_id, chapter_id,
|
||||
/// page_ids).
|
||||
async fn seed_manga_chapter_pages(
|
||||
pool: &PgPool,
|
||||
n: i32,
|
||||
) -> (uuid::Uuid, uuid::Uuid, Vec<uuid::Uuid>) {
|
||||
let manga_id: uuid::Uuid =
|
||||
sqlx::query_scalar("INSERT INTO mangas (title) VALUES ('M') RETURNING id")
|
||||
.fetch_one(pool)
|
||||
@@ -48,13 +51,18 @@ async fn seed_pages(pool: &PgPool, n: i32) -> Vec<uuid::Uuid> {
|
||||
)
|
||||
.bind(chapter_id)
|
||||
.bind(i)
|
||||
.bind(format!("k/{i}.png"))
|
||||
.bind(format!("k/{manga_id}/{i}.png"))
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
ids.push(id);
|
||||
}
|
||||
ids
|
||||
(manga_id, chapter_id, ids)
|
||||
}
|
||||
|
||||
/// Convenience wrapper for tests that only need the page ids.
|
||||
async fn seed_pages(pool: &PgPool, n: i32) -> Vec<uuid::Uuid> {
|
||||
seed_manga_chapter_pages(pool, n).await.2
|
||||
}
|
||||
|
||||
async fn analyze_job_count(pool: &PgPool) -> i64 {
|
||||
@@ -186,6 +194,140 @@ async fn reenqueue_backfills_existing_pages(pool: PgPool) {
|
||||
assert_eq!(analyze_job_count(&pool).await, 3);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn reenqueue_scoped_to_manga(pool: PgPool) {
|
||||
let h = common::harness_with_analysis(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
let (manga_a, _ch_a, _pa) = seed_manga_chapter_pages(&pool, 2).await;
|
||||
seed_manga_chapter_pages(&pool, 3).await; // a different manga
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::post_json_with_cookie(
|
||||
"/api/v1/admin/analysis/reenqueue",
|
||||
json!({ "manga_id": manga_a }),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = common::body_json(resp).await;
|
||||
assert_eq!(body["enqueued"], 2, "only manga_a's pages enqueued");
|
||||
assert_eq!(analyze_job_count(&pool).await, 2);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn reenqueue_scoped_to_chapter(pool: PgPool) {
|
||||
let h = common::harness_with_analysis(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
let (_m, chapter_id, _ids) = seed_manga_chapter_pages(&pool, 4).await;
|
||||
seed_manga_chapter_pages(&pool, 5).await; // unrelated chapter
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::post_json_with_cookie(
|
||||
"/api/v1/admin/analysis/reenqueue",
|
||||
json!({ "chapter_id": chapter_id }),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = common::body_json(resp).await;
|
||||
assert_eq!(body["enqueued"], 4);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn reenqueue_including_analyzed_sets_force_and_covers_done_pages(pool: PgPool) {
|
||||
use mangalord::domain::page_analysis::{SafetyFlag, VisionAnalysis};
|
||||
|
||||
let h = common::harness_with_analysis(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
let ids = seed_pages(&pool, 2).await;
|
||||
// First page already analyzed.
|
||||
repo::page_analysis::persist_analysis(
|
||||
&pool,
|
||||
ids[0],
|
||||
&VisionAnalysis {
|
||||
ocr_results: vec![],
|
||||
tagging_results: vec![],
|
||||
scene_description: String::new(),
|
||||
safety_flag: SafetyFlag::default(),
|
||||
},
|
||||
"m",
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// only_unanalyzed=false → include the done page, with force=true.
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::post_json_with_cookie(
|
||||
"/api/v1/admin/analysis/reenqueue",
|
||||
json!({ "only_unanalyzed": false }),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = common::body_json(resp).await;
|
||||
assert_eq!(body["enqueued"], 2, "both pages enqueued (incl. the done one)");
|
||||
|
||||
// The job for the already-done page must carry force=true so the worker
|
||||
// re-analyzes instead of skipping it.
|
||||
let force: bool = sqlx::query_scalar(
|
||||
"SELECT (payload->>'force')::boolean FROM crawler_jobs \
|
||||
WHERE payload->>'page_id' = $1",
|
||||
)
|
||||
.bind(ids[0].to_string())
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(force, "include-analyzed jobs must force re-analysis");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn reenqueue_manga_and_chapter_are_mutually_exclusive(pool: PgPool) {
|
||||
let h = common::harness_with_analysis(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::post_json_with_cookie(
|
||||
"/api/v1/admin/analysis/reenqueue",
|
||||
json!({ "manga_id": uuid::Uuid::new_v4(), "chapter_id": uuid::Uuid::new_v4() }),
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::UNPROCESSABLE_ENTITY);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn reenqueue_unknown_scope_target_is_404(pool: PgPool) {
|
||||
let h = common::harness_with_analysis(pool.clone());
|
||||
let cookie = seed_admin(&pool, &h.app).await;
|
||||
|
||||
for body in [
|
||||
json!({ "manga_id": uuid::Uuid::new_v4() }),
|
||||
json!({ "chapter_id": uuid::Uuid::new_v4() }),
|
||||
] {
|
||||
let resp = h
|
||||
.app
|
||||
.clone()
|
||||
.oneshot(common::post_json_with_cookie(
|
||||
"/api/v1/admin/analysis/reenqueue",
|
||||
body,
|
||||
&cookie,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::NOT_FOUND);
|
||||
}
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn reenqueue_only_unanalyzed_skips_done_pages(pool: PgPool) {
|
||||
use mangalord::domain::page_analysis::{SafetyFlag, VisionAnalysis};
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "mangalord-frontend",
|
||||
"version": "0.71.0",
|
||||
"version": "0.72.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
|
||||
Reference in New Issue
Block a user