feat(analysis): enqueue page-analysis on upload/crawl + admin backfill endpoints
Wires the analyze_page job kind into the page-create paths and adds admin controls, all gated on a new ANALYSIS_ENABLED flag (AppState.analysis_enabled, config::AnalysisConfig): - Chapter upload (api::chapters) enqueues one analyze_page job per page after the tx commits (best-effort; failures logged, never fatal). - Crawler content sync (persist_pages → RETURNING id) enqueues per persisted page when the daemon dispatcher's analysis_enabled flag is set; threaded through sync_chapter_content (CLI/resync pass false). - repo::page_analysis::enqueue_all_pages: bulk backfill via INSERT..SELECT, skipping done pages (only_unanalyzed) and existing pending jobs. - New admin endpoints (RequireAdmin, 503 when disabled, audited): POST /admin/analysis/reenqueue and POST /admin/pages/:id/analyze (force). Tests: upload enqueues N jobs / no-op when disabled; crawler persist_pages enqueue gate; admin reenqueue (backfill, idempotent, only-unanalyzed, 503, non-admin 403) and force-analyze (force flag, 404). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -215,6 +215,10 @@ pub async fn sync_chapter_content(
|
||||
// dispatcher passes the shared handle (the chapter has already been
|
||||
// registered via `begin_chapter`); the CLI / admin resync pass `None`.
|
||||
progress: Option<&crate::crawler::status::StatusHandle>,
|
||||
// When `true`, enqueue an `analyze_page` job per persisted page. The
|
||||
// daemon dispatcher passes the configured flag; CLI / resync pass
|
||||
// `false`.
|
||||
enqueue_analysis: bool,
|
||||
) -> anyhow::Result<SyncOutcome> {
|
||||
// Skip if already fetched, unless caller explicitly forces.
|
||||
if !force_refetch {
|
||||
@@ -316,7 +320,7 @@ pub async fn sync_chapter_content(
|
||||
// Short transaction: page rows + page_count only, no network I/O. On
|
||||
// failure, roll back the stored bytes so the chapter stays at
|
||||
// page_count=0 and is retried cleanly next run.
|
||||
if let Err(e) = persist_pages(db, chapter_id, &stored).await {
|
||||
if let Err(e) = persist_pages(db, chapter_id, &stored, enqueue_analysis).await {
|
||||
cleanup_orphans(storage, &written_keys).await;
|
||||
return Err(e);
|
||||
}
|
||||
@@ -450,23 +454,27 @@ pub(crate) async fn persist_pages(
|
||||
db: &PgPool,
|
||||
chapter_id: Uuid,
|
||||
stored: &[StoredPage],
|
||||
enqueue_analysis: bool,
|
||||
) -> anyhow::Result<()> {
|
||||
let mut tx = db.begin().await.context("open chapter sync tx")?;
|
||||
let mut page_ids: Vec<Uuid> = Vec::with_capacity(stored.len());
|
||||
for page in stored {
|
||||
sqlx::query(
|
||||
let (id,): (Uuid,) = sqlx::query_as(
|
||||
"INSERT INTO pages (chapter_id, page_number, storage_key, content_type)
|
||||
VALUES ($1, $2, $3, $4)
|
||||
ON CONFLICT (chapter_id, page_number) DO UPDATE
|
||||
SET storage_key = EXCLUDED.storage_key,
|
||||
content_type = EXCLUDED.content_type",
|
||||
content_type = EXCLUDED.content_type
|
||||
RETURNING id",
|
||||
)
|
||||
.bind(chapter_id)
|
||||
.bind(page.page_number)
|
||||
.bind(&page.storage_key)
|
||||
.bind(&page.content_type)
|
||||
.execute(&mut *tx)
|
||||
.fetch_one(&mut *tx)
|
||||
.await
|
||||
.with_context(|| format!("insert page row {}", page.page_number))?;
|
||||
page_ids.push(id);
|
||||
}
|
||||
sqlx::query("UPDATE chapters SET page_count = $1 WHERE id = $2")
|
||||
.bind(stored.len() as i32)
|
||||
@@ -475,6 +483,20 @@ pub(crate) async fn persist_pages(
|
||||
.await
|
||||
.context("update page_count")?;
|
||||
tx.commit().await.context("commit chapter sync")?;
|
||||
|
||||
// Enqueue AI content-analysis for the crawled pages once their rows
|
||||
// are committed. Best-effort: a failed enqueue is logged, never fatal
|
||||
// (the admin re-enqueue endpoint can backfill). Gated by the caller so
|
||||
// jobs don't pile up when the analysis worker is disabled.
|
||||
if enqueue_analysis {
|
||||
for page_id in page_ids {
|
||||
if let Err(e) =
|
||||
crate::repo::page_analysis::enqueue_for_page(db, page_id, false).await
|
||||
{
|
||||
tracing::warn!(%page_id, error = %e, "failed to enqueue page analysis after crawl");
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -561,7 +583,7 @@ mod tests {
|
||||
content_type: "image/jpeg".into(),
|
||||
},
|
||||
];
|
||||
persist_pages(&pool, chapter_id, &stored).await.unwrap();
|
||||
persist_pages(&pool, chapter_id, &stored, false).await.unwrap();
|
||||
|
||||
let page_count: i32 =
|
||||
sqlx::query_scalar("SELECT page_count FROM chapters WHERE id = $1")
|
||||
@@ -579,7 +601,7 @@ mod tests {
|
||||
assert_eq!(rows, 2);
|
||||
|
||||
// Idempotent re-run (force refetch path): same rows, page_count stable.
|
||||
persist_pages(&pool, chapter_id, &stored).await.unwrap();
|
||||
persist_pages(&pool, chapter_id, &stored, false).await.unwrap();
|
||||
let rows2: i64 =
|
||||
sqlx::query_scalar("SELECT COUNT(*) FROM pages WHERE chapter_id = $1")
|
||||
.bind(chapter_id)
|
||||
@@ -589,6 +611,48 @@ mod tests {
|
||||
assert_eq!(rows2, 2, "re-run is idempotent via ON CONFLICT");
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn persist_pages_enqueues_analysis_only_when_flag_set(pool: PgPool) {
|
||||
let manga_id = Uuid::new_v4();
|
||||
let chapter_id = Uuid::new_v4();
|
||||
sqlx::query("INSERT INTO mangas (id, title) VALUES ($1, 'T')")
|
||||
.bind(manga_id)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("INSERT INTO chapters (id, manga_id, number) VALUES ($1, $2, 1)")
|
||||
.bind(chapter_id)
|
||||
.bind(manga_id)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let stored = vec![StoredPage {
|
||||
page_number: 1,
|
||||
storage_key: "k/0001.jpg".into(),
|
||||
content_type: "image/jpeg".into(),
|
||||
}];
|
||||
|
||||
// Flag off: no analyze_page jobs.
|
||||
persist_pages(&pool, chapter_id, &stored, false).await.unwrap();
|
||||
let jobs_off: i64 = sqlx::query_scalar(
|
||||
"SELECT COUNT(*) FROM crawler_jobs WHERE payload->>'kind' = 'analyze_page'",
|
||||
)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(jobs_off, 0, "flag off must not enqueue analysis");
|
||||
|
||||
// Flag on: one analyze_page job for the upserted page.
|
||||
persist_pages(&pool, chapter_id, &stored, true).await.unwrap();
|
||||
let jobs_on: i64 = sqlx::query_scalar(
|
||||
"SELECT COUNT(*) FROM crawler_jobs WHERE payload->>'kind' = 'analyze_page'",
|
||||
)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(jobs_on, 1, "flag on enqueues one job per persisted page");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_chapter_pages_skips_loader_and_sorts_by_id() {
|
||||
// Loader image, two real pages out of order, and one with no id.
|
||||
|
||||
@@ -259,6 +259,9 @@ impl ResyncService for RealResyncService {
|
||||
self.tor.as_deref(),
|
||||
// Admin resync isn't a daemon worker slot — no live status.
|
||||
None,
|
||||
// Resync re-fetches existing pages (same ids); analysis isn't
|
||||
// re-enqueued here — use the admin force re-analyze endpoint.
|
||||
false,
|
||||
)
|
||||
.await;
|
||||
drop(lease);
|
||||
|
||||
Reference in New Issue
Block a user