feat(analysis): enqueue page-analysis on upload/crawl + admin backfill endpoints

Wires the analyze_page job kind into the page-create paths and adds admin
controls, all gated on a new ANALYSIS_ENABLED flag (AppState.analysis_enabled,
config::AnalysisConfig):

- Chapter upload (api::chapters) enqueues one analyze_page job per page
  after the tx commits (best-effort; failures logged, never fatal).
- Crawler content sync (persist_pages → RETURNING id) enqueues per
  persisted page when the daemon dispatcher's analysis_enabled flag is set;
  threaded through sync_chapter_content (CLI/resync pass false).
- repo::page_analysis::enqueue_all_pages: bulk backfill via INSERT..SELECT,
  skipping done pages (only_unanalyzed) and existing pending jobs.
- New admin endpoints (RequireAdmin, 503 when disabled, audited):
  POST /admin/analysis/reenqueue and POST /admin/pages/:id/analyze (force).

Tests: upload enqueues N jobs / no-op when disabled; crawler persist_pages
enqueue gate; admin reenqueue (backfill, idempotent, only-unanalyzed, 503,
non-admin 403) and force-analyze (force flag, 404).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
MechaCat02
2026-06-13 18:39:03 +02:00
parent 63e1aa5484
commit 2bbf1595ff
15 changed files with 592 additions and 11 deletions

View File

@@ -215,6 +215,10 @@ pub async fn sync_chapter_content(
// dispatcher passes the shared handle (the chapter has already been
// registered via `begin_chapter`); the CLI / admin resync pass `None`.
progress: Option<&crate::crawler::status::StatusHandle>,
// When `true`, enqueue an `analyze_page` job per persisted page. The
// daemon dispatcher passes the configured flag; CLI / resync pass
// `false`.
enqueue_analysis: bool,
) -> anyhow::Result<SyncOutcome> {
// Skip if already fetched, unless caller explicitly forces.
if !force_refetch {
@@ -316,7 +320,7 @@ pub async fn sync_chapter_content(
// Short transaction: page rows + page_count only, no network I/O. On
// failure, roll back the stored bytes so the chapter stays at
// page_count=0 and is retried cleanly next run.
if let Err(e) = persist_pages(db, chapter_id, &stored).await {
if let Err(e) = persist_pages(db, chapter_id, &stored, enqueue_analysis).await {
cleanup_orphans(storage, &written_keys).await;
return Err(e);
}
@@ -450,23 +454,27 @@ pub(crate) async fn persist_pages(
db: &PgPool,
chapter_id: Uuid,
stored: &[StoredPage],
enqueue_analysis: bool,
) -> anyhow::Result<()> {
let mut tx = db.begin().await.context("open chapter sync tx")?;
let mut page_ids: Vec<Uuid> = Vec::with_capacity(stored.len());
for page in stored {
sqlx::query(
let (id,): (Uuid,) = sqlx::query_as(
"INSERT INTO pages (chapter_id, page_number, storage_key, content_type)
VALUES ($1, $2, $3, $4)
ON CONFLICT (chapter_id, page_number) DO UPDATE
SET storage_key = EXCLUDED.storage_key,
content_type = EXCLUDED.content_type",
content_type = EXCLUDED.content_type
RETURNING id",
)
.bind(chapter_id)
.bind(page.page_number)
.bind(&page.storage_key)
.bind(&page.content_type)
.execute(&mut *tx)
.fetch_one(&mut *tx)
.await
.with_context(|| format!("insert page row {}", page.page_number))?;
page_ids.push(id);
}
sqlx::query("UPDATE chapters SET page_count = $1 WHERE id = $2")
.bind(stored.len() as i32)
@@ -475,6 +483,20 @@ pub(crate) async fn persist_pages(
.await
.context("update page_count")?;
tx.commit().await.context("commit chapter sync")?;
// Enqueue AI content-analysis for the crawled pages once their rows
// are committed. Best-effort: a failed enqueue is logged, never fatal
// (the admin re-enqueue endpoint can backfill). Gated by the caller so
// jobs don't pile up when the analysis worker is disabled.
if enqueue_analysis {
for page_id in page_ids {
if let Err(e) =
crate::repo::page_analysis::enqueue_for_page(db, page_id, false).await
{
tracing::warn!(%page_id, error = %e, "failed to enqueue page analysis after crawl");
}
}
}
Ok(())
}
@@ -561,7 +583,7 @@ mod tests {
content_type: "image/jpeg".into(),
},
];
persist_pages(&pool, chapter_id, &stored).await.unwrap();
persist_pages(&pool, chapter_id, &stored, false).await.unwrap();
let page_count: i32 =
sqlx::query_scalar("SELECT page_count FROM chapters WHERE id = $1")
@@ -579,7 +601,7 @@ mod tests {
assert_eq!(rows, 2);
// Idempotent re-run (force refetch path): same rows, page_count stable.
persist_pages(&pool, chapter_id, &stored).await.unwrap();
persist_pages(&pool, chapter_id, &stored, false).await.unwrap();
let rows2: i64 =
sqlx::query_scalar("SELECT COUNT(*) FROM pages WHERE chapter_id = $1")
.bind(chapter_id)
@@ -589,6 +611,48 @@ mod tests {
assert_eq!(rows2, 2, "re-run is idempotent via ON CONFLICT");
}
#[sqlx::test(migrations = "./migrations")]
async fn persist_pages_enqueues_analysis_only_when_flag_set(pool: PgPool) {
let manga_id = Uuid::new_v4();
let chapter_id = Uuid::new_v4();
sqlx::query("INSERT INTO mangas (id, title) VALUES ($1, 'T')")
.bind(manga_id)
.execute(&pool)
.await
.unwrap();
sqlx::query("INSERT INTO chapters (id, manga_id, number) VALUES ($1, $2, 1)")
.bind(chapter_id)
.bind(manga_id)
.execute(&pool)
.await
.unwrap();
let stored = vec![StoredPage {
page_number: 1,
storage_key: "k/0001.jpg".into(),
content_type: "image/jpeg".into(),
}];
// Flag off: no analyze_page jobs.
persist_pages(&pool, chapter_id, &stored, false).await.unwrap();
let jobs_off: i64 = sqlx::query_scalar(
"SELECT COUNT(*) FROM crawler_jobs WHERE payload->>'kind' = 'analyze_page'",
)
.fetch_one(&pool)
.await
.unwrap();
assert_eq!(jobs_off, 0, "flag off must not enqueue analysis");
// Flag on: one analyze_page job for the upserted page.
persist_pages(&pool, chapter_id, &stored, true).await.unwrap();
let jobs_on: i64 = sqlx::query_scalar(
"SELECT COUNT(*) FROM crawler_jobs WHERE payload->>'kind' = 'analyze_page'",
)
.fetch_one(&pool)
.await
.unwrap();
assert_eq!(jobs_on, 1, "flag on enqueues one job per persisted page");
}
#[test]
fn parse_chapter_pages_skips_loader_and_sorts_by_id() {
// Loader image, two real pages out of order, and one with no id.

View File

@@ -259,6 +259,9 @@ impl ResyncService for RealResyncService {
self.tor.as_deref(),
// Admin resync isn't a daemon worker slot — no live status.
None,
// Resync re-fetches existing pages (same ids); analysis isn't
// re-enqueued here — use the admin force re-analyze endpoint.
false,
)
.await;
drop(lease);