feat(analysis): enqueue page-analysis on upload/crawl + admin backfill endpoints
Wires the analyze_page job kind into the page-create paths and adds admin controls, all gated on a new ANALYSIS_ENABLED flag (AppState.analysis_enabled, config::AnalysisConfig): - Chapter upload (api::chapters) enqueues one analyze_page job per page after the tx commits (best-effort; failures logged, never fatal). - Crawler content sync (persist_pages → RETURNING id) enqueues per persisted page when the daemon dispatcher's analysis_enabled flag is set; threaded through sync_chapter_content (CLI/resync pass false). - repo::page_analysis::enqueue_all_pages: bulk backfill via INSERT..SELECT, skipping done pages (only_unanalyzed) and existing pending jobs. - New admin endpoints (RequireAdmin, 503 when disabled, audited): POST /admin/analysis/reenqueue and POST /admin/pages/:id/analyze (force). Tests: upload enqueues N jobs / no-op when disabled; crawler persist_pages enqueue gate; admin reenqueue (backfill, idempotent, only-unanalyzed, 503, non-admin 403) and force-analyze (force flag, 404). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
116
backend/src/api/admin/analysis.rs
Normal file
116
backend/src/api/admin/analysis.rs
Normal file
@@ -0,0 +1,116 @@
|
||||
//! Admin controls for the AI content-analysis worker.
|
||||
//!
|
||||
//! Two endpoints, both admin-only (`RequireAdmin`, cookie-only) and gated
|
||||
//! on `AppState.analysis_enabled` (503 when the feature is off):
|
||||
//!
|
||||
//! * `POST /admin/analysis/reenqueue` — bulk backfill: enqueue
|
||||
//! `analyze_page` jobs for existing pages. Body `{ only_unanalyzed }`
|
||||
//! (default true) skips pages that already have a completed analysis.
|
||||
//! * `POST /admin/pages/:id/analyze` — force re-analysis of a single page
|
||||
//! (re-runs even if already `done`).
|
||||
|
||||
use axum::extract::{Path, State};
|
||||
use axum::routing::post;
|
||||
use axum::{Json, Router};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::json;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::app::AppState;
|
||||
use crate::auth::extractor::RequireAdmin;
|
||||
use crate::error::{AppError, AppResult};
|
||||
use crate::repo;
|
||||
|
||||
pub fn routes() -> Router<AppState> {
|
||||
Router::new()
|
||||
.route("/admin/analysis/reenqueue", post(reenqueue))
|
||||
.route("/admin/pages/:id/analyze", post(analyze_page))
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Deserialize)]
|
||||
pub struct ReenqueueBody {
|
||||
/// Skip pages that already have a `done` analysis row. Defaults to
|
||||
/// true so a routine backfill only touches the gaps.
|
||||
#[serde(default = "default_true")]
|
||||
pub only_unanalyzed: bool,
|
||||
}
|
||||
|
||||
fn default_true() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct ReenqueueResponse {
|
||||
pub enqueued: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct AnalyzePageResponse {
|
||||
pub enqueued: bool,
|
||||
}
|
||||
|
||||
/// Reject the request with 503 when the analysis worker is disabled, so
|
||||
/// an operator doesn't pile up jobs that nothing will ever drain.
|
||||
fn ensure_enabled(state: &AppState) -> AppResult<()> {
|
||||
if state.analysis_enabled {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(AppError::ServiceUnavailable(
|
||||
"content-analysis worker is disabled (ANALYSIS_ENABLED=false)".into(),
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
async fn reenqueue(
|
||||
State(state): State<AppState>,
|
||||
admin: RequireAdmin,
|
||||
body: Option<Json<ReenqueueBody>>,
|
||||
) -> AppResult<Json<ReenqueueResponse>> {
|
||||
ensure_enabled(&state)?;
|
||||
let only_unanalyzed = body.map(|b| b.0.only_unanalyzed).unwrap_or(true);
|
||||
|
||||
let enqueued = repo::page_analysis::enqueue_all_pages(&state.db, only_unanalyzed).await?;
|
||||
|
||||
repo::admin_audit::insert(
|
||||
&state.db,
|
||||
admin.0.id,
|
||||
"analysis_reenqueue",
|
||||
"analysis",
|
||||
None,
|
||||
json!({ "only_unanalyzed": only_unanalyzed, "enqueued": enqueued }),
|
||||
)
|
||||
.await?;
|
||||
|
||||
Ok(Json(ReenqueueResponse { enqueued }))
|
||||
}
|
||||
|
||||
async fn analyze_page(
|
||||
State(state): State<AppState>,
|
||||
admin: RequireAdmin,
|
||||
Path(page_id): Path<Uuid>,
|
||||
) -> AppResult<Json<AnalyzePageResponse>> {
|
||||
ensure_enabled(&state)?;
|
||||
|
||||
let exists: bool =
|
||||
sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM pages WHERE id = $1)")
|
||||
.bind(page_id)
|
||||
.fetch_one(&state.db)
|
||||
.await?;
|
||||
if !exists {
|
||||
return Err(AppError::NotFound);
|
||||
}
|
||||
|
||||
repo::page_analysis::enqueue_for_page(&state.db, page_id, true).await?;
|
||||
|
||||
repo::admin_audit::insert(
|
||||
&state.db,
|
||||
admin.0.id,
|
||||
"analysis_force_page",
|
||||
"page",
|
||||
Some(page_id),
|
||||
json!({ "force": true }),
|
||||
)
|
||||
.await?;
|
||||
|
||||
Ok(Json(AnalyzePageResponse { enqueued: true }))
|
||||
}
|
||||
@@ -4,6 +4,7 @@
|
||||
//! bot/API tokens cannot reach admin routes (see
|
||||
//! `crate::auth::extractor::RequireAdmin`).
|
||||
|
||||
pub mod analysis;
|
||||
pub mod crawler;
|
||||
pub mod mangas;
|
||||
pub mod resync;
|
||||
@@ -21,4 +22,5 @@ pub fn routes() -> Router<AppState> {
|
||||
.merge(resync::routes())
|
||||
.merge(system::routes())
|
||||
.merge(crawler::routes())
|
||||
.merge(analysis::routes())
|
||||
}
|
||||
|
||||
@@ -137,6 +137,7 @@ async fn create(
|
||||
)
|
||||
.await?;
|
||||
|
||||
let mut page_ids: Vec<Uuid> = Vec::with_capacity(pages.len());
|
||||
for (idx, page) in pages.iter().enumerate() {
|
||||
let page_number = (idx + 1) as i32;
|
||||
let nnnn = format!("{:04}", page_number);
|
||||
@@ -145,7 +146,9 @@ async fn create(
|
||||
manga_id, chapter.id, nnnn, page.ext
|
||||
);
|
||||
state.storage.put(&key, &page.bytes).await?;
|
||||
repo::page::create(&mut *tx, chapter.id, page_number, &key, page.mime).await?;
|
||||
let created =
|
||||
repo::page::create(&mut *tx, chapter.id, page_number, &key, page.mime).await?;
|
||||
page_ids.push(created.id);
|
||||
}
|
||||
|
||||
let page_count = pages.len() as i32;
|
||||
@@ -154,6 +157,20 @@ async fn create(
|
||||
|
||||
tx.commit().await?;
|
||||
|
||||
// Enqueue AI content-analysis for each new page. Done after commit so a
|
||||
// rolled-back upload never leaves jobs pointing at nonexistent pages; a
|
||||
// failed enqueue is logged but doesn't fail the upload (the admin
|
||||
// re-enqueue endpoint can backfill).
|
||||
if state.analysis_enabled {
|
||||
for page_id in page_ids {
|
||||
if let Err(e) =
|
||||
repo::page_analysis::enqueue_for_page(&state.db, page_id, false).await
|
||||
{
|
||||
tracing::warn!(%page_id, error = %e, "failed to enqueue page analysis");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok((StatusCode::CREATED, Json(chapter)))
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user