feat(analysis): enqueue page-analysis on upload/crawl + admin backfill endpoints
Wires the analyze_page job kind into the page-create paths and adds admin controls, all gated on a new ANALYSIS_ENABLED flag (AppState.analysis_enabled, config::AnalysisConfig): - Chapter upload (api::chapters) enqueues one analyze_page job per page after the tx commits (best-effort; failures logged, never fatal). - Crawler content sync (persist_pages → RETURNING id) enqueues per persisted page when the daemon dispatcher's analysis_enabled flag is set; threaded through sync_chapter_content (CLI/resync pass false). - repo::page_analysis::enqueue_all_pages: bulk backfill via INSERT..SELECT, skipping done pages (only_unanalyzed) and existing pending jobs. - New admin endpoints (RequireAdmin, 503 when disabled, audited): POST /admin/analysis/reenqueue and POST /admin/pages/:id/analyze (force). Tests: upload enqueues N jobs / no-op when disabled; crawler persist_pages enqueue gate; admin reenqueue (backfill, idempotent, only-unanalyzed, 503, non-admin 403) and force-analyze (force flag, 404). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -56,6 +56,11 @@ pub struct AppState {
|
||||
/// `Arc` keeps the clone cheap. Empty list → check is skipped
|
||||
/// (operator opt-out documented in `.env.example`).
|
||||
pub admin_allowed_origins: Arc<Vec<String>>,
|
||||
/// When `true`, page-create paths (upload + crawler) enqueue
|
||||
/// `analyze_page` jobs and the admin re-enqueue endpoints are active.
|
||||
/// Mirrors `config.analysis.enabled`; defaults to `false` so the
|
||||
/// feature is opt-in and the test harness inherits a no-op gate.
|
||||
pub analysis_enabled: bool,
|
||||
}
|
||||
|
||||
/// Shared handle the admin crawler endpoints use to observe and control
|
||||
@@ -113,7 +118,13 @@ pub async fn build(config: Config) -> anyhow::Result<AppHandle> {
|
||||
let storage: Arc<dyn Storage> = Arc::new(LocalStorage::new(config.storage_dir.clone()));
|
||||
|
||||
let (daemon, resync, crawler) = if config.crawler.daemon_enabled {
|
||||
let spawned = spawn_crawler_daemon(db.clone(), Arc::clone(&storage), &config.crawler).await?;
|
||||
let spawned = spawn_crawler_daemon(
|
||||
db.clone(),
|
||||
Arc::clone(&storage),
|
||||
&config.crawler,
|
||||
config.analysis.enabled,
|
||||
)
|
||||
.await?;
|
||||
(Some(spawned.handle), Some(spawned.resync), Some(spawned.crawler))
|
||||
} else {
|
||||
tracing::info!("crawler daemon disabled (CRAWLER_DAEMON=false)");
|
||||
@@ -130,6 +141,7 @@ pub async fn build(config: Config) -> anyhow::Result<AppHandle> {
|
||||
resync,
|
||||
crawler,
|
||||
admin_allowed_origins: Arc::new(config.admin_allowed_origins.clone()),
|
||||
analysis_enabled: config.analysis.enabled,
|
||||
};
|
||||
let router = router(state).layer(cors_layer(&config.cors_allowed_origins));
|
||||
Ok(AppHandle { router, daemon })
|
||||
@@ -149,6 +161,7 @@ async fn spawn_crawler_daemon(
|
||||
db: PgPool,
|
||||
storage: Arc<dyn Storage>,
|
||||
cfg: &CrawlerConfig,
|
||||
analysis_enabled: bool,
|
||||
) -> anyhow::Result<SpawnedDaemon> {
|
||||
// Reqwest client with a shared cookie jar so CDN image fetches include
|
||||
// PHPSESSID. The same `Arc<Jar>` is held by the SessionController, so a
|
||||
@@ -288,6 +301,7 @@ async fn spawn_crawler_daemon(
|
||||
rate: Arc::clone(&rate),
|
||||
download_allowlist: cfg.download_allowlist.clone(),
|
||||
max_image_bytes: cfg.max_image_bytes,
|
||||
analysis_enabled,
|
||||
transient_failures: Arc::new(AtomicU32::new(0)),
|
||||
restart_threshold: cfg.browser_restart_threshold,
|
||||
drain_deadline: cfg.job_timeout,
|
||||
@@ -446,6 +460,9 @@ struct RealChapterDispatcher {
|
||||
rate: Arc<HostRateLimiters>,
|
||||
download_allowlist: DownloadAllowlist,
|
||||
max_image_bytes: usize,
|
||||
/// Enqueue `analyze_page` jobs for freshly-crawled pages. Mirrors
|
||||
/// `config.analysis.enabled`.
|
||||
analysis_enabled: bool,
|
||||
/// Consecutive transient chapter failures; resets on any success.
|
||||
/// Drives the automatic coordinated browser restart.
|
||||
transient_failures: Arc<std::sync::atomic::AtomicU32>,
|
||||
@@ -501,6 +518,7 @@ impl ChapterDispatcher for RealChapterDispatcher {
|
||||
self.max_image_bytes,
|
||||
self.tor.as_deref(),
|
||||
Some(&self.status),
|
||||
self.analysis_enabled,
|
||||
)
|
||||
.await;
|
||||
drop(lease);
|
||||
|
||||
Reference in New Issue
Block a user