feat(analysis): worker daemon + real dispatcher + app wiring
The background analysis worker that drains analyze_page jobs: - analysis::daemon: a lean sibling of the crawler daemon — leases only KIND_ANALYZE_PAGE, skip-if-done-unless-force, lease heartbeat, panic + timeout isolation, ack done/failed, and a failed page_analysis row on terminal (dead-lettered) failure. AnalyzeDispatcher trait seam. - RealAnalyzeDispatcher: load page → storage.get → VisionClient.analyze → persist_analysis (skips a deleted page; caps image bytes). - app::build spawns the daemon (own plain reqwest client) when ANALYSIS_ENABLED; AppHandle/main shut it down alongside the crawler. - repo::page::find_by_id. Tests: dispatch+ack-done, skip-when-done, force re-dispatch, terminal failure writes failed row, panic isolation, ignores non-analyze jobs. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
MechaCat02
281
backend/src/analysis/daemon.rs
Normal file
281
backend/src/analysis/daemon.rs
Normal file
@@ -0,0 +1,281 @@
|
||||
//! The AI page-analysis worker daemon.
|
||||
//!
|
||||
//! A lean sibling of [`crate::crawler::daemon`]: it leases `analyze_page`
|
||||
//! jobs from the SAME `crawler_jobs` queue (filtered by kind, so it never
|
||||
//! contends with crawl jobs), dispatches each through an [`AnalyzeDispatcher`]
|
||||
//! seam, and acks done/failed. No cron, no browser, no session state — it
|
||||
//! runs whenever `ANALYSIS_ENABLED` is set, independent of the crawler.
|
||||
//!
|
||||
//! Per job: skip if already `done` (unless the payload sets `force`),
|
||||
//! heartbeat the lease while the (slow) vision call runs, isolate panics
|
||||
//! and an outer timeout, and on terminal failure write a `failed`
|
||||
//! `page_analysis` row so the page's state is observable.
|
||||
|
||||
use std::panic::AssertUnwindSafe;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use futures_util::FutureExt;
|
||||
use sqlx::PgPool;
|
||||
use tokio::task::JoinSet;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::analysis::vision::VisionClient;
|
||||
use crate::crawler::jobs::{self, JobPayload, Lease, KIND_ANALYZE_PAGE};
|
||||
use crate::repo;
|
||||
use crate::storage::Storage;
|
||||
|
||||
/// Lease window; continuously extended by the per-job heartbeat.
|
||||
const LEASE_DURATION: Duration = Duration::from_secs(60);
|
||||
/// Heartbeat cadence — a third of the lease window.
|
||||
const LEASE_HEARTBEAT: Duration = Duration::from_secs(20);
|
||||
|
||||
/// The unit of work: analyze one page. Implemented by
|
||||
/// [`RealAnalyzeDispatcher`] in production and stubbed in tests.
|
||||
#[async_trait]
|
||||
pub trait AnalyzeDispatcher: Send + Sync {
|
||||
async fn dispatch(&self, page_id: Uuid) -> anyhow::Result<()>;
|
||||
}
|
||||
|
||||
pub struct AnalysisDaemonConfig {
|
||||
pub dispatcher: Arc<dyn AnalyzeDispatcher>,
|
||||
pub workers: usize,
|
||||
pub job_timeout: Duration,
|
||||
}
|
||||
|
||||
pub struct AnalysisDaemonHandle {
|
||||
cancel: CancellationToken,
|
||||
join: JoinSet<()>,
|
||||
}
|
||||
|
||||
impl AnalysisDaemonHandle {
|
||||
pub async fn shutdown(mut self) {
|
||||
self.cancel.cancel();
|
||||
while self.join.join_next().await.is_some() {}
|
||||
}
|
||||
}
|
||||
|
||||
/// Spawn the analysis workers. Returns immediately; tasks run in the
|
||||
/// background until the handle is shut down (or `cancel` fires).
|
||||
pub fn spawn(
|
||||
pool: PgPool,
|
||||
cancel: CancellationToken,
|
||||
cfg: AnalysisDaemonConfig,
|
||||
) -> AnalysisDaemonHandle {
|
||||
let mut join = JoinSet::new();
|
||||
for id in 0..cfg.workers.max(1) {
|
||||
let ctx = WorkerContext {
|
||||
pool: pool.clone(),
|
||||
cancel: cancel.clone(),
|
||||
dispatcher: Arc::clone(&cfg.dispatcher),
|
||||
job_timeout: cfg.job_timeout,
|
||||
id,
|
||||
};
|
||||
join.spawn(async move { ctx.run().await });
|
||||
}
|
||||
AnalysisDaemonHandle { cancel, join }
|
||||
}
|
||||
|
||||
struct WorkerContext {
|
||||
pool: PgPool,
|
||||
cancel: CancellationToken,
|
||||
dispatcher: Arc<dyn AnalyzeDispatcher>,
|
||||
job_timeout: Duration,
|
||||
id: usize,
|
||||
}
|
||||
|
||||
impl WorkerContext {
|
||||
async fn run(self) {
|
||||
loop {
|
||||
if self.cancel.is_cancelled() {
|
||||
tracing::info!(worker = self.id, "analysis worker: shutdown");
|
||||
return;
|
||||
}
|
||||
let leases =
|
||||
match jobs::lease(&self.pool, Some(KIND_ANALYZE_PAGE), 1, LEASE_DURATION).await {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
tracing::warn!(worker = self.id, ?e, "analysis worker: lease failed");
|
||||
if self.sleep_or_cancel(Duration::from_secs(5)).await {
|
||||
return;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let Some(lease) = leases.into_iter().next() else {
|
||||
if self.sleep_or_cancel(Duration::from_secs(1)).await {
|
||||
return;
|
||||
}
|
||||
continue;
|
||||
};
|
||||
self.process_lease(lease).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Sleep `dur` or return `true` if cancelled while waiting.
|
||||
async fn sleep_or_cancel(&self, dur: Duration) -> bool {
|
||||
tokio::select! {
|
||||
_ = tokio::time::sleep(dur) => false,
|
||||
_ = self.cancel.cancelled() => true,
|
||||
}
|
||||
}
|
||||
|
||||
async fn process_lease(&self, lease: Lease) {
|
||||
let JobPayload::AnalyzePage { page_id, force } = lease.payload else {
|
||||
// Shouldn't happen — we lease only analyze_page — but ack done
|
||||
// so a misrouted job doesn't loop forever.
|
||||
tracing::warn!(worker = self.id, "analysis worker: non-analyze payload leased");
|
||||
let _ = jobs::ack_done(&self.pool, lease.id).await;
|
||||
return;
|
||||
};
|
||||
|
||||
// Skip-if-done net: a non-forced job for an already-analyzed page is
|
||||
// a no-op (e.g. a duplicate enqueue). Re-analysis goes through
|
||||
// `force` or a fresh page row.
|
||||
if !force {
|
||||
if let Ok(Some(row)) = repo::page_analysis::load(&self.pool, page_id).await {
|
||||
if row.status == crate::domain::page_analysis::AnalysisStatus::Done {
|
||||
let _ = jobs::ack_done(&self.pool, lease.id).await;
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Heartbeat the lease while the vision call runs.
|
||||
let heartbeat = {
|
||||
let hb_pool = self.pool.clone();
|
||||
let hb_id = lease.id;
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
tokio::time::sleep(LEASE_HEARTBEAT).await;
|
||||
match jobs::renew(&hb_pool, hb_id, LEASE_DURATION).await {
|
||||
Ok(true) => {}
|
||||
Ok(false) => break,
|
||||
Err(e) => tracing::warn!(lease_id = %hb_id, ?e, "heartbeat renew failed"),
|
||||
}
|
||||
}
|
||||
})
|
||||
};
|
||||
|
||||
let dispatch = AssertUnwindSafe(self.dispatcher.dispatch(page_id)).catch_unwind();
|
||||
let outcome = tokio::time::timeout(self.job_timeout, dispatch).await;
|
||||
heartbeat.abort();
|
||||
|
||||
// Flatten timeout / panic / dispatch error into one Result + message.
|
||||
let result: Result<(), String> = match outcome {
|
||||
Err(_elapsed) => Err("analysis dispatch timed out".to_string()),
|
||||
Ok(Err(_panic)) => Err("analysis dispatcher panicked".to_string()),
|
||||
Ok(Ok(Err(e))) => Err(format!("{e:#}")),
|
||||
Ok(Ok(Ok(()))) => Ok(()),
|
||||
};
|
||||
|
||||
match result {
|
||||
Ok(()) => {
|
||||
let _ = jobs::ack_done(&self.pool, lease.id).await;
|
||||
}
|
||||
Err(msg) => {
|
||||
tracing::warn!(
|
||||
worker = self.id,
|
||||
%page_id,
|
||||
error = %msg,
|
||||
"analysis worker: dispatch failed — ack failed"
|
||||
);
|
||||
let _ = jobs::ack_failed(
|
||||
&self.pool,
|
||||
lease.id,
|
||||
&msg,
|
||||
lease.attempts,
|
||||
lease.max_attempts,
|
||||
)
|
||||
.await;
|
||||
// Terminal failure (retries exhausted) → record a `failed`
|
||||
// row so the page's analysis state is observable rather than
|
||||
// silently absent.
|
||||
if lease.attempts >= lease.max_attempts {
|
||||
let _ = repo::page_analysis::mark_failed(&self.pool, page_id, &msg).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Production dispatcher: load the page, read its image from storage, call
|
||||
/// the vision model, and persist the analysis.
|
||||
pub struct RealAnalyzeDispatcher {
|
||||
pub db: PgPool,
|
||||
pub storage: Arc<dyn Storage>,
|
||||
pub vision: VisionClient,
|
||||
pub model: String,
|
||||
pub max_image_bytes: usize,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl AnalyzeDispatcher for RealAnalyzeDispatcher {
|
||||
async fn dispatch(&self, page_id: Uuid) -> anyhow::Result<()> {
|
||||
let Some(page) = repo::page::find_by_id(&self.db, page_id).await? else {
|
||||
// Page was deleted between enqueue and dispatch — nothing to do.
|
||||
return Ok(());
|
||||
};
|
||||
let bytes = self
|
||||
.storage
|
||||
.get(&page.storage_key)
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("read page image {}: {e}", page.storage_key))?;
|
||||
if bytes.len() > self.max_image_bytes {
|
||||
anyhow::bail!(
|
||||
"page image {} is {} bytes, over the {} cap",
|
||||
page.storage_key,
|
||||
bytes.len(),
|
||||
self.max_image_bytes
|
||||
);
|
||||
}
|
||||
let analysis = self.vision.analyze(&bytes, &page.content_type).await?;
|
||||
repo::page_analysis::persist_analysis(&self.db, page_id, &analysis, &self.model).await?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Stubs for the daemon's integration tests. Public because the tests live
|
||||
/// in the `tests/` dir (a separate crate).
|
||||
pub mod test_support {
|
||||
use super::*;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
/// Counts dispatch calls and returns a configurable result.
|
||||
pub struct CountingDispatcher {
|
||||
pub calls: AtomicUsize,
|
||||
pub fail: bool,
|
||||
pub panic: bool,
|
||||
}
|
||||
|
||||
impl CountingDispatcher {
|
||||
pub fn ok() -> Arc<Self> {
|
||||
Arc::new(Self { calls: AtomicUsize::new(0), fail: false, panic: false })
|
||||
}
|
||||
pub fn failing() -> Arc<Self> {
|
||||
Arc::new(Self { calls: AtomicUsize::new(0), fail: true, panic: false })
|
||||
}
|
||||
pub fn panicking() -> Arc<Self> {
|
||||
Arc::new(Self { calls: AtomicUsize::new(0), fail: false, panic: true })
|
||||
}
|
||||
pub fn call_count(&self) -> usize {
|
||||
self.calls.load(Ordering::Acquire)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl AnalyzeDispatcher for CountingDispatcher {
|
||||
async fn dispatch(&self, _page_id: Uuid) -> anyhow::Result<()> {
|
||||
self.calls.fetch_add(1, Ordering::AcqRel);
|
||||
if self.panic {
|
||||
panic!("intentional analysis dispatcher panic");
|
||||
}
|
||||
if self.fail {
|
||||
anyhow::bail!("intentional analysis dispatch failure");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -7,5 +7,6 @@
|
||||
//! * [`vision`] — the HTTP client: downscale → request → parse → sanitize.
|
||||
//! * [`daemon`] — the job-leasing worker loop (added in the worker phase).
|
||||
|
||||
pub mod daemon;
|
||||
pub mod prompt;
|
||||
pub mod vision;
|
||||
|
||||
Reference in New Issue
Block a user