feat(analysis): worker daemon + real dispatcher + app wiring
The background analysis worker that drains analyze_page jobs: - analysis::daemon: a lean sibling of the crawler daemon — leases only KIND_ANALYZE_PAGE, skip-if-done-unless-force, lease heartbeat, panic + timeout isolation, ack done/failed, and a failed page_analysis row on terminal (dead-lettered) failure. AnalyzeDispatcher trait seam. - RealAnalyzeDispatcher: load page → storage.get → VisionClient.analyze → persist_analysis (skips a deleted page; caps image bytes). - app::build spawns the daemon (own plain reqwest client) when ANALYSIS_ENABLED; AppHandle/main shut it down alongside the crawler. - repo::page::find_by_id. Tests: dispatch+ack-done, skip-when-done, force re-dispatch, terminal failure writes failed row, panic isolation, ignores non-analyze jobs. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -91,6 +91,9 @@ pub struct CrawlerControl {
|
||||
pub struct AppHandle {
|
||||
pub router: Router,
|
||||
pub daemon: Option<daemon::DaemonHandle>,
|
||||
/// AI content-analysis worker daemon; `None` when `ANALYSIS_ENABLED`
|
||||
/// is off. Awaited alongside the crawler daemon on shutdown.
|
||||
pub analysis_daemon: Option<crate::analysis::daemon::AnalysisDaemonHandle>,
|
||||
}
|
||||
|
||||
impl AppHandle {
|
||||
@@ -98,6 +101,9 @@ impl AppHandle {
|
||||
if let Some(d) = self.daemon {
|
||||
d.shutdown().await;
|
||||
}
|
||||
if let Some(a) = self.analysis_daemon {
|
||||
a.shutdown().await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -131,6 +137,41 @@ pub async fn build(config: Config) -> anyhow::Result<AppHandle> {
|
||||
(None, None, None)
|
||||
};
|
||||
|
||||
// AI content-analysis worker. Independent of the crawler daemon (works
|
||||
// for uploads with the crawler off), gated on ANALYSIS_ENABLED. Uses a
|
||||
// plain reqwest client — no cookie jar / proxy, unlike the crawler's.
|
||||
let analysis_daemon = if config.analysis.enabled {
|
||||
let http = reqwest::Client::builder()
|
||||
.timeout(config.analysis.request_timeout)
|
||||
.build()
|
||||
.context("build analysis http client")?;
|
||||
let vision = crate::analysis::vision::VisionClient::new(http, &config.analysis);
|
||||
let dispatcher = Arc::new(crate::analysis::daemon::RealAnalyzeDispatcher {
|
||||
db: db.clone(),
|
||||
storage: Arc::clone(&storage),
|
||||
vision,
|
||||
model: config.analysis.model.clone(),
|
||||
max_image_bytes: config.analysis.max_image_bytes,
|
||||
});
|
||||
let handle = crate::analysis::daemon::spawn(
|
||||
db.clone(),
|
||||
tokio_util::sync::CancellationToken::new(),
|
||||
crate::analysis::daemon::AnalysisDaemonConfig {
|
||||
dispatcher,
|
||||
workers: config.analysis.workers,
|
||||
job_timeout: config.analysis.job_timeout,
|
||||
},
|
||||
);
|
||||
tracing::info!(
|
||||
workers = config.analysis.workers,
|
||||
model = %config.analysis.model,
|
||||
"analysis worker daemon started"
|
||||
);
|
||||
Some(handle)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let auth_limiter = Arc::new(AuthRateLimiter::new(config.auth.rate_limit));
|
||||
let state = AppState {
|
||||
db,
|
||||
@@ -144,7 +185,7 @@ pub async fn build(config: Config) -> anyhow::Result<AppHandle> {
|
||||
analysis_enabled: config.analysis.enabled,
|
||||
};
|
||||
let router = router(state).layer(cors_layer(&config.cors_allowed_origins));
|
||||
Ok(AppHandle { router, daemon })
|
||||
Ok(AppHandle { router, daemon, analysis_daemon })
|
||||
}
|
||||
|
||||
/// Bundle returned by [`spawn_crawler_daemon`]. The handle owns the
|
||||
|
||||
Reference in New Issue
Block a user