feat(analysis): worker daemon + real dispatcher + app wiring

The background analysis worker that drains analyze_page jobs:

- analysis::daemon: a lean sibling of the crawler daemon — leases only
  KIND_ANALYZE_PAGE, skip-if-done-unless-force, lease heartbeat, panic +
  timeout isolation, ack done/failed, and a failed page_analysis row on
  terminal (dead-lettered) failure. AnalyzeDispatcher trait seam.
- RealAnalyzeDispatcher: load page → storage.get → VisionClient.analyze →
  persist_analysis (skips a deleted page; caps image bytes).
- app::build spawns the daemon (own plain reqwest client) when
  ANALYSIS_ENABLED; AppHandle/main shut it down alongside the crawler.
- repo::page::find_by_id.

Tests: dispatch+ack-done, skip-when-done, force re-dispatch, terminal
failure writes failed row, panic isolation, ignores non-analyze jobs.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
MechaCat02
2026-06-13 18:51:53 +02:00
parent dbde6e02c4
commit 7d80a437bf
9 changed files with 600 additions and 5 deletions

View File

@@ -91,6 +91,9 @@ pub struct CrawlerControl {
pub struct AppHandle {
pub router: Router,
pub daemon: Option<daemon::DaemonHandle>,
/// AI content-analysis worker daemon; `None` when `ANALYSIS_ENABLED`
/// is off. Awaited alongside the crawler daemon on shutdown.
pub analysis_daemon: Option<crate::analysis::daemon::AnalysisDaemonHandle>,
}
impl AppHandle {
@@ -98,6 +101,9 @@ impl AppHandle {
if let Some(d) = self.daemon {
d.shutdown().await;
}
if let Some(a) = self.analysis_daemon {
a.shutdown().await;
}
}
}
@@ -131,6 +137,41 @@ pub async fn build(config: Config) -> anyhow::Result<AppHandle> {
(None, None, None)
};
// AI content-analysis worker. Independent of the crawler daemon (works
// for uploads with the crawler off), gated on ANALYSIS_ENABLED. Uses a
// plain reqwest client — no cookie jar / proxy, unlike the crawler's.
let analysis_daemon = if config.analysis.enabled {
let http = reqwest::Client::builder()
.timeout(config.analysis.request_timeout)
.build()
.context("build analysis http client")?;
let vision = crate::analysis::vision::VisionClient::new(http, &config.analysis);
let dispatcher = Arc::new(crate::analysis::daemon::RealAnalyzeDispatcher {
db: db.clone(),
storage: Arc::clone(&storage),
vision,
model: config.analysis.model.clone(),
max_image_bytes: config.analysis.max_image_bytes,
});
let handle = crate::analysis::daemon::spawn(
db.clone(),
tokio_util::sync::CancellationToken::new(),
crate::analysis::daemon::AnalysisDaemonConfig {
dispatcher,
workers: config.analysis.workers,
job_timeout: config.analysis.job_timeout,
},
);
tracing::info!(
workers = config.analysis.workers,
model = %config.analysis.model,
"analysis worker daemon started"
);
Some(handle)
} else {
None
};
let auth_limiter = Arc::new(AuthRateLimiter::new(config.auth.rate_limit));
let state = AppState {
db,
@@ -144,7 +185,7 @@ pub async fn build(config: Config) -> anyhow::Result<AppHandle> {
analysis_enabled: config.analysis.enabled,
};
let router = router(state).layer(cors_layer(&config.cors_allowed_origins));
Ok(AppHandle { router, daemon })
Ok(AppHandle { router, daemon, analysis_daemon })
}
/// Bundle returned by [`spawn_crawler_daemon`]. The handle owns the