feat(analysis): gate page leasing on a vision readiness probe
The analysis worker leases an analyze_page job (incrementing attempts in SQL) before calling vision, so a stopped/loading vision burns all retries and writes a permanent failed page_analysis row. With the vision-manager autoscaler idle-stopping the container, that would poison the queue on every cold start. Add an optional VisionReadiness seam (HTTP GET /health in production) wired via the new env-only ANALYSIS_VISION_HEALTH_URL. When set, the worker parks without leasing until the probe answers 2xx; jobs stay pending with attempts untouched and resume the instant vision is ready. None preserves today's behavior for always-on endpoints. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -32,6 +32,10 @@ use crate::storage::Storage;
|
||||
const LEASE_DURATION: Duration = Duration::from_secs(60);
|
||||
/// Heartbeat cadence — a third of the lease window.
|
||||
const LEASE_HEARTBEAT: Duration = Duration::from_secs(20);
|
||||
/// How long to wait before re-probing vision when the readiness gate is
|
||||
/// closed. Short enough to resume promptly after an autoscaler cold start,
|
||||
/// long enough not to hammer `/health` while vision is down.
|
||||
const READINESS_POLL: Duration = Duration::from_secs(2);
|
||||
|
||||
/// The unit of work: analyze one page. Implemented by
|
||||
/// [`RealAnalyzeDispatcher`] in production and stubbed in tests.
|
||||
@@ -40,12 +44,45 @@ pub trait AnalyzeDispatcher: Send + Sync {
|
||||
async fn dispatch(&self, page_id: Uuid) -> anyhow::Result<()>;
|
||||
}
|
||||
|
||||
/// Probe answering "is the vision backend up and the model loaded?". When a
|
||||
/// probe is wired in, the worker refuses to *lease* a job until it reports
|
||||
/// ready — so an idle-stopped vision (the autoscaler's normal state) never
|
||||
/// burns a job's retries or writes a `failed` row. Implemented by
|
||||
/// [`HttpVisionReadiness`] in production (a `GET /health`) and stubbed in
|
||||
/// tests.
|
||||
#[async_trait]
|
||||
pub trait VisionReadiness: Send + Sync {
|
||||
async fn ready(&self) -> bool;
|
||||
}
|
||||
|
||||
/// Production readiness probe: `GET {health_url}`, ready iff it answers 2xx.
|
||||
/// Any transport error (connection refused while stopped, 503 while the
|
||||
/// model loads) is treated as not-ready.
|
||||
pub struct HttpVisionReadiness {
|
||||
pub http: reqwest::Client,
|
||||
pub health_url: String,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl VisionReadiness for HttpVisionReadiness {
|
||||
async fn ready(&self) -> bool {
|
||||
match self.http.get(&self.health_url).send().await {
|
||||
Ok(resp) => resp.status().is_success(),
|
||||
Err(_) => false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct AnalysisDaemonConfig {
|
||||
pub dispatcher: Arc<dyn AnalyzeDispatcher>,
|
||||
pub workers: usize,
|
||||
pub job_timeout: Duration,
|
||||
/// Live-event sink (Started/Completed/Failed) for admin SSE.
|
||||
pub events: Arc<AnalysisEvents>,
|
||||
/// Optional vision readiness gate. `None` ⇒ no gate (today's behavior,
|
||||
/// for always-on endpoints). `Some` ⇒ the worker parks instead of
|
||||
/// leasing while the probe is not ready.
|
||||
pub readiness: Option<Arc<dyn VisionReadiness>>,
|
||||
}
|
||||
|
||||
pub struct AnalysisDaemonHandle {
|
||||
@@ -75,6 +112,7 @@ pub fn spawn(
|
||||
dispatcher: Arc::clone(&cfg.dispatcher),
|
||||
job_timeout: cfg.job_timeout,
|
||||
events: Arc::clone(&cfg.events),
|
||||
readiness: cfg.readiness.clone(),
|
||||
id,
|
||||
};
|
||||
join.spawn(async move { ctx.run().await });
|
||||
@@ -88,6 +126,7 @@ struct WorkerContext {
|
||||
dispatcher: Arc<dyn AnalyzeDispatcher>,
|
||||
job_timeout: Duration,
|
||||
events: Arc<AnalysisEvents>,
|
||||
readiness: Option<Arc<dyn VisionReadiness>>,
|
||||
id: usize,
|
||||
}
|
||||
|
||||
@@ -98,6 +137,18 @@ impl WorkerContext {
|
||||
tracing::info!(worker = self.id, "analysis worker: shutdown");
|
||||
return;
|
||||
}
|
||||
// Readiness gate: park (without leasing) while vision is down or
|
||||
// still loading its model, so the autoscaler's idle-stop never
|
||||
// costs a job its retries. Probe *before* the lease — leasing
|
||||
// increments `attempts` in SQL and can't be undone.
|
||||
if let Some(readiness) = &self.readiness {
|
||||
if !readiness.ready().await {
|
||||
if self.sleep_or_cancel(READINESS_POLL).await {
|
||||
return;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
}
|
||||
let leases =
|
||||
match jobs::lease(&self.pool, Some(KIND_ANALYZE_PAGE), 1, LEASE_DURATION).await {
|
||||
Ok(v) => v,
|
||||
@@ -269,7 +320,29 @@ impl AnalyzeDispatcher for RealAnalyzeDispatcher {
|
||||
/// in the `tests/` dir (a separate crate).
|
||||
pub mod test_support {
|
||||
use super::*;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||
|
||||
/// A readiness probe whose answer can be flipped from the test thread, to
|
||||
/// drive the "vision is down, then comes up" transition.
|
||||
pub struct ToggleReadiness {
|
||||
ready: AtomicBool,
|
||||
}
|
||||
|
||||
impl ToggleReadiness {
|
||||
pub fn new(ready: bool) -> Arc<Self> {
|
||||
Arc::new(Self { ready: AtomicBool::new(ready) })
|
||||
}
|
||||
pub fn set(&self, ready: bool) {
|
||||
self.ready.store(ready, Ordering::Release);
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl VisionReadiness for ToggleReadiness {
|
||||
async fn ready(&self) -> bool {
|
||||
self.ready.load(Ordering::Acquire)
|
||||
}
|
||||
}
|
||||
|
||||
/// Counts dispatch calls and returns a configurable result.
|
||||
pub struct CountingDispatcher {
|
||||
|
||||
Reference in New Issue
Block a user