feat(analysis): gate page leasing on a vision readiness probe

The analysis worker leases an analyze_page job (incrementing attempts in
SQL) before calling vision, so a stopped/loading vision burns all retries
and writes a permanent failed page_analysis row. With the vision-manager
autoscaler idle-stopping the container, that would poison the queue on
every cold start.

Add an optional VisionReadiness seam (HTTP GET /health in production) wired
via the new env-only ANALYSIS_VISION_HEALTH_URL. When set, the worker parks
without leasing until the probe answers 2xx; jobs stay pending with
attempts untouched and resume the instant vision is ready. None preserves
today's behavior for always-on endpoints.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

backend/src/analysis/daemon.rs

@@ -32,6 +32,10 @@ use crate::storage::Storage;
 const LEASE_DURATION: Duration = Duration::from_secs(60);
 /// Heartbeat cadence — a third of the lease window.
 const LEASE_HEARTBEAT: Duration = Duration::from_secs(20);
+/// How long to wait before re-probing vision when the readiness gate is
+/// closed. Short enough to resume promptly after an autoscaler cold start,
+/// long enough not to hammer `/health` while vision is down.
+const READINESS_POLL: Duration = Duration::from_secs(2);
 
 /// The unit of work: analyze one page. Implemented by
 /// [`RealAnalyzeDispatcher`] in production and stubbed in tests.
@@ -40,12 +44,45 @@ pub trait AnalyzeDispatcher: Send + Sync {
     async fn dispatch(&self, page_id: Uuid) -> anyhow::Result<()>;
 }
 
+/// Probe answering "is the vision backend up and the model loaded?". When a
+/// probe is wired in, the worker refuses to *lease* a job until it reports
+/// ready — so an idle-stopped vision (the autoscaler's normal state) never
+/// burns a job's retries or writes a `failed` row. Implemented by
+/// [`HttpVisionReadiness`] in production (a `GET /health`) and stubbed in
+/// tests.
+#[async_trait]
+pub trait VisionReadiness: Send + Sync {
+    async fn ready(&self) -> bool;
+}
+
+/// Production readiness probe: `GET {health_url}`, ready iff it answers 2xx.
+/// Any transport error (connection refused while stopped, 503 while the
+/// model loads) is treated as not-ready.
+pub struct HttpVisionReadiness {
+    pub http: reqwest::Client,
+    pub health_url: String,
+}
+
+#[async_trait]
+impl VisionReadiness for HttpVisionReadiness {
+    async fn ready(&self) -> bool {
+        match self.http.get(&self.health_url).send().await {
+            Ok(resp) => resp.status().is_success(),
+            Err(_) => false,
+        }
+    }
+}
+
 pub struct AnalysisDaemonConfig {
     pub dispatcher: Arc<dyn AnalyzeDispatcher>,
     pub workers: usize,
     pub job_timeout: Duration,
     /// Live-event sink (Started/Completed/Failed) for admin SSE.
     pub events: Arc<AnalysisEvents>,
+    /// Optional vision readiness gate. `None` ⇒ no gate (today's behavior,
+    /// for always-on endpoints). `Some` ⇒ the worker parks instead of
+    /// leasing while the probe is not ready.
+    pub readiness: Option<Arc<dyn VisionReadiness>>,
 }
 
 pub struct AnalysisDaemonHandle {
@@ -75,6 +112,7 @@ pub fn spawn(
             dispatcher: Arc::clone(&cfg.dispatcher),
             job_timeout: cfg.job_timeout,
             events: Arc::clone(&cfg.events),
+            readiness: cfg.readiness.clone(),
             id,
         };
         join.spawn(async move { ctx.run().await });
@@ -88,6 +126,7 @@ struct WorkerContext {
     dispatcher: Arc<dyn AnalyzeDispatcher>,
     job_timeout: Duration,
     events: Arc<AnalysisEvents>,
+    readiness: Option<Arc<dyn VisionReadiness>>,
     id: usize,
 }
 
@@ -98,6 +137,18 @@ impl WorkerContext {
                 tracing::info!(worker = self.id, "analysis worker: shutdown");
                 return;
             }
+            // Readiness gate: park (without leasing) while vision is down or
+            // still loading its model, so the autoscaler's idle-stop never
+            // costs a job its retries. Probe *before* the lease — leasing
+            // increments `attempts` in SQL and can't be undone.
+            if let Some(readiness) = &self.readiness {
+                if !readiness.ready().await {
+                    if self.sleep_or_cancel(READINESS_POLL).await {
+                        return;
+                    }
+                    continue;
+                }
+            }
             let leases =
                 match jobs::lease(&self.pool, Some(KIND_ANALYZE_PAGE), 1, LEASE_DURATION).await {
                     Ok(v) => v,
@@ -269,7 +320,29 @@ impl AnalyzeDispatcher for RealAnalyzeDispatcher {
 /// in the `tests/` dir (a separate crate).
 pub mod test_support {
     use super::*;
-    use std::sync::atomic::{AtomicUsize, Ordering};
+    use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+
+    /// A readiness probe whose answer can be flipped from the test thread, to
+    /// drive the "vision is down, then comes up" transition.
+    pub struct ToggleReadiness {
+        ready: AtomicBool,
+    }
+
+    impl ToggleReadiness {
+        pub fn new(ready: bool) -> Arc<Self> {
+            Arc::new(Self { ready: AtomicBool::new(ready) })
+        }
+        pub fn set(&self, ready: bool) {
+            self.ready.store(ready, Ordering::Release);
+        }
+    }
+
+    #[async_trait]
+    impl VisionReadiness for ToggleReadiness {
+        async fn ready(&self) -> bool {
+            self.ready.load(Ordering::Acquire)
+        }
+    }
 
     /// Counts dispatch calls and returns a configurable result.
     pub struct CountingDispatcher {

backend/src/app.rs

@@ -420,6 +420,24 @@ fn spawn_analysis_daemon(
         model: cfg.model.clone(),
         max_image_bytes: cfg.max_image_bytes,
     });
+    // When a readiness URL is configured, gate leasing on it so an
+    // autoscaler that idle-stops the vision container never lets a job burn
+    // its retries. A dedicated short-timeout client keeps the probe snappy
+    // and independent of the (long) per-request analysis timeout.
+    let readiness: Option<Arc<dyn crate::analysis::daemon::VisionReadiness>> =
+        match &cfg.vision_health_url {
+            Some(url) if !url.is_empty() => {
+                let probe = reqwest::Client::builder()
+                    .timeout(std::time::Duration::from_secs(5))
+                    .build()
+                    .context("build vision readiness http client")?;
+                Some(Arc::new(crate::analysis::daemon::HttpVisionReadiness {
+                    http: probe,
+                    health_url: url.clone(),
+                }))
+            }
+            _ => None,
+        };
     let handle = crate::analysis::daemon::spawn(
         db,
         CancellationToken::new(),
@@ -428,6 +446,7 @@ fn spawn_analysis_daemon(
             workers: cfg.workers,
             job_timeout: cfg.job_timeout,
             events,
+            readiness,
         },
     );
     tracing::info!(workers = cfg.workers, model = %cfg.model, "analysis worker daemon started");

backend/src/config.rs

@@ -124,6 +124,13 @@ pub struct AnalysisConfig {
     pub workers: usize,
     /// OpenAI-compatible chat/completions URL (`ANALYSIS_VISION_URL`).
     pub endpoint: String,
+    /// Optional vision readiness probe URL (`ANALYSIS_VISION_HEALTH_URL`),
+    /// e.g. `http://mangalord-vision:8000/health`. When set, the worker
+    /// refuses to lease a job until this answers 2xx — so an autoscaler that
+    /// idle-stops the vision container never lets a job burn its retries or
+    /// land a `failed` row. `None` (the default) disables the gate, matching
+    /// the prior behavior for always-on endpoints. Env-only, like `api_key`.
+    pub vision_health_url: Option<String>,
     /// Model id to request (`ANALYSIS_MODEL`).
     pub model: String,
     /// Optional bearer token (`ANALYSIS_API_KEY`); local servers usually
@@ -190,6 +197,7 @@ impl Default for AnalysisConfig {
             enabled: false,
             workers: 1,
             endpoint: "http://localhost:8000/v1/chat/completions".to_string(),
+            vision_health_url: None,
             model: String::new(),
             api_key: None,
             request_timeout: Duration::from_secs(120),
@@ -221,6 +229,9 @@ impl AnalysisConfig {
             enabled: env_bool("ANALYSIS_ENABLED", d.enabled),
             workers: env_usize("ANALYSIS_WORKERS", d.workers).max(1),
             endpoint: std::env::var("ANALYSIS_VISION_URL").unwrap_or(d.endpoint),
+            vision_health_url: std::env::var("ANALYSIS_VISION_HEALTH_URL")
+                .ok()
+                .filter(|s| !s.is_empty()),
             model: std::env::var("ANALYSIS_MODEL").unwrap_or(d.model),
             api_key: std::env::var("ANALYSIS_API_KEY")
                 .ok()

backend/src/settings.rs

@@ -410,6 +410,8 @@ impl AnalysisSettings {
             grounding_prompt: prompt(&self.grounding_prompt, GROUNDING_PROMPT_DEFAULT),
             // Env-only secret preserved from the base.
             api_key: base.api_key.clone(),
+            // Env-only readiness probe URL preserved from the base.
+            vision_health_url: base.vision_health_url.clone(),
         })
     }
 }