fix(analysis): harden readiness gate and vision-manager per self-review
Backend: - Log on readiness state transitions (ready<->not-ready) so a misconfigured ANALYSIS_VISION_HEALTH_URL, which otherwise parks the worker silently forever, is diagnosable. - Add unit tests for HttpVisionReadiness: 2xx->ready, non-2xx->not-ready, connection error->not-ready (the production status mapping had no test). vision-manager: - Guard the backlog count against non-numeric output before `-gt`, so a stray value can't exit-2 and kill the loop under `set -e`. - Throttle the crawl-mutex "deferring start" log to once per episode. - Only reset the idle/uptime timers when `docker stop` actually succeeds, so a failed stop retries next tick instead of waiting a full debounce window. - Decouple the per-probe curl timeout (HEALTH_TIMEOUT) from the warm-up poll cadence. Docs: - Correct the docker-socket-proxy comment: CONTAINERS+POST permits the full container lifecycle (not just start/stop); state the real trust boundary. - Document that the externally-defined mangalord-vision container must share the compose network for name resolution. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -132,6 +132,9 @@ struct WorkerContext {
|
||||
|
||||
impl WorkerContext {
|
||||
async fn run(self) {
|
||||
// Last observed readiness, so we log only on transitions (not every
|
||||
// poll). `None` until the first probe.
|
||||
let mut was_ready: Option<bool> = None;
|
||||
loop {
|
||||
if self.cancel.is_cancelled() {
|
||||
tracing::info!(worker = self.id, "analysis worker: shutdown");
|
||||
@@ -142,7 +145,21 @@ impl WorkerContext {
|
||||
// costs a job its retries. Probe *before* the lease — leasing
|
||||
// increments `attempts` in SQL and can't be undone.
|
||||
if let Some(readiness) = &self.readiness {
|
||||
if !readiness.ready().await {
|
||||
let ready = readiness.ready().await;
|
||||
// Log on flip so a typo'd health URL (which parks the worker
|
||||
// forever) is diagnosable, without spamming every poll.
|
||||
if was_ready != Some(ready) {
|
||||
if ready {
|
||||
tracing::info!(worker = self.id, "analysis worker: vision ready — resuming");
|
||||
} else {
|
||||
tracing::warn!(
|
||||
worker = self.id,
|
||||
"analysis worker: vision not ready — parking until /health is 2xx"
|
||||
);
|
||||
}
|
||||
was_ready = Some(ready);
|
||||
}
|
||||
if !ready {
|
||||
if self.sleep_or_cancel(READINESS_POLL).await {
|
||||
return;
|
||||
}
|
||||
@@ -380,3 +397,56 @@ pub mod test_support {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::net::TcpListener;
|
||||
|
||||
/// Bind an ephemeral port that answers every request with `status_line`
|
||||
/// (e.g. `"200 OK"`), and return its `/health` URL. The probe under test
|
||||
/// only inspects the status code, so a zero-length body is enough.
|
||||
async fn serve(status_line: &'static str) -> String {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
tokio::spawn(async move {
|
||||
while let Ok((mut sock, _)) = listener.accept().await {
|
||||
let mut buf = [0u8; 1024];
|
||||
let _ = sock.read(&mut buf).await;
|
||||
let resp = format!(
|
||||
"HTTP/1.1 {status_line}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
|
||||
);
|
||||
let _ = sock.write_all(resp.as_bytes()).await;
|
||||
}
|
||||
});
|
||||
format!("http://{addr}/health")
|
||||
}
|
||||
|
||||
fn probe(url: String) -> HttpVisionReadiness {
|
||||
HttpVisionReadiness {
|
||||
http: reqwest::Client::builder()
|
||||
.timeout(Duration::from_millis(500))
|
||||
.build()
|
||||
.unwrap(),
|
||||
health_url: url,
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn http_readiness_2xx_is_ready() {
|
||||
assert!(probe(serve("200 OK").await).ready().await);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn http_readiness_non_2xx_is_not_ready() {
|
||||
// llama-server answers 503 while the model is still loading.
|
||||
assert!(!probe(serve("503 Service Unavailable").await).ready().await);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn http_readiness_connection_error_is_not_ready() {
|
||||
// Nothing listening (vision stopped) → not ready, never an error.
|
||||
assert!(!probe("http://127.0.0.1:1/health".to_string()).ready().await);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user