fix(analysis): harden readiness gate and vision-manager per self-review
Backend: - Log on readiness state transitions (ready<->not-ready) so a misconfigured ANALYSIS_VISION_HEALTH_URL, which otherwise parks the worker silently forever, is diagnosable. - Add unit tests for HttpVisionReadiness: 2xx->ready, non-2xx->not-ready, connection error->not-ready (the production status mapping had no test). vision-manager: - Guard the backlog count against non-numeric output before `-gt`, so a stray value can't exit-2 and kill the loop under `set -e`. - Throttle the crawl-mutex "deferring start" log to once per episode. - Only reset the idle/uptime timers when `docker stop` actually succeeds, so a failed stop retries next tick instead of waiting a full debounce window. - Decouple the per-probe curl timeout (HEALTH_TIMEOUT) from the warm-up poll cadence. Docs: - Correct the docker-socket-proxy comment: CONTAINERS+POST permits the full container lifecycle (not just start/stop); state the real trust boundary. - Document that the externally-defined mangalord-vision container must share the compose network for name resolution. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -24,6 +24,7 @@ POLL_INTERVAL="${POLL_INTERVAL:-20}" # seconds between backlog checks
|
||||
STOP_DEBOUNCE="${STOP_DEBOUNCE:-600}" # idle seconds before stopping vision
|
||||
START_HEALTH_TIMEOUT="${START_HEALTH_TIMEOUT:-300}" # max wait for /health 200
|
||||
HEALTH_POLL_INTERVAL="${HEALTH_POLL_INTERVAL:-5}" # poll cadence while warming
|
||||
HEALTH_TIMEOUT="${HEALTH_TIMEOUT:-5}" # per-probe curl timeout, seconds
|
||||
MAX_UPTIME="${MAX_UPTIME:-0}" # >0: force-stop after N idle-or-not seconds running (backstop); 0 disables
|
||||
# When >0, refuse to start vision while a crawl is running (Chromium + vision
|
||||
# together OOM the 8 GiB box). Set to 0 on roomier hosts.
|
||||
@@ -60,7 +61,7 @@ vision_running() {
|
||||
}
|
||||
|
||||
vision_ready() {
|
||||
curl -fsS -m "$HEALTH_POLL_INTERVAL" -o /dev/null "$VISION_HEALTH_URL" 2>/dev/null
|
||||
curl -fsS -m "$HEALTH_TIMEOUT" -o /dev/null "$VISION_HEALTH_URL" 2>/dev/null
|
||||
}
|
||||
|
||||
start_vision() {
|
||||
@@ -84,24 +85,34 @@ start_vision() {
|
||||
|
||||
stop_vision() {
|
||||
log "stopping $VISION_CONTAINER (idle ${STOP_DEBOUNCE}s)"
|
||||
docker stop "$VISION_CONTAINER" >/dev/null 2>&1 || log "WARNING: docker stop failed"
|
||||
if docker stop "$VISION_CONTAINER" >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
log "WARNING: docker stop failed; will retry next tick"
|
||||
return 1
|
||||
}
|
||||
|
||||
# ---- Main loop -------------------------------------------------------------
|
||||
log "started: container=$VISION_CONTAINER health=$VISION_HEALTH_URL poll=${POLL_INTERVAL}s debounce=${STOP_DEBOUNCE}s docker_host=$DOCKER_HOST"
|
||||
|
||||
idle_for=0 # seconds the queue has been empty while vision is running
|
||||
up_for=0 # seconds vision has been running (for MAX_UPTIME backstop)
|
||||
idle_for=0 # seconds the queue has been empty while vision is running
|
||||
up_for=0 # seconds vision has been running (for MAX_UPTIME backstop)
|
||||
crawl_deferred=0 # 1 while a start is held off by the crawl mutex (log on flip only)
|
||||
|
||||
while true; do
|
||||
pending="$(pending_analysis)"
|
||||
running="$(vision_running)"
|
||||
|
||||
if [ "$pending" = "ERR" ]; then
|
||||
log "WARNING: backlog query failed; will retry next tick"
|
||||
sleep "$POLL_INTERVAL"
|
||||
continue
|
||||
fi
|
||||
# Treat anything non-numeric (the "ERR" sentinel, or a stray psql notice on
|
||||
# stdout) as "query failed" — never feed it to `-gt`, which under
|
||||
# `set -e` would otherwise exit-2 and kill the loop.
|
||||
case "$pending" in
|
||||
''|*[!0-9]*)
|
||||
log "WARNING: backlog query failed (got '${pending}'); will retry next tick"
|
||||
sleep "$POLL_INTERVAL"
|
||||
continue
|
||||
;;
|
||||
esac
|
||||
|
||||
if [ "$running" = "true" ]; then
|
||||
up_for=$((up_for + POLL_INTERVAL))
|
||||
@@ -113,21 +124,34 @@ while true; do
|
||||
idle_for=0
|
||||
if [ "$running" != "true" ]; then
|
||||
if [ "$RESPECT_CRAWL_MUTEX" != "0" ] && [ "$(crawl_running)" != "0" ]; then
|
||||
log "deferring start: $pending analysis job(s) pending but a crawl is running (RAM mutex)"
|
||||
# Log once per deferral episode, not every poll, so a long crawl
|
||||
# doesn't flood the log.
|
||||
if [ "$crawl_deferred" != "1" ]; then
|
||||
log "deferring start: $pending analysis job(s) pending but a crawl is running (RAM mutex)"
|
||||
crawl_deferred=1
|
||||
fi
|
||||
else
|
||||
crawl_deferred=0
|
||||
start_vision || true
|
||||
fi
|
||||
else
|
||||
crawl_deferred=0
|
||||
fi
|
||||
else
|
||||
crawl_deferred=0
|
||||
# No work. Debounce the stop so bursty enqueues don't thrash the load
|
||||
# cycle, and own the stop on our OWN timer (never wait for a backend
|
||||
# "drained" signal — leak safety).
|
||||
if [ "$running" = "true" ]; then
|
||||
idle_for=$((idle_for + POLL_INTERVAL))
|
||||
if [ "$idle_for" -ge "$STOP_DEBOUNCE" ]; then
|
||||
stop_vision
|
||||
idle_for=0
|
||||
up_for=0
|
||||
# Only reset the timers if the stop actually took; otherwise let them
|
||||
# keep counting so we retry on the next tick rather than waiting out
|
||||
# another full debounce window.
|
||||
if stop_vision; then
|
||||
idle_for=0
|
||||
up_for=0
|
||||
fi
|
||||
fi
|
||||
else
|
||||
idle_for=0
|
||||
@@ -137,9 +161,10 @@ while true; do
|
||||
# Backstop: bound how long vision can stay up regardless of debounce state.
|
||||
if [ "$MAX_UPTIME" -gt 0 ] && [ "$running" = "true" ] && [ "$up_for" -ge "$MAX_UPTIME" ]; then
|
||||
log "MAX_UPTIME ${MAX_UPTIME}s reached; force-stopping $VISION_CONTAINER"
|
||||
stop_vision
|
||||
idle_for=0
|
||||
up_for=0
|
||||
if stop_vision; then
|
||||
idle_for=0
|
||||
up_for=0
|
||||
fi
|
||||
fi
|
||||
|
||||
sleep "$POLL_INTERVAL"
|
||||
|
||||
Reference in New Issue
Block a user