fix(analysis): harden readiness gate and vision-manager per self-review
Some checks failed
deploy / test-backend (push) Failing after 7m32s
deploy / build-and-push (push) Has been cancelled
deploy / deploy (push) Has been cancelled
deploy / test-frontend (push) Has been cancelled

Backend:
- Log on readiness state transitions (ready<->not-ready) so a misconfigured
  ANALYSIS_VISION_HEALTH_URL, which otherwise parks the worker silently
  forever, is diagnosable.
- Add unit tests for HttpVisionReadiness: 2xx->ready, non-2xx->not-ready,
  connection error->not-ready (the production status mapping had no test).

vision-manager:
- Guard the backlog count against non-numeric output before `-gt`, so a stray
  value can't exit-2 and kill the loop under `set -e`.
- Throttle the crawl-mutex "deferring start" log to once per episode.
- Only reset the idle/uptime timers when `docker stop` actually succeeds, so a
  failed stop retries next tick instead of waiting a full debounce window.
- Decouple the per-probe curl timeout (HEALTH_TIMEOUT) from the warm-up poll
  cadence.

Docs:
- Correct the docker-socket-proxy comment: CONTAINERS+POST permits the full
  container lifecycle (not just start/stop); state the real trust boundary.
- Document that the externally-defined mangalord-vision container must share
  the compose network for name resolution.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
MechaCat02
2026-06-14 15:15:40 +02:00
parent 64a9dceb67
commit f441425519
4 changed files with 132 additions and 22 deletions

View File

@@ -24,6 +24,7 @@ POLL_INTERVAL="${POLL_INTERVAL:-20}" # seconds between backlog checks
STOP_DEBOUNCE="${STOP_DEBOUNCE:-600}" # idle seconds before stopping vision
START_HEALTH_TIMEOUT="${START_HEALTH_TIMEOUT:-300}" # max wait for /health 200
HEALTH_POLL_INTERVAL="${HEALTH_POLL_INTERVAL:-5}" # poll cadence while warming
HEALTH_TIMEOUT="${HEALTH_TIMEOUT:-5}" # per-probe curl timeout, seconds
MAX_UPTIME="${MAX_UPTIME:-0}" # >0: force-stop after N idle-or-not seconds running (backstop); 0 disables
# When >0, refuse to start vision while a crawl is running (Chromium + vision
# together OOM the 8 GiB box). Set to 0 on roomier hosts.
@@ -60,7 +61,7 @@ vision_running() {
}
vision_ready() {
curl -fsS -m "$HEALTH_POLL_INTERVAL" -o /dev/null "$VISION_HEALTH_URL" 2>/dev/null
curl -fsS -m "$HEALTH_TIMEOUT" -o /dev/null "$VISION_HEALTH_URL" 2>/dev/null
}
start_vision() {
@@ -84,24 +85,34 @@ start_vision() {
stop_vision() {
log "stopping $VISION_CONTAINER (idle ${STOP_DEBOUNCE}s)"
docker stop "$VISION_CONTAINER" >/dev/null 2>&1 || log "WARNING: docker stop failed"
if docker stop "$VISION_CONTAINER" >/dev/null 2>&1; then
return 0
fi
log "WARNING: docker stop failed; will retry next tick"
return 1
}
# ---- Main loop -------------------------------------------------------------
log "started: container=$VISION_CONTAINER health=$VISION_HEALTH_URL poll=${POLL_INTERVAL}s debounce=${STOP_DEBOUNCE}s docker_host=$DOCKER_HOST"
idle_for=0 # seconds the queue has been empty while vision is running
up_for=0 # seconds vision has been running (for MAX_UPTIME backstop)
idle_for=0 # seconds the queue has been empty while vision is running
up_for=0 # seconds vision has been running (for MAX_UPTIME backstop)
crawl_deferred=0 # 1 while a start is held off by the crawl mutex (log on flip only)
while true; do
pending="$(pending_analysis)"
running="$(vision_running)"
if [ "$pending" = "ERR" ]; then
log "WARNING: backlog query failed; will retry next tick"
sleep "$POLL_INTERVAL"
continue
fi
# Treat anything non-numeric (the "ERR" sentinel, or a stray psql notice on
# stdout) as "query failed" — never feed it to `-gt`, which under
# `set -e` would otherwise exit-2 and kill the loop.
case "$pending" in
''|*[!0-9]*)
log "WARNING: backlog query failed (got '${pending}'); will retry next tick"
sleep "$POLL_INTERVAL"
continue
;;
esac
if [ "$running" = "true" ]; then
up_for=$((up_for + POLL_INTERVAL))
@@ -113,21 +124,34 @@ while true; do
idle_for=0
if [ "$running" != "true" ]; then
if [ "$RESPECT_CRAWL_MUTEX" != "0" ] && [ "$(crawl_running)" != "0" ]; then
log "deferring start: $pending analysis job(s) pending but a crawl is running (RAM mutex)"
# Log once per deferral episode, not every poll, so a long crawl
# doesn't flood the log.
if [ "$crawl_deferred" != "1" ]; then
log "deferring start: $pending analysis job(s) pending but a crawl is running (RAM mutex)"
crawl_deferred=1
fi
else
crawl_deferred=0
start_vision || true
fi
else
crawl_deferred=0
fi
else
crawl_deferred=0
# No work. Debounce the stop so bursty enqueues don't thrash the load
# cycle, and own the stop on our OWN timer (never wait for a backend
# "drained" signal — leak safety).
if [ "$running" = "true" ]; then
idle_for=$((idle_for + POLL_INTERVAL))
if [ "$idle_for" -ge "$STOP_DEBOUNCE" ]; then
stop_vision
idle_for=0
up_for=0
# Only reset the timers if the stop actually took; otherwise let them
# keep counting so we retry on the next tick rather than waiting out
# another full debounce window.
if stop_vision; then
idle_for=0
up_for=0
fi
fi
else
idle_for=0
@@ -137,9 +161,10 @@ while true; do
# Backstop: bound how long vision can stay up regardless of debounce state.
if [ "$MAX_UPTIME" -gt 0 ] && [ "$running" = "true" ] && [ "$up_for" -ge "$MAX_UPTIME" ]; then
log "MAX_UPTIME ${MAX_UPTIME}s reached; force-stopping $VISION_CONTAINER"
stop_vision
idle_for=0
up_for=0
if stop_vision; then
idle_for=0
up_for=0
fi
fi
sleep "$POLL_INTERVAL"