fix(vision-manager): disambiguate docker errors, fail-symmetrically on psql, stop wedged container (0.87.8)

Three medium vision-manager findings:

1. **`vision_running()` conflated "container absent" with "inspect
   failed".** A transient docker-socket-proxy hiccup made the manager
   think vision had stopped, which reset `up_for`/`idle_for` and
   defeated MAX_UPTIME. Distinguishes: exit 0 → echo true/false;
   "No such container" → false; else → ERR (loop skips tick).

2. **`crawl_running()` fail-closed where `analysis_enabled()` fail-opens.**
   A psql `ERR` logged a misleading "RAM mutex" line every tick.
   Numeric-guard at the call site with a distinct log.

3. **`start_vision` left wedged containers running** on
   `START_HEALTH_TIMEOUT` expiry — pinned forever in not-ready state.
   Now `stop_vision` so next tick retries from a clean state.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
MechaCat02
2026-06-22 21:47:33 +02:00
parent 6444ddee29
commit c0281f7e9b
4 changed files with 63 additions and 8 deletions

View File

@@ -91,10 +91,27 @@ read_mem_used_pct() {
}
# ---- Docker helpers --------------------------------------------------------
# `docker inspect .State.Running` → true/false; empty if the container is
# absent (defined elsewhere and not yet created) — treated as not-running.
# `docker inspect .State.Running` → true/false. Disambiguate "container
# absent" (legitimate not-running) from "inspect failed" (transient
# docker-socket-proxy hiccup) so the loop can skip a tick instead of
# resetting timers as if vision had just been stopped — without this
# distinction, an inspect blip every poll resets `up_for` and defeats
# MAX_UPTIME entirely.
vision_running() {
docker inspect -f '{{.State.Running}}' "$VISION_CONTAINER" 2>/dev/null || echo "false"
local out rc
out="$(docker inspect -f '{{.State.Running}}' "$VISION_CONTAINER" 2>&1)"
rc=$?
if [ "$rc" -eq 0 ]; then
# `true` or `false`. Echo as-is.
echo "$out"
elif echo "$out" | grep -qE 'No such container|No such object|not found'; then
# Container is intentionally not defined yet. Treat as not-running.
echo "false"
else
# Transient docker / socket-proxy error. Mirror the psql callers'
# `ERR` sentinel; the loop short-circuits the tick on this.
echo "ERR"
fi
}
vision_ready() {
@@ -111,7 +128,19 @@ start_vision() {
local waited=0
while ! vision_ready; do
if [ "$waited" -ge "$START_HEALTH_TIMEOUT" ]; then
log "WARNING: $VISION_CONTAINER not healthy after ${START_HEALTH_TIMEOUT}s; leaving it running"
# Previously: log a warning and leave the container up. That made
# the wedge permanent — next loop tick sees running=true, the
# pending>0 branch falls through to `crawl_deferred=0` without
# retrying start, and `idle_for` never accumulates because
# `pending>0` resets it. The container stays running forever
# without ever answering /health.
#
# Stop it explicitly: a fresh `docker start` next tick takes the
# cold-load hit again but at least produces a clean state. If the
# wedge is persistent the operator sees a start/stop loop in the
# log instead of a silent hang.
log "WARNING: $VISION_CONTAINER not healthy after ${START_HEALTH_TIMEOUT}s; stopping so next tick can retry from a clean state"
stop_vision "unhealthy after ${START_HEALTH_TIMEOUT}s start probe" || true
return 1
fi
sleep "$HEALTH_POLL_INTERVAL"
@@ -184,6 +213,16 @@ while true; do
;;
esac
# Same treatment for the docker-inspect status. Without this, a transient
# docker-socket-proxy hiccup would mis-classify a running vision as stopped
# — every poll under load would reset `up_for`/`idle_for` and defeat
# MAX_UPTIME and the idle debounce both. Skip the tick and try again.
if [ "$running" = "ERR" ]; then
log "WARNING: docker inspect failed; will retry next tick"
sleep "$POLL_INTERVAL"
continue
fi
# Gate on the runtime enable flag. If analysis is OFF, force the backlog to 0
# so the normal idle path runs the STOP_DEBOUNCE timer and shuts vision down.
# We deliberately reuse the idle debounce (rather than stopping instantly) so
@@ -238,7 +277,23 @@ while true; do
# A RUNNING vision never reaches here; idle_for stayed 0 above because
# pending>0, so a working vision is never idle-stopped under pressure.
:
elif [ "$RESPECT_CRAWL_MUTEX" != "0" ] && [ "$(crawl_running)" != "0" ]; then
elif [ "$RESPECT_CRAWL_MUTEX" != "0" ] && {
cr="$(crawl_running)"
# Numeric guard for symmetry with analysis_enabled() (which
# fail-OPENs on ERR — see line 74). Without this, a psql
# blip falls through to the != "0" branch and we log the
# misleading "RAM mutex" line on every tick of the outage.
case "$cr" in
''|*[!0-9]*)
if [ "$crawl_deferred" != "1" ]; then
log "WARNING: crawl_running query failed (got '${cr}') — deferring start until next clean read"
crawl_deferred=1
fi
true
;;
*) [ "$cr" != "0" ] ;;
esac
}; then
# Log once per deferral episode, not every poll, so a long crawl
# doesn't flood the log.
if [ "$crawl_deferred" != "1" ]; then