fix(vision-manager): disambiguate docker errors, fail-symmetrically on psql, stop wedged container (0.87.8)
Three medium vision-manager findings: 1. **`vision_running()` conflated "container absent" with "inspect failed".** A transient docker-socket-proxy hiccup made the manager think vision had stopped, which reset `up_for`/`idle_for` and defeated MAX_UPTIME. Distinguishes: exit 0 → echo true/false; "No such container" → false; else → ERR (loop skips tick). 2. **`crawl_running()` fail-closed where `analysis_enabled()` fail-opens.** A psql `ERR` logged a misleading "RAM mutex" line every tick. Numeric-guard at the call site with a distinct log. 3. **`start_vision` left wedged containers running** on `START_HEALTH_TIMEOUT` expiry — pinned forever in not-ready state. Now `stop_vision` so next tick retries from a clean state. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
2
backend/Cargo.lock
generated
2
backend/Cargo.lock
generated
@@ -1517,7 +1517,7 @@ checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
|
||||
|
||||
[[package]]
|
||||
name = "mangalord"
|
||||
version = "0.87.7"
|
||||
version = "0.87.8"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"argon2",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "mangalord"
|
||||
version = "0.87.7"
|
||||
version = "0.87.8"
|
||||
edition = "2021"
|
||||
default-run = "mangalord"
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "mangalord-frontend",
|
||||
"version": "0.87.7",
|
||||
"version": "0.87.8",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
|
||||
@@ -91,10 +91,27 @@ read_mem_used_pct() {
|
||||
}
|
||||
|
||||
# ---- Docker helpers --------------------------------------------------------
|
||||
# `docker inspect .State.Running` → true/false; empty if the container is
|
||||
# absent (defined elsewhere and not yet created) — treated as not-running.
|
||||
# `docker inspect .State.Running` → true/false. Disambiguate "container
|
||||
# absent" (legitimate not-running) from "inspect failed" (transient
|
||||
# docker-socket-proxy hiccup) so the loop can skip a tick instead of
|
||||
# resetting timers as if vision had just been stopped — without this
|
||||
# distinction, an inspect blip every poll resets `up_for` and defeats
|
||||
# MAX_UPTIME entirely.
|
||||
vision_running() {
|
||||
docker inspect -f '{{.State.Running}}' "$VISION_CONTAINER" 2>/dev/null || echo "false"
|
||||
local out rc
|
||||
out="$(docker inspect -f '{{.State.Running}}' "$VISION_CONTAINER" 2>&1)"
|
||||
rc=$?
|
||||
if [ "$rc" -eq 0 ]; then
|
||||
# `true` or `false`. Echo as-is.
|
||||
echo "$out"
|
||||
elif echo "$out" | grep -qE 'No such container|No such object|not found'; then
|
||||
# Container is intentionally not defined yet. Treat as not-running.
|
||||
echo "false"
|
||||
else
|
||||
# Transient docker / socket-proxy error. Mirror the psql callers'
|
||||
# `ERR` sentinel; the loop short-circuits the tick on this.
|
||||
echo "ERR"
|
||||
fi
|
||||
}
|
||||
|
||||
vision_ready() {
|
||||
@@ -111,7 +128,19 @@ start_vision() {
|
||||
local waited=0
|
||||
while ! vision_ready; do
|
||||
if [ "$waited" -ge "$START_HEALTH_TIMEOUT" ]; then
|
||||
log "WARNING: $VISION_CONTAINER not healthy after ${START_HEALTH_TIMEOUT}s; leaving it running"
|
||||
# Previously: log a warning and leave the container up. That made
|
||||
# the wedge permanent — next loop tick sees running=true, the
|
||||
# pending>0 branch falls through to `crawl_deferred=0` without
|
||||
# retrying start, and `idle_for` never accumulates because
|
||||
# `pending>0` resets it. The container stays running forever
|
||||
# without ever answering /health.
|
||||
#
|
||||
# Stop it explicitly: a fresh `docker start` next tick takes the
|
||||
# cold-load hit again but at least produces a clean state. If the
|
||||
# wedge is persistent the operator sees a start/stop loop in the
|
||||
# log instead of a silent hang.
|
||||
log "WARNING: $VISION_CONTAINER not healthy after ${START_HEALTH_TIMEOUT}s; stopping so next tick can retry from a clean state"
|
||||
stop_vision "unhealthy after ${START_HEALTH_TIMEOUT}s start probe" || true
|
||||
return 1
|
||||
fi
|
||||
sleep "$HEALTH_POLL_INTERVAL"
|
||||
@@ -184,6 +213,16 @@ while true; do
|
||||
;;
|
||||
esac
|
||||
|
||||
# Same treatment for the docker-inspect status. Without this, a transient
|
||||
# docker-socket-proxy hiccup would mis-classify a running vision as stopped
|
||||
# — every poll under load would reset `up_for`/`idle_for` and defeat
|
||||
# MAX_UPTIME and the idle debounce both. Skip the tick and try again.
|
||||
if [ "$running" = "ERR" ]; then
|
||||
log "WARNING: docker inspect failed; will retry next tick"
|
||||
sleep "$POLL_INTERVAL"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Gate on the runtime enable flag. If analysis is OFF, force the backlog to 0
|
||||
# so the normal idle path runs the STOP_DEBOUNCE timer and shuts vision down.
|
||||
# We deliberately reuse the idle debounce (rather than stopping instantly) so
|
||||
@@ -238,7 +277,23 @@ while true; do
|
||||
# A RUNNING vision never reaches here; idle_for stayed 0 above because
|
||||
# pending>0, so a working vision is never idle-stopped under pressure.
|
||||
:
|
||||
elif [ "$RESPECT_CRAWL_MUTEX" != "0" ] && [ "$(crawl_running)" != "0" ]; then
|
||||
elif [ "$RESPECT_CRAWL_MUTEX" != "0" ] && {
|
||||
cr="$(crawl_running)"
|
||||
# Numeric guard for symmetry with analysis_enabled() (which
|
||||
# fail-OPENs on ERR — see line 74). Without this, a psql
|
||||
# blip falls through to the != "0" branch and we log the
|
||||
# misleading "RAM mutex" line on every tick of the outage.
|
||||
case "$cr" in
|
||||
''|*[!0-9]*)
|
||||
if [ "$crawl_deferred" != "1" ]; then
|
||||
log "WARNING: crawl_running query failed (got '${cr}') — deferring start until next clean read"
|
||||
crawl_deferred=1
|
||||
fi
|
||||
true
|
||||
;;
|
||||
*) [ "$cr" != "0" ] ;;
|
||||
esac
|
||||
}; then
|
||||
# Log once per deferral episode, not every poll, so a long crawl
|
||||
# doesn't flood the log.
|
||||
if [ "$crawl_deferred" != "1" ]; then
|
||||
|
||||
Reference in New Issue
Block a user