fix(vision-manager): symmetric ERR handling + smoke tests + analysis reload reclaim test (0.87.15)
0.87.7 + 0.87.8 follow-ups: 1. mem_check_and_stop now defers on docker ERR instead of silently no-op'ing the SIGTERM under pressure. 2. Post-mem_check_and_stop loop refresh now also handles the ERR sentinel (continues the loop) rather than letting it reset up_for and defeat MAX_UPTIME. 3. New `vision-manager/test_manager.sh` — 7 smoke tests covering vision_running's three exit shapes and mem_check_and_stop's ERR symmetry, via stubbed docker/psql/curl on PATH. Production loop guarded by MANAGER_TEST_NO_MAIN. 4. New `app::tests::spawn_analysis_daemon_reclaims_orphaned_analyze_leases` pins the 0.87.7 wire-through: seeds an expired analyze_page lease, spawns + shuts down, asserts pending + attempts refunded. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -164,17 +164,34 @@ stop_vision() {
|
||||
# SIGTERM it (the friendly, right-victim alternative to an OOM kill) and arm a
|
||||
# cooldown. Cheap on the hot path — only inspects docker once used% is already
|
||||
# over the mark. Reads `mem_yield_until`/`idle_for`/`up_for` from the loop scope.
|
||||
#
|
||||
# Symmetric ERR handling with the main loop's `running` refresh: an inspect
|
||||
# blip under genuine memory pressure must NOT silently no-op the SIGTERM.
|
||||
# When `vision_running` returns `ERR`, log it and skip THIS check rather than
|
||||
# treating the ambiguous state as "not running" (which would let memory
|
||||
# pressure go un-yielded until the next tick).
|
||||
mem_check_and_stop() {
|
||||
[ "$MEM_YIELD_ENABLED" = "0" ] && return 0
|
||||
local used; used="$(read_mem_used_pct)"
|
||||
case "$used" in ''|*[!0-9]*) return 0 ;; esac # ignore a malformed read
|
||||
if [ "$used" -ge "$MEM_HIGH_WATERMARK_PCT" ] && [ "$(vision_running)" = "true" ]; then
|
||||
if stop_vision "memory-yield: used ${used}% >= ${MEM_HIGH_WATERMARK_PCT}% high watermark"; then
|
||||
mem_yield_until=$(( $(date +%s) + MEM_YIELD_COOLDOWN ))
|
||||
idle_for=0
|
||||
up_for=0
|
||||
fi
|
||||
fi
|
||||
[ "$used" -lt "$MEM_HIGH_WATERMARK_PCT" ] && return 0
|
||||
local vr; vr="$(vision_running)"
|
||||
case "$vr" in
|
||||
ERR)
|
||||
log "WARNING: memory-yield wanted to stop vision (used ${used}% >= ${MEM_HIGH_WATERMARK_PCT}%), but vision_running returned ERR — deferring to next tick"
|
||||
return 0
|
||||
;;
|
||||
true)
|
||||
if stop_vision "memory-yield: used ${used}% >= ${MEM_HIGH_WATERMARK_PCT}% high watermark"; then
|
||||
mem_yield_until=$(( $(date +%s) + MEM_YIELD_COOLDOWN ))
|
||||
idle_for=0
|
||||
up_for=0
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
# Not running — nothing to yield from. Cheap no-op.
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
# Start inhibit: echo 1 while a cooldown is in effect OR used% is at/above the
|
||||
@@ -198,6 +215,14 @@ analysis_off=0 # 1 while analysis is disabled in settings (log on flip only)
|
||||
mem_yield_until=0 # epoch deadline of the active memory-yield cooldown (0 = none)
|
||||
mem_inhibited=0 # 1 while starts are inhibited by memory pressure (log on flip only)
|
||||
|
||||
# Tests source this script with `MANAGER_TEST_NO_MAIN=1` set so they can
|
||||
# exercise the helper functions above with stubbed `docker`/`psql` on PATH
|
||||
# without entering the production poll loop. Production callers (and the
|
||||
# Dockerfile entrypoint) leave the var unset, so the loop runs as before.
|
||||
if [ -n "${MANAGER_TEST_NO_MAIN:-}" ]; then
|
||||
return 0 2>/dev/null || exit 0
|
||||
fi
|
||||
|
||||
while true; do
|
||||
pending="$(pending_analysis)"
|
||||
running="$(vision_running)"
|
||||
@@ -250,7 +275,18 @@ while true; do
|
||||
# LOW (idle stack + LLM ≈ 80% here). LOW is a START gate, not a stop
|
||||
# threshold.
|
||||
mem_check_and_stop
|
||||
running="$(vision_running)" # refresh: the active stop may have just flipped it
|
||||
# Refresh: the active stop may have just flipped vision_running.
|
||||
# Symmetric ERR handling with the top-of-loop check — without this,
|
||||
# a docker-socket-proxy hiccup between mem_check_and_stop and here
|
||||
# makes `running=ERR`, the next `[ "$running" = "true" ]` test is
|
||||
# false, and `up_for=0` resets — defeating MAX_UPTIME exactly the
|
||||
# way the top-of-loop guard was meant to prevent.
|
||||
running="$(vision_running)"
|
||||
if [ "$running" = "ERR" ]; then
|
||||
log "WARNING: docker inspect failed after mem_check_and_stop; will retry next tick"
|
||||
sleep "$POLL_INTERVAL"
|
||||
continue
|
||||
fi
|
||||
mem_block_start=0
|
||||
if [ "$(mem_yield_active)" = "1" ]; then
|
||||
mem_block_start=1
|
||||
|
||||
Reference in New Issue
Block a user