fix(vision-manager): symmetric ERR handling + smoke tests + analysis reload reclaim test (0.87.15)

0.87.7 + 0.87.8 follow-ups:

1. mem_check_and_stop now defers on docker ERR instead of silently
   no-op'ing the SIGTERM under pressure.
2. Post-mem_check_and_stop loop refresh now also handles the ERR
   sentinel (continues the loop) rather than letting it reset up_for
   and defeat MAX_UPTIME.
3. New `vision-manager/test_manager.sh` — 7 smoke tests covering
   vision_running's three exit shapes and mem_check_and_stop's ERR
   symmetry, via stubbed docker/psql/curl on PATH. Production loop
   guarded by MANAGER_TEST_NO_MAIN.
4. New `app::tests::spawn_analysis_daemon_reclaims_orphaned_analyze_leases`
   pins the 0.87.7 wire-through: seeds an expired analyze_page lease,
   spawns + shuts down, asserts pending + attempts refunded.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
MechaCat02
2026-06-23 19:18:04 +02:00
parent 95de02f583
commit 27fc1a52f6
6 changed files with 314 additions and 11 deletions

View File

@@ -164,17 +164,34 @@ stop_vision() {
# SIGTERM it (the friendly, right-victim alternative to an OOM kill) and arm a
# cooldown. Cheap on the hot path — only inspects docker once used% is already
# over the mark. Reads `mem_yield_until`/`idle_for`/`up_for` from the loop scope.
#
# Symmetric ERR handling with the main loop's `running` refresh: an inspect
# blip under genuine memory pressure must NOT silently no-op the SIGTERM.
# When `vision_running` returns `ERR`, log it and skip THIS check rather than
# treating the ambiguous state as "not running" (which would let memory
# pressure go un-yielded until the next tick).
mem_check_and_stop() {
[ "$MEM_YIELD_ENABLED" = "0" ] && return 0
local used; used="$(read_mem_used_pct)"
case "$used" in ''|*[!0-9]*) return 0 ;; esac # ignore a malformed read
if [ "$used" -ge "$MEM_HIGH_WATERMARK_PCT" ] && [ "$(vision_running)" = "true" ]; then
if stop_vision "memory-yield: used ${used}% >= ${MEM_HIGH_WATERMARK_PCT}% high watermark"; then
mem_yield_until=$(( $(date +%s) + MEM_YIELD_COOLDOWN ))
idle_for=0
up_for=0
fi
fi
[ "$used" -lt "$MEM_HIGH_WATERMARK_PCT" ] && return 0
local vr; vr="$(vision_running)"
case "$vr" in
ERR)
log "WARNING: memory-yield wanted to stop vision (used ${used}% >= ${MEM_HIGH_WATERMARK_PCT}%), but vision_running returned ERR — deferring to next tick"
return 0
;;
true)
if stop_vision "memory-yield: used ${used}% >= ${MEM_HIGH_WATERMARK_PCT}% high watermark"; then
mem_yield_until=$(( $(date +%s) + MEM_YIELD_COOLDOWN ))
idle_for=0
up_for=0
fi
;;
*)
# Not running — nothing to yield from. Cheap no-op.
;;
esac
}
# Start inhibit: echo 1 while a cooldown is in effect OR used% is at/above the
@@ -198,6 +215,14 @@ analysis_off=0 # 1 while analysis is disabled in settings (log on flip only)
mem_yield_until=0 # epoch deadline of the active memory-yield cooldown (0 = none)
mem_inhibited=0 # 1 while starts are inhibited by memory pressure (log on flip only)
# Tests source this script with `MANAGER_TEST_NO_MAIN=1` set so they can
# exercise the helper functions above with stubbed `docker`/`psql` on PATH
# without entering the production poll loop. Production callers (and the
# Dockerfile entrypoint) leave the var unset, so the loop runs as before.
if [ -n "${MANAGER_TEST_NO_MAIN:-}" ]; then
return 0 2>/dev/null || exit 0
fi
while true; do
pending="$(pending_analysis)"
running="$(vision_running)"
@@ -250,7 +275,18 @@ while true; do
# LOW (idle stack + LLM ≈ 80% here). LOW is a START gate, not a stop
# threshold.
mem_check_and_stop
running="$(vision_running)" # refresh: the active stop may have just flipped it
# Refresh: the active stop may have just flipped vision_running.
# Symmetric ERR handling with the top-of-loop check — without this,
# a docker-socket-proxy hiccup between mem_check_and_stop and here
# makes `running=ERR`, the next `[ "$running" = "true" ]` test is
# false, and `up_for=0` resets — defeating MAX_UPTIME exactly the
# way the top-of-loop guard was meant to prevent.
running="$(vision_running)"
if [ "$running" = "ERR" ]; then
log "WARNING: docker inspect failed after mem_check_and_stop; will retry next tick"
sleep "$POLL_INTERVAL"
continue
fi
mem_block_start=0
if [ "$(mem_yield_active)" = "1" ]; then
mem_block_start=1