test(vision-manager): extract refresh_running_after_mem_check; assert log distinguishes fix from bug (0.87.18)
Two 0.87.15 ship-blockers: 1. mem_check_and_stop test now ALSO greps for the "deferring to next tick" warning — distinguishes fix from bug. Mutation-confirmed. 2. Extract `refresh_running_after_mem_check` (sets global instead of echoing so loop-scope flags persist across calls) and drive it directly from the harness, closing the structurally-untestable post-mem refresh ERR guard. Adjacent: log-on-flip gating on both ERR warnings; `log` to stderr; dangling-TempDir in app::tests reclaim test fixed. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -43,7 +43,7 @@ MEM_POLL_INTERVAL="${MEM_POLL_INTERVAL:-5}" # mem sub-poll cadence,
|
||||
|
||||
PSQL=(psql "$DATABASE_URL" -At -v ON_ERROR_STOP=1)
|
||||
|
||||
log() { echo "[vision-manager] $(date -u +%FT%TZ) $*"; }
|
||||
log() { echo "[vision-manager] $(date -u +%FT%TZ) $*" >&2; }
|
||||
|
||||
# ---- Queries ---------------------------------------------------------------
|
||||
# Pending analysis work: pending OR currently-leased analyze_page jobs. Note
|
||||
@@ -178,10 +178,14 @@ mem_check_and_stop() {
|
||||
local vr; vr="$(vision_running)"
|
||||
case "$vr" in
|
||||
ERR)
|
||||
log "WARNING: memory-yield wanted to stop vision (used ${used}% >= ${MEM_HIGH_WATERMARK_PCT}%), but vision_running returned ERR — deferring to next tick"
|
||||
if [ "${mem_check_err_logged:-0}" != "1" ]; then
|
||||
log "WARNING: memory-yield wanted to stop vision (used ${used}% >= ${MEM_HIGH_WATERMARK_PCT}%), but vision_running returned ERR — deferring to next tick"
|
||||
mem_check_err_logged=1
|
||||
fi
|
||||
return 0
|
||||
;;
|
||||
true)
|
||||
mem_check_err_logged=0
|
||||
if stop_vision "memory-yield: used ${used}% >= ${MEM_HIGH_WATERMARK_PCT}% high watermark"; then
|
||||
mem_yield_until=$(( $(date +%s) + MEM_YIELD_COOLDOWN ))
|
||||
idle_for=0
|
||||
@@ -190,10 +194,42 @@ mem_check_and_stop() {
|
||||
;;
|
||||
*)
|
||||
# Not running — nothing to yield from. Cheap no-op.
|
||||
mem_check_err_logged=0
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
# Refresh the cached vision_running after `mem_check_and_stop` may have
|
||||
# stopped the container. Sets the global `REFRESH_RUNNING_RESULT` to the
|
||||
# new running value (the literal `ERR` sentinel when docker inspect
|
||||
# fails) so the caller can short-circuit the tick instead of falling
|
||||
# through into branches whose `[ "$running" = "true" ]` test would treat
|
||||
# ERR as "stopped" (and then reset `up_for=0`, defeating MAX_UPTIME).
|
||||
# Logs the underlying error only on TRANSITION via the loop-scope
|
||||
# `inspect_err_logged` flag so a long-running docker-socket-proxy outage
|
||||
# doesn't flood the log.
|
||||
#
|
||||
# We set a global instead of echoing because the test harness drives
|
||||
# this via direct call (not `$()` subshell); preserving the loop-scope
|
||||
# log-on-flip flag requires staying in the caller's shell. Production
|
||||
# also avoids the subshell hop, which is a small perf win on every tick.
|
||||
#
|
||||
# Extracted from inline so test_manager.sh can drive each branch with
|
||||
# stubbed `docker` on PATH without entering the production loop.
|
||||
refresh_running_after_mem_check() {
|
||||
local r
|
||||
r="$(vision_running)"
|
||||
if [ "$r" = "ERR" ]; then
|
||||
if [ "${inspect_err_logged:-0}" != "1" ]; then
|
||||
log "WARNING: docker inspect failed after mem_check_and_stop; will retry next tick"
|
||||
inspect_err_logged=1
|
||||
fi
|
||||
else
|
||||
inspect_err_logged=0
|
||||
fi
|
||||
REFRESH_RUNNING_RESULT="$r"
|
||||
}
|
||||
|
||||
# Start inhibit: echo 1 while a cooldown is in effect OR used% is at/above the
|
||||
# LOW watermark. The two distinct watermarks give the mandatory hysteresis; the
|
||||
# cooldown gives whatever needed the RAM room to finish before vision restarts.
|
||||
@@ -214,6 +250,8 @@ crawl_deferred=0 # 1 while a start is held off by the crawl mutex (log on flip
|
||||
analysis_off=0 # 1 while analysis is disabled in settings (log on flip only)
|
||||
mem_yield_until=0 # epoch deadline of the active memory-yield cooldown (0 = none)
|
||||
mem_inhibited=0 # 1 while starts are inhibited by memory pressure (log on flip only)
|
||||
inspect_err_logged=0 # 1 once a docker inspect ERR has been warned about (log on flip only)
|
||||
mem_check_err_logged=0 # 1 once mem_check_and_stop has deferred on ERR (log on flip only)
|
||||
|
||||
# Tests source this script with `MANAGER_TEST_NO_MAIN=1` set so they can
|
||||
# exercise the helper functions above with stubbed `docker`/`psql` on PATH
|
||||
@@ -275,15 +313,9 @@ while true; do
|
||||
# LOW (idle stack + LLM ≈ 80% here). LOW is a START gate, not a stop
|
||||
# threshold.
|
||||
mem_check_and_stop
|
||||
# Refresh: the active stop may have just flipped vision_running.
|
||||
# Symmetric ERR handling with the top-of-loop check — without this,
|
||||
# a docker-socket-proxy hiccup between mem_check_and_stop and here
|
||||
# makes `running=ERR`, the next `[ "$running" = "true" ]` test is
|
||||
# false, and `up_for=0` resets — defeating MAX_UPTIME exactly the
|
||||
# way the top-of-loop guard was meant to prevent.
|
||||
running="$(vision_running)"
|
||||
refresh_running_after_mem_check
|
||||
running="$REFRESH_RUNNING_RESULT"
|
||||
if [ "$running" = "ERR" ]; then
|
||||
log "WARNING: docker inspect failed after mem_check_and_stop; will retry next tick"
|
||||
sleep "$POLL_INTERVAL"
|
||||
continue
|
||||
fi
|
||||
|
||||
Reference in New Issue
Block a user