test(vision-manager): extract refresh_running_after_mem_check; assert log distinguishes fix from bug (0.87.18)

Two 0.87.15 ship-blockers:

1. mem_check_and_stop test now ALSO greps for the "deferring to next
   tick" warning — distinguishes fix from bug. Mutation-confirmed.

2. Extract `refresh_running_after_mem_check` (sets global instead of
   echoing so loop-scope flags persist across calls) and drive it
   directly from the harness, closing the structurally-untestable
   post-mem refresh ERR guard.

Adjacent: log-on-flip gating on both ERR warnings; `log` to stderr;
dangling-TempDir in app::tests reclaim test fixed.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
MechaCat02
2026-06-23 20:17:47 +02:00
parent e3d16e49b7
commit d6a4fd668c
6 changed files with 177 additions and 27 deletions

View File

@@ -43,7 +43,7 @@ MEM_POLL_INTERVAL="${MEM_POLL_INTERVAL:-5}" # mem sub-poll cadence,
PSQL=(psql "$DATABASE_URL" -At -v ON_ERROR_STOP=1)
log() { echo "[vision-manager] $(date -u +%FT%TZ) $*"; }
log() { echo "[vision-manager] $(date -u +%FT%TZ) $*" >&2; }
# ---- Queries ---------------------------------------------------------------
# Pending analysis work: pending OR currently-leased analyze_page jobs. Note
@@ -178,10 +178,14 @@ mem_check_and_stop() {
local vr; vr="$(vision_running)"
case "$vr" in
ERR)
log "WARNING: memory-yield wanted to stop vision (used ${used}% >= ${MEM_HIGH_WATERMARK_PCT}%), but vision_running returned ERR — deferring to next tick"
if [ "${mem_check_err_logged:-0}" != "1" ]; then
log "WARNING: memory-yield wanted to stop vision (used ${used}% >= ${MEM_HIGH_WATERMARK_PCT}%), but vision_running returned ERR — deferring to next tick"
mem_check_err_logged=1
fi
return 0
;;
true)
mem_check_err_logged=0
if stop_vision "memory-yield: used ${used}% >= ${MEM_HIGH_WATERMARK_PCT}% high watermark"; then
mem_yield_until=$(( $(date +%s) + MEM_YIELD_COOLDOWN ))
idle_for=0
@@ -190,10 +194,42 @@ mem_check_and_stop() {
;;
*)
# Not running — nothing to yield from. Cheap no-op.
mem_check_err_logged=0
;;
esac
}
# Refresh the cached vision_running after `mem_check_and_stop` may have
# stopped the container. Sets the global `REFRESH_RUNNING_RESULT` to the
# new running value (the literal `ERR` sentinel when docker inspect
# fails) so the caller can short-circuit the tick instead of falling
# through into branches whose `[ "$running" = "true" ]` test would treat
# ERR as "stopped" (and then reset `up_for=0`, defeating MAX_UPTIME).
# Logs the underlying error only on TRANSITION via the loop-scope
# `inspect_err_logged` flag so a long-running docker-socket-proxy outage
# doesn't flood the log.
#
# We set a global instead of echoing because the test harness drives
# this via direct call (not `$()` subshell); preserving the loop-scope
# log-on-flip flag requires staying in the caller's shell. Production
# also avoids the subshell hop, which is a small perf win on every tick.
#
# Extracted from inline so test_manager.sh can drive each branch with
# stubbed `docker` on PATH without entering the production loop.
refresh_running_after_mem_check() {
local r
r="$(vision_running)"
if [ "$r" = "ERR" ]; then
if [ "${inspect_err_logged:-0}" != "1" ]; then
log "WARNING: docker inspect failed after mem_check_and_stop; will retry next tick"
inspect_err_logged=1
fi
else
inspect_err_logged=0
fi
REFRESH_RUNNING_RESULT="$r"
}
# Start inhibit: echo 1 while a cooldown is in effect OR used% is at/above the
# LOW watermark. The two distinct watermarks give the mandatory hysteresis; the
# cooldown gives whatever needed the RAM room to finish before vision restarts.
@@ -214,6 +250,8 @@ crawl_deferred=0 # 1 while a start is held off by the crawl mutex (log on flip
analysis_off=0 # 1 while analysis is disabled in settings (log on flip only)
mem_yield_until=0 # epoch deadline of the active memory-yield cooldown (0 = none)
mem_inhibited=0 # 1 while starts are inhibited by memory pressure (log on flip only)
inspect_err_logged=0 # 1 once a docker inspect ERR has been warned about (log on flip only)
mem_check_err_logged=0 # 1 once mem_check_and_stop has deferred on ERR (log on flip only)
# Tests source this script with `MANAGER_TEST_NO_MAIN=1` set so they can
# exercise the helper functions above with stubbed `docker`/`psql` on PATH
@@ -275,15 +313,9 @@ while true; do
# LOW (idle stack + LLM ≈ 80% here). LOW is a START gate, not a stop
# threshold.
mem_check_and_stop
# Refresh: the active stop may have just flipped vision_running.
# Symmetric ERR handling with the top-of-loop check — without this,
# a docker-socket-proxy hiccup between mem_check_and_stop and here
# makes `running=ERR`, the next `[ "$running" = "true" ]` test is
# false, and `up_for=0` resets — defeating MAX_UPTIME exactly the
# way the top-of-loop guard was meant to prevent.
running="$(vision_running)"
refresh_running_after_mem_check
running="$REFRESH_RUNNING_RESULT"
if [ "$running" = "ERR" ]; then
log "WARNING: docker inspect failed after mem_check_and_stop; will retry next tick"
sleep "$POLL_INTERVAL"
continue
fi