feat(vision-manager): memory-pressure yield gate
Add a fourth gate to the vision autoscaler poll loop: when host memory gets tight, proactively SIGTERM-stop the mangalord-vision container so a transient spike elsewhere (a crawl, CI, a backend burst) can finish instead of the kernel OOM-killer shooting something stateful. - read_mem_used_pct: host-wide used% from /proc/meminfo MemAvailable (not MemFree), ERR sentinel when MemTotal or MemAvailable is missing. - mem_check_and_stop: active stop over MEM_HIGH_WATERMARK_PCT + arm a restart cooldown; only inspects docker once already over the mark. - mem_yield_active: start inhibit while cooldown active or used% >= MEM_LOW_WATERMARK_PCT (two watermarks = hysteresis). - Inhibit reuses the existing idle path via pending=0; flip-only logging; stop_vision gains a reason arg so idle/MAX_UPTIME/memory-yield stops are distinctly logged. - Faster MEM_POLL_INTERVAL sub-poll between backlog ticks to catch spikes. Backend verifications V1-V4 (see VISION-MEMORY-YIELD.md) all pass against the current queue, so no backend changes are needed. RESPECT_CRAWL_MUTEX is kept for now. New VISION_MEM_* knobs wired through docker-compose.yml and documented in .env.example. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
MechaCat02
@@ -30,6 +30,17 @@ MAX_UPTIME="${MAX_UPTIME:-0}" # >0: force-stop after N idle-or-not
|
||||
# together OOM the 8 GiB box). Set to 0 on roomier hosts.
|
||||
RESPECT_CRAWL_MUTEX="${RESPECT_CRAWL_MUTEX:-1}"
|
||||
|
||||
# ---- Memory-pressure yield (see VISION-MEMORY-YIELD.md) --------------------
|
||||
# Dynamic generalization of the crawl mutex: stop a RUNNING vision when the
|
||||
# HOST is actually short on RAM, so a transient spike elsewhere (a crawl, CI,
|
||||
# a backend burst) can finish instead of the kernel OOM-killer shooting
|
||||
# something stateful. Uses host-wide MemAvailable from /proc/meminfo.
|
||||
MEM_YIELD_ENABLED="${MEM_YIELD_ENABLED:-1}" # 0 disables the whole gate
|
||||
MEM_HIGH_WATERMARK_PCT="${MEM_HIGH_WATERMARK_PCT:-92}" # used% >= → active-stop a running vision
|
||||
MEM_LOW_WATERMARK_PCT="${MEM_LOW_WATERMARK_PCT:-80}" # used% >= → inhibit starts (hysteresis)
|
||||
MEM_YIELD_COOLDOWN="${MEM_YIELD_COOLDOWN:-300}" # s after a pressure-stop before restart allowed
|
||||
MEM_POLL_INTERVAL="${MEM_POLL_INTERVAL:-5}" # mem sub-poll cadence, <= POLL_INTERVAL
|
||||
|
||||
PSQL=(psql "$DATABASE_URL" -At -v ON_ERROR_STOP=1)
|
||||
|
||||
log() { echo "[vision-manager] $(date -u +%FT%TZ) $*"; }
|
||||
@@ -53,6 +64,18 @@ crawl_running() {
|
||||
AND state = 'running';" 2>/dev/null || echo "ERR"
|
||||
}
|
||||
|
||||
# ---- Memory pressure -------------------------------------------------------
|
||||
# Host-wide used% from MemAvailable (NOT MemFree, which sits near "full" thanks
|
||||
# to page cache and would false-trigger constantly). /proc/meminfo in this
|
||||
# container reports HOST totals (it is not namespaced under this kernel).
|
||||
# Emits the "ERR" sentinel on a malformed read so callers can ignore it.
|
||||
read_mem_used_pct() {
|
||||
# Require BOTH fields: a partial read, or a pre-3.14 kernel without
|
||||
# MemAvailable, would otherwise leave `a` at 0 → used=100% → a false stop.
|
||||
awk '/^MemTotal:/{t=$2} /^MemAvailable:/{a=$2}
|
||||
END { if (t>0 && a!="") printf "%d", 100*(1-a/t); else print "ERR" }' /proc/meminfo
|
||||
}
|
||||
|
||||
# ---- Docker helpers --------------------------------------------------------
|
||||
# `docker inspect .State.Running` → true/false; empty if the container is
|
||||
# absent (defined elsewhere and not yet created) — treated as not-running.
|
||||
@@ -84,7 +107,8 @@ start_vision() {
|
||||
}
|
||||
|
||||
stop_vision() {
|
||||
log "stopping $VISION_CONTAINER (idle ${STOP_DEBOUNCE}s)"
|
||||
local reason="${1:-idle ${STOP_DEBOUNCE}s}"
|
||||
log "stopping $VISION_CONTAINER ($reason)"
|
||||
if docker stop "$VISION_CONTAINER" >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
@@ -92,12 +116,43 @@ stop_vision() {
|
||||
return 1
|
||||
}
|
||||
|
||||
# ---- Main loop -------------------------------------------------------------
|
||||
log "started: container=$VISION_CONTAINER health=$VISION_HEALTH_URL poll=${POLL_INTERVAL}s debounce=${STOP_DEBOUNCE}s docker_host=$DOCKER_HOST"
|
||||
# ---- Memory-yield gates ----------------------------------------------------
|
||||
# Active stop: when the host is over the HIGH watermark and vision is running,
|
||||
# SIGTERM it (the friendly, right-victim alternative to an OOM kill) and arm a
|
||||
# cooldown. Cheap on the hot path — only inspects docker once used% is already
|
||||
# over the mark. Reads `mem_yield_until`/`idle_for`/`up_for` from the loop scope.
|
||||
mem_check_and_stop() {
|
||||
[ "$MEM_YIELD_ENABLED" = "0" ] && return 0
|
||||
local used; used="$(read_mem_used_pct)"
|
||||
case "$used" in ''|*[!0-9]*) return 0 ;; esac # ignore a malformed read
|
||||
if [ "$used" -ge "$MEM_HIGH_WATERMARK_PCT" ] && [ "$(vision_running)" = "true" ]; then
|
||||
if stop_vision "memory-yield: used ${used}% >= ${MEM_HIGH_WATERMARK_PCT}% high watermark"; then
|
||||
mem_yield_until=$(( $(date +%s) + MEM_YIELD_COOLDOWN ))
|
||||
idle_for=0
|
||||
up_for=0
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
idle_for=0 # seconds the queue has been empty while vision is running
|
||||
up_for=0 # seconds vision has been running (for MAX_UPTIME backstop)
|
||||
crawl_deferred=0 # 1 while a start is held off by the crawl mutex (log on flip only)
|
||||
# Start inhibit: echo 1 while a cooldown is in effect OR used% is at/above the
|
||||
# LOW watermark. The two distinct watermarks give the mandatory hysteresis; the
|
||||
# cooldown gives whatever needed the RAM room to finish before vision restarts.
|
||||
mem_yield_active() {
|
||||
[ "$MEM_YIELD_ENABLED" = "0" ] && { echo 0; return; }
|
||||
[ "$(date +%s)" -lt "$mem_yield_until" ] && { echo 1; return; }
|
||||
local used; used="$(read_mem_used_pct)"
|
||||
case "$used" in ''|*[!0-9]*) echo 0; return ;; esac
|
||||
if [ "$used" -ge "$MEM_LOW_WATERMARK_PCT" ]; then echo 1; else echo 0; fi
|
||||
}
|
||||
|
||||
# ---- Main loop -------------------------------------------------------------
|
||||
log "started: container=$VISION_CONTAINER health=$VISION_HEALTH_URL poll=${POLL_INTERVAL}s debounce=${STOP_DEBOUNCE}s docker_host=$DOCKER_HOST mem_yield=$([ "$MEM_YIELD_ENABLED" = 0 ] && echo off || echo "${MEM_LOW_WATERMARK_PCT}/${MEM_HIGH_WATERMARK_PCT}% cooldown=${MEM_YIELD_COOLDOWN}s mempoll=${MEM_POLL_INTERVAL}s")"
|
||||
|
||||
idle_for=0 # seconds the queue has been empty while vision is running
|
||||
up_for=0 # seconds vision has been running (for MAX_UPTIME backstop)
|
||||
crawl_deferred=0 # 1 while a start is held off by the crawl mutex (log on flip only)
|
||||
mem_yield_until=0 # epoch deadline of the active memory-yield cooldown (0 = none)
|
||||
mem_inhibited=0 # 1 while starts are inhibited by memory pressure (log on flip only)
|
||||
|
||||
while true; do
|
||||
pending="$(pending_analysis)"
|
||||
@@ -114,6 +169,22 @@ while true; do
|
||||
;;
|
||||
esac
|
||||
|
||||
# Memory-yield gates (before the backlog logic). The active stop handles a
|
||||
# running vision the idle path can't; the inhibit forces pending=0 so the
|
||||
# existing idle-debounce path keeps it down (same shape as the crawl mutex).
|
||||
mem_check_and_stop
|
||||
running="$(vision_running)" # refresh: the active stop may have just flipped it
|
||||
if [ "$(mem_yield_active)" = "1" ]; then
|
||||
if [ "$mem_inhibited" != "1" ]; then
|
||||
log "memory-yield: inhibiting vision start (used >= ${MEM_LOW_WATERMARK_PCT}% or cooldown active)"
|
||||
mem_inhibited=1
|
||||
fi
|
||||
pending=0
|
||||
elif [ "$mem_inhibited" = "1" ]; then
|
||||
log "memory-yield: pressure cleared — start inhibit lifted"
|
||||
mem_inhibited=0
|
||||
fi
|
||||
|
||||
if [ "$running" = "true" ]; then
|
||||
up_for=$((up_for + POLL_INTERVAL))
|
||||
else
|
||||
@@ -160,12 +231,24 @@ while true; do
|
||||
|
||||
# Backstop: bound how long vision can stay up regardless of debounce state.
|
||||
if [ "$MAX_UPTIME" -gt 0 ] && [ "$running" = "true" ] && [ "$up_for" -ge "$MAX_UPTIME" ]; then
|
||||
log "MAX_UPTIME ${MAX_UPTIME}s reached; force-stopping $VISION_CONTAINER"
|
||||
if stop_vision; then
|
||||
if stop_vision "MAX_UPTIME ${MAX_UPTIME}s reached"; then
|
||||
idle_for=0
|
||||
up_for=0
|
||||
fi
|
||||
fi
|
||||
|
||||
sleep "$POLL_INTERVAL"
|
||||
# Sleep until the next backlog poll, but re-check memory pressure on the
|
||||
# faster MEM_POLL_INTERVAL sub-cadence so a spike between backlog ticks can
|
||||
# trigger an active stop before the kernel OOM-killer does. Degenerates to a
|
||||
# single sleep when MEM_POLL_INTERVAL >= POLL_INTERVAL.
|
||||
slept=0
|
||||
while [ "$slept" -lt "$POLL_INTERVAL" ]; do
|
||||
step="$MEM_POLL_INTERVAL"
|
||||
remaining=$((POLL_INTERVAL - slept))
|
||||
[ "$step" -gt "$remaining" ] && step="$remaining"
|
||||
[ "$step" -lt 1 ] && step=1
|
||||
sleep "$step"
|
||||
slept=$((slept + step))
|
||||
mem_check_and_stop
|
||||
done
|
||||
done
|
||||
|
||||
Reference in New Issue
Block a user