feat(vision-manager): memory-pressure yield gate (+ retain analysis gate) (#8)

vision-manager/manager.sh

@@ -30,6 +30,17 @@ MAX_UPTIME="${MAX_UPTIME:-0}"               # >0: force-stop after N idle-or-not
 # together OOM the 8 GiB box). Set to 0 on roomier hosts.
 RESPECT_CRAWL_MUTEX="${RESPECT_CRAWL_MUTEX:-1}"
 
+# ---- Memory-pressure yield (see VISION-MEMORY-YIELD.md) --------------------
+# Dynamic generalization of the crawl mutex: stop a RUNNING vision when the
+# HOST is actually short on RAM, so a transient spike elsewhere (a crawl, CI,
+# a backend burst) can finish instead of the kernel OOM-killer shooting
+# something stateful. Uses host-wide MemAvailable from /proc/meminfo.
+MEM_YIELD_ENABLED="${MEM_YIELD_ENABLED:-1}"             # 0 disables the whole gate
+MEM_HIGH_WATERMARK_PCT="${MEM_HIGH_WATERMARK_PCT:-92}"  # used% >= → active-stop a running vision
+MEM_LOW_WATERMARK_PCT="${MEM_LOW_WATERMARK_PCT:-80}"    # used% >= → inhibit starts (hysteresis)
+MEM_YIELD_COOLDOWN="${MEM_YIELD_COOLDOWN:-300}"         # s after a pressure-stop before restart allowed
+MEM_POLL_INTERVAL="${MEM_POLL_INTERVAL:-5}"             # mem sub-poll cadence, <= POLL_INTERVAL
+
 PSQL=(psql "$DATABASE_URL" -At -v ON_ERROR_STOP=1)
 
 log() { echo "[vision-manager] $(date -u +%FT%TZ) $*"; }
@@ -53,6 +64,32 @@ crawl_running() {
         AND state = 'running';" 2>/dev/null || echo "ERR"
 }
 
+# Is the analysis subsystem enabled in Admin → Settings? The runtime source of
+# truth is the `app_settings` row (key='analysis'), NOT the .env seed. When
+# analysis is OFF the worker drains nothing, so leftover pending jobs would
+# otherwise keep vision pinned forever — we must gate on this too, not just the
+# queue depth. Returns the literal 'true'/'false'; 'ERR' (or anything else) on
+# query failure is treated as enabled by the caller (fail-open: never yank
+# vision out from under a live analysis on a transient DB hiccup).
+analysis_enabled() {
+  "${PSQL[@]}" -c \
+    "SELECT CASE WHEN COALESCE((value->>'enabled')::boolean, false)
+                 THEN 'true' ELSE 'false' END
+       FROM app_settings WHERE key = 'analysis';" 2>/dev/null || echo "ERR"
+}
+
+# ---- Memory pressure -------------------------------------------------------
+# Host-wide used% from MemAvailable (NOT MemFree, which sits near "full" thanks
+# to page cache and would false-trigger constantly). /proc/meminfo in this
+# container reports HOST totals (it is not namespaced under this kernel).
+# Emits the "ERR" sentinel on a malformed read so callers can ignore it.
+read_mem_used_pct() {
+  # Require BOTH fields: a partial read, or a pre-3.14 kernel without
+  # MemAvailable, would otherwise leave `a` at 0 → used=100% → a false stop.
+  awk '/^MemTotal:/{t=$2} /^MemAvailable:/{a=$2}
+       END { if (t>0 && a!="") printf "%d", 100*(1-a/t); else print "ERR" }' /proc/meminfo
+}
+
 # ---- Docker helpers --------------------------------------------------------
 # `docker inspect .State.Running` → true/false; empty if the container is
 # absent (defined elsewhere and not yet created) — treated as not-running.
@@ -84,7 +121,8 @@ start_vision() {
 }
 
 stop_vision() {
-  log "stopping $VISION_CONTAINER (idle ${STOP_DEBOUNCE}s)"
+  local reason="${1:-idle ${STOP_DEBOUNCE}s}"
+  log "stopping $VISION_CONTAINER ($reason)"
   if docker stop "$VISION_CONTAINER" >/dev/null 2>&1; then
     return 0
   fi
@@ -92,12 +130,44 @@ stop_vision() {
   return 1
 }
 
-# ---- Main loop -------------------------------------------------------------
-log "started: container=$VISION_CONTAINER health=$VISION_HEALTH_URL poll=${POLL_INTERVAL}s debounce=${STOP_DEBOUNCE}s docker_host=$DOCKER_HOST"
+# ---- Memory-yield gates ----------------------------------------------------
+# Active stop: when the host is over the HIGH watermark and vision is running,
+# SIGTERM it (the friendly, right-victim alternative to an OOM kill) and arm a
+# cooldown. Cheap on the hot path — only inspects docker once used% is already
+# over the mark. Reads `mem_yield_until`/`idle_for`/`up_for` from the loop scope.
+mem_check_and_stop() {
+  [ "$MEM_YIELD_ENABLED" = "0" ] && return 0
+  local used; used="$(read_mem_used_pct)"
+  case "$used" in ''|*[!0-9]*) return 0 ;; esac   # ignore a malformed read
+  if [ "$used" -ge "$MEM_HIGH_WATERMARK_PCT" ] && [ "$(vision_running)" = "true" ]; then
+    if stop_vision "memory-yield: used ${used}% >= ${MEM_HIGH_WATERMARK_PCT}% high watermark"; then
+      mem_yield_until=$(( $(date +%s) + MEM_YIELD_COOLDOWN ))
+      idle_for=0
+      up_for=0
+    fi
+  fi
+}
 
-idle_for=0       # seconds the queue has been empty while vision is running
-up_for=0         # seconds vision has been running (for MAX_UPTIME backstop)
-crawl_deferred=0 # 1 while a start is held off by the crawl mutex (log on flip only)
+# Start inhibit: echo 1 while a cooldown is in effect OR used% is at/above the
+# LOW watermark. The two distinct watermarks give the mandatory hysteresis; the
+# cooldown gives whatever needed the RAM room to finish before vision restarts.
+mem_yield_active() {
+  [ "$MEM_YIELD_ENABLED" = "0" ] && { echo 0; return; }
+  [ "$(date +%s)" -lt "$mem_yield_until" ] && { echo 1; return; }
+  local used; used="$(read_mem_used_pct)"
+  case "$used" in ''|*[!0-9]*) echo 0; return ;; esac
+  if [ "$used" -ge "$MEM_LOW_WATERMARK_PCT" ]; then echo 1; else echo 0; fi
+}
+
+# ---- Main loop -------------------------------------------------------------
+log "started: container=$VISION_CONTAINER health=$VISION_HEALTH_URL poll=${POLL_INTERVAL}s debounce=${STOP_DEBOUNCE}s docker_host=$DOCKER_HOST mem_yield=$([ "$MEM_YIELD_ENABLED" = 0 ] && echo off || echo "${MEM_LOW_WATERMARK_PCT}/${MEM_HIGH_WATERMARK_PCT}% cooldown=${MEM_YIELD_COOLDOWN}s mempoll=${MEM_POLL_INTERVAL}s")"
+
+idle_for=0        # seconds the queue has been empty while vision is running
+up_for=0          # seconds vision has been running (for MAX_UPTIME backstop)
+crawl_deferred=0  # 1 while a start is held off by the crawl mutex (log on flip only)
+analysis_off=0    # 1 while analysis is disabled in settings (log on flip only)
+mem_yield_until=0 # epoch deadline of the active memory-yield cooldown (0 = none)
+mem_inhibited=0   # 1 while starts are inhibited by memory pressure (log on flip only)
 
 while true; do
   pending="$(pending_analysis)"
@@ -114,6 +184,38 @@ while true; do
       ;;
   esac
 
+  # Gate on the runtime enable flag. If analysis is OFF, force the backlog to 0
+  # so the normal idle path runs the STOP_DEBOUNCE timer and shuts vision down.
+  # We deliberately reuse the idle debounce (rather than stopping instantly) so
+  # a quick toggle off→on inside the debounce window keeps vision warm — no
+  # cold reload for a momentary flip. Fail-open: only a literal 'false' counts
+  # as disabled; 'ERR'/empty leaves the real queue depth in effect.
+  if [ "$(analysis_enabled)" = "false" ]; then
+    if [ "$analysis_off" != "1" ]; then
+      log "analysis disabled in settings — ignoring ${pending} queued job(s); idle timer will stop vision"
+      analysis_off=1
+    fi
+    pending=0
+  else
+    analysis_off=0
+  fi
+
+  # Memory-yield gates (before the backlog logic). The active stop handles a
+  # running vision the idle path can't; the inhibit forces pending=0 so the
+  # existing idle-debounce path keeps it down (same shape as the crawl mutex).
+  mem_check_and_stop
+  running="$(vision_running)"   # refresh: the active stop may have just flipped it
+  if [ "$(mem_yield_active)" = "1" ]; then
+    if [ "$mem_inhibited" != "1" ]; then
+      log "memory-yield: inhibiting vision start (used >= ${MEM_LOW_WATERMARK_PCT}% or cooldown active)"
+      mem_inhibited=1
+    fi
+    pending=0
+  elif [ "$mem_inhibited" = "1" ]; then
+    log "memory-yield: pressure cleared — start inhibit lifted"
+    mem_inhibited=0
+  fi
+
   if [ "$running" = "true" ]; then
     up_for=$((up_for + POLL_INTERVAL))
   else
@@ -160,12 +262,24 @@ while true; do
 
   # Backstop: bound how long vision can stay up regardless of debounce state.
   if [ "$MAX_UPTIME" -gt 0 ] && [ "$running" = "true" ] && [ "$up_for" -ge "$MAX_UPTIME" ]; then
-    log "MAX_UPTIME ${MAX_UPTIME}s reached; force-stopping $VISION_CONTAINER"
-    if stop_vision; then
+    if stop_vision "MAX_UPTIME ${MAX_UPTIME}s reached"; then
       idle_for=0
       up_for=0
     fi
   fi
 
-  sleep "$POLL_INTERVAL"
+  # Sleep until the next backlog poll, but re-check memory pressure on the
+  # faster MEM_POLL_INTERVAL sub-cadence so a spike between backlog ticks can
+  # trigger an active stop before the kernel OOM-killer does. Degenerates to a
+  # single sleep when MEM_POLL_INTERVAL >= POLL_INTERVAL.
+  slept=0
+  while [ "$slept" -lt "$POLL_INTERVAL" ]; do
+    step="$MEM_POLL_INTERVAL"
+    remaining=$((POLL_INTERVAL - slept))
+    [ "$step" -gt "$remaining" ] && step="$remaining"
+    [ "$step" -lt 1 ] && step=1
+    sleep "$step"
+    slept=$((slept + step))
+    mem_check_and_stop
+  done
 done