d6a4fd668c
Two 0.87.15 ship-blockers: 1. mem_check_and_stop test now ALSO greps for the "deferring to next tick" warning — distinguishes fix from bug. Mutation-confirmed. 2. Extract `refresh_running_after_mem_check` (sets global instead of echoing so loop-scope flags persist across calls) and drive it directly from the harness, closing the structurally-untestable post-mem refresh ERR guard. Adjacent: log-on-flip gating on both ERR warnings; `log` to stderr; dangling-TempDir in app::tests reclaim test fixed. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
422 lines
19 KiB
Bash
422 lines
19 KiB
Bash
#!/usr/bin/env bash
|
|
# vision-manager — start/stop the mangalord-vision (llama.cpp) container by
|
|
# name according to the analysis backlog in Postgres.
|
|
#
|
|
# Why this exists: llama-server has no idle-unload and pins ~4.4 GiB. On the
|
|
# 8 GiB box vision must only run while there is analysis work. This sidecar is
|
|
# the SINGLE owner of the vision lifecycle and the only component with Docker
|
|
# access (scoped through docker-socket-proxy) — the internet-facing backend
|
|
# never gets the socket. See VISION-AUTOSCALE.md.
|
|
#
|
|
# It is deliberately dumb: a poll loop with psql + curl + docker. No web
|
|
# surface, no state beyond an in-memory idle timer. If it dies, vision keeps
|
|
# running (harmless RAM); a MAX_UPTIME backstop bounds that.
|
|
set -euo pipefail
|
|
|
|
# ---- Config (all overridable via env) --------------------------------------
|
|
: "${DATABASE_URL:?DATABASE_URL must be set (use the read-only vision_manager role)}"
|
|
VISION_CONTAINER="${VISION_CONTAINER:-mangalord-vision}"
|
|
VISION_HEALTH_URL="${VISION_HEALTH_URL:-http://mangalord-vision:8000/health}"
|
|
# DOCKER_HOST points at the scoped docker-socket-proxy by default.
|
|
export DOCKER_HOST="${DOCKER_HOST:-tcp://docker-socket-proxy:2375}"
|
|
|
|
POLL_INTERVAL="${POLL_INTERVAL:-20}" # seconds between backlog checks
|
|
STOP_DEBOUNCE="${STOP_DEBOUNCE:-600}" # idle seconds before stopping vision
|
|
START_HEALTH_TIMEOUT="${START_HEALTH_TIMEOUT:-300}" # max wait for /health 200
|
|
HEALTH_POLL_INTERVAL="${HEALTH_POLL_INTERVAL:-5}" # poll cadence while warming
|
|
HEALTH_TIMEOUT="${HEALTH_TIMEOUT:-5}" # per-probe curl timeout, seconds
|
|
MAX_UPTIME="${MAX_UPTIME:-0}" # >0: force-stop after N idle-or-not seconds running (backstop); 0 disables
|
|
# When >0, refuse to start vision while a crawl is running (Chromium + vision
|
|
# together OOM the 8 GiB box). Set to 0 on roomier hosts.
|
|
RESPECT_CRAWL_MUTEX="${RESPECT_CRAWL_MUTEX:-1}"
|
|
|
|
# ---- Memory-pressure yield (see VISION-MEMORY-YIELD.md) --------------------
|
|
# Dynamic generalization of the crawl mutex: stop a RUNNING vision when the
|
|
# HOST is actually short on RAM, so a transient spike elsewhere (a crawl, CI,
|
|
# a backend burst) can finish instead of the kernel OOM-killer shooting
|
|
# something stateful. Uses host-wide MemAvailable from /proc/meminfo.
|
|
MEM_YIELD_ENABLED="${MEM_YIELD_ENABLED:-1}" # 0 disables the whole gate
|
|
MEM_HIGH_WATERMARK_PCT="${MEM_HIGH_WATERMARK_PCT:-92}" # used% >= → active-stop a running vision
|
|
MEM_LOW_WATERMARK_PCT="${MEM_LOW_WATERMARK_PCT:-80}" # used% >= → inhibit starts (hysteresis)
|
|
MEM_YIELD_COOLDOWN="${MEM_YIELD_COOLDOWN:-300}" # s after a pressure-stop before restart allowed
|
|
MEM_POLL_INTERVAL="${MEM_POLL_INTERVAL:-5}" # mem sub-poll cadence, <= POLL_INTERVAL
|
|
|
|
PSQL=(psql "$DATABASE_URL" -At -v ON_ERROR_STOP=1)
|
|
|
|
log() { echo "[vision-manager] $(date -u +%FT%TZ) $*" >&2; }
|
|
|
|
# ---- Queries ---------------------------------------------------------------
|
|
# Pending analysis work: pending OR currently-leased analyze_page jobs. Note
|
|
# the queue column is `state` (not `status`) and the kind lives in the JSONB
|
|
# payload — see backend/migrations/0012_crawler.sql.
|
|
pending_analysis() {
|
|
"${PSQL[@]}" -c \
|
|
"SELECT count(*) FROM crawler_jobs
|
|
WHERE payload->>'kind' = 'analyze_page'
|
|
AND state IN ('pending','running');" 2>/dev/null || echo "ERR"
|
|
}
|
|
|
|
# A crawl is in-flight if any non-analyze job is currently running.
|
|
crawl_running() {
|
|
"${PSQL[@]}" -c \
|
|
"SELECT count(*) FROM crawler_jobs
|
|
WHERE payload->>'kind' <> 'analyze_page'
|
|
AND state = 'running';" 2>/dev/null || echo "ERR"
|
|
}
|
|
|
|
# Is the analysis subsystem enabled in Admin → Settings? The runtime source of
|
|
# truth is the `app_settings` row (key='analysis'), NOT the .env seed. When
|
|
# analysis is OFF the worker drains nothing, so leftover pending jobs would
|
|
# otherwise keep vision pinned forever — we must gate on this too, not just the
|
|
# queue depth. Returns the literal 'true'/'false'; 'ERR' (or anything else) on
|
|
# query failure is treated as enabled by the caller (fail-open: never yank
|
|
# vision out from under a live analysis on a transient DB hiccup).
|
|
analysis_enabled() {
|
|
"${PSQL[@]}" -c \
|
|
"SELECT CASE WHEN COALESCE((value->>'enabled')::boolean, false)
|
|
THEN 'true' ELSE 'false' END
|
|
FROM app_settings WHERE key = 'analysis';" 2>/dev/null || echo "ERR"
|
|
}
|
|
|
|
# ---- Memory pressure -------------------------------------------------------
|
|
# Host-wide used% from MemAvailable (NOT MemFree, which sits near "full" thanks
|
|
# to page cache and would false-trigger constantly). /proc/meminfo in this
|
|
# container reports HOST totals (it is not namespaced under this kernel).
|
|
# Emits the "ERR" sentinel on a malformed read so callers can ignore it.
|
|
read_mem_used_pct() {
|
|
# Require BOTH fields: a partial read, or a pre-3.14 kernel without
|
|
# MemAvailable, would otherwise leave `a` at 0 → used=100% → a false stop.
|
|
awk '/^MemTotal:/{t=$2} /^MemAvailable:/{a=$2}
|
|
END { if (t>0 && a!="") printf "%d", 100*(1-a/t); else print "ERR" }' /proc/meminfo
|
|
}
|
|
|
|
# ---- Docker helpers --------------------------------------------------------
|
|
# `docker inspect .State.Running` → true/false. Disambiguate "container
|
|
# absent" (legitimate not-running) from "inspect failed" (transient
|
|
# docker-socket-proxy hiccup) so the loop can skip a tick instead of
|
|
# resetting timers as if vision had just been stopped — without this
|
|
# distinction, an inspect blip every poll resets `up_for` and defeats
|
|
# MAX_UPTIME entirely.
|
|
vision_running() {
|
|
local out rc
|
|
out="$(docker inspect -f '{{.State.Running}}' "$VISION_CONTAINER" 2>&1)"
|
|
rc=$?
|
|
if [ "$rc" -eq 0 ]; then
|
|
# `true` or `false`. Echo as-is.
|
|
echo "$out"
|
|
elif echo "$out" | grep -qE 'No such container|No such object|not found'; then
|
|
# Container is intentionally not defined yet. Treat as not-running.
|
|
echo "false"
|
|
else
|
|
# Transient docker / socket-proxy error. Mirror the psql callers'
|
|
# `ERR` sentinel; the loop short-circuits the tick on this.
|
|
echo "ERR"
|
|
fi
|
|
}
|
|
|
|
vision_ready() {
|
|
curl -fsS -m "$HEALTH_TIMEOUT" -o /dev/null "$VISION_HEALTH_URL" 2>/dev/null
|
|
}
|
|
|
|
start_vision() {
|
|
log "starting $VISION_CONTAINER"
|
|
# Idempotent: starting an already-running container is a docker no-op.
|
|
if ! docker start "$VISION_CONTAINER" >/dev/null 2>&1; then
|
|
log "ERROR: docker start failed (is $VISION_CONTAINER defined on this host?)"
|
|
return 1
|
|
fi
|
|
local waited=0
|
|
while ! vision_ready; do
|
|
if [ "$waited" -ge "$START_HEALTH_TIMEOUT" ]; then
|
|
# Previously: log a warning and leave the container up. That made
|
|
# the wedge permanent — next loop tick sees running=true, the
|
|
# pending>0 branch falls through to `crawl_deferred=0` without
|
|
# retrying start, and `idle_for` never accumulates because
|
|
# `pending>0` resets it. The container stays running forever
|
|
# without ever answering /health.
|
|
#
|
|
# Stop it explicitly: a fresh `docker start` next tick takes the
|
|
# cold-load hit again but at least produces a clean state. If the
|
|
# wedge is persistent the operator sees a start/stop loop in the
|
|
# log instead of a silent hang.
|
|
log "WARNING: $VISION_CONTAINER not healthy after ${START_HEALTH_TIMEOUT}s; stopping so next tick can retry from a clean state"
|
|
stop_vision "unhealthy after ${START_HEALTH_TIMEOUT}s start probe" || true
|
|
return 1
|
|
fi
|
|
sleep "$HEALTH_POLL_INTERVAL"
|
|
waited=$((waited + HEALTH_POLL_INTERVAL))
|
|
done
|
|
log "$VISION_CONTAINER healthy after ~${waited}s"
|
|
}
|
|
|
|
stop_vision() {
|
|
local reason="${1:-idle ${STOP_DEBOUNCE}s}"
|
|
log "stopping $VISION_CONTAINER ($reason)"
|
|
if docker stop "$VISION_CONTAINER" >/dev/null 2>&1; then
|
|
return 0
|
|
fi
|
|
log "WARNING: docker stop failed; will retry next tick"
|
|
return 1
|
|
}
|
|
|
|
# ---- Memory-yield gates ----------------------------------------------------
|
|
# Active stop: when the host is over the HIGH watermark and vision is running,
|
|
# SIGTERM it (the friendly, right-victim alternative to an OOM kill) and arm a
|
|
# cooldown. Cheap on the hot path — only inspects docker once used% is already
|
|
# over the mark. Reads `mem_yield_until`/`idle_for`/`up_for` from the loop scope.
|
|
#
|
|
# Symmetric ERR handling with the main loop's `running` refresh: an inspect
|
|
# blip under genuine memory pressure must NOT silently no-op the SIGTERM.
|
|
# When `vision_running` returns `ERR`, log it and skip THIS check rather than
|
|
# treating the ambiguous state as "not running" (which would let memory
|
|
# pressure go un-yielded until the next tick).
|
|
mem_check_and_stop() {
|
|
[ "$MEM_YIELD_ENABLED" = "0" ] && return 0
|
|
local used; used="$(read_mem_used_pct)"
|
|
case "$used" in ''|*[!0-9]*) return 0 ;; esac # ignore a malformed read
|
|
[ "$used" -lt "$MEM_HIGH_WATERMARK_PCT" ] && return 0
|
|
local vr; vr="$(vision_running)"
|
|
case "$vr" in
|
|
ERR)
|
|
if [ "${mem_check_err_logged:-0}" != "1" ]; then
|
|
log "WARNING: memory-yield wanted to stop vision (used ${used}% >= ${MEM_HIGH_WATERMARK_PCT}%), but vision_running returned ERR — deferring to next tick"
|
|
mem_check_err_logged=1
|
|
fi
|
|
return 0
|
|
;;
|
|
true)
|
|
mem_check_err_logged=0
|
|
if stop_vision "memory-yield: used ${used}% >= ${MEM_HIGH_WATERMARK_PCT}% high watermark"; then
|
|
mem_yield_until=$(( $(date +%s) + MEM_YIELD_COOLDOWN ))
|
|
idle_for=0
|
|
up_for=0
|
|
fi
|
|
;;
|
|
*)
|
|
# Not running — nothing to yield from. Cheap no-op.
|
|
mem_check_err_logged=0
|
|
;;
|
|
esac
|
|
}
|
|
|
|
# Refresh the cached vision_running after `mem_check_and_stop` may have
|
|
# stopped the container. Sets the global `REFRESH_RUNNING_RESULT` to the
|
|
# new running value (the literal `ERR` sentinel when docker inspect
|
|
# fails) so the caller can short-circuit the tick instead of falling
|
|
# through into branches whose `[ "$running" = "true" ]` test would treat
|
|
# ERR as "stopped" (and then reset `up_for=0`, defeating MAX_UPTIME).
|
|
# Logs the underlying error only on TRANSITION via the loop-scope
|
|
# `inspect_err_logged` flag so a long-running docker-socket-proxy outage
|
|
# doesn't flood the log.
|
|
#
|
|
# We set a global instead of echoing because the test harness drives
|
|
# this via direct call (not `$()` subshell); preserving the loop-scope
|
|
# log-on-flip flag requires staying in the caller's shell. Production
|
|
# also avoids the subshell hop, which is a small perf win on every tick.
|
|
#
|
|
# Extracted from inline so test_manager.sh can drive each branch with
|
|
# stubbed `docker` on PATH without entering the production loop.
|
|
refresh_running_after_mem_check() {
|
|
local r
|
|
r="$(vision_running)"
|
|
if [ "$r" = "ERR" ]; then
|
|
if [ "${inspect_err_logged:-0}" != "1" ]; then
|
|
log "WARNING: docker inspect failed after mem_check_and_stop; will retry next tick"
|
|
inspect_err_logged=1
|
|
fi
|
|
else
|
|
inspect_err_logged=0
|
|
fi
|
|
REFRESH_RUNNING_RESULT="$r"
|
|
}
|
|
|
|
# Start inhibit: echo 1 while a cooldown is in effect OR used% is at/above the
|
|
# LOW watermark. The two distinct watermarks give the mandatory hysteresis; the
|
|
# cooldown gives whatever needed the RAM room to finish before vision restarts.
|
|
mem_yield_active() {
|
|
[ "$MEM_YIELD_ENABLED" = "0" ] && { echo 0; return; }
|
|
[ "$(date +%s)" -lt "$mem_yield_until" ] && { echo 1; return; }
|
|
local used; used="$(read_mem_used_pct)"
|
|
case "$used" in ''|*[!0-9]*) echo 0; return ;; esac
|
|
if [ "$used" -ge "$MEM_LOW_WATERMARK_PCT" ]; then echo 1; else echo 0; fi
|
|
}
|
|
|
|
# ---- Main loop -------------------------------------------------------------
|
|
log "started: container=$VISION_CONTAINER health=$VISION_HEALTH_URL poll=${POLL_INTERVAL}s debounce=${STOP_DEBOUNCE}s docker_host=$DOCKER_HOST mem_yield=$([ "$MEM_YIELD_ENABLED" = 0 ] && echo off || echo "${MEM_LOW_WATERMARK_PCT}/${MEM_HIGH_WATERMARK_PCT}% cooldown=${MEM_YIELD_COOLDOWN}s mempoll=${MEM_POLL_INTERVAL}s")"
|
|
|
|
idle_for=0 # seconds the queue has been empty while vision is running
|
|
up_for=0 # seconds vision has been running (for MAX_UPTIME backstop)
|
|
crawl_deferred=0 # 1 while a start is held off by the crawl mutex (log on flip only)
|
|
analysis_off=0 # 1 while analysis is disabled in settings (log on flip only)
|
|
mem_yield_until=0 # epoch deadline of the active memory-yield cooldown (0 = none)
|
|
mem_inhibited=0 # 1 while starts are inhibited by memory pressure (log on flip only)
|
|
inspect_err_logged=0 # 1 once a docker inspect ERR has been warned about (log on flip only)
|
|
mem_check_err_logged=0 # 1 once mem_check_and_stop has deferred on ERR (log on flip only)
|
|
|
|
# Tests source this script with `MANAGER_TEST_NO_MAIN=1` set so they can
|
|
# exercise the helper functions above with stubbed `docker`/`psql` on PATH
|
|
# without entering the production poll loop. Production callers (and the
|
|
# Dockerfile entrypoint) leave the var unset, so the loop runs as before.
|
|
if [ -n "${MANAGER_TEST_NO_MAIN:-}" ]; then
|
|
return 0 2>/dev/null || exit 0
|
|
fi
|
|
|
|
while true; do
|
|
pending="$(pending_analysis)"
|
|
running="$(vision_running)"
|
|
|
|
# Treat anything non-numeric (the "ERR" sentinel, or a stray psql notice on
|
|
# stdout) as "query failed" — never feed it to `-gt`, which under
|
|
# `set -e` would otherwise exit-2 and kill the loop.
|
|
case "$pending" in
|
|
''|*[!0-9]*)
|
|
log "WARNING: backlog query failed (got '${pending}'); will retry next tick"
|
|
sleep "$POLL_INTERVAL"
|
|
continue
|
|
;;
|
|
esac
|
|
|
|
# Same treatment for the docker-inspect status. Without this, a transient
|
|
# docker-socket-proxy hiccup would mis-classify a running vision as stopped
|
|
# — every poll under load would reset `up_for`/`idle_for` and defeat
|
|
# MAX_UPTIME and the idle debounce both. Skip the tick and try again.
|
|
if [ "$running" = "ERR" ]; then
|
|
log "WARNING: docker inspect failed; will retry next tick"
|
|
sleep "$POLL_INTERVAL"
|
|
continue
|
|
fi
|
|
|
|
# Gate on the runtime enable flag. If analysis is OFF, force the backlog to 0
|
|
# so the normal idle path runs the STOP_DEBOUNCE timer and shuts vision down.
|
|
# We deliberately reuse the idle debounce (rather than stopping instantly) so
|
|
# a quick toggle off→on inside the debounce window keeps vision warm — no
|
|
# cold reload for a momentary flip. Fail-open: only a literal 'false' counts
|
|
# as disabled; 'ERR'/empty leaves the real queue depth in effect.
|
|
if [ "$(analysis_enabled)" = "false" ]; then
|
|
if [ "$analysis_off" != "1" ]; then
|
|
log "analysis disabled in settings — ignoring ${pending} queued job(s); idle timer will stop vision"
|
|
analysis_off=1
|
|
fi
|
|
pending=0
|
|
else
|
|
analysis_off=0
|
|
fi
|
|
|
|
# Memory-yield: two SEPARATE levers, never conflated.
|
|
# - active stop (HIGH watermark): mem_check_and_stop SIGTERMs a RUNNING
|
|
# vision when the host is critically short — the ONLY thing that stops a
|
|
# vision that is actively working.
|
|
# - start block (LOW watermark / cooldown): holds off STARTING a stopped
|
|
# vision when memory is already tight, via the mem_block_start flag.
|
|
# It must NOT zero `pending`: doing so made the idle-debounce path read
|
|
# "no work" and idle-stop a vision mid-analysis once the box floated past
|
|
# LOW (idle stack + LLM ≈ 80% here). LOW is a START gate, not a stop
|
|
# threshold.
|
|
mem_check_and_stop
|
|
refresh_running_after_mem_check
|
|
running="$REFRESH_RUNNING_RESULT"
|
|
if [ "$running" = "ERR" ]; then
|
|
sleep "$POLL_INTERVAL"
|
|
continue
|
|
fi
|
|
mem_block_start=0
|
|
if [ "$(mem_yield_active)" = "1" ]; then
|
|
mem_block_start=1
|
|
if [ "$mem_inhibited" != "1" ]; then
|
|
log "memory-yield: blocking vision START (used >= ${MEM_LOW_WATERMARK_PCT}% or cooldown); a running vision keeps working until the ${MEM_HIGH_WATERMARK_PCT}% active stop"
|
|
mem_inhibited=1
|
|
fi
|
|
elif [ "$mem_inhibited" = "1" ]; then
|
|
log "memory-yield: pressure cleared — start block lifted"
|
|
mem_inhibited=0
|
|
fi
|
|
|
|
if [ "$running" = "true" ]; then
|
|
up_for=$((up_for + POLL_INTERVAL))
|
|
else
|
|
up_for=0
|
|
fi
|
|
|
|
if [ "$pending" -gt 0 ]; then
|
|
idle_for=0
|
|
if [ "$running" != "true" ]; then
|
|
if [ "$mem_block_start" = "1" ]; then
|
|
# Memory pressure / cooldown — hold off STARTING (flip already logged).
|
|
# A RUNNING vision never reaches here; idle_for stayed 0 above because
|
|
# pending>0, so a working vision is never idle-stopped under pressure.
|
|
:
|
|
elif [ "$RESPECT_CRAWL_MUTEX" != "0" ] && {
|
|
cr="$(crawl_running)"
|
|
# Numeric guard for symmetry with analysis_enabled() (which
|
|
# fail-OPENs on ERR — see line 74). Without this, a psql
|
|
# blip falls through to the != "0" branch and we log the
|
|
# misleading "RAM mutex" line on every tick of the outage.
|
|
case "$cr" in
|
|
''|*[!0-9]*)
|
|
if [ "$crawl_deferred" != "1" ]; then
|
|
log "WARNING: crawl_running query failed (got '${cr}') — deferring start until next clean read"
|
|
crawl_deferred=1
|
|
fi
|
|
true
|
|
;;
|
|
*) [ "$cr" != "0" ] ;;
|
|
esac
|
|
}; then
|
|
# Log once per deferral episode, not every poll, so a long crawl
|
|
# doesn't flood the log.
|
|
if [ "$crawl_deferred" != "1" ]; then
|
|
log "deferring start: $pending analysis job(s) pending but a crawl is running (RAM mutex)"
|
|
crawl_deferred=1
|
|
fi
|
|
else
|
|
crawl_deferred=0
|
|
start_vision || true
|
|
fi
|
|
else
|
|
crawl_deferred=0
|
|
fi
|
|
else
|
|
crawl_deferred=0
|
|
# No work. Debounce the stop so bursty enqueues don't thrash the load
|
|
# cycle, and own the stop on our OWN timer (never wait for a backend
|
|
# "drained" signal — leak safety).
|
|
if [ "$running" = "true" ]; then
|
|
idle_for=$((idle_for + POLL_INTERVAL))
|
|
if [ "$idle_for" -ge "$STOP_DEBOUNCE" ]; then
|
|
# Only reset the timers if the stop actually took; otherwise let them
|
|
# keep counting so we retry on the next tick rather than waiting out
|
|
# another full debounce window.
|
|
if stop_vision; then
|
|
idle_for=0
|
|
up_for=0
|
|
fi
|
|
fi
|
|
else
|
|
idle_for=0
|
|
fi
|
|
fi
|
|
|
|
# Backstop: bound how long vision can stay up regardless of debounce state.
|
|
if [ "$MAX_UPTIME" -gt 0 ] && [ "$running" = "true" ] && [ "$up_for" -ge "$MAX_UPTIME" ]; then
|
|
if stop_vision "MAX_UPTIME ${MAX_UPTIME}s reached"; then
|
|
idle_for=0
|
|
up_for=0
|
|
fi
|
|
fi
|
|
|
|
# Sleep until the next backlog poll, but re-check memory pressure on the
|
|
# faster MEM_POLL_INTERVAL sub-cadence so a spike between backlog ticks can
|
|
# trigger an active stop before the kernel OOM-killer does. Degenerates to a
|
|
# single sleep when MEM_POLL_INTERVAL >= POLL_INTERVAL.
|
|
slept=0
|
|
while [ "$slept" -lt "$POLL_INTERVAL" ]; do
|
|
step="$MEM_POLL_INTERVAL"
|
|
remaining=$((POLL_INTERVAL - slept))
|
|
[ "$step" -gt "$remaining" ] && step="$remaining"
|
|
[ "$step" -lt 1 ] && step=1
|
|
sleep "$step"
|
|
slept=$((slept + step))
|
|
mem_check_and_stop
|
|
done
|
|
done
|