#!/usr/bin/env bash
# vision-manager — start/stop the mangalord-vision (llama.cpp) container by
# name according to the analysis backlog in Postgres.
#
# Why this exists: llama-server has no idle-unload and pins ~4.4 GiB. On the
# 8 GiB box vision must only run while there is analysis work. This sidecar is
# the SINGLE owner of the vision lifecycle and the only component with Docker
# access (scoped through docker-socket-proxy) — the internet-facing backend
# never gets the socket. See VISION-AUTOSCALE.md.
#
# It is deliberately dumb: a poll loop with psql + curl + docker. No web
# surface, no state beyond an in-memory idle timer. If it dies, vision keeps
# running (harmless RAM); a MAX_UPTIME backstop bounds that.
set -euo pipefail

# ---- Config (all overridable via env) --------------------------------------
: "${DATABASE_URL:?DATABASE_URL must be set (use the read-only vision_manager role)}"
VISION_CONTAINER="${VISION_CONTAINER:-mangalord-vision}"
VISION_HEALTH_URL="${VISION_HEALTH_URL:-http://mangalord-vision:8000/health}"
# DOCKER_HOST points at the scoped docker-socket-proxy by default.
export DOCKER_HOST="${DOCKER_HOST:-tcp://docker-socket-proxy:2375}"

POLL_INTERVAL="${POLL_INTERVAL:-20}"        # seconds between backlog checks
STOP_DEBOUNCE="${STOP_DEBOUNCE:-600}"       # idle seconds before stopping vision
START_HEALTH_TIMEOUT="${START_HEALTH_TIMEOUT:-300}"  # max wait for /health 200
HEALTH_POLL_INTERVAL="${HEALTH_POLL_INTERVAL:-5}"    # poll cadence while warming
HEALTH_TIMEOUT="${HEALTH_TIMEOUT:-5}"                # per-probe curl timeout, seconds
MAX_UPTIME="${MAX_UPTIME:-0}"               # >0: force-stop after N idle-or-not seconds running (backstop); 0 disables
# When >0, refuse to start vision while a crawl is running (Chromium + vision
# together OOM the 8 GiB box). Set to 0 on roomier hosts.
RESPECT_CRAWL_MUTEX="${RESPECT_CRAWL_MUTEX:-1}"

# ---- Memory-pressure yield (see VISION-MEMORY-YIELD.md) --------------------
# Dynamic generalization of the crawl mutex: stop a RUNNING vision when the
# HOST is actually short on RAM, so a transient spike elsewhere (a crawl, CI,
# a backend burst) can finish instead of the kernel OOM-killer shooting
# something stateful. Uses host-wide MemAvailable from /proc/meminfo.
MEM_YIELD_ENABLED="${MEM_YIELD_ENABLED:-1}"             # 0 disables the whole gate
MEM_HIGH_WATERMARK_PCT="${MEM_HIGH_WATERMARK_PCT:-92}"  # used% >= → active-stop a running vision
MEM_LOW_WATERMARK_PCT="${MEM_LOW_WATERMARK_PCT:-80}"    # used% >= → inhibit starts (hysteresis)
MEM_YIELD_COOLDOWN="${MEM_YIELD_COOLDOWN:-300}"         # s after a pressure-stop before restart allowed
MEM_POLL_INTERVAL="${MEM_POLL_INTERVAL:-5}"             # mem sub-poll cadence, <= POLL_INTERVAL

PSQL=(psql "$DATABASE_URL" -At -v ON_ERROR_STOP=1)

log() { echo "[vision-manager] $(date -u +%FT%TZ) $*"; }

# ---- Queries ---------------------------------------------------------------
# Pending analysis work: pending OR currently-leased analyze_page jobs. Note
# the queue column is `state` (not `status`) and the kind lives in the JSONB
# payload — see backend/migrations/0012_crawler.sql.
pending_analysis() {
  "${PSQL[@]}" -c \
    "SELECT count(*) FROM crawler_jobs
      WHERE payload->>'kind' = 'analyze_page'
        AND state IN ('pending','running');" 2>/dev/null || echo "ERR"
}

# A crawl is in-flight if any non-analyze job is currently running.
crawl_running() {
  "${PSQL[@]}" -c \
    "SELECT count(*) FROM crawler_jobs
      WHERE payload->>'kind' <> 'analyze_page'
        AND state = 'running';" 2>/dev/null || echo "ERR"
}

# Is the analysis subsystem enabled in Admin → Settings? The runtime source of
# truth is the `app_settings` row (key='analysis'), NOT the .env seed. When
# analysis is OFF the worker drains nothing, so leftover pending jobs would
# otherwise keep vision pinned forever — we must gate on this too, not just the
# queue depth. Returns the literal 'true'/'false'; 'ERR' (or anything else) on
# query failure is treated as enabled by the caller (fail-open: never yank
# vision out from under a live analysis on a transient DB hiccup).
analysis_enabled() {
  "${PSQL[@]}" -c \
    "SELECT CASE WHEN COALESCE((value->>'enabled')::boolean, false)
                 THEN 'true' ELSE 'false' END
       FROM app_settings WHERE key = 'analysis';" 2>/dev/null || echo "ERR"
}

# ---- Memory pressure -------------------------------------------------------
# Host-wide used% from MemAvailable (NOT MemFree, which sits near "full" thanks
# to page cache and would false-trigger constantly). /proc/meminfo in this
# container reports HOST totals (it is not namespaced under this kernel).
# Emits the "ERR" sentinel on a malformed read so callers can ignore it.
read_mem_used_pct() {
  # Require BOTH fields: a partial read, or a pre-3.14 kernel without
  # MemAvailable, would otherwise leave `a` at 0 → used=100% → a false stop.
  awk '/^MemTotal:/{t=$2} /^MemAvailable:/{a=$2}
       END { if (t>0 && a!="") printf "%d", 100*(1-a/t); else print "ERR" }' /proc/meminfo
}

# ---- Docker helpers --------------------------------------------------------
# `docker inspect .State.Running` → true/false; empty if the container is
# absent (defined elsewhere and not yet created) — treated as not-running.
vision_running() {
  docker inspect -f '{{.State.Running}}' "$VISION_CONTAINER" 2>/dev/null || echo "false"
}

vision_ready() {
  curl -fsS -m "$HEALTH_TIMEOUT" -o /dev/null "$VISION_HEALTH_URL" 2>/dev/null
}

start_vision() {
  log "starting $VISION_CONTAINER"
  # Idempotent: starting an already-running container is a docker no-op.
  if ! docker start "$VISION_CONTAINER" >/dev/null 2>&1; then
    log "ERROR: docker start failed (is $VISION_CONTAINER defined on this host?)"
    return 1
  fi
  local waited=0
  while ! vision_ready; do
    if [ "$waited" -ge "$START_HEALTH_TIMEOUT" ]; then
      log "WARNING: $VISION_CONTAINER not healthy after ${START_HEALTH_TIMEOUT}s; leaving it running"
      return 1
    fi
    sleep "$HEALTH_POLL_INTERVAL"
    waited=$((waited + HEALTH_POLL_INTERVAL))
  done
  log "$VISION_CONTAINER healthy after ~${waited}s"
}

stop_vision() {
  local reason="${1:-idle ${STOP_DEBOUNCE}s}"
  log "stopping $VISION_CONTAINER ($reason)"
  if docker stop "$VISION_CONTAINER" >/dev/null 2>&1; then
    return 0
  fi
  log "WARNING: docker stop failed; will retry next tick"
  return 1
}

# ---- Memory-yield gates ----------------------------------------------------
# Active stop: when the host is over the HIGH watermark and vision is running,
# SIGTERM it (the friendly, right-victim alternative to an OOM kill) and arm a
# cooldown. Cheap on the hot path — only inspects docker once used% is already
# over the mark. Reads `mem_yield_until`/`idle_for`/`up_for` from the loop scope.
mem_check_and_stop() {
  [ "$MEM_YIELD_ENABLED" = "0" ] && return 0
  local used; used="$(read_mem_used_pct)"
  case "$used" in ''|*[!0-9]*) return 0 ;; esac   # ignore a malformed read
  if [ "$used" -ge "$MEM_HIGH_WATERMARK_PCT" ] && [ "$(vision_running)" = "true" ]; then
    if stop_vision "memory-yield: used ${used}% >= ${MEM_HIGH_WATERMARK_PCT}% high watermark"; then
      mem_yield_until=$(( $(date +%s) + MEM_YIELD_COOLDOWN ))
      idle_for=0
      up_for=0
    fi
  fi
}

# Start inhibit: echo 1 while a cooldown is in effect OR used% is at/above the
# LOW watermark. The two distinct watermarks give the mandatory hysteresis; the
# cooldown gives whatever needed the RAM room to finish before vision restarts.
mem_yield_active() {
  [ "$MEM_YIELD_ENABLED" = "0" ] && { echo 0; return; }
  [ "$(date +%s)" -lt "$mem_yield_until" ] && { echo 1; return; }
  local used; used="$(read_mem_used_pct)"
  case "$used" in ''|*[!0-9]*) echo 0; return ;; esac
  if [ "$used" -ge "$MEM_LOW_WATERMARK_PCT" ]; then echo 1; else echo 0; fi
}

# ---- Main loop -------------------------------------------------------------
log "started: container=$VISION_CONTAINER health=$VISION_HEALTH_URL poll=${POLL_INTERVAL}s debounce=${STOP_DEBOUNCE}s docker_host=$DOCKER_HOST mem_yield=$([ "$MEM_YIELD_ENABLED" = 0 ] && echo off || echo "${MEM_LOW_WATERMARK_PCT}/${MEM_HIGH_WATERMARK_PCT}% cooldown=${MEM_YIELD_COOLDOWN}s mempoll=${MEM_POLL_INTERVAL}s")"

idle_for=0        # seconds the queue has been empty while vision is running
up_for=0          # seconds vision has been running (for MAX_UPTIME backstop)
crawl_deferred=0  # 1 while a start is held off by the crawl mutex (log on flip only)
analysis_off=0    # 1 while analysis is disabled in settings (log on flip only)
mem_yield_until=0 # epoch deadline of the active memory-yield cooldown (0 = none)
mem_inhibited=0   # 1 while starts are inhibited by memory pressure (log on flip only)

while true; do
  pending="$(pending_analysis)"
  running="$(vision_running)"

  # Treat anything non-numeric (the "ERR" sentinel, or a stray psql notice on
  # stdout) as "query failed" — never feed it to `-gt`, which under
  # `set -e` would otherwise exit-2 and kill the loop.
  case "$pending" in
    ''|*[!0-9]*)
      log "WARNING: backlog query failed (got '${pending}'); will retry next tick"
      sleep "$POLL_INTERVAL"
      continue
      ;;
  esac

  # Gate on the runtime enable flag. If analysis is OFF, force the backlog to 0
  # so the normal idle path runs the STOP_DEBOUNCE timer and shuts vision down.
  # We deliberately reuse the idle debounce (rather than stopping instantly) so
  # a quick toggle off→on inside the debounce window keeps vision warm — no
  # cold reload for a momentary flip. Fail-open: only a literal 'false' counts
  # as disabled; 'ERR'/empty leaves the real queue depth in effect.
  if [ "$(analysis_enabled)" = "false" ]; then
    if [ "$analysis_off" != "1" ]; then
      log "analysis disabled in settings — ignoring ${pending} queued job(s); idle timer will stop vision"
      analysis_off=1
    fi
    pending=0
  else
    analysis_off=0
  fi

  # Memory-yield gates (before the backlog logic). The active stop handles a
  # running vision the idle path can't; the inhibit forces pending=0 so the
  # existing idle-debounce path keeps it down (same shape as the crawl mutex).
  mem_check_and_stop
  running="$(vision_running)"   # refresh: the active stop may have just flipped it
  if [ "$(mem_yield_active)" = "1" ]; then
    if [ "$mem_inhibited" != "1" ]; then
      log "memory-yield: inhibiting vision start (used >= ${MEM_LOW_WATERMARK_PCT}% or cooldown active)"
      mem_inhibited=1
    fi
    pending=0
  elif [ "$mem_inhibited" = "1" ]; then
    log "memory-yield: pressure cleared — start inhibit lifted"
    mem_inhibited=0
  fi

  if [ "$running" = "true" ]; then
    up_for=$((up_for + POLL_INTERVAL))
  else
    up_for=0
  fi

  if [ "$pending" -gt 0 ]; then
    idle_for=0
    if [ "$running" != "true" ]; then
      if [ "$RESPECT_CRAWL_MUTEX" != "0" ] && [ "$(crawl_running)" != "0" ]; then
        # Log once per deferral episode, not every poll, so a long crawl
        # doesn't flood the log.
        if [ "$crawl_deferred" != "1" ]; then
          log "deferring start: $pending analysis job(s) pending but a crawl is running (RAM mutex)"
          crawl_deferred=1
        fi
      else
        crawl_deferred=0
        start_vision || true
      fi
    else
      crawl_deferred=0
    fi
  else
    crawl_deferred=0
    # No work. Debounce the stop so bursty enqueues don't thrash the load
    # cycle, and own the stop on our OWN timer (never wait for a backend
    # "drained" signal — leak safety).
    if [ "$running" = "true" ]; then
      idle_for=$((idle_for + POLL_INTERVAL))
      if [ "$idle_for" -ge "$STOP_DEBOUNCE" ]; then
        # Only reset the timers if the stop actually took; otherwise let them
        # keep counting so we retry on the next tick rather than waiting out
        # another full debounce window.
        if stop_vision; then
          idle_for=0
          up_for=0
        fi
      fi
    else
      idle_for=0
    fi
  fi

  # Backstop: bound how long vision can stay up regardless of debounce state.
  if [ "$MAX_UPTIME" -gt 0 ] && [ "$running" = "true" ] && [ "$up_for" -ge "$MAX_UPTIME" ]; then
    if stop_vision "MAX_UPTIME ${MAX_UPTIME}s reached"; then
      idle_for=0
      up_for=0
    fi
  fi

  # Sleep until the next backlog poll, but re-check memory pressure on the
  # faster MEM_POLL_INTERVAL sub-cadence so a spike between backlog ticks can
  # trigger an active stop before the kernel OOM-killer does. Degenerates to a
  # single sleep when MEM_POLL_INTERVAL >= POLL_INTERVAL.
  slept=0
  while [ "$slept" -lt "$POLL_INTERVAL" ]; do
    step="$MEM_POLL_INTERVAL"
    remaining=$((POLL_INTERVAL - slept))
    [ "$step" -gt "$remaining" ] && step="$remaining"
    [ "$step" -lt 1 ] && step=1
    sleep "$step"
    slept=$((slept + step))
    mem_check_and_stop
  done
done