Mangalord/vision-manager/test_manager.sh

#!/usr/bin/env bash
# Smoke tests for vision-manager's helper functions. Sources `manager.sh`
# with the `MANAGER_TEST_NO_MAIN=1` guard so the production poll loop is
# skipped, then re-points `docker`/`psql` at stub binaries in a temp dir
# on PATH to drive each branch.
#
# Run: `vision-manager/test_manager.sh` (no arguments).
#
# Covers:
#   * vision_running's three exit shapes (true / false / ERR)
#   * mem_check_and_stop's ERR symmetry — including a positive grep on
#     the "deferring to next tick" warning that DISTINGUISHES the fix
#     from the bug (the prior version of this file only asserted
#     `docker stop` was absent, which is true for BOTH branches)
#   * refresh_running_after_mem_check (extracted in 0.87.18) — the
#     post-mem_check ERR guard that was structurally untestable in the
#     0.87.15 harness because MANAGER_TEST_NO_MAIN=1 skips the loop
#   * log-on-flip gating for the new WARNING lines so a long outage
#     doesn't flood the log
set -euo pipefail

script_dir="$(cd "$(dirname "$0")" && pwd)"

# ---------------------------------------------------------------------------
# Stub harness
# ---------------------------------------------------------------------------

stub_dir="$(mktemp -d)"
trap 'rm -rf "$stub_dir"' EXIT

# `state` file is written by stubs to record calls; read by tests to verify
# argv/exit code combinations.
state_file="$stub_dir/.state"
: > "$state_file"

# `log_file` captures everything `log` emits during a single test so the
# bug-vs-fix-distinguishing tests can grep for the WARNING line that the
# 0.87.15 test failed to assert.
log_file="$stub_dir/.log"
: > "$log_file"

# Configurable stubs: each stub looks for a per-call env var and emits
# whatever the test set. Stubs append to $state_file so a test can assert
# "stop_vision called N times" etc.

cat > "$stub_dir/docker" <<'STUB'
#!/usr/bin/env bash
echo "docker $*" >> "$STUB_STATE"
case "$1" in
  inspect)
    : "${STUB_DOCKER_INSPECT_OUT:=true}"
    : "${STUB_DOCKER_INSPECT_RC:=0}"
    printf '%s' "$STUB_DOCKER_INSPECT_OUT"
    exit "$STUB_DOCKER_INSPECT_RC"
    ;;
  stop|start)
    : "${STUB_DOCKER_RC:=0}"
    exit "$STUB_DOCKER_RC"
    ;;
esac
exit 0
STUB
chmod +x "$stub_dir/docker"

cat > "$stub_dir/psql" <<'STUB'
#!/usr/bin/env bash
echo "psql $*" >> "$STUB_STATE"
: "${STUB_PSQL_OUT:=0}"
: "${STUB_PSQL_RC:=0}"
printf '%s' "$STUB_PSQL_OUT"
exit "$STUB_PSQL_RC"
STUB
chmod +x "$stub_dir/psql"

cat > "$stub_dir/curl" <<'STUB'
#!/usr/bin/env bash
echo "curl $*" >> "$STUB_STATE"
: "${STUB_CURL_RC:=0}"
exit "$STUB_CURL_RC"
STUB
chmod +x "$stub_dir/curl"

# Source manager.sh with the no-main guard. DATABASE_URL is required by
# the script's preamble; a stub value is fine because we override `psql`.
export PATH="$stub_dir:$PATH"
export STUB_STATE="$state_file"
export DATABASE_URL="postgres://stub@stub/stub"
export MANAGER_TEST_NO_MAIN=1
# Disable `set -e` propagation into our own asserts so a failing test
# doesn't kill the harness.
set +e
# shellcheck disable=SC1091
source "$script_dir/manager.sh"
set -e

# ---------------------------------------------------------------------------
# Asserts
# ---------------------------------------------------------------------------

pass=0
fail=0

assert_eq() {
  local got="$1" want="$2" desc="$3"
  if [ "$got" = "$want" ]; then
    pass=$((pass + 1))
    echo "  ok: $desc"
  else
    fail=$((fail + 1))
    echo "  FAIL: $desc"
    echo "    want: ${want}"
    echo "    got:  ${got}"
  fi
}

reset() {
  : > "$state_file"
  : > "$log_file"
  unset STUB_DOCKER_INSPECT_OUT STUB_DOCKER_INSPECT_RC STUB_DOCKER_RC
  unset STUB_PSQL_OUT STUB_PSQL_RC STUB_CURL_RC
  # Reset the loop-scope log-on-flip flags so each test starts from a
  # clean slate. Initially zero per manager.sh's main-loop init block.
  mem_check_err_logged=0
  inspect_err_logged=0
}

# Capture stdout from a function call so a test can assert on what the
# manager logged, not just on what it didn't do. `log` writes to stdout
# (see manager.sh:46); redirect it through this helper to a file and
# return the function's exit code so the chained `$?` is preserved.
run_and_capture_log() {
  "$@" >>"$log_file" 2>&1
}

assert_log_contains() {
  local want="$1" desc="$2"
  if grep -qF -- "$want" "$log_file"; then
    pass=$((pass + 1))
    echo "  ok: $desc"
  else
    fail=$((fail + 1))
    echo "  FAIL: $desc"
    echo "    expected log to contain: ${want}"
    echo "    log was:"
    sed 's/^/      /' "$log_file"
  fi
}

assert_log_count() {
  local pattern="$1" want="$2" desc="$3"
  local got
  got="$(grep -cF -- "$pattern" "$log_file" || true)"
  if [ "$got" = "$want" ]; then
    pass=$((pass + 1))
    echo "  ok: $desc"
  else
    fail=$((fail + 1))
    echo "  FAIL: $desc"
    echo "    pattern '${pattern}' expected count ${want}, got ${got}"
  fi
}

# ---------------------------------------------------------------------------
# Tests
# ---------------------------------------------------------------------------

echo "vision_running: disambiguates running/absent/error"

reset
export STUB_DOCKER_INSPECT_OUT="true" STUB_DOCKER_INSPECT_RC=0
assert_eq "$(vision_running)" "true" "running container → 'true'"

reset
export STUB_DOCKER_INSPECT_OUT="false" STUB_DOCKER_INSPECT_RC=0
assert_eq "$(vision_running)" "false" "stopped container → 'false'"

reset
export STUB_DOCKER_INSPECT_OUT="Error: No such container: mangalord-vision" STUB_DOCKER_INSPECT_RC=1
assert_eq "$(vision_running)" "false" "absent container ('No such container') → 'false' (not ERR)"

reset
export STUB_DOCKER_INSPECT_OUT="Cannot connect to the Docker daemon at tcp://..." STUB_DOCKER_INSPECT_RC=2
assert_eq "$(vision_running)" "ERR" "socket-proxy hiccup (unrelated stderr) → 'ERR' sentinel"

echo
echo "mem_check_and_stop: ERR symmetry under genuine memory pressure"

# Force read_mem_used_pct to report 95% so mem_check_and_stop wants to stop.
# Easiest: override the function in this test shell.
read_mem_used_pct() { echo "95"; }

reset
export STUB_DOCKER_INSPECT_OUT="Cannot connect to the Docker daemon" STUB_DOCKER_INSPECT_RC=2
MEM_YIELD_ENABLED=1 MEM_HIGH_WATERMARK_PCT=90 run_and_capture_log mem_check_and_stop
# Two distinct assertions: no docker stop (the absence) AND the
# "deferring to next tick" warning (the presence). The 0.87.15 test
# checked only the first, which both buggy AND fixed branches satisfy.
# Without the warning assertion, mechanical revert of the ERR carve-out
# to `[ "$vr" = "true" ]` ships green — the rereview verified this.
if grep -q "^docker stop " "$state_file"; then
  fail=$((fail + 1))
  echo "  FAIL: mem_check_and_stop must NOT call docker stop when vision_running returns ERR"
else
  pass=$((pass + 1))
  echo "  ok: mem_check_and_stop defers stop on inspect ERR (no SIGTERM)"
fi
assert_log_contains \
  "but vision_running returned ERR — deferring to next tick" \
  "mem_check_and_stop logs the deferral warning on ERR (distinguishes fix from bug)"

# Log-on-flip: a second mem_check_and_stop call with the same ERR must
# NOT re-log the warning. Mirrors the sibling mem_inhibited/crawl_deferred
# pattern in the production code.
run_and_capture_log mem_check_and_stop
assert_log_count \
  "but vision_running returned ERR — deferring to next tick" \
  "1" \
  "mem_check_and_stop log-on-flip: WARNING doesn't repeat across ticks"

reset
export STUB_DOCKER_INSPECT_OUT="true" STUB_DOCKER_INSPECT_RC=0
MEM_YIELD_ENABLED=1 MEM_HIGH_WATERMARK_PCT=90 run_and_capture_log mem_check_and_stop
if grep -q "^docker stop " "$state_file"; then
  pass=$((pass + 1))
  echo "  ok: mem_check_and_stop stops vision when vision_running='true' and pressure"
else
  fail=$((fail + 1))
  echo "  FAIL: mem_check_and_stop should call docker stop when vision is running under pressure"
fi

reset
export STUB_DOCKER_INSPECT_OUT="false" STUB_DOCKER_INSPECT_RC=0
MEM_YIELD_ENABLED=1 MEM_HIGH_WATERMARK_PCT=90 run_and_capture_log mem_check_and_stop
if grep -q "^docker stop " "$state_file"; then
  fail=$((fail + 1))
  echo "  FAIL: mem_check_and_stop should NOT call docker stop when vision isn't running"
else
  pass=$((pass + 1))
  echo "  ok: mem_check_and_stop no-ops when vision isn't running"
fi

echo
echo "refresh_running_after_mem_check: post-mem ERR guard (extracted helper)"

reset
export STUB_DOCKER_INSPECT_OUT="true" STUB_DOCKER_INSPECT_RC=0
run_and_capture_log refresh_running_after_mem_check
assert_eq "$REFRESH_RUNNING_RESULT" "true" "refresh_running_after_mem_check sets 'true' when running"
if grep -q "WARNING" "$log_file"; then
  fail=$((fail + 1))
  echo "  FAIL: refresh_running_after_mem_check must not log on the happy path"
else
  pass=$((pass + 1))
  echo "  ok: refresh_running_after_mem_check is quiet on the happy path"
fi

reset
export STUB_DOCKER_INSPECT_OUT="Cannot connect to the Docker daemon" STUB_DOCKER_INSPECT_RC=2
run_and_capture_log refresh_running_after_mem_check
assert_eq "$REFRESH_RUNNING_RESULT" "ERR" "refresh_running_after_mem_check sets 'ERR' on docker inspect failure"
assert_log_contains \
  "docker inspect failed after mem_check_and_stop" \
  "refresh_running_after_mem_check warns on ERR (distinguishes fix from bug)"

# Log-on-flip: second ERR call must NOT re-log.
run_and_capture_log refresh_running_after_mem_check
assert_eq "$REFRESH_RUNNING_RESULT" "ERR" "second ERR also sets 'ERR'"
assert_log_count \
  "docker inspect failed after mem_check_and_stop" \
  "1" \
  "refresh_running_after_mem_check log-on-flip: WARNING doesn't repeat across ticks"

# Transition back to OK must reset the flip-flag so a future ERR
# re-logs.
export STUB_DOCKER_INSPECT_OUT="true" STUB_DOCKER_INSPECT_RC=0
run_and_capture_log refresh_running_after_mem_check
assert_eq "$REFRESH_RUNNING_RESULT" "true" "transition to OK resets the helper"
export STUB_DOCKER_INSPECT_OUT="Cannot connect" STUB_DOCKER_INSPECT_RC=2
run_and_capture_log refresh_running_after_mem_check
assert_eq "$REFRESH_RUNNING_RESULT" "ERR" "re-ERR after recovery sets 'ERR'"
assert_log_count \
  "docker inspect failed after mem_check_and_stop" \
  "2" \
  "refresh_running_after_mem_check log-on-flip: re-ERR after recovery DOES log"

echo
echo "Result: ${pass} pass, ${fail} fail"
[ "$fail" -eq 0 ]