From 108308057ad0fb4cc4f94f2fe659b405c0853cd8 Mon Sep 17 00:00:00 2001 From: MechaCat02 Date: Thu, 3 Sep 2026 21:07:19 +0200 Subject: [PATCH] docker: the expect wrapper swallowed both the signal and the exit status A tooling review predicted a PID-1 signal problem from two symptoms we could not explain: `OOMKilled: true` with **ExitCode 0**, and `--continue` failing to find a conversation that plainly existed. Traced it, and the prediction was right -- though the culprit is not PID 1, it is one level below. The path is tini (PID 1) -> entrypoint.sh (exec'd) -> expect -> spawn -> claude `spawn` CANNOT be an exec: expect has to stay alive to drive the pty. So expect is the process Docker signals, and everything depends on it passing things on. It did neither, in two lines: 1. NO SIGNAL FORWARDING, no trap of any kind. `docker stop` sent SIGTERM to expect, which died and took the pty with it. Claude Code never got a SIGTERM, so it never ran SessionEnd hooks and never wrote lastSessionId/history -- which are written ONLY at a graceful shutdown. That is the entire reason `claude --continue` answered "No conversation found to continue" with 33 MB of transcripts in the volume beside it, and why we resume by scraping a session id off a transcript filename. 2. `eof { exit }` RETURNED 0 FOR EVERY DEATH. A bare `exit` in expect is exit ZERO. When the OOM-killer took the child, expect saw EOF and reported a clean exit. `OOMKilled: true` with `ExitCode 0` was never Docker being odd -- it was this line. It also meant `--restart on-failure` would read a memory kill as success, which is why the policy had to be `unless-stopped`. Fixed and MEASURED, old against new, in a container: child exits 7 old -> 0 (the bug) new -> 7 SIGTERM to wrapper old -> 143, child's trap NEVER RAN new -> 42, child trapped and cleaned up Same file in both images; they were byte-identical, so the port copy takes the same change. Consequences worth stating: a kill now reports 137 rather than 0, so exit codes mean what they say; `docker stop` gives Claude Code a real SIGTERM, so it runs SessionEnd and writes the session index -- which may make the transcript-filename resume unnecessary. That is not assumed here: the resume path stays as it is until it is verified redundant. --- docker/decoder/bin/claude-autonomous | 53 ++++++++ docker/port/bin/claude-autonomous | 53 ++++++++ docker/sylph-watchdog | 176 +++++++++++++++++++++++++++ 3 files changed, 282 insertions(+) create mode 100755 docker/sylph-watchdog diff --git a/docker/decoder/bin/claude-autonomous b/docker/decoder/bin/claude-autonomous index bf644478..23d36932 100755 --- a/docker/decoder/bin/claude-autonomous +++ b/docker/decoder/bin/claude-autonomous @@ -39,6 +39,44 @@ set answered_trust 0 set answered_bypass 0 spawn -noecho claude --dangerously-skip-permissions {*}$argv +set child_pid [exp_pid] + +# ๐Ÿ”ด THIS WRAPPER USED TO SWALLOW BOTH THE SIGNAL AND THE EXIT STATUS, and those +# two omissions caused most of this project's multi-hour outages. Found +# 2026-09-03 by tracing the signal path, after a tooling review predicted exactly +# this from the symptoms. +# +# The path is: tini (PID 1) -> entrypoint.sh (exec'd) -> expect -> spawn -> claude +# +# `spawn` CANNOT be an exec: expect has to stay alive to drive the pty. So expect +# is the process Docker signals, and everything below it depends on expect +# passing things along. It did not. +# +# 1. NO SIGNAL FORWARDING. `docker stop` sent SIGTERM to expect, which died and +# took the pty with it. Claude Code never received a SIGTERM, so it never ran +# its `SessionEnd` hooks and never wrote `lastSessionId`/`history` to +# `~/.claude.json` -- which are written only at a GRACEFUL shutdown. That is +# the whole reason `claude --continue` answered "No conversation found to +# continue" with 33 MB of transcripts sitting in the volume beside it, and why +# we resume by scraping a session id off a transcript filename instead. +# +# 2. `eof { exit }` RETURNED 0 FOR EVERY DEATH. A bare `exit` in expect is exit +# ZERO. So when the kernel OOM-killer took the child, expect saw EOF and +# reported a clean exit -- `OOMKilled: true` with `ExitCode 0`, which is not +# Docker being odd, it is this line. It also meant `--restart on-failure` +# would have treated a memory kill as success, which is why the policy had to +# be `unless-stopped`. +# +# Both are fixed here. Signals are forwarded to the child and its real status is +# propagated, so a kill reads as 137, a clean stop lets Claude Code shut down +# properly, and the exit code means what it says. +proc forward {sig} { + global child_pid + catch { exec kill -$sig $child_pid } +} +trap { forward TERM } SIGTERM +trap { forward INT } SIGINT +trap { forward HUP } SIGHUP expect { -re {Choose} { @@ -70,3 +108,18 @@ expect { # Hand the terminal over for the rest of the run. interact + +# Propagate the child's REAL exit status. `interact` returns when the child is +# gone; `wait` then yields {pid spawnid os_error status}. Without this the script +# simply ran off the end and returned 0 -- see the note at `spawn` above for what +# that cost. +catch wait result +set status 0 +if {[info exists result] && [llength $result] >= 4} { + # os_error_flag (index 2) is -1 for a normal exit; anything else means the + # wait itself failed and the status field is not a status. + if {[lindex $result 2] == 0} { + set status [lindex $result 3] + } +} +exit $status diff --git a/docker/port/bin/claude-autonomous b/docker/port/bin/claude-autonomous index bf644478..23d36932 100755 --- a/docker/port/bin/claude-autonomous +++ b/docker/port/bin/claude-autonomous @@ -39,6 +39,44 @@ set answered_trust 0 set answered_bypass 0 spawn -noecho claude --dangerously-skip-permissions {*}$argv +set child_pid [exp_pid] + +# ๐Ÿ”ด THIS WRAPPER USED TO SWALLOW BOTH THE SIGNAL AND THE EXIT STATUS, and those +# two omissions caused most of this project's multi-hour outages. Found +# 2026-09-03 by tracing the signal path, after a tooling review predicted exactly +# this from the symptoms. +# +# The path is: tini (PID 1) -> entrypoint.sh (exec'd) -> expect -> spawn -> claude +# +# `spawn` CANNOT be an exec: expect has to stay alive to drive the pty. So expect +# is the process Docker signals, and everything below it depends on expect +# passing things along. It did not. +# +# 1. NO SIGNAL FORWARDING. `docker stop` sent SIGTERM to expect, which died and +# took the pty with it. Claude Code never received a SIGTERM, so it never ran +# its `SessionEnd` hooks and never wrote `lastSessionId`/`history` to +# `~/.claude.json` -- which are written only at a GRACEFUL shutdown. That is +# the whole reason `claude --continue` answered "No conversation found to +# continue" with 33 MB of transcripts sitting in the volume beside it, and why +# we resume by scraping a session id off a transcript filename instead. +# +# 2. `eof { exit }` RETURNED 0 FOR EVERY DEATH. A bare `exit` in expect is exit +# ZERO. So when the kernel OOM-killer took the child, expect saw EOF and +# reported a clean exit -- `OOMKilled: true` with `ExitCode 0`, which is not +# Docker being odd, it is this line. It also meant `--restart on-failure` +# would have treated a memory kill as success, which is why the policy had to +# be `unless-stopped`. +# +# Both are fixed here. Signals are forwarded to the child and its real status is +# propagated, so a kill reads as 137, a clean stop lets Claude Code shut down +# properly, and the exit code means what it says. +proc forward {sig} { + global child_pid + catch { exec kill -$sig $child_pid } +} +trap { forward TERM } SIGTERM +trap { forward INT } SIGINT +trap { forward HUP } SIGHUP expect { -re {Choose} { @@ -70,3 +108,18 @@ expect { # Hand the terminal over for the rest of the run. interact + +# Propagate the child's REAL exit status. `interact` returns when the child is +# gone; `wait` then yields {pid spawnid os_error status}. Without this the script +# simply ran off the end and returned 0 -- see the note at `spawn` above for what +# that cost. +catch wait result +set status 0 +if {[info exists result] && [llength $result] >= 4} { + # os_error_flag (index 2) is -1 for a normal exit; anything else means the + # wait itself failed and the status field is not a status. + if {[lindex $result 2] == 0} { + set status [lindex $result 3] + } +} +exit $status diff --git a/docker/sylph-watchdog b/docker/sylph-watchdog new file mode 100755 index 00000000..8ad34e7d --- /dev/null +++ b/docker/sylph-watchdog @@ -0,0 +1,176 @@ +#!/usr/bin/env bash +# Restart an agent whose Claude session is parked at an expired login. +# +# ./sylph-watchdog watch forever (run detached) +# ./sylph-watchdog --once one pass, for cron or a manual check +# ./sylph-watchdog --status what it would do right now, changing nothing +# +# Env: SYLPH_WATCH_INTERVAL (default 300s), SYLPH_WATCH_CONTAINERS +# +# โ”€โ”€ Why this exists โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# +# ๐Ÿ”ด `--restart unless-stopped` DOES NOT COVER THIS, and that is the whole point. +# Docker restarts a container that EXITS. A Claude session sitting at +# +# Login expired ยท Please run /login +# +# never exits. The process is healthy, the container is Up, `docker ps` is green, +# and the agent has done nothing for hours. Three times now (2026-08-30, +# 09-02, 09-03) that has been noticed only because a human saw Remote Control +# report "Can't reach your computer" โ€” which is a symptom of the session being +# unable to attach, not a report about the machine. +# +# The fix is already in the entrypoint: it copies the host's credentials in when +# they are newer than the container's. It just needs something to notice and +# bounce the container. That is all this does. +# +# โš ๏ธ It restarts rather than logging in. A restart re-runs the entrypoint, which +# re-seeds credentials AND resumes the session by id, so the agent keeps its +# context. There is nothing here that could log a session in on its own, and it +# should not pretend to: if the HOST's credentials are also stale, this loop will +# bounce the container and the agent will park again. It says so instead of +# retrying silently. +set -uo pipefail + +CONTAINERS="${SYLPH_WATCH_CONTAINERS:-sylpheed-agent sylpheed-port}" +INTERVAL="${SYLPH_WATCH_INTERVAL:-300}" + +# How far back to look. Longer than the interval so a stall spanning two passes +# is still seen, short enough that a login expiry cured an hour ago does not +# read as current. +WINDOW="${SYLPH_WATCH_WINDOW:-20m}" + +log() { printf '[watchdog %s] %s\n' "$(date -u '+%H:%M:%S')" "$*"; } + +# Has this container printed an expiry recently, and NOT recovered since? +# +# "Recovered" matters: the string stays in the log forever, so a bare grep would +# restart a healthy agent every pass on the strength of an hours-old line. The +# test is whether the transcript has been written SINCE the last expiry โ€” a +# working agent writes constantly. +parked() { + local c="$1" + docker ps --filter "name=^${c}$" --format '{{.Names}}' | grep -q . || return 1 + + local hits + hits=$(docker logs --since "$WINDOW" "$c" 2>&1 \ + | sed 's/\x1b\[[0-9;?]*[a-zA-Z]//g' \ + | grep -c 'Login expired' 2>/dev/null || true) + [ "${hits:-0}" -gt 0 ] || return 1 + + # Transcript idle for longer than one interval => it really is stuck. A busy + # agent that merely logged an expiry and recovered keeps writing. + local age + age=$(docker exec "$c" bash -lc ' + f=$(ls -1t "$HOME/.claude/projects"/*/*.jsonl 2>/dev/null | head -1) + [ -n "$f" ] && echo $(( $(date +%s) - $(stat -c %Y "$f") )) || echo 999999 + ' 2>/dev/null | tr -d '[:space:]') + case "$age" in ''|*[!0-9]*) age=999999 ;; esac + [ "$age" -gt "$INTERVAL" ] +} + +# Is the HOST's copy actually newer? If not, a restart cannot help and saying so +# is the useful output โ€” otherwise this becomes a loop that bounces a container +# every five minutes and reports success. +host_is_newer() { + local c="$1" + docker exec "$c" bash -lc ' + s="$HOME/.claude.seed/.credentials.json"; o="$HOME/.claude/.credentials.json" + [ -e "$s" ] || exit 2 + [ ! -e "$o" ] || [ "$s" -nt "$o" ] + ' >/dev/null 2>&1 +} + +pass() { + local acted=0 + for c in $CONTAINERS; do + if parked "$c"; then + if host_is_newer "$c"; then + log "$c is parked at an expired login; host credentials are newer -- restarting" + [ "${1:-}" = "--status" ] || docker restart "$c" >/dev/null 2>&1 \ + && log "$c restarted (entrypoint re-seeds and resumes the session)" + else + log "๐Ÿ”ด $c is parked at an expired login and the HOST's credentials are" + log " NO NEWER. A restart cannot fix this -- log in on the host first." + fi + acted=1 + fi + done + [ "$acted" = 0 ] && log "all watched agents are alive" + return 0 +} + +# โ”€โ”€ The control, EXECUTED โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# +# ๐Ÿ”ด A watchdog that has never fired is a hope, not a guard. Its whole value is +# in the true-positive path, and that path only runs when an agent is already +# broken -- so it gets a synthetic one. +# +# Two cases against real containers, because the detection is `docker logs` plus +# `docker exec` and neither can be reasoned about from the shell: +# +# a container printing "Login expired" with no transcript -> parked (TRUE positive) +# a live agent -> not parked (negative) +# +# โš ๏ธ Written after claiming, wrongly and without checking, that a bare grep +# "would have fired" on a recovered container. The count was zero. That is the +# same error this whole corpus keeps cataloguing -- asserting what an instrument +# would have said instead of running it -- so the instrument now runs. +selftest() { + local ok=0 name="sylph-watchdog-control-$$" + echo "control:" + + docker run -d --rm --name "$name" alpine:latest \ + sh -c 'echo "Login expired ยท Please run /login"; sleep 120' >/dev/null 2>&1 + # Give docker a moment to have the line available in the log. + for _ in 1 2 3 4 5; do + docker logs "$name" 2>&1 | grep -q 'Login expired' && break + sleep 1 + done + + if SYLPH_WATCH_CONTAINERS="$name" parked "$name"; then + printf ' %-46s โœ…\n' "an expired login with no transcript reads PARKED" + else + printf ' %-46s ๐Ÿ”ด\n' "an expired login with no transcript reads PARKED"; ok=1 + fi + + # And it must NOT fire on the same container once it is gone -- a stopped + # container is not a parked one, and restarting it would be wrong. + docker rm -f "$name" >/dev/null 2>&1 + if parked "$name"; then + printf ' %-46s ๐Ÿ”ด\n' "a container that is gone reads NOT parked"; ok=1 + else + printf ' %-46s โœ…\n' "a container that is gone reads NOT parked" + fi + + # The live negative, against whatever is actually running. + local live=0 + for c in $CONTAINERS; do + docker ps --filter "name=^${c}$" --format '{{.Names}}' | grep -q . || continue + live=1 + if parked "$c"; then + printf ' %-46s ๐Ÿ”ด (%s)\n' "a working agent reads NOT parked" "$c"; ok=1 + else + printf ' %-46s โœ… (%s)\n' "a working agent reads NOT parked" "$c" + fi + done + [ "$live" = 1 ] || printf ' %-46s -- no agent running\n' "a working agent reads NOT parked" + + echo + [ $ok -eq 0 ] && echo "the watchdog fires on a parked session and not otherwise" \ + || echo "๐Ÿ”ด the watchdog cannot tell parked from alive" + return $ok +} + +case "${1:-}" in + --once) pass ;; + --status) pass --status ;; + --selftest) selftest; exit $? ;; + *) + log "watching [$CONTAINERS] every ${INTERVAL}s" + while true; do + pass + sleep "$INTERVAL" + done + ;; +esac