From e1749c83e1cf38d30e0e45b728fb516204611d56 Mon Sep 17 00:00:00 2001 From: MechaCat02 Date: Tue, 1 Sep 2026 20:20:51 +0200 Subject: [PATCH] docker: auto-restart, and resume the session the agent was actually in The decoder died mid-task and it took four separate findings to explain, each of which read as something else: 1. OOM-KILLED, REPORTED AS A CLEAN EXIT. `OOMKilled: true` with **ExitCode 0**. So `--restart on-failure` would treat a memory kill as a successful finish and leave the agent down -- the policy has to be `unless-stopped`. 2. THE JOB CAP WAS SET AND THEN REMOVED THREE LINES LATER. build-reborn has always exported CARGO_BUILD_JOBS, but a raw `cargo test --release -p sylpheed-formats` never reaches the wrapper. Adding `-e CARGO_BUILD_JOBS` to the launcher did not help either: the entrypoint recomputes and exports over it unconditionally. An explicit value now wins, and says so in the log. 3. THE MEMORY CONSTANT WAS WRONG. `mem_gib * 2 / 3` assumes ~1.5 GB per job; release rustc on this workspace needs ~2 GB, and 4 jobs in 6 GB is what died. Divisor is now 2. 4. `--continue` CANNOT RESUME AN ABRUPT DEATH, which is the only kind we get. It resolves through ~/.claude.json's per-project `history`/`lastSessionId`, and MEASURED mid-session both are None -- they are written at a graceful shutdown. A killed container never writes them, so `--continue` answered "No conversation found to continue" with 33 MB of transcripts in the volume beside it. Persisting .claude.json did not help, because the fields were never populated in the first place; that attempt is removed rather than left in looking useful. The TRANSCRIPTS are durable and named by session id, so the entrypoint reads the id off the newest one for its cwd and passes `--resume `. Verified on both agents: each reattached to its exact prior session and appended to the same file rather than opening a new one. The /loop prompt is still passed alongside `--resume`, so the loop is RE-ARMED rather than merely restored -- a resumed conversation with no wake-up scheduled answers once and stops, which looks like resuming and is not. Restarting into the same death is guarded at the other end: a start less than 120 s after the previous one begins FRESH instead of continuing back into whatever killed it. That fired correctly during this work. On resume the agent is told it was restarted, that its in-progress work is uncommitted in the tree, that any build or capture it had running did not finish and its absence is not a result, and which wrapper to prefer over a raw release build. --- docker/decoder/entrypoint.sh | 94 +++++++++++++++++++++++++++++++++++- docker/decoder/sylph-decoder | 25 +++++++++- docker/port/entrypoint.sh | 68 ++++++++++++++++++++++++++ docker/port/sylph-port | 13 ++++- 4 files changed, 196 insertions(+), 4 deletions(-) diff --git a/docker/decoder/entrypoint.sh b/docker/decoder/entrypoint.sh index da5ce0e5..dee593f8 100755 --- a/docker/decoder/entrypoint.sh +++ b/docker/decoder/entrypoint.sh @@ -84,11 +84,26 @@ if [ -r /sys/fs/cgroup/memory.max ]; then [ "$_m" != max ] && mem_gib=$(( _m / 1073741824 )) fi [ "${mem_gib:-0}" -lt 1 ] && mem_gib=1 -by_mem=$(( mem_gib * 2 / 3 )) +# 🔴 THIS CONSTANT WAS WRONG, AND IT COST A RUN. `mem_gib * 2 / 3` assumes +# ~1.5 GB per job. On 2026-09-01 a raw `cargo test --release -p sylpheed-formats` +# ran 4 jobs in a 6 GB container and was OOM-killed mid-task. Release-mode rustc +# on this workspace needs closer to 2 GB, so the divisor is 2, not 3/2. +# +# ⚠️ And the kill reported `OOMKilled: true` with **ExitCode 0**, so it read as a +# clean exit — which is why the restart policy is `unless-stopped` rather than +# `on-failure`. +by_mem=$(( mem_gib / 2 )) [ "$by_mem" -lt 1 ] && by_mem=1 jobs=$(( cpus < by_mem ? cpus : by_mem )) +# An EXPLICIT cap from the launcher wins. Without this the launcher's +# `-e CARGO_BUILD_JOBS=...` was computed, exported over, and silently discarded +# — the guardrail was set and then removed three lines later. +if [ -n "${CARGO_BUILD_JOBS:-}" ] && [ "${CARGO_BUILD_JOBS}" -ge 1 ] 2>/dev/null; then + jobs="$CARGO_BUILD_JOBS" + _why=" (explicit, from the launcher)" +fi export SYLPH_JOBS="$jobs" CARGO_BUILD_JOBS="$jobs" CMAKE_BUILD_PARALLEL_LEVEL="$jobs" -log "build parallelism: $jobs (cpus=$cpus, mem=${mem_gib}GiB avail)" +log "build parallelism: $jobs${_why:-} (cpus=$cpus, mem=${mem_gib}GiB avail)" mkdir -p "$HOME/shots" "$HOME/logs" @@ -160,6 +175,10 @@ fi if [ -f "$HOME/.claude.host.json" ] && [ ! -s "$HOME/.claude.json" ]; then cp "$HOME/.claude.host.json" "$HOME/.claude.json" 2>/dev/null || true fi + +# Nothing is restored into `~/.claude.json` on purpose. Resuming is done by +# SESSION ID off the transcript instead — see the resume block below for why +# the index is useless for this. # `credential.helper=store` rewrites this file by rename-over-target, which # fails with EBUSY on a bind mount -- reported as `fatal: unable to write # credential store`, while the push itself succeeds. A fatal line that is @@ -181,6 +200,77 @@ if [ "${SYLPH_AUTONOMOUS:-0}" = "1" ]; then if [ "$#" -eq 1 ] && [ "$1" = "bash" ]; then set -- fi + # ── Resume across a restart ──────────────────────────────────────────────── + # + # The container restarts automatically now, and a restart that opens a BLANK + # session throws away everything the agent knew. That is not hypothetical: on + # 2026-09-01 an OOM kill ended a run mid-task with a 2.7 MB transcript and two + # files uncommitted in the volume. + # + # 🔴 RESUME BY SESSION ID, NOT BY `--continue`. Measured 2026-09-01: + # + # `--continue` resolves through `~/.claude.json`'s per-project `history` and + # `lastSessionId`. Those are written at a GRACEFUL SHUTDOWN — mid-session the + # live file has `history: None`, `lastSessionId: None`. A container that is + # OOM-killed or `docker rm -f`ed never writes them, which is exactly the case + # this feature exists for. So `--continue` answered "No conversation found to + # continue" with 33 MB of perfectly good transcripts in the volume beside it, + # and persisting `.claude.json` did not help because the fields were never + # populated in the first place. + # + # The TRANSCRIPTS are durable and are named by session id, so read the id off + # the newest one for this working directory. Claude Code has not started yet + # at this point, so the newest is the previous run's. + # + # The /loop prompt is still passed, so the loop is RE-ARMED rather than merely + # restored — a resumed conversation with no wake-up scheduled answers once and + # stops, which looks like resuming and is not. + SYLPH_STAMP="$HOME/.claude/.sylph-last-start" + SYLPH_RESUME=0 + SYLPH_SESSION="" + SYLPH_PROJ="$HOME/.claude/projects/$(printf '%s' "$PWD" | sed 's#/#-#g')" + if [ -d "$SYLPH_PROJ" ]; then + _newest=$(ls -1t "$SYLPH_PROJ"/*.jsonl 2>/dev/null | head -1) + if [ -n "$_newest" ]; then + SYLPH_SESSION=$(basename "$_newest" .jsonl) + SYLPH_RESUME=1 + fi + fi + # 🔴 A POISONED TRANSCRIPT MUST NOT CRASH-LOOP. If the last start was under + # two minutes ago we are already in a restart loop, and continuing back into + # whatever killed us is the one thing guaranteed not to help. Start fresh and + # say so, rather than burning tokens on the same death repeatedly. + if [ "$SYLPH_RESUME" = "1" ] && [ -f "$SYLPH_STAMP" ]; then + _last=$(cat "$SYLPH_STAMP" 2>/dev/null || echo 0) + _now=$(date +%s) + if [ $((_now - _last)) -lt 120 ]; then + SYLPH_RESUME=0 + log "restarted <120s after the last start — restart loop suspected;" + log " starting a FRESH session rather than continuing into the same death" + fi + fi + mkdir -p "$HOME/.claude" 2>/dev/null || true + date +%s > "$SYLPH_STAMP" 2>/dev/null || true + + if [ "$SYLPH_RESUME" = "1" ] && [ "$#" -eq 1 ]; then + set -- "$1 + +⚠️ YOU WERE RESTARTED, and this session was resumed — your context is intact, +but the process that was running when it died is gone. Before anything else: + +1. \`git -C /work status\`. Whatever you had in progress is still in the tree, + UNCOMMITTED. Commit it and \`push-work\` before starting anything new. +2. Any build, test or capture you had running did NOT finish. Do not read its + absence as a result. +3. The likeliest cause is an OOM kill — this container is capped at 6 GB. + \`CARGO_BUILD_JOBS\` is now set for you in the environment; do not raise it, + and prefer \`build-reborn test\` over a raw \`cargo test --release\`, which + bypasses the wrapper's job cap. That is exactly what killed the run on + 2026-09-01." + log "resuming session ${SYLPH_SESSION%%-*}… with a restart notice" + fi + [ "$SYLPH_RESUME" = "1" ] && set -- --resume "$SYLPH_SESSION" "$@" + # The flag the user asked for. It is refused under root, which is why this # image runs as `agent`. # Remote Control registers the session with your account so you can chat with diff --git a/docker/decoder/sylph-decoder b/docker/decoder/sylph-decoder index b35f15df..e61c7de4 100755 --- a/docker/decoder/sylph-decoder +++ b/docker/decoder/sylph-decoder @@ -130,6 +130,19 @@ docker_args() { -e "PROJECT_DIR=/work" -e "SYLPH_EXCHANGE=/exchange" -e "SYLPH_AGENT=decoder" + # 🔴 THE JOB CAP LIVES IN THE ENVIRONMENT, NOT IN THE WRAPPER. + # + # `build-reborn` has always exported CARGO_BUILD_JOBS, and on 2026-09-01 + # that was not enough: the agent ran a RAW `cargo test --release -p + # sylpheed-formats`, which never touches the wrapper, got one rustc per + # granted CPU, and the container was OOM-killed at its 6 GB cap mid-task. + # Docker reported ExitCode 0 with OOMKilled true, so it read as a clean + # exit and cost a diagnosis. + # + # A guardrail reachable only through a wrapper protects the calls that use + # the wrapper. This one is inherited by every process in the container, so + # bypassing it takes an explicit override rather than forgetting. + -e "CARGO_BUILD_JOBS=${SYLPH_JOBS:-2}" -e "SYLPH_REPO_URL=https://git.mc02.dev/fabi/Sylpheed.git" -e "XENIA_SRC=/canary" # ── claude ── @@ -269,7 +282,17 @@ case "${1:-}" in echo "==> repo: own clone in volume sylpheed-decoder-repo -> /work" echo "==> pacing: ${INTERVAL:-self-paced}" docker rm -f "$NAME" >/dev/null 2>&1 || true - docker run -d -i -t "${ARGS[@]}" "$IMAGE" "/loop ${INTERVAL:+$INTERVAL }$TASK" >/dev/null + # 🔴 `unless-stopped`, NOT `on-failure` -- and the reason is a trap worth + # keeping. When this container was OOM-killed on 2026-09-01, Docker reported + # `OOMKilled: true` with **ExitCode 0**. `on-failure` keys off the exit code, + # so it would have treated a memory kill as a clean finish and left the agent + # down. `unless-stopped` restarts regardless, and still honours an explicit + # `./sylph-agent stop`. + # + # Restarting into the same death is handled at the other end: the entrypoint + # refuses to `--continue` if the last start was under two minutes ago. + docker run -d -i -t --restart unless-stopped "${ARGS[@]}" "$IMAGE" \ + "/loop ${INTERVAL:+$INTERVAL }$TASK" >/dev/null echo echo " running detached as '$NAME'." echo " ./sylph-agent remote link to chat with it from anywhere" diff --git a/docker/port/entrypoint.sh b/docker/port/entrypoint.sh index 8fa94dcc..d74991a6 100755 --- a/docker/port/entrypoint.sh +++ b/docker/port/entrypoint.sh @@ -57,6 +57,10 @@ fi if [ -f "$HOME/.claude.host.json" ] && [ ! -s "$HOME/.claude.json" ]; then cp "$HOME/.claude.host.json" "$HOME/.claude.json" 2>/dev/null || true fi + +# Nothing is restored into `~/.claude.json` on purpose. Resuming is done by +# SESSION ID off the transcript instead — see the resume block below for why the +# index cannot serve. # Same reason as .claude.json above: `credential.helper=store` rewrites this # file by rename-over-target, which fails with EBUSY on a bind mount. Copy it to # a writable path; nothing is ever written back to the host's file. @@ -110,6 +114,70 @@ if [ "${SYLPH_AUTONOMOUS:-0}" = "1" ]; then if [ "$#" -eq 1 ] && [ "$1" = "bash" ]; then set -- fi + # ── Resume across a restart ──────────────────────────────────────────────── + # + # The container restarts automatically now, and a restart that opens a BLANK + # session throws away everything the agent knew. + # + # 🔴 RESUME BY SESSION ID, NOT BY `--continue`. Measured on the decoder + # 2026-09-01: `--continue` resolves through `~/.claude.json`'s per-project + # `history` / `lastSessionId`, and those are written at a GRACEFUL SHUTDOWN -- + # mid-session the live file has both as `None`. A container that is OOM-killed + # or `docker rm -f`ed never writes them, which is exactly the case this exists + # for, so `--continue` answered "No conversation found to continue" with the + # transcripts sitting in the volume beside it. + # + # The TRANSCRIPTS are durable and named by session id. Claude Code has not + # started yet here, so the newest is the previous run's. + # + # The /loop prompt is still passed so the loop is RE-ARMED rather than merely + # restored -- a resumed conversation with no wake-up scheduled answers once + # and stops, which looks like resuming and is not. + SYLPH_STAMP="$HOME/.claude/.sylph-last-start" + SYLPH_RESUME=0 + SYLPH_SESSION="" + SYLPH_PROJ="$HOME/.claude/projects/$(printf '%s' "$PWD" | sed 's#/#-#g')" + if [ -d "$SYLPH_PROJ" ]; then + _newest=$(ls -1t "$SYLPH_PROJ"/*.jsonl 2>/dev/null | head -1) + if [ -n "$_newest" ]; then + SYLPH_SESSION=$(basename "$_newest" .jsonl) + SYLPH_RESUME=1 + fi + fi + # 🔴 A POISONED TRANSCRIPT MUST NOT CRASH-LOOP. Restarted under two minutes + # after the last start, we are already looping: continuing back into whatever + # killed us is the one thing guaranteed not to help. + if [ "$SYLPH_RESUME" = "1" ] && [ -f "$SYLPH_STAMP" ]; then + _last=$(cat "$SYLPH_STAMP" 2>/dev/null || echo 0) + _now=$(date +%s) + if [ $((_now - _last)) -lt 120 ]; then + SYLPH_RESUME=0 + echo "[entrypoint] restarted <120s after the last start -- restart loop" + echo "[entrypoint] suspected; starting FRESH rather than continuing" + fi + fi + mkdir -p "$HOME/.claude" 2>/dev/null || true + date +%s > "$SYLPH_STAMP" 2>/dev/null || true + + if [ "$SYLPH_RESUME" = "1" ] && [ "$#" -eq 1 ]; then + set -- "$1 + +⚠️ YOU WERE RESTARTED, and this session was resumed — your context is intact, +but the process that was running when it died is gone. Before anything else: + +1. \`git -C /work status\`. Whatever you had in progress is still in the tree, + UNCOMMITTED. Commit it and \`push-work\` before starting anything new. +2. Any build, test, export or Godot run you had going did NOT finish. Do not + read its absence as a result. +3. The likeliest cause is an OOM kill — this container is capped at 4 GB. + \`CARGO_BUILD_JOBS\` is now set for you in the environment; do not raise it, + and prefer \`build-export\` / \`build-reference-cli\` over a raw + \`cargo build --release\`, which bypasses the wrapper's job cap. That is what + killed the decoder's run on 2026-09-01." + echo "[entrypoint] resuming session ${SYLPH_SESSION%%-*}… with a restart notice" + fi + [ "$SYLPH_RESUME" = "1" ] && set -- --resume "$SYLPH_SESSION" "$@" + # Remote Control registers the session with the account so the agent can be # reached from claude.ai -- the point of a detached run being that nobody is # sitting in front of it. The name is passed EXPLICITLY: the flag's value is diff --git a/docker/port/sylph-port b/docker/port/sylph-port index 8cbca2d7..cd7d554f 100755 --- a/docker/port/sylph-port +++ b/docker/port/sylph-port @@ -67,6 +67,12 @@ docker_args() { -v "${SYLPH_CLAUDE_JSON:-$HOME/.claude.json}:/sylph-home/port/.claude.host.json:ro" -v "sylpheed-exchange:/exchange" -e "PROJECT_DIR=/work" + # Same guardrail as the decoder, added the same day and for its reason: the + # decoder was OOM-killed mid-task by a RAW `cargo test --release`, which + # never reaches `build-export`/`build-reference-cli` and so never saw their + # CARGO_BUILD_JOBS. This container is smaller (4 GB, 3 CPUs), so the same + # bypass is at least as easy to hit here. + -e "CARGO_BUILD_JOBS=${SYLPH_PORT_JOBS:-2}" -e "SYLPH_EXCHANGE=/exchange" -e "SYLPH_AGENT=port" -e "SYLPH_REPO_URL=https://git.mc02.dev/fabi/Sylpheed.git" @@ -140,7 +146,12 @@ case "${1:-}" in INTERVAL="${SYLPH_LOOP_INTERVAL-45m}" echo "==> loose | cpus=$CPUS mem=${MEM_GB}g pacing=${INTERVAL:-self}" echo "==> repo: own clone in volume sylpheed-port-repo -> /work" - docker run -d -i -t "${ARGS[@]}" -e SYLPH_AUTONOMOUS=1 -w /work "$IMAGE" \ + # `unless-stopped`, NOT `on-failure`: an OOM kill on this setup reports + # `OOMKilled: true` with **ExitCode 0**, so `on-failure` would read a memory + # kill as a clean finish and leave the agent down. Restarting into the same + # death is handled in the entrypoint, which refuses to `--continue` when the + # last start was under two minutes ago. + docker run -d -i -t --restart unless-stopped "${ARGS[@]}" -e SYLPH_AUTONOMOUS=1 -w /work "$IMAGE" \ "/loop ${INTERVAL:+$INTERVAL }$TASK" >/dev/null echo echo " running detached as '$NAME'."