diff --git a/docker/decoder/entrypoint.sh b/docker/decoder/entrypoint.sh index da5ce0e5..dee593f8 100755 --- a/docker/decoder/entrypoint.sh +++ b/docker/decoder/entrypoint.sh @@ -84,11 +84,26 @@ if [ -r /sys/fs/cgroup/memory.max ]; then [ "$_m" != max ] && mem_gib=$(( _m / 1073741824 )) fi [ "${mem_gib:-0}" -lt 1 ] && mem_gib=1 -by_mem=$(( mem_gib * 2 / 3 )) +# 🔴 THIS CONSTANT WAS WRONG, AND IT COST A RUN. `mem_gib * 2 / 3` assumes +# ~1.5 GB per job. On 2026-09-01 a raw `cargo test --release -p sylpheed-formats` +# ran 4 jobs in a 6 GB container and was OOM-killed mid-task. Release-mode rustc +# on this workspace needs closer to 2 GB, so the divisor is 2, not 3/2. +# +# ⚠️ And the kill reported `OOMKilled: true` with **ExitCode 0**, so it read as a +# clean exit — which is why the restart policy is `unless-stopped` rather than +# `on-failure`. +by_mem=$(( mem_gib / 2 )) [ "$by_mem" -lt 1 ] && by_mem=1 jobs=$(( cpus < by_mem ? cpus : by_mem )) +# An EXPLICIT cap from the launcher wins. Without this the launcher's +# `-e CARGO_BUILD_JOBS=...` was computed, exported over, and silently discarded +# — the guardrail was set and then removed three lines later. +if [ -n "${CARGO_BUILD_JOBS:-}" ] && [ "${CARGO_BUILD_JOBS}" -ge 1 ] 2>/dev/null; then + jobs="$CARGO_BUILD_JOBS" + _why=" (explicit, from the launcher)" +fi export SYLPH_JOBS="$jobs" CARGO_BUILD_JOBS="$jobs" CMAKE_BUILD_PARALLEL_LEVEL="$jobs" -log "build parallelism: $jobs (cpus=$cpus, mem=${mem_gib}GiB avail)" +log "build parallelism: $jobs${_why:-} (cpus=$cpus, mem=${mem_gib}GiB avail)" mkdir -p "$HOME/shots" "$HOME/logs" @@ -160,6 +175,10 @@ fi if [ -f "$HOME/.claude.host.json" ] && [ ! -s "$HOME/.claude.json" ]; then cp "$HOME/.claude.host.json" "$HOME/.claude.json" 2>/dev/null || true fi + +# Nothing is restored into `~/.claude.json` on purpose. Resuming is done by +# SESSION ID off the transcript instead — see the resume block below for why +# the index is useless for this. # `credential.helper=store` rewrites this file by rename-over-target, which # fails with EBUSY on a bind mount -- reported as `fatal: unable to write # credential store`, while the push itself succeeds. A fatal line that is @@ -181,6 +200,77 @@ if [ "${SYLPH_AUTONOMOUS:-0}" = "1" ]; then if [ "$#" -eq 1 ] && [ "$1" = "bash" ]; then set -- fi + # ── Resume across a restart ──────────────────────────────────────────────── + # + # The container restarts automatically now, and a restart that opens a BLANK + # session throws away everything the agent knew. That is not hypothetical: on + # 2026-09-01 an OOM kill ended a run mid-task with a 2.7 MB transcript and two + # files uncommitted in the volume. + # + # 🔴 RESUME BY SESSION ID, NOT BY `--continue`. Measured 2026-09-01: + # + # `--continue` resolves through `~/.claude.json`'s per-project `history` and + # `lastSessionId`. Those are written at a GRACEFUL SHUTDOWN — mid-session the + # live file has `history: None`, `lastSessionId: None`. A container that is + # OOM-killed or `docker rm -f`ed never writes them, which is exactly the case + # this feature exists for. So `--continue` answered "No conversation found to + # continue" with 33 MB of perfectly good transcripts in the volume beside it, + # and persisting `.claude.json` did not help because the fields were never + # populated in the first place. + # + # The TRANSCRIPTS are durable and are named by session id, so read the id off + # the newest one for this working directory. Claude Code has not started yet + # at this point, so the newest is the previous run's. + # + # The /loop prompt is still passed, so the loop is RE-ARMED rather than merely + # restored — a resumed conversation with no wake-up scheduled answers once and + # stops, which looks like resuming and is not. + SYLPH_STAMP="$HOME/.claude/.sylph-last-start" + SYLPH_RESUME=0 + SYLPH_SESSION="" + SYLPH_PROJ="$HOME/.claude/projects/$(printf '%s' "$PWD" | sed 's#/#-#g')" + if [ -d "$SYLPH_PROJ" ]; then + _newest=$(ls -1t "$SYLPH_PROJ"/*.jsonl 2>/dev/null | head -1) + if [ -n "$_newest" ]; then + SYLPH_SESSION=$(basename "$_newest" .jsonl) + SYLPH_RESUME=1 + fi + fi + # 🔴 A POISONED TRANSCRIPT MUST NOT CRASH-LOOP. If the last start was under + # two minutes ago we are already in a restart loop, and continuing back into + # whatever killed us is the one thing guaranteed not to help. Start fresh and + # say so, rather than burning tokens on the same death repeatedly. + if [ "$SYLPH_RESUME" = "1" ] && [ -f "$SYLPH_STAMP" ]; then + _last=$(cat "$SYLPH_STAMP" 2>/dev/null || echo 0) + _now=$(date +%s) + if [ $((_now - _last)) -lt 120 ]; then + SYLPH_RESUME=0 + log "restarted <120s after the last start — restart loop suspected;" + log " starting a FRESH session rather than continuing into the same death" + fi + fi + mkdir -p "$HOME/.claude" 2>/dev/null || true + date +%s > "$SYLPH_STAMP" 2>/dev/null || true + + if [ "$SYLPH_RESUME" = "1" ] && [ "$#" -eq 1 ]; then + set -- "$1 + +⚠️ YOU WERE RESTARTED, and this session was resumed — your context is intact, +but the process that was running when it died is gone. Before anything else: + +1. \`git -C /work status\`. Whatever you had in progress is still in the tree, + UNCOMMITTED. Commit it and \`push-work\` before starting anything new. +2. Any build, test or capture you had running did NOT finish. Do not read its + absence as a result. +3. The likeliest cause is an OOM kill — this container is capped at 6 GB. + \`CARGO_BUILD_JOBS\` is now set for you in the environment; do not raise it, + and prefer \`build-reborn test\` over a raw \`cargo test --release\`, which + bypasses the wrapper's job cap. That is exactly what killed the run on + 2026-09-01." + log "resuming session ${SYLPH_SESSION%%-*}… with a restart notice" + fi + [ "$SYLPH_RESUME" = "1" ] && set -- --resume "$SYLPH_SESSION" "$@" + # The flag the user asked for. It is refused under root, which is why this # image runs as `agent`. # Remote Control registers the session with your account so you can chat with diff --git a/docker/decoder/sylph-decoder b/docker/decoder/sylph-decoder index b35f15df..e61c7de4 100755 --- a/docker/decoder/sylph-decoder +++ b/docker/decoder/sylph-decoder @@ -130,6 +130,19 @@ docker_args() { -e "PROJECT_DIR=/work" -e "SYLPH_EXCHANGE=/exchange" -e "SYLPH_AGENT=decoder" + # 🔴 THE JOB CAP LIVES IN THE ENVIRONMENT, NOT IN THE WRAPPER. + # + # `build-reborn` has always exported CARGO_BUILD_JOBS, and on 2026-09-01 + # that was not enough: the agent ran a RAW `cargo test --release -p + # sylpheed-formats`, which never touches the wrapper, got one rustc per + # granted CPU, and the container was OOM-killed at its 6 GB cap mid-task. + # Docker reported ExitCode 0 with OOMKilled true, so it read as a clean + # exit and cost a diagnosis. + # + # A guardrail reachable only through a wrapper protects the calls that use + # the wrapper. This one is inherited by every process in the container, so + # bypassing it takes an explicit override rather than forgetting. + -e "CARGO_BUILD_JOBS=${SYLPH_JOBS:-2}" -e "SYLPH_REPO_URL=https://git.mc02.dev/fabi/Sylpheed.git" -e "XENIA_SRC=/canary" # ── claude ── @@ -269,7 +282,17 @@ case "${1:-}" in echo "==> repo: own clone in volume sylpheed-decoder-repo -> /work" echo "==> pacing: ${INTERVAL:-self-paced}" docker rm -f "$NAME" >/dev/null 2>&1 || true - docker run -d -i -t "${ARGS[@]}" "$IMAGE" "/loop ${INTERVAL:+$INTERVAL }$TASK" >/dev/null + # 🔴 `unless-stopped`, NOT `on-failure` -- and the reason is a trap worth + # keeping. When this container was OOM-killed on 2026-09-01, Docker reported + # `OOMKilled: true` with **ExitCode 0**. `on-failure` keys off the exit code, + # so it would have treated a memory kill as a clean finish and left the agent + # down. `unless-stopped` restarts regardless, and still honours an explicit + # `./sylph-agent stop`. + # + # Restarting into the same death is handled at the other end: the entrypoint + # refuses to `--continue` if the last start was under two minutes ago. + docker run -d -i -t --restart unless-stopped "${ARGS[@]}" "$IMAGE" \ + "/loop ${INTERVAL:+$INTERVAL }$TASK" >/dev/null echo echo " running detached as '$NAME'." echo " ./sylph-agent remote link to chat with it from anywhere" diff --git a/docker/port/entrypoint.sh b/docker/port/entrypoint.sh index 8fa94dcc..d74991a6 100755 --- a/docker/port/entrypoint.sh +++ b/docker/port/entrypoint.sh @@ -57,6 +57,10 @@ fi if [ -f "$HOME/.claude.host.json" ] && [ ! -s "$HOME/.claude.json" ]; then cp "$HOME/.claude.host.json" "$HOME/.claude.json" 2>/dev/null || true fi + +# Nothing is restored into `~/.claude.json` on purpose. Resuming is done by +# SESSION ID off the transcript instead — see the resume block below for why the +# index cannot serve. # Same reason as .claude.json above: `credential.helper=store` rewrites this # file by rename-over-target, which fails with EBUSY on a bind mount. Copy it to # a writable path; nothing is ever written back to the host's file. @@ -110,6 +114,70 @@ if [ "${SYLPH_AUTONOMOUS:-0}" = "1" ]; then if [ "$#" -eq 1 ] && [ "$1" = "bash" ]; then set -- fi + # ── Resume across a restart ──────────────────────────────────────────────── + # + # The container restarts automatically now, and a restart that opens a BLANK + # session throws away everything the agent knew. + # + # 🔴 RESUME BY SESSION ID, NOT BY `--continue`. Measured on the decoder + # 2026-09-01: `--continue` resolves through `~/.claude.json`'s per-project + # `history` / `lastSessionId`, and those are written at a GRACEFUL SHUTDOWN -- + # mid-session the live file has both as `None`. A container that is OOM-killed + # or `docker rm -f`ed never writes them, which is exactly the case this exists + # for, so `--continue` answered "No conversation found to continue" with the + # transcripts sitting in the volume beside it. + # + # The TRANSCRIPTS are durable and named by session id. Claude Code has not + # started yet here, so the newest is the previous run's. + # + # The /loop prompt is still passed so the loop is RE-ARMED rather than merely + # restored -- a resumed conversation with no wake-up scheduled answers once + # and stops, which looks like resuming and is not. + SYLPH_STAMP="$HOME/.claude/.sylph-last-start" + SYLPH_RESUME=0 + SYLPH_SESSION="" + SYLPH_PROJ="$HOME/.claude/projects/$(printf '%s' "$PWD" | sed 's#/#-#g')" + if [ -d "$SYLPH_PROJ" ]; then + _newest=$(ls -1t "$SYLPH_PROJ"/*.jsonl 2>/dev/null | head -1) + if [ -n "$_newest" ]; then + SYLPH_SESSION=$(basename "$_newest" .jsonl) + SYLPH_RESUME=1 + fi + fi + # 🔴 A POISONED TRANSCRIPT MUST NOT CRASH-LOOP. Restarted under two minutes + # after the last start, we are already looping: continuing back into whatever + # killed us is the one thing guaranteed not to help. + if [ "$SYLPH_RESUME" = "1" ] && [ -f "$SYLPH_STAMP" ]; then + _last=$(cat "$SYLPH_STAMP" 2>/dev/null || echo 0) + _now=$(date +%s) + if [ $((_now - _last)) -lt 120 ]; then + SYLPH_RESUME=0 + echo "[entrypoint] restarted <120s after the last start -- restart loop" + echo "[entrypoint] suspected; starting FRESH rather than continuing" + fi + fi + mkdir -p "$HOME/.claude" 2>/dev/null || true + date +%s > "$SYLPH_STAMP" 2>/dev/null || true + + if [ "$SYLPH_RESUME" = "1" ] && [ "$#" -eq 1 ]; then + set -- "$1 + +⚠️ YOU WERE RESTARTED, and this session was resumed — your context is intact, +but the process that was running when it died is gone. Before anything else: + +1. \`git -C /work status\`. Whatever you had in progress is still in the tree, + UNCOMMITTED. Commit it and \`push-work\` before starting anything new. +2. Any build, test, export or Godot run you had going did NOT finish. Do not + read its absence as a result. +3. The likeliest cause is an OOM kill — this container is capped at 4 GB. + \`CARGO_BUILD_JOBS\` is now set for you in the environment; do not raise it, + and prefer \`build-export\` / \`build-reference-cli\` over a raw + \`cargo build --release\`, which bypasses the wrapper's job cap. That is what + killed the decoder's run on 2026-09-01." + echo "[entrypoint] resuming session ${SYLPH_SESSION%%-*}… with a restart notice" + fi + [ "$SYLPH_RESUME" = "1" ] && set -- --resume "$SYLPH_SESSION" "$@" + # Remote Control registers the session with the account so the agent can be # reached from claude.ai -- the point of a detached run being that nobody is # sitting in front of it. The name is passed EXPLICITLY: the flag's value is diff --git a/docker/port/sylph-port b/docker/port/sylph-port index 8cbca2d7..cd7d554f 100755 --- a/docker/port/sylph-port +++ b/docker/port/sylph-port @@ -67,6 +67,12 @@ docker_args() { -v "${SYLPH_CLAUDE_JSON:-$HOME/.claude.json}:/sylph-home/port/.claude.host.json:ro" -v "sylpheed-exchange:/exchange" -e "PROJECT_DIR=/work" + # Same guardrail as the decoder, added the same day and for its reason: the + # decoder was OOM-killed mid-task by a RAW `cargo test --release`, which + # never reaches `build-export`/`build-reference-cli` and so never saw their + # CARGO_BUILD_JOBS. This container is smaller (4 GB, 3 CPUs), so the same + # bypass is at least as easy to hit here. + -e "CARGO_BUILD_JOBS=${SYLPH_PORT_JOBS:-2}" -e "SYLPH_EXCHANGE=/exchange" -e "SYLPH_AGENT=port" -e "SYLPH_REPO_URL=https://git.mc02.dev/fabi/Sylpheed.git" @@ -140,7 +146,12 @@ case "${1:-}" in INTERVAL="${SYLPH_LOOP_INTERVAL-45m}" echo "==> loose | cpus=$CPUS mem=${MEM_GB}g pacing=${INTERVAL:-self}" echo "==> repo: own clone in volume sylpheed-port-repo -> /work" - docker run -d -i -t "${ARGS[@]}" -e SYLPH_AUTONOMOUS=1 -w /work "$IMAGE" \ + # `unless-stopped`, NOT `on-failure`: an OOM kill on this setup reports + # `OOMKilled: true` with **ExitCode 0**, so `on-failure` would read a memory + # kill as a clean finish and leave the agent down. Restarting into the same + # death is handled in the entrypoint, which refuses to `--continue` when the + # last start was under two minutes ago. + docker run -d -i -t --restart unless-stopped "${ARGS[@]}" -e SYLPH_AUTONOMOUS=1 -w /work "$IMAGE" \ "/loop ${INTERVAL:+$INTERVAL }$TASK" >/dev/null echo echo " running detached as '$NAME'."