test/soak.sh
Ref: Size: 8.4 KiB History
#!/bin/sh
# Runs the full e2e suite SOAK_N times (default 10, ~25min) and reports a
# per-failure table in the decisions.md convention: which FAIL line, from
# which run. Serial on purpose — each e2e run isolates by its own $$, and
# the between-runs hygiene check below assumes nothing else is using the
# same temp-file patterns while it looks (a concurrently running suite
# would read as a leak).
set -u
# The same list e2e.sh takes, in the same order — see the note in build.zig;
# soak IS that suite run N times, so an argument added to one and not the
# other makes every run abort on an unbound variable.
MUX="$1"; RAWMODE="$2"; DELAYPIPE="$3"; RENDER="$4"; PTYCLIENT="$5"; WSCLIENT="$6"
E2E="$(dirname "$0")/e2e.sh"
# pid_rss_kb and pid_fd_count, from the file that owns every OS-specific
# spelling in this suite. Sourced rather than copied: os_oracle.sh brings
# no trap and no cleanup registry, so it composes with the ones this script
# already has, and a second OS's arm is written once for both readers.
# shellcheck source=test/os_oracle.sh
. "$(dirname "$0")/os_oracle.sh"
N="${SOAK_N:-10}"
TMP="${TMPDIR:-/tmp}"
FAILDIR="$TMP/mux-soak-$$-failures"
SUMMARY="$TMP/mux-soak-$$.summary"
LOG="$TMP/mux-soak-$$.log"
: > "$SUMMARY"
# Residue predating the soak is not this soak's leak. Refuse to start rather
# than baseline-subtract: the files are probably evidence a previous failing
# e2e deliberately left behind, and the per-run hygiene check below cannot
# tell them apart from a leak — forcing a look is better than silently
# discarding them.
BASE_STRAYS=$(find "$TMP" -maxdepth 1 \( -name 'muxd-e2e-*' -o -name 'mux-e2e-*' \) 2>/dev/null | wc -l)
[ "$BASE_STRAYS" -eq 0 ] || {
echo "soak FAIL: $BASE_STRAYS mux e2e temp files already in $TMP before run 1."
echo " They are probably evidence from an earlier failing e2e."
echo " Inspect them, then clear them — the per-run hygiene check"
echo " cannot tell them apart from a leak."
exit 1
}
FAILED=0
i=1
while [ "$i" -le "$N" ]; do
if "$E2E" "$MUX" d "$MUX" "$RAWMODE" "$DELAYPIPE" "$RENDER" "$PTYCLIENT" "$WSCLIENT" "$MUX" web "$MUX" a > "$LOG" 2>&1; then
echo "soak run $i/$N: PASS"
else
FAILED=$((FAILED + 1))
FL=$(grep 'e2e FAIL' "$LOG" | head -1)
FL="${FL:-exited nonzero with no FAIL line}"
echo "soak run $i/$N: FAIL — $FL"
printf 'run %s: %s\n' "$i" "$FL" >> "$SUMMARY"
mkdir -p "$FAILDIR"
cp "$LOG" "$FAILDIR/run$i.log"
# A failing run deliberately leaves grid evidence behind; sweep it
# into the failure dir so the hygiene check below stays meaningful
# for the NEXT run.
find "$TMP" -maxdepth 1 \( -name 'muxd-e2e-*' -o -name 'mux-e2e-*' \) \
-exec mv {} "$FAILDIR/" \; 2>/dev/null
fi
# Per-run tmp hygiene: a leak in run 3 must not blame run 7.
STRAYS=$(find "$TMP" -maxdepth 1 \( -name 'muxd-e2e-*' -o -name 'mux-e2e-*' \) 2>/dev/null | wc -l)
if [ "$STRAYS" -ne 0 ]; then
echo "soak FAIL: run $i left $STRAYS temp files behind:"
find "$TMP" -maxdepth 1 \( -name 'muxd-e2e-*' -o -name 'mux-e2e-*' \)
FAILED=$((FAILED + 1))
printf 'run %s: left %s temp files\n' "$i" "$STRAYS" >> "$SUMMARY"
fi
i=$((i + 1))
done
# ---- persistence phase (hygiene kit, 6b) ----
# One daemon, SOAK_CYCLES client lifecycles. Catches what the run loop
# structurally cannot: C-side growth, fd leaks, unbounded accumulation —
# the classes the Zig-side LEAK marker (6a) never sees. Baseline is taken
# AFTER a warmup: the first attaches pay one-time allocations (grid,
# history) that are capacity, not leakage.
PSOCK="$TMP/mux-soak-persist-$$.sock"
PLOG="$TMP/mux-soak-persist-$$.log"
PCAP="$TMP/mux-soak-persist-$$.cap"
# e2e.sh isolates its own XDG homes; this leg attaches directly and must
# too, or every soak adds a dead socket to the developer's real wall file.
XDG_STATE_HOME="$TMP/mux-soak-persist-$$-state"; export XDG_STATE_HOME
CYCLES="${SOAK_CYCLES:-20}"
WARMUP=3
RSS_BOUND_KB=4096
PERSIST_FAILED_BEFORE=$FAILED
"$MUX" d start --sock "$PSOCK" --shell /bin/sh > "$PLOG" 2>&1 &
PDPID=$!
_i=0
while [ ! -S "$PSOCK" ] && [ "$_i" -lt 100 ]; do sleep 0.05; _i=$((_i + 1)); done
if [ ! -S "$PSOCK" ]; then
echo "soak FAIL: persistence daemon never bound $PSOCK"
FAILED=$((FAILED + 1))
printf 'persistence: daemon never bound\n' >> "$SUMMARY"
else
BASE_RSS=0; BASE_FD=0; RSS=0; FD=0
ATTACH_FAILED=0
c=1
while [ "$c" -le "$CYCLES" ]; do
{ printf 'echo cycle-%s\n' "$c"; sleep 1; printf '\034\034'; } | \
timeout 30 "$MUX" --sock "$PSOCK" > "$PCAP" 2>/dev/null
# Vacuous-green guard: a regression that makes the client fail
# instantly would leave fds/RSS trivially flat and this phase green
# without ever having driven the daemon. Require the echo to land.
if ! grep -q "cycle-$c" "$PCAP"; then
echo "soak FAIL: persistence cycle $c client never attached/echoed"
FAILED=$((FAILED + 1))
printf 'persistence: cycle %s no attach\n' "$c" >> "$SUMMARY"
ATTACH_FAILED=1
break
fi
RSS=$(pid_rss_kb "$PDPID")
# The daemon pid can be an unreaped zombie by now: the OS still has
# the pid but reports no resident size for it, so pid_rss_kb prints
# an empty string rather than failing. Guard it directly so the
# died-mid-phase check below isn't skipped by `[ "" -eq 0 ]`
# erroring-as-false.
RSS=${RSS:-0}
FD=$(pid_fd_count "$PDPID")
[ "$c" -eq "$WARMUP" ] && { BASE_RSS=$RSS; BASE_FD=$FD; }
c=$((c + 1))
done
echo "soak persistence: $CYCLES cycles, RSS ${BASE_RSS}->${RSS} kB, fds ${BASE_FD}->${FD}"
if [ "$ATTACH_FAILED" -eq 1 ]; then
: # cycle loop already recorded the no-attach failure above
elif [ "$RSS" -eq 0 ] || [ "$BASE_RSS" -eq 0 ]; then
echo "soak FAIL: persistence daemon died mid-phase"
FAILED=$((FAILED + 1))
printf 'persistence: daemon died mid-phase\n' >> "$SUMMARY"
else
# fds must RETURN to baseline exactly: every attach opens, every
# detach must close. RSS gets a bound, not equality — allocators
# retain pages — but growth past it over this few cycles is a leak.
if [ "$FD" -ne "$BASE_FD" ]; then
# Detach is fire-and-forget from the client's side; the daemon
# may not have closed the fd yet at the instant we sampled. One
# settle beat kills the flake without weakening the equality
# check itself — a real leak won't self-heal in a second.
sleep 1
FD=$(pid_fd_count "$PDPID")
fi
if [ "$FD" -ne "$BASE_FD" ]; then
echo "soak FAIL: persistence fd count $BASE_FD -> $FD across detached cycles"
FAILED=$((FAILED + 1))
printf 'persistence: fd leak %s->%s\n' "$BASE_FD" "$FD" >> "$SUMMARY"
fi
if [ $((RSS - BASE_RSS)) -gt "$RSS_BOUND_KB" ]; then
echo "soak FAIL: persistence RSS grew $((RSS - BASE_RSS)) kB (bound $RSS_BOUND_KB)"
FAILED=$((FAILED + 1))
printf 'persistence: RSS grew %s kB\n' "$((RSS - BASE_RSS))" >> "$SUMMARY"
fi
fi
fi
# Kill unconditionally, not just on the bound-socket path — a daemon that
# never bound its socket is still a running process; never assume it died.
kill "$PDPID" 2>/dev/null
wait "$PDPID" 2>/dev/null
# The daemon's own Zig-side verdict rides along for free (6a).
if grep -q "LEAK:" "$PLOG" 2>/dev/null; then
echo "soak FAIL: persistence daemon reported leaked allocations:"
grep "LEAK:" "$PLOG"
FAILED=$((FAILED + 1))
printf 'persistence: LEAK marker\n' >> "$SUMMARY"
fi
if [ "$FAILED" -gt "$PERSIST_FAILED_BEFORE" ]; then
# Same convention as the run loop above: a failure destroys nothing
# that would explain it.
mkdir -p "$FAILDIR"
cp "$PLOG" "$FAILDIR/persistence.log" 2>/dev/null
[ -f "$PCAP" ] && cp "$PCAP" "$FAILDIR/persistence.cap" 2>/dev/null
fi
rm -rf "$PLOG" "$PSOCK" "$PCAP" "$XDG_STATE_HOME"
rm -f "$LOG"
echo "---"
if [ "$FAILED" -eq 0 ]; then
echo "soak OK: $N/$N runs green"
rm -f "$SUMMARY"
exit 0
fi
# The table: failure line -> count/N, with run attribution underneath.
echo "soak FAIL: $FAILED of $N runs failed (logs in $FAILDIR):"
sed 's/^run [0-9]*: //' "$SUMMARY" | sort | uniq -c | sort -rn | \
while read -r c l; do echo " $c/$N $l"; done
cat "$SUMMARY"
exit 1