test/e2e_lib.sh
Ref: Size: 67.6 KiB History
# shellcheck shell=sh
# e2e_lib.sh — sourced by test/e2e.sh, before any group file.
#
# What a group file may assume this has already done: the XDG homes, $SHELL
# and $SSH_AUTH_SOCK are hermetic; $SOCK and $OUT are spelled; the cleanup
# registry exists and the EXIT trap is armed; every helper below is defined.
# What a group file must do in return: register what it creates. A daemon
# through start_daemon, anything else through defer_kill, defer_sock or
# defer_rm. Captures need no registration — spell them from $OUT and the
# trap sweeps them by pattern.
#
# What is NOT here: the scenario count. ok() counts, and the pin that
# gates the count is the runner's, at the bottom of test/e2e.sh — a group
# file knows nothing about how many scenarios the suite has.
# ---- $TMPDIR, trailing slash removed -----------------------------------
# Every temporary path in this suite is spelled `${TMPDIR:-/tmp}/name`, and
# a good few of them are then compared as STRINGS — a layout leaf against
# the socket the daemon bound, a hosts line against the daemon the client
# attached to. macOS sets $TMPDIR to a per-user directory ending in `/`, so
# those paths came out with `//` in the middle: mux records the spelling it
# was handed, the suite spells its expectation the same way, and the two
# still differ when only one of them went through a normalizing step. Two
# groups failed on exactly that. Stripped once here rather than at every
# call site that spells it, and before the first of them.
while :; do
case "${TMPDIR:-}" in
*/) TMPDIR="${TMPDIR%/}" ;;
*) break ;;
esac
done
# An empty $TMPDIR is not a directory; leave it unset so `${TMPDIR:-/tmp}`
# falls back the way every caller here expects.
[ -n "${TMPDIR:-}" ] && export TMPDIR || unset TMPDIR
# ---- the cleanup registry ---------------------------------------------
# Every process and every artifact a leg creates is REGISTERED where it is
# created, and the EXIT trap walks the registers. Before this, cleanup()
# carried four hand-maintained lists naming ~137 $SOCKnn/$DnnPID variables,
# and a leg added without all four edits leaked: 102 captures were missing
# from them by 2026-08-19, enough that `make soak` refused to start. A list
# kept somewhere other than the code it describes is a list that goes
# stale, so there is no list — registering IS the statement that creates
# the thing.
#
# Captures are the exception, and deliberately so: every one of them is
# spelled from $OUT, and the trap removes that whole set by the same
# pattern leak_sweep already reads it by. A pattern cannot forget a leg,
# and forgetting was the failure. What has to be registered is what no
# pattern can find — pids, and the artifacts spelled somewhere other than
# $OUT.
#
# Newline-joined strings rather than arrays: the shebang is /bin/sh and the
# build's shell gate lints this file as POSIX sh. No path here can hold a
# newline (every one is spelled from $TMPDIR, $OUT and $$ in this file),
# and cleanup sets IFS to a newline before walking them, so a $TMPDIR with
# a space in it still reads as one path.
E2E_KILL=""
E2E_SOCK=""
E2E_RM=""
# defer_kill PID... — a process the trap must end. Newest first, so
# teardown runs in the reverse of creation order and a client is signalled
# before the daemon it is attached to.
#
# An `if` rather than `[ -n "$_dk" ] && ...`, for wait_sock's reason: a
# false guard as the last command in a function becomes that function's
# exit status, and under `set -e` registering an empty pid would abort the
# caller.
defer_kill() {
for _dk in "$@"; do
if [ -n "$_dk" ]; then
# Already registered is not an error: pipe_mux names the same
# FIFO on every call, and a leg that starts two daemons on one
# path registers it twice. Registering twice must cost nothing,
# or callers start reasoning about who registered first.
case "
$E2E_KILL" in
*"
$_dk
"*) continue ;;
esac
E2E_KILL="$_dk
$E2E_KILL"
fi
done
}
# defer_sock PATH... — a path a daemon may be listening on. The trap asks
# `mux d stop` there before it kills anything and long before it unlinks the
# path: a daemon this shell never forked — one a proxy, `mux d start -d` or the
# handoff spawned — has no pid here, and the socket is the only handle
# there is.
defer_sock() {
for _ds in "$@"; do
if [ -n "$_ds" ]; then
# Already registered is not an error: pipe_mux names the same
# FIFO on every call, and a leg that starts two daemons on one
# path registers it twice. Registering twice must cost nothing,
# or callers start reasoning about who registered first.
case "
$E2E_SOCK" in
*"
$_ds
"*) continue ;;
esac
E2E_SOCK="$_ds
$E2E_SOCK"
fi
done
}
# hostroom NAME — a hosts file of one leg's own, spelled into $HOSTROOM
# for the command that follows to pass as XDG_STATE_HOME.
#
# A wall is the hosts FILE now: every `mux TARGET` records the daemon it
# attached to, and every `mux` on a TERMINAL opens on every line it finds.
# The suite's shared state home is therefore every daemon the suite has
# ever attached to — so a leg that read it would open on its own tile plus
# a tile for every session every daemon still running holds, and the
# geometry every pty leg asserts would be somebody else's.
#
# Per LEG and not per group: a group's earlier legs leave their own daemons
# running while the later ones attach, so a wall opened on a home the leg
# before it wrote shows that leg's sessions as tiles beside its own.
hostroom() {
HOSTROOM="${TMPDIR:-/tmp}/mux-e2e-room-$1-$$"
rm -rf "$HOSTROOM"
defer_rm "$HOSTROOM"
}
# defer_rm PATH... — an artifact the trap must remove: a key, a generated
# script, a private HOME, a state home. Registered at the line that spells
# the path, so the two can never drift apart. Files go out through the leak
# sweep's banking; directories are removed whole.
defer_rm() {
for _dr in "$@"; do
if [ -n "$_dr" ]; then
# Already registered is not an error: pipe_mux names the same
# FIFO on every call, and a leg that starts two daemons on one
# path registers it twice. Registering twice must cost nothing,
# or callers start reasoning about who registered first.
case "
$E2E_RM" in
*"
$_dr
"*) continue ;;
esac
E2E_RM="$_dr
$E2E_RM"
fi
done
}
SOCK="${TMPDIR:-/tmp}/muxd-e2e-$$.sock"
defer_sock "$SOCK"
OUT="${TMPDIR:-/tmp}/mux-e2e-out-$$"
# M10: hermetic XDG homes. Key-default scenarios must see OUR key or none,
# never the developer's real ~/.config/mux/key.
XDG_CONFIG_HOME="${TMPDIR:-/tmp}/mux-e2e-cfg-$$"
defer_rm "$XDG_CONFIG_HOME"
XDG_STATE_HOME="${TMPDIR:-/tmp}/mux-e2e-state-$$"
defer_rm "$XDG_STATE_HOME"
# M14: and the same for the handoff's per-host cache. It holds a KEY, and
# the M14 scenarios both read and poison it — neither of which may ever
# touch the developer's real ~/.cache/mux.
XDG_CACHE_HOME="${TMPDIR:-/tmp}/mux-e2e-cache-$$"
defer_rm "$XDG_CACHE_HOME"
# And XDG_RUNTIME_DIR, which is not a home but names the DEFAULT SOCKET:
# every leg here spells --sock, but a leg with a bug in it does not — the
# 16_push group's first RED run sent `d upgrade` to the developer's LIVE
# daemon at /run/user/*/muxd.sock, and only the version rule stood between
# that and an e2e Debug build exec'd over their sessions. Isolated, a stray
# default dial reads "nothing listening" instead of someone's real daemon.
XDG_RUNTIME_DIR="${TMPDIR:-/tmp}/mux-e2e-run-$$"
mkdir -p "$XDG_RUNTIME_DIR"
defer_rm "$XDG_RUNTIME_DIR"
export XDG_CONFIG_HOME XDG_STATE_HOME XDG_CACHE_HOME XDG_RUNTIME_DIR
# ...and the same argument for $SHELL, which is not an XDG home but is read
# the same way: every daemon this suite AUTO-STARTS gets no --shell flag and
# resolves $SHELL, so without this the suite runs the developer's login
# shell and its whole rc — arbitrary code, on the session under test.
#
# Found by soak, not by reasoning. The M10 `mux d start -d` block inherited zsh,
# whose plugin manager roots itself at $XDG_CACHE_HOME; pointing that at a
# fresh directory (the line above) made every session re-clone its plugins
# from the network before the shell would answer, and the scenario's marker
# missed its window. The daemon was healthy the whole time. The M13 blocks
# already pin SHELL per command for this reason; hoisting it here covers the
# M10 block too, and any scenario added later that forgets.
SHELL=/bin/sh
export SHELL
# MUX_KEY_FILE belongs to the same family and is cleared here for the same
# forgets-proofing reason: `mux d endpoint` reads it BEFORE the default key
# path, so an operator who happens to have one exported would silently
# change which key the handoff announces — and the scenarios would still
# pass, against the wrong key, until one of them did not.
unset MUX_KEY_FILE
# ...and SSH_AUTH_SOCK, the same family again and the sharpest case in it.
# A daemon that inherited the developer's real agent would hand it to every
# session whose own agent socket failed to bind — and the refusal leg, whose
# whole subject is a shell finding nobody to sign for it, would then be
# asserting against the developer's keyring. The legs that want an agent
# export one per command, at a path this file made.
unset SSH_AUTH_SOCK
# One counter out of a MUX_PREDICT_STATS line. The client prints exactly one
# such line on exit; every field is a key=value pair, so a rename or reorder
# in the client shows up here as an empty read rather than a wrong number.
predict_stat() {
sed -n "s/.*predict .*$2=\([0-9]*\).*/\1/p" "$1" | head -1
}
# Assert one counter, with the whole line in the failure so a wrong number is
# read in context rather than alone.
want_stat() {
_got=$(predict_stat "$1" "$2")
[ -n "$_got" ] || {
echo "e2e FAIL: $4: no predict stats line (wanted $2=$3); got:"
cat "$1"; exit 1;
}
[ "$_got" = "$3" ] || {
echo "e2e FAIL: $4: $2=$_got, want $3"
grep "^predict " "$1" || true
exit 1;
}
}
# The same, as a floor. Its own helper rather than a mode on want_stat:
# exact equality is the right assertion for every correctness counter, and
# a shared one would make it easy to weaken those by habit. Only counters
# that are genuinely timing-dependent belong here.
want_stat_ge() {
_got=$(predict_stat "$1" "$2")
[ -n "$_got" ] || {
echo "e2e FAIL: $4: no predict stats line (wanted $2>=$3); got:"
cat "$1"; exit 1;
}
[ "$_got" -ge "$3" ] || {
echo "e2e FAIL: $4: $2=$_got, want >=$3"
grep "^predict " "$1" || true
exit 1;
}
}
# Wait until PATTERN shows up in FILE (default 15s). Timing that keys off the
# session's own output instead of a fixed sleep: the marker is proof the
# client is attached and idle, which is exactly the state the tear needs.
# E2E_TIME_SCALE multiplies the polling BUDGETS below, and nothing else. It
# exists for test/coverage.sh: a ptrace-traced binary runs about half again
# slower, which is enough to blow a 25s convergence budget that is generous
# at native speed — and the failure reads as "the attach never converged"
# rather than "the tracer is slow", which is a lie about the product.
#
# Budgets only. Not the sleeps between polls, not a `timeout` that a scenario
# asserts on, and never a threshold: scaling this changes how long the suite
# is WILLING TO WAIT, never what it demands to see. A run at scale 4 that
# passes proves the same facts as a run at scale 1, just later.
TIME_SCALE="${E2E_TIME_SCALE:-1}"
case "$TIME_SCALE" in
''|*[!0-9]*) echo "e2e FAIL: E2E_TIME_SCALE must be a positive integer"; exit 1 ;;
esac
[ "$TIME_SCALE" -ge 1 ] || { echo "e2e FAIL: E2E_TIME_SCALE must be >= 1"; exit 1; }
# Validated here rather than at its use in ok(), for the reason above: a bad
# value would otherwise surface as a bare test(1) error at the first passing
# scenario, with no e2e FAIL line to say what was wrong.
case "${E2E_STOP_AFTER:-1}" in
''|*[!0-9]*) echo "e2e FAIL: E2E_STOP_AFTER must be a positive integer"; exit 1 ;;
esac
# rc0 MESSAGE [FILE...] — the assert under a `set +e … RC=$? … set -e`
# capture: RC must be 0 or the suite fails, printing MESSAGE and then the
# FILEs. MESSAGE is the caller's own prose and already holds $RC wherever
# it wants it — rc0 appends no "(rc N)" of its own, because the sites this
# replaced carry rc legends ("134 = panic, 124 = hung") in positions no
# fixed suffix could reproduce. Only the plain rc==0 shape converts to
# this; a block that compares RC to a specific value, kills something, or
# dumps state on the way out keeps its bespoke form.
rc0() {
[ "$RC" -eq 0 ] && return 0
echo "e2e FAIL: $1"; shift
for _f in "$@"; do cat "$_f" 2>/dev/null; done
exit 1
}
wait_for() {
_file="$1"; _pat="$2"; _ticks=$(( ${3:-15} * 10 * TIME_SCALE )); _i=0
while [ "$_i" -lt "$_ticks" ]; do
if [ -f "$_file" ] && grep -q "$_pat" "$_file" 2>/dev/null; then return 0; fi
sleep 0.1; _i=$((_i+1))
done
return 1
}
# hardkill PID — SIGKILL a pid and anything it fathered, children first.
#
# Plain `kill -9 "$pid"` is right only while the pid the suite holds IS the
# process under test. Under the coverage harness (test/coverage.sh) it is a
# kcov wrapper instead, and SIGKILL is the one signal a wrapper cannot
# forward: killing the node alone would leave the real daemon alive holding
# its socket, and every leg that waits for that daemon to die would hang
# rather than fail. Children first so a traced child's death is observed by
# its tracer, which is when kcov writes the coverage it has collected.
#
# With no wrapper in the picture there are no children and this is exactly
# `kill -9`, which is why the abort legs keep the semantics they assert on:
# the daemon still dies by SIGKILL, still without unlinking its socket.
hardkill() {
# The test is what the process IS, not whether it has children. A daemon
# has children too — a session shell per attach — and killing those first
# ends the session CLEANLY, which is the one thing the abort legs must not
# see: they assert on a client whose daemon vanished under it, and a tidy
# session exit takes a different path out of the client (measured: exit
# 128 where the leg wants 0). Only a wrapper gets the two-step treatment.
if [ "$(ps -o comm= -p "$1" 2>/dev/null)" = kcov ]; then
# The CHILD is the daemon the suite means to kill; the pid it holds is
# only the tracer. Kill the child first and the wrapper exits on its
# own, writing the coverage it has collected — measured: a SIGKILLed
# wrapper writes no coverage.db at any --output-interval, so a clean
# exit is the only exit that keeps the data. Then wait for it, because
# killing both at once is the same as never killing the child at all.
for _c in $(pid_children "$1"); do
kill -9 "$_c" 2>/dev/null || true
done
_i=0
while kill -0 "$1" 2>/dev/null && [ "$_i" -lt $(( 20 * TIME_SCALE )) ]; do
sleep 0.05; _i=$((_i + 1))
done
fi
kill -9 "$1" 2>/dev/null || true
}
# hardkill's TERM twin. A signal sent to a kcov wrapper is dropped: ptrace
# intercepts signals bound for the TRACEE, and the tracer has no handler of
# its own — measured on the hub leg, where `kill $W3PID` left both alive and
# wait_pid_gone timed out. TERM the tracee; the wrapper exits with it and
# writes its database. Off the tracer this is a plain kill, same exit status.
# A /proc claim about the daemon is read off the tracee: under `make coverage`
# the pid the suite holds is the tracer's (hardkill's reason), and kcov's own
# cmdline is what a resumed-argv check would otherwise read.
real_pid() {
if [ "$(ps -o comm= -p "$1" 2>/dev/null)" = kcov ]; then
pid_children "$1" | head -1
else
echo "$1"
fi
}
softkill() {
if [ "$(ps -o comm= -p "$1" 2>/dev/null)" = kcov ]; then
_rc=1
for _c in $(pid_children "$1"); do
kill "$_c" 2>/dev/null && _rc=0
done
return $_rc
fi
kill "$1" 2>/dev/null
}
# ---- the OS oracle ------------------------------------------------------
# The helpers themselves are in os_oracle.sh, sourced here where they used
# to be written out. They moved because soak.sh reads two of them as well,
# and its copies were a second spelling of a question this repo means to
# ask in one place. That file is trap-free on purpose: the registry and the
# EXIT trap below are this file's, and it must not bring a second of either.
# shellcheck source=test/os_oracle.sh
. "$(dirname "$0")/os_oracle.sh"
# And the oracle's own pin, which moved out of this file for the same
# reason: it has to be runnable on a box where nothing else here works
# yet, which is the first thing a new OS arm needs. It defines
# oracle_selftest and nothing else on this path.
# shellcheck source=test/oracle_selftest.sh
. "$(dirname "$0")/oracle_selftest.sh"
# Poll until nothing answers on a socket path (2s). Keyed off the daemon's
# own liveness rather than a fixed sleep, same reasoning as wait_for.
wait_gone() {
_i=0
while "$MUX" d dump --sock "$1" > /dev/null 2>&1; do
_i=$((_i + 1)); [ "$_i" -lt $(( 40 * TIME_SCALE )) ] || { echo "e2e FAIL: daemon on $1 never died"; exit 1; }
sleep 0.05
done
}
# wait_sock PATH LOG LABEL — poll until a daemon has bound PATH (5s), then
# ASSERT it. Every `mux d start &` in this file needs this: the bind happens
# after the fork, so the very next command would otherwise race it. LOG is
# the spawn's own capture, printed on failure because a daemon that failed
# to bind almost always said why; pass "" for the spawns that have none.
wait_sock() {
_i=0
while [ ! -S "$1" ] && [ "$_i" -lt $(( 50 * TIME_SCALE )) ]; do sleep 0.1; _i=$((_i+1)); done
[ -S "$1" ] || {
echo "e2e FAIL: $3"
# An `if` rather than `[ -n "$2" ] && cat "$2"`, for converged_quiet's
# reason: a false guard must not become this block's exit status.
if [ -n "$2" ]; then cat "$2"; fi
exit 1
}
}
# start_daemon SOCK LOG LABEL [run flags...] — a daemon in the background,
# registered before it is waited for. Sets $DPID, the pid to end it by.
# $DPID is scratch: the NEXT start_daemon overwrites it, so a leg that
# still needs its daemon further down keeps the pid under a name of its
# own. The long-lived daemon is $D1PID for exactly that reason — it is
# read some 5,000 lines after it is started.
#
# The registration is the whole reason this exists. A `mux d start &` written
# out by hand is three statements — the spawn, the pid, the wait — plus a
# fourth in cleanup() saying how to end it, and the fourth is the one that
# got forgotten. Here the spawn IS the registration and there is no fourth
# place to edit.
#
# Only for the plain shape. A daemon that needs an environment in front of
# it writes its own spawn and calls defer_kill/defer_sock itself: `env` or
# a VAR=VAL prefix on a FUNCTION call does not scope to the function — in
# POSIX sh it assigns in THIS shell and the value stays there for every
# scenario after it.
start_daemon() {
_sds="$1"; _sdl="$2"; _sdlab="$3"; shift 3
"$MUX" d start --sock "$_sds" "$@" > "$_sdl" 2>&1 &
DPID=$!
defer_kill "$DPID"
defer_sock "$_sds"
wait_sock "$_sds" "$_sdl" "$_sdlab"
}
# pipe_mux OUT ERR CMD... — a client on a plain pipe, and the pipe kept
# open. ERR is the stderr capture, "" to leave it on the suite's own.
# The suite's non-tty scenarios used to be `{ printf cmd; sleep 2; printf
# detach; } | "$MUX"`: two seconds bought for output that lands in
# milliseconds (measured: the needle is in the capture 2ms after the send),
# and 120s of the suite's runtime was exactly that purchase. Here the stdin
# is a FIFO held open on fd 9, so a scenario sends, then WAITS FOR THE NEEDLE
# with await_out, then detaches — the same assertion it was going to make
# anyway, moved to where it ends the wait. fd 9 is opened read-write so
# neither the open nor the client's EOF blocks on the other side; the client
# sees EOF only when pipe_detach closes it.
#
# Not for the scenarios that pace on purpose: the prediction legs snapshot
# the capture N ms after a keystroke, and a sleep there IS the test.
pipe_mux() {
PIPE_OUT="$1"; PIPE_ERR="$2"; shift 2
PIPE_IN="$PIPE_OUT.in"
defer_rm "$PIPE_IN"
rm -f "$PIPE_IN"
mkfifo "$PIPE_IN"
exec 9<>"$PIPE_IN"
if [ -n "$PIPE_ERR" ]; then
"$@" < "$PIPE_IN" > "$PIPE_OUT" 2> "$PIPE_ERR" &
else
"$@" < "$PIPE_IN" > "$PIPE_OUT" &
fi
PIPE_PID=$!
defer_kill "$PIPE_PID"
}
# pipe_send FMT [ARGS] — printf into the open client, same spelling the
# inline blocks used so a conversion moves the format string verbatim.
pipe_send() {
# shellcheck disable=SC2059
printf "$@" >&9
}
# await_out FILE NEEDLE LABEL — wait_for with the verdict attached: FAIL
# with the capture when NEEDLE never comes (5s, scaled). The same `grep -q`
# the scenario asserts with afterwards, so waiting on it costs no new claim.
await_out() {
wait_for "$1" "$2" 5 || {
echo "e2e FAIL: $3; never saw '$2' in $1, which holds:"
cat "$1"
exit 1
}
}
# fill_sessions SOCK STATE PREFIX FROM TO — one throwaway pipe attach per
# name so the daemon holds a slot for each. The session outlives the client
# that made it, which is what lets these detach immediately.
#
# THREE at a time, and the number is `max_observers` minus one, not
# `max_clients` minus one. `acceptConn` parks every new connection in an
# observer slot and promotes it to a client only when its attach frame
# arrives, so simultaneous dials contend for the four observer slots.
# Until 2026-09-05 the ones with nowhere to land were closed outright —
# measured at batch 7 on a 32-slot daemon: 11 of 31 fills died with
# "connection to the daemon lost" and the table never filled; at batch 3,
# 31 of 31 land. The daemon now leaves a fifth dial in the kernel backlog
# instead (`Server.freeObserverSlot`), so the batch is no longer what
# keeps the fills alive; it stays because the count arithmetic below is
# written for it and a fixture that races the daemon on purpose belongs
# in a leg that says so, not in a helper every group calls.
#
# STATE is a scratch home so the fills never write a wall line the leg
# later counts. The last name is asked back through mux a so a fill that
# never landed fails here with its own log rather than as the refusal leg
# proving nothing.
fill_sessions() {
_fs_sock="$1"; _fs_state="$2"; _fs_pfx="$3"; _fs_i="$4"; _fs_to="$5"
while [ "$_fs_i" -le "$_fs_to" ]; do
_fs_pids=""; _fs_j=0
while [ "$_fs_j" -lt 3 ] && [ "$_fs_i" -le "$_fs_to" ]; do
{ sleep 1.5; printf '\034\034'; } | XDG_STATE_HOME="$_fs_state" timeout 40 \
"$MUX" --sock "$_fs_sock" --session "$_fs_pfx$_fs_i" \
> "$OUT.fill.$_fs_pfx$_fs_i" 2>&1 &
_fs_pids="$_fs_pids $!"
_fs_i=$((_fs_i + 1)); _fs_j=$((_fs_j + 1))
done
for _fs_p in $_fs_pids; do wait "$_fs_p" || {
echo "e2e FAIL: fill_sessions $_fs_pfx: a fill attach on $_fs_sock exited nonzero:"
cat "$OUT".fill."$_fs_pfx"*; exit 1; }; done
done
"$MUX" a status --sock "$_fs_sock" --session "$_fs_pfx$_fs_to" > "$OUT.fill.$_fs_pfx.st" 2>&1 || {
echo "e2e FAIL: fill_sessions: $_fs_pfx$_fs_to was never created, the daemon is not full:"
cat "$OUT.fill.$_fs_pfx.st"; exit 1; }
}
# hold_clients SOCK STATE COUNT NAMES — open COUNT attaches that STAY open,
# spread round-robin over NAMES (a space-separated list of sessions that
# already exist), so the daemon's CLIENT table fills without its session
# table growing. `fill_sessions` cannot do this job: its attaches detach as
# they go, which is what makes them cheap, and a slot is spent per ATTACH
# rather than per session — so filling a 32-slot table out of sessions alone
# would need all 32 names and leave the session table full too, and then a
# refusal under test could be either table's.
#
# ONE fifo for every holder, opened read-write HERE so it has a writer from
# this shell: each holder gets past `open()` and then blocks on a read that
# never comes. Per-holder fifos would need a spare fd each, which POSIX sh
# has no way to allocate.
#
# Closing that fd does NOT end them, and assuming it did is the trap this
# comment exists to close. A wall treats stdin EOF as "the script that was
# typing has gone", not as a goodbye: `wallview.zig` sets `stdin_open = false`
# and CONTINUES, because the session outlives its typist. That is the same
# reason `fill_sessions` types the detach chord rather than just closing —
# and one shared fifo cannot carry a chord per holder, since whichever holder
# read first would eat it. So `release_holds` signals them and then WATCHES
# the daemon's gauge come back down.
#
# THREE at a time, and for `fill_sessions`' reason: `acceptConn` parks every
# new connection in one of four OBSERVER slots and promotes it to a client
# only when its attach frame lands, so a wider batch contends for those four
# and the ones with nowhere to land are closed outright. Each batch is
# confirmed against the daemon's own gauge before the next dials, so a holder
# that never landed fails here with its own log rather than as a later
# refusal that proves nothing.
HOLD_FIFO=""
HOLD_PIDS=""
HOLD_SOCK=""
HOLD_BASE=""
hold_clients() {
_hc_sock="$1"; _hc_state="$2"; _hc_want="$3"; _hc_names="$4"
# fd 8 is the one fifo (fd 9 is pipe_mux's), so a second open would
# silently strand the first set still attached.
[ -z "$HOLD_FIFO" ] || {
echo "e2e FAIL: hold_clients: holders are already open on $HOLD_FIFO"
echo " — call release_holds before opening another set"
exit 1; }
HOLD_FIFO="$OUT.holdfifo"
HOLD_SOCK="$_hc_sock"
HOLD_PIDS=""
defer_rm "$HOLD_FIFO"
rm -f "$HOLD_FIFO"
mkfifo "$HOLD_FIFO"
exec 8<>"$HOLD_FIFO"
HOLD_BASE=$(clients_now "$_hc_sock")
[ -n "$HOLD_BASE" ] || {
echo "e2e FAIL: hold_clients: $_hc_sock gave no clients= reading to start from"
exit 1; }
_hc_i=0
while [ "$_hc_i" -lt "$_hc_want" ]; do
_hc_j=0
while [ "$_hc_j" -lt 3 ] && [ "$_hc_i" -lt "$_hc_want" ]; do
# Round-robin over the names: `set --` re-splits the list for
# every holder, then shifts to the one this index wants.
# shellcheck disable=SC2086
set -- $_hc_names
_hc_k=$(( _hc_i % $# ))
while [ "$_hc_k" -gt 0 ]; do shift; _hc_k=$((_hc_k - 1)); done
XDG_STATE_HOME="$_hc_state" "$MUX" --sock "$_hc_sock" --session "$1" \
< "$HOLD_FIFO" > "$OUT.hold.$_hc_i" 2>&1 &
HOLD_PIDS="$HOLD_PIDS $!"
defer_kill "$!"
_hc_i=$((_hc_i + 1)); _hc_j=$((_hc_j + 1))
done
_hc_target=$(( HOLD_BASE + _hc_i ))
wait_until 300 \
"hold_clients: $_hc_sock never reached clients=$_hc_target — a holder never attached" \
"[ \"\$(clients_now $_hc_sock)\" -ge $_hc_target ]" \
"\"\$MUX\" d stats --sock $_hc_sock; cat $OUT.hold.*"
done
}
# release_holds — send every holder away and WAIT for the daemon to say the
# slots came back. The wait is the point: closing the fifo alone leaves them
# all attached (see hold_clients above), and a leg that assumed otherwise
# would go on to measure a table that never emptied. Asking the daemon rather
# than trusting the signal is the same discipline as the rest of this file —
# the gauge is the witness, not the kill.
release_holds() {
[ -n "$HOLD_FIFO" ] || return 0
for _rh_p in $HOLD_PIDS; do
kill -TERM "$_rh_p" 2>/dev/null || true
done
exec 8>&-
wait_until 300 \
"release_holds: $HOLD_SOCK never fell back to clients=$HOLD_BASE — a holder kept its slot" \
"[ \"\$(clients_now $HOLD_SOCK)\" -le $HOLD_BASE ]" \
"\"\$MUX\" d stats --sock $HOLD_SOCK"
HOLD_FIFO=""; HOLD_PIDS=""
}
# repaints FILE — how many full repaints (ESC[2J) a capture holds. The
# first paint after a reconnect is always a full one (interact.zig
# `repaint_after_resync`), so a count that rose is the resume itself, seen
# from the terminal's side. Keystrokes typed into a tear are dropped, not
# held, so a scenario that tears the transport must see this before it
# types again.
repaints() {
_clr=$(printf '\033[2J')
grep -a -F -o "$_clr" "$1" 2>/dev/null | wc -l
}
# ssh_shim_head FILE — the prologue every fake ssh in this suite needs,
# written fresh over FILE; the caller appends its own body and chmods.
#
# Real ssh takes options before the host word, and the client spells some:
# `handoff.recipeFor` adds `-o BatchMode=yes` to every recipe that must not
# ask, and the wall's poller runs those. A shim that read `$1` as the host
# would exec `BatchMode=yes ...` and fail exactly like an unreachable box —
# the state the no-start legs assert, so the fixture would be lying in the
# direction of a pass. The arity guard is the other half: `ssh HOST` with
# no command leaves `"$@"` empty after the shift, and a bare exec there is
# a no-op that exits 0 — silence shaped like success. 97 converts that into
# a visible "no announce".
#
# Here, not per group, because the coupling was invisible: the option loop
# is correct in one shim and absent in the other only because those legs
# happen to drive asking recipes, and nothing said so.
ssh_shim_head() {
cat > "$1" <<'SHIMHEAD'
#!/bin/sh
while [ $# -gt 0 ]; do
case "$1" in
-o) shift 2 ;;
-*) shift ;;
*) break ;;
esac
done
[ $# -ge 2 ] || exit 97
SHIMHEAD
}
# rail_cols FILE — the column of every vertical rail in a capture,
# ascending and deduplicated. A rail paints as a CUP followed by reverse
# video, which is how a label bar starts too, so column 1 is a bar and
# never a rail; a caller that cares filters it (`awk '$1 > 1'`).
#
# Here rather than in the four legs that had spelled it out: an escape
# sequence copied per leg is four places a terminal change has to be found,
# and a function is hermetic in the way the `_rail_re` variable those
# copies shared a name for was not.
rail_cols() {
grep -ao $'\x1b\\[[0-9][0-9]*;[0-9][0-9]*H\x1b\\[7m' "$1" |
sed 's/.*\x1b\[[0-9]*;\([0-9]*\)H\x1b\[7m/\1/' | sort -n | uniq
}
# no_saved_tree STATE — forget the layout sidecar under STATE.
#
# A wall on a TERMINAL saves its pane tree on the way out and restore is
# VERBATIM, so a leg that asserts the default cut must not inherit the tree
# an earlier ptyclient run left behind. Whether any run has actually shared
# this home is not the caller's to know: legs move, homes get reused, and
# the failure — a wall cut by somebody else's resize — reads as this leg's
# geometry assertion being wrong.
no_saved_tree() {
rm -f "$1/mux/layout"
}
# seed_layout STATE ORIENT SPELLING... — the wall a `mux` under STATE opens.
#
# The layout file IS the wall: the poll grades panes and adds none, so a
# session nobody wrote into this file is on no wall at all. A leg that used
# to get its tiles from a daemon's list therefore has to AUTHOR them, and
# this is the one place that spells the grammar — `mux-layout 1`, then a
# single `leaf` or an ORIENT container with one indented `leaf` per
# spelling. Every leaf weighs 1, which is an equal cut: a leg that cares
# about the weights writes the file itself and says why.
#
# A SPELLING is `HOST#SESSION` where HOST is the hosts file's own line
# (`--sock PATH`, `box`, `quic://box`) byte for byte — the seed matches
# leaves against host spellings and refuses the whole file over one that
# names no listed daemon.
seed_layout() {
_sl_state="$1"; _sl_orient="$2"; shift 2
mkdir -p "$_sl_state/mux"
{
printf 'mux-layout 1\n'
if [ "$#" -eq 1 ]; then
printf 'leaf 1 %s\n' "$1"
else
printf '%s 0\n' "$_sl_orient"
for _sl in "$@"; do printf ' leaf 1 %s\n' "$_sl"; done
fi
} > "$_sl_state/mux/layout"
}
# await_repaint FILE BASELINE LABEL — wait until repaints FILE exceeds
# BASELINE (5s, scaled), FAIL with the capture otherwise.
await_repaint() {
_i=0
while [ "$(repaints "$1")" -le "$2" ] && [ "$_i" -lt $(( 50 * TIME_SCALE )) ]; do
sleep 0.1; _i=$((_i+1))
done
[ "$(repaints "$1")" -gt "$2" ] || {
echo "e2e FAIL: $3; no repaint after the tear in $1, which holds:"
cat "$1"
exit 1
}
}
# pipe_detach [LABEL] — the Ctrl-\ chord, then EOF, then the client's own
# exit, which must be 0: a foreground `| "$MUX"` under set -e asserted that
# by accident, and this asserts it on purpose. LABEL names the client in
# the verdict; PIPE_RC is left for a scenario with more to say.
pipe_detach() {
printf '\034\034' >&9
pipe_waitexit "$@"
}
# pipe_waitexit [LABEL [WANT]] — the client leaves on its own (its session
# ended); wait for that, then release the FIFO. Stdin stays open until the
# exit so the client never sees an EOF it was not sent. WANT is the exit
# status the scenario means (default 0): the exit-semantics legs want the
# shell's own.
pipe_waitexit() {
set +e
wait "$PIPE_PID"
PIPE_RC=$?
set -e
exec 9>&-
PIPE_PID=""
rm -f "$PIPE_IN"
[ "$PIPE_RC" -eq "${2:-0}" ] || {
echo "e2e FAIL: ${1:-piped client} exited $PIPE_RC (want ${2:-0}; 124 means it hung)"
cat "$PIPE_OUT"
if [ -n "$PIPE_ERR" ]; then cat "$PIPE_ERR"; fi
exit 1
}
}
# dump_session SOCK [NAME] — `mux d dump` against one session (M18). An
# EMPTY or absent NAME passes no --session flag AT ALL rather than an empty
# one, and that is the load-bearing part: no tail is the wire's own
# default-session spelling and the pre-M18 compat path (decision 3), so
# every call site here that never heard of sessions keeps sending exactly
# the bytes it always did. One helper, so a scenario asking about a named
# session and one asking about the default cannot drift into two spellings.
dump_session() {
if [ -n "${2:-}" ]; then
"$MUX" d dump --sock "$1" --session "$2"
else
"$MUX" d dump --sock "$1"
fi
}
# wait_grid SOCK NEEDLE LABEL [SESSION] — poll the daemon's own grid for
# NEEDLE (10s) and ASSERT it landed. The assert is the whole point. The bare
# form of this loop — poll, `&& break`, carry on — falls out silently when
# the marker never arrives, so a session that died at birth is discovered by
# whatever runs next, 15s later and in the wrong layer. Three M-web
# scenarios had grown exactly that shape.
#
# SESSION is optional and defaults to the default session: a marker typed
# into session `a` is not on the default session's grid, so a named-session
# scenario that forgot the argument would poll a grid the marker can never
# reach and fail 10s later blaming the wrong thing.
wait_grid() {
_i=0
while [ "$_i" -lt $(( 100 * TIME_SCALE )) ]; do
dump_session "$1" "${4:-}" 2>/dev/null | grep -q "$2" && return 0
sleep 0.1; _i=$((_i+1))
done
echo "e2e FAIL: $3: '$2' never reached the daemon's grid; it holds:"
dump_session "$1" "${4:-}" || echo "(nothing answers on $1)"
exit 1
}
# wait_sessions SOCK N LABEL — poll until `mux d stats` reports N live
# sessions (10s) and ASSERT it. wait_grid's shape for wait_grid's reason: a
# session is freed asynchronously — its shell exits, the daemon reaps on a
# later pump — so "it died" has to be waited FOR, and a wait that falls out
# silently turns "the session never died" into a puzzling failure two
# assertions further down.
wait_sessions() {
_i=0
while [ "$_i" -lt $(( 100 * TIME_SCALE )) ]; do
"$MUX" d stats --sock "$1" 2>/dev/null | grep -q "sessions=$2" && return 0
sleep 0.1; _i=$((_i+1))
done
echo "e2e FAIL: $3: stats never reported sessions=$2; it holds:"
"$MUX" d stats --sock "$1" || echo "(nothing answers on $1)"
exit 1
}
# wait_until TENTHS LABEL PREDICATE [DUMP] — poll a shell PREDICATE until it
# succeeds, for TENTHS tenths of a second scaled by TIME_SCALE, then ASSERT
# it and print DUMP on failure. wait_grid's discipline for the waits whose
# subject is not a daemon grid — an HTTP body from the browser hub, a
# fixture's capture file. The budget is spelled in tenths because that is
# what the hand-rolled loops it replaces counted, so a conversion is one
# number moved and nothing else.
#
# PREDICATE and DUMP are EVALUATED ON EVERY TICK, so pass them single-quoted
# and let the expansion happen here: a `$(curl ...)` that the call site
# expanded once would poll its own first answer forever and time out on a
# wall that changed a tick later.
#
# eval rather than running "$@" as argv: every predicate these waits need is
# a pipeline or a command substitution, and neither survives being carried
# as a command and its arguments.
wait_until() {
_i=0
while [ "$_i" -lt $(( $1 * TIME_SCALE )) ]; do
# First command of an AND-OR list, which set -e exempts (wait_grid's
# shape): a predicate that is false on this tick is the normal case
# and must not abort the group.
eval "$3" && return 0
sleep 0.1; _i=$((_i+1))
done
echo "e2e FAIL: $2"
# `|| true` because the dump is read in the failure state that provoked
# it — a curl against a hub that died exits 7, and that status would
# become this function's under set -e, replacing the exit 1 the runner
# reads with a number that means something else.
if [ -n "${4:-}" ]; then eval "$4" || true; fi
exit 1
}
# --- tiles attach once each: did the daemon see a second? -------------
#
# The focus model's whole claim is that a tile's connection is its own —
# no new attach, no dial on a focus move. Proving a negative needs a
# daemon-side witness, and there are two here doing different jobs.
#
# THE ASSERTION is `attaches=`, a cumulative counter of every attach this
# daemon accepted (server.zig Stats). Read once before the wall starts and
# once after it exits, the delta is exactly how many times anything attached
# across the whole leg — which for a wall of N tiles must be N, one per tile
# at startup, and never N+1 however far the focus moved. It is read from
# `mux d stats`, which is plain human text and connects as an OBSERVER: the
# reading itself never attaches, so it cannot pollute what it measures.
#
# THE GAUGE is `session NAME clients=`, sampled every 200ms into a file and
# asserted on its peak. It is kept because it localises a failure — it says
# WHICH session grew a second watcher and roughly when — but it is the
# weaker of the two and must never be the only one: a connection that closes
# as another opens never exceeds 1, and one that lives less than a sample
# interval is invisible to it. The counter cannot miss either.
#
# Nothing else may attach while a leg is being measured (`mux a capture` and
# `wait_grid` are clients too), so both readings bracket the ptyclient leg
# and nothing more.
#
# attaches_now SOCK — the daemon's cumulative accepted-attach count.
attaches_now() {
timeout 5 "$MUX" d stats --sock "$1" 2>/dev/null |
sed -n 's/.*[^_]attaches=\([0-9]*\).*/\1/p'
}
# clients_now SOCK — how many of the daemon's `max_clients` slots are HELD
# right now. The daemon-global gauge, not a session's: the same stats line
# goes on to say `session 0 clients=N`, so the match is anchored on the
# ` attaches=` that only ever follows the global one, and `head -1` refuses
# a second reading rather than running two together.
clients_now() {
timeout 5 "$MUX" d stats --sock "$1" 2>/dev/null |
sed -n 's/.* clients=\([0-9]*\) attaches=.*/\1/p' | head -1
}
# assert_attach_delta BEFORE AFTER WANT LABEL — how many attaches happened.
# Empty readings fail loudly rather than arithmetically: `$(())` on an empty
# string is 0, and 0-0=0 would pass this check having measured nothing at
# all — the vacuous green this whole block exists to refuse.
assert_attach_delta() {
[ -n "$1" ] && [ -n "$2" ] || {
echo "e2e FAIL: $4: stats gave no attaches= reading (before='$1' after='$2')"
echo " — the flat-attach claim would be vacuous"
exit 1; }
[ "$(($2 - $1))" = "$3" ] || {
echo "e2e FAIL: $4: the daemon accepted $(($2 - $1)) attaches (want $3)"
echo " — the wall's tiles attach once each and a focus move never does"
exit 1; }
}
# watch_clients SOCK FILE — start sampling in the background.
WATCH_PID=""
watch_clients() {
touch "$2.on"
( while [ -e "$2.on" ]; do
timeout 5 "$MUX" d stats --sock "$1" 2>/dev/null
sleep 0.2
done ) > "$2" &
WATCH_PID=$!
defer_kill "$WATCH_PID"
}
# unwatch_clients FILE — stop sampling.
unwatch_clients() {
rm -f "$1.on"
wait "$WATCH_PID" 2>/dev/null || true
WATCH_PID=""
}
# assert_never_two_clients FILE NAME LABEL — the peak `clients=` this
# session ever showed is 1. Also asserts it was ever 1, which is the anchor:
# a watcher that sampled an empty file, or a wall whose tiles never attached,
# would otherwise pass this leg by having witnessed nothing at all.
#
# Localisation, not proof — `assert_attach_delta` is the proof. This says
# which session grew a second watcher; the counter says whether anything
# attached at all.
assert_never_two_clients() {
_peak=$(grep -o "session $2 clients=[0-9]*" "$1" | sed 's/.*=//' | sort -n | tail -1)
[ -n "$_peak" ] || {
echo "e2e FAIL: $3: no stats sample ever named session $2 — the watch"
echo " saw nothing, so its flat-count claim is vacuous"
exit 1; }
[ "$_peak" = "1" ] || {
echo "e2e FAIL: $3: session $2 was watched by $_peak clients at once"
echo " (want 1) — a focus move attached instead of being local"
exit 1; }
}
# wait_pid_gone PID LABEL — poll until a tracked pid is gone (2s, the same
# budget `mux d stop` gives itself). Its own helper rather than wait_gone's
# socket probe, because the two answer different questions: `mux d: stopped`
# is printed on the first probe that gets a REFUSAL, which is the socket
# being unlinked, and the process can still be a fraction behind that. Only
# a pid can say the daemon itself ended, which is the assertion the M13
# teardowns owe — never the stop command's own claim.
#
# M14 gave it a second kind of caller: the cold-handoff scenario asks the
# same question of the ssh the client is supposed to have killed once QUIC
# took over. So the message names the pid and leaves the EXPECTATION to
# each call site's label — the two are "stop said it stopped" and "QUIC
# said it had taken over", and one wording cannot honestly claim both.
wait_pid_gone() {
_i=0
while kill -0 "$1" 2>/dev/null; do
_i=$((_i + 1)); [ "$_i" -lt 40 ] || {
echo "e2e FAIL: $2: pid $1 is still running 2s later"; exit 1; }
sleep 0.05
done
}
# assert_stopped SOCK PID LABEL ERRFILE — teardown by the sanctioned verb,
# asserted four ways. An exit code says the command RETURNED, not that it
# did the job: `mux d: stopped` is the daemon's own account of it, the absent
# socket is the filesystem's, and only the pid can say the process itself
# ended (see wait_pid_gone). A stop that exited 0 while leaving any of the
# three behind is precisely the regression this shape exists to catch.
# ERRFILE holds the stop's stderr — captured rather than let through,
# because a stop that fails exits 1 and would abort under `set -e` with no
# line of its own, after which the trap deletes the evidence.
#
# The caller nulls its own pid variable afterwards, the way every scenario
# here does: the trap reads those variables, and a pid it still holds is
# how a failing run says which daemon it leaked.
assert_stopped() {
set +e
"$MUX" d stop --sock "$1" 2> "$4"
_rc=$?
set -e
[ "$_rc" = "0" ] || {
echo "e2e FAIL: $3: stop exited $_rc, want 0"; cat "$4"; exit 1; }
grep -q '^mux d: stopped' "$4" || {
echo "e2e FAIL: $3: stop did not report stopped"; cat "$4"; exit 1; }
[ ! -S "$1" ] || {
echo "e2e FAIL: $3: stop left the socket"; ls -l "$1"; exit 1; }
wait_pid_gone "$2" "$3: stop reported stopped"
}
# The transport child for a given socket: a `mux` whose own first two words
# are `d proxy`, on that path. The word test is positional and not a match
# anywhere in the line, because one binary means the CLIENT is a `mux` too
# and the delayed-link scripts name the same socket — a pattern match here
# takes out the very client under test, and so would `pkill -f proxy`.
#
# Read out of `args` alone, and the program name taken as a BASENAME of its
# first word. A `comm` column cannot carry this question across the two
# OSes: BSD ps prints comm as the executable's full path where Linux prints
# the basename, and in a multi-column format it truncates that path to the
# column width, so `$2=="mux"` matched nothing at all on macOS and every
# scenario that tears a transport failed as "could not find the proxy".
proxy_pid() {
ps -eo pid,args |
awk -v s="$1" '{ n = split($2, _p, "/") }
_p[n]=="mux" && $3=="d" && $4=="proxy" && index($0,s) {print $1}' |
head -1
}
# --- M11: render-vs-dump convergence -----------------------------------
# converged_quiet CLIENT_OUT SOCK [COLS ROWS] — render the captured client
# stream and diff it against the daemon's grid, plain and styled. Nonzero on
# divergence, leaving CLIENT_OUT.{render,dump,diff,rvt,dvt} behind for
# inspection. CLIENT_OUT must be PURE client stdout: a capture taken with
# 2>&1 has exit messages and predict stats mixed into the escape stream.
# The optional size is for pty scenarios whose grids are not the non-tty
# 80x24 default; render must replay into the same dimensions the daemon
# holds or the diff compares two honest grids of different shapes.
# It is an INPUT, not something this check validates, and the error is
# one-sided: too small re-wraps or clips a row and diverges loudly (the
# 80x24 default against tp2b's 100-wide final grid does exactly that),
# while too large only adds trailing blanks that the normalization strips,
# and passes. Pass the size the scenario actually ran at.
converged_quiet() {
_co="$1"; _cs="$2"; _sz=""; _drop=""
# An `if` rather than `[ ... ] && _sz=...` so it stays correct if it
# ever ends up the last command in this function: there, a false guard
# would be the function's exit status under `set -e` and a 2-argument
# call would report a divergence it never looked for.
if [ $# -ge 4 ]; then _sz="--cols $3 --rows $4"; fi
# A tty client's top row is the wall's label bar, which the daemon
# grid never held; a fifth argument says how many such rows to cut
# before diffing. assert_converged_pty proves the bar exists first.
if [ $# -ge 5 ]; then _drop="--drop-top $5"; fi
# $_sz/$_drop are flags or nothing, never data.
# shellcheck disable=SC2086
"$RENDER" $_sz $_drop < "$_co" > "$_co.render" || return 1
"$MUX" d dump --sock "$_cs" > "$_co.dump" || return 1
# Trailing whitespace is a formatting difference between two correct
# grids (padded vs unpadded row ends), not a divergence.
sed 's/[[:space:]]*$//' "$_co.render" > "$_co.render.n"
sed 's/[[:space:]]*$//' "$_co.dump" > "$_co.dump.n"
diff -u "$_co.dump.n" "$_co.render.n" > "$_co.diff" || return 1
# Styled, byte-for-byte: both sides come out of the same formatter, so
# equal grids are equal bytes. This is the half that sees a bled SGR
# or a leftover prediction underline — plain text dumps the same glyph
# either way, which is exactly why plain alone cannot carry M9's
# overlay-never-becomes-state invariant.
# Same as above: flags or nothing.
# shellcheck disable=SC2086
"$RENDER" --vt $_sz $_drop < "$_co" > "$_co.rvt" || return 1
"$MUX" d dump --vt --sock "$_cs" > "$_co.dvt" || return 1
# Trailing DEFAULT spaces are normalized away on both sides for the same
# reason the plain leg strips them: the row encoder stops a row at its
# last cell that is not a default blank, so a space the shell wrote at
# the end of a row never leaves the daemon and the client's grid holds an
# erased cell where the daemon's holds a space. The two paint identically.
# A trailing space carrying a colour is NOT a default blank — it is sent,
# and it arrives wrapped in SGR bytes, so the line does not end in
# whitespace and this leaves it alone.
_cr=$(printf '\r')
sed "s/ *\\(${_cr}\\{0,1\\}\\)\$/\\1/" "$_co.dvt" > "$_co.dvt.n"
sed "s/ *\\(${_cr}\\{0,1\\}\\)\$/\\1/" "$_co.rvt" > "$_co.rvt.n"
cmp -s "$_co.dvt.n" "$_co.rvt.n" || return 1
rm -f "$_co.render" "$_co.dump" "$_co.render.n" "$_co.dump.n" \
"$_co.diff" "$_co.rvt" "$_co.dvt" "$_co.rvt.n" "$_co.dvt.n"
return 0
}
# assert_converged CLIENT_OUT SOCK NAME [COLS ROWS] — a scenario's LAST act,
# after quiesce and after the client detached: mid-scenario the dump is still
# moving, and another attach would claim the grid.
CONV_COUNT=0
assert_converged() {
CONV_COUNT=$((CONV_COUNT + 1))
if [ $# -ge 5 ]; then
converged_quiet "$1" "$2" "$4" "$5" ${6:+"$6"}
else
converged_quiet "$1" "$2"
fi || {
echo "e2e FAIL: $3: client render diverges from daemon grid (-daemon +client):"
# An empty diff file is the styled-only case, not a passing one —
# the glyphs agree and the pens do not, which is precisely what the
# byte-exact leg exists to catch, so say so instead of printing
# forty lines of nothing.
if [ -s "$1.diff" ]; then
head -40 "$1.diff"
else
echo "(no plain diff: the grids agree on glyphs and differ on STYLE —"
echo " compare $1.dvt against $1.rvt)"
fi
echo "grids left in $1.render / $1.dump / $1.rvt / $1.dvt"
exit 1
}
}
# assert_converged_pty CLIENT_OUT SOCK NAME COLS ROWS — a TTY client's
# convergence. The wall paints a label bar on the tile's top row (the bar
# follows the tty), so the session grid is the ROWS-1 rows under it.
# Assert the bar really is there FIRST — a --drop-top that cut a content
# row would hide the very divergence the diff exists to catch — then
# converge on the rows below. Counted in CONV_COUNT via assert_converged.
assert_converged_pty() {
"$RENDER" --cols "$4" --rows "$5" < "$1" > "$1.bar" || {
echo "e2e FAIL: $3: render failed on the tty capture"; exit 1; }
head -1 "$1.bar" | grep -Eq '[0-9]+> .+ \[' || {
echo "e2e FAIL: $3: no label bar on the tty client's top row; got:"
head -3 "$1.bar"; exit 1; }
rm -f "$1.bar"
assert_converged "$1" "$2" "$3" "$4" "$5" 1
}
# assert_ws_converged WSOUT SOCK LABEL [SESSION] — the WebSocket leg's
# convergence check. WSOUT is the stand-in's `dumpexit` file: its replica's
# grid, in `mux d dump`'s own format, built from frames that crossed the hub.
# Diff it against the daemon's grid, then doctor a copy of that grid and
# assert the SAME diff catches the doctored one — the wan.sh rule (M9): a
# convergence check that cannot fail proves nothing.
#
# SESSION (M18) names which grid the daemon side of that diff comes from. A
# tile attached to session `a` must be diffed against `a`, and pointing it
# at the default session instead would compare two unrelated grids — the
# check would fail, loudly and for the wrong reason.
#
# Deliberately NOT counted in CONV_COUNT. That pin counts assert_converged
# call sites — render-replays-the-client-stream — and this is different
# machinery answering a different question. Folding the two counts together
# would make either pin's number stop meaning anything.
assert_ws_converged() {
dump_session "$2" "${4:-}" > "$1.dump"
# Trailing whitespace is a formatting difference between two correct
# grids, exactly as in converged_quiet.
sed 's/[[:space:]]*$//' "$1" > "$1.n"
sed 's/[[:space:]]*$//' "$1.dump" > "$1.dump.n"
diff -u "$1.dump.n" "$1.n" > "$1.diff" || {
echo "e2e FAIL: $3: wsclient replica diverges from daemon grid (-daemon +ws):"
head -40 "$1.diff"; exit 1; }
cp "$1.dump.n" "$1.doc"
printf 'doctored-row\n' >> "$1.doc"
if diff -u "$1.doc" "$1.n" > /dev/null 2>&1; then
echo "e2e FAIL: $3: the convergence diff cannot fail (doctored dump passed)"; exit 1
fi
}
# tile_id_of ORIGIN LABEL SESSION — the hub's id for one tile of `GET
# /tiles`, or empty. Ids are BIRTH order and a birth can come from the
# file, from the page's `+` or from a restart, so no leg may predict them;
# the label AND the session together are what names a tile, because a
# daemon with two sessions wears one label twice.
#
# Matched up to the comma after the session and not to the object's `}`:
# `state` follows the session now, so a pattern anchored on the brace
# would find nothing at all — and the comma is what keeps `1` from
# matching `11`.
tile_id_of() {
curl -s "$1/tiles" | tr '{' '\n' |
grep -F "\"label\":\"$2\",\"session\":\"$3\"," |
sed -n 's/^"id":\([0-9][0-9]*\),.*/\1/p' | head -1
}
# tiles_shape ORIGIN — `GET /tiles` with the ids and the states struck
# out: the ORDER and the labels are a leg's to assert, the numbering is
# not, and neither is a grade that changes with every poll. A leg whose
# claim IS the grade greps the raw answer for it.
tiles_shape() {
curl -s "$1/tiles" | sed 's/"id":[0-9]*,//g; s/,"state":"[a-z]*"//g'
}
# Scenario checkpoints. The suite's final line asserts the COUNT of these
# against a literal: adding or removing a scenario means updating that
# literal, and the friction is the feature — a scenario that silently
# stops running is the failure mode the pin exists for.
OK_COUNT=0
ok() {
OK_COUNT=$((OK_COUNT + 1))
echo "e2e OK: $1"
# Scenario boundaries, stamped for whoever needs to attribute something
# to the scenario that produced it. Nothing in this suite reads the file:
# test/coverage.sh maps each kcov database to the scenario that was
# running when the traced process wrote it, and the timestamps are also
# the only per-scenario timing this suite has ever been able to report.
# Milliseconds through the oracle's now_ms, not `date +%s.%N`: BSD date
# has no %N and printed a literal N into the column on a Mac.
#
# An `if` rather than `[ -n ... ] && printf`, for wait_sock's reason: a
# false guard as the last command in a function becomes that function's
# exit status, and under `set -e` every unstamped run would abort at its
# first passing scenario.
if [ -n "${E2E_OK_LOG:-}" ]; then
printf '%s\t%s\t%s\n' "$OK_COUNT" "$(now_ms)" "$1" >> "$E2E_OK_LOG"
fi
# Prefix slicing. This suite is linear and stateful — the M13 scenarios
# run inside sessions the M10 scenarios created — so a prefix is the only
# slice that means anything, and a "run just scenario 40" filter would be
# a filter that lies. Exiting rather than skipping keeps that honest, and
# exiting 0 goes out through the trap, so a sliced run still tears down
# its daemons and still reports its leak verdict.
if [ -n "${E2E_STOP_AFTER:-}" ] && [ "$OK_COUNT" -ge "$E2E_STOP_AFTER" ]; then
echo "e2e STOP: sliced after $OK_COUNT scenarios (E2E_STOP_AFTER)"
exit 0
fi
}
# ---- the leak sweep's two halves (hygiene kit, 6a) ---------------------
# Every capture this suite writes — daemon stderr AND client output — is a
# lifecycle log: a binary that leaked printed a `LEAK:` marker into one. The
# verdict is read in the EXIT trap, which is the only place that sees a
# FAILING run too; under `set -e` the bottom of this file is reached by
# passing runs alone, and a leak introduced alongside a defect is exactly
# the pair a suite should report together.
#
# That leaves the captures the scenarios delete as they go. A file removed
# at line 900 is not there for a trap at line 3000 to read, so removal is
# where its verdict has to be observed: rm_swept banks the marker first and
# deletes second, and the bank is what the trap reads for everything that no
# longer exists. Mid-suite removal of a capture goes through here — a plain
# `rm -f` of one is a verdict thrown away.
LEAKBANK="$OUT.leakbank"
# leak_line FILE — the verdict FILE holds, named and made readable. Never the
# matched line verbatim: half these captures are escape streams, whose one
# "line" is the entire session replay, and printing that repaints the reader's
# terminal instead of reporting to it. A log line short enough to BE a log
# line is kept whole (it names the binary, which is worth having); anything
# longer is cut back to the verdict. `grep -a` because the same captures make
# grep answer "Binary file ... matches", which does not even contain the
# marker a reader — or the sweep's own grep over the bank — is looking for.
leak_line() {
_ll=$(grep -a "LEAK:" "$1" 2>/dev/null | head -1 | tr -d '\000-\011\013-\037')
if [ "${#_ll}" -gt 120 ]; then
_ll=$(printf '%s\n' "$_ll" | sed 's/.*\(LEAK:\)/\1/' | cut -c1-120)
fi
case "$_ll" in
*LEAK:*) ;;
*) _ll="LEAK: marker present, unreadable as text" ;;
esac
echo "$1: $_ll"
}
rm_swept() {
for _rs in "$@"; do
# Regular files only: this list carries sockets and generated
# scripts too, and a socket path is not something to open.
[ -f "$_rs" ] || continue
if grep -q "LEAK:" "$_rs" 2>/dev/null; then
leak_line "$_rs" >> "$LEAKBANK"
fi
done
rm -f "$@"
}
# leak_sweep RC — the whole-suite verdict. RC is the suite's own exit status
# and decides one thing: what an empty sweep means.
#
# On a green run, nothing to read is a failure — the vacuous-green guard this
# gate has always had. Every capture is spelled from $OUT and lives until
# this trap, and the bank exists once a verdict has been banked out of a
# deleted capture, so neither being there means the capture convention broke
# and this gate is reading nothing. Any capture, not one named leg's: keying
# this on $OUT.d1.d failed every E2E_ONLY group but the one that spawns
# daemon one, which is a gate reporting on a fixture rather than on the run
# in front of it. On a run that is ALREADY
# failing, the same emptiness means something else entirely: an early exit,
# before the first daemon ever existed. That is not a second defect, and
# reporting it as one would stack a fabricated failure on top of the real
# one, so it says what it saw and returns clean.
#
# It never speaks for the suite either way: the caller promotes this verdict
# only over a green run (see cleanup).
leak_sweep() {
_lsrc="$1"
_lsbad=0
_lsany=""
for _lsf in "$OUT" "$OUT".*; do
if [ -f "$_lsf" ]; then _lsany=1; break; fi
done
if [ -z "$_lsany" ] && [ ! -e "$LEAKBANK" ]; then
if [ "$_lsrc" -eq 0 ]; then
echo "e2e FAIL: leak sweep found no captures to read"
return 1
fi
echo "e2e note: leak sweep had nothing to read — the run exited $_lsrc" \
"before the first daemon"
return 0
fi
# The captures still on disk, one report line each. `"$OUT"` is named
# alongside the glob because the first client capture is written to the
# bare path, and `"$OUT".*` does not match it.
_lssaid=""
for _lsf in "$OUT" "$OUT".*; do
if [ ! -f "$_lsf" ]; then continue; fi
# The bank is not a capture; it is printed whole below, and running it
# through leak_line would report only its first entry.
if [ "$_lsf" = "$LEAKBANK" ]; then continue; fi
if grep -q "LEAK:" "$_lsf" 2>/dev/null; then
if [ -z "$_lssaid" ]; then
echo "e2e FAIL: a binary reported leaked allocations:"
_lssaid=1
fi
leak_line "$_lsf"
_lsbad=1
fi
done
# ...and the verdicts banked out of captures the suite deleted as it went.
if [ -s "$LEAKBANK" ]; then
echo "e2e FAIL: a binary reported leaked allocations into a capture" \
"the suite has since deleted:"
cat "$LEAKBANK"
_lsbad=1
fi
# The detached (`mux d start -d`) daemons log via XDG_STATE_HOME — and every
# wall on a terminal now runs under a state home of its OWN (hostroom,
# and the per-leg $*STATE dirs), so reading the suite's shared home alone
# swept none of the daemons those legs start: the local auto-start, the
# empty-file wall, the no-start boxes. A gate keyed to a variable legs are
# expected to override is a gate that stops running without saying so.
#
# The REGISTER is the list, not a glob: a state home is a directory, and
# every directory a leg creates goes through defer_rm at the line that
# spells it. One that skipped the register would fail the residue guard in
# cleanup instead, so there is no third place for a home to hide.
#
# The file is APPENDED to at every spawn, so one home holds every
# daemon started under it and the grep below sees all of them.
_lsoifs=$IFS
IFS='
'
_lsseen=""
# shellcheck disable=SC2086 # IFS is a newline: each entry is one path
for _lsd in "$XDG_STATE_HOME" $E2E_RM; do
_lslog="$_lsd/mux/muxd.log"
[ -f "$_lslog" ] || continue
case "
$_lsseen" in
*"
$_lslog
"*) continue ;;
esac
_lsseen="$_lslog
$_lsseen"
if grep -q "LEAK:" "$_lslog"; then
echo "e2e FAIL: a detached daemon reported leaked allocations:"
grep -H "LEAK:" "$_lslog" || true
_lsbad=1
fi
done
IFS=$_lsoifs
[ "$_lsbad" -eq 0 ]
}
# A killed daemon writes its leak verdict on the way OUT, so a sweep that
# reads before the process is gone reads a file the verdict has not reached.
# Bounded, and shared across the pids rather than per-pid: a trap must never
# be the thing that hangs, and a daemon that outlives the wait is swept for
# whatever it did write.
reap_briefly() {
_rbi=0
while [ "$_rbi" -lt 40 ]; do
_rblive=""
for _rbp in "$@"; do
[ -n "$_rbp" ] || continue
if kill -0 "$_rbp" 2>/dev/null; then _rblive=1; fi
done
[ -n "$_rblive" ] || return 0
sleep 0.05
_rbi=$((_rbi + 1))
done
return 0
}
cleanup() {
# The suite's own status, captured before anything in here can overwrite
# it. Every verdict below is composed onto THIS number, never in place of
# it: a run that failed at scenario 6 exits with scenario 6's failure.
_rc=$?
# The registers are newline-joined and the paths in them may hold spaces
# (a $TMPDIR nobody here chose); a newline IFS is what keeps each entry
# one entry.
_oifs=$IFS
IFS='
'
# Sockets first, and by the daemon's own verb. `mux d stop` on a path
# nobody serves is a no-op that exits 0, and on a daemon this shell never
# forked — one a proxy, `mux d start -d` or the handoff spawned — it is the
# only handle there is. Before the kills, and long before the unlink
# below: unlinking first would leave a live daemon nothing could reach
# by path.
for _cs in $E2E_SOCK; do
if [ -S "$_cs" ] && [ -n "${MUX:-}" ]; then
"$MUX" d stop --sock "$_cs" > /dev/null 2>&1 || true
fi
done
# Then every registered process. softkill rather than a plain `kill`,
# which is what this trap used to send: under the coverage harness the
# pid the suite holds is a kcov TRACER, and a signal sent to a tracer is
# dropped — measured on the hub leg, where `kill $W3PID` left both alive.
# TERM rather than hardkill's KILL, because a daemon writes its leak
# verdict on the way OUT and a SIGKILLed one writes nothing: the sweep
# below would then be reading a lifecycle that never ended.
#
# CONT first, and unconditionally: two legs hold their subject under
# SIGSTOP (the mute-offerer's agent, the refusal leg's daemon), and a
# stopped process does not see TERM until it runs again. On anything not
# stopped it is a no-op.
#
# `|| true` on both, for the reason every kill in this trap has always
# carried one: under `set -e` a signal to an already-dead pid would abort
# the trap itself and skip everything below it.
for _ck in $E2E_KILL; do
kill -CONT "$_ck" 2>/dev/null || true
softkill "$_ck" || true
done
# ---- the leak sweep (hygiene kit, 6a) ----
# Here rather than at the bottom of the file, which `set -e` reaches only
# on a passing run: a leak that arrives alongside a defect is reported
# with it. Between the kills above and the rms below, which is the one
# window where every daemon has finished its exit path and every capture
# still exists.
#
# A killed daemon writes its verdict on the way out, so the sweep waits
# for the processes to be gone first.
# shellcheck disable=SC2086 # IFS is a newline: each entry is one word
reap_briefly $E2E_KILL
_leak=0
leak_sweep "$_rc" || _leak=1
# The registered artifacts: sockets, keys, generated scripts, private
# HOMEs, state homes. Through rm_swept, so a registered FILE still banks
# its leak verdict on the way out exactly as a mid-suite removal does;
# directories go whole.
for _cr in $E2E_SOCK $E2E_RM; do
if [ -d "$_cr" ]; then
rm -rf "$_cr"
else
rm_swept "$_cr"
fi
done
IFS=$_oifs
# ...and the captures, by the pattern leak_sweep just read them by rather
# than by a list of some 350 names. The list is what failed: every
# capture had to be remembered in two places, 102 of them were not, and
# by 2026-08-19 `make soak` refused to start. A pattern cannot forget a
# leg.
#
# Green runs only. A failing run keeps its captures — the .render/.dump/
# .diff files a failed assert_converged leaves behind are that failure's
# evidence, and soak moves the whole set into its failure dir for the
# next run's sake.
if [ "$_rc" -eq 0 ]; then
for _co in "$OUT" "$OUT".*; do
[ -e "$_co" ] || continue
rm -rf "$_co"
done
fi
# Residue, in the leak sweep's shape and for its reason: a run that PASSED
# must leave nothing behind, and this trap is the last place that can
# still say so. Pinned to this run's pid so a suite running concurrently
# is never counted, and never deleted.
#
# What it catches now is a path that is neither registered nor spelled
# from $OUT — the one shape both mechanisms above are blind to.
#
# BOTH directories when they differ. Two legs put their daemon sockets
# in /tmp on purpose — a socket path has to fit a label bar as well as
# sun_path — and a sweep that only read $TMPDIR would be blind to
# exactly the paths that were moved out of it.
_stray=0
if [ "$_rc" -eq 0 ]; then
_sweep="${TMPDIR:-/tmp}"
[ "$_sweep" = /tmp ] || _sweep="$_sweep /tmp"
# shellcheck disable=SC2086 # two directory words, and split is the point
_left=$(find $_sweep -maxdepth 1 \
\( -name "mux*-$$" -o -name "mux*-$$.*" \) 2>/dev/null)
if [ -n "$_left" ]; then
_n=$(printf '%s\n' "$_left" | wc -l)
if [ "$_n" -eq 1 ]; then _w=file; else _w=files; fi
echo "e2e FAIL: the suite passed but left $_n $_w in $_sweep:"
printf '%s\n' "$_left" | sed 's/^/ /'
echo " A capture is spelled from \$OUT and swept by pattern;"
echo " anything else is registered where it is created, with"
echo " defer_rm, defer_sock or start_daemon. A path that is"
echo " neither is invisible until soak refuses to start."
printf '%s\n' "$_left" | xargs -r rm -rf
_stray=1
fi
fi
# This trap can change the suite's answer in exactly one direction: a run
# that was green and swept up a leak. Every other path returns normally
# and leaves the status alone — a suite that failed at scenario 6 must
# exit with scenario 6's failure, not with the trap's opinion of it.
if [ "$_rc" -eq 0 ] && { [ "$_leak" -eq 1 ] || [ "$_stray" -eq 1 ]; }; then
exit 1
fi
}
trap cleanup EXIT INT TERM