test/agent.sh
Ref: Size: 46.3 KiB History
#!/bin/sh
# End-to-end for the agent surface: `mux d` and `mux a`, binary level.
# Every verb mux a has prints one JSON object, so every assertion here is made
# against a PARSED object rather than a grep of the line — a field that got
# renamed, or a number that became a string, is a failure this suite can see.
#
# Unlike test/e2e.sh this one does NOT exit on the first failure: each
# scenario is bounded on its own and reports PASS/FAIL/SKIP, and the suite
# exits nonzero at the end if any failed. A run that finds two defects should
# report two, not the first one and a silence.
set -u
# The binary under test, defaulting to the build output next to this
# script's repo. A positional override keeps e2e.sh's convention for a caller
# (build.zig, a packaging check) that wants to name it explicitly.
ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
MUX="${1:-$ROOT/zig-out/bin/mux}"
[ -x "$MUX" ] || { echo "agent FAIL: no mux at $MUX (run: zig build)"; exit 1; }
# The two tools every scenario needs, checked here rather than per-scenario
# because a box without them cannot run ANY of this: python3 parses every
# assertion (see jget) and builds the relay, and `timeout` is what makes a hung
# mux a a failure instead of a wedged suite. Missing either is a refusal to run,
# not a skip — nine SKIP lines and exit 0 would be this suite reporting success
# for work it did not do.
for _tool in python3 timeout; do
command -v "$_tool" >/dev/null 2>&1 ||
{ echo "agent FAIL: no $_tool — this suite cannot assert or bound anything without it"; exit 1; }
done
# One directory for everything this run writes: sockets, keys, captures, the
# relay, the TUI's HOME. Removed by the trap, so a failing run leaves nothing
# behind but its output.
TMP="${TMPDIR:-/tmp}/mux-agent-$$"
mkdir -p "$TMP" || exit 1
# Hermetic homes, for e2e.sh's reasons: the key-bearing scenarios must see OUR
# key and never the developer's ~/.config/mux/key, and $SHELL is read the same
# way — a daemon started without --shell would otherwise run the operator's
# login shell and its whole rc on the session under test.
XDG_CONFIG_HOME="$TMP/cfg"
XDG_STATE_HOME="$TMP/state"
XDG_CACHE_HOME="$TMP/cache"
export XDG_CONFIG_HOME XDG_STATE_HOME XDG_CACHE_HOME
SHELL=/bin/sh
export SHELL
# mux a reads MUX_KEY_FILE BEFORE the default key path, so an operator who
# happens to have one exported would silently change which key every QUIC
# scenario below presents — and they would still pass, against the wrong key.
unset MUX_KEY_FILE
# The OSC 133 injection is opt-IN as of this commit, and marks are the whole
# subject of this suite. Every daemon below is started from this environment,
# so the one export covers them all; the two scenarios that pin the DEGRADED
# path (settle, quiet2) use /bin/sh, which shellint has no scripts for, and
# are unaffected by it.
MUX_SHELL_INTEGRATION=1
export MUX_SHELL_INTEGRATION
# Unix-socket daemons, one per scenario that owns its session's shell.
SOCK_MARKS="$TMP/marks.sock"
SOCK_TUI="$TMP/tui.sock"
SOCK_SETTLE="$TMP/settle.sock"
# QUIC daemons: the one behind the tearable relay, and the one the quiet-await
# scenario dials directly (it must keep the DEFAULT idle timeout, which is the
# very thing it is pinning, so it cannot share the reduced-idle daemon).
SOCK_TEAR="$TMP/tear.sock"
SOCK_QUIET="$TMP/quiet.sock"
# The four UDP ports are picked further down, after the trap: picking one
# binds a socket, and everything that can fail from here on must be able to
# take the tmpdir with it.
PORT_TEAR=""
PORT_RELAY=""
PORT_QUIET=""
PORT_SINK=""
KEY="$TMP/key"
RELAY="$TMP/relay.py"
RELAY_LOG="$TMP/relay.log"
SINK_LOG="$TMP/sink.log"
# The relay's two control files. Creating one is the tear; the relay removes it
# and narrates what it did, so the log is evidence rather than the test's own
# claim about what it asked for.
CTL_FLOW="$TMP/ctl.flow"
CTL_ALL="$TMP/ctl.all"
TUISH="$TMP/tui.sh"
# Every pid the trap may have to kill, declared before the trap is installed:
# under `set -u` a bare $VAR the trap reads would abort the trap itself on a
# failure that happened before the assignment, and the tmpdir would survive.
D_MARKS=""
D_TUI=""
D_SETTLE=""
D_TEAR=""
D_QUIET=""
RELAY_PID=""
SINK_PID=""
CLI_PID=""
cleanup() {
# Every pid this run started, including the ones already dead: a kill that
# finds nothing is not a problem here, which is why the status of each one
# is discarded rather than tested. The `return 0` at the bottom is the load-
# bearing part — without it the trap would exit with the status of whatever
# ran last, and a cleanup that fired on a PASSING run could fail the suite.
for p in "$D_MARKS" "$D_TUI" "$D_SETTLE" "$D_TEAR" "$D_QUIET" \
"$RELAY_PID" "$SINK_PID" "$CLI_PID"; do
[ -n "$p" ] && kill "$p" 2>/dev/null
done
# ...and by socket, for the window between `mux d start -d`'s fork and the
# up-line this script reads its pid from. `mux d stop` on a path nobody
# serves is a no-op. These must precede the rm -rf: unlinking the sockets
# first would leave a live daemon nothing could reach by path.
for s in "$SOCK_MARKS" "$SOCK_TUI" "$SOCK_SETTLE" "$SOCK_TEAR" "$SOCK_QUIET"; do
[ -S "$s" ] && "$MUX" d stop --sock "$s" >/dev/null 2>&1
done
rm -rf "$TMP"
return 0
}
trap 'cleanup' EXIT INT TERM
# --- ports ------------------------------------------------------------------
# A band of its own so a concurrent test/e2e.sh (11000..50000, in 5000-wide
# slots) cannot collide, and derived from $$ so two agent suites usually start
# from different numbers. All four are ports we BIND — but "we bind it" makes
# a derived number a GUESS, not a fact, and this band sits INSIDE the
# ephemeral range on a stock box (32768-60999) and on a tuned one
# (ip_local_port_range is routinely lowered into five figures). Two ways it
# goes wrong, and they are the same bind failure: an unrelated outgoing
# connection is already holding the number, or two suites whose pids are
# congruent mod 900 derived the same four.
#
# So the number is turned into an observation before it is used: free_port
# BINDS each candidate and hands back the first one the kernel actually gave
# it, stepping by PORT_STEP. That leaves only the window between the probe's
# close and the daemon's own bind, which start_quic below covers by retrying
# on a fresh candidate — see there for why the daemon's up-line, not this
# probe, is what finally decides which port a scenario dials.
PORT_STEP=13
PORT_TRIES=12
free_port() {
python3 - "$1" "$PORT_TRIES" "$PORT_STEP" <<'PY'
import socket, sys
port, tries, step = int(sys.argv[1]), int(sys.argv[2]), int(sys.argv[3])
for _ in range(tries):
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
try:
s.bind(("127.0.0.1", port))
except OSError:
port += step
continue
finally:
s.close()
print(port)
raise SystemExit(0)
raise SystemExit(1)
PY
}
# pick_port VAR BASE — free_port, with the refusal spelled out. A box that
# cannot hand out one of twelve candidates is not a box this suite can run
# on, and saying so beats every downstream scenario failing at its dial.
pick_port() {
_pp=$(free_port "$2") || {
echo "agent FAIL: no bindable UDP port in $PORT_TRIES candidates from $2"
exit 1
}
eval "$1=\$_pp"
}
pick_port PORT_TEAR $((51000 + ($$ % 900)))
pick_port PORT_RELAY $((52000 + ($$ % 900)))
pick_port PORT_QUIET $((53000 + ($$ % 900)))
pick_port PORT_SINK $((54000 + ($$ % 900)))
PASSES=0
FAILS=0
SKIPS=0
pass() { PASSES=$((PASSES + 1)); echo "agent PASS: $1"; }
fail() { FAILS=$((FAILS + 1)); echo "agent FAIL: $1"; }
skip() { SKIPS=$((SKIPS + 1)); echo "agent SKIP: $1"; }
# The reason a scenario function gave for stopping. Set by `why`, read by the
# caller: a scenario reports ONE line, and this is how the first failed
# assertion inside it gets into that line.
#
# It also reaps the scenario's in-flight client, because this is the ONLY path
# out of a QUIC scenario that skips the `wait` below it. Without this a failing
# tear scenario leaves a mux a still awaiting on the relay, and the scenario
# after it counts that stranger's flows as its own — one failure would read as
# two, and the second one would be fiction.
WHY=""
why() {
WHY="$1"
[ -n "$CLI_PID" ] && kill "$CLI_PID" 2>/dev/null
CLI_PID=""
return 1
}
# Run a scenario function: 0 passes, 1 fails with $WHY, 2 skips with $WHY.
# One line per scenario, which is what makes the count pin at the bottom mean
# "every scenario ran" rather than "some number of assertions ran".
run_scenario() {
_name="$1"
shift
WHY=""
"$@"
case $? in
0) pass "$_name" ;;
2) skip "$_name: $WHY" ;;
*) fail "$_name: $WHY" ;;
esac
}
# One field out of a JSON object, re-encoded as JSON: `null`, `true`, `0`, and
# `"marks"` WITH its quotes. The quotes are the point. mux a's contract is a
# typed one — exit_code is a number, alt_screen a boolean, mechanism a string —
# and a `str(v)` here would print all three the same way, so a daemon that
# started spelling exit_code as "0" or alt_screen as "true" would sail past
# every assertion below. Re-encoding makes the type part of the comparison, and
# the cost is that string expectations at the call sites carry their quotes too.
#
# The three sentinels cannot collide with any of that: a field whose value were
# literally the text `<missing>` re-encodes to `"<missing>"`, quotes and all. A
# mux a that printed a stack trace fails as `<unparseable>`, not as a mismatch.
jget() {
python3 - "$1" "$2" <<'PY'
import json, sys
try:
obj = json.load(open(sys.argv[1]))
except Exception:
print("<unparseable>"); raise SystemExit(0)
if not isinstance(obj, dict):
print("<not-an-object>"); raise SystemExit(0)
if sys.argv[2] not in obj:
print("<missing>"); raise SystemExit(0)
print(json.dumps(obj[sys.argv[2]]))
PY
}
# want FILE FIELD VALUE — assert one field, naming the whole body on a miss so
# a wrong answer is read in context rather than alone. VALUE is JSON: bare for
# null/true/false/numbers, quoted for strings.
want() {
_got=$(jget "$1" "$2")
[ "$_got" = "$3" ] && return 0
why "$2=$_got, want $3 [$(tr -d '\n' < "$1")]"
}
# Wait until PATTERN shows up in FILE. Keyed off the process's own output
# rather than a fixed sleep, e2e.sh's convention.
wait_for() {
_i=0
while [ "$_i" -lt $((${3:-10} * 20)) ]; do
[ -f "$1" ] && grep -q "$2" "$1" 2>/dev/null && return 0
sleep 0.05
_i=$((_i + 1))
done
return 1
}
# Where a given daemon's OWN stdout+stderr went — its allocator verdict
# among them (hygiene kit, 6a). One per daemon, by tag; see start_daemon.
daemon_log() { echo "$TMP/state/$1/mux/muxd.log"; }
# After a clean `mux d stop`, the daemon's whole lifecycle has run and its
# log carries the allocator's verdict (hygiene kit, 6a). The check runs only
# after the process is actually gone, or the grep races the exit path it is
# asserting about.
#
# leakcheck PID TAG — TAG names the daemon's own log, so this asserts about
# THAT daemon rather than about whichever one spawned most recently.
leakcheck() {
_i=0
while kill -0 "$1" 2>/dev/null && [ "$_i" -lt 100 ]; do
sleep 0.05
_i=$((_i + 1))
done
# An expired poll is not a clean answer: the verdict is written on the way
# out, so a daemon still alive here has written nothing and the grep below
# would read a stale log and call it healthy — swallowing both the
# unreadable verdict AND the `mux d stop` that failed to kill anything.
kill -0 "$1" 2>/dev/null && { why "daemon still alive ${_i}x50ms after stop — leak verdict unreadable"; return 1; }
_dl=$(daemon_log "$2")
# A missing log is a broken convention, not a clean daemon: `mux d start -d`
# creates this file before it forks, so every daemon that ever existed has
# one. Passing on its absence is how this gate would go quietly vacuous.
[ -f "$_dl" ] || { why "no daemon log at $_dl — the leak verdict was never captured"; return 1; }
grep -q "LEAK:" "$_dl" || return 0
why "daemon leaked: $(grep 'LEAK:' "$_dl" | head -1)"
}
# Start a detached daemon and hand back the pid IT reported. Never a pid this
# script guessed from a process name: the suite kills what it started, and a
# name match can only ever name a bystander.
#
# $_log catches what `mux d start -d` ITSELF prints (the up-line, or the refusal).
# The daemon's own output goes somewhere else entirely: `-d` detaches the
# child onto $XDG_STATE_HOME/mux/muxd.log — a path computed from the
# environment `mux d start -d` is run with — and APPENDS to it. One state
# home for the whole suite would therefore interleave every daemon's
# allocator verdict into one file, and a leak found there would name no
# scenario. So each daemon gets a state home of its own, tagged by its
# log's name, and every verdict is attributable at leakcheck and at the
# sweep at the bottom of this file.
start_daemon() {
_log="$1"
shift
_tag=$(basename "$_log" .log)
mkdir -p "$TMP/state/$_tag/mux" || return 1
XDG_STATE_HOME="$TMP/state/$_tag" "$MUX" d start -d "$@" >"$_log" 2>&1
sed -n 's/^up .*pid=\([0-9]*\).*/\1/p' "$_log" | head -1
}
# Poll until mux a can answer on a socket: the session's shell has to have been
# exec'd and the daemon's listener accepted before any assertion means
# anything, and how long that takes is the box's business, not a constant here.
wait_ready() {
_i=0
while [ "$_i" -lt 100 ]; do
"$MUX" a status --sock "$1" --timeout 1000 >/dev/null 2>&1 && return 0
sleep 0.05
_i=$((_i + 1))
done
return 1
}
# start_ready PIDVAR LOG SOCK ARGS... — bring a daemon up and wait for it to
# answer, storing its pid in the named variable and calling `why` with the
# whole story if either half fails. The bring-up is identical in every scenario
# that needs a plain daemon, and a scenario that got only PART of it right — a
# pid but no readiness wait — would fail later, somewhere else, as a flake.
#
# PIDVAR is set the INSTANT the pid is known, before the readiness wait and
# whatever that wait decides. That ordering is the load-bearing part: a daemon
# that came up and then never answered is still a daemon this run started, and
# a version that only assigned on success would leave it running past cleanup.
#
# The two scenarios that do NOT use this (the TUI and the QUIC tear) want a
# daemon that failed to come up to be a stashed message rather than an
# immediate return, and folding that in would cost more than it saves.
start_ready() {
_var="$1"
_log="$2"
_sock="$3"
shift 3
eval "$_var=\$(start_daemon \"\$_log\" \"\$@\")"
eval "_pid=\$$_var"
[ -n "$_pid" ] || why "daemon never printed an up-line [$(cat "$_log")]" || return 1
wait_ready "$_sock" || why "daemon never answered on $_sock" || return 1
return 0
}
# start_quic PORTVAR PIDVAR LOG SOCK ARGS... — start_ready for a daemon that
# also holds a UDP port, brought up on the first candidate port it can
# actually BIND. `--sock SOCK` and `--quic 127.0.0.1:PORT` are supplied here;
# ARGS carries the rest.
#
# The retry is the load-bearing half. free_port proved the number bindable a
# moment ago, and between that proof and this bind a concurrent suite or an
# outgoing connection can take it — the daemon holds no SO_REUSEADDR and refuses to
# share a port rather than split its datagrams, so it prints `a daemon is
# already listening on udp ...` into its own log and exits, and `mux d start -d`
# reports no up-line. That refusal is the ONLY failure another port can fix,
# and it is told apart from the rest by the daemon's own words: anything else
# stops here and is reported, because a daemon that cannot start is a defect and
# marching it through twelve ports would only bury it under a port sweep.
#
# PORTVAR is written back with the port the daemon reported an up-line on, so
# everything downstream — mux a's --quic, the relay's target — dials what was
# BOUND rather than what this suite first derived.
start_quic() {
_qpv="$1"
_qpidv="$2"
_qlog="$3"
_qsock="$4"
shift 4
eval "_qp=\$$_qpv"
_qn=0
while : ; do
_qpid=$(start_daemon "$_qlog" --sock "$_qsock" --quic "127.0.0.1:$_qp" "$@")
if [ -n "$_qpid" ]; then
# Both written the INSTANT they are known, before the readiness
# wait and whatever it decides — start_ready's rule, for its
# reason: a daemon that came up and then never answered is still a
# daemon this run started, and cleanup has to be able to reach it.
eval "$_qpidv=\$_qpid"
eval "$_qpv=\$_qp"
wait_ready "$_qsock" ||
{ why "the QUIC daemon (pid $_qpid) never answered on $_qsock"; return 1; }
return 0
fi
grep -q "listening on udp" "$(daemon_log "$(basename "$_qlog" .log)")" 2>/dev/null || {
why "the QUIC daemon printed no up-line [$(tr -d '\n' < "$_qlog")]"
return 1
}
_qn=$((_qn + 1))
[ "$_qn" -lt "$PORT_TRIES" ] ||
{ why "$_qn candidate udp ports, up to $_qp, were all taken"; return 1; }
_qp=$(free_port $((_qp + PORT_STEP))) ||
{ why "no bindable UDP port left above $_qp"; return 1; }
done
}
now_ms() { python3 -c 'import time; print(int(time.time() * 1000))'; }
# The ports as picked. The two daemon ports can still move from here — a
# daemon that loses a race for one is restarted on the next candidate (see
# start_quic), and the scenarios dial whatever it bound.
echo "agent: ports ${PORT_TEAR}/${PORT_RELAY}/${PORT_QUIET}/${PORT_SINK}, tmp $TMP"
# --- 1: marks. A shell with OSC 133 injected knows its own exit codes -------
# The only mechanism that can report an exit code at all, so all three
# assertions here are really one: `mechanism=marks` is what makes the number
# in `exit_code` the command's rather than a guess.
scen_marks() {
[ -x /bin/bash ] || { WHY="no /bin/bash to inject marks into"; return 2; }
start_ready D_MARKS "$TMP/marks.log" "$SOCK_MARKS" --sock "$SOCK_MARKS" --shell /bin/bash || return 1
timeout 20 "$MUX" a run --sock "$SOCK_MARKS" --timeout 8000 'true' >"$TMP/m1" 2>&1
_rc=$?
[ "$_rc" -eq 0 ] || why "run 'true' exited $_rc [$(tr -d '\n' < "$TMP/m1")]" || return 1
want "$TMP/m1" reason '"returned"' || return 1
want "$TMP/m1" mechanism '"marks"' || return 1
want "$TMP/m1" exit_code 0 || return 1
timeout 20 "$MUX" a run --sock "$SOCK_MARKS" --timeout 8000 'false' >"$TMP/m2" 2>&1
want "$TMP/m2" reason '"returned"' || return 1
want "$TMP/m2" mechanism '"marks"' || return 1
# The command failed; mux a did not. A nonzero exit_code is an ANSWER, and
# an agent that branches on mux a's own status must not see it as an error.
want "$TMP/m2" exit_code 1 || return 1
# A marker the session's shell cannot expand differently than we spell it,
# and one that is unique per run so a stale grid can never satisfy it.
_mark="out-$$"
timeout 20 "$MUX" a run --sock "$SOCK_MARKS" --timeout 8000 "echo $_mark" >"$TMP/m3" 2>&1
want "$TMP/m3" reason '"returned"' || return 1
want "$TMP/m3" mechanism '"marks"' || return 1
want "$TMP/m3" exit_code 0 || return 1
# Exactly the output, not "contains": the span between the two marks is
# the command's transcript, and a prompt or an echoed command line leaking
# into it is the bug this equality is here to catch.
want "$TMP/m3" output "\"$_mark\"" || return 1
"$MUX" d stop --sock "$SOCK_MARKS" >/dev/null 2>&1
leakcheck "$D_MARKS" marks || return 1
D_MARKS=""
return 0
}
run_scenario "marks: exit codes and output come back from a bash session" scen_marks
# --- The ephemeral TUI, which two scenarios share --------------------------
# The spec's field specimen: a daemon whose session is not a shell at all but a
# throwaway full-screen program. `mux d start -d` has no `--` argv, and --shell
# execs whatever path it is given, so a one-line wrapper carries the argument.
# HOME points into the tmpdir for e2e.sh's $SHELL reason: ~/.lesskey and
# ~/.vimrc are arbitrary code on the session under test.
#
# The outcome splits three ways, and the split is the point: no TUI on the box
# is the environment's business and skips, but a TUI that is here and whose
# daemon did not come up is a DEFECT and must fail. Blanking D_TUI for both
# would report a broken daemon as a skip — and would also lose the pid, orphaning
# a daemon that is merely unresponsive rather than dead.
TUI_BIN=""
TUI_QUIT=""
TUI_SKIP=""
TUI_FAIL=""
TUI_OK=""
if command -v vi >/dev/null 2>&1; then
TUI_BIN=$(command -v vi)
TUI_QUIT=':q!\n'
elif command -v less >/dev/null 2>&1; then
TUI_BIN=$(command -v less)
TUI_QUIT='q'
fi
if [ -z "$TUI_BIN" ]; then
TUI_SKIP="neither vi nor less on this box"
else
mkdir -p "$TMP/home"
# Single-quoted in the generated script, both of them: $TMP contains $$ and
# is usually tame, but a TMPDIR with a space in it would otherwise split
# HOME in half and hand `exec` an argument it never meant to have.
{
echo '#!/bin/sh'
echo "HOME='$TMP/home'; export HOME"
# less reads its own switches out of the environment; a developer with
# -F exported would make the session exit before it was ever driven.
echo 'LESS=; export LESS'
echo 'unset LESSOPEN LESSCLOSE'
echo "exec '$TUI_BIN' /etc/hostname"
} > "$TUISH"
chmod +x "$TUISH"
# D_TUI keeps the pid whatever happens next, so cleanup can always reach a
# daemon that came up but never answered.
D_TUI=$(start_daemon "$TMP/tui.log" --sock "$SOCK_TUI" --shell "$TUISH")
if [ -z "$D_TUI" ]; then
TUI_FAIL="the TUI daemon printed no up-line [$(tr -d '\n' < "$TMP/tui.log")]"
elif wait_ready "$SOCK_TUI"; then
TUI_OK=1
else
TUI_FAIL="the TUI daemon (pid $D_TUI) never answered on $SOCK_TUI"
fi
fi
# The gate both TUI scenarios open with: 1 for a defect, 2 for a box that has
# no TUI to drive. `return $?` propagates whichever it was.
tui_gate() {
[ -z "$TUI_FAIL" ] || { WHY="$TUI_FAIL"; return 1; }
[ -n "$TUI_OK" ] || { WHY="$TUI_SKIP"; return 2; }
return 0
}
# --- 2: the alt-screen guard ------------------------------------------------
# Ordered before the drive below because that one ENDS this session. Nothing
# on a full-screen program's grid can mean "the command returned" — there are
# no marks, no prompt, and no rows to attribute — so the honest answer to
# `run` is that the wait timed out. A fabricated `returned` here would be the
# worst failure in the surface: an agent would read an exit code that no
# command ever produced.
scen_altguard() {
tui_gate || return $?
timeout 20 "$MUX" a run --sock "$SOCK_TUI" --timeout 1500 'true' >"$TMP/g1" 2>&1
_rc=$?
want "$TMP/g1" reason '"timeout"' || return 1
# Exit 3 is the whole point of having a code for it: `returned` and
# `settled` are answers and exit 0, a timeout is a question still open.
[ "$_rc" -eq 3 ] || why "exit $_rc, want 3 [$(tr -d '\n' < "$TMP/g1")]" || return 1
return 0
}
run_scenario "alt-screen: run times out rather than fabricating a return" scen_altguard
# --- 3: the ephemeral TUI, driven and quit ---------------------------------
scen_tui() {
tui_gate || return $?
timeout 20 "$MUX" a status --sock "$SOCK_TUI" --timeout 5000 >"$TMP/t1" 2>&1
want "$TMP/t1" alt_screen true || return 1
# Marks are a shell's doing. A program that is not a shell cannot have
# them, and claiming otherwise is what scenario 2 would then read.
_mech=$(jget "$TMP/t1" mechanism)
[ "$_mech" != '"marks"' ] || why "mechanism=marks on a TUI that no shell started" || return 1
timeout 10 "$MUX" a send --sock "$SOCK_TUI" -- "$TUI_QUIT" >"$TMP/t2" 2>&1
want "$TMP/t2" sent true || return 1
# The SESSION ended and the daemon did NOT: `x` ends a session, never a
# box. So the quit is witnessed by the daemon's own table going empty,
# and then by the pid still answering — the daemon has to be alive to
# have said `sessions=0` in the first place, and the `kill -0` is what
# says the process is here rather than merely that the socket is.
_i=0
while ! "$MUX" d stats --sock "$SOCK_TUI" 2>/dev/null | grep -q 'sessions=0'; do
_i=$((_i + 1))
[ "$_i" -lt 100 ] || why "the TUI quit but its daemon still holds a session 5s later" || return 1
sleep 0.05
done
kill -0 "$D_TUI" 2>/dev/null ||
why "the TUI daemon (pid $D_TUI) left with its last session — only mux d stop ends one" || return 1
# The corpse the contract below needs, made deliberately: a daemon that
# is really gone is what an agent's next call has to meet, and nothing
# but a stop produces one now.
"$MUX" d stop --sock "$SOCK_TUI" >/dev/null 2>&1
_i=0
while kill -0 "$D_TUI" 2>/dev/null; do
_i=$((_i + 1))
[ "$_i" -lt 100 ] || why "mux d stop left the TUI daemon (pid $D_TUI) up 5s later" || return 1
sleep 0.05
done
D_TUI=""
# ...and the next call against the corpse is a JSON OBJECT, which is the
# contract an agent depends on: there is no reply mux a can give that an
# agent has to parse as prose, and no path here that panics.
timeout 10 "$MUX" a status --sock "$SOCK_TUI" --timeout 2000 >"$TMP/t3" 2>&1
_rc=$?
[ "$_rc" -ne 0 ] || why "status against a dead daemon exited 0 [$(tr -d '\n' < "$TMP/t3")]" || return 1
_err=$(jget "$TMP/t3" error)
case "$_err" in
"<unparseable>"|"<missing>"|"<not-an-object>")
why "no JSON error object after the session ended: [$(tr -d '\n' < "$TMP/t3")]" || return 1 ;;
esac
# A panic prints a trace and an error message; the grep is what tells the
# two apart when the object above happens to parse anyway.
! grep -qi 'panic\|segmentation\|\.zig:[0-9]' "$TMP/t3" ||
why "a stack trace, not an error object: [$(cat "$TMP/t3")]" || return 1
return 0
}
run_scenario "ephemeral TUI: alt_screen seen, quit driven, death reported as JSON" scen_tui
# --- 4: settle, on a shell with no marks -----------------------------------
# `settled` and `returned` are BOTH honest here and which one arrives is a
# race: --settle 300 accepts 300ms of quiet as the end of the command, and the
# pgid probe sees the foreground group go back to the shell at the second the
# sleep exits. What must never come back is `timeout` — the client asked a
# question a markless session can answer two different ways, and "I don't
# know" is not one of them.
scen_settle() {
start_ready D_SETTLE "$TMP/settle.log" "$SOCK_SETTLE" --sock "$SOCK_SETTLE" --shell /bin/sh || return 1
timeout 20 "$MUX" a run --sock "$SOCK_SETTLE" --settle 300 --timeout 10000 'sleep 1' >"$TMP/s1" 2>&1
_rc=$?
[ "$_rc" -eq 0 ] || why "run exited $_rc [$(tr -d '\n' < "$TMP/s1")]" || return 1
_reason=$(jget "$TMP/s1" reason)
case "$_reason" in
'"settled"'|'"returned"') ;;
*) why "reason=$_reason, want \"settled\" or \"returned\" [$(tr -d '\n' < "$TMP/s1")]" || return 1 ;;
esac
# No marks means no exit code, and mux a says so with a null rather than a
# zero — an agent must never read "it worked" out of a mechanism that
# cannot know.
want "$TMP/s1" exit_code null || return 1
"$MUX" d stop --sock "$SOCK_SETTLE" >/dev/null 2>&1
leakcheck "$D_SETTLE" settle || return 1
D_SETTLE=""
return 0
}
run_scenario "settle: a markless sleep returns an answer, never a timeout" scen_settle
# --- The QUIC half ----------------------------------------------------------
# Everything below needs a key, a daemon holding a UDP port, and — for the two
# tear scenarios — a path this suite can break on purpose WITHOUT root. The
# relay is that path: a UDP forwarder in front of the daemon's port, which
# mux a dials instead. Tearing is a control file, not a signal and not a kill,
# because the flow has to keep being ABSORBED after it breaks: a relay that
# died would have the kernel answer with ICMP port-unreachable, and a refusal
# is the fast path, not the loss this is modelling.
#
# The tear blackholes the flow it is told about and keeps forwarding NEW ones.
# That is exactly a path that went away and a client that came back on another
# one, and it is what makes the heal deterministic: no timing window to hit,
# because the redial's fresh source port is never the torn one.
cat > "$RELAY" <<'PY'
import os, socket, select, sys
listen_port, target_port, ctl_flow, ctl_all = int(sys.argv[1]), int(sys.argv[2]), sys.argv[3], sys.argv[4]
front = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
front.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
front.bind(("127.0.0.1", listen_port))
backs, owner, blocked, block_all = {}, {}, set(), False
sys.stderr.write("relay up %d -> %d\n" % (listen_port, target_port)); sys.stderr.flush()
while True:
if os.path.exists(ctl_flow):
blocked |= set(backs.keys()); os.remove(ctl_flow)
sys.stderr.write("relay: tore %d flow(s)\n" % len(blocked)); sys.stderr.flush()
if os.path.exists(ctl_all):
block_all = True; os.remove(ctl_all)
sys.stderr.write("relay: blackholed everything\n"); sys.stderr.flush()
ready, _, _ = select.select([front] + list(backs.values()), [], [], 0.05)
for s in ready:
if s is front:
data, addr = front.recvfrom(65535)
if block_all or addr in blocked:
continue
b = backs.get(addr)
if b is None:
b = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
b.connect(("127.0.0.1", target_port))
backs[addr], owner[b.fileno()] = b, addr
sys.stderr.write("relay: flow %d\n" % len(backs)); sys.stderr.flush()
try:
b.send(data)
except OSError:
pass
else:
try:
data = s.recv(65535)
except OSError:
continue
addr = owner.get(s.fileno())
if addr is not None and not block_all and addr not in blocked:
front.sendto(data, addr)
PY
# Every way this setup can go wrong is a DEFECT, and every one of them fails
# rather than skips. There is no environmental escape hatch left at this point:
# python3 and `timeout` were made hard prerequisites at the top of the file, so
# what remains — keygen refusing, the key not landing where keygen said it did,
# the QUIC daemon not coming up, the relay not binding — is either the daemon
# misbehaving or this box handing out a port twice. A skip here would be the
# one door in the suite wide enough for a real regression to walk through
# wearing green: `mux d --quic` breaking outright would have reported "4 passed,
# 5 skipped" and exited 0, which is CI saying yes to a broken binary.
QUIC_FAIL=""
if ! "$MUX" d keygen >"$TMP/keygen.log" 2>&1; then
QUIC_FAIL="mux d keygen failed [$(tr -d '\n' < "$TMP/keygen.log")]"
elif ! cp "$XDG_CONFIG_HOME/mux/key" "$KEY" 2>/dev/null; then
QUIC_FAIL="keygen wrote no key where it said it did [$(tr -d '\n' < "$TMP/keygen.log")]"
fi
if [ -z "$QUIC_FAIL" ]; then
# The reduced idle is the schedule the tear scenarios wait for. It is a
# transport parameter, so the NEGOTIATED value is the min of the two ends
# and this daemon's 4s governs both — mux a has no idle flag of its own,
# and waiting out its 15s default twice would be most of this suite's
# runtime. The quiet-await scenario below deliberately does not use it.
start_quic PORT_TEAR D_TEAR "$TMP/tear.log" "$SOCK_TEAR" --shell /bin/bash \
--key "$KEY" --quic-idle-ms 4000 || QUIC_FAIL="$WHY"
fi
if [ -z "$QUIC_FAIL" ]; then
python3 "$RELAY" "$PORT_RELAY" "$PORT_TEAR" "$CTL_FLOW" "$CTL_ALL" >"$RELAY_LOG" 2>&1 &
RELAY_PID=$!
wait_for "$RELAY_LOG" "relay up" 5 || QUIC_FAIL="the relay never bound $PORT_RELAY [$(tr -d '\n' < "$RELAY_LOG")]"
fi
# The gate every QUIC scenario opens with. One outcome only — there is nothing
# left here that a box could legitimately be excused from.
quic_gate() {
[ -z "$QUIC_FAIL" ] || { WHY="$QUIC_FAIL"; return 1; }
return 0
}
# --- 5: a tear mid-await heals, and the command still ran exactly once ------
# The two claims an agent's whole reconnect story rests on. The reply that
# arrives after the heal carries the ORIGINAL command's return — mux a re-issued
# its await from the watermark it already held, so the daemon answered about
# the same command rather than starting a new wait — and the command ran once,
# which is the no-input-resend rule: a client that re-sent its cmdline on
# reconnect would have run it twice, and on anything but `sleep` that is a
# second deploy, not a second read.
scen_tear_heal() {
quic_gate || return $?
_tally="$TMP/tally"
rm -f "$_tally"
# Counted in the FILESYSTEM, not in the grid: a grid count would also see
# the echoed command line, and a wrapped row would make it a guess.
timeout 40 "$MUX" a run --quic "127.0.0.1:$PORT_RELAY" --key "$KEY" --timeout 25000 \
"sleep 12; echo ran >> $_tally" >"$TMP/q1" 2>&1 &
CLI_PID=$!
# Tear once the await is genuinely in flight — proved by the relay having
# opened the flow, not by a sleep that hopes it has.
wait_for "$RELAY_LOG" "relay: flow 1" 10 || why "the client never reached the relay [$(cat "$RELAY_LOG")]" || return 1
sleep 1
: > "$CTL_FLOW"
wait_for "$RELAY_LOG" "relay: tore" 5 || why "the relay never acted on the tear [$(cat "$RELAY_LOG")]" || return 1
wait "$CLI_PID"
_rc=$?
CLI_PID=""
[ "$_rc" -eq 0 ] || why "run exited $_rc after the heal [$(tr -d '\n' < "$TMP/q1")]" || return 1
want "$TMP/q1" reason '"returned"' || return 1
want "$TMP/q1" mechanism '"marks"' || return 1
want "$TMP/q1" exit_code 0 || return 1
# A second flow through the relay is the reconnect, observed from outside
# mux a. Without it the reply could only mean the tear never landed, and
# this scenario would be asserting nothing at all.
grep -q "relay: flow 2" "$RELAY_LOG" ||
why "no second flow: the reply came back without a redial [$(cat "$RELAY_LOG")]" || return 1
_ran=$(wc -l < "$_tally" 2>/dev/null || echo 0)
[ "$_ran" -eq 1 ] || why "the command ran $_ran time(s), want exactly 1" || return 1
return 0
}
run_scenario "quic: a tear mid-await heals, and the command ran exactly once" scen_tear_heal
# --- 6: a tear with nothing to come back to is fatal, and says so ----------
# The reconnect is spent once. When the redial cannot complete, the failure an
# agent reads must be the WHOLE story: the wait died because the path tore, and
# it stayed dead because the redial could not finish. An agent told only
# `Timeout` goes and checks its own command; an agent told `connection lost;
# reconnect failed: Timeout` knows to check the network.
scen_tear_fatal() {
quic_gate || return $?
# A hang is the failure mode here, so the deadline is asserted twice: the
# outer `timeout` makes one impossible to sit through, and the wall clock
# below makes one impossible to pass with.
# The flow number this client will be given, read off the log rather than
# assumed: a scenario above that failed before its redial would leave a
# different count, and a hardcoded 3 would then wait for a flow that never
# comes and report THAT as this scenario's failure.
_flow=$(( $(grep -c "relay: flow" "$RELAY_LOG") + 1 ))
_t0=$(now_ms)
timeout 30 "$MUX" a run --quic "127.0.0.1:$PORT_RELAY" --key "$KEY" --timeout 12000 \
'sleep 20' >"$TMP/q2" 2>&1 &
CLI_PID=$!
wait_for "$RELAY_LOG" "relay: flow $_flow" 10 || why "the client never opened a new flow [$(cat "$RELAY_LOG")]" || return 1
sleep 1
: > "$CTL_ALL"
wait_for "$RELAY_LOG" "blackholed" 5 || why "the relay never blackholed [$(cat "$RELAY_LOG")]" || return 1
wait "$CLI_PID"
_rc=$?
CLI_PID=""
_spent=$(( $(now_ms) - _t0 ))
[ "$_rc" -ne 0 ] || why "run exited 0 with the path gone [$(tr -d '\n' < "$TMP/q2")]" || return 1
[ "$_rc" -ne 124 ] || why "mux a hung past the outer 30s bound" || return 1
# --timeout plus mux a's 2s grace over the daemon's own window, plus room
# for the box. Anything near 30s means the deadline was not honoured.
[ "$_spent" -lt 20000 ] || why "took ${_spent}ms for a 12000ms timeout" || return 1
# The COMPOSED narrative, not merely a prefix of it. mux a has three endings
# for a lost connection and only one of them is honest here: this client
# redialled and the redial could not complete. A bare `connection lost`
# means nothing tried to redial, and `connection lost again, after the one
# reconnect` means the redial was already spent — both would be regressions
# in this setup, and a `"connection lost"*` glob would pass for either.
_detail=$(jget "$TMP/q2" detail)
case "$_detail" in
'"connection lost; reconnect failed: '*) ;;
*) why "detail=$_detail, want '\"connection lost; reconnect failed: ...' [$(tr -d '\n' < "$TMP/q2")]" || return 1 ;;
esac
return 0
}
run_scenario "quic: a tear with no path back fails with the whole story" scen_tear_fatal
# --- 7: a quiet await outlives the idle timeout ----------------------------
# A DIFFERENT daemon, with the default 15s idle: the connection has to be kept
# alive by keepalives across a wait during which neither end has anything to
# say. The failure this pins is a timeout at ~15s reported as a lost
# connection — an agent would go looking for a network fault that never
# happened, and the honest answer (still running) would have been one field.
scen_keepalive() {
quic_gate || return $?
start_quic PORT_QUIET D_QUIET "$TMP/quiet.log" "$SOCK_QUIET" --shell /bin/bash \
--key "$KEY" || return 1
timeout 40 "$MUX" a await --quic "127.0.0.1:$PORT_QUIET" --key "$KEY" --timeout 20000 >"$TMP/q3" 2>&1
_rc=$?
[ "$_rc" -eq 3 ] || why "await exited $_rc, want 3 [$(tr -d '\n' < "$TMP/q3")]" || return 1
want "$TMP/q3" reason '"timeout"' || return 1
# The number is the assertion: 15000 would be the idle timeout wearing a
# timeout's clothes, and only a duration past it proves the keepalives ran.
_dur=$(jget "$TMP/q3" duration_ms)
[ "$_dur" -ge 18000 ] 2>/dev/null ||
why "duration_ms=$_dur — the wait did not survive the 15s idle timeout" || return 1
"$MUX" d stop --sock "$SOCK_QUIET" >/dev/null 2>&1
leakcheck "$D_QUIET" quiet || return 1
D_QUIET=""
return 0
}
run_scenario "quic: a quiet 20s await outlives the 15s idle timeout" scen_keepalive
# --- 8: the session's death beats the connection's ------------------------
# Both ends of this race end the wait, and only one of them is the truth. The
# shell exited 5; the connection then closed BECAUSE it did. Reporting the
# close is reporting the consequence and losing the cause, and the exit code
# is the one thing the agent came for.
#
# Coupling worth naming: this dials the tear daemon DIRECTLY, but it is the
# third scenario to drive that one bash session — 5 and 6 reached it through
# the relay. Scenario 6 leaves a `sleep 20` running there whether it passes or
# fails: killing the client does not kill what the session was already typed.
# What covers it is scenario 7, which spends 20s of its own in between, so the
# sleep is long finished before `exit 5` is ever sent. If 7 itself fails fast
# that margin narrows, hence the 12s bound rather than a snug one — the queued
# `exit 5` still lands, just late. Never a hang either way: the outer `timeout`
# is the backstop.
#
# Not given its own daemon because the session's death IS the assertion — this
# scenario destroys what it runs on, so it goes last among the three regardless.
scen_session_exit() {
quic_gate || return $?
timeout 30 "$MUX" a run --quic "127.0.0.1:$PORT_TEAR" --key "$KEY" --timeout 12000 \
'exit 5' >"$TMP/q4" 2>&1
_rc=$?
# An ANSWER, not a failure: the command is over and this is how.
[ "$_rc" -eq 0 ] || why "run exited $_rc [$(tr -d '\n' < "$TMP/q4")]" || return 1
want "$TMP/q4" reason '"session_ended"' || return 1
want "$TMP/q4" exit_code 5 || return 1
return 0
}
run_scenario "quic: a session that exits 5 reports 5, not a lost connection" scen_session_exit
# The pid is KEPT. The scenario ended this daemon's only session, and a
# daemon lives until `mux d stop` — so it is still running, and the sweep
# below has to wait for its leak verdict like any other daemon's.
# --- 9: a destination that swallows still honours --timeout ----------------
# Not a refusal: a never-listening port answers with ICMP and mux a fails
# instantly, which proves nothing about the deadline. A UDP listener that reads
# and never replies makes the HANDSHAKE hang, and the only thing that can end
# it is mux a's own clock.
scen_blackhole() {
quic_gate || return $?
# The relay, blackholing from birth: its control file exists before it
# starts, so it swallows the first packet it ever sees.
: > "$TMP/sink.all"
python3 "$RELAY" "$PORT_SINK" "$PORT_TEAR" "$TMP/sink.flow" "$TMP/sink.all" >"$SINK_LOG" 2>&1 &
SINK_PID=$!
wait_for "$SINK_LOG" "relay up" 5 || why "the sink never bound $PORT_SINK [$(cat "$SINK_LOG")]" || return 1
_t0=$(now_ms)
timeout 20 "$MUX" a status --quic "127.0.0.1:$PORT_SINK" --key "$KEY" --timeout 2000 >"$TMP/q5" 2>&1
_rc=$?
_spent=$(( $(now_ms) - _t0 ))
kill "$SINK_PID" 2>/dev/null
SINK_PID=""
[ "$_rc" -ne 0 ] || why "status exited 0 against a blackhole [$(tr -d '\n' < "$TMP/q5")]" || return 1
[ "$_rc" -ne 124 ] || why "mux a hung past the outer 20s bound" || return 1
# The ceiling that must NOT be hit is the 15s handshake idle timeout: a
# mux a that ignored --timeout would land there, and this bound is under it
# by enough that only the flag can explain the number.
[ "$_spent" -lt 6000 ] || why "took ${_spent}ms for a 2000ms timeout — the flag was not honoured" || return 1
_err=$(jget "$TMP/q5" error)
case "$_err" in
"<unparseable>"|"<missing>"|"<not-an-object>")
why "no JSON error object [$(tr -d '\n' < "$TMP/q5")]" || return 1 ;;
esac
return 0
}
run_scenario "quic: a blackholed destination fails on --timeout, not on the idle ceiling" scen_blackhole
# --- 10: a live daemon actually answers mux a status --quic -----------------
# The blackhole probe above proves mux a REFUSES a daemon it cannot reach —
# on its own that is consistent with two very different bugs: "refuses dead
# daemons" (the claim) and "refuses every daemon, QUIC included" (what an
# M18 review found: handleFrame's status_req arm dropped a session-less
# slot outright via `orelse return`, so a QUIC connection — promoted to a
# client slot at handshake, before any attach gives it a session — never
# got an answer, and `mux a status --quic` against a perfectly live daemon
# just sat until its own --timeout). This is the other half of that pair:
# the same verb, against a daemon that DOES answer, must actually get a
# reply. SOCK_QUIET/PORT_QUIET are free again — scenario 7 already stopped
# that daemon and leakchecked it — so they are reused rather than adding a
# fourth port this suite has to pick and sweep.
scen_quic_status_live() {
quic_gate || return $?
start_quic PORT_QUIET D_QUIET "$TMP/quiet2.log" "$SOCK_QUIET" --shell /bin/sh \
--key "$KEY" || return 1
timeout 20 "$MUX" a status --quic "127.0.0.1:$PORT_QUIET" --key "$KEY" --timeout 5000 >"$TMP/q6" 2>&1
_rc=$?
[ "$_rc" -eq 0 ] || why "status exited $_rc against a live daemon [$(tr -d '\n' < "$TMP/q6")]" || return 1
# Not just "exit 0" — a real StatusReply decoded into the JSON shape
# mux a's own contract promises, which a silently-empty object could
# still slip past a bare exit-code check.
_cols=$(jget "$TMP/q6" cols)
case "$_cols" in
''|'<missing>'|'<unparseable>'|'<not-an-object>')
why "no cols in the status reply [$(tr -d '\n' < "$TMP/q6")]" || return 1 ;;
esac
"$MUX" d stop --sock "$SOCK_QUIET" >/dev/null 2>&1
leakcheck "$D_QUIET" quiet2 || return 1
D_QUIET=""
return 0
}
run_scenario "quic: mux a status succeeds against a live daemon (not just refuses dead ones)" scen_quic_status_live
# The count, pinned against a literal for e2e.sh's reason: a scenario that
# silently stops running is the failure mode no assertion inside it can catch.
TOTAL=$((PASSES + FAILS + SKIPS))
if [ "$TOTAL" -ne 10 ]; then
echo "agent FAIL: $TOTAL scenarios reported, want 10 — one did not run"
FAILS=$((FAILS + 1))
fi
# ---- per-daemon leak sweep (hygiene kit, 6a) ------------------------------
# Every daemon this run started wrote its verdict into a log of its own (see
# start_daemon), so this reads ALL of them rather than the most recent one's.
# leakcheck already gated the three scenarios that stop their own daemon; what
# lands here is everything else — the QUIC daemon scenario 8 emptied, and any
# daemon a FAILING scenario left behind, which is exactly the case that used
# to go unswept. Emptying a daemon does not remove it from this list: nothing
# but `mux d stop` ends one.
#
# After the count pin on purpose: a leak is not a scenario, and folding it
# into PASSES+FAILS+SKIPS would make that pin's number stop meaning "nine
# scenarios ran".
for _s in "$SOCK_MARKS" "$SOCK_TUI" "$SOCK_SETTLE" "$SOCK_TEAR" "$SOCK_QUIET"; do
[ -S "$_s" ] && "$MUX" d stop --sock "$_s" >/dev/null 2>&1
done
# The verdict is written on the way out, so a daemon still running has not
# written one yet. Bounded, and bounded rather than `wait`ed on because these
# are not this shell's children and because a sweep must never be the thing
# that hangs: a daemon that outlives the wait is read anyway and reports
# whatever it had written, which is the honest answer for a daemon that would
# not die.
_i=0
while [ "$_i" -lt 60 ]; do
_live=""
for _p in "$D_MARKS" "$D_TUI" "$D_SETTLE" "$D_TEAR" "$D_QUIET"; do
[ -n "$_p" ] || continue
if kill -0 "$_p" 2>/dev/null; then _live=1; fi
done
[ -n "$_live" ] || break
sleep 0.05
_i=$((_i + 1))
done
_logs=0
for _dl in "$TMP"/state/*/mux/muxd.log; do
[ -f "$_dl" ] || continue
_logs=$((_logs + 1))
if grep -q "LEAK:" "$_dl"; then
fail "leak sweep: the $(basename "$(dirname "$(dirname "$_dl")")") daemon leaked: $(grep 'LEAK:' "$_dl" | head -1)"
fi
done
# Vacuous-green guard: at least one daemon is started before any scenario can
# report anything, so a sweep with nothing to read means the log convention
# broke and this gate looked at nothing — a failure, never a silent pass.
[ "$_logs" -gt 0 ] || fail "leak sweep found no daemon logs to read under $TMP/state"
echo "agent: $PASSES passed, $FAILS failed, $SKIPS skipped"
[ "$FAILS" -eq 0 ] || exit 1
exit 0