test/e2e_14_upgrade.sh
Ref: Size: 31.0 KiB History
# shellcheck shell=sh
# e2e_14_upgrade.sh — sourced by test/e2e.sh after e2e_lib.sh. Scenarios run in
# the order they stand in; see the lib's header for what this file may
# assume and what it must register.
# `mux d upgrade` — four daemons, because each one is a different daemon
# LIFECYCLE and no two can share a process: the same-binary leg's daemon
# ends up running a second image, the rollback leg's is born with the abort
# armed in its environment, the agent leg's holds an `-A` client, and the
# QUIC leg's owns a UDP socket. They run one after another, so the last of
# them is the only leg here that needs a port.
SOCK69="${TMPDIR:-/tmp}/muxd-e2e-upgrade-$$.sock"
defer_sock "$SOCK69"
SOCK70="${TMPDIR:-/tmp}/muxd-e2e-uproll-$$.sock"
defer_sock "$SOCK70"
SOCK71="${TMPDIR:-/tmp}/muxd-e2e-upagent-$$.sock"
defer_sock "$SOCK71"
SOCK72="${TMPDIR:-/tmp}/muxd-e2e-upquic-$$.sock"
defer_sock "$SOCK72"
SOCK73="${TMPDIR:-/tmp}/muxd-e2e-stopgone-$$.sock"
defer_sock "$SOCK73"
# The next 4000-wide band DOWN from the hydrate leg's 6000: every base from
# 11000 up is taken, and the 61000+ tail above the ephemeral range is spoken
# for by the two browser ports. A collision here reads as this leg's daemon
# failing to bind, the same verdict every other band gives.
UPQPORT=$(( 2000 + ($$ % 4000) ))
UPKEY="${TMPDIR:-/tmp}/mux-e2e-upkey-$$"
defer_rm "$UPKEY"
# A HOME of its own for the session shell. The title assertion below is
# about a title the SESSION set, and a distribution's rc file repaints the
# window title from PROMPT_COMMAND on every prompt (shellint.bash_init says
# so of Arch's) — read out of the developer's own dotfiles, this leg would
# assert on whatever their prompt happens to spell.
UPHOME="${TMPDIR:-/tmp}/mux-e2e-uphome-$$"
defer_rm "$UPHOME"
# The candidate binary the refusal leg offers: a copy, so its exec bit can
# be taken away without touching the one every other scenario runs. Fully
# resolved, because the exec witness below compares it against pid_exe,
# which answers with the resolved image — a $TMPDIR reached through a
# symlink would never equal a logical path and the wait would time out on a
# copy that had in fact exec'd.
UPBIN=$(real_path "${TMPDIR:-/tmp}/mux-e2e-upcand-$$")
defer_rm "$UPBIN"
UPAGENT="${TMPDIR:-/tmp}/mux-e2e-upagent-$$.sock"
defer_rm "$UPAGENT"
UPAGKEY="${TMPDIR:-/tmp}/mux-e2e-upagkey-$$"
defer_rm "$UPAGKEY" "$UPAGKEY.pub"
# --- mux d upgrade: the daemon becomes the new binary, holding everything ---
#
# `mux d upgrade` execs the candidate OVER the running daemon: same pid, same
# children, same descriptors. Nothing on the wire can carry the proof — a
# client reconnecting to a freshly started daemon sees exactly the same
# snapshot — so the witness is the session shell's own pid, put on the grid
# before the exec and read off it after. A restart cannot fake that: a new
# daemon forks a new shell, and a new shell has a new pid.
#
# One daemon, one attached client, and the whole story in order: what the
# session had before, two refusals that must change nothing, the exec, and
# then every one of those things again on the far side.
#
# bash with marks on, because two of the claims are the mechanism's:
# `mux a run` answers under `marks` before the exec and must answer under
# `marks` after it. The FIRST one after is the assertion that matters —
# the manifest carries a return watermark, and one carried into the new
# seq space is a watermark from the future that no return can pass, which
# measured as the first await after every upgrade timing out at 30s while
# the shell had already answered.
UPVER=$("$MUX" d --version | awk '{print $2}')
case "$UPVER" in
?*.?*) ;;
*) echo "e2e FAIL: upgrade: mux d --version gave no version to refuse with: [$UPVER]"
exit 1;;
esac
mkdir -p "$UPHOME"
MUX_SHELL_INTEGRATION=1 HOME="$UPHOME" "$MUX" d start --sock "$SOCK69" \
--shell /bin/bash > "$OUT.up.d" 2>&1 &
D69PID=$!
defer_kill "$D69PID"
wait_sock "$SOCK69" "$OUT.up.d" "upgrade daemon never bound"
# Marks BEFORE the exec, so "marks after" is a comparison rather than a
# hope: a daemon that never had them would fail this line first.
timeout 20 "$MUX" a run --sock "$SOCK69" --timeout 8000 'echo up-marks-pre' > "$OUT.upm1" 2>&1 || {
echo "e2e FAIL: upgrade: mux a run failed before the upgrade even started:"
cat "$OUT.upm1"; exit 1; }
grep -qF '"mechanism":"marks"' "$OUT.upm1" || {
echo "e2e FAIL: upgrade: the session had no marks to lose:"
cat "$OUT.upm1"; exit 1; }
# The attached client, on a real pty, held across the exec. Its script
# stops on `expect up-resumed`, which nothing in the script produces — the
# suite's own post-upgrade `mux a run` does, below. That is the rendezvous:
# the fixture reads the master the whole time it waits, so the capture is
# continuous across the tear, and the needle can only arrive through the
# connection the client re-dialled.
#
# `unset PROMPT_COMMAND; PROMPT_COMMAND=_mux_precmd` keeps mux's own precmd
# (shellint's `precmd_fn`, which is where the marks come from) and drops
# every other member. A distribution rc appends a title-setting member
# there, and it repaints the title on every prompt — the title this leg
# sets would be gone by the next one. `unset` first because a scalar
# assignment onto bash's array spelling lands on element 0 and leaves the
# rest.
hostroom upgrade
XDG_STATE_HOME="$HOSTROOM" timeout 150 "$PTYCLIENT" --cols 100 --rows 30 \
--out "$OUT.upcap" --err "$OUT.upcap.err" -- \
"$MUX" --sock "$SOCK69" > "$OUT.uppc" 2>&1 <<'EOF' &
settle 1000 25000
send unset PROMPT_COMMAND; PROMPT_COMMAND=_mux_precmd\n
settle 800 25000
send printf '\\033]0;uptitle\\007'\n
expect \x1b]0;uptitle\x07 25000
send printf 'up-%s\n' pre\n
expect up-pre 25000
send echo shpid=$$\n
expect shpid= 25000
settle 800 25000
expect \x1b]0;uptitle\x07 90000
expect up-resumed 60000
settle 800 25000
send printf 'up-%s\n' typed\n
expect up-typed 25000
send \x1cd
waitexit 15000
EOF
UPPCPID=$!
defer_kill "$UPPCPID"
# Daemon truth for the pid, and the number this leg is built around. The
# echoed command line spells `$$`, so only the shell's own answer has
# digits after the `=` — grepping for a bare `shpid=` would match the echo.
wait_grid "$SOCK69" "shpid=[0-9]" "upgrade: the session shell never printed its pid"
UPSHPID=$(dump_session "$SOCK69" | sed -n 's/.*shpid=\([0-9][0-9]*\).*/\1/p' | head -1)
[ -n "$UPSHPID" ] || {
echo "e2e FAIL: upgrade: no shell pid read off the grid — the witness would be vacuous"
dump_session "$SOCK69"; exit 1; }
"$MUX" d stats --sock "$SOCK69" > "$OUT.upst1" 2>&1 || {
echo "e2e FAIL: upgrade: stats did not answer before the upgrade"; exit 1; }
UPATT1=$(sed -n 's/.*attaches=\([0-9]*\).*/\1/p' "$OUT.upst1")
UPSNAP1=$(sed -n 's/^snapshots=\([0-9]*\).*/\1/p' "$OUT.upst1")
[ -n "$UPATT1" ] && [ -n "$UPSNAP1" ] || {
echo "e2e FAIL: upgrade: no counters read out of stats; the carry pin would be vacuous:"
cat "$OUT.upst1"; exit 1; }
# 1. Refused: same version, no flag. The reason names BOTH versions, which
# is the difference between a policy an operator can act on and a "no".
set +e
"$MUX" d upgrade --sock "$SOCK69" > "$OUT.upref1" 2>&1
UPRC=$?
set -e
[ "$UPRC" -eq 1 ] || {
echo "e2e FAIL: upgrade: a same-version upgrade exited $UPRC, want 1:"
cat "$OUT.upref1"; exit 1; }
grep -qF "refused: version: $UPVER is not newer than $UPVER" "$OUT.upref1" || {
echo "e2e FAIL: upgrade: the refusal did not name both versions:"
cat "$OUT.upref1"; exit 1; }
# 2. Refused: a candidate that is not executable. `mux d upgrade` offers its
# OWN path (/proc/self/exe), so the only way to spell this refusal is to
# take the exec bit off a copy AFTER it has started — and the ordering is
# made deterministic rather than hoped for. The daemon is held under
# SIGSTOP so it cannot read the request until the chmod has landed; the
# image the OS says that pid is running is the witness that the copy has
# already exec'd, so the chmod cannot beat it and turn the leg into an
# exec failure that asserts nothing.
cp "$MUX_ELF" "$UPBIN"
chmod 755 "$UPBIN"
kill -STOP "$D69PID"
"$UPBIN" d upgrade --sock "$SOCK69" --allow-same-version > "$OUT.upref2" 2>&1 &
UPXPID=$!
defer_kill "$UPXPID"
_i=0
while [ "$(pid_exe "$UPXPID")" != "$UPBIN" ]; do
_i=$((_i + 1))
[ "$_i" -lt $(( 100 * TIME_SCALE )) ] || {
kill -CONT "$D69PID" 2>/dev/null || true
echo "e2e FAIL: upgrade: the candidate copy never exec'd; the refusal below"
echo " would have been about a binary that never asked"
exit 1; }
sleep 0.05
done
chmod 000 "$UPBIN"
kill -CONT "$D69PID"
set +e
wait "$UPXPID"
UPRC=$?
set -e
chmod 755 "$UPBIN"
[ "$UPRC" -eq 1 ] || {
echo "e2e FAIL: upgrade: an unexecutable candidate exited $UPRC, want 1:"
cat "$OUT.upref2"; exit 1; }
grep -qF "refused: path: not executable" "$OUT.upref2" || {
echo "e2e FAIL: upgrade: the refusal did not name the path check:"
cat "$OUT.upref2"; exit 1; }
# ...and two refusals changed NOTHING. A refusal that had torn a client
# down, or written the manifest and left the fd table half open, would
# still have printed the words above.
timeout 20 "$MUX" a status --sock "$SOCK69" > "$OUT.upsta" 2>&1 || {
echo "e2e FAIL: upgrade: the daemon stopped answering after refusing:"
cat "$OUT.upsta"; exit 1; }
wait_grid "$SOCK69" "shpid=$UPSHPID" "upgrade: a refusal cost the session its shell"
# 3. The exec itself.
set +e
"$MUX" d upgrade --sock "$SOCK69" --allow-same-version > "$OUT.upok" 2>&1
UPRC=$?
set -e
[ "$UPRC" -eq 0 ] || {
echo "e2e FAIL: upgrade: the upgrade exited $UPRC, want 0:"
cat "$OUT.upok" "$OUT.up.d"; exit 1; }
grep -qF "mux d: upgraded to $UPVER" "$OUT.upok" || {
echo "e2e FAIL: upgrade: no verdict naming the version it landed on:"
cat "$OUT.upok"; exit 1; }
# The pid is the feature. Not "a daemon is answering" — the same process,
# now running a different image, which the OS's account of its argv says in
# one word: a daemon that had restarted would carry the original
# `run --sock` argv.
kill -0 "$D69PID" 2>/dev/null || {
echo "e2e FAIL: upgrade: the daemon pid $D69PID is gone — that is a restart, not an upgrade"
cat "$OUT.up.d"; exit 1; }
pid_args "$(real_pid "$D69PID")" > "$OUT.upst2" 2>/dev/null || true
grep -qF -- "--resume-fd" "$OUT.upst2" || {
echo "e2e FAIL: upgrade: pid $D69PID is not running the resumed argv; it holds:"
cat "$OUT.upst2"; exit 1; }
# Counters CARRY, and it is asserted HERE rather than at the end for a
# reason about blame: the re-attach poll below counts `attaches` up from
# the reading taken before the exec, so a daemon that had zeroed its
# counters fails THAT poll — reported as a client that never came back,
# which is the wrong diagnosis of the wrong bug. `-ge`, because the
# re-attach this reading races has not necessarily landed yet.
"$MUX" d stats --sock "$SOCK69" > "$OUT.upst2" 2>&1 || {
echo "e2e FAIL: upgrade: stats did not answer after the exec"; cat "$OUT.up.d"; exit 1; }
UPATT2=$(sed -n 's/.*attaches=\([0-9]*\).*/\1/p' "$OUT.upst2")
UPSNAP2=$(sed -n 's/^snapshots=\([0-9]*\).*/\1/p' "$OUT.upst2")
[ -n "$UPATT2" ] && [ -n "$UPSNAP2" ] || {
echo "e2e FAIL: upgrade: no counters read out of stats after the exec:"
cat "$OUT.upst2"; exit 1; }
[ "$UPATT2" -ge "$UPATT1" ] || {
echo "e2e FAIL: upgrade: attaches went $UPATT1 -> $UPATT2 across the exec; the"
echo " counters were reset, not carried, and an upgrade now looks like"
echo " a restart to anything sampling this daemon"
exit 1; }
[ "$UPSNAP2" -ge "$UPSNAP1" ] || {
echo "e2e FAIL: upgrade: snapshots went $UPSNAP1 -> $UPSNAP2 across the exec; the"
echo " counters were reset, not carried"
exit 1; }
# The client came back on its own. Counted with `attaches=`, not with
# `clients=`: the client is re-dialled within a millisecond of the exec, so
# a poll that only looked for clients=1 would be satisfied by the reading it
# took before the tear and would pass on a daemon that never dropped anyone.
_i=0
while : ; do
_upst=$("$MUX" d stats --sock "$SOCK69" 2>/dev/null || true)
_upatt=$(printf '%s' "$_upst" | sed -n 's/.*attaches=\([0-9]*\).*/\1/p')
case "$_upst" in
*"session 0 clients=1"*)
if [ -n "$_upatt" ] && [ "$_upatt" -gt "$UPATT1" ]; then break; fi ;;
esac
_i=$((_i + 1))
[ "$_i" -lt $(( 200 * TIME_SCALE )) ] || {
echo "e2e FAIL: upgrade: the attached client never re-attached after the exec;"
echo " stats holds: $_upst"
cat "$OUT.up.d"; exit 1; }
sleep 0.1
done
# The grid crossed with the process: the marker the client typed, and the
# pid of the shell that is still running under the new image.
wait_grid "$SOCK69" "up-pre" "upgrade: the pre-upgrade marker did not survive the exec"
wait_grid "$SOCK69" "shpid=$UPSHPID" "upgrade: the session is a NEW shell, not the one that crossed"
# The FIRST mux a run after the exec, and it is also what releases the
# attached client's `expect up-resumed` above — one command, two claims.
set +e
timeout 40 "$MUX" a run --sock "$SOCK69" --timeout 20000 'echo up-resumed' > "$OUT.upm2" 2>&1
UPRC=$?
set -e
[ "$UPRC" -eq 0 ] || {
echo "e2e FAIL: upgrade: the FIRST mux a run after the exec exited $UPRC — a carried"
echo " return watermark is a watermark from the future and no return passes it:"
cat "$OUT.upm2"; exit 1; }
grep -qF '"mechanism":"marks"' "$OUT.upm2" || {
echo "e2e FAIL: upgrade: the first await after the exec fell off marks:"
cat "$OUT.upm2"; exit 1; }
grep -qF '"exit_code":0' "$OUT.upm2" || {
echo "e2e FAIL: upgrade: no real exit code came back after the exec:"
cat "$OUT.upm2"; exit 1; }
# The client's own verdict: the title came back on the re-attach, the
# marker arrived through the new image, and a keystroke typed AFTER the
# exec reached the shell.
set +e
wait "$UPPCPID"
UPRC=$?
set -e
UPPCPID=""
[ "$UPRC" -eq 0 ] || {
echo "e2e FAIL: upgrade: the attached client exited $UPRC across the exec:"
cat "$OUT.uppc" "$OUT.upcap.err"; exit 1; }
# A NONZERO exit code too, because `"exit_code":0` is also what a mechanism
# that guesses would say. After the client has gone, not beside it: two
# writers typing into one shell's pty interleave, and the leg would be
# racing itself. Spelled as a CHILD, not as `exit 7`, which is the session
# shell's own exit — the session would end, the daemon with it, and the
# counter reading below would be taken from nothing. Not `(exit 7)` either:
# a subshell sets `_mux_ran` in the subshell, so shellint's `precmd_fn`
# sees an untouched prompt in the parent and emits no `D;` mark at all
# (measured: the await ran its whole budget out).
timeout 40 "$MUX" a run --sock "$SOCK69" --timeout 20000 'sh -c "exit 7"' > "$OUT.upm3" 2>&1 || true
grep -qF '"exit_code":7' "$OUT.upm3" || {
echo "e2e FAIL: upgrade: the shell's own exit code did not survive the exec:"
cat "$OUT.upm3"; exit 1; }
# ...and the re-attach was COUNTED onto them rather than replacing them.
# The carry was asserted above with `-ge`; this is the other half, taken
# once the client's own snapshot has certainly been served.
"$MUX" d stats --sock "$SOCK69" > "$OUT.upst2" 2>&1
UPATT2=$(sed -n 's/.*attaches=\([0-9]*\).*/\1/p' "$OUT.upst2")
UPSNAP2=$(sed -n 's/^snapshots=\([0-9]*\).*/\1/p' "$OUT.upst2")
[ -n "$UPATT2" ] && [ -n "$UPSNAP2" ] || {
echo "e2e FAIL: upgrade: no counters read out of stats after the client left:"
cat "$OUT.upst2"; exit 1; }
[ "$UPATT2" -gt "$UPATT1" ] || {
echo "e2e FAIL: upgrade: attaches is still $UPATT2 — the re-attach after the exec"
echo " was never counted"
exit 1; }
[ "$UPSNAP2" -gt "$UPSNAP1" ] || {
echo "e2e FAIL: upgrade: snapshots is still $UPSNAP2 — the re-attach was served"
echo " without a snapshot, which no fresh epoch allows"
exit 1; }
assert_stopped "$SOCK69" "$D69PID" "upgrade" "$OUT.upstop"
D69PID=""
rm -f "$UPBIN"
rm -rf "$UPHOME"
ok "mux d upgrade keeps the shell, its pid, its title, its marks and its counters"
# --- ...and a candidate that cannot adopt hands the daemon back -----------
#
# The rollback: the new image fails partway through adoption and execs the
# OLD binary — whose path the manifest carries — with the same manifest fd.
# Armed through the ENVIRONMENT rather than the flag it shares a name with,
# and that is not a convenience: `execUpgrade` builds a fixed argv
# (`run --resume-fd N`), so nothing typed here can put `--resume-fail-at`
# in front of the candidate. The daemon's environment is what crosses an
# exec, so the abort is armed on the daemon before any upgrade asks.
#
# The daemon's own log is the only thing that can say a rollback HAPPENED —
# `mux d upgrade` prints its verdict the moment the daemon accepts, and the
# process that would have corrected it is gone. Without that line every
# assertion below would pass just as well on an upgrade that simply worked.
mkdir -p "$UPHOME"
MUX_SHELL_INTEGRATION=1 MUX_RESUME_FAIL_AT=session HOME="$UPHOME" \
start_daemon "$SOCK70" "$OUT.url.d" "rollback daemon never bound" --shell /bin/bash
D70PID=$DPID
timeout 20 "$MUX" a run --sock "$SOCK70" --timeout 8000 'echo rollpid=$$' > "$OUT.urlm1" 2>&1 || {
echo "e2e FAIL: rollback: mux a run failed before the upgrade:"
cat "$OUT.urlm1"; exit 1; }
wait_grid "$SOCK70" "rollpid=[0-9]" "rollback: the session shell never printed its pid"
UPROLLPID=$(dump_session "$SOCK70" | sed -n 's/.*rollpid=\([0-9][0-9]*\).*/\1/p' | head -1)
[ -n "$UPROLLPID" ] || {
echo "e2e FAIL: rollback: no shell pid read off the grid"; dump_session "$SOCK70"; exit 1; }
set +e
"$MUX" d upgrade --sock "$SOCK70" --allow-same-version > "$OUT.urlup" 2>&1
UPRC=$?
set -e
[ "$UPRC" -eq 0 ] || {
echo "e2e FAIL: rollback: upgrade exited $UPRC — the rolled-back daemon must serve"
echo " well enough for the confirm to pass:"
cat "$OUT.urlup" "$OUT.url.d"; exit 1; }
wait_for "$OUT.url.d" "adoption failed at session" 15 || {
echo "e2e FAIL: rollback: the candidate adopted instead of aborting — the abort"
echo " was not armed, and nothing below tests a rollback:"
cat "$OUT.url.d"; exit 1; }
grep -qF "exec'ing" "$OUT.url.d" || {
echo "e2e FAIL: rollback: the abort never named the binary it went back to:"
cat "$OUT.url.d"; exit 1; }
kill -0 "$D70PID" 2>/dev/null || {
echo "e2e FAIL: rollback: the daemon pid $D70PID is gone; a failed adoption took"
echo " the sessions with it"
cat "$OUT.url.d"; exit 1; }
wait_grid "$SOCK70" "rollpid=$UPROLLPID" "rollback: the session lost its shell to a failed adoption"
timeout 40 "$MUX" a run --sock "$SOCK70" --timeout 20000 'echo roll-post' > "$OUT.urlm2" 2>&1 || {
echo "e2e FAIL: rollback: the daemon that came back cannot run a command:"
cat "$OUT.urlm2" "$OUT.url.d"; exit 1; }
grep -qF '"mechanism":"marks"' "$OUT.urlm2" || {
echo "e2e FAIL: rollback: the re-adopted session lost its marks:"
cat "$OUT.urlm2"; exit 1; }
assert_stopped "$SOCK70" "$D70PID" "rollback" "$OUT.urlstop"
D70PID=""
rm -rf "$UPHOME"
ok "a candidate that cannot adopt execs the old binary back, session and marks intact"
# --- ...and the session's agent socket crosses the exec -------------------
#
# The per-session agent listener is a descriptor the daemon owns and the
# shell reaches by path ($SSH_AUTH_SOCK, named after the daemon's pid). It
# has to cross the exec on both counts: the fd by inheritance, the path
# because the pid does not change. Nothing else in this suite would notice
# if it did not — the agent channels themselves are deliberately NOT
# carried, so the only thing that can say the listener survived is a real
# `ssh-add -l` answering through a re-dialled `-A` client.
#
# Its own agent rather than the forwarding legs' AGENT48: that one is left
# under SIGSTOP by the mute-offerer leg, and a mute agent is exactly what
# this leg cannot tell from a broken listener.
start_daemon "$SOCK71" "$OUT.uag.d" "agent-upgrade daemon never bound" --shell /bin/sh
D71PID=$DPID
ssh-agent -a "$UPAGENT" > "$OUT.uagenv" 2>&1
UPAGPID=$(sed -n 's/.*SSH_AGENT_PID=\([0-9]*\).*/\1/p' "$OUT.uagenv")
defer_kill "$UPAGPID"
[ -n "$UPAGPID" ] || {
echo "e2e FAIL: agent-upgrade: ssh-agent printed no pid for the trap to hold it by:"
cat "$OUT.uagenv"; exit 1; }
ssh-keygen -q -t ed25519 -N '' -C mux-e2e-upgrade -f "$UPAGKEY"
SSH_AUTH_SOCK="$UPAGENT" ssh-add "$UPAGKEY" > /dev/null 2>&1
UPFP=$(ssh-keygen -lf "$UPAGKEY" | awk '{print $2}')
case "$UPFP" in
SHA256:?*) ;;
*) echo "e2e FAIL: agent-upgrade: ssh-keygen -lf gave no fingerprint: [$UPFP]"; exit 1;;
esac
pipe_mux "$OUT.uag" "$OUT.uag.err" env SSH_AUTH_SOCK="$UPAGENT" timeout 90 \
"$MUX" -A --sock "$SOCK71"
UPATT1=$("$MUX" d stats --sock "$SOCK71" | sed -n 's/.*attaches=\([0-9]*\).*/\1/p')
[ -n "$UPATT1" ] || { echo "e2e FAIL: agent-upgrade: no attaches counter to compare against"; exit 1; }
# `agpre=` rather than the fingerprint alone: the shell echoes what is
# typed, and `$?` is not expanded in the echo, so only ssh-add's own status
# can put a digit there. The fingerprint is asserted beside it because a
# status of 0 says an agent answered, not that it was the client's.
pipe_send 'ssh-add -l; echo agpre=$?\n'
await_out "$OUT.uag" "agpre=0" "agent-upgrade: ssh-add never answered before the upgrade"
# Counted by OCCURRENCE, not by matching line: this capture is an escape
# stream whose one "line" is the whole session, so `grep -c` would answer 1
# for both answers and the comparison after the exec would be vacuous.
UPFP1=$(grep -aoF -- "$UPFP" "$OUT.uag" | wc -l)
[ "$UPFP1" -ge 1 ] || {
echo "e2e FAIL: agent-upgrade: the client's key never came back before the upgrade"
echo " (wanted $UPFP)"; exit 1; }
set +e
"$MUX" d upgrade --sock "$SOCK71" --allow-same-version > "$OUT.uagup" 2>&1
UPRC=$?
set -e
[ "$UPRC" -eq 0 ] || {
echo "e2e FAIL: agent-upgrade: upgrade exited $UPRC:"
cat "$OUT.uagup" "$OUT.uag.d"; exit 1; }
_i=0
while : ; do
_upst=$("$MUX" d stats --sock "$SOCK71" 2>/dev/null || true)
_upatt=$(printf '%s' "$_upst" | sed -n 's/.*attaches=\([0-9]*\).*/\1/p')
case "$_upst" in
*"session 0 clients=1"*)
if [ -n "$_upatt" ] && [ "$_upatt" -gt "$UPATT1" ]; then break; fi ;;
esac
_i=$((_i + 1))
[ "$_i" -lt $(( 200 * TIME_SCALE )) ] || {
echo "e2e FAIL: agent-upgrade: the -A client never re-attached; stats holds: $_upst"
cat "$OUT.uag.d"; exit 1; }
sleep 0.1
done
# The same question a second time, and `agdone=` is a needle the first
# answer cannot satisfy.
pipe_send 'ssh-add -l; echo agdone=$?\n'
await_out "$OUT.uag" "agdone=0" "agent-upgrade: ssh-add stopped answering across the exec"
UPFP2=$(grep -aoF -- "$UPFP" "$OUT.uag" | wc -l)
[ "$UPFP2" -gt "$UPFP1" ] || {
echo "e2e FAIL: agent-upgrade: ssh-add exited 0 but the key came back $UPFP1 time(s)"
echo " before the exec and $UPFP2 after — no NEW answer crossed"; exit 1; }
pipe_detach "agent-upgrade client"
# A shell born of the UPGRADED daemon: a fresh named session, so its shell
# was forked by the exec'd image. Its fd table is the witness — asked of
# the OS, because a daemon cannot see its own leak. Before the fix every
# post-exec shell held the daemon's listener and agent sockets.
pipe_mux "$OUT.uagn" "$OUT.uagn.err" timeout 60 "$MUX" --sock "$SOCK71" --session post
pipe_send 'echo newsh=$$\n'
wait_grid "$SOCK71" "newsh=[0-9]" "agent-upgrade: the post-exec session never printed its pid" post
UPNEWSH=$(dump_session "$SOCK71" post | sed -n 's/.*newsh=\([0-9][0-9]*\).*/\1/p' | head -1)
[ -n "$UPNEWSH" ] || {
echo "e2e FAIL: agent-upgrade: no post-exec shell pid read off the grid"; dump_session "$SOCK71" post; exit 1; }
# The three fd KINDS a daemon holds and a session shell must not: a
# listening socket, the upgrade manifest's carrier, and a pty master. The
# master is a class of its own — a shell's OWN terminal is the SLAVE
# (`/dev/pts/N`, dup2'd onto 0/1/2), so a master in a shell's fd table can
# only be an adopted session's, inherited. Asked through the oracle by KIND
# rather than by the strings Linux happens to print, so a second OS answers
# the same three questions with whatever names it uses.
UPLEAK=$(( $(pid_holds_fd_kind "$UPNEWSH" socket) \
+ $(pid_holds_fd_kind "$UPNEWSH" carrier) \
+ $(pid_holds_fd_kind "$UPNEWSH" ptymaster) ))
[ "$UPLEAK" -eq 0 ] || {
echo "e2e FAIL: agent-upgrade: the post-exec shell holds $UPLEAK daemon fd(s) — listeners, pty"
echo " masters or the manifest carrier crossed the exec without FD_CLOEXEC and were inherited:"
pid_fd_targets "$UPNEWSH"; exit 1; }
pipe_detach "post-exec session client"
# The other half of the same claim, asked of the daemon's own fd table: the
# manifest carrier is adoption scaffolding, so once adoption is final
# nothing should still hold it open — a survivor accumulates one per
# upgrade. `carrier` is the only KIND asked for, because the manifest is
# the only carrier this daemon ever opens.
UPDPID=$(real_pid "$D71PID")
UPDMEM=$(pid_holds_fd_kind "$UPDPID" carrier)
[ "$UPDMEM" -eq 0 ] || {
echo "e2e FAIL: agent-upgrade: the daemon still holds $UPDMEM manifest carrier(s) after the exec"
pid_fd_targets "$UPDPID"; exit 1; }
assert_stopped "$SOCK71" "$D71PID" "agent-upgrade" "$OUT.uagstop"
D71PID=""
softkill "$UPAGPID" || true
wait_pid_gone "$UPAGPID" "agent-upgrade: ssh-agent killed by tracked pid"
UPAGPID=""
ok "the session's agent socket crosses the exec: ssh-add -l answers again"
# --- ...and a QUIC client is serving again in a breath, not after a timeout
#
# Unix clients need no goodbye — their fds are close-on-exec and the exec is
# their EOF. A QUIC peer has no such thing: silence is indistinguishable from
# a bad network, and a client that waited it out would sit on a blank wall
# for the whole idle timeout. So the daemon sends CONNECTION_CLOSE to every
# connection before it execs, and what that buys is the number asserted
# here: back in service in a fraction of `--quic-idle-ms`, not after it.
head -c 32 /dev/urandom > "$UPKEY"
chmod 600 "$UPKEY"
start_daemon "$SOCK72" "$OUT.uqc.d" "quic-upgrade daemon never bound" --shell /bin/sh \
--quic "127.0.0.1:$UPQPORT" --key "$UPKEY" --quic-idle-ms 15000
D72PID=$DPID
pipe_mux "$OUT.uqc" "$OUT.uqc.err" timeout 90 "$MUX" "quic://127.0.0.1:$UPQPORT" \
--key "$UPKEY" --quic-idle-ms 15000
pipe_send 'printf "uq-%%s\\n" pre\n'
await_out "$OUT.uqc" "uq-pre" "quic-upgrade: the QUIC client never got its pre-upgrade marker"
UPATT1=$("$MUX" d stats --sock "$SOCK72" | sed -n 's/.*attaches=\([0-9]*\).*/\1/p')
[ -n "$UPATT1" ] || { echo "e2e FAIL: quic-upgrade: no attaches counter to compare against"; exit 1; }
UPT0=$(now_ms)
set +e
"$MUX" d upgrade --sock "$SOCK72" --allow-same-version > "$OUT.uqup" 2>&1
UPRC=$?
set -e
[ "$UPRC" -eq 0 ] || {
echo "e2e FAIL: quic-upgrade: upgrade exited $UPRC:"
cat "$OUT.uqup" "$OUT.uqc.d"; exit 1; }
_i=0
while : ; do
_upst=$("$MUX" d stats --sock "$SOCK72" 2>/dev/null || true)
_upatt=$(printf '%s' "$_upst" | sed -n 's/.*attaches=\([0-9]*\).*/\1/p')
case "$_upst" in
*"session 0 clients=1"*)
if [ -n "$_upatt" ] && [ "$_upatt" -gt "$UPATT1" ]; then break; fi ;;
esac
_i=$((_i + 1))
[ "$_i" -lt $(( 400 * TIME_SCALE )) ] || {
echo "e2e FAIL: quic-upgrade: the QUIC client never re-attached; stats holds: $_upst"
cat "$OUT.uqc.d"; exit 1; }
sleep 0.05
done
UPT1=$(now_ms)
UPMS=$(( UPT1 - UPT0 ))
# Half the idle timeout, which is the only bound that means anything here:
# a client that had waited the connection out would arrive at 15000ms and
# still pass a generous absolute budget. Measured at ~60ms.
[ "$UPMS" -lt $(( 7500 * TIME_SCALE )) ] || {
echo "e2e FAIL: quic-upgrade: the QUIC client took ${UPMS}ms to come back —"
echo " that is the idle timeout expiring, not a CONNECTION_CLOSE"
exit 1; }
# ...and it is a session, not just a connection: a round trip that did not
# exist before the exec.
pipe_send 'printf "uq-%%s\\n" post\n'
await_out "$OUT.uqc" "uq-post" "quic-upgrade: the reconnected QUIC client cannot reach its shell"
pipe_detach "quic-upgrade client"
assert_stopped "$SOCK72" "$D72PID" "quic-upgrade" "$OUT.uqstop"
D72PID=""
ok "a QUIC client is served again within a breath of the exec, not after the idle timeout"
# --- mux d stop returns when the PROCESS is gone, not when the path is
#
# The unlink is the first thing a stopping daemon does; reaping its shells
# and deleting its dirs come after. A stop that said "stopped" at the
# unlink handed a scripted `mux d start -d`, or a supervisor's "is it down",
# a daemon still running — `wait_pid_gone` after every stop in this suite
# was that gap, papered. Shells that ignore TERM make the window a real
# grace rather than a race the assertion could win by luck, and two
# sessions make it a table's grace, not one shell's.
printf '#!/bin/sh\ntrap "" HUP TERM\nwhile :; do sleep 1; done\n' > "$OUT.sg.sh"
chmod +x "$OUT.sg.sh"
start_daemon "$SOCK73" "$OUT.sg.d" "stop-gone daemon never bound" --shell "$OUT.sg.sh"
D73PID=$DPID
pipe_mux "$OUT.sg" "" timeout 60 "$MUX" --sock "$SOCK73" --session second
sleep 0.5
pipe_detach "stop-gone: the second session's client"
[ "$("$MUX" d stats --sock "$SOCK73" | sed -n 's/.*sessions=\([0-9]*\).*/\1/p')" = "2" ] || {
echo "e2e FAIL: stop-gone: wanted two sessions of stubborn shells"; exit 1; }
D73REAL=$(real_pid "$D73PID")
"$MUX" d stop --sock "$SOCK73" 2> "$OUT.sg.stop" || {
echo "e2e FAIL: stop-gone: stop failed"; cat "$OUT.sg.stop"; exit 1; }
# Asked of the OS the instant stop returns — no wait, no retry.
if kill -0 "$D73REAL" 2>/dev/null; then
echo "e2e FAIL: stop said stopped, but pid $D73PID is still running"; cat "$OUT.sg.stop"; exit 1
fi
grep -q '^mux d: stopped' "$OUT.sg.stop" || {
echo "e2e FAIL: stop-gone: stop did not report stopped"; cat "$OUT.sg.stop"; exit 1; }
D73PID=""
ok "mux d stop returns when the process is gone, not when the socket is"
# The pins. Literals, not variables set from counting something else —
# "assert the literal, never the constant the code under test reads"
# (decisions.md, M10). The two figures live at the END of this file and
# nowhere else: a count repeated in prose is a count nothing gates, and
# both of the ones that used to stand here had drifted away from the pins
# they described. Adding a scenario means editing those two literals by
# hand, on purpose.
#
# Why most legs carry no convergence point. `assert_converged` replays one
# client capture into an engine and diffs it against one daemon grid, so it
# can only speak where BOTH exist and the grid is the subject. It says
# nothing about a side channel (OSC 52, a window title, pty modes, a
# fingerprint ssh printed, bytes counted rather than read), about a file an
# editor or the wall wrote, about an exit code or a status field, about
# WHICH of two sessions a keystroke reached, about scrollback the live grid
# does not hold, about the absence of bytes two agreeing empties would
# agree on anyway, or about a leg that never attaches at all. It also does
# not claim to reproduce a full-screen application's redraws byte for byte,
# which is why the alternate-screen legs stay out. A leg that asserts on
# one of those and adds a convergence point would be pinning a fact every
# leg above already establishes.