test/e2e_04_handoff.sh

Ref: main worktree-web-client-spec   Size: 34.8 KiB   History

# shellcheck shell=sh
# e2e_04_handoff.sh — sourced by test/e2e.sh after e2e_lib.sh. Scenarios run in
# the order they stand in; see the lib's header for what this file may
# assume and what it must register.
# M14 ssh→QUIC handoff. The shim that stands in for ssh (built where the
# scenarios run), and a RUNTIME DIR
# per daemon: the handoff dials the DEFAULT socket on the far side, and the
# far side is this box, so each scenario's default socket has to be one of
# ours rather than the operator's. Two daemons, no more — the one a cold
# attach gets because its own `mux d endpoint --start` ensured it over the
# shim, and the key-mismatch one that cannot be shared because it holds a
# key nothing else on this box has.
SSHIM_DIR="${TMPDIR:-/tmp}/muxd-e2e-sshim-$$"
defer_rm "$SSHIM_DIR"
SSHIM_PIDLOG="$SSHIM_DIR/pids"
export SSHIM_PIDLOG
# What the OS saw, not what the client claims: one line per shim run,
# `<own comm> <parent comm>`, taken BEFORE the exec below replaces this
# process. `mux` execs the ssh argv itself, so both words are pinned — a
# client that went back to running the recipe through `/bin/sh -c` would
# name a shell in one of them.
SSHIM_COMMLOG="$SSHIM_DIR/comms"
export SSHIM_COMMLOG
# The remote COMMAND each run was handed, one line per run. The pidlog says
# how many ssh runs there were and the commlog says what they were; neither
# can tell a reading dial from an asking one, and that distinction is now
# the whole of how a daemon comes to exist.
SSHIM_ARGLOG="$SSHIM_DIR/args"
export SSHIM_ARGLOG
HRUN="${TMPDIR:-/tmp}/mux-e2e-hrun-$$"
defer_rm "$HRUN"
HRUN2="${TMPDIR:-/tmp}/mux-e2e-hrun2-$$"
defer_rm "$HRUN2"
SOCK16="$HRUN/muxd.sock"
SOCK17="$HRUN2/muxd.sock"
defer_sock "$SOCK16" "$SOCK17"
HKEY="${TMPDIR:-/tmp}/mux-e2e-hkey-$$"
defer_rm "$HKEY"
# A config home that is a FILE: `mux d endpoint` cannot create a key under
# it and cannot find one, which is the announce-none scenario's lever.
HCFGBAD="${TMPDIR:-/tmp}/mux-e2e-hnokey-$$"
defer_rm "$HCFGBAD"
# Bands of their own, like every other port here, so a concurrent suite
# cannot collide — and BELOW the ephemeral range (32768–60999 here), which
# for HDEADPORT is not housekeeping but the scenario's correctness. It is
# the port the poisoned cache points at, and the daemon it is poisoning
# against holds a KERNEL-ASSIGNED ephemeral port: land on that one and the
# "dead" port is live, the dial succeeds, and the self-heal scenario
# quietly asserts a heal that never had to happen.
HQPORT=$(( 11000 + ($$ % 4000) ))
HDEADPORT=$(( 16000 + ($$ % 4000) ))

# --- M14: the ssh→QUIC handoff -----------------------------------------
#
# `mux HOST` fetches QUIC coordinates over ssh once, caches them, and
# attaches over pure QUIC thereafter — falling back to that same ssh, one
# deadline later, when QUIC cannot get through. Six scenarios: cold, warm,
# a poisoned cache, a key mismatch, a warm miss on the coordinates that
# mismatch left cached, and a remote that can offer nothing.
#
# The shim IS ssh as far as the client can tell: `ssh HOST CMD...` drops
# HOST and execs CMD here, so `ssh whatever mux d endpoint` runs the real
# binary against a socket of this suite's own making and every line of the
# client's handoff code runs for real, with no network anywhere.
#
# Every invocation appends its pid to SSHIM_PIDLOG BEFORE the exec, and
# that log is how these scenarios know whether ssh ran. Never `ps | grep
# ssh`: this box belongs to someone who is probably ssh'd into something,
# and a warm attach's entire claim is that no ssh existed.
#
# The contract it depends on — real ssh's own: everything after HOST is
# joined with spaces and handed to the REMOTE user's shell. The shim
# models that with `sh -c "$*"`, which is what lets the client's quoted
# remote command (the PATH-suffix form recipeFor builds) run here the
# same way sshd would run it. A bare `exec "$@"` was enough while the
# command was plain argv; it execs a program literally named
# `PATH=... mux d endpoint` now, and dies at 127 without an announce.
#
# The option loop and the arity guard come from `ssh_shim_head`, which
# both of this suite's fake sshs share: the recipes these scenarios drive
# happen to be the asking ones, which pass no `-o`, and a shim that is
# right only by that coincidence breaks as "no announce" the day the entry
# dial gains a flag.
mkdir -p "$SSHIM_DIR" "$HRUN" "$HRUN2"
: > "$SSHIM_PIDLOG"
: > "$SSHIM_COMMLOG"
: > "$SSHIM_ARGLOG"
ssh_shim_head "$SSHIM_DIR/ssh"
cat >> "$SSHIM_DIR/ssh" <<'SHIM'
echo $$ >> "${SSHIM_PIDLOG:?}"
# ps rather than the lib's pid_comm: this shim is written to disk and run
# as its own `sh` by the client under test, so nothing the suite sourced is
# in scope here. Basenames taken by hand, for pid_comm's own reason: BSD ps
# prints comm as the executable's full path where Linux prints the
# basename.
_sc=$(ps -o comm= -p $$); _sp=$(ps -o comm= -p $PPID)
printf '%s %s\n' "${_sc##*/}" "${_sp##*/}" >> "${SSHIM_COMMLOG:?}"
shift
printf '%s\n' "$*" >> "${SSHIM_ARGLOG:?}"
exec /bin/sh -c "$*"
SHIM
chmod +x "$SSHIM_DIR/ssh"
# The PATH every scenario below runs the client under: the shim shadows any
# real ssh, and `mux d` resolves to the binary under test rather than to
# whatever is installed.
HPATH="$SSHIM_DIR:$(dirname "$MUX"):$PATH"
# `user@` on the host is not decoration. The QUIC dial strips it
# (handoff.dialHost) and gets a loopback literal, which is what makes the
# dial reach the daemon at all; the CACHE keys on the whole word, which is
# what keeps these three scenarios' cache file distinct from the bare
# `127.0.0.1` that (d) and (e) deliberately SHARE — (e)'s whole premise is
# that it attaches onto the coordinates (d) left cached. A fake name would
# resolve to nothing and every QUIC attempt below would fail at DNS without
# ever dialling.
HHOST="mux-e2e@127.0.0.1"
HCACHE="$XDG_CACHE_HOME/mux/hosts/$HHOST"
# The bare word's cache, which (d) writes and (e) then attaches onto.
HCACHE_D="$XDG_CACHE_HOME/mux/hosts/127.0.0.1"
# (a) COLD: no daemon, no cache. The attach has to produce the daemon —
# the REMOTE does it, because an asked dial spells `mux d endpoint --start`
# and that verb ensures a daemon before it announces — fetch coordinates
# over the shim,
# and END UP ON QUIC — and that last part is proven by observation rather than
# inference: the ssh the announce arrived on is killed by the client on
# QUIC success, so once its pid is OBSERVED gone, a second marker still
# making the round trip can only be riding QUIC. The pipe's owner is dead.
pipe_mux "$OUT.h1" "$OUT.h1.err" env SHELL=/bin/sh XDG_RUNTIME_DIR="$HRUN" PATH="$HPATH" timeout 60 \
    "$MUX" "$HHOST"
pipe_send 'printf "cold-%%s\\n" one\n'
# The bounded wait, with a message of its own, because of what it is FOR:
# a client blocked on an announce that is never coming has no other
# symptom. It cannot be left to the suite's timeout or the trap — those
# report a dead process, not which line stopped arriving.
wait_for "$OUT.h1" "cold-one" 25 || {
    echo "e2e FAIL: cold handoff: no marker in 25s — the attach never converged."
    echo "          A client that blocks here read no announce line off the ssh pipe."
    cat "$OUT.h1" "$OUT.h1.err" 2>/dev/null; exit 1; }

# Exactly ONE ssh run, from the log rather than from ps. Three was the cold
# shape while the CLIENT decided to start: `mux d endpoint` refused an empty
# box, the client reaped the exit code to tell that refusal from ssh's own
# 255, ran `mux d start -d`, and asked again. The remote decides now, so the
# run that starts the daemon is the run that announces it.
HSHIMS=$(wc -l < "$SSHIM_PIDLOG")
[ "$HSHIMS" -eq 1 ] || {
    echo "e2e FAIL: cold handoff ran $HSHIMS ssh invocations, want exactly 1"
    echo "          (one mux d endpoint --start, which ensures and announces)"
    cat "$SSHIM_PIDLOG"; cat "$SSHIM_ARGLOG"; exit 1; }
# The WORD, not just the count. A client that spent its one run on the
# reading verb would show the same 1 here and would have started nothing —
# it is the flag that makes this cold attach work at all, and the flag is
# the only thing separating this run from a wall poll's.
HASKS=$(grep -c -- 'mux d endpoint --start$' "$SSHIM_ARGLOG" || true)
[ "$HASKS" -eq 1 ] || {
    echo "e2e FAIL: cold handoff ran the asking word $HASKS times, want exactly 1"
    cat "$SSHIM_ARGLOG"; exit 1; }
# Asked of the OS, not of the client: that child sat directly under `mux`,
# with nothing interposed. Only the PARENT word is pinned. The child's own
# comm is not comparable across the two OSes — Linux names a shebang
# script's process after the SCRIPT (`ssh`) and Darwin after the
# INTERPRETER (`sh`), so both words are correct answers and neither says
# anything about the product. It is still recorded, because a failure here
# is read by eye.
#
# What this does NOT catch is a `/bin/sh -c` line put back — bash and dash
# both exec a single simple command in place, so the shell is gone by the
# time the process table is read. That regression is build.zig folder rule
# 5's, measured there; this leg pins the shape the rule cannot see, that
# the argv reaches a real ssh unmangled.
HBADCOMM=$(awk '$2 != "mux" {n++} END {print n+0}' "$SSHIM_COMMLOG")
[ "$HBADCOMM" -eq 0 ] || {
    echo "e2e FAIL: the handoff spawned something other than ssh straight off mux:"
    cat "$SSHIM_COMMLOG"; exit 1; }
ok "the handoff's child IS ssh, forked off mux with no shell between"
# The one run there was, and the one that carried the announce: its death
# is what proves QUIC took over.
HSHIMPID=$(tail -1 "$SSHIM_PIDLOG")
wait_pid_gone "$HSHIMPID" "cold handoff: QUIC took over, so ssh must be gone"
# The second marker is typed only now, after the death was OBSERVED, so its
# round trip is traffic that moved after the pipe's owner died — by
# construction, not by a clock. What guarantees the ordering on the product
# side: openHandoff kills the ssh child the moment QUIC is ready, before
# Transport.open returns, so before the session can emit a single byte.
# Anyone who moves that kill (to after the first frame, into a deferred
# teardown) takes this scenario's premise with them.
pipe_send 'printf "cold-%%s\\n" two\n'
wait_for "$OUT.h1" "cold-two" 20 || {
    echo "e2e FAIL: cold handoff: the session stopped converging once ssh was gone,"
    echo "          so the bytes were riding the pipe rather than QUIC"
    cat "$OUT.h1" "$OUT.h1.err" 2>/dev/null; exit 1; }
pipe_detach "cold handoff client"

# The daemon the REMOTE started, by the pid its own up-line reported — the
# only handle this suite has on a process that is nobody's child. The line
# is `mux d endpoint`'s now, and it reaches this file the way ssh's own
# stderr does: RELAYED, byte for byte, because the entry dial is the one
# caller that `narrate`s the pipe it reads. Same
# accepted gap as the M13 blocks: a failure above leaves it untracked, and
# the trap's stop-by-socket is what covers that.
grep -q '^mux d endpoint: starting' "$OUT.h1.err" || {
    echo "e2e FAIL: cold handoff printed no starting line — mux d endpoint --start"
    echo "          did not ensure the daemon it was asked for"
    cat "$OUT.h1.err"; exit 1; }
HAPID=$(sed -n 's/.* pid=\([0-9]*\).*/\1/p' "$OUT.h1.err" | head -1)
defer_kill "$HAPID"
[ -n "$HAPID" ] || {
    echo "e2e FAIL: mux d endpoint --start's up-line carries no pid"
    cat "$OUT.h1.err"; exit 1; }
kill -0 "$HAPID" 2>/dev/null || {
    echo "e2e FAIL: the handoff daemon (pid $HAPID) is not alive"; exit 1; }

# The cache: present, private (it holds the key), and naming a port that is
# really being listened on. The last one is the assertion that matters —
# a remembered number proves nothing, a remembered number that maps to the
# daemon's actual UDP socket proves the announce was true. Asked through
# udp_local_bound for the same reasons the M8 block uses it, and 00000000
# because the lazy bind takes 0.0.0.0.
[ -f "$HCACHE" ] || {
    echo "e2e FAIL: cold handoff left no cache at $HCACHE"
    ls -la "$XDG_CACHE_HOME/mux/hosts" 2>/dev/null; exit 1; }
HCPERM=$(file_mode "$HCACHE")
[ "$HCPERM" = "600" ] || {
    echo "e2e FAIL: handoff cache perms $HCPERM, want 600 (the file holds a key)"; exit 1; }
HCDPERM=$(file_mode "$XDG_CACHE_HOME/mux/hosts")
[ "$HCDPERM" = "700" ] || {
    echo "e2e FAIL: handoff cache dir perms $HCDPERM, want 700"; exit 1; }
HPORT=$(sed -n 's/^endpoint \([0-9][0-9]*\) [0-9a-f]*$/\1/p' "$HCACHE")
[ -n "$HPORT" ] || {
    echo "e2e FAIL: the cache is not an announce line:"; cat -v "$HCACHE"; exit 1; }
HHEXUDP=$(printf '00000000:%04X' "$HPORT")
udp_local_bound "$HHEXUDP" || {
    echo "e2e FAIL: the cached port $HPORT names no UDP listener ($HHEXUDP);"
    echo "          the announce the client believed was not the daemon's"
    exit 1; }
assert_converged "$OUT.h1" "$SOCK16" "cold handoff"
ok "cold handoff: ssh fetches the coordinates, QUIC carries the session"

# (b) WARM: the same daemon, the cache from (a). No ssh may run at all —
# the whole point of remembering coordinates — so the pidlog count is
# snapshotted and must not move. The transport is also silent: the M13
# contract that a warm attach prints no spawn progress, extended to the
# handoff's own line, which has nothing to report when nothing fell back.
HSHIMS_B=$(wc -l < "$SSHIM_PIDLOG")
pipe_mux "$OUT.h2" "$OUT.h2.err" env SHELL=/bin/sh XDG_RUNTIME_DIR="$HRUN" PATH="$HPATH" timeout 40 "$MUX" "$HHOST"
pipe_send 'printf "warm-%%s\\n" ok\n'
await_out "$OUT.h2" "warm-ok" "warm-ok never reached the client"
pipe_detach "warm handoff client"
grep -q "warm-ok" "$OUT.h2" || {
    echo "e2e FAIL: warm handoff served no session"
    cat "$OUT.h2" "$OUT.h2.err" 2>/dev/null; exit 1; }
HSHIMS_B2=$(wc -l < "$SSHIM_PIDLOG")
# This also carries M13's warm-attach silence contract, which is why there
# is no separate grep for spawn progress here: `mux d endpoint` is the only
# thing that prints a starting line, it only runs under ssh, and ssh
# provably did not run. A progress grep could therefore never fail on its
# own — it would be a check the line above has already made.
[ "$HSHIMS_B2" -eq "$HSHIMS_B" ] || {
    echo "e2e FAIL: warm handoff spawned ssh: pidlog went $HSHIMS_B -> $HSHIMS_B2"
    echo "          (want unchanged — a cache hit dials QUIC and never shells out)"
    exit 1; }
grep -q 'unreachable, attaching over ssh' "$OUT.h2.err" && {
    echo "e2e FAIL: warm handoff printed the fallback line"; cat "$OUT.h2.err"; exit 1; }
assert_converged "$OUT.h2" "$SOCK16" "warm handoff"
ok "warm handoff: the cache dials QUIC, ssh never runs (pidlog still $HSHIMS_B2)"

# (c) STALE CACHE: a well-formed announce naming a port nothing holds. The
# invariant under test is that a stale cache costs a ROUND TRIP and never
# correctness: the dial fails, the cold path refetches, QUIC succeeds on
# the fresh coordinates, and the user is told nothing — a fallback line
# here would be reporting a failure that did not happen.
#
# The poison is written by hand because shell cannot import handoff.zig.
# The grammar is `endpoint <port> <64 hex>\n`; the key is STOLEN from the
# real cache above and only the port is swapped, so the line differs from a
# genuine one in exactly the way the scenario is about.
#
# Until M15 this cost a full deadline (~2s), because the ICMP refusal from
# the dead port landed on a sendto inside the QUIC drain and was discarded
# there. Both socket paths act on it now, so the dial dies in about one
# loopback round trip and the whole heal measures ~200ms. The bound below
# was a floor on that spent budget and is now a ceiling on its absence.
#
# The detach is sent only after the marker has rendered, and that ordering
# is load-bearing: `waitReady` answers Ctrl-\ DURING a handshake (M8 pins
# exactly that), so a detach landing inside the dial window aborts the
# attach — no session, no error, exit 0 — and a scenario that paced the
# detach on a clock could pass its exit check having tested nothing.
HHEXKEY=$(sed -n 's/^endpoint [0-9][0-9]* \([0-9a-f]*\)$/\1/p' "$HCACHE")
[ -n "$HHEXKEY" ] || { echo "e2e FAIL: could not read the cached key"; exit 1; }
printf 'endpoint %s %s\n' "$HDEADPORT" "$HHEXKEY" > "$HCACHE"
chmod 600 "$HCACHE"
HSHIMS_C=$(wc -l < "$SSHIM_PIDLOG")
HT4=$(now_ms)
pipe_mux "$OUT.h3" "$OUT.h3.err" env SHELL=/bin/sh XDG_RUNTIME_DIR="$HRUN" PATH="$HPATH" timeout 40 \
    "$MUX" "$HHOST"
pipe_send 'printf "heal-%%s\\n" ok\n'
wait_for "$OUT.h3" "heal-ok" 30 || {
    echo "e2e FAIL: stale-cache handoff never served a session"
    cat "$OUT.h3" "$OUT.h3.err" 2>/dev/null; exit 1; }
HT5=$(now_ms)
HMS_C=$(( HT5 - HT4 ))
pipe_detach "stale-cache handoff client"
# The absent-grep, and the control that proves it can fire is scenario (d)
# below: the same pattern, asserted PRESENT, against a build where the
# fallback really happened.
grep -q 'unreachable, attaching over ssh' "$OUT.h3.err" && {
    echo "e2e FAIL: a stale cache printed the fallback line; it costs time, not correctness"
    cat "$OUT.h3.err"; exit 1; }
HPORT_C=$(sed -n 's/^endpoint \([0-9][0-9]*\) [0-9a-f]*$/\1/p' "$HCACHE")
[ "$HPORT_C" = "$HPORT" ] || {
    echo "e2e FAIL: the cache did not heal: holds $HPORT_C, want the live $HPORT"
    cat -v "$HCACHE"; exit 1; }
HSHIMS_C2=$(wc -l < "$SSHIM_PIDLOG")
[ "$((HSHIMS_C2 - HSHIMS_C))" -eq 1 ] || {
    echo "e2e FAIL: the healing attach ran $((HSHIMS_C2 - HSHIMS_C)) ssh invocations, want 1"
    exit 1; }
# The other half of the invariant, and it changed direction in M15. A
# floor used to live here: while a refused port ran the whole budget out,
# spending it was proof the poisoned coordinates had been dialled, which
# no other check in this scenario makes. That proof is gone — the heal now
# costs ~200ms, which is indistinguishable from a build that read no cache
# and simply went cold. Scenario (b) is what catches THAT build (a warm
# attach there must spawn no ssh at all), and this scenario now leans on
# it rather than pretending to stand alone.
#
# What replaces the floor is its mirror: the refusal must be SEEN, not
# waited out. A build that swallowed the ICMP again — the pre-M15 bug —
# would push this back over 2s, and the ceiling is what says so. 1500
# clears the measured ~200ms by a wide margin and still sits under the
# 2000ms budget, so nothing between "fast" and "spent the budget" hides.
[ "$HMS_C" -lt 1500 ] || {
    echo "e2e FAIL: the stale-cache attach took ${HMS_C}ms, where a refused port is"
    echo "          meant to fail in about one round trip (~200ms end to end). Either"
    echo "          the ICMP refusal is being swallowed again and this dial spent the"
    echo "          2000ms QUIC budget, or this box is loaded enough to stretch a"
    echo "          200ms attach past 1500 — the daemon log and the load average"
    echo "          tell those apart"
    exit 1; }
assert_converged "$OUT.h3" "$SOCK16" "stale-cache self-heal"
ok "a stale cache self-heals: one refetch (${HMS_C}ms, no budget spent), no fallback line, real port cached"

# (d) FALLBACK LINE, by key mismatch. A second daemon holding a key nothing
# else on this box has; `mux d endpoint` announces the DEFAULT key (that is
# what the endpoint process resolves) together with this daemon's real
# port, so the coordinates are perfectly well-formed and the handshake can
# never complete. PSK auth is mutual and a listener does not answer a peer
# it cannot authenticate, so the client gets silence and spends the whole
# deadline — which is exactly the case the fallback line exists for.
#
# The host is the literal `127.0.0.1`, and it has to be: a fake name would
# fail at DNS instantly, skip the deadline, and leave this scenario pinning
# nothing. Its cache file is `127.0.0.1`, distinct from (a)'s.
#
# This scenario costs ~2s of wall time by design. That is the price of
# pinning the budget, and it is the only place the suite pays it.
#
# Six seconds to the detach byte, for the reason spelled out in (c): the
# stdin script's clock runs in parallel with a deadline-long attach, and a
# Ctrl-\ that lands inside the dial is answered by `waitReady` as an abort
# — which would leave this scenario asserting a fallback line that a
# correct build never got as far as printing.
head -c 32 /dev/urandom > "$HKEY"
chmod 600 "$HKEY"
# No --quic-idle-ms: the daemon's default is already 15000, and spelling
# it here would suggest the ceiling's reasoning below depends on this
# flag. It does not — the 15000 that bounds an overrunning dial is the
# CLIENT's `quic_idle_ms_default`, which this daemon cannot influence.
start_daemon "$SOCK17" "$OUT.d17.d" "key-mismatch daemon never bound" --shell /bin/sh \
    --quic "127.0.0.1:$HQPORT" --key "$HKEY"
HDPID=$DPID

# ---- a QUIC poller holds one client slot, not one per second -----------
#
# Every QUIC connection takes a client slot at the handshake, and the wall
# polls each QUIC host once a second on a connection of its own. A client
# that dropped its socket without CONNECTION_CLOSE left the daemon to
# learn of it from the 15 s idle timer, so the polls filled the table at a
# slot a second and every attach after that was refused — found on a
# laptop whose only clients were another wall's polls, when the daemon had
# eight slots and so took eight seconds (2026-09-02). The table is
# `max_clients` deep now, which only moves that deadline: the ceiling
# below is on slots held at ONCE and does not move with it. Sampled off
# `mux d stats`, which counts slots HELD, while a hub polls this daemon.
HPOLLSTATE="${TMPDIR:-/tmp}/mux-e2e-hpoll-$$"
defer_rm "$HPOLLSTATE"
mkdir -p "$HPOLLSTATE"
XDG_STATE_HOME="$HPOLLSTATE" "$MUX" web "quic://127.0.0.1:$HQPORT" --key "$HKEY" \
    --port "$((HQPORT + 1))" > "$OUT.hpoll" 2>&1 &
HPOLLPID=$!
defer_kill "$HPOLLPID"
# The daemon's gauge is the `clients=N attaches=` pair; the same line goes
# on to say `session 0 clients=1`, which a greedy match reads instead (the
# first draft of this leg did, and passed against the bug).
HPOLLMAX=0
HPOLLSEEN=""
for _ in 1 2 3 4 5 6 7 8; do
    sleep 1
    HPOLLNOW=$(clients_now "$SOCK17")
    HPOLLSEEN="$HPOLLSEEN ${HPOLLNOW:-?}"
    [ "${HPOLLNOW:-0}" -gt "$HPOLLMAX" ] && HPOLLMAX=$HPOLLNOW
done
softkill "$HPOLLPID"
# At most three: the hub's tile on session 0, the poll in flight, and the
# one before it still draining. Without the goodbye this reads 8 by the
# eighth second — a slot a second, and the table's depth never enters it.
[ "$HPOLLMAX" -le 3 ] || {
    echo "e2e FAIL: eight seconds of polling held $HPOLLMAX client slots at once (want at most 3); per second:$HPOLLSEEN"
    cat "$OUT.hpoll"; exit 1; }
ok "a QUIC poller holds one client slot, not one per second"

HSHIMS_D=$(wc -l < "$SSHIM_PIDLOG")
HT0=$(now_ms)
pipe_mux "$OUT.h4" "$OUT.h4.err" env SHELL=/bin/sh XDG_RUNTIME_DIR="$HRUN2" PATH="$HPATH" timeout 40 \
    "$MUX" "127.0.0.1"
pipe_send 'printf "fallback-%%s\\n" ok\n'
# Timed to the LINE rather than to the client's exit, so no session sleeps
# are folded in. The elapsed time is the dial's budget plus launch overhead
# (measured 26-129ms), which only inflates it — the floor is safe.
# Only the stderr is printed on failure, deliberately: the complaint is
# about a line that is missing from it, and the capture beside it is a
# screenful of escape sequences that would bury the answer.
wait_for "$OUT.h4.err" "unreachable, attaching over ssh" 15 || {
    echo "e2e FAIL: key mismatch printed no fallback line in 15s; its stderr was:"
    cat "$OUT.h4.err" 2>/dev/null; exit 1; }
HT1=$(now_ms)
HMS=$(( HT1 - HT0 ))
# The exact line, with the port the daemon really holds: a fallback that
# named the wrong coordinates would be a different bug wearing this one's
# message.
grep -q "^mux: quic://127.0.0.1:$HQPORT unreachable, attaching over ssh$" "$OUT.h4.err" || {
    echo "e2e FAIL: the fallback line is not the pinned one (want port $HQPORT):"
    cat "$OUT.h4.err"; exit 1; }
# Bounded on BOTH sides. The floor is what proves the dial actually
# happened — an implementation that never tried would fall back instantly —
# and it sits below the 2000ms budget only by the slack the measurement
# itself needs. The ceiling is what proves the budget bounds anything at
# all: without it a hang reads as a pass that took a while.
#
# That ceiling is 10000 and not the plan's deadline+3s, and the difference
# is a decision rather than an oversight. It separates "bounded" from
# "hung" and is not asked to do anything finer: the observed spread is
# 2026-2129ms across 13 runs, and a regression that overruns the budget
# does not overrun it by a few hundred milliseconds — it overruns to the
# connection's idle timeout, 15000ms, because that is the only other bound
# in the dial. A tighter ceiling would therefore catch nothing this one
# misses, while making the single scenario that already pays 2s by design
# the most load-sensitive check in the suite.
#
# The wait_for above is the backstop beyond it: an overrun past 15s is
# reported there, with its own message, rather than here.
[ "$HMS" -ge 1500 ] || {
    echo "e2e FAIL: the fallback came after ${HMS}ms, too fast to have spent the"
    echo "          2000ms QUIC budget — the dial cannot have happened"
    exit 1; }
[ "$HMS" -lt 10000 ] || {
    echo "e2e FAIL: the fallback took ${HMS}ms; the deadline is meant to bound it"
    exit 1; }
# ...and the session it fell back to is a real one, over the pipe that
# carried the announce.
await_out "$OUT.h4" "fallback-ok" "key-mismatch attach printed the line but served no session"
pipe_detach "key-mismatch client"
HSHIMS_D2=$(wc -l < "$SSHIM_PIDLOG")
[ "$((HSHIMS_D2 - HSHIMS_D))" -eq 1 ] || {
    echo "e2e FAIL: the fallback attach ran $((HSHIMS_D2 - HSHIMS_D)) ssh invocations, want 1"
    exit 1; }
assert_converged "$OUT.h4" "$SOCK17" "handoff fallback to ssh"
# This leg's postcondition IS (e)'s precondition: the announce it fetched is
# on disk under the bare host word, and (e) attaches onto that file.
[ -s "$HCACHE_D" ] || {
    echo "e2e FAIL: the key-mismatch attach cached nothing at $HCACHE_D"; exit 1; }
ok "a key mismatch falls back to the ssh pipe: one deadline (${HMS}ms), one line"

# (e) WARM MISS on coordinates the remote still means. The same daemon,
# the same runtime dir and the same host word as (d), so the cache file (d)
# wrote holds the very endpoint that just went silent — which is the state
# a genuinely unreachable QUIC route is in on EVERY attach after the first,
# and therefore the common case rather than the exotic one.
#
# The warm dial spends the budget on silence, the refetch names the same
# port and the same key, and the client has to use the ssh pipe it is
# already holding. Dialling them again buys nothing: ssh is authoritative
# and has just said the coordinates have not moved.
# The premise, CHECKED rather than inherited from (d). Every assertion this
# leg makes also holds for a COLD run through the table — one shim, one
# fallback line, one budget — so a (d) whose cache write had regressed would
# leave this leg green and pinning nothing at all. Three facts make the warm
# dial certain: the file is there, `handoff.readCache` will ACCEPT it (it
# refuses anything looser than 0600 and the attach then runs cold), and it
# names the port that just went silent rather than some other daemon's.
[ -f "$HCACHE_D" ] || {
    echo "e2e FAIL: no cache at $HCACHE_D — (d) wrote none, so this leg would"
    echo "          attach COLD and prove nothing about a warm miss"
    exit 1; }
HWPORT=$(sed -n 's/^endpoint \([0-9][0-9]*\) [0-9a-f]*$/\1/p' "$HCACHE_D")
[ "$HWPORT" = "$HQPORT" ] || {
    echo "e2e FAIL: the cache at $HCACHE_D names port '$HWPORT', want the silent $HQPORT"
    cat -v "$HCACHE_D"; exit 1; }
HWMODE=$(file_mode "$HCACHE_D")
[ "$HWMODE" = "600" ] || {
    echo "e2e FAIL: the cache at $HCACHE_D is mode $HWMODE; readCache refuses"
    echo "          anything looser, so this leg would attach COLD"
    exit 1; }
HSHIMS_W=$(wc -l < "$SSHIM_PIDLOG")
HT4=$(now_ms)
pipe_mux "$OUT.h6" "$OUT.h6.err" env SHELL=/bin/sh XDG_RUNTIME_DIR="$HRUN2" PATH="$HPATH" timeout 40 \
    "$MUX" "127.0.0.1"
pipe_send 'printf "warmmiss-%%s\\n" ok\n'
# Timed to the LINE, as (d) is, so no session sleeps are folded in.
wait_for "$OUT.h6.err" "unreachable, attaching over ssh" 15 || {
    echo "e2e FAIL: the warm miss printed no fallback line in 15s; its stderr was:"
    cat "$OUT.h6.err" 2>/dev/null; exit 1; }
HT5=$(now_ms)
HMS_W=$(( HT5 - HT4 ))
# ONCE, and naming the port the daemon really holds. Twice would be two
# dials wearing one message.
HFB_W=$(grep -c "^mux: quic://127.0.0.1:$HQPORT unreachable, attaching over ssh$" "$OUT.h6.err" || true)
[ "$HFB_W" -eq 1 ] || {
    echo "e2e FAIL: the warm miss printed the fallback line $HFB_W times, want 1 (port $HQPORT):"
    cat "$OUT.h6.err"; exit 1; }
# The floor is (d)'s, and for (d)'s reason: below it the warm dial cannot
# have happened, and this leg would be pinning nothing.
#
# The CEILING is what this leg exists for, and it is 3500 = two budgets
# minus 500ms of slack rather than (d)'s 10000. One budget is 2000 and the
# spread measured there is 2026-2139ms; a client that dialled the refetched
# coordinates a second time pays two of them and lands past 4000 — measured
# at 4264ms on 2026-08-11, which is the reading that stood against a spec
# criterion of "within one deadline" until the step table. So the gap
# between a pass and that regression is ~1900ms, and 500ms of slack sits
# inside it with room to spare. This bound is not load-sensitive in the way
# a tighter one would be: the budget is spent WAITING for a silent peer, not
# computing, so the Debug build's ~600x penalty on output does not touch it.
[ "$HMS_W" -ge 1500 ] || {
    echo "e2e FAIL: the warm miss fell back after ${HMS_W}ms, too fast to have spent"
    echo "          the 2000ms QUIC budget — the warm dial cannot have happened"
    exit 1; }
[ "$HMS_W" -lt 3500 ] || {
    echo "e2e FAIL: the warm miss took ${HMS_W}ms. One QUIC budget is 2000ms; at or"
    echo "          above ~4000 the client dialled the refetched coordinates a SECOND"
    echo "          time, which is the defect this leg pins (4264ms measured before"
    echo "          the fix). Between 3500 and 4000 it is one budget plus an unusually"
    echo "          slow ssh: this attach ran $(( $(wc -l < "$SSHIM_PIDLOG") - HSHIMS_W ))"
    echo "          ssh invocations, and one is the correct number."
    exit 1; }
# ...and the session it fell back to is a real one, over that same pipe.
await_out "$OUT.h6" "warmmiss-ok" "warm miss printed the line but served no session"
pipe_detach "warm-miss client"
# ONE ssh: the refetch. The warm dial runs none, and there is no second
# fetch to pay a connect timeout or a password prompt for.
HSHIMS_W2=$(wc -l < "$SSHIM_PIDLOG")
[ "$((HSHIMS_W2 - HSHIMS_W))" -eq 1 ] || {
    echo "e2e FAIL: the warm miss ran $((HSHIMS_W2 - HSHIMS_W)) ssh invocations, want 1"
    exit 1; }
assert_converged "$OUT.h6" "$SOCK17" "warm miss falls back to ssh"
ok "a warm miss on silent coordinates costs ONE deadline (${HMS_W}ms), not two"

# (f) ANNOUNCE-NONE: a remote that cannot produce coordinates at all. The
# lever is a config home that is a FILE, so the default key can be neither
# created nor found — `mux d endpoint` says so on stderr (ssh carries it to
# the user) and announces `endpoint none`.
#
# Against that, no deadline may be paid: no coordinates were ever in play,
# so there is nothing to dial and nothing to report as unreachable. Its own
# host, uncached by construction — reusing (a)'s would hit the cache and
# never reach the announce at all.
: > "$HCFGBAD"
HSHIMS_E=$(wc -l < "$SSHIM_PIDLOG")
HT2=$(now_ms)
pipe_mux "$OUT.h5" "$OUT.h5.err" env SHELL=/bin/sh XDG_RUNTIME_DIR="$HRUN" XDG_CONFIG_HOME="$HCFGBAD" \
    PATH="$HPATH" timeout 40 \
    "$MUX" "mux-e2e-none@127.0.0.1"
pipe_send 'printf "none-%%s\\n" ok\n'
wait_for "$OUT.h5" "none-ok" 20 || {
    echo "e2e FAIL: announce-none handoff never served a session"
    cat "$OUT.h5" "$OUT.h5.err" 2>/dev/null; exit 1; }
HT3=$(now_ms)
HMS_E=$(( HT3 - HT2 ))
pipe_detach "announce-none client"
# The remote said why, in one line, on the stderr mux relays from ssh.
grep -q '^mux d endpoint: .*staying on ssh' "$OUT.h5.err" || {
    echo "e2e FAIL: announce-none said nothing about why it stayed on ssh"
    cat "$OUT.h5.err"; exit 1; }
# ...and said nothing about a fallback, because nothing fell back: the
# client never held coordinates to fail with.
#
# EVIDENCE NOTE, and it is written down precisely because a comment that
# implied otherwise would be worse than the gap. This scenario makes seven
# assertions in all; three of them are specific to what it claims — the
# staying-on-ssh line above, this absent fallback line, and the budget
# bound below — and only the FIRST has been shown able to fail (deleting
# announceKey's create_failed message kills it, at 19 checkpoints). The
# other four are the shared shape every scenario here uses: exit status,
# marker, pidlog delta, convergence, each exercised by mutations
# elsewhere.
#
# So these two have never been exercised by any mutation — the suite stops
# at its first failure and every mutation tried so far dies in an earlier
# scenario. Neither is unfalsifiable: this grep is the same shape as (c)'s,
# which the deleted-fallback-line mutation did kill, and the bound below
# fires on any number at or above it. They are untested, which is a
# different thing, and Task 8's regrade can take one if it comes cheap.
grep -q 'unreachable, attaching over ssh' "$OUT.h5.err" && {
    echo "e2e FAIL: announce-none printed the fallback line; no dial was ever possible"
    cat "$OUT.h5.err"; exit 1; }
HSHIMS_E2=$(wc -l < "$SSHIM_PIDLOG")
[ "$((HSHIMS_E2 - HSHIMS_E))" -eq 1 ] || {
    echo "e2e FAIL: announce-none ran $((HSHIMS_E2 - HSHIMS_E)) ssh invocations, want 1"
    exit 1; }
# No budget spent, measured to the first marker rather than to exit. The
# bound sits at 1500 rather than at the 2000ms budget itself so that a
# slow-but-honest attach still passes; what it cannot do is name a cause
# it has not proved. A dial that ran its full budget lands at 2000+
# (measured 2026-2129 in scenario (d)); anything between 1500 and 2000 is
# an attach that was merely slow, or one that started a dial it should
# never have begun. The message therefore reports the number and says
# which readings are which, rather than asserting a spent budget on
# evidence that cannot distinguish one.
[ "$HMS_E" -lt 1500 ] || {
    echo "e2e FAIL: announce-none took ${HMS_E}ms to converge, over the 1500ms bound."
    echo "          At or above ~2000ms a full QUIC budget was spent, on coordinates"
    echo "          the remote said it did not have. Below that it is either load or"
    echo "          a dial that began and was cut short — check which before blaming"
    echo "          the machine."
    exit 1; }
assert_converged "$OUT.h5" "$SOCK16" "announce-none stays on ssh"
ok "announce-none stays on ssh: no deadline paid (${HMS_E}ms), no fallback line"

# Both daemons down by the sanctioned verb, and OBSERVED dead by pid — the
# M13 teardown shape, all four assertions of it (see assert_stopped).
assert_stopped "$SOCK16" "$HAPID" "handoff daemon" "$OUT.stop"
HAPID=""
assert_stopped "$SOCK17" "$HDPID" "key-mismatch daemon" "$OUT.stop"
HDPID=""
rm_swept "$OUT.h1" "$OUT.h1.err" "$OUT.h2" "$OUT.h2.err" "$OUT.h3" "$OUT.h3.err" \
    "$OUT.h4" "$OUT.h4.err" "$OUT.h5" "$OUT.h5.err" "$OUT.h6" "$OUT.h6.err" \
    "$OUT.stop" "$HKEY" "$HCFGBAD"
rm -rf "$SSHIM_DIR" "$HRUN" "$HRUN2"