d6888aa5
test: e2e — quality batch: full SOCK17 teardown, trap backstops, fixture hardening, honest comments
a73x 2026-08-12 04:25
Commit message
test/e2e.sh
| Old | New | ||
|---|---|---|---|
| @@ -37,6 +37,12 @@ export XDG_CONFIG_HOME XDG_STATE_HOME XDG_CACHE_HOME | |||
| 37 | # M10 block too, and any scenario added later that forgets. | 37 | # M10 block too, and any scenario added later that forgets. |
| 38 | SHELL=/bin/sh | 38 | SHELL=/bin/sh |
| 39 | export SHELL | 39 | export SHELL |
| 40 | # MUX_KEY_FILE belongs to the same family and is cleared here for the same | ||
| 41 | # forgets-proofing reason: `muxd endpoint` reads it BEFORE the default key | ||
| 42 | # path, so an operator who happens to have one exported would silently | ||
| 43 | # change which key the handoff announces — and the scenarios would still | ||
| 44 | # pass, against the wrong key, until one of them did not. | ||
| 45 | unset MUX_KEY_FILE | ||
| 40 | # Second daemon, used only by the M7 abort scenario; declared here so the | 46 | # Second daemon, used only by the M7 abort scenario; declared here so the |
| 41 | # trap below can reference them under `set -u` before they are ever started. | 47 | # trap below can reference them under `set -u` before they are ever started. |
| 42 | SOCK2="${TMPDIR:-/tmp}/muxd-e2e-abort-$$.sock" | 48 | SOCK2="${TMPDIR:-/tmp}/muxd-e2e-abort-$$.sock" |
| @@ -132,6 +138,14 @@ HQPORT=$(( 11000 + ($$ % 4000) )) | |||
| 132 | HDEADPORT=$(( 16000 + ($$ % 4000) )) | 138 | HDEADPORT=$(( 16000 + ($$ % 4000) )) |
| 133 | HAPID="" | 139 | HAPID="" |
| 134 | HDPID="" | 140 | HDPID="" |
| 141 | # The three handoff clients that run in the BACKGROUND. Each is already | ||
| 142 | # bounded by its own `timeout`, which stays the primary guarantee; these | ||
| 143 | # are the trap's backstop for the window where a scenario exits between | ||
| 144 | # the launch and the wait, leaving a client the shell would otherwise | ||
| 145 | # orphan. Declared here so the trap can read them under `set -u`. | ||
| 146 | H1PID="" | ||
| 147 | H4PID="" | ||
| 148 | H5PID="" | ||
| 135 | 149 | ||
| 136 | # One counter out of a MUX_PREDICT_STATS line. The client prints exactly one | 150 | # One counter out of a MUX_PREDICT_STATS line. The client prints exactly one |
| 137 | # such line on exit; every field is a key=value pair, so a rename or reorder | 151 | # such line on exit; every field is a key=value pair, so a rename or reorder |
| @@ -351,6 +365,12 @@ cleanup() { | |||
| 351 | # could no longer reach by path. | 365 | # could no longer reach by path. |
| 352 | [ -n "$HDPID" ] && kill "$HDPID" 2>/dev/null || true | 366 | [ -n "$HDPID" ] && kill "$HDPID" 2>/dev/null || true |
| 353 | [ -n "$HAPID" ] && kill "$HAPID" 2>/dev/null || true | 367 | [ -n "$HAPID" ] && kill "$HAPID" 2>/dev/null || true |
| 368 | # The backgrounded clients, by tracked pid. Their `timeout` bounds them | ||
| 369 | # anyway; this only shortens the wait on a run that failed between a | ||
| 370 | # launch and its wait. | ||
| 371 | [ -n "$H1PID" ] && kill "$H1PID" 2>/dev/null || true | ||
| 372 | [ -n "$H4PID" ] && kill "$H4PID" 2>/dev/null || true | ||
| 373 | [ -n "$H5PID" ] && kill "$H5PID" 2>/dev/null || true | ||
| 354 | [ -n "${MUXD:-}" ] && "$MUXD" stop --sock "$SOCK16" >/dev/null 2>&1 || true | 374 | [ -n "${MUXD:-}" ] && "$MUXD" stop --sock "$SOCK16" >/dev/null 2>&1 || true |
| 355 | [ -n "${MUXD:-}" ] && "$MUXD" stop --sock "$SOCK17" >/dev/null 2>&1 || true | 375 | [ -n "${MUXD:-}" ] && "$MUXD" stop --sock "$SOCK17" >/dev/null 2>&1 || true |
| 356 | rm -f "$SOCK8" "$SOCK8T" "$SOCK11" "$OUT.start" "$OUT.start2" "$OUT.s8" \ | 376 | rm -f "$SOCK8" "$SOCK8T" "$SOCK11" "$OUT.start" "$OUT.start2" "$OUT.s8" \ |
| @@ -2263,10 +2283,20 @@ ok "local mux auto-start under the pty fixture" | |||
| 2263 | # that log is how these scenarios know whether ssh ran. Never `ps | grep | 2283 | # that log is how these scenarios know whether ssh ran. Never `ps | grep |
| 2264 | # ssh`: this box belongs to someone who is probably ssh'd into something, | 2284 | # ssh`: this box belongs to someone who is probably ssh'd into something, |
| 2265 | # and a warm attach's entire claim is that no ssh existed. | 2285 | # and a warm attach's entire claim is that no ssh existed. |
| 2286 | # | ||
| 2287 | # The contract it depends on: the client spawns `ssh HOST CMD...` as | ||
| 2288 | # separate shell words, so dropping HOST is a `shift` and the rest is a | ||
| 2289 | # command. A future client that quoted the remote command instead would | ||
| 2290 | # arrive here as two arguments whose second is one word, and `exec` would | ||
| 2291 | # fail to find a program by that name — loudly, at 127, rather than by | ||
| 2292 | # quietly not being ssh. The arity guard below is the cheaper half of the | ||
| 2293 | # same insurance: a fixture whose silence cannot be told from success is | ||
| 2294 | # worth two shell words to rule out. | ||
| 2266 | mkdir -p "$SSHIM_DIR" "$HRUN" "$HRUN2" | 2295 | mkdir -p "$SSHIM_DIR" "$HRUN" "$HRUN2" |
| 2267 | : > "$SSHIM_PIDLOG" | 2296 | : > "$SSHIM_PIDLOG" |
| 2268 | cat > "$SSHIM_DIR/ssh" <<'SHIM' | 2297 | cat > "$SSHIM_DIR/ssh" <<'SHIM' |
| 2269 | #!/bin/sh | 2298 | #!/bin/sh |
| 2299 | [ $# -ge 2 ] || exit 97 | ||
| 2270 | echo $$ >> "${SSHIM_PIDLOG:?}" | 2300 | echo $$ >> "${SSHIM_PIDLOG:?}" |
| 2271 | shift | 2301 | shift |
| 2272 | exec "$@" | 2302 | exec "$@" |
| @@ -2284,22 +2314,19 @@ HPATH="$SSHIM_DIR:$(dirname "$MUXD"):$PATH" | |||
| 2284 | # attempt below would fail at DNS without ever dialling. | 2314 | # attempt below would fail at DNS without ever dialling. |
| 2285 | HHOST="mux-e2e@127.0.0.1" | 2315 | HHOST="mux-e2e@127.0.0.1" |
| 2286 | HCACHE="$XDG_CACHE_HOME/mux/hosts/$HHOST" | 2316 | HCACHE="$XDG_CACHE_HOME/mux/hosts/$HHOST" |
| 2287 | # Nothing may be set in the environment that redirects the key resolution: | ||
| 2288 | # `muxd endpoint` reads MUX_KEY_FILE first, and an operator who has one | ||
| 2289 | # exported would silently change which key gets announced. | ||
| 2290 | unset MUX_KEY_FILE | ||
| 2291 | |||
| 2292 | # (a) COLD: no daemon, no cache. The attach has to produce the daemon (via | 2317 | # (a) COLD: no daemon, no cache. The attach has to produce the daemon (via |
| 2293 | # `muxd endpoint`'s own auto-start), fetch coordinates over the shim, and | 2318 | # `muxd endpoint`'s own auto-start), fetch coordinates over the shim, and |
| 2294 | # END UP ON QUIC — and that last part is proven by observation rather than | 2319 | # END UP ON QUIC — and that last part is proven by observation rather than |
| 2295 | # inference: the ssh the announce arrived on is killed by the client on | 2320 | # inference: the ssh the announce arrived on is killed by the client on |
| 2296 | # QUIC success, so once its pid is OBSERVED gone, a second marker still | 2321 | # QUIC success, so once its pid is OBSERVED gone, a second marker still |
| 2297 | # making the round trip can only be riding QUIC. The pipe's owner is dead. | 2322 | # making the round trip can only be riding QUIC. The pipe's owner is dead. |
| 2323 | set +e | ||
| 2298 | { printf 'printf "cold-%%s\\n" one\n'; sleep 6; \ | 2324 | { printf 'printf "cold-%%s\\n" one\n'; sleep 6; \ |
| 2299 | printf 'printf "cold-%%s\\n" two\n'; sleep 2.5; printf '\034'; } | \ | 2325 | printf 'printf "cold-%%s\\n" two\n'; sleep 2.5; printf '\034'; } | \ |
| 2300 | SHELL=/bin/sh XDG_RUNTIME_DIR="$HRUN" PATH="$HPATH" timeout 60 \ | 2326 | SHELL=/bin/sh XDG_RUNTIME_DIR="$HRUN" PATH="$HPATH" timeout 60 \ |
| 2301 | "$MUX" "$HHOST" > "$OUT.h1" 2> "$OUT.h1.err" & | 2327 | "$MUX" "$HHOST" > "$OUT.h1" 2> "$OUT.h1.err" & |
| 2302 | H1PID=$! | 2328 | H1PID=$! |
| 2329 | set -e | ||
| 2303 | # The bounded wait, with a message of its own, because of what it is FOR: | 2330 | # The bounded wait, with a message of its own, because of what it is FOR: |
| 2304 | # a client blocked on an announce that is never coming has no other | 2331 | # a client blocked on an announce that is never coming has no other |
| 2305 | # symptom. It cannot be left to the suite's timeout or the trap — those | 2332 | # symptom. It cannot be left to the suite's timeout or the trap — those |
| @@ -2317,20 +2344,33 @@ HSHIMS=$(wc -l < "$SSHIM_PIDLOG") | |||
| 2317 | HSHIMPID=$(head -1 "$SSHIM_PIDLOG") | 2344 | HSHIMPID=$(head -1 "$SSHIM_PIDLOG") |
| 2318 | # The BASELINE, and it is what makes the pair below an ordering proof | 2345 | # The BASELINE, and it is what makes the pair below an ordering proof |
| 2319 | # rather than two independent facts: the second marker must not have | 2346 | # rather than two independent facts: the second marker must not have |
| 2320 | # arrived YET. Without it the scenario has a hole with a real failure mode | 2347 | # arrived YET. |
| 2321 | # behind it — if an attach ever took longer than the script's 6s sleep, | 2348 | # |
| 2322 | # both markers would be typed before the session existed, both would be | 2349 | # What actually guarantees the ordering is the PRODUCT, and saying so is |
| 2323 | # delivered out of the carry buffer at attach, and cold-two would already | 2350 | # the point of this comment. openHandoff kills the ssh child the moment |
| 2324 | # be in the capture when ssh was killed. Every check below would then pass | 2351 | # QUIC is ready, before Transport.open returns — so before the session can |
| 2325 | # while proving nothing about bytes moving AFTER the pipe's owner died. | 2352 | # emit a single byte, and therefore before cold-one, never mind cold-two. |
| 2353 | # The baseline pins the weaker, checkable half of that: cold-two had not | ||
| 2354 | # arrived at the instant we observed the death. Anyone who moves WHEN the | ||
| 2355 | # client kills ssh — to after the first frame, say, or into a deferred | ||
| 2356 | # teardown — takes this scenario's premise with it, and should expect to | ||
| 2357 | # rewrite the block rather than to nudge a sleep. | ||
| 2326 | # | 2358 | # |
| 2327 | # A failure here is the scenario's own premise breaking (an attach slower | 2359 | # The hole it closes is real rather than theoretical. Both markers are |
| 2328 | # than 6s against ~300ms observed), not a product defect, and it says so. | 2360 | # typed by a script on its own clock: if an attach ever outran the 6s |
| 2361 | # between them, both would be buffered before the session existed and | ||
| 2362 | # delivered together, and cold-two could already be on screen here. The | ||
| 2363 | # scenario could then no longer WITNESS post-death traffic — not because | ||
| 2364 | # the product misbehaved (the kill still preceded both) but because the | ||
| 2365 | # two events stopped being separable. That is what this reports, and the | ||
| 2366 | # message says so rather than accusing the product of an ordering it did | ||
| 2367 | # not get wrong. | ||
| 2329 | grep -q "cold-two" "$OUT.h1" && { | 2368 | grep -q "cold-two" "$OUT.h1" && { |
| 2330 | echo "e2e FAIL: cold handoff: the second marker arrived before ssh was killed," | 2369 | echo "e2e FAIL: cold handoff: the second marker is already on screen at the" |
| 2331 | echo " so it cannot witness QUIC carrying the session. The attach was" | 2370 | echo " baseline, so it cannot witness bytes moving after ssh died —" |
| 2332 | echo " slower than this scenario's 6s script — a timing premise broke," | 2371 | echo " both markers were delivered together, which is what happens" |
| 2333 | echo " not necessarily the product." | 2372 | echo " when the attach outruns this scenario's 6s script (~300ms" |
| 2373 | echo " observed). A timing premise broke, not necessarily the product." | ||
| 2334 | exit 1; } | 2374 | exit 1; } |
| 2335 | wait_pid_gone "$HSHIMPID" "cold handoff: QUIC took over, so ssh must be gone" | 2375 | wait_pid_gone "$HSHIMPID" "cold handoff: QUIC took over, so ssh must be gone" |
| 2336 | # ...and the session did not go with it. | 2376 | # ...and the session did not go with it. |
| @@ -2551,6 +2591,14 @@ grep -q "^mux: quic://127.0.0.1:$HQPORT unreachable, attaching over ssh$" "$OUT. | |||
| 2551 | # in the dial. A tighter ceiling would therefore catch nothing this one | 2591 | # in the dial. A tighter ceiling would therefore catch nothing this one |
| 2552 | # misses, while making the single scenario that already pays 2s by design | 2592 | # misses, while making the single scenario that already pays 2s by design |
| 2553 | # the most load-sensitive check in the suite. | 2593 | # the most load-sensitive check in the suite. |
| 2594 | # | ||
| 2595 | # And it is worth knowing what this bound can actually SEE, which is a | ||
| 2596 | # narrow band. Past about six seconds the script's own detach byte lands | ||
| 2597 | # inside the dial, `waitReady` answers it as an abort, no fallback line is | ||
| 2598 | # ever printed, and the overrun is reported by the wait_for above — with | ||
| 2599 | # its own message — rather than here. So this check owns the range between | ||
| 2600 | # the budget and that: long enough to be a real overrun, short enough that | ||
| 2601 | # the line still gets printed. The wait_for is the backstop beyond it. | ||
| 2554 | [ "$HMS" -ge 1500 ] || { | 2602 | [ "$HMS" -ge 1500 ] || { |
| 2555 | echo "e2e FAIL: the fallback came after ${HMS}ms, too fast to have spent the" | 2603 | echo "e2e FAIL: the fallback came after ${HMS}ms, too fast to have spent the" |
| 2556 | echo " 2000ms QUIC budget — the dial cannot have happened" | 2604 | echo " 2000ms QUIC budget — the dial cannot have happened" |
| @@ -2665,6 +2713,16 @@ RC_STOP=$? | |||
| 2665 | set -e | 2713 | set -e |
| 2666 | [ "$RC_STOP" = "0" ] || { | 2714 | [ "$RC_STOP" = "0" ] || { |
| 2667 | echo "e2e FAIL: key-mismatch daemon stop exited $RC_STOP, want 0"; cat "$OUT.stop"; exit 1; } | 2715 | echo "e2e FAIL: key-mismatch daemon stop exited $RC_STOP, want 0"; cat "$OUT.stop"; exit 1; } |
| 2716 | # The same three assertions the SOCK16 leg makes, and for the same reason: | ||
| 2717 | # an exit code says the command returned, not that it did the job. The | ||
| 2718 | # stopped line is the daemon's own account of it and the absent socket is | ||
| 2719 | # the filesystem's — a stop that exited 0 while leaving either behind is | ||
| 2720 | # the regression this shape exists to catch, and it would be invisible | ||
| 2721 | # here without them. | ||
| 2722 | grep -q '^muxd: stopped' "$OUT.stop" || { | ||
| 2723 | echo "e2e FAIL: key-mismatch daemon stop did not report stopped"; cat "$OUT.stop"; exit 1; } | ||
| 2724 | [ ! -S "$SOCK17" ] || { | ||
| 2725 | echo "e2e FAIL: stop left the key-mismatch daemon's socket"; ls -l "$SOCK17"; exit 1; } | ||
| 2668 | wait_pid_gone "$HDPID" "key-mismatch daemon: stop reported stopped" | 2726 | wait_pid_gone "$HDPID" "key-mismatch daemon: stop reported stopped" |
| 2669 | HDPID="" | 2727 | HDPID="" |
| 2670 | rm -f "$OUT.h1" "$OUT.h1.err" "$OUT.h2" "$OUT.h2.err" "$OUT.h3" "$OUT.h3.err" \ | 2728 | rm -f "$OUT.h1" "$OUT.h1.err" "$OUT.h2" "$OUT.h2.err" "$OUT.h3" "$OUT.h3.err" \ |