01b5ae24
refactor: the helpers, the registry and the trap become e2e_lib.sh
a73x 2026-08-26 18:42
Commit message
test/e2e.sh
| Old | New | ||
|---|---|---|---|
| @@ -94,1066 +94,11 @@ done | |||
| 94 | echo " daemon can still report a command's real exit code" | 94 | echo " daemon can still report a command's real exit code" |
| 95 | exit 1; } | 95 | exit 1; } |
| 96 | 96 | ||
| 97 | # ---- the cleanup registry --------------------------------------------- | 97 | # The helpers, the registry and the trap live in e2e_lib.sh; the scenarios |
| 98 | # Every process and every artifact a leg creates is REGISTERED where it is | 98 | # are still below. This file is the runner: it hands them a hermetic |
| 99 | # created, and the EXIT trap walks the registers. Before this, cleanup() | 99 | # environment and an armed trap, and owns the count pin at the bottom. |
| 100 | # carried four hand-maintained lists naming ~137 $SOCKnn/$DnnPID variables, | 100 | E2E_DIR=$(dirname "$0") |
| 101 | # and a leg added without all four edits leaked: 102 captures were missing | 101 | . "$E2E_DIR/e2e_lib.sh" |
| 102 | # from them by 2026-08-19, enough that `make soak` refused to start. A list | ||
| 103 | # kept somewhere other than the code it describes is a list that goes | ||
| 104 | # stale, so there is no list — registering IS the statement that creates | ||
| 105 | # the thing. | ||
| 106 | # | ||
| 107 | # Captures are the exception, and deliberately so: every one of them is | ||
| 108 | # spelled from $OUT, and the trap removes that whole set by the same | ||
| 109 | # pattern leak_sweep already reads it by. A pattern cannot forget a leg, | ||
| 110 | # and forgetting was the failure. What has to be registered is what no | ||
| 111 | # pattern can find — pids, and the artifacts spelled somewhere other than | ||
| 112 | # $OUT. | ||
| 113 | # | ||
| 114 | # Newline-joined strings rather than arrays: the shebang is /bin/sh and the | ||
| 115 | # build's shell gate lints this file as POSIX sh. No path here can hold a | ||
| 116 | # newline (every one is spelled from $TMPDIR, $OUT and $$ in this file), | ||
| 117 | # and cleanup sets IFS to a newline before walking them, so a $TMPDIR with | ||
| 118 | # a space in it still reads as one path. | ||
| 119 | E2E_KILL="" | ||
| 120 | E2E_SOCK="" | ||
| 121 | E2E_RM="" | ||
| 122 | |||
| 123 | # defer_kill PID... — a process the trap must end. Newest first, so | ||
| 124 | # teardown runs in the reverse of creation order and a client is signalled | ||
| 125 | # before the daemon it is attached to. | ||
| 126 | # | ||
| 127 | # An `if` rather than `[ -n "$_dk" ] && ...`, for wait_sock's reason: a | ||
| 128 | # false guard as the last command in a function becomes that function's | ||
| 129 | # exit status, and under `set -e` registering an empty pid would abort the | ||
| 130 | # caller. | ||
| 131 | defer_kill() { | ||
| 132 | for _dk in "$@"; do | ||
| 133 | if [ -n "$_dk" ]; then | ||
| 134 | # Already registered is not an error: pipe_mux names the same | ||
| 135 | # FIFO on every call, and a leg that starts two daemons on one | ||
| 136 | # path registers it twice. Registering twice must cost nothing, | ||
| 137 | # or callers start reasoning about who registered first. | ||
| 138 | case " | ||
| 139 | $E2E_KILL" in | ||
| 140 | *" | ||
| 141 | $_dk | ||
| 142 | "*) continue ;; | ||
| 143 | esac | ||
| 144 | E2E_KILL="$_dk | ||
| 145 | $E2E_KILL" | ||
| 146 | fi | ||
| 147 | done | ||
| 148 | } | ||
| 149 | |||
| 150 | # defer_sock PATH... — a path a daemon may be listening on. The trap asks | ||
| 151 | # `muxd stop` there before it kills anything and long before it unlinks the | ||
| 152 | # path: a daemon this shell never forked — one a proxy, `muxd start` or the | ||
| 153 | # handoff spawned — has no pid here, and the socket is the only handle | ||
| 154 | # there is. | ||
| 155 | defer_sock() { | ||
| 156 | for _ds in "$@"; do | ||
| 157 | if [ -n "$_ds" ]; then | ||
| 158 | # Already registered is not an error: pipe_mux names the same | ||
| 159 | # FIFO on every call, and a leg that starts two daemons on one | ||
| 160 | # path registers it twice. Registering twice must cost nothing, | ||
| 161 | # or callers start reasoning about who registered first. | ||
| 162 | case " | ||
| 163 | $E2E_SOCK" in | ||
| 164 | *" | ||
| 165 | $_ds | ||
| 166 | "*) continue ;; | ||
| 167 | esac | ||
| 168 | E2E_SOCK="$_ds | ||
| 169 | $E2E_SOCK" | ||
| 170 | fi | ||
| 171 | done | ||
| 172 | } | ||
| 173 | |||
| 174 | # defer_rm PATH... — an artifact the trap must remove: a key, a generated | ||
| 175 | # script, a private HOME, a state home. Registered at the line that spells | ||
| 176 | # the path, so the two can never drift apart. Files go out through the leak | ||
| 177 | # sweep's banking; directories are removed whole. | ||
| 178 | defer_rm() { | ||
| 179 | for _dr in "$@"; do | ||
| 180 | if [ -n "$_dr" ]; then | ||
| 181 | # Already registered is not an error: pipe_mux names the same | ||
| 182 | # FIFO on every call, and a leg that starts two daemons on one | ||
| 183 | # path registers it twice. Registering twice must cost nothing, | ||
| 184 | # or callers start reasoning about who registered first. | ||
| 185 | case " | ||
| 186 | $E2E_RM" in | ||
| 187 | *" | ||
| 188 | $_dr | ||
| 189 | "*) continue ;; | ||
| 190 | esac | ||
| 191 | E2E_RM="$_dr | ||
| 192 | $E2E_RM" | ||
| 193 | fi | ||
| 194 | done | ||
| 195 | } | ||
| 196 | |||
| 197 | |||
| 198 | SOCK="${TMPDIR:-/tmp}/muxd-e2e-$$.sock" | ||
| 199 | defer_sock "$SOCK" | ||
| 200 | OUT="${TMPDIR:-/tmp}/mux-e2e-out-$$" | ||
| 201 | # M10: hermetic XDG homes. Key-default scenarios must see OUR key or none, | ||
| 202 | # never the developer's real ~/.config/mux/key. | ||
| 203 | XDG_CONFIG_HOME="${TMPDIR:-/tmp}/mux-e2e-cfg-$$" | ||
| 204 | defer_rm "$XDG_CONFIG_HOME" | ||
| 205 | XDG_STATE_HOME="${TMPDIR:-/tmp}/mux-e2e-state-$$" | ||
| 206 | defer_rm "$XDG_STATE_HOME" | ||
| 207 | # M14: and the same for the handoff's per-host cache. It holds a KEY, and | ||
| 208 | # the M14 scenarios both read and poison it — neither of which may ever | ||
| 209 | # touch the developer's real ~/.cache/mux. | ||
| 210 | XDG_CACHE_HOME="${TMPDIR:-/tmp}/mux-e2e-cache-$$" | ||
| 211 | defer_rm "$XDG_CACHE_HOME" | ||
| 212 | export XDG_CONFIG_HOME XDG_STATE_HOME XDG_CACHE_HOME | ||
| 213 | # ...and the same argument for $SHELL, which is not an XDG home but is read | ||
| 214 | # the same way: every daemon this suite AUTO-STARTS gets no --shell flag and | ||
| 215 | # resolves $SHELL, so without this the suite runs the developer's login | ||
| 216 | # shell and its whole rc — arbitrary code, on the session under test. | ||
| 217 | # | ||
| 218 | # Found by soak, not by reasoning. The M10 `muxd start` block inherited zsh, | ||
| 219 | # whose plugin manager roots itself at $XDG_CACHE_HOME; pointing that at a | ||
| 220 | # fresh directory (the line above) made every session re-clone its plugins | ||
| 221 | # from the network before the shell would answer, and the scenario's marker | ||
| 222 | # missed its window. The daemon was healthy the whole time. The M13 blocks | ||
| 223 | # already pin SHELL per command for this reason; hoisting it here covers the | ||
| 224 | # M10 block too, and any scenario added later that forgets. | ||
| 225 | SHELL=/bin/sh | ||
| 226 | export SHELL | ||
| 227 | # MUX_KEY_FILE belongs to the same family and is cleared here for the same | ||
| 228 | # forgets-proofing reason: `muxd endpoint` reads it BEFORE the default key | ||
| 229 | # path, so an operator who happens to have one exported would silently | ||
| 230 | # change which key the handoff announces — and the scenarios would still | ||
| 231 | # pass, against the wrong key, until one of them did not. | ||
| 232 | unset MUX_KEY_FILE | ||
| 233 | # ...and SSH_AUTH_SOCK, the same family again and the sharpest case in it. | ||
| 234 | # A daemon that inherited the developer's real agent would hand it to every | ||
| 235 | # session whose own agent socket failed to bind — and the refusal leg, whose | ||
| 236 | # whole subject is a shell finding nobody to sign for it, would then be | ||
| 237 | # asserting against the developer's keyring. The legs that want an agent | ||
| 238 | # export one per command, at a path this file made. | ||
| 239 | unset SSH_AUTH_SOCK | ||
| 240 | |||
| 241 | # One counter out of a MUX_PREDICT_STATS line. The client prints exactly one | ||
| 242 | # such line on exit; every field is a key=value pair, so a rename or reorder | ||
| 243 | # in the client shows up here as an empty read rather than a wrong number. | ||
| 244 | predict_stat() { | ||
| 245 | sed -n "s/.*predict .*$2=\([0-9]*\).*/\1/p" "$1" | head -1 | ||
| 246 | } | ||
| 247 | |||
| 248 | # Assert one counter, with the whole line in the failure so a wrong number is | ||
| 249 | # read in context rather than alone. | ||
| 250 | want_stat() { | ||
| 251 | _got=$(predict_stat "$1" "$2") | ||
| 252 | [ -n "$_got" ] || { | ||
| 253 | echo "e2e FAIL: $4: no predict stats line (wanted $2=$3); got:" | ||
| 254 | cat "$1"; exit 1; | ||
| 255 | } | ||
| 256 | [ "$_got" = "$3" ] || { | ||
| 257 | echo "e2e FAIL: $4: $2=$_got, want $3" | ||
| 258 | grep "^predict " "$1" || true | ||
| 259 | exit 1; | ||
| 260 | } | ||
| 261 | } | ||
| 262 | |||
| 263 | # The same, as a floor. Its own helper rather than a mode on want_stat: | ||
| 264 | # exact equality is the right assertion for every correctness counter, and | ||
| 265 | # a shared one would make it easy to weaken those by habit. Only counters | ||
| 266 | # that are genuinely timing-dependent belong here. | ||
| 267 | want_stat_ge() { | ||
| 268 | _got=$(predict_stat "$1" "$2") | ||
| 269 | [ -n "$_got" ] || { | ||
| 270 | echo "e2e FAIL: $4: no predict stats line (wanted $2>=$3); got:" | ||
| 271 | cat "$1"; exit 1; | ||
| 272 | } | ||
| 273 | [ "$_got" -ge "$3" ] || { | ||
| 274 | echo "e2e FAIL: $4: $2=$_got, want >=$3" | ||
| 275 | grep "^predict " "$1" || true | ||
| 276 | exit 1; | ||
| 277 | } | ||
| 278 | } | ||
| 279 | |||
| 280 | # Wait until PATTERN shows up in FILE (default 15s). Timing that keys off the | ||
| 281 | # session's own output instead of a fixed sleep: the marker is proof the | ||
| 282 | # client is attached and idle, which is exactly the state the tear needs. | ||
| 283 | # E2E_TIME_SCALE multiplies the polling BUDGETS below, and nothing else. It | ||
| 284 | # exists for test/coverage.sh: a ptrace-traced binary runs about half again | ||
| 285 | # slower, which is enough to blow a 25s convergence budget that is generous | ||
| 286 | # at native speed — and the failure reads as "the attach never converged" | ||
| 287 | # rather than "the tracer is slow", which is a lie about the product. | ||
| 288 | # | ||
| 289 | # Budgets only. Not the sleeps between polls, not a `timeout` that a scenario | ||
| 290 | # asserts on, and never a threshold: scaling this changes how long the suite | ||
| 291 | # is WILLING TO WAIT, never what it demands to see. A run at scale 4 that | ||
| 292 | # passes proves the same facts as a run at scale 1, just later. | ||
| 293 | TIME_SCALE="${E2E_TIME_SCALE:-1}" | ||
| 294 | case "$TIME_SCALE" in | ||
| 295 | ''|*[!0-9]*) echo "e2e FAIL: E2E_TIME_SCALE must be a positive integer"; exit 1 ;; | ||
| 296 | esac | ||
| 297 | [ "$TIME_SCALE" -ge 1 ] || { echo "e2e FAIL: E2E_TIME_SCALE must be >= 1"; exit 1; } | ||
| 298 | |||
| 299 | # Validated here rather than at its use in ok(), for the reason above: a bad | ||
| 300 | # value would otherwise surface as a bare test(1) error at the first passing | ||
| 301 | # scenario, with no e2e FAIL line to say what was wrong. | ||
| 302 | case "${E2E_STOP_AFTER:-1}" in | ||
| 303 | ''|*[!0-9]*) echo "e2e FAIL: E2E_STOP_AFTER must be a positive integer"; exit 1 ;; | ||
| 304 | esac | ||
| 305 | |||
| 306 | wait_for() { | ||
| 307 | _file="$1"; _pat="$2"; _ticks=$(( ${3:-15} * 10 * TIME_SCALE )); _i=0 | ||
| 308 | while [ "$_i" -lt "$_ticks" ]; do | ||
| 309 | if [ -f "$_file" ] && grep -q "$_pat" "$_file" 2>/dev/null; then return 0; fi | ||
| 310 | sleep 0.1; _i=$((_i+1)) | ||
| 311 | done | ||
| 312 | return 1 | ||
| 313 | } | ||
| 314 | |||
| 315 | # hardkill PID — SIGKILL a pid and anything it fathered, children first. | ||
| 316 | # | ||
| 317 | # Plain `kill -9 "$pid"` is right only while the pid the suite holds IS the | ||
| 318 | # process under test. Under the coverage harness (test/coverage.sh) it is a | ||
| 319 | # kcov wrapper instead, and SIGKILL is the one signal a wrapper cannot | ||
| 320 | # forward: killing the node alone would leave the real daemon alive holding | ||
| 321 | # its socket, and every leg that waits for that daemon to die would hang | ||
| 322 | # rather than fail. Children first so a traced child's death is observed by | ||
| 323 | # its tracer, which is when kcov writes the coverage it has collected. | ||
| 324 | # | ||
| 325 | # With no wrapper in the picture there are no children and this is exactly | ||
| 326 | # `kill -9`, which is why the abort legs keep the semantics they assert on: | ||
| 327 | # the daemon still dies by SIGKILL, still without unlinking its socket. | ||
| 328 | hardkill() { | ||
| 329 | # The test is what the process IS, not whether it has children. A daemon | ||
| 330 | # has children too — a session shell per attach — and killing those first | ||
| 331 | # ends the session CLEANLY, which is the one thing the abort legs must not | ||
| 332 | # see: they assert on a client whose daemon vanished under it, and a tidy | ||
| 333 | # session exit takes a different path out of the client (measured: exit | ||
| 334 | # 128 where the leg wants 0). Only a wrapper gets the two-step treatment. | ||
| 335 | if [ "$(ps -o comm= -p "$1" 2>/dev/null)" = kcov ]; then | ||
| 336 | # The CHILD is the daemon the suite means to kill; the pid it holds is | ||
| 337 | # only the tracer. Kill the child first and the wrapper exits on its | ||
| 338 | # own, writing the coverage it has collected — measured: a SIGKILLed | ||
| 339 | # wrapper writes no coverage.db at any --output-interval, so a clean | ||
| 340 | # exit is the only exit that keeps the data. Then wait for it, because | ||
| 341 | # killing both at once is the same as never killing the child at all. | ||
| 342 | for _c in $(ps -o pid= --ppid "$1" 2>/dev/null); do | ||
| 343 | kill -9 "$_c" 2>/dev/null || true | ||
| 344 | done | ||
| 345 | _i=0 | ||
| 346 | while kill -0 "$1" 2>/dev/null && [ "$_i" -lt $(( 20 * TIME_SCALE )) ]; do | ||
| 347 | sleep 0.05; _i=$((_i + 1)) | ||
| 348 | done | ||
| 349 | fi | ||
| 350 | kill -9 "$1" 2>/dev/null || true | ||
| 351 | } | ||
| 352 | |||
| 353 | # hardkill's TERM twin. A signal sent to a kcov wrapper is dropped: ptrace | ||
| 354 | # intercepts signals bound for the TRACEE, and the tracer has no handler of | ||
| 355 | # its own — measured on the hub leg, where `kill $W3PID` left both alive and | ||
| 356 | # wait_pid_gone timed out. TERM the tracee; the wrapper exits with it and | ||
| 357 | # writes its database. Off the tracer this is a plain kill, same exit status. | ||
| 358 | # A /proc claim about the daemon is read off the tracee: under `make coverage` | ||
| 359 | # the pid the suite holds is the tracer's (hardkill's reason), and kcov's own | ||
| 360 | # cmdline is what a resumed-argv check would otherwise read. | ||
| 361 | real_pid() { | ||
| 362 | if [ "$(ps -o comm= -p "$1" 2>/dev/null)" = kcov ]; then | ||
| 363 | ps -o pid= --ppid "$1" 2>/dev/null | head -1 | tr -d ' ' | ||
| 364 | else | ||
| 365 | echo "$1" | ||
| 366 | fi | ||
| 367 | } | ||
| 368 | |||
| 369 | softkill() { | ||
| 370 | if [ "$(ps -o comm= -p "$1" 2>/dev/null)" = kcov ]; then | ||
| 371 | _rc=1 | ||
| 372 | for _c in $(ps -o pid= --ppid "$1" 2>/dev/null); do | ||
| 373 | kill "$_c" 2>/dev/null && _rc=0 | ||
| 374 | done | ||
| 375 | return $_rc | ||
| 376 | fi | ||
| 377 | kill "$1" 2>/dev/null | ||
| 378 | } | ||
| 379 | |||
| 380 | # Poll until nothing answers on a socket path (2s). Keyed off the daemon's | ||
| 381 | # own liveness rather than a fixed sleep, same reasoning as wait_for. | ||
| 382 | wait_gone() { | ||
| 383 | _i=0 | ||
| 384 | while "$MUXD" dump --sock "$1" > /dev/null 2>&1; do | ||
| 385 | _i=$((_i + 1)); [ "$_i" -lt $(( 40 * TIME_SCALE )) ] || { echo "e2e FAIL: daemon on $1 never died"; exit 1; } | ||
| 386 | sleep 0.05 | ||
| 387 | done | ||
| 388 | } | ||
| 389 | |||
| 390 | # wait_sock PATH LOG LABEL — poll until a daemon has bound PATH (5s), then | ||
| 391 | # ASSERT it. Every `muxd run &` in this file needs this: the bind happens | ||
| 392 | # after the fork, so the very next command would otherwise race it. LOG is | ||
| 393 | # the spawn's own capture, printed on failure because a daemon that failed | ||
| 394 | # to bind almost always said why; pass "" for the spawns that have none. | ||
| 395 | wait_sock() { | ||
| 396 | _i=0 | ||
| 397 | while [ ! -S "$1" ] && [ "$_i" -lt $(( 50 * TIME_SCALE )) ]; do sleep 0.1; _i=$((_i+1)); done | ||
| 398 | [ -S "$1" ] || { | ||
| 399 | echo "e2e FAIL: $3" | ||
| 400 | # An `if` rather than `[ -n "$2" ] && cat "$2"`, for converged_quiet's | ||
| 401 | # reason: a false guard must not become this block's exit status. | ||
| 402 | if [ -n "$2" ]; then cat "$2"; fi | ||
| 403 | exit 1 | ||
| 404 | } | ||
| 405 | } | ||
| 406 | |||
| 407 | # start_daemon SOCK LOG LABEL [muxd args...] — a daemon in the background, | ||
| 408 | # registered before it is waited for. Sets $DPID, the pid to end it by. | ||
| 409 | # $DPID is scratch: the NEXT start_daemon overwrites it, so a leg that | ||
| 410 | # still needs its daemon further down keeps the pid under a name of its | ||
| 411 | # own. The long-lived daemon is $D1PID for exactly that reason — it is | ||
| 412 | # read some 5,000 lines after it is started. | ||
| 413 | # | ||
| 414 | # The registration is the whole reason this exists. A `muxd run &` written | ||
| 415 | # out by hand is three statements — the spawn, the pid, the wait — plus a | ||
| 416 | # fourth in cleanup() saying how to end it, and the fourth is the one that | ||
| 417 | # got forgotten. Here the spawn IS the registration and there is no fourth | ||
| 418 | # place to edit. | ||
| 419 | # | ||
| 420 | # Only for the plain shape. A daemon that needs an environment in front of | ||
| 421 | # it writes its own spawn and calls defer_kill/defer_sock itself: `env` or | ||
| 422 | # a VAR=VAL prefix on a FUNCTION call does not scope to the function — in | ||
| 423 | # POSIX sh it assigns in THIS shell and the value stays there for every | ||
| 424 | # scenario after it. | ||
| 425 | start_daemon() { | ||
| 426 | _sds="$1"; _sdl="$2"; _sdlab="$3"; shift 3 | ||
| 427 | "$MUXD" run --sock "$_sds" "$@" > "$_sdl" 2>&1 & | ||
| 428 | DPID=$! | ||
| 429 | defer_kill "$DPID" | ||
| 430 | defer_sock "$_sds" | ||
| 431 | wait_sock "$_sds" "$_sdl" "$_sdlab" | ||
| 432 | } | ||
| 433 | |||
| 434 | |||
| 435 | # pipe_mux OUT ERR CMD... — a client on a plain pipe, and the pipe kept | ||
| 436 | # open. ERR is the stderr capture, "" to leave it on the suite's own. | ||
| 437 | # The suite's non-tty scenarios used to be `{ printf cmd; sleep 2; printf | ||
| 438 | # detach; } | "$MUX"`: two seconds bought for output that lands in | ||
| 439 | # milliseconds (measured: the needle is in the capture 2ms after the send), | ||
| 440 | # and 120s of the suite's runtime was exactly that purchase. Here the stdin | ||
| 441 | # is a FIFO held open on fd 9, so a scenario sends, then WAITS FOR THE NEEDLE | ||
| 442 | # with await_out, then detaches — the same assertion it was going to make | ||
| 443 | # anyway, moved to where it ends the wait. fd 9 is opened read-write so | ||
| 444 | # neither the open nor the client's EOF blocks on the other side; the client | ||
| 445 | # sees EOF only when pipe_detach closes it. | ||
| 446 | # | ||
| 447 | # Not for the scenarios that pace on purpose: the prediction legs snapshot | ||
| 448 | # the capture N ms after a keystroke, and a sleep there IS the test. | ||
| 449 | pipe_mux() { | ||
| 450 | PIPE_OUT="$1"; PIPE_ERR="$2"; shift 2 | ||
| 451 | PIPE_IN="$PIPE_OUT.in" | ||
| 452 | defer_rm "$PIPE_IN" | ||
| 453 | rm -f "$PIPE_IN" | ||
| 454 | mkfifo "$PIPE_IN" | ||
| 455 | exec 9<>"$PIPE_IN" | ||
| 456 | if [ -n "$PIPE_ERR" ]; then | ||
| 457 | "$@" < "$PIPE_IN" > "$PIPE_OUT" 2> "$PIPE_ERR" & | ||
| 458 | else | ||
| 459 | "$@" < "$PIPE_IN" > "$PIPE_OUT" & | ||
| 460 | fi | ||
| 461 | PIPE_PID=$! | ||
| 462 | defer_kill "$PIPE_PID" | ||
| 463 | } | ||
| 464 | |||
| 465 | # pipe_send FMT [ARGS] — printf into the open client, same spelling the | ||
| 466 | # inline blocks used so a conversion moves the format string verbatim. | ||
| 467 | pipe_send() { | ||
| 468 | # shellcheck disable=SC2059 | ||
| 469 | printf "$@" >&9 | ||
| 470 | } | ||
| 471 | |||
| 472 | # await_out FILE NEEDLE LABEL — wait_for with the verdict attached: FAIL | ||
| 473 | # with the capture when NEEDLE never comes (5s, scaled). The same `grep -q` | ||
| 474 | # the scenario asserts with afterwards, so waiting on it costs no new claim. | ||
| 475 | await_out() { | ||
| 476 | wait_for "$1" "$2" 5 || { | ||
| 477 | echo "e2e FAIL: $3; never saw '$2' in $1, which holds:" | ||
| 478 | cat "$1" | ||
| 479 | exit 1 | ||
| 480 | } | ||
| 481 | } | ||
| 482 | |||
| 483 | # fill_sessions SOCK STATE PREFIX FROM TO — one throwaway pipe attach per | ||
| 484 | # name so the daemon holds a slot for each. The session outlives the client | ||
| 485 | # that made it, which is what lets these detach immediately. | ||
| 486 | # | ||
| 487 | # THREE at a time, and the number is `max_observers` minus one, not | ||
| 488 | # `max_clients` minus one: `acceptConn` parks EVERY new connection in an | ||
| 489 | # observer slot and promotes it to a client only when its attach frame | ||
| 490 | # arrives, so simultaneous dials contend for the four observer slots, and | ||
| 491 | # the ones with nowhere to land are closed outright. Measured at batch 7 | ||
| 492 | # on a 32-slot daemon: 11 of 31 fills died with "connection to muxd lost" | ||
| 493 | # and the table never filled; at batch 3, 31 of 31 land. | ||
| 494 | # | ||
| 495 | # STATE is a scratch home so the fills never write a wall line the leg | ||
| 496 | # later counts. The last name is asked back through muxa so a fill that | ||
| 497 | # never landed fails here with its own log rather than as the refusal leg | ||
| 498 | # proving nothing. | ||
| 499 | fill_sessions() { | ||
| 500 | _fs_sock="$1"; _fs_state="$2"; _fs_pfx="$3"; _fs_i="$4"; _fs_to="$5" | ||
| 501 | while [ "$_fs_i" -le "$_fs_to" ]; do | ||
| 502 | _fs_pids=""; _fs_j=0 | ||
| 503 | while [ "$_fs_j" -lt 3 ] && [ "$_fs_i" -le "$_fs_to" ]; do | ||
| 504 | { sleep 1.5; printf '\034\034'; } | XDG_STATE_HOME="$_fs_state" timeout 40 \ | ||
| 505 | "$MUX" --sock "$_fs_sock" --session "$_fs_pfx$_fs_i" \ | ||
| 506 | > "$OUT.fill.$_fs_pfx$_fs_i" 2>&1 & | ||
| 507 | _fs_pids="$_fs_pids $!" | ||
| 508 | _fs_i=$((_fs_i + 1)); _fs_j=$((_fs_j + 1)) | ||
| 509 | done | ||
| 510 | for _fs_p in $_fs_pids; do wait "$_fs_p" || { | ||
| 511 | echo "e2e FAIL: fill_sessions $_fs_pfx: a fill attach on $_fs_sock exited nonzero:" | ||
| 512 | cat "$OUT".fill."$_fs_pfx"*; exit 1; }; done | ||
| 513 | done | ||
| 514 | "$MUXA" status --sock "$_fs_sock" --session "$_fs_pfx$_fs_to" > "$OUT.fill.$_fs_pfx.st" 2>&1 || { | ||
| 515 | echo "e2e FAIL: fill_sessions: $_fs_pfx$_fs_to was never created, the daemon is not full:" | ||
| 516 | cat "$OUT.fill.$_fs_pfx.st"; exit 1; } | ||
| 517 | } | ||
| 518 | |||
| 519 | # repaints FILE — how many full repaints (ESC[2J) a capture holds. The | ||
| 520 | # first paint after a reconnect is always a full one (interact.zig | ||
| 521 | # `repaint_after_resync`), so a count that rose is the resume itself, seen | ||
| 522 | # from the terminal's side. Keystrokes typed into a tear are dropped, not | ||
| 523 | # held, so a scenario that tears the transport must see this before it | ||
| 524 | # types again. | ||
| 525 | repaints() { | ||
| 526 | _clr=$(printf '\033[2J') | ||
| 527 | grep -a -F -o "$_clr" "$1" 2>/dev/null | wc -l | ||
| 528 | } | ||
| 529 | |||
| 530 | # await_repaint FILE BASELINE LABEL — wait until repaints FILE exceeds | ||
| 531 | # BASELINE (5s, scaled), FAIL with the capture otherwise. | ||
| 532 | await_repaint() { | ||
| 533 | _i=0 | ||
| 534 | while [ "$(repaints "$1")" -le "$2" ] && [ "$_i" -lt $(( 50 * TIME_SCALE )) ]; do | ||
| 535 | sleep 0.1; _i=$((_i+1)) | ||
| 536 | done | ||
| 537 | [ "$(repaints "$1")" -gt "$2" ] || { | ||
| 538 | echo "e2e FAIL: $3; no repaint after the tear in $1, which holds:" | ||
| 539 | cat "$1" | ||
| 540 | exit 1 | ||
| 541 | } | ||
| 542 | } | ||
| 543 | |||
| 544 | # pipe_detach [LABEL] — the Ctrl-\ chord, then EOF, then the client's own | ||
| 545 | # exit, which must be 0: a foreground `| "$MUX"` under set -e asserted that | ||
| 546 | # by accident, and this asserts it on purpose. LABEL names the client in | ||
| 547 | # the verdict; PIPE_RC is left for a scenario with more to say. | ||
| 548 | pipe_detach() { | ||
| 549 | printf '\034\034' >&9 | ||
| 550 | pipe_waitexit "$@" | ||
| 551 | } | ||
| 552 | |||
| 553 | # pipe_waitexit [LABEL [WANT]] — the client leaves on its own (its session | ||
| 554 | # ended); wait for that, then release the FIFO. Stdin stays open until the | ||
| 555 | # exit so the client never sees an EOF it was not sent. WANT is the exit | ||
| 556 | # status the scenario means (default 0): the exit-semantics legs want the | ||
| 557 | # shell's own. | ||
| 558 | pipe_waitexit() { | ||
| 559 | set +e | ||
| 560 | wait "$PIPE_PID" | ||
| 561 | PIPE_RC=$? | ||
| 562 | set -e | ||
| 563 | exec 9>&- | ||
| 564 | PIPE_PID="" | ||
| 565 | rm -f "$PIPE_IN" | ||
| 566 | [ "$PIPE_RC" -eq "${2:-0}" ] || { | ||
| 567 | echo "e2e FAIL: ${1:-piped client} exited $PIPE_RC (want ${2:-0}; 124 means it hung)" | ||
| 568 | cat "$PIPE_OUT" | ||
| 569 | if [ -n "$PIPE_ERR" ]; then cat "$PIPE_ERR"; fi | ||
| 570 | exit 1 | ||
| 571 | } | ||
| 572 | } | ||
| 573 | |||
| 574 | # dump_session SOCK [NAME] — `muxd dump` against one session (M18). An | ||
| 575 | # EMPTY or absent NAME passes no --session flag AT ALL rather than an empty | ||
| 576 | # one, and that is the load-bearing part: no tail is the wire's own | ||
| 577 | # default-session spelling and the pre-M18 compat path (decision 3), so | ||
| 578 | # every call site here that never heard of sessions keeps sending exactly | ||
| 579 | # the bytes it always did. One helper, so a scenario asking about a named | ||
| 580 | # session and one asking about the default cannot drift into two spellings. | ||
| 581 | dump_session() { | ||
| 582 | if [ -n "${2:-}" ]; then | ||
| 583 | "$MUXD" dump --sock "$1" --session "$2" | ||
| 584 | else | ||
| 585 | "$MUXD" dump --sock "$1" | ||
| 586 | fi | ||
| 587 | } | ||
| 588 | |||
| 589 | # wait_grid SOCK NEEDLE LABEL [SESSION] — poll the daemon's own grid for | ||
| 590 | # NEEDLE (10s) and ASSERT it landed. The assert is the whole point. The bare | ||
| 591 | # form of this loop — poll, `&& break`, carry on — falls out silently when | ||
| 592 | # the marker never arrives, so a session that died at birth is discovered by | ||
| 593 | # whatever runs next, 15s later and in the wrong layer. Three M-web | ||
| 594 | # scenarios had grown exactly that shape. | ||
| 595 | # | ||
| 596 | # SESSION is optional and defaults to the default session: a marker typed | ||
| 597 | # into session `a` is not on the default session's grid, so a named-session | ||
| 598 | # scenario that forgot the argument would poll a grid the marker can never | ||
| 599 | # reach and fail 10s later blaming the wrong thing. | ||
| 600 | wait_grid() { | ||
| 601 | _i=0 | ||
| 602 | while [ "$_i" -lt $(( 100 * TIME_SCALE )) ]; do | ||
| 603 | dump_session "$1" "${4:-}" 2>/dev/null | grep -q "$2" && return 0 | ||
| 604 | sleep 0.1; _i=$((_i+1)) | ||
| 605 | done | ||
| 606 | echo "e2e FAIL: $3: '$2' never reached the daemon's grid; it holds:" | ||
| 607 | dump_session "$1" "${4:-}" || echo "(nothing answers on $1)" | ||
| 608 | exit 1 | ||
| 609 | } | ||
| 610 | |||
| 611 | # wait_sessions SOCK N LABEL — poll until `muxd stats` reports N live | ||
| 612 | # sessions (10s) and ASSERT it. wait_grid's shape for wait_grid's reason: a | ||
| 613 | # session is freed asynchronously — its shell exits, the daemon reaps on a | ||
| 614 | # later pump — so "it died" has to be waited FOR, and a wait that falls out | ||
| 615 | # silently turns "the session never died" into a puzzling failure two | ||
| 616 | # assertions further down. | ||
| 617 | wait_sessions() { | ||
| 618 | _i=0 | ||
| 619 | while [ "$_i" -lt $(( 100 * TIME_SCALE )) ]; do | ||
| 620 | "$MUXD" stats --sock "$1" 2>/dev/null | grep -q "sessions=$2" && return 0 | ||
| 621 | sleep 0.1; _i=$((_i+1)) | ||
| 622 | done | ||
| 623 | echo "e2e FAIL: $3: stats never reported sessions=$2; it holds:" | ||
| 624 | "$MUXD" stats --sock "$1" || echo "(nothing answers on $1)" | ||
| 625 | exit 1 | ||
| 626 | } | ||
| 627 | |||
| 628 | # --- tiles attach once each: did the daemon see a second? ------------- | ||
| 629 | # | ||
| 630 | # The focus model's whole claim is that a tile's connection is its own — | ||
| 631 | # no new attach, no dial on a focus move. Proving a negative needs a | ||
| 632 | # daemon-side witness, and there are two here doing different jobs. | ||
| 633 | # | ||
| 634 | # THE ASSERTION is `attaches=`, a cumulative counter of every attach this | ||
| 635 | # daemon accepted (server.zig Stats). Read once before the wall starts and | ||
| 636 | # once after it exits, the delta is exactly how many times anything attached | ||
| 637 | # across the whole leg — which for a wall of N tiles must be N, one per tile | ||
| 638 | # at startup, and never N+1 however far the focus moved. It is read from | ||
| 639 | # `muxd stats`, which is plain human text and connects as an OBSERVER: the | ||
| 640 | # reading itself never attaches, so it cannot pollute what it measures. | ||
| 641 | # | ||
| 642 | # THE GAUGE is `session NAME clients=`, sampled every 200ms into a file and | ||
| 643 | # asserted on its peak. It is kept because it localises a failure — it says | ||
| 644 | # WHICH session grew a second watcher and roughly when — but it is the | ||
| 645 | # weaker of the two and must never be the only one: a connection that closes | ||
| 646 | # as another opens never exceeds 1, and one that lives less than a sample | ||
| 647 | # interval is invisible to it. The counter cannot miss either. | ||
| 648 | # | ||
| 649 | # Nothing else may attach while a leg is being measured (`muxa capture` and | ||
| 650 | # `wait_grid` are clients too), so both readings bracket the ptyclient leg | ||
| 651 | # and nothing more. | ||
| 652 | # | ||
| 653 | # attaches_now SOCK — the daemon's cumulative accepted-attach count. | ||
| 654 | attaches_now() { | ||
| 655 | timeout 5 "$MUXD" stats --sock "$1" 2>/dev/null | | ||
| 656 | sed -n 's/.*[^_]attaches=\([0-9]*\).*/\1/p' | ||
| 657 | } | ||
| 658 | |||
| 659 | # assert_attach_delta BEFORE AFTER WANT LABEL — how many attaches happened. | ||
| 660 | # Empty readings fail loudly rather than arithmetically: `$(())` on an empty | ||
| 661 | # string is 0, and 0-0=0 would pass this check having measured nothing at | ||
| 662 | # all — the vacuous green this whole block exists to refuse. | ||
| 663 | assert_attach_delta() { | ||
| 664 | [ -n "$1" ] && [ -n "$2" ] || { | ||
| 665 | echo "e2e FAIL: $4: stats gave no attaches= reading (before='$1' after='$2')" | ||
| 666 | echo " — the flat-attach claim would be vacuous" | ||
| 667 | exit 1; } | ||
| 668 | [ "$(($2 - $1))" = "$3" ] || { | ||
| 669 | echo "e2e FAIL: $4: the daemon accepted $(($2 - $1)) attaches (want $3)" | ||
| 670 | echo " — the wall's tiles attach once each and a focus move never does" | ||
| 671 | exit 1; } | ||
| 672 | } | ||
| 673 | |||
| 674 | # watch_clients SOCK FILE — start sampling in the background. | ||
| 675 | WATCH_PID="" | ||
| 676 | watch_clients() { | ||
| 677 | touch "$2.on" | ||
| 678 | ( while [ -e "$2.on" ]; do | ||
| 679 | timeout 5 "$MUXD" stats --sock "$1" 2>/dev/null | ||
| 680 | sleep 0.2 | ||
| 681 | done ) > "$2" & | ||
| 682 | WATCH_PID=$! | ||
| 683 | defer_kill "$WATCH_PID" | ||
| 684 | } | ||
| 685 | |||
| 686 | # unwatch_clients FILE — stop sampling. | ||
| 687 | unwatch_clients() { | ||
| 688 | rm -f "$1.on" | ||
| 689 | wait "$WATCH_PID" 2>/dev/null || true | ||
| 690 | WATCH_PID="" | ||
| 691 | } | ||
| 692 | |||
| 693 | # assert_never_two_clients FILE NAME LABEL — the peak `clients=` this | ||
| 694 | # session ever showed is 1. Also asserts it was ever 1, which is the anchor: | ||
| 695 | # a watcher that sampled an empty file, or a wall whose tiles never attached, | ||
| 696 | # would otherwise pass this leg by having witnessed nothing at all. | ||
| 697 | # | ||
| 698 | # Localisation, not proof — `assert_attach_delta` is the proof. This says | ||
| 699 | # which session grew a second watcher; the counter says whether anything | ||
| 700 | # attached at all. | ||
| 701 | assert_never_two_clients() { | ||
| 702 | _peak=$(grep -o "session $2 clients=[0-9]*" "$1" | sed 's/.*=//' | sort -n | tail -1) | ||
| 703 | [ -n "$_peak" ] || { | ||
| 704 | echo "e2e FAIL: $3: no stats sample ever named session $2 — the watch" | ||
| 705 | echo " saw nothing, so its flat-count claim is vacuous" | ||
| 706 | exit 1; } | ||
| 707 | [ "$_peak" = "1" ] || { | ||
| 708 | echo "e2e FAIL: $3: session $2 was watched by $_peak clients at once" | ||
| 709 | echo " (want 1) — a focus move attached instead of being local" | ||
| 710 | exit 1; } | ||
| 711 | } | ||
| 712 | |||
| 713 | # wait_pid_gone PID LABEL — poll until a tracked pid is gone (2s, the same | ||
| 714 | # budget `muxd stop` gives itself). Its own helper rather than wait_gone's | ||
| 715 | # socket probe, because the two answer different questions: `muxd: stopped` | ||
| 716 | # is printed on the first probe that gets a REFUSAL, which is the socket | ||
| 717 | # being unlinked, and the process can still be a fraction behind that. Only | ||
| 718 | # a pid can say the daemon itself ended, which is the assertion the M13 | ||
| 719 | # teardowns owe — never the stop command's own claim. | ||
| 720 | # | ||
| 721 | # M14 gave it a second kind of caller: the cold-handoff scenario asks the | ||
| 722 | # same question of the ssh the client is supposed to have killed once QUIC | ||
| 723 | # took over. So the message names the pid and leaves the EXPECTATION to | ||
| 724 | # each call site's label — the two are "stop said it stopped" and "QUIC | ||
| 725 | # said it had taken over", and one wording cannot honestly claim both. | ||
| 726 | wait_pid_gone() { | ||
| 727 | _i=0 | ||
| 728 | while kill -0 "$1" 2>/dev/null; do | ||
| 729 | _i=$((_i + 1)); [ "$_i" -lt 40 ] || { | ||
| 730 | echo "e2e FAIL: $2: pid $1 is still running 2s later"; exit 1; } | ||
| 731 | sleep 0.05 | ||
| 732 | done | ||
| 733 | } | ||
| 734 | |||
| 735 | # assert_stopped SOCK PID LABEL ERRFILE — teardown by the sanctioned verb, | ||
| 736 | # asserted four ways. An exit code says the command RETURNED, not that it | ||
| 737 | # did the job: `muxd: stopped` is the daemon's own account of it, the absent | ||
| 738 | # socket is the filesystem's, and only the pid can say the process itself | ||
| 739 | # ended (see wait_pid_gone). A stop that exited 0 while leaving any of the | ||
| 740 | # three behind is precisely the regression this shape exists to catch. | ||
| 741 | # ERRFILE holds the stop's stderr — captured rather than let through, | ||
| 742 | # because a stop that fails exits 1 and would abort under `set -e` with no | ||
| 743 | # line of its own, after which the trap deletes the evidence. | ||
| 744 | # | ||
| 745 | # The caller nulls its own pid variable afterwards, the way every scenario | ||
| 746 | # here does: the trap reads those variables, and a pid it still holds is | ||
| 747 | # how a failing run says which daemon it leaked. | ||
| 748 | assert_stopped() { | ||
| 749 | set +e | ||
| 750 | "$MUXD" stop --sock "$1" 2> "$4" | ||
| 751 | _rc=$? | ||
| 752 | set -e | ||
| 753 | [ "$_rc" = "0" ] || { | ||
| 754 | echo "e2e FAIL: $3: stop exited $_rc, want 0"; cat "$4"; exit 1; } | ||
| 755 | grep -q '^muxd: stopped' "$4" || { | ||
| 756 | echo "e2e FAIL: $3: stop did not report stopped"; cat "$4"; exit 1; } | ||
| 757 | [ ! -S "$1" ] || { | ||
| 758 | echo "e2e FAIL: $3: stop left the socket"; ls -l "$1"; exit 1; } | ||
| 759 | wait_pid_gone "$2" "$3: stop reported stopped" | ||
| 760 | } | ||
| 761 | |||
| 762 | # The transport child for a given socket: a process named muxd running the | ||
| 763 | # `proxy` subcommand on that path. NEVER `pkill -f proxy` — the client's own | ||
| 764 | # argv contains the --via command string, so a pattern kill takes out the | ||
| 765 | # very client under test. | ||
| 766 | proxy_pid() { | ||
| 767 | ps -eo pid,comm,args | awk -v s="$1" '$2=="muxd" && /proxy/ && index($0,s) {print $1}' | head -1 | ||
| 768 | } | ||
| 769 | |||
| 770 | # --- M11: render-vs-dump convergence ----------------------------------- | ||
| 771 | # converged_quiet CLIENT_OUT SOCK [COLS ROWS] — render the captured client | ||
| 772 | # stream and diff it against the daemon's grid, plain and styled. Nonzero on | ||
| 773 | # divergence, leaving CLIENT_OUT.{render,dump,diff,rvt,dvt} behind for | ||
| 774 | # inspection. CLIENT_OUT must be PURE client stdout: a capture taken with | ||
| 775 | # 2>&1 has exit messages and predict stats mixed into the escape stream. | ||
| 776 | # The optional size is for pty scenarios whose grids are not the non-tty | ||
| 777 | # 80x24 default; render must replay into the same dimensions the daemon | ||
| 778 | # holds or the diff compares two honest grids of different shapes. | ||
| 779 | # It is an INPUT, not something this check validates, and the error is | ||
| 780 | # one-sided: too small re-wraps or clips a row and diverges loudly (the | ||
| 781 | # 80x24 default against tp2b's 100-wide final grid does exactly that), | ||
| 782 | # while too large only adds trailing blanks that the normalization strips, | ||
| 783 | # and passes. Pass the size the scenario actually ran at. | ||
| 784 | converged_quiet() { | ||
| 785 | _co="$1"; _cs="$2"; _sz="" | ||
| 786 | # An `if` rather than `[ ... ] && _sz=...` so it stays correct if it | ||
| 787 | # ever ends up the last command in this function: there, a false guard | ||
| 788 | # would be the function's exit status under `set -e` and a 2-argument | ||
| 789 | # call would report a divergence it never looked for. | ||
| 790 | if [ $# -ge 4 ]; then _sz="--cols $3 --rows $4"; fi | ||
| 791 | # $_sz is two flags or nothing, never data. | ||
| 792 | # shellcheck disable=SC2086 | ||
| 793 | "$RENDER" $_sz < "$_co" > "$_co.render" || return 1 | ||
| 794 | "$MUXD" dump --sock "$_cs" > "$_co.dump" || return 1 | ||
| 795 | # Trailing whitespace is a formatting difference between two correct | ||
| 796 | # grids (padded vs unpadded row ends), not a divergence. | ||
| 797 | sed 's/[[:space:]]*$//' "$_co.render" > "$_co.render.n" | ||
| 798 | sed 's/[[:space:]]*$//' "$_co.dump" > "$_co.dump.n" | ||
| 799 | diff -u "$_co.dump.n" "$_co.render.n" > "$_co.diff" || return 1 | ||
| 800 | # Styled, byte-for-byte: both sides come out of the same formatter, so | ||
| 801 | # equal grids are equal bytes. This is the half that sees a bled SGR | ||
| 802 | # or a leftover prediction underline — plain text dumps the same glyph | ||
| 803 | # either way, which is exactly why plain alone cannot carry M9's | ||
| 804 | # overlay-never-becomes-state invariant. | ||
| 805 | # Same as above: flags or nothing. | ||
| 806 | # shellcheck disable=SC2086 | ||
| 807 | "$RENDER" --vt $_sz < "$_co" > "$_co.rvt" || return 1 | ||
| 808 | "$MUXD" dump --vt --sock "$_cs" > "$_co.dvt" || return 1 | ||
| 809 | cmp -s "$_co.dvt" "$_co.rvt" || return 1 | ||
| 810 | rm -f "$_co.render" "$_co.dump" "$_co.render.n" "$_co.dump.n" \ | ||
| 811 | "$_co.diff" "$_co.rvt" "$_co.dvt" | ||
| 812 | return 0 | ||
| 813 | } | ||
| 814 | |||
| 815 | # assert_converged CLIENT_OUT SOCK NAME [COLS ROWS] — a scenario's LAST act, | ||
| 816 | # after quiesce and after the client detached: mid-scenario the dump is still | ||
| 817 | # moving, and another attach would claim the grid. | ||
| 818 | CONV_COUNT=0 | ||
| 819 | assert_converged() { | ||
| 820 | CONV_COUNT=$((CONV_COUNT + 1)) | ||
| 821 | if [ $# -ge 5 ]; then | ||
| 822 | converged_quiet "$1" "$2" "$4" "$5" | ||
| 823 | else | ||
| 824 | converged_quiet "$1" "$2" | ||
| 825 | fi || { | ||
| 826 | echo "e2e FAIL: $3: client render diverges from daemon grid (-daemon +client):" | ||
| 827 | # An empty diff file is the styled-only case, not a passing one — | ||
| 828 | # the glyphs agree and the pens do not, which is precisely what the | ||
| 829 | # byte-exact leg exists to catch, so say so instead of printing | ||
| 830 | # forty lines of nothing. | ||
| 831 | if [ -s "$1.diff" ]; then | ||
| 832 | head -40 "$1.diff" | ||
| 833 | else | ||
| 834 | echo "(no plain diff: the grids agree on glyphs and differ on STYLE —" | ||
| 835 | echo " compare $1.dvt against $1.rvt)" | ||
| 836 | fi | ||
| 837 | echo "grids left in $1.render / $1.dump / $1.rvt / $1.dvt" | ||
| 838 | exit 1 | ||
| 839 | } | ||
| 840 | } | ||
| 841 | |||
| 842 | # assert_ws_converged WSOUT SOCK LABEL [SESSION] — the WebSocket leg's | ||
| 843 | # convergence check. WSOUT is the stand-in's `dumpexit` file: its replica's | ||
| 844 | # grid, in `muxd dump`'s own format, built from frames that crossed the hub. | ||
| 845 | # Diff it against the daemon's grid, then doctor a copy of that grid and | ||
| 846 | # assert the SAME diff catches the doctored one — the wan.sh rule (M9): a | ||
| 847 | # convergence check that cannot fail proves nothing. | ||
| 848 | # | ||
| 849 | # SESSION (M18) names which grid the daemon side of that diff comes from. A | ||
| 850 | # tile attached to session `a` must be diffed against `a`, and pointing it | ||
| 851 | # at the default session instead would compare two unrelated grids — the | ||
| 852 | # check would fail, loudly and for the wrong reason. | ||
| 853 | # | ||
| 854 | # Deliberately NOT counted in CONV_COUNT. That pin counts assert_converged | ||
| 855 | # call sites — render-replays-the-client-stream — and this is different | ||
| 856 | # machinery answering a different question. Folding the two counts together | ||
| 857 | # would make either pin's number stop meaning anything. | ||
| 858 | assert_ws_converged() { | ||
| 859 | dump_session "$2" "${4:-}" > "$1.dump" | ||
| 860 | # Trailing whitespace is a formatting difference between two correct | ||
| 861 | # grids, exactly as in converged_quiet. | ||
| 862 | sed 's/[[:space:]]*$//' "$1" > "$1.n" | ||
| 863 | sed 's/[[:space:]]*$//' "$1.dump" > "$1.dump.n" | ||
| 864 | diff -u "$1.dump.n" "$1.n" > "$1.diff" || { | ||
| 865 | echo "e2e FAIL: $3: wsclient replica diverges from daemon grid (-daemon +ws):" | ||
| 866 | head -40 "$1.diff"; exit 1; } | ||
| 867 | cp "$1.dump.n" "$1.doc" | ||
| 868 | printf 'doctored-row\n' >> "$1.doc" | ||
| 869 | if diff -u "$1.doc" "$1.n" > /dev/null 2>&1; then | ||
| 870 | echo "e2e FAIL: $3: the convergence diff cannot fail (doctored dump passed)"; exit 1 | ||
| 871 | fi | ||
| 872 | } | ||
| 873 | |||
| 874 | # Scenario checkpoints. The suite's final line asserts the COUNT of these | ||
| 875 | # against a literal: adding or removing a scenario means updating that | ||
| 876 | # literal, and the friction is the feature — a scenario that silently | ||
| 877 | # stops running is the failure mode the pin exists for. | ||
| 878 | OK_COUNT=0 | ||
| 879 | ok() { | ||
| 880 | OK_COUNT=$((OK_COUNT + 1)) | ||
| 881 | echo "e2e OK: $1" | ||
| 882 | # Scenario boundaries, stamped for whoever needs to attribute something | ||
| 883 | # to the scenario that produced it. Nothing in this suite reads the file: | ||
| 884 | # test/coverage.sh maps each kcov database to the scenario that was | ||
| 885 | # running when the traced process wrote it, and the timestamps are also | ||
| 886 | # the only per-scenario timing this suite has ever been able to report. | ||
| 887 | # | ||
| 888 | # An `if` rather than `[ -n ... ] && printf`, for wait_sock's reason: a | ||
| 889 | # false guard as the last command in a function becomes that function's | ||
| 890 | # exit status, and under `set -e` every unstamped run would abort at its | ||
| 891 | # first passing scenario. | ||
| 892 | if [ -n "${E2E_OK_LOG:-}" ]; then | ||
| 893 | printf '%s\t%s\t%s\n' "$OK_COUNT" "$(date +%s.%N)" "$1" >> "$E2E_OK_LOG" | ||
| 894 | fi | ||
| 895 | # Prefix slicing. This suite is linear and stateful — the M13 scenarios | ||
| 896 | # run inside sessions the M10 scenarios created — so a prefix is the only | ||
| 897 | # slice that means anything, and a "run just scenario 40" filter would be | ||
| 898 | # a filter that lies. Exiting rather than skipping keeps that honest, and | ||
| 899 | # exiting 0 goes out through the trap, so a sliced run still tears down | ||
| 900 | # its daemons and still reports its leak verdict. | ||
| 901 | if [ -n "${E2E_STOP_AFTER:-}" ] && [ "$OK_COUNT" -ge "$E2E_STOP_AFTER" ]; then | ||
| 902 | echo "e2e STOP: sliced after $OK_COUNT scenarios (E2E_STOP_AFTER)" | ||
| 903 | exit 0 | ||
| 904 | fi | ||
| 905 | } | ||
| 906 | |||
| 907 | # ---- the leak sweep's two halves (hygiene kit, 6a) --------------------- | ||
| 908 | # Every capture this suite writes — daemon stderr AND client output — is a | ||
| 909 | # lifecycle log: a binary that leaked printed a `LEAK:` marker into one. The | ||
| 910 | # verdict is read in the EXIT trap, which is the only place that sees a | ||
| 911 | # FAILING run too; under `set -e` the bottom of this file is reached by | ||
| 912 | # passing runs alone, and a leak introduced alongside a defect is exactly | ||
| 913 | # the pair a suite should report together. | ||
| 914 | # | ||
| 915 | # That leaves the captures the scenarios delete as they go. A file removed | ||
| 916 | # at line 900 is not there for a trap at line 3000 to read, so removal is | ||
| 917 | # where its verdict has to be observed: rm_swept banks the marker first and | ||
| 918 | # deletes second, and the bank is what the trap reads for everything that no | ||
| 919 | # longer exists. Mid-suite removal of a capture goes through here — a plain | ||
| 920 | # `rm -f` of one is a verdict thrown away. | ||
| 921 | LEAKBANK="$OUT.leakbank" | ||
| 922 | |||
| 923 | # leak_line FILE — the verdict FILE holds, named and made readable. Never the | ||
| 924 | # matched line verbatim: half these captures are escape streams, whose one | ||
| 925 | # "line" is the entire session replay, and printing that repaints the reader's | ||
| 926 | # terminal instead of reporting to it. A log line short enough to BE a log | ||
| 927 | # line is kept whole (it names the binary, which is worth having); anything | ||
| 928 | # longer is cut back to the verdict. `grep -a` because the same captures make | ||
| 929 | # grep answer "Binary file ... matches", which does not even contain the | ||
| 930 | # marker a reader — or the sweep's own grep over the bank — is looking for. | ||
| 931 | leak_line() { | ||
| 932 | _ll=$(grep -a "LEAK:" "$1" 2>/dev/null | head -1 | tr -d '\000-\011\013-\037') | ||
| 933 | if [ "${#_ll}" -gt 120 ]; then | ||
| 934 | _ll=$(printf '%s\n' "$_ll" | sed 's/.*\(LEAK:\)/\1/' | cut -c1-120) | ||
| 935 | fi | ||
| 936 | case "$_ll" in | ||
| 937 | *LEAK:*) ;; | ||
| 938 | *) _ll="LEAK: marker present, unreadable as text" ;; | ||
| 939 | esac | ||
| 940 | echo "$1: $_ll" | ||
| 941 | } | ||
| 942 | |||
| 943 | rm_swept() { | ||
| 944 | for _rs in "$@"; do | ||
| 945 | # Regular files only: this list carries sockets and generated | ||
| 946 | # scripts too, and a socket path is not something to open. | ||
| 947 | [ -f "$_rs" ] || continue | ||
| 948 | if grep -q "LEAK:" "$_rs" 2>/dev/null; then | ||
| 949 | leak_line "$_rs" >> "$LEAKBANK" | ||
| 950 | fi | ||
| 951 | done | ||
| 952 | rm -f "$@" | ||
| 953 | } | ||
| 954 | |||
| 955 | # leak_sweep RC — the whole-suite verdict. RC is the suite's own exit status | ||
| 956 | # and decides one thing: what an empty sweep means. | ||
| 957 | # | ||
| 958 | # On a green run, nothing to read is a failure — the vacuous-green guard this | ||
| 959 | # gate has always had. $OUT.d1.d is created when daemon one is spawned and | ||
| 960 | # lives until this trap, and the bank exists once a verdict has been banked | ||
| 961 | # out of a deleted capture, so neither being there means the capture | ||
| 962 | # convention broke and this gate is reading nothing. On a run that is ALREADY | ||
| 963 | # failing, the same emptiness means something else entirely: an early exit, | ||
| 964 | # before the first daemon ever existed. That is not a second defect, and | ||
| 965 | # reporting it as one would stack a fabricated failure on top of the real | ||
| 966 | # one, so it says what it saw and returns clean. | ||
| 967 | # | ||
| 968 | # It never speaks for the suite either way: the caller promotes this verdict | ||
| 969 | # only over a green run (see cleanup). | ||
| 970 | leak_sweep() { | ||
| 971 | _lsrc="$1" | ||
| 972 | _lsbad=0 | ||
| 973 | if [ ! -e "$OUT.d1.d" ] && [ ! -e "$LEAKBANK" ]; then | ||
| 974 | if [ "$_lsrc" -eq 0 ]; then | ||
| 975 | echo "e2e FAIL: leak sweep found no captures to read" | ||
| 976 | return 1 | ||
| 977 | fi | ||
| 978 | echo "e2e note: leak sweep had nothing to read — the run exited $_lsrc" \ | ||
| 979 | "before the first daemon" | ||
| 980 | return 0 | ||
| 981 | fi | ||
| 982 | # The captures still on disk, one report line each. `"$OUT"` is named | ||
| 983 | # alongside the glob because the first client capture is written to the | ||
| 984 | # bare path, and `"$OUT".*` does not match it. | ||
| 985 | _lssaid="" | ||
| 986 | for _lsf in "$OUT" "$OUT".*; do | ||
| 987 | if [ ! -f "$_lsf" ]; then continue; fi | ||
| 988 | # The bank is not a capture; it is printed whole below, and running it | ||
| 989 | # through leak_line would report only its first entry. | ||
| 990 | if [ "$_lsf" = "$LEAKBANK" ]; then continue; fi | ||
| 991 | if grep -q "LEAK:" "$_lsf" 2>/dev/null; then | ||
| 992 | if [ -z "$_lssaid" ]; then | ||
| 993 | echo "e2e FAIL: a binary reported leaked allocations:" | ||
| 994 | _lssaid=1 | ||
| 995 | fi | ||
| 996 | leak_line "$_lsf" | ||
| 997 | _lsbad=1 | ||
| 998 | fi | ||
| 999 | done | ||
| 1000 | # ...and the verdicts banked out of captures the suite deleted as it went. | ||
| 1001 | if [ -s "$LEAKBANK" ]; then | ||
| 1002 | echo "e2e FAIL: a binary reported leaked allocations into a capture" \ | ||
| 1003 | "the suite has since deleted:" | ||
| 1004 | cat "$LEAKBANK" | ||
| 1005 | _lsbad=1 | ||
| 1006 | fi | ||
| 1007 | # The detached (`muxd start`) daemons log via XDG_STATE_HOME; the file is | ||
| 1008 | # truncated at every spawn, so this asserts the LAST such daemon only — | ||
| 1009 | # stated, not hidden. | ||
| 1010 | if [ -f "$XDG_STATE_HOME/mux/muxd.log" ] && | ||
| 1011 | grep -q "LEAK:" "$XDG_STATE_HOME/mux/muxd.log"; then | ||
| 1012 | echo "e2e FAIL: a detached daemon reported leaked allocations:" | ||
| 1013 | grep -H "LEAK:" "$XDG_STATE_HOME/mux/muxd.log" || true | ||
| 1014 | _lsbad=1 | ||
| 1015 | fi | ||
| 1016 | [ "$_lsbad" -eq 0 ] | ||
| 1017 | } | ||
| 1018 | |||
| 1019 | # A killed daemon writes its leak verdict on the way OUT, so a sweep that | ||
| 1020 | # reads before the process is gone reads a file the verdict has not reached. | ||
| 1021 | # Bounded, and shared across the pids rather than per-pid: a trap must never | ||
| 1022 | # be the thing that hangs, and a daemon that outlives the wait is swept for | ||
| 1023 | # whatever it did write. | ||
| 1024 | reap_briefly() { | ||
| 1025 | _rbi=0 | ||
| 1026 | while [ "$_rbi" -lt 40 ]; do | ||
| 1027 | _rblive="" | ||
| 1028 | for _rbp in "$@"; do | ||
| 1029 | [ -n "$_rbp" ] || continue | ||
| 1030 | if kill -0 "$_rbp" 2>/dev/null; then _rblive=1; fi | ||
| 1031 | done | ||
| 1032 | [ -n "$_rblive" ] || return 0 | ||
| 1033 | sleep 0.05 | ||
| 1034 | _rbi=$((_rbi + 1)) | ||
| 1035 | done | ||
| 1036 | return 0 | ||
| 1037 | } | ||
| 1038 | |||
| 1039 | cleanup() { | ||
| 1040 | # The suite's own status, captured before anything in here can overwrite | ||
| 1041 | # it. Every verdict below is composed onto THIS number, never in place of | ||
| 1042 | # it: a run that failed at scenario 6 exits with scenario 6's failure. | ||
| 1043 | _rc=$? | ||
| 1044 | # The registers are newline-joined and the paths in them may hold spaces | ||
| 1045 | # (a $TMPDIR nobody here chose); a newline IFS is what keeps each entry | ||
| 1046 | # one entry. | ||
| 1047 | _oifs=$IFS | ||
| 1048 | IFS=' | ||
| 1049 | ' | ||
| 1050 | # Sockets first, and by the daemon's own verb. `muxd stop` on a path | ||
| 1051 | # nobody serves is a no-op that exits 0, and on a daemon this shell never | ||
| 1052 | # forked — one a proxy, `muxd start` or the handoff spawned — it is the | ||
| 1053 | # only handle there is. Before the kills, and long before the unlink | ||
| 1054 | # below: unlinking first would leave a live daemon nothing could reach | ||
| 1055 | # by path. | ||
| 1056 | for _cs in $E2E_SOCK; do | ||
| 1057 | if [ -S "$_cs" ] && [ -n "${MUXD:-}" ]; then | ||
| 1058 | "$MUXD" stop --sock "$_cs" > /dev/null 2>&1 || true | ||
| 1059 | fi | ||
| 1060 | done | ||
| 1061 | # Then every registered process. softkill rather than a plain `kill`, | ||
| 1062 | # which is what this trap used to send: under the coverage harness the | ||
| 1063 | # pid the suite holds is a kcov TRACER, and a signal sent to a tracer is | ||
| 1064 | # dropped — measured on the hub leg, where `kill $W3PID` left both alive. | ||
| 1065 | # TERM rather than hardkill's KILL, because a daemon writes its leak | ||
| 1066 | # verdict on the way OUT and a SIGKILLed one writes nothing: the sweep | ||
| 1067 | # below would then be reading a lifecycle that never ended. | ||
| 1068 | # | ||
| 1069 | # CONT first, and unconditionally: two legs hold their subject under | ||
| 1070 | # SIGSTOP (the mute-offerer's agent, the refusal leg's daemon), and a | ||
| 1071 | # stopped process does not see TERM until it runs again. On anything not | ||
| 1072 | # stopped it is a no-op. | ||
| 1073 | # | ||
| 1074 | # `|| true` on both, for the reason every kill in this trap has always | ||
| 1075 | # carried one: under `set -e` a signal to an already-dead pid would abort | ||
| 1076 | # the trap itself and skip everything below it. | ||
| 1077 | for _ck in $E2E_KILL; do | ||
| 1078 | kill -CONT "$_ck" 2>/dev/null || true | ||
| 1079 | softkill "$_ck" || true | ||
| 1080 | done | ||
| 1081 | # ---- the leak sweep (hygiene kit, 6a) ---- | ||
| 1082 | # Here rather than at the bottom of the file, which `set -e` reaches only | ||
| 1083 | # on a passing run: a leak that arrives alongside a defect is reported | ||
| 1084 | # with it. Between the kills above and the rms below, which is the one | ||
| 1085 | # window where every daemon has finished its exit path and every capture | ||
| 1086 | # still exists. | ||
| 1087 | # | ||
| 1088 | # A killed daemon writes its verdict on the way out, so the sweep waits | ||
| 1089 | # for the processes to be gone first. | ||
| 1090 | # shellcheck disable=SC2086 # IFS is a newline: each entry is one word | ||
| 1091 | reap_briefly $E2E_KILL | ||
| 1092 | _leak=0 | ||
| 1093 | leak_sweep "$_rc" || _leak=1 | ||
| 1094 | |||
| 1095 | # The registered artifacts: sockets, keys, generated scripts, private | ||
| 1096 | # HOMEs, state homes. Through rm_swept, so a registered FILE still banks | ||
| 1097 | # its leak verdict on the way out exactly as a mid-suite removal does; | ||
| 1098 | # directories go whole. | ||
| 1099 | for _cr in $E2E_SOCK $E2E_RM; do | ||
| 1100 | if [ -d "$_cr" ]; then | ||
| 1101 | rm -rf "$_cr" | ||
| 1102 | else | ||
| 1103 | rm_swept "$_cr" | ||
| 1104 | fi | ||
| 1105 | done | ||
| 1106 | IFS=$_oifs | ||
| 1107 | |||
| 1108 | # ...and the captures, by the pattern leak_sweep just read them by rather | ||
| 1109 | # than by a list of some 350 names. The list is what failed: every | ||
| 1110 | # capture had to be remembered in two places, 102 of them were not, and | ||
| 1111 | # by 2026-08-19 `make soak` refused to start. A pattern cannot forget a | ||
| 1112 | # leg. | ||
| 1113 | # | ||
| 1114 | # Green runs only. A failing run keeps its captures — the .render/.dump/ | ||
| 1115 | # .diff files a failed assert_converged leaves behind are that failure's | ||
| 1116 | # evidence, and soak moves the whole set into its failure dir for the | ||
| 1117 | # next run's sake. | ||
| 1118 | if [ "$_rc" -eq 0 ]; then | ||
| 1119 | for _co in "$OUT" "$OUT".*; do | ||
| 1120 | [ -e "$_co" ] || continue | ||
| 1121 | rm -rf "$_co" | ||
| 1122 | done | ||
| 1123 | fi | ||
| 1124 | # Residue, in the leak sweep's shape and for its reason: a run that PASSED | ||
| 1125 | # must leave nothing behind, and this trap is the last place that can | ||
| 1126 | # still say so. Pinned to this run's pid so a suite running concurrently | ||
| 1127 | # is never counted, and never deleted. | ||
| 1128 | # | ||
| 1129 | # What it catches now is a path that is neither registered nor spelled | ||
| 1130 | # from $OUT — the one shape both mechanisms above are blind to. | ||
| 1131 | _stray=0 | ||
| 1132 | if [ "$_rc" -eq 0 ]; then | ||
| 1133 | _left=$(find "${TMPDIR:-/tmp}" -maxdepth 1 \ | ||
| 1134 | \( -name "mux*-$$" -o -name "mux*-$$.*" \) 2>/dev/null) | ||
| 1135 | if [ -n "$_left" ]; then | ||
| 1136 | _n=$(printf '%s\n' "$_left" | wc -l) | ||
| 1137 | if [ "$_n" -eq 1 ]; then _w=file; else _w=files; fi | ||
| 1138 | echo "e2e FAIL: the suite passed but left $_n $_w in ${TMPDIR:-/tmp}:" | ||
| 1139 | printf '%s\n' "$_left" | sed 's/^/ /' | ||
| 1140 | echo " A capture is spelled from \$OUT and swept by pattern;" | ||
| 1141 | echo " anything else is registered where it is created, with" | ||
| 1142 | echo " defer_rm, defer_sock or start_daemon. A path that is" | ||
| 1143 | echo " neither is invisible until soak refuses to start." | ||
| 1144 | printf '%s\n' "$_left" | xargs -r rm -rf | ||
| 1145 | _stray=1 | ||
| 1146 | fi | ||
| 1147 | fi | ||
| 1148 | # This trap can change the suite's answer in exactly one direction: a run | ||
| 1149 | # that was green and swept up a leak. Every other path returns normally | ||
| 1150 | # and leaves the status alone — a suite that failed at scenario 6 must | ||
| 1151 | # exit with scenario 6's failure, not with the trap's opinion of it. | ||
| 1152 | if [ "$_rc" -eq 0 ] && { [ "$_leak" -eq 1 ] || [ "$_stray" -eq 1 ]; }; then | ||
| 1153 | exit 1 | ||
| 1154 | fi | ||
| 1155 | } | ||
| 1156 | trap cleanup EXIT INT TERM | ||
| 1157 | 102 | ||
| 1158 | # Second daemon, used only by the M7 abort scenario. | 103 | # Second daemon, used only by the M7 abort scenario. |
| 1159 | SOCK2="${TMPDIR:-/tmp}/muxd-e2e-abort-$$.sock" | 104 | SOCK2="${TMPDIR:-/tmp}/muxd-e2e-abort-$$.sock" |
test/e2e_lib.sh
| Old | New | ||
|---|---|---|---|
| @@ -0,0 +1,1074 @@ | |||
| 1 | # shellcheck shell=sh | ||
| 2 | # e2e_lib.sh — sourced by test/e2e.sh, before any group file. | ||
| 3 | # | ||
| 4 | # What a group file may assume this has already done: the XDG homes, $SHELL | ||
| 5 | # and $SSH_AUTH_SOCK are hermetic; $SOCK and $OUT are spelled; the cleanup | ||
| 6 | # registry exists and the EXIT trap is armed; every helper below is defined. | ||
| 7 | # What a group file must do in return: register what it creates. A daemon | ||
| 8 | # through start_daemon, anything else through defer_kill, defer_sock or | ||
| 9 | # defer_rm. Captures need no registration — spell them from $OUT and the | ||
| 10 | # trap sweeps them by pattern. | ||
| 11 | # | ||
| 12 | # What is NOT here: the scenario count. ok() counts, and the pin that | ||
| 13 | # gates the count is the runner's, at the bottom of test/e2e.sh — a group | ||
| 14 | # file knows nothing about how many scenarios the suite has. | ||
| 15 | # ---- the cleanup registry --------------------------------------------- | ||
| 16 | # Every process and every artifact a leg creates is REGISTERED where it is | ||
| 17 | # created, and the EXIT trap walks the registers. Before this, cleanup() | ||
| 18 | # carried four hand-maintained lists naming ~137 $SOCKnn/$DnnPID variables, | ||
| 19 | # and a leg added without all four edits leaked: 102 captures were missing | ||
| 20 | # from them by 2026-08-19, enough that `make soak` refused to start. A list | ||
| 21 | # kept somewhere other than the code it describes is a list that goes | ||
| 22 | # stale, so there is no list — registering IS the statement that creates | ||
| 23 | # the thing. | ||
| 24 | # | ||
| 25 | # Captures are the exception, and deliberately so: every one of them is | ||
| 26 | # spelled from $OUT, and the trap removes that whole set by the same | ||
| 27 | # pattern leak_sweep already reads it by. A pattern cannot forget a leg, | ||
| 28 | # and forgetting was the failure. What has to be registered is what no | ||
| 29 | # pattern can find — pids, and the artifacts spelled somewhere other than | ||
| 30 | # $OUT. | ||
| 31 | # | ||
| 32 | # Newline-joined strings rather than arrays: the shebang is /bin/sh and the | ||
| 33 | # build's shell gate lints this file as POSIX sh. No path here can hold a | ||
| 34 | # newline (every one is spelled from $TMPDIR, $OUT and $$ in this file), | ||
| 35 | # and cleanup sets IFS to a newline before walking them, so a $TMPDIR with | ||
| 36 | # a space in it still reads as one path. | ||
| 37 | E2E_KILL="" | ||
| 38 | E2E_SOCK="" | ||
| 39 | E2E_RM="" | ||
| 40 | |||
| 41 | # defer_kill PID... — a process the trap must end. Newest first, so | ||
| 42 | # teardown runs in the reverse of creation order and a client is signalled | ||
| 43 | # before the daemon it is attached to. | ||
| 44 | # | ||
| 45 | # An `if` rather than `[ -n "$_dk" ] && ...`, for wait_sock's reason: a | ||
| 46 | # false guard as the last command in a function becomes that function's | ||
| 47 | # exit status, and under `set -e` registering an empty pid would abort the | ||
| 48 | # caller. | ||
| 49 | defer_kill() { | ||
| 50 | for _dk in "$@"; do | ||
| 51 | if [ -n "$_dk" ]; then | ||
| 52 | # Already registered is not an error: pipe_mux names the same | ||
| 53 | # FIFO on every call, and a leg that starts two daemons on one | ||
| 54 | # path registers it twice. Registering twice must cost nothing, | ||
| 55 | # or callers start reasoning about who registered first. | ||
| 56 | case " | ||
| 57 | $E2E_KILL" in | ||
| 58 | *" | ||
| 59 | $_dk | ||
| 60 | "*) continue ;; | ||
| 61 | esac | ||
| 62 | E2E_KILL="$_dk | ||
| 63 | $E2E_KILL" | ||
| 64 | fi | ||
| 65 | done | ||
| 66 | } | ||
| 67 | |||
| 68 | # defer_sock PATH... — a path a daemon may be listening on. The trap asks | ||
| 69 | # `muxd stop` there before it kills anything and long before it unlinks the | ||
| 70 | # path: a daemon this shell never forked — one a proxy, `muxd start` or the | ||
| 71 | # handoff spawned — has no pid here, and the socket is the only handle | ||
| 72 | # there is. | ||
| 73 | defer_sock() { | ||
| 74 | for _ds in "$@"; do | ||
| 75 | if [ -n "$_ds" ]; then | ||
| 76 | # Already registered is not an error: pipe_mux names the same | ||
| 77 | # FIFO on every call, and a leg that starts two daemons on one | ||
| 78 | # path registers it twice. Registering twice must cost nothing, | ||
| 79 | # or callers start reasoning about who registered first. | ||
| 80 | case " | ||
| 81 | $E2E_SOCK" in | ||
| 82 | *" | ||
| 83 | $_ds | ||
| 84 | "*) continue ;; | ||
| 85 | esac | ||
| 86 | E2E_SOCK="$_ds | ||
| 87 | $E2E_SOCK" | ||
| 88 | fi | ||
| 89 | done | ||
| 90 | } | ||
| 91 | |||
| 92 | # defer_rm PATH... — an artifact the trap must remove: a key, a generated | ||
| 93 | # script, a private HOME, a state home. Registered at the line that spells | ||
| 94 | # the path, so the two can never drift apart. Files go out through the leak | ||
| 95 | # sweep's banking; directories are removed whole. | ||
| 96 | defer_rm() { | ||
| 97 | for _dr in "$@"; do | ||
| 98 | if [ -n "$_dr" ]; then | ||
| 99 | # Already registered is not an error: pipe_mux names the same | ||
| 100 | # FIFO on every call, and a leg that starts two daemons on one | ||
| 101 | # path registers it twice. Registering twice must cost nothing, | ||
| 102 | # or callers start reasoning about who registered first. | ||
| 103 | case " | ||
| 104 | $E2E_RM" in | ||
| 105 | *" | ||
| 106 | $_dr | ||
| 107 | "*) continue ;; | ||
| 108 | esac | ||
| 109 | E2E_RM="$_dr | ||
| 110 | $E2E_RM" | ||
| 111 | fi | ||
| 112 | done | ||
| 113 | } | ||
| 114 | |||
| 115 | |||
| 116 | SOCK="${TMPDIR:-/tmp}/muxd-e2e-$$.sock" | ||
| 117 | defer_sock "$SOCK" | ||
| 118 | OUT="${TMPDIR:-/tmp}/mux-e2e-out-$$" | ||
| 119 | # M10: hermetic XDG homes. Key-default scenarios must see OUR key or none, | ||
| 120 | # never the developer's real ~/.config/mux/key. | ||
| 121 | XDG_CONFIG_HOME="${TMPDIR:-/tmp}/mux-e2e-cfg-$$" | ||
| 122 | defer_rm "$XDG_CONFIG_HOME" | ||
| 123 | XDG_STATE_HOME="${TMPDIR:-/tmp}/mux-e2e-state-$$" | ||
| 124 | defer_rm "$XDG_STATE_HOME" | ||
| 125 | # M14: and the same for the handoff's per-host cache. It holds a KEY, and | ||
| 126 | # the M14 scenarios both read and poison it — neither of which may ever | ||
| 127 | # touch the developer's real ~/.cache/mux. | ||
| 128 | XDG_CACHE_HOME="${TMPDIR:-/tmp}/mux-e2e-cache-$$" | ||
| 129 | defer_rm "$XDG_CACHE_HOME" | ||
| 130 | export XDG_CONFIG_HOME XDG_STATE_HOME XDG_CACHE_HOME | ||
| 131 | # ...and the same argument for $SHELL, which is not an XDG home but is read | ||
| 132 | # the same way: every daemon this suite AUTO-STARTS gets no --shell flag and | ||
| 133 | # resolves $SHELL, so without this the suite runs the developer's login | ||
| 134 | # shell and its whole rc — arbitrary code, on the session under test. | ||
| 135 | # | ||
| 136 | # Found by soak, not by reasoning. The M10 `muxd start` block inherited zsh, | ||
| 137 | # whose plugin manager roots itself at $XDG_CACHE_HOME; pointing that at a | ||
| 138 | # fresh directory (the line above) made every session re-clone its plugins | ||
| 139 | # from the network before the shell would answer, and the scenario's marker | ||
| 140 | # missed its window. The daemon was healthy the whole time. The M13 blocks | ||
| 141 | # already pin SHELL per command for this reason; hoisting it here covers the | ||
| 142 | # M10 block too, and any scenario added later that forgets. | ||
| 143 | SHELL=/bin/sh | ||
| 144 | export SHELL | ||
| 145 | # MUX_KEY_FILE belongs to the same family and is cleared here for the same | ||
| 146 | # forgets-proofing reason: `muxd endpoint` reads it BEFORE the default key | ||
| 147 | # path, so an operator who happens to have one exported would silently | ||
| 148 | # change which key the handoff announces — and the scenarios would still | ||
| 149 | # pass, against the wrong key, until one of them did not. | ||
| 150 | unset MUX_KEY_FILE | ||
| 151 | # ...and SSH_AUTH_SOCK, the same family again and the sharpest case in it. | ||
| 152 | # A daemon that inherited the developer's real agent would hand it to every | ||
| 153 | # session whose own agent socket failed to bind — and the refusal leg, whose | ||
| 154 | # whole subject is a shell finding nobody to sign for it, would then be | ||
| 155 | # asserting against the developer's keyring. The legs that want an agent | ||
| 156 | # export one per command, at a path this file made. | ||
| 157 | unset SSH_AUTH_SOCK | ||
| 158 | |||
| 159 | # One counter out of a MUX_PREDICT_STATS line. The client prints exactly one | ||
| 160 | # such line on exit; every field is a key=value pair, so a rename or reorder | ||
| 161 | # in the client shows up here as an empty read rather than a wrong number. | ||
| 162 | predict_stat() { | ||
| 163 | sed -n "s/.*predict .*$2=\([0-9]*\).*/\1/p" "$1" | head -1 | ||
| 164 | } | ||
| 165 | |||
| 166 | # Assert one counter, with the whole line in the failure so a wrong number is | ||
| 167 | # read in context rather than alone. | ||
| 168 | want_stat() { | ||
| 169 | _got=$(predict_stat "$1" "$2") | ||
| 170 | [ -n "$_got" ] || { | ||
| 171 | echo "e2e FAIL: $4: no predict stats line (wanted $2=$3); got:" | ||
| 172 | cat "$1"; exit 1; | ||
| 173 | } | ||
| 174 | [ "$_got" = "$3" ] || { | ||
| 175 | echo "e2e FAIL: $4: $2=$_got, want $3" | ||
| 176 | grep "^predict " "$1" || true | ||
| 177 | exit 1; | ||
| 178 | } | ||
| 179 | } | ||
| 180 | |||
| 181 | # The same, as a floor. Its own helper rather than a mode on want_stat: | ||
| 182 | # exact equality is the right assertion for every correctness counter, and | ||
| 183 | # a shared one would make it easy to weaken those by habit. Only counters | ||
| 184 | # that are genuinely timing-dependent belong here. | ||
| 185 | want_stat_ge() { | ||
| 186 | _got=$(predict_stat "$1" "$2") | ||
| 187 | [ -n "$_got" ] || { | ||
| 188 | echo "e2e FAIL: $4: no predict stats line (wanted $2>=$3); got:" | ||
| 189 | cat "$1"; exit 1; | ||
| 190 | } | ||
| 191 | [ "$_got" -ge "$3" ] || { | ||
| 192 | echo "e2e FAIL: $4: $2=$_got, want >=$3" | ||
| 193 | grep "^predict " "$1" || true | ||
| 194 | exit 1; | ||
| 195 | } | ||
| 196 | } | ||
| 197 | |||
| 198 | # Wait until PATTERN shows up in FILE (default 15s). Timing that keys off the | ||
| 199 | # session's own output instead of a fixed sleep: the marker is proof the | ||
| 200 | # client is attached and idle, which is exactly the state the tear needs. | ||
| 201 | # E2E_TIME_SCALE multiplies the polling BUDGETS below, and nothing else. It | ||
| 202 | # exists for test/coverage.sh: a ptrace-traced binary runs about half again | ||
| 203 | # slower, which is enough to blow a 25s convergence budget that is generous | ||
| 204 | # at native speed — and the failure reads as "the attach never converged" | ||
| 205 | # rather than "the tracer is slow", which is a lie about the product. | ||
| 206 | # | ||
| 207 | # Budgets only. Not the sleeps between polls, not a `timeout` that a scenario | ||
| 208 | # asserts on, and never a threshold: scaling this changes how long the suite | ||
| 209 | # is WILLING TO WAIT, never what it demands to see. A run at scale 4 that | ||
| 210 | # passes proves the same facts as a run at scale 1, just later. | ||
| 211 | TIME_SCALE="${E2E_TIME_SCALE:-1}" | ||
| 212 | case "$TIME_SCALE" in | ||
| 213 | ''|*[!0-9]*) echo "e2e FAIL: E2E_TIME_SCALE must be a positive integer"; exit 1 ;; | ||
| 214 | esac | ||
| 215 | [ "$TIME_SCALE" -ge 1 ] || { echo "e2e FAIL: E2E_TIME_SCALE must be >= 1"; exit 1; } | ||
| 216 | |||
| 217 | # Validated here rather than at its use in ok(), for the reason above: a bad | ||
| 218 | # value would otherwise surface as a bare test(1) error at the first passing | ||
| 219 | # scenario, with no e2e FAIL line to say what was wrong. | ||
| 220 | case "${E2E_STOP_AFTER:-1}" in | ||
| 221 | ''|*[!0-9]*) echo "e2e FAIL: E2E_STOP_AFTER must be a positive integer"; exit 1 ;; | ||
| 222 | esac | ||
| 223 | |||
| 224 | wait_for() { | ||
| 225 | _file="$1"; _pat="$2"; _ticks=$(( ${3:-15} * 10 * TIME_SCALE )); _i=0 | ||
| 226 | while [ "$_i" -lt "$_ticks" ]; do | ||
| 227 | if [ -f "$_file" ] && grep -q "$_pat" "$_file" 2>/dev/null; then return 0; fi | ||
| 228 | sleep 0.1; _i=$((_i+1)) | ||
| 229 | done | ||
| 230 | return 1 | ||
| 231 | } | ||
| 232 | |||
| 233 | # hardkill PID — SIGKILL a pid and anything it fathered, children first. | ||
| 234 | # | ||
| 235 | # Plain `kill -9 "$pid"` is right only while the pid the suite holds IS the | ||
| 236 | # process under test. Under the coverage harness (test/coverage.sh) it is a | ||
| 237 | # kcov wrapper instead, and SIGKILL is the one signal a wrapper cannot | ||
| 238 | # forward: killing the node alone would leave the real daemon alive holding | ||
| 239 | # its socket, and every leg that waits for that daemon to die would hang | ||
| 240 | # rather than fail. Children first so a traced child's death is observed by | ||
| 241 | # its tracer, which is when kcov writes the coverage it has collected. | ||
| 242 | # | ||
| 243 | # With no wrapper in the picture there are no children and this is exactly | ||
| 244 | # `kill -9`, which is why the abort legs keep the semantics they assert on: | ||
| 245 | # the daemon still dies by SIGKILL, still without unlinking its socket. | ||
| 246 | hardkill() { | ||
| 247 | # The test is what the process IS, not whether it has children. A daemon | ||
| 248 | # has children too — a session shell per attach — and killing those first | ||
| 249 | # ends the session CLEANLY, which is the one thing the abort legs must not | ||
| 250 | # see: they assert on a client whose daemon vanished under it, and a tidy | ||
| 251 | # session exit takes a different path out of the client (measured: exit | ||
| 252 | # 128 where the leg wants 0). Only a wrapper gets the two-step treatment. | ||
| 253 | if [ "$(ps -o comm= -p "$1" 2>/dev/null)" = kcov ]; then | ||
| 254 | # The CHILD is the daemon the suite means to kill; the pid it holds is | ||
| 255 | # only the tracer. Kill the child first and the wrapper exits on its | ||
| 256 | # own, writing the coverage it has collected — measured: a SIGKILLed | ||
| 257 | # wrapper writes no coverage.db at any --output-interval, so a clean | ||
| 258 | # exit is the only exit that keeps the data. Then wait for it, because | ||
| 259 | # killing both at once is the same as never killing the child at all. | ||
| 260 | for _c in $(ps -o pid= --ppid "$1" 2>/dev/null); do | ||
| 261 | kill -9 "$_c" 2>/dev/null || true | ||
| 262 | done | ||
| 263 | _i=0 | ||
| 264 | while kill -0 "$1" 2>/dev/null && [ "$_i" -lt $(( 20 * TIME_SCALE )) ]; do | ||
| 265 | sleep 0.05; _i=$((_i + 1)) | ||
| 266 | done | ||
| 267 | fi | ||
| 268 | kill -9 "$1" 2>/dev/null || true | ||
| 269 | } | ||
| 270 | |||
| 271 | # hardkill's TERM twin. A signal sent to a kcov wrapper is dropped: ptrace | ||
| 272 | # intercepts signals bound for the TRACEE, and the tracer has no handler of | ||
| 273 | # its own — measured on the hub leg, where `kill $W3PID` left both alive and | ||
| 274 | # wait_pid_gone timed out. TERM the tracee; the wrapper exits with it and | ||
| 275 | # writes its database. Off the tracer this is a plain kill, same exit status. | ||
| 276 | # A /proc claim about the daemon is read off the tracee: under `make coverage` | ||
| 277 | # the pid the suite holds is the tracer's (hardkill's reason), and kcov's own | ||
| 278 | # cmdline is what a resumed-argv check would otherwise read. | ||
| 279 | real_pid() { | ||
| 280 | if [ "$(ps -o comm= -p "$1" 2>/dev/null)" = kcov ]; then | ||
| 281 | ps -o pid= --ppid "$1" 2>/dev/null | head -1 | tr -d ' ' | ||
| 282 | else | ||
| 283 | echo "$1" | ||
| 284 | fi | ||
| 285 | } | ||
| 286 | |||
| 287 | softkill() { | ||
| 288 | if [ "$(ps -o comm= -p "$1" 2>/dev/null)" = kcov ]; then | ||
| 289 | _rc=1 | ||
| 290 | for _c in $(ps -o pid= --ppid "$1" 2>/dev/null); do | ||
| 291 | kill "$_c" 2>/dev/null && _rc=0 | ||
| 292 | done | ||
| 293 | return $_rc | ||
| 294 | fi | ||
| 295 | kill "$1" 2>/dev/null | ||
| 296 | } | ||
| 297 | |||
| 298 | # Poll until nothing answers on a socket path (2s). Keyed off the daemon's | ||
| 299 | # own liveness rather than a fixed sleep, same reasoning as wait_for. | ||
| 300 | wait_gone() { | ||
| 301 | _i=0 | ||
| 302 | while "$MUXD" dump --sock "$1" > /dev/null 2>&1; do | ||
| 303 | _i=$((_i + 1)); [ "$_i" -lt $(( 40 * TIME_SCALE )) ] || { echo "e2e FAIL: daemon on $1 never died"; exit 1; } | ||
| 304 | sleep 0.05 | ||
| 305 | done | ||
| 306 | } | ||
| 307 | |||
| 308 | # wait_sock PATH LOG LABEL — poll until a daemon has bound PATH (5s), then | ||
| 309 | # ASSERT it. Every `muxd run &` in this file needs this: the bind happens | ||
| 310 | # after the fork, so the very next command would otherwise race it. LOG is | ||
| 311 | # the spawn's own capture, printed on failure because a daemon that failed | ||
| 312 | # to bind almost always said why; pass "" for the spawns that have none. | ||
| 313 | wait_sock() { | ||
| 314 | _i=0 | ||
| 315 | while [ ! -S "$1" ] && [ "$_i" -lt $(( 50 * TIME_SCALE )) ]; do sleep 0.1; _i=$((_i+1)); done | ||
| 316 | [ -S "$1" ] || { | ||
| 317 | echo "e2e FAIL: $3" | ||
| 318 | # An `if` rather than `[ -n "$2" ] && cat "$2"`, for converged_quiet's | ||
| 319 | # reason: a false guard must not become this block's exit status. | ||
| 320 | if [ -n "$2" ]; then cat "$2"; fi | ||
| 321 | exit 1 | ||
| 322 | } | ||
| 323 | } | ||
| 324 | |||
| 325 | # start_daemon SOCK LOG LABEL [muxd args...] — a daemon in the background, | ||
| 326 | # registered before it is waited for. Sets $DPID, the pid to end it by. | ||
| 327 | # $DPID is scratch: the NEXT start_daemon overwrites it, so a leg that | ||
| 328 | # still needs its daemon further down keeps the pid under a name of its | ||
| 329 | # own. The long-lived daemon is $D1PID for exactly that reason — it is | ||
| 330 | # read some 5,000 lines after it is started. | ||
| 331 | # | ||
| 332 | # The registration is the whole reason this exists. A `muxd run &` written | ||
| 333 | # out by hand is three statements — the spawn, the pid, the wait — plus a | ||
| 334 | # fourth in cleanup() saying how to end it, and the fourth is the one that | ||
| 335 | # got forgotten. Here the spawn IS the registration and there is no fourth | ||
| 336 | # place to edit. | ||
| 337 | # | ||
| 338 | # Only for the plain shape. A daemon that needs an environment in front of | ||
| 339 | # it writes its own spawn and calls defer_kill/defer_sock itself: `env` or | ||
| 340 | # a VAR=VAL prefix on a FUNCTION call does not scope to the function — in | ||
| 341 | # POSIX sh it assigns in THIS shell and the value stays there for every | ||
| 342 | # scenario after it. | ||
| 343 | start_daemon() { | ||
| 344 | _sds="$1"; _sdl="$2"; _sdlab="$3"; shift 3 | ||
| 345 | "$MUXD" run --sock "$_sds" "$@" > "$_sdl" 2>&1 & | ||
| 346 | DPID=$! | ||
| 347 | defer_kill "$DPID" | ||
| 348 | defer_sock "$_sds" | ||
| 349 | wait_sock "$_sds" "$_sdl" "$_sdlab" | ||
| 350 | } | ||
| 351 | |||
| 352 | |||
| 353 | # pipe_mux OUT ERR CMD... — a client on a plain pipe, and the pipe kept | ||
| 354 | # open. ERR is the stderr capture, "" to leave it on the suite's own. | ||
| 355 | # The suite's non-tty scenarios used to be `{ printf cmd; sleep 2; printf | ||
| 356 | # detach; } | "$MUX"`: two seconds bought for output that lands in | ||
| 357 | # milliseconds (measured: the needle is in the capture 2ms after the send), | ||
| 358 | # and 120s of the suite's runtime was exactly that purchase. Here the stdin | ||
| 359 | # is a FIFO held open on fd 9, so a scenario sends, then WAITS FOR THE NEEDLE | ||
| 360 | # with await_out, then detaches — the same assertion it was going to make | ||
| 361 | # anyway, moved to where it ends the wait. fd 9 is opened read-write so | ||
| 362 | # neither the open nor the client's EOF blocks on the other side; the client | ||
| 363 | # sees EOF only when pipe_detach closes it. | ||
| 364 | # | ||
| 365 | # Not for the scenarios that pace on purpose: the prediction legs snapshot | ||
| 366 | # the capture N ms after a keystroke, and a sleep there IS the test. | ||
| 367 | pipe_mux() { | ||
| 368 | PIPE_OUT="$1"; PIPE_ERR="$2"; shift 2 | ||
| 369 | PIPE_IN="$PIPE_OUT.in" | ||
| 370 | defer_rm "$PIPE_IN" | ||
| 371 | rm -f "$PIPE_IN" | ||
| 372 | mkfifo "$PIPE_IN" | ||
| 373 | exec 9<>"$PIPE_IN" | ||
| 374 | if [ -n "$PIPE_ERR" ]; then | ||
| 375 | "$@" < "$PIPE_IN" > "$PIPE_OUT" 2> "$PIPE_ERR" & | ||
| 376 | else | ||
| 377 | "$@" < "$PIPE_IN" > "$PIPE_OUT" & | ||
| 378 | fi | ||
| 379 | PIPE_PID=$! | ||
| 380 | defer_kill "$PIPE_PID" | ||
| 381 | } | ||
| 382 | |||
| 383 | # pipe_send FMT [ARGS] — printf into the open client, same spelling the | ||
| 384 | # inline blocks used so a conversion moves the format string verbatim. | ||
| 385 | pipe_send() { | ||
| 386 | # shellcheck disable=SC2059 | ||
| 387 | printf "$@" >&9 | ||
| 388 | } | ||
| 389 | |||
| 390 | # await_out FILE NEEDLE LABEL — wait_for with the verdict attached: FAIL | ||
| 391 | # with the capture when NEEDLE never comes (5s, scaled). The same `grep -q` | ||
| 392 | # the scenario asserts with afterwards, so waiting on it costs no new claim. | ||
| 393 | await_out() { | ||
| 394 | wait_for "$1" "$2" 5 || { | ||
| 395 | echo "e2e FAIL: $3; never saw '$2' in $1, which holds:" | ||
| 396 | cat "$1" | ||
| 397 | exit 1 | ||
| 398 | } | ||
| 399 | } | ||
| 400 | |||
| 401 | # fill_sessions SOCK STATE PREFIX FROM TO — one throwaway pipe attach per | ||
| 402 | # name so the daemon holds a slot for each. The session outlives the client | ||
| 403 | # that made it, which is what lets these detach immediately. | ||
| 404 | # | ||
| 405 | # THREE at a time, and the number is `max_observers` minus one, not | ||
| 406 | # `max_clients` minus one: `acceptConn` parks EVERY new connection in an | ||
| 407 | # observer slot and promotes it to a client only when its attach frame | ||
| 408 | # arrives, so simultaneous dials contend for the four observer slots, and | ||
| 409 | # the ones with nowhere to land are closed outright. Measured at batch 7 | ||
| 410 | # on a 32-slot daemon: 11 of 31 fills died with "connection to muxd lost" | ||
| 411 | # and the table never filled; at batch 3, 31 of 31 land. | ||
| 412 | # | ||
| 413 | # STATE is a scratch home so the fills never write a wall line the leg | ||
| 414 | # later counts. The last name is asked back through muxa so a fill that | ||
| 415 | # never landed fails here with its own log rather than as the refusal leg | ||
| 416 | # proving nothing. | ||
| 417 | fill_sessions() { | ||
| 418 | _fs_sock="$1"; _fs_state="$2"; _fs_pfx="$3"; _fs_i="$4"; _fs_to="$5" | ||
| 419 | while [ "$_fs_i" -le "$_fs_to" ]; do | ||
| 420 | _fs_pids=""; _fs_j=0 | ||
| 421 | while [ "$_fs_j" -lt 3 ] && [ "$_fs_i" -le "$_fs_to" ]; do | ||
| 422 | { sleep 1.5; printf '\034\034'; } | XDG_STATE_HOME="$_fs_state" timeout 40 \ | ||
| 423 | "$MUX" --sock "$_fs_sock" --session "$_fs_pfx$_fs_i" \ | ||
| 424 | > "$OUT.fill.$_fs_pfx$_fs_i" 2>&1 & | ||
| 425 | _fs_pids="$_fs_pids $!" | ||
| 426 | _fs_i=$((_fs_i + 1)); _fs_j=$((_fs_j + 1)) | ||
| 427 | done | ||
| 428 | for _fs_p in $_fs_pids; do wait "$_fs_p" || { | ||
| 429 | echo "e2e FAIL: fill_sessions $_fs_pfx: a fill attach on $_fs_sock exited nonzero:" | ||
| 430 | cat "$OUT".fill."$_fs_pfx"*; exit 1; }; done | ||
| 431 | done | ||
| 432 | "$MUXA" status --sock "$_fs_sock" --session "$_fs_pfx$_fs_to" > "$OUT.fill.$_fs_pfx.st" 2>&1 || { | ||
| 433 | echo "e2e FAIL: fill_sessions: $_fs_pfx$_fs_to was never created, the daemon is not full:" | ||
| 434 | cat "$OUT.fill.$_fs_pfx.st"; exit 1; } | ||
| 435 | } | ||
| 436 | |||
| 437 | # repaints FILE — how many full repaints (ESC[2J) a capture holds. The | ||
| 438 | # first paint after a reconnect is always a full one (interact.zig | ||
| 439 | # `repaint_after_resync`), so a count that rose is the resume itself, seen | ||
| 440 | # from the terminal's side. Keystrokes typed into a tear are dropped, not | ||
| 441 | # held, so a scenario that tears the transport must see this before it | ||
| 442 | # types again. | ||
| 443 | repaints() { | ||
| 444 | _clr=$(printf '\033[2J') | ||
| 445 | grep -a -F -o "$_clr" "$1" 2>/dev/null | wc -l | ||
| 446 | } | ||
| 447 | |||
| 448 | # await_repaint FILE BASELINE LABEL — wait until repaints FILE exceeds | ||
| 449 | # BASELINE (5s, scaled), FAIL with the capture otherwise. | ||
| 450 | await_repaint() { | ||
| 451 | _i=0 | ||
| 452 | while [ "$(repaints "$1")" -le "$2" ] && [ "$_i" -lt $(( 50 * TIME_SCALE )) ]; do | ||
| 453 | sleep 0.1; _i=$((_i+1)) | ||
| 454 | done | ||
| 455 | [ "$(repaints "$1")" -gt "$2" ] || { | ||
| 456 | echo "e2e FAIL: $3; no repaint after the tear in $1, which holds:" | ||
| 457 | cat "$1" | ||
| 458 | exit 1 | ||
| 459 | } | ||
| 460 | } | ||
| 461 | |||
| 462 | # pipe_detach [LABEL] — the Ctrl-\ chord, then EOF, then the client's own | ||
| 463 | # exit, which must be 0: a foreground `| "$MUX"` under set -e asserted that | ||
| 464 | # by accident, and this asserts it on purpose. LABEL names the client in | ||
| 465 | # the verdict; PIPE_RC is left for a scenario with more to say. | ||
| 466 | pipe_detach() { | ||
| 467 | printf '\034\034' >&9 | ||
| 468 | pipe_waitexit "$@" | ||
| 469 | } | ||
| 470 | |||
| 471 | # pipe_waitexit [LABEL [WANT]] — the client leaves on its own (its session | ||
| 472 | # ended); wait for that, then release the FIFO. Stdin stays open until the | ||
| 473 | # exit so the client never sees an EOF it was not sent. WANT is the exit | ||
| 474 | # status the scenario means (default 0): the exit-semantics legs want the | ||
| 475 | # shell's own. | ||
| 476 | pipe_waitexit() { | ||
| 477 | set +e | ||
| 478 | wait "$PIPE_PID" | ||
| 479 | PIPE_RC=$? | ||
| 480 | set -e | ||
| 481 | exec 9>&- | ||
| 482 | PIPE_PID="" | ||
| 483 | rm -f "$PIPE_IN" | ||
| 484 | [ "$PIPE_RC" -eq "${2:-0}" ] || { | ||
| 485 | echo "e2e FAIL: ${1:-piped client} exited $PIPE_RC (want ${2:-0}; 124 means it hung)" | ||
| 486 | cat "$PIPE_OUT" | ||
| 487 | if [ -n "$PIPE_ERR" ]; then cat "$PIPE_ERR"; fi | ||
| 488 | exit 1 | ||
| 489 | } | ||
| 490 | } | ||
| 491 | |||
| 492 | # dump_session SOCK [NAME] — `muxd dump` against one session (M18). An | ||
| 493 | # EMPTY or absent NAME passes no --session flag AT ALL rather than an empty | ||
| 494 | # one, and that is the load-bearing part: no tail is the wire's own | ||
| 495 | # default-session spelling and the pre-M18 compat path (decision 3), so | ||
| 496 | # every call site here that never heard of sessions keeps sending exactly | ||
| 497 | # the bytes it always did. One helper, so a scenario asking about a named | ||
| 498 | # session and one asking about the default cannot drift into two spellings. | ||
| 499 | dump_session() { | ||
| 500 | if [ -n "${2:-}" ]; then | ||
| 501 | "$MUXD" dump --sock "$1" --session "$2" | ||
| 502 | else | ||
| 503 | "$MUXD" dump --sock "$1" | ||
| 504 | fi | ||
| 505 | } | ||
| 506 | |||
| 507 | # wait_grid SOCK NEEDLE LABEL [SESSION] — poll the daemon's own grid for | ||
| 508 | # NEEDLE (10s) and ASSERT it landed. The assert is the whole point. The bare | ||
| 509 | # form of this loop — poll, `&& break`, carry on — falls out silently when | ||
| 510 | # the marker never arrives, so a session that died at birth is discovered by | ||
| 511 | # whatever runs next, 15s later and in the wrong layer. Three M-web | ||
| 512 | # scenarios had grown exactly that shape. | ||
| 513 | # | ||
| 514 | # SESSION is optional and defaults to the default session: a marker typed | ||
| 515 | # into session `a` is not on the default session's grid, so a named-session | ||
| 516 | # scenario that forgot the argument would poll a grid the marker can never | ||
| 517 | # reach and fail 10s later blaming the wrong thing. | ||
| 518 | wait_grid() { | ||
| 519 | _i=0 | ||
| 520 | while [ "$_i" -lt $(( 100 * TIME_SCALE )) ]; do | ||
| 521 | dump_session "$1" "${4:-}" 2>/dev/null | grep -q "$2" && return 0 | ||
| 522 | sleep 0.1; _i=$((_i+1)) | ||
| 523 | done | ||
| 524 | echo "e2e FAIL: $3: '$2' never reached the daemon's grid; it holds:" | ||
| 525 | dump_session "$1" "${4:-}" || echo "(nothing answers on $1)" | ||
| 526 | exit 1 | ||
| 527 | } | ||
| 528 | |||
| 529 | # wait_sessions SOCK N LABEL — poll until `muxd stats` reports N live | ||
| 530 | # sessions (10s) and ASSERT it. wait_grid's shape for wait_grid's reason: a | ||
| 531 | # session is freed asynchronously — its shell exits, the daemon reaps on a | ||
| 532 | # later pump — so "it died" has to be waited FOR, and a wait that falls out | ||
| 533 | # silently turns "the session never died" into a puzzling failure two | ||
| 534 | # assertions further down. | ||
| 535 | wait_sessions() { | ||
| 536 | _i=0 | ||
| 537 | while [ "$_i" -lt $(( 100 * TIME_SCALE )) ]; do | ||
| 538 | "$MUXD" stats --sock "$1" 2>/dev/null | grep -q "sessions=$2" && return 0 | ||
| 539 | sleep 0.1; _i=$((_i+1)) | ||
| 540 | done | ||
| 541 | echo "e2e FAIL: $3: stats never reported sessions=$2; it holds:" | ||
| 542 | "$MUXD" stats --sock "$1" || echo "(nothing answers on $1)" | ||
| 543 | exit 1 | ||
| 544 | } | ||
| 545 | |||
| 546 | # --- tiles attach once each: did the daemon see a second? ------------- | ||
| 547 | # | ||
| 548 | # The focus model's whole claim is that a tile's connection is its own — | ||
| 549 | # no new attach, no dial on a focus move. Proving a negative needs a | ||
| 550 | # daemon-side witness, and there are two here doing different jobs. | ||
| 551 | # | ||
| 552 | # THE ASSERTION is `attaches=`, a cumulative counter of every attach this | ||
| 553 | # daemon accepted (server.zig Stats). Read once before the wall starts and | ||
| 554 | # once after it exits, the delta is exactly how many times anything attached | ||
| 555 | # across the whole leg — which for a wall of N tiles must be N, one per tile | ||
| 556 | # at startup, and never N+1 however far the focus moved. It is read from | ||
| 557 | # `muxd stats`, which is plain human text and connects as an OBSERVER: the | ||
| 558 | # reading itself never attaches, so it cannot pollute what it measures. | ||
| 559 | # | ||
| 560 | # THE GAUGE is `session NAME clients=`, sampled every 200ms into a file and | ||
| 561 | # asserted on its peak. It is kept because it localises a failure — it says | ||
| 562 | # WHICH session grew a second watcher and roughly when — but it is the | ||
| 563 | # weaker of the two and must never be the only one: a connection that closes | ||
| 564 | # as another opens never exceeds 1, and one that lives less than a sample | ||
| 565 | # interval is invisible to it. The counter cannot miss either. | ||
| 566 | # | ||
| 567 | # Nothing else may attach while a leg is being measured (`muxa capture` and | ||
| 568 | # `wait_grid` are clients too), so both readings bracket the ptyclient leg | ||
| 569 | # and nothing more. | ||
| 570 | # | ||
| 571 | # attaches_now SOCK — the daemon's cumulative accepted-attach count. | ||
| 572 | attaches_now() { | ||
| 573 | timeout 5 "$MUXD" stats --sock "$1" 2>/dev/null | | ||
| 574 | sed -n 's/.*[^_]attaches=\([0-9]*\).*/\1/p' | ||
| 575 | } | ||
| 576 | |||
| 577 | # assert_attach_delta BEFORE AFTER WANT LABEL — how many attaches happened. | ||
| 578 | # Empty readings fail loudly rather than arithmetically: `$(())` on an empty | ||
| 579 | # string is 0, and 0-0=0 would pass this check having measured nothing at | ||
| 580 | # all — the vacuous green this whole block exists to refuse. | ||
| 581 | assert_attach_delta() { | ||
| 582 | [ -n "$1" ] && [ -n "$2" ] || { | ||
| 583 | echo "e2e FAIL: $4: stats gave no attaches= reading (before='$1' after='$2')" | ||
| 584 | echo " — the flat-attach claim would be vacuous" | ||
| 585 | exit 1; } | ||
| 586 | [ "$(($2 - $1))" = "$3" ] || { | ||
| 587 | echo "e2e FAIL: $4: the daemon accepted $(($2 - $1)) attaches (want $3)" | ||
| 588 | echo " — the wall's tiles attach once each and a focus move never does" | ||
| 589 | exit 1; } | ||
| 590 | } | ||
| 591 | |||
| 592 | # watch_clients SOCK FILE — start sampling in the background. | ||
| 593 | WATCH_PID="" | ||
| 594 | watch_clients() { | ||
| 595 | touch "$2.on" | ||
| 596 | ( while [ -e "$2.on" ]; do | ||
| 597 | timeout 5 "$MUXD" stats --sock "$1" 2>/dev/null | ||
| 598 | sleep 0.2 | ||
| 599 | done ) > "$2" & | ||
| 600 | WATCH_PID=$! | ||
| 601 | defer_kill "$WATCH_PID" | ||
| 602 | } | ||
| 603 | |||
| 604 | # unwatch_clients FILE — stop sampling. | ||
| 605 | unwatch_clients() { | ||
| 606 | rm -f "$1.on" | ||
| 607 | wait "$WATCH_PID" 2>/dev/null || true | ||
| 608 | WATCH_PID="" | ||
| 609 | } | ||
| 610 | |||
| 611 | # assert_never_two_clients FILE NAME LABEL — the peak `clients=` this | ||
| 612 | # session ever showed is 1. Also asserts it was ever 1, which is the anchor: | ||
| 613 | # a watcher that sampled an empty file, or a wall whose tiles never attached, | ||
| 614 | # would otherwise pass this leg by having witnessed nothing at all. | ||
| 615 | # | ||
| 616 | # Localisation, not proof — `assert_attach_delta` is the proof. This says | ||
| 617 | # which session grew a second watcher; the counter says whether anything | ||
| 618 | # attached at all. | ||
| 619 | assert_never_two_clients() { | ||
| 620 | _peak=$(grep -o "session $2 clients=[0-9]*" "$1" | sed 's/.*=//' | sort -n | tail -1) | ||
| 621 | [ -n "$_peak" ] || { | ||
| 622 | echo "e2e FAIL: $3: no stats sample ever named session $2 — the watch" | ||
| 623 | echo " saw nothing, so its flat-count claim is vacuous" | ||
| 624 | exit 1; } | ||
| 625 | [ "$_peak" = "1" ] || { | ||
| 626 | echo "e2e FAIL: $3: session $2 was watched by $_peak clients at once" | ||
| 627 | echo " (want 1) — a focus move attached instead of being local" | ||
| 628 | exit 1; } | ||
| 629 | } | ||
| 630 | |||
| 631 | # wait_pid_gone PID LABEL — poll until a tracked pid is gone (2s, the same | ||
| 632 | # budget `muxd stop` gives itself). Its own helper rather than wait_gone's | ||
| 633 | # socket probe, because the two answer different questions: `muxd: stopped` | ||
| 634 | # is printed on the first probe that gets a REFUSAL, which is the socket | ||
| 635 | # being unlinked, and the process can still be a fraction behind that. Only | ||
| 636 | # a pid can say the daemon itself ended, which is the assertion the M13 | ||
| 637 | # teardowns owe — never the stop command's own claim. | ||
| 638 | # | ||
| 639 | # M14 gave it a second kind of caller: the cold-handoff scenario asks the | ||
| 640 | # same question of the ssh the client is supposed to have killed once QUIC | ||
| 641 | # took over. So the message names the pid and leaves the EXPECTATION to | ||
| 642 | # each call site's label — the two are "stop said it stopped" and "QUIC | ||
| 643 | # said it had taken over", and one wording cannot honestly claim both. | ||
| 644 | wait_pid_gone() { | ||
| 645 | _i=0 | ||
| 646 | while kill -0 "$1" 2>/dev/null; do | ||
| 647 | _i=$((_i + 1)); [ "$_i" -lt 40 ] || { | ||
| 648 | echo "e2e FAIL: $2: pid $1 is still running 2s later"; exit 1; } | ||
| 649 | sleep 0.05 | ||
| 650 | done | ||
| 651 | } | ||
| 652 | |||
| 653 | # assert_stopped SOCK PID LABEL ERRFILE — teardown by the sanctioned verb, | ||
| 654 | # asserted four ways. An exit code says the command RETURNED, not that it | ||
| 655 | # did the job: `muxd: stopped` is the daemon's own account of it, the absent | ||
| 656 | # socket is the filesystem's, and only the pid can say the process itself | ||
| 657 | # ended (see wait_pid_gone). A stop that exited 0 while leaving any of the | ||
| 658 | # three behind is precisely the regression this shape exists to catch. | ||
| 659 | # ERRFILE holds the stop's stderr — captured rather than let through, | ||
| 660 | # because a stop that fails exits 1 and would abort under `set -e` with no | ||
| 661 | # line of its own, after which the trap deletes the evidence. | ||
| 662 | # | ||
| 663 | # The caller nulls its own pid variable afterwards, the way every scenario | ||
| 664 | # here does: the trap reads those variables, and a pid it still holds is | ||
| 665 | # how a failing run says which daemon it leaked. | ||
| 666 | assert_stopped() { | ||
| 667 | set +e | ||
| 668 | "$MUXD" stop --sock "$1" 2> "$4" | ||
| 669 | _rc=$? | ||
| 670 | set -e | ||
| 671 | [ "$_rc" = "0" ] || { | ||
| 672 | echo "e2e FAIL: $3: stop exited $_rc, want 0"; cat "$4"; exit 1; } | ||
| 673 | grep -q '^muxd: stopped' "$4" || { | ||
| 674 | echo "e2e FAIL: $3: stop did not report stopped"; cat "$4"; exit 1; } | ||
| 675 | [ ! -S "$1" ] || { | ||
| 676 | echo "e2e FAIL: $3: stop left the socket"; ls -l "$1"; exit 1; } | ||
| 677 | wait_pid_gone "$2" "$3: stop reported stopped" | ||
| 678 | } | ||
| 679 | |||
| 680 | # The transport child for a given socket: a process named muxd running the | ||
| 681 | # `proxy` subcommand on that path. NEVER `pkill -f proxy` — the client's own | ||
| 682 | # argv contains the --via command string, so a pattern kill takes out the | ||
| 683 | # very client under test. | ||
| 684 | proxy_pid() { | ||
| 685 | ps -eo pid,comm,args | awk -v s="$1" '$2=="muxd" && /proxy/ && index($0,s) {print $1}' | head -1 | ||
| 686 | } | ||
| 687 | |||
| 688 | # --- M11: render-vs-dump convergence ----------------------------------- | ||
| 689 | # converged_quiet CLIENT_OUT SOCK [COLS ROWS] — render the captured client | ||
| 690 | # stream and diff it against the daemon's grid, plain and styled. Nonzero on | ||
| 691 | # divergence, leaving CLIENT_OUT.{render,dump,diff,rvt,dvt} behind for | ||
| 692 | # inspection. CLIENT_OUT must be PURE client stdout: a capture taken with | ||
| 693 | # 2>&1 has exit messages and predict stats mixed into the escape stream. | ||
| 694 | # The optional size is for pty scenarios whose grids are not the non-tty | ||
| 695 | # 80x24 default; render must replay into the same dimensions the daemon | ||
| 696 | # holds or the diff compares two honest grids of different shapes. | ||
| 697 | # It is an INPUT, not something this check validates, and the error is | ||
| 698 | # one-sided: too small re-wraps or clips a row and diverges loudly (the | ||
| 699 | # 80x24 default against tp2b's 100-wide final grid does exactly that), | ||
| 700 | # while too large only adds trailing blanks that the normalization strips, | ||
| 701 | # and passes. Pass the size the scenario actually ran at. | ||
| 702 | converged_quiet() { | ||
| 703 | _co="$1"; _cs="$2"; _sz="" | ||
| 704 | # An `if` rather than `[ ... ] && _sz=...` so it stays correct if it | ||
| 705 | # ever ends up the last command in this function: there, a false guard | ||
| 706 | # would be the function's exit status under `set -e` and a 2-argument | ||
| 707 | # call would report a divergence it never looked for. | ||
| 708 | if [ $# -ge 4 ]; then _sz="--cols $3 --rows $4"; fi | ||
| 709 | # $_sz is two flags or nothing, never data. | ||
| 710 | # shellcheck disable=SC2086 | ||
| 711 | "$RENDER" $_sz < "$_co" > "$_co.render" || return 1 | ||
| 712 | "$MUXD" dump --sock "$_cs" > "$_co.dump" || return 1 | ||
| 713 | # Trailing whitespace is a formatting difference between two correct | ||
| 714 | # grids (padded vs unpadded row ends), not a divergence. | ||
| 715 | sed 's/[[:space:]]*$//' "$_co.render" > "$_co.render.n" | ||
| 716 | sed 's/[[:space:]]*$//' "$_co.dump" > "$_co.dump.n" | ||
| 717 | diff -u "$_co.dump.n" "$_co.render.n" > "$_co.diff" || return 1 | ||
| 718 | # Styled, byte-for-byte: both sides come out of the same formatter, so | ||
| 719 | # equal grids are equal bytes. This is the half that sees a bled SGR | ||
| 720 | # or a leftover prediction underline — plain text dumps the same glyph | ||
| 721 | # either way, which is exactly why plain alone cannot carry M9's | ||
| 722 | # overlay-never-becomes-state invariant. | ||
| 723 | # Same as above: flags or nothing. | ||
| 724 | # shellcheck disable=SC2086 | ||
| 725 | "$RENDER" --vt $_sz < "$_co" > "$_co.rvt" || return 1 | ||
| 726 | "$MUXD" dump --vt --sock "$_cs" > "$_co.dvt" || return 1 | ||
| 727 | cmp -s "$_co.dvt" "$_co.rvt" || return 1 | ||
| 728 | rm -f "$_co.render" "$_co.dump" "$_co.render.n" "$_co.dump.n" \ | ||
| 729 | "$_co.diff" "$_co.rvt" "$_co.dvt" | ||
| 730 | return 0 | ||
| 731 | } | ||
| 732 | |||
| 733 | # assert_converged CLIENT_OUT SOCK NAME [COLS ROWS] — a scenario's LAST act, | ||
| 734 | # after quiesce and after the client detached: mid-scenario the dump is still | ||
| 735 | # moving, and another attach would claim the grid. | ||
| 736 | CONV_COUNT=0 | ||
| 737 | assert_converged() { | ||
| 738 | CONV_COUNT=$((CONV_COUNT + 1)) | ||
| 739 | if [ $# -ge 5 ]; then | ||
| 740 | converged_quiet "$1" "$2" "$4" "$5" | ||
| 741 | else | ||
| 742 | converged_quiet "$1" "$2" | ||
| 743 | fi || { | ||
| 744 | echo "e2e FAIL: $3: client render diverges from daemon grid (-daemon +client):" | ||
| 745 | # An empty diff file is the styled-only case, not a passing one — | ||
| 746 | # the glyphs agree and the pens do not, which is precisely what the | ||
| 747 | # byte-exact leg exists to catch, so say so instead of printing | ||
| 748 | # forty lines of nothing. | ||
| 749 | if [ -s "$1.diff" ]; then | ||
| 750 | head -40 "$1.diff" | ||
| 751 | else | ||
| 752 | echo "(no plain diff: the grids agree on glyphs and differ on STYLE —" | ||
| 753 | echo " compare $1.dvt against $1.rvt)" | ||
| 754 | fi | ||
| 755 | echo "grids left in $1.render / $1.dump / $1.rvt / $1.dvt" | ||
| 756 | exit 1 | ||
| 757 | } | ||
| 758 | } | ||
| 759 | |||
| 760 | # assert_ws_converged WSOUT SOCK LABEL [SESSION] — the WebSocket leg's | ||
| 761 | # convergence check. WSOUT is the stand-in's `dumpexit` file: its replica's | ||
| 762 | # grid, in `muxd dump`'s own format, built from frames that crossed the hub. | ||
| 763 | # Diff it against the daemon's grid, then doctor a copy of that grid and | ||
| 764 | # assert the SAME diff catches the doctored one — the wan.sh rule (M9): a | ||
| 765 | # convergence check that cannot fail proves nothing. | ||
| 766 | # | ||
| 767 | # SESSION (M18) names which grid the daemon side of that diff comes from. A | ||
| 768 | # tile attached to session `a` must be diffed against `a`, and pointing it | ||
| 769 | # at the default session instead would compare two unrelated grids — the | ||
| 770 | # check would fail, loudly and for the wrong reason. | ||
| 771 | # | ||
| 772 | # Deliberately NOT counted in CONV_COUNT. That pin counts assert_converged | ||
| 773 | # call sites — render-replays-the-client-stream — and this is different | ||
| 774 | # machinery answering a different question. Folding the two counts together | ||
| 775 | # would make either pin's number stop meaning anything. | ||
| 776 | assert_ws_converged() { | ||
| 777 | dump_session "$2" "${4:-}" > "$1.dump" | ||
| 778 | # Trailing whitespace is a formatting difference between two correct | ||
| 779 | # grids, exactly as in converged_quiet. | ||
| 780 | sed 's/[[:space:]]*$//' "$1" > "$1.n" | ||
| 781 | sed 's/[[:space:]]*$//' "$1.dump" > "$1.dump.n" | ||
| 782 | diff -u "$1.dump.n" "$1.n" > "$1.diff" || { | ||
| 783 | echo "e2e FAIL: $3: wsclient replica diverges from daemon grid (-daemon +ws):" | ||
| 784 | head -40 "$1.diff"; exit 1; } | ||
| 785 | cp "$1.dump.n" "$1.doc" | ||
| 786 | printf 'doctored-row\n' >> "$1.doc" | ||
| 787 | if diff -u "$1.doc" "$1.n" > /dev/null 2>&1; then | ||
| 788 | echo "e2e FAIL: $3: the convergence diff cannot fail (doctored dump passed)"; exit 1 | ||
| 789 | fi | ||
| 790 | } | ||
| 791 | |||
| 792 | # Scenario checkpoints. The suite's final line asserts the COUNT of these | ||
| 793 | # against a literal: adding or removing a scenario means updating that | ||
| 794 | # literal, and the friction is the feature — a scenario that silently | ||
| 795 | # stops running is the failure mode the pin exists for. | ||
| 796 | OK_COUNT=0 | ||
| 797 | ok() { | ||
| 798 | OK_COUNT=$((OK_COUNT + 1)) | ||
| 799 | echo "e2e OK: $1" | ||
| 800 | # Scenario boundaries, stamped for whoever needs to attribute something | ||
| 801 | # to the scenario that produced it. Nothing in this suite reads the file: | ||
| 802 | # test/coverage.sh maps each kcov database to the scenario that was | ||
| 803 | # running when the traced process wrote it, and the timestamps are also | ||
| 804 | # the only per-scenario timing this suite has ever been able to report. | ||
| 805 | # | ||
| 806 | # An `if` rather than `[ -n ... ] && printf`, for wait_sock's reason: a | ||
| 807 | # false guard as the last command in a function becomes that function's | ||
| 808 | # exit status, and under `set -e` every unstamped run would abort at its | ||
| 809 | # first passing scenario. | ||
| 810 | if [ -n "${E2E_OK_LOG:-}" ]; then | ||
| 811 | printf '%s\t%s\t%s\n' "$OK_COUNT" "$(date +%s.%N)" "$1" >> "$E2E_OK_LOG" | ||
| 812 | fi | ||
| 813 | # Prefix slicing. This suite is linear and stateful — the M13 scenarios | ||
| 814 | # run inside sessions the M10 scenarios created — so a prefix is the only | ||
| 815 | # slice that means anything, and a "run just scenario 40" filter would be | ||
| 816 | # a filter that lies. Exiting rather than skipping keeps that honest, and | ||
| 817 | # exiting 0 goes out through the trap, so a sliced run still tears down | ||
| 818 | # its daemons and still reports its leak verdict. | ||
| 819 | if [ -n "${E2E_STOP_AFTER:-}" ] && [ "$OK_COUNT" -ge "$E2E_STOP_AFTER" ]; then | ||
| 820 | echo "e2e STOP: sliced after $OK_COUNT scenarios (E2E_STOP_AFTER)" | ||
| 821 | exit 0 | ||
| 822 | fi | ||
| 823 | } | ||
| 824 | |||
| 825 | # ---- the leak sweep's two halves (hygiene kit, 6a) --------------------- | ||
| 826 | # Every capture this suite writes — daemon stderr AND client output — is a | ||
| 827 | # lifecycle log: a binary that leaked printed a `LEAK:` marker into one. The | ||
| 828 | # verdict is read in the EXIT trap, which is the only place that sees a | ||
| 829 | # FAILING run too; under `set -e` the bottom of this file is reached by | ||
| 830 | # passing runs alone, and a leak introduced alongside a defect is exactly | ||
| 831 | # the pair a suite should report together. | ||
| 832 | # | ||
| 833 | # That leaves the captures the scenarios delete as they go. A file removed | ||
| 834 | # at line 900 is not there for a trap at line 3000 to read, so removal is | ||
| 835 | # where its verdict has to be observed: rm_swept banks the marker first and | ||
| 836 | # deletes second, and the bank is what the trap reads for everything that no | ||
| 837 | # longer exists. Mid-suite removal of a capture goes through here — a plain | ||
| 838 | # `rm -f` of one is a verdict thrown away. | ||
| 839 | LEAKBANK="$OUT.leakbank" | ||
| 840 | |||
| 841 | # leak_line FILE — the verdict FILE holds, named and made readable. Never the | ||
| 842 | # matched line verbatim: half these captures are escape streams, whose one | ||
| 843 | # "line" is the entire session replay, and printing that repaints the reader's | ||
| 844 | # terminal instead of reporting to it. A log line short enough to BE a log | ||
| 845 | # line is kept whole (it names the binary, which is worth having); anything | ||
| 846 | # longer is cut back to the verdict. `grep -a` because the same captures make | ||
| 847 | # grep answer "Binary file ... matches", which does not even contain the | ||
| 848 | # marker a reader — or the sweep's own grep over the bank — is looking for. | ||
| 849 | leak_line() { | ||
| 850 | _ll=$(grep -a "LEAK:" "$1" 2>/dev/null | head -1 | tr -d '\000-\011\013-\037') | ||
| 851 | if [ "${#_ll}" -gt 120 ]; then | ||
| 852 | _ll=$(printf '%s\n' "$_ll" | sed 's/.*\(LEAK:\)/\1/' | cut -c1-120) | ||
| 853 | fi | ||
| 854 | case "$_ll" in | ||
| 855 | *LEAK:*) ;; | ||
| 856 | *) _ll="LEAK: marker present, unreadable as text" ;; | ||
| 857 | esac | ||
| 858 | echo "$1: $_ll" | ||
| 859 | } | ||
| 860 | |||
| 861 | rm_swept() { | ||
| 862 | for _rs in "$@"; do | ||
| 863 | # Regular files only: this list carries sockets and generated | ||
| 864 | # scripts too, and a socket path is not something to open. | ||
| 865 | [ -f "$_rs" ] || continue | ||
| 866 | if grep -q "LEAK:" "$_rs" 2>/dev/null; then | ||
| 867 | leak_line "$_rs" >> "$LEAKBANK" | ||
| 868 | fi | ||
| 869 | done | ||
| 870 | rm -f "$@" | ||
| 871 | } | ||
| 872 | |||
| 873 | # leak_sweep RC — the whole-suite verdict. RC is the suite's own exit status | ||
| 874 | # and decides one thing: what an empty sweep means. | ||
| 875 | # | ||
| 876 | # On a green run, nothing to read is a failure — the vacuous-green guard this | ||
| 877 | # gate has always had. $OUT.d1.d is created when daemon one is spawned and | ||
| 878 | # lives until this trap, and the bank exists once a verdict has been banked | ||
| 879 | # out of a deleted capture, so neither being there means the capture | ||
| 880 | # convention broke and this gate is reading nothing. On a run that is ALREADY | ||
| 881 | # failing, the same emptiness means something else entirely: an early exit, | ||
| 882 | # before the first daemon ever existed. That is not a second defect, and | ||
| 883 | # reporting it as one would stack a fabricated failure on top of the real | ||
| 884 | # one, so it says what it saw and returns clean. | ||
| 885 | # | ||
| 886 | # It never speaks for the suite either way: the caller promotes this verdict | ||
| 887 | # only over a green run (see cleanup). | ||
| 888 | leak_sweep() { | ||
| 889 | _lsrc="$1" | ||
| 890 | _lsbad=0 | ||
| 891 | if [ ! -e "$OUT.d1.d" ] && [ ! -e "$LEAKBANK" ]; then | ||
| 892 | if [ "$_lsrc" -eq 0 ]; then | ||
| 893 | echo "e2e FAIL: leak sweep found no captures to read" | ||
| 894 | return 1 | ||
| 895 | fi | ||
| 896 | echo "e2e note: leak sweep had nothing to read — the run exited $_lsrc" \ | ||
| 897 | "before the first daemon" | ||
| 898 | return 0 | ||
| 899 | fi | ||
| 900 | # The captures still on disk, one report line each. `"$OUT"` is named | ||
| 901 | # alongside the glob because the first client capture is written to the | ||
| 902 | # bare path, and `"$OUT".*` does not match it. | ||
| 903 | _lssaid="" | ||
| 904 | for _lsf in "$OUT" "$OUT".*; do | ||
| 905 | if [ ! -f "$_lsf" ]; then continue; fi | ||
| 906 | # The bank is not a capture; it is printed whole below, and running it | ||
| 907 | # through leak_line would report only its first entry. | ||
| 908 | if [ "$_lsf" = "$LEAKBANK" ]; then continue; fi | ||
| 909 | if grep -q "LEAK:" "$_lsf" 2>/dev/null; then | ||
| 910 | if [ -z "$_lssaid" ]; then | ||
| 911 | echo "e2e FAIL: a binary reported leaked allocations:" | ||
| 912 | _lssaid=1 | ||
| 913 | fi | ||
| 914 | leak_line "$_lsf" | ||
| 915 | _lsbad=1 | ||
| 916 | fi | ||
| 917 | done | ||
| 918 | # ...and the verdicts banked out of captures the suite deleted as it went. | ||
| 919 | if [ -s "$LEAKBANK" ]; then | ||
| 920 | echo "e2e FAIL: a binary reported leaked allocations into a capture" \ | ||
| 921 | "the suite has since deleted:" | ||
| 922 | cat "$LEAKBANK" | ||
| 923 | _lsbad=1 | ||
| 924 | fi | ||
| 925 | # The detached (`muxd start`) daemons log via XDG_STATE_HOME; the file is | ||
| 926 | # truncated at every spawn, so this asserts the LAST such daemon only — | ||
| 927 | # stated, not hidden. | ||
| 928 | if [ -f "$XDG_STATE_HOME/mux/muxd.log" ] && | ||
| 929 | grep -q "LEAK:" "$XDG_STATE_HOME/mux/muxd.log"; then | ||
| 930 | echo "e2e FAIL: a detached daemon reported leaked allocations:" | ||
| 931 | grep -H "LEAK:" "$XDG_STATE_HOME/mux/muxd.log" || true | ||
| 932 | _lsbad=1 | ||
| 933 | fi | ||
| 934 | [ "$_lsbad" -eq 0 ] | ||
| 935 | } | ||
| 936 | |||
| 937 | # A killed daemon writes its leak verdict on the way OUT, so a sweep that | ||
| 938 | # reads before the process is gone reads a file the verdict has not reached. | ||
| 939 | # Bounded, and shared across the pids rather than per-pid: a trap must never | ||
| 940 | # be the thing that hangs, and a daemon that outlives the wait is swept for | ||
| 941 | # whatever it did write. | ||
| 942 | reap_briefly() { | ||
| 943 | _rbi=0 | ||
| 944 | while [ "$_rbi" -lt 40 ]; do | ||
| 945 | _rblive="" | ||
| 946 | for _rbp in "$@"; do | ||
| 947 | [ -n "$_rbp" ] || continue | ||
| 948 | if kill -0 "$_rbp" 2>/dev/null; then _rblive=1; fi | ||
| 949 | done | ||
| 950 | [ -n "$_rblive" ] || return 0 | ||
| 951 | sleep 0.05 | ||
| 952 | _rbi=$((_rbi + 1)) | ||
| 953 | done | ||
| 954 | return 0 | ||
| 955 | } | ||
| 956 | |||
| 957 | cleanup() { | ||
| 958 | # The suite's own status, captured before anything in here can overwrite | ||
| 959 | # it. Every verdict below is composed onto THIS number, never in place of | ||
| 960 | # it: a run that failed at scenario 6 exits with scenario 6's failure. | ||
| 961 | _rc=$? | ||
| 962 | # The registers are newline-joined and the paths in them may hold spaces | ||
| 963 | # (a $TMPDIR nobody here chose); a newline IFS is what keeps each entry | ||
| 964 | # one entry. | ||
| 965 | _oifs=$IFS | ||
| 966 | IFS=' | ||
| 967 | ' | ||
| 968 | # Sockets first, and by the daemon's own verb. `muxd stop` on a path | ||
| 969 | # nobody serves is a no-op that exits 0, and on a daemon this shell never | ||
| 970 | # forked — one a proxy, `muxd start` or the handoff spawned — it is the | ||
| 971 | # only handle there is. Before the kills, and long before the unlink | ||
| 972 | # below: unlinking first would leave a live daemon nothing could reach | ||
| 973 | # by path. | ||
| 974 | for _cs in $E2E_SOCK; do | ||
| 975 | if [ -S "$_cs" ] && [ -n "${MUXD:-}" ]; then | ||
| 976 | "$MUXD" stop --sock "$_cs" > /dev/null 2>&1 || true | ||
| 977 | fi | ||
| 978 | done | ||
| 979 | # Then every registered process. softkill rather than a plain `kill`, | ||
| 980 | # which is what this trap used to send: under the coverage harness the | ||
| 981 | # pid the suite holds is a kcov TRACER, and a signal sent to a tracer is | ||
| 982 | # dropped — measured on the hub leg, where `kill $W3PID` left both alive. | ||
| 983 | # TERM rather than hardkill's KILL, because a daemon writes its leak | ||
| 984 | # verdict on the way OUT and a SIGKILLed one writes nothing: the sweep | ||
| 985 | # below would then be reading a lifecycle that never ended. | ||
| 986 | # | ||
| 987 | # CONT first, and unconditionally: two legs hold their subject under | ||
| 988 | # SIGSTOP (the mute-offerer's agent, the refusal leg's daemon), and a | ||
| 989 | # stopped process does not see TERM until it runs again. On anything not | ||
| 990 | # stopped it is a no-op. | ||
| 991 | # | ||
| 992 | # `|| true` on both, for the reason every kill in this trap has always | ||
| 993 | # carried one: under `set -e` a signal to an already-dead pid would abort | ||
| 994 | # the trap itself and skip everything below it. | ||
| 995 | for _ck in $E2E_KILL; do | ||
| 996 | kill -CONT "$_ck" 2>/dev/null || true | ||
| 997 | softkill "$_ck" || true | ||
| 998 | done | ||
| 999 | # ---- the leak sweep (hygiene kit, 6a) ---- | ||
| 1000 | # Here rather than at the bottom of the file, which `set -e` reaches only | ||
| 1001 | # on a passing run: a leak that arrives alongside a defect is reported | ||
| 1002 | # with it. Between the kills above and the rms below, which is the one | ||
| 1003 | # window where every daemon has finished its exit path and every capture | ||
| 1004 | # still exists. | ||
| 1005 | # | ||
| 1006 | # A killed daemon writes its verdict on the way out, so the sweep waits | ||
| 1007 | # for the processes to be gone first. | ||
| 1008 | # shellcheck disable=SC2086 # IFS is a newline: each entry is one word | ||
| 1009 | reap_briefly $E2E_KILL | ||
| 1010 | _leak=0 | ||
| 1011 | leak_sweep "$_rc" || _leak=1 | ||
| 1012 | |||
| 1013 | # The registered artifacts: sockets, keys, generated scripts, private | ||
| 1014 | # HOMEs, state homes. Through rm_swept, so a registered FILE still banks | ||
| 1015 | # its leak verdict on the way out exactly as a mid-suite removal does; | ||
| 1016 | # directories go whole. | ||
| 1017 | for _cr in $E2E_SOCK $E2E_RM; do | ||
| 1018 | if [ -d "$_cr" ]; then | ||
| 1019 | rm -rf "$_cr" | ||
| 1020 | else | ||
| 1021 | rm_swept "$_cr" | ||
| 1022 | fi | ||
| 1023 | done | ||
| 1024 | IFS=$_oifs | ||
| 1025 | |||
| 1026 | # ...and the captures, by the pattern leak_sweep just read them by rather | ||
| 1027 | # than by a list of some 350 names. The list is what failed: every | ||
| 1028 | # capture had to be remembered in two places, 102 of them were not, and | ||
| 1029 | # by 2026-08-19 `make soak` refused to start. A pattern cannot forget a | ||
| 1030 | # leg. | ||
| 1031 | # | ||
| 1032 | # Green runs only. A failing run keeps its captures — the .render/.dump/ | ||
| 1033 | # .diff files a failed assert_converged leaves behind are that failure's | ||
| 1034 | # evidence, and soak moves the whole set into its failure dir for the | ||
| 1035 | # next run's sake. | ||
| 1036 | if [ "$_rc" -eq 0 ]; then | ||
| 1037 | for _co in "$OUT" "$OUT".*; do | ||
| 1038 | [ -e "$_co" ] || continue | ||
| 1039 | rm -rf "$_co" | ||
| 1040 | done | ||
| 1041 | fi | ||
| 1042 | # Residue, in the leak sweep's shape and for its reason: a run that PASSED | ||
| 1043 | # must leave nothing behind, and this trap is the last place that can | ||
| 1044 | # still say so. Pinned to this run's pid so a suite running concurrently | ||
| 1045 | # is never counted, and never deleted. | ||
| 1046 | # | ||
| 1047 | # What it catches now is a path that is neither registered nor spelled | ||
| 1048 | # from $OUT — the one shape both mechanisms above are blind to. | ||
| 1049 | _stray=0 | ||
| 1050 | if [ "$_rc" -eq 0 ]; then | ||
| 1051 | _left=$(find "${TMPDIR:-/tmp}" -maxdepth 1 \ | ||
| 1052 | \( -name "mux*-$$" -o -name "mux*-$$.*" \) 2>/dev/null) | ||
| 1053 | if [ -n "$_left" ]; then | ||
| 1054 | _n=$(printf '%s\n' "$_left" | wc -l) | ||
| 1055 | if [ "$_n" -eq 1 ]; then _w=file; else _w=files; fi | ||
| 1056 | echo "e2e FAIL: the suite passed but left $_n $_w in ${TMPDIR:-/tmp}:" | ||
| 1057 | printf '%s\n' "$_left" | sed 's/^/ /' | ||
| 1058 | echo " A capture is spelled from \$OUT and swept by pattern;" | ||
| 1059 | echo " anything else is registered where it is created, with" | ||
| 1060 | echo " defer_rm, defer_sock or start_daemon. A path that is" | ||
| 1061 | echo " neither is invisible until soak refuses to start." | ||
| 1062 | printf '%s\n' "$_left" | xargs -r rm -rf | ||
| 1063 | _stray=1 | ||
| 1064 | fi | ||
| 1065 | fi | ||
| 1066 | # This trap can change the suite's answer in exactly one direction: a run | ||
| 1067 | # that was green and swept up a leak. Every other path returns normally | ||
| 1068 | # and leaves the status alone — a suite that failed at scenario 6 must | ||
| 1069 | # exit with scenario 6's failure, not with the trap's opinion of it. | ||
| 1070 | if [ "$_rc" -eq 0 ] && { [ "$_leak" -eq 1 ] || [ "$_stray" -eq 1 ]; }; then | ||
| 1071 | exit 1 | ||
| 1072 | fi | ||
| 1073 | } | ||
| 1074 | trap cleanup EXIT INT TERM | ||