3ef748c2
test: make soak repeats the suite and attributes failures to runs
a73x 2026-08-09 17:55
Commit message
Makefile
| Old | New | ||
|---|---|---|---|
| @@ -2,7 +2,7 @@ | |||
| 2 | # default zig is 0.17-dev. Override with ZIG=... if yours lives elsewhere. | 2 | # default zig is 0.17-dev. Override with ZIG=... if yours lives elsewhere. |
| 3 | ZIG ?= $(HOME)/Downloads/zig-x86_64-linux-0.15.2/zig | 3 | ZIG ?= $(HOME)/Downloads/zig-x86_64-linux-0.15.2/zig |
| 4 | 4 | ||
| 5 | .PHONY: build test e2e bench deps clean clean-deps | 5 | .PHONY: build test e2e soak bench deps clean clean-deps |
| 6 | 6 | ||
| 7 | # The QUIC stack (deps/quic) is built on demand by build.zig, so no target | 7 | # The QUIC stack (deps/quic) is built on demand by build.zig, so no target |
| 8 | # here needs to depend on this one. It exists to make the one-time cost | 8 | # here needs to depend on this one. It exists to make the one-time cost |
| @@ -22,6 +22,9 @@ test: | |||
| 22 | e2e: | 22 | e2e: |
| 23 | $(ZIG) build e2e | 23 | $(ZIG) build e2e |
| 24 | 24 | ||
| 25 | soak: | ||
| 26 | $(ZIG) build soak | ||
| 27 | |||
| 25 | bench: | 28 | bench: |
| 26 | $(ZIG) build bench | 29 | $(ZIG) build bench |
| 27 | 30 | ||
build.zig
| Old | New | ||
|---|---|---|---|
| @@ -306,6 +306,15 @@ pub fn build(b: *std.Build) void { | |||
| 306 | const e2e_step = b.step("e2e", "Run end-to-end test"); | 306 | const e2e_step = b.step("e2e", "Run end-to-end test"); |
| 307 | e2e_step.dependOn(&e2e.step); | 307 | e2e_step.dependOn(&e2e.step); |
| 308 | 308 | ||
| 309 | const soak = b.addSystemCommand(&.{"test/soak.sh"}); | ||
| 310 | soak.addArtifactArg(exe); | ||
| 311 | soak.addArtifactArg(mux_exe); | ||
| 312 | soak.addArtifactArg(rawmode_exe); | ||
| 313 | soak.addArtifactArg(delaypipe_exe); | ||
| 314 | soak.addArtifactArg(render_exe); | ||
| 315 | const soak_step = b.step("soak", "Run the e2e suite SOAK_N times (default 10)"); | ||
| 316 | soak_step.dependOn(&soak.step); | ||
| 317 | |||
| 309 | const bench = b.addSystemCommand(&.{"test/bench.sh"}); | 318 | const bench = b.addSystemCommand(&.{"test/bench.sh"}); |
| 310 | bench.addArtifactArg(exe); | 319 | bench.addArtifactArg(exe); |
| 311 | bench.addArtifactArg(mux_exe); | 320 | bench.addArtifactArg(mux_exe); |
test/soak.sh
| Old | New | ||
|---|---|---|---|
| @@ -0,0 +1,58 @@ | |||
| 1 | #!/bin/sh | ||
| 2 | # Runs the full e2e suite SOAK_N times (default 10, ~25min) and reports a | ||
| 3 | # per-failure table in the decisions.md convention: which FAIL line, from | ||
| 4 | # which run. Serial on purpose — each e2e run isolates by its own $$, and | ||
| 5 | # the between-runs hygiene check below assumes nothing else is using the | ||
| 6 | # same temp-file patterns while it looks (a concurrently running suite | ||
| 7 | # would read as a leak). | ||
| 8 | set -u | ||
| 9 | MUXD="$1"; MUX="$2"; RAWMODE="$3"; DELAYPIPE="$4"; RENDER="$5" | ||
| 10 | E2E="$(dirname "$0")/e2e.sh" | ||
| 11 | N="${SOAK_N:-10}" | ||
| 12 | TMP="${TMPDIR:-/tmp}" | ||
| 13 | FAILDIR="$TMP/mux-soak-$$-failures" | ||
| 14 | SUMMARY="$TMP/mux-soak-$$.summary" | ||
| 15 | LOG="$TMP/mux-soak-$$.log" | ||
| 16 | : > "$SUMMARY" | ||
| 17 | FAILED=0 | ||
| 18 | i=1 | ||
| 19 | while [ "$i" -le "$N" ]; do | ||
| 20 | if "$E2E" "$MUXD" "$MUX" "$RAWMODE" "$DELAYPIPE" "$RENDER" > "$LOG" 2>&1; then | ||
| 21 | echo "soak run $i/$N: PASS" | ||
| 22 | else | ||
| 23 | FAILED=$((FAILED + 1)) | ||
| 24 | FL=$(grep 'e2e FAIL' "$LOG" | head -1) | ||
| 25 | FL="${FL:-exited nonzero with no FAIL line}" | ||
| 26 | echo "soak run $i/$N: FAIL — $FL" | ||
| 27 | printf 'run %s: %s\n' "$i" "$FL" >> "$SUMMARY" | ||
| 28 | mkdir -p "$FAILDIR" | ||
| 29 | cp "$LOG" "$FAILDIR/run$i.log" | ||
| 30 | # A failing run deliberately leaves grid evidence behind; sweep it | ||
| 31 | # into the failure dir so the hygiene check below stays meaningful | ||
| 32 | # for the NEXT run. | ||
| 33 | find "$TMP" -maxdepth 1 \( -name 'muxd-e2e-*' -o -name 'mux-e2e-*' \) \ | ||
| 34 | -exec mv {} "$FAILDIR/" \; 2>/dev/null | ||
| 35 | fi | ||
| 36 | # Per-run tmp hygiene: a leak in run 3 must not blame run 7. | ||
| 37 | STRAYS=$(find "$TMP" -maxdepth 1 \( -name 'muxd-e2e-*' -o -name 'mux-e2e-*' \) 2>/dev/null | wc -l) | ||
| 38 | if [ "$STRAYS" -ne 0 ]; then | ||
| 39 | echo "soak FAIL: run $i left $STRAYS temp files behind:" | ||
| 40 | find "$TMP" -maxdepth 1 \( -name 'muxd-e2e-*' -o -name 'mux-e2e-*' \) | ||
| 41 | FAILED=$((FAILED + 1)) | ||
| 42 | printf 'run %s: left %s temp files\n' "$i" "$STRAYS" >> "$SUMMARY" | ||
| 43 | fi | ||
| 44 | i=$((i + 1)) | ||
| 45 | done | ||
| 46 | rm -f "$LOG" | ||
| 47 | echo "---" | ||
| 48 | if [ "$FAILED" -eq 0 ]; then | ||
| 49 | echo "soak OK: $N/$N runs green" | ||
| 50 | rm -f "$SUMMARY" | ||
| 51 | exit 0 | ||
| 52 | fi | ||
| 53 | # The table: failure line -> count/N, with run attribution underneath. | ||
| 54 | echo "soak FAIL: $FAILED of $N runs failed (logs in $FAILDIR):" | ||
| 55 | sed 's/^run [0-9]*: //' "$SUMMARY" | sort | uniq -c | sort -rn | \ | ||
| 56 | while read -r c l; do echo " $c/$N $l"; done | ||
| 57 | cat "$SUMMARY" | ||
| 58 | exit 1 | ||