20016508

test: e2e proves resume-by-delta across a transport tear and resume-by-snapshot across a daemon restart

a73x 2026-08-08 14:08

Commit message

test: e2e proves resume-by-delta across a transport tear and resume-by-snapshot across a daemon restart

test/e2e.sh

@@ @@ -10,12 +10,40 @@ OUT="${TMPDIR:-/tmp}/mux-e2e-out-$$" @@
+# trap below can reference them under `set -u` before they are ever started.
+SOCK2="${TMPDIR:-/tmp}/muxd-e2e-abort-$$.sock"
+D2PID=""
+# Third daemon, for the restart scenario: it gets killed and started again on
+# the same path, so it cannot share the long-lived one.
+SOCK3="${TMPDIR:-/tmp}/muxd-e2e-restart-$$.sock"
+D3PID=""
+# Wait until PATTERN shows up in FILE (default 15s). Timing that keys off the
+# session's own output instead of a fixed sleep: the marker is proof the
+# client is attached and idle, which is exactly the state the tear needs.
+wait_for() {
+    _file="$1"; _pat="$2"; _ticks=$(( ${3:-15} * 10 )); _i=0
+    while [ "$_i" -lt "$_ticks" ]; do
+        if [ -f "$_file" ] && grep -q "$_pat" "$_file" 2>/dev/null; then return 0; fi
+        sleep 0.1; _i=$((_i+1))
+    done
+    return 1
+}
+# The transport child for a given socket: a process named muxd running the
+# `proxy` subcommand on that path. NEVER `pkill -f proxy` — the client's own
+# argv contains the --via command string, so a pattern kill takes out the
+# very client under test.
+proxy_pid() {
+    ps -eo pid,comm,args | awk -v s="$1" '$2=="muxd" && /proxy/ && index($0,s) {print $1}' | head -1
+}
+cleanup() {
+    kill "$DPID" 2>/dev/null || true
+    # `|| true` on every one of these: under `set -e` a kill of an
+    # already-dead pid would abort the trap itself, skipping the rm below and
+    # failing a suite that had actually passed.
+    [ -n "$D2PID" ] && kill "$D2PID" 2>/dev/null || true
+    [ -n "$D3PID" ] && kill "$D3PID" 2>/dev/null || true
+    rm -f "$SOCK" "$SOCK2" "$SOCK3" "$OUT" "$OUT.kill" "$OUT.re" "$OUT.a" \
+        "$OUT.b" "$OUT.via" "$OUT.dead" "$OUT.abort" "$OUT.m7" "$OUT.m7b"
+trap cleanup EXIT INT TERM
@@ @@ -145,4 +173,119 @@ grep -q "detached while reconnecting" "$OUT.abort" || { @@
+rm -f "$OUT.abort" "$SOCK2"
+# --- M7 Scenario A: kill the transport mid-session; the client must resume
+# by DELTA. The proxy is the transport, so killing it stands in for an ssh
+# drop while the daemon and its session carry on untouched.
+#
+# The counter is the real assertion, not the markers: a snapshot-served
+# resume renders "m7-after" perfectly well, so only `snapshots` holding
+# still across the tear proves the resume was a delta — which is the whole
+# milestone claim.
+set +e
+{ sleep 0.5; printf 'printf "m7-%%s\\n" before\n'; sleep 8; \
+  printf 'printf "m7-%%s\\n" after\n'; sleep 2.5; printf '\034'; } | \
+    XDG_RUNTIME_DIR=/nonexistent-mux-e2e timeout 40 \
+    "$MUX" --via "$MUXD proxy --sock $SOCK" > "$OUT.m7" 2>&1 &
+M7PID=$!
+set -e
+# Tear only once the session is provably live and idle: the marker is the
+# shell's own output, so seeing it means attached, drained, mid-sleep.
+wait_for "$OUT.m7" "m7-before" 20 || {
+    echo "e2e FAIL: m7 pre-tear output never arrived"; cat "$OUT.m7"; exit 1;
+}
+SNAPS_BEFORE=$("$MUXD" stats --sock "$SOCK" | sed -n 's/.*snapshots=\([0-9]*\).*/\1/p')
+PPID_PROXY=$(proxy_pid "$SOCK")
+[ -n "$PPID_PROXY" ] || { echo "e2e FAIL: could not find the proxy to kill"; exit 1; }
+kill -9 "$PPID_PROXY"
+# The tear must have hit the transport only; the daemon is what we are
+# proving survives, so say so out loud rather than inferring it later.
+kill -0 "$DPID" || { echo "e2e FAIL: the tear killed the daemon, not the proxy"; exit 1; }
+set +e
+wait "$M7PID"
+RC=$?
+set -e
+[ "$RC" -eq 0 ] || {
+    echo "e2e FAIL: m7 client exited $RC after a transport tear (want 0)"
+    cat "$OUT.m7"; exit 1;
+}
+grep -q "m7-before" "$OUT.m7" || { echo "e2e FAIL: m7 pre-tear output missing"; exit 1; }
+grep -q "m7-after" "$OUT.m7" || {
+    echo "e2e FAIL: m7 client did not resume after the transport was killed"
+    cat "$OUT.m7"; exit 1;
+}
+SNAPS_AFTER=$("$MUXD" stats --sock "$SOCK" | sed -n 's/.*snapshots=\([0-9]*\).*/\1/p')
+[ -n "$SNAPS_BEFORE" ] && [ -n "$SNAPS_AFTER" ] || {
+    echo "e2e FAIL: could not read the snapshots counter"; exit 1;
+}
+[ "$SNAPS_BEFORE" = "$SNAPS_AFTER" ] || {
+    echo "e2e FAIL: reconnect was served a snapshot ($SNAPS_BEFORE -> $SNAPS_AFTER), not a delta"
+    exit 1;
+}
+rm -f "$OUT.m7"
+# --- M7 Scenario B: kill the DAEMON under an attached client and start a new
+# one on the same path. The seq the client holds belongs to a session that no
+# longer exists, so the epoch fence must refuse it and serve a snapshot of the
+# fresh session instead. Needs its own daemon, since this one gets killed.
+"$MUXD" run --sock "$SOCK3" --shell /bin/sh &
+D3PID=$!
+i=0
+while [ ! -S "$SOCK3" ] && [ "$i" -lt 50 ]; do sleep 0.1; i=$((i+1)); done
+[ -S "$SOCK3" ] || { echo "e2e FAIL: restart-scenario socket never appeared"; exit 1; }
+set +e
+{ sleep 0.5; printf 'printf "m7b-%%s\\n" pre-restart\n'; sleep 8; \
+  printf 'printf "m7b-%%s\\n" post-restart\n'; sleep 3; printf '\034'; } | \
+    timeout 40 "$MUX" --sock "$SOCK3" > "$OUT.m7b" 2>&1 &
+M7BPID=$!
+set -e
+wait_for "$OUT.m7b" "m7b-pre-restart" 20 || {
+    echo "e2e FAIL: m7b pre-restart output never arrived"; cat "$OUT.m7b"; exit 1;
+}
+# SIGKILL leaves the socket file behind; the new daemon's stale-socket
+# recovery (ECONNREFUSED + S_ISSOCK -> unlink) is what lets it rebind here.
+kill -9 "$D3PID"
+sleep 0.5
+"$MUXD" run --sock "$SOCK3" --shell /bin/sh &
+D3PID=$!
+i=0
+while [ ! -S "$SOCK3" ] && [ "$i" -lt 50 ]; do sleep 0.1; i=$((i+1)); done
+[ -S "$SOCK3" ] || { echo "e2e FAIL: daemon did not rebind the stale socket"; exit 1; }
+set +e
+wait "$M7BPID"
+RC=$?
+set -e
+[ "$RC" -eq 0 ] || {
+    echo "e2e FAIL: m7b client exited $RC across a daemon restart (want 0)"
+    cat "$OUT.m7b"; exit 1;
+}
+grep -q "m7b-post-restart" "$OUT.m7b" || {
+    echo "e2e FAIL: m7b client did not resume into the restarted daemon"
+    cat "$OUT.m7b"; exit 1;
+}
+# The fresh session really is fresh: a new daemon means a new shell, so the
+# pre-restart marker cannot be in its grid. If it were, we would be looking
+# at a client that reconnected to something with the old session's history —
+# i.e. the epoch fence letting a stale seq through.
+if "$MUXD" dump --sock "$SOCK3" | grep -q "m7b-pre-restart"; then
+    echo "e2e FAIL: restarted daemon's grid still holds the pre-restart marker"
+    exit 1
+fi
+# Markers are blind to the kind of resume — a broken epoch fence handing back
+# a garbage delta would render "m7b-post-restart" just as well. Only the
+# counter sees it: the reconnecting client is this daemon's sole client, so a
+# snapshot in its stats is proof it was resynced from scratch rather than
+# resumed off a seq that belongs to a session that no longer exists.
+SNAPS_NEW=$("$MUXD" stats --sock "$SOCK3" | sed -n 's/.*snapshots=\([0-9]*\).*/\1/p')
+[ -n "$SNAPS_NEW" ] || { echo "e2e FAIL: could not read the restarted daemon's counters"; exit 1; }
+[ "$SNAPS_NEW" -ge 1 ] || {
+    echo "e2e FAIL: restarted daemon served no snapshot ($SNAPS_NEW); the stale seq was honoured"
+    exit 1;
+}
+rm -f "$OUT.m7b"
+echo "e2e OK"