c35a9687

test: e2e over QUIC — same semantics, same counters, new transport

a73x 2026-08-08 14:08

Commit message

test: e2e over QUIC — same semantics, same counters, new transport

An audit of the plan's Task 4 list against what Task 3's scenarios already
covered, plus the two that were missing and one small piece of observability.

**Reattach.** A detach leaves the session running, so coming back has to find
it — and has to be served as its OWN snapshot, because a client that left
deliberately holds nothing to resume from. Asserted on the counter, not the
rendering, since the two are indistinguishable on screen. It also checks the
slot is free again afterwards, which the new stats field makes visible.

**Kill -9 and restart, on the same paths.** Two things at once. The client
resumes into a session that no longer exists, which can only be a snapshot
under a new epoch — a delta there would mean a stale seq was honoured against
content that is gone. And the restarted daemon has to rebind the UDP port at
all: with SO_REUSEADDR removed, anything lingering from the killed process
shows up as a bind failure rather than as two daemons quietly sharing a port.
Both are asserted; the resume kind on the counter, the rebind on
/proc/net/udp.

**`muxd stats` gains `clients=N`.** Appended, never reordered, so the
harnesses that split on these pairs keep working. It is a gauge rather than a
counter — the only field here that can go down — and that is the point:
slot occupancy was previously unobservable from outside, which is exactly
what made the half-open finding hard to characterise. Task 5's policy
question wants this number.

Two test-quality fixes the mutations forced. Every scenario now names its
`--quic-idle-ms` explicitly instead of inheriting a default that another
scenario's timing depends on. And the refusal paths run under `timeout`: a
regression that ACCEPTS a refusal used to run a daemon forever and hang the
suite rather than failing it — restoring SO_REUSEADDR hung for ten minutes
before this, and now reports "exit 124, want 1" in ten seconds. A test that
hangs on regression is worth much less than one that fails.

Suite runtime 65-69s across three runs; sleeps with slack were trimmed to
pay for the additions.

Banked for Task 5, no action: `timeoutMs` does not short-circuit when a whole
frame is already buffered, so a frame that arrives with no packet behind it
can wait out the poll — up to ~100ms of render lag, one line to fix if it
ever shows up against a real RTT.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

src/server.zig

Old		New
@@ -1366,16 +1366,35 @@ pub const Server = struct {
1366	try proto.writeFrame(fd, .stats_reply, try self.statsText(&buf));	1366	try proto.writeFrame(fd, .stats_reply, try self.statsText(&buf));
1367	}	1367	}
1368		1368
1369	const stats_text_len = 256;	1369	pub const stats_text_len = 256;
		1370
		1371	/// How many client slots are occupied right now.
		1372	///
		1373	/// Not a counter but a gauge, and the only field here that can go down.
		1374	/// It exists because slot occupancy was previously unobservable from
		1375	/// outside: a QUIC connection that completes its handshake and never
		1376	/// attaches holds a slot until its idle timeout, and there was no way to
		1377	/// see that happening — or to see it clear — without attaching a debugger.
		1378	fn liveClients(self: *const Server) usize {
		1379	var n: usize = 0;
		1380	for (self.clients) \|slot\| {
		1381	if (slot != null) n += 1;
		1382	}
		1383	return n;
		1384	}
1370		1385
1371	fn statsText(self: *const Server, buf: []u8) ![]const u8 {	1386	fn statsText(self: *const Server, buf: []u8) ![]const u8 {
		1387	// Appended, never reordered: the bench harness and the e2e tests
		1388	// split on these key=value pairs.
1372	return std.fmt.bufPrint(	1389	return std.fmt.bufPrint(
1373	buf,	1390	buf,
1374	"seq={d} snapshots={d} snapshot_bytes={d} deltas={d} delta_bytes={d} snapshot_equiv_bytes={d}",	1391	"seq={d} snapshots={d} snapshot_bytes={d} deltas={d} delta_bytes={d}" ++
		1392	" snapshot_equiv_bytes={d} clients={d}",
1375	.{	1393	.{
1376	self.tracker.seq, self.stats.snapshots,	1394	self.tracker.seq, self.stats.snapshots,
1377	self.stats.snapshot_bytes, self.stats.deltas,	1395	self.stats.snapshot_bytes, self.stats.deltas,
1378	self.stats.delta_bytes, self.stats.snapshot_equiv_bytes,	1396	self.stats.delta_bytes, self.stats.snapshot_equiv_bytes,
		1397	self.liveClients(),
1379	},	1398	},
1380	);	1399	);
1381	}	1400	}
@@ -3734,3 +3753,44 @@ test "Server: a socket that replaced ours is not ours to delete" {
3734	try std.testing.expect(std.posix.S.ISSOCK(st.mode));	3753	try std.testing.expect(std.posix.S.ISSOCK(st.mode));
3735	std.fs.cwd().deleteFile(sock_path) catch {};	3754	std.fs.cwd().deleteFile(sock_path) catch {};
3736	}	3755	}
		3756
		3757	test "Server: stats reports live client slots, and the number comes down again" {
		3758	const alloc = std.testing.allocator;
		3759	var tmp = try TmpDir.make();
		3760	defer tmp.cleanup();
		3761	const dir_path = tmp.path();
		3762	const sock_path = try std.fmt.allocPrint(alloc, "{s}/gauge.sock", .{dir_path});
		3763	defer alloc.free(sock_path);
		3764
		3765	var srv = try Server.init(alloc, .{ .sock_path = sock_path, .shell = "/bin/sh" });
		3766	defer srv.deinit();
		3767
		3768	var buf: [Server.stats_text_len]u8 = undefined;
		3769	try std.testing.expect(std.mem.endsWith(u8, try srv.statsText(&buf), "clients=0"));
		3770
		3771	// Slots filled directly: what is under test is the gauge, not the
		3772	// machinery that fills them. Emptied by defer so that a failed
		3773	// assertion below still leaves teardown a valid Server — otherwise the
		3774	// real failure is buried under an abort from closing fd -1.
		3775	defer {
		3776	srv.clients[0] = null;
		3777	srv.clients[3] = null;
		3778	}
		3779	srv.clients[0] = .{ .sink = .{ .socket = -1 } };
		3780	srv.clients[3] = .{ .sink = .{ .socket = -1 } };
		3781	try std.testing.expect(std.mem.endsWith(u8, try srv.statsText(&buf), "clients=2"));
		3782
		3783	// A gauge, not a counter: the whole reason for adding it is watching
		3784	// occupancy clear, so it has to be able to go down.
		3785	srv.clients[0] = null;
		3786	try std.testing.expect(std.mem.endsWith(u8, try srv.statsText(&buf), "clients=1"));
		3787	srv.clients[3] = null;
		3788	try std.testing.expect(std.mem.endsWith(u8, try srv.statsText(&buf), "clients=0"));
		3789
		3790	// The fields the harnesses parse are still where they were: appended,
		3791	// never reordered.
		3792	const text = try srv.statsText(&buf);
		3793	try std.testing.expect(std.mem.startsWith(u8, text, "seq="));
		3794	try std.testing.expect(std.mem.indexOf(u8, text, " snapshots=") != null);
		3795	try std.testing.expect(std.mem.indexOf(u8, text, " snapshot_equiv_bytes=") != null);
		3796	}

test/e2e.sh

Old		New
@@ -54,7 +54,7 @@ cleanup() {
54	rm -f "$SOCK" "$SOCK2" "$SOCK3" "$SOCK4" "$SOCK4.second" "$QKEY" "$QKEY.bad" \	54	rm -f "$SOCK" "$SOCK2" "$SOCK3" "$SOCK4" "$SOCK4.second" "$QKEY" "$QKEY.bad" \
55	"$OUT" "$OUT.kill" "$OUT.re" "$OUT.a" \	55	"$OUT" "$OUT.kill" "$OUT.re" "$OUT.a" \
56	"$OUT.b" "$OUT.via" "$OUT.dead" "$OUT.abort" "$OUT.m7" "$OUT.m7b" \	56	"$OUT.b" "$OUT.via" "$OUT.dead" "$OUT.abort" "$OUT.m7" "$OUT.m7b" \
57	"$OUT.q" "$OUT.qc" "$OUT.qr" "$OUT.qa" "$QKEY.wrong"	57	"$OUT.q" "$OUT.qc" "$OUT.qr" "$OUT.qa" "$OUT.qk" "$QKEY.wrong"
58	}	58	}
59	trap cleanup EXIT INT TERM	59	trap cleanup EXIT INT TERM
60		60
@@ -311,7 +311,10 @@ rm -f "$OUT.m7b"
311	refuse() {	311	refuse() {
312	_want="$1"; shift	312	_want="$1"; shift
313	set +e	313	set +e
314	"$MUXD" run --sock "$SOCK4" --shell /bin/sh "$@" > "$OUT.q" 2>&1	314	# Timed out rather than trusted to exit: every case here is a refusal, so
		315	# a regression that ACCEPTS one would otherwise run a daemon forever and
		316	# hang the suite instead of failing it. 124 is a distinguishable answer.
		317	timeout 10 "$MUXD" run --sock "$SOCK4" --shell /bin/sh "$@" > "$OUT.q" 2>&1
315	_rc=$?	318	_rc=$?
316	set -e	319	set -e
317	[ "$_rc" -eq "$_want" ] \|\| {	320	[ "$_rc" -eq "$_want" ] \|\| {
@@ -367,12 +370,15 @@ grep -qi " $QHEX " /proc/net/udp \|\| {
367	# no error anywhere. This is the QUIC edition of the stale-socket story, and	370	# no error anywhere. This is the QUIC edition of the stale-socket story, and
368	# it needs two processes to test, which is why it lives here.	371	# it needs two processes to test, which is why it lives here.
369	set +e	372	set +e
370	"$MUXD" run --sock "$SOCK4.second" --shell /bin/sh \	373	# Same reasoning as refuse(): if the second daemon ever succeeds it runs
371	--quic "127.0.0.1:$QPORT" --key "$QKEY" > "$OUT.q" 2>&1	374	# until killed, so the failure has to be a timeout rather than a hang.
		375	timeout 10 "$MUXD" run --sock "$SOCK4.second" --shell /bin/sh \
		376	--quic "127.0.0.1:$QPORT" --key "$QKEY" --quic-idle-ms 15000 > "$OUT.q" 2>&1
372	RC=$?	377	RC=$?
373	set -e	378	set -e
374	[ "$RC" -eq 1 ] \|\| {	379	[ "$RC" -eq 1 ] \|\| {
375	echo "e2e FAIL: a second daemon took udp $QPORT (exit $RC, want 1)"; cat "$OUT.q"; exit 1;	380	echo "e2e FAIL: a second daemon took udp $QPORT (exit $RC, want 1; 124 means it bound and ran)"
		381	cat "$OUT.q"; exit 1;
376	}	382	}
377	grep -q "already listening" "$OUT.q" \|\| {	383	grep -q "already listening" "$OUT.q" \|\| {
378	echo "e2e FAIL: second daemon refused, but not with the already-listening message:"	384	echo "e2e FAIL: second daemon refused, but not with the already-listening message:"
@@ -401,7 +407,8 @@ kill -0 "$D4PID" \|\| { echo "e2e FAIL: --quic daemon died"; exit 1; }
401	# as the socket client does.	407	# as the socket client does.
402	set +e	408	set +e
403	{ printf 'printf "quic-%%s\\n" attach-ok\n'; sleep 2; printf '\034'; } \| \	409	{ printf 'printf "quic-%%s\\n" attach-ok\n'; sleep 2; printf '\034'; } \| \
404	timeout 30 "$MUX" "quic://127.0.0.1:$QPORT" --key "$QKEY" > "$OUT.qc" 2>&1	410	timeout 30 "$MUX" "quic://127.0.0.1:$QPORT" --key "$QKEY" \
		411	--quic-idle-ms 15000 > "$OUT.qc" 2>&1
405	RC=$?	412	RC=$?
406	set -e	413	set -e
407	[ "$RC" -eq 0 ] \|\| {	414	[ "$RC" -eq 0 ] \|\| {
@@ -414,6 +421,40 @@ grep -q "quic-attach-ok" "$OUT.qc" \|\| {
414	echo "e2e FAIL: daemon grid missing the quic:// client's output"; exit 1;	421	echo "e2e FAIL: daemon grid missing the quic:// client's output"; exit 1;
415	}	422	}
416		423
		424	# 1b. Reattach. A detach leaves the session running, so coming back must
		425	# find it — and must be served as a fresh attach (its own snapshot),
		426	# because a client that detached deliberately holds nothing to resume
		427	# from. The counter is what tells that apart from a delta.
		428	SNAPS_RA=$("$MUXD" stats --sock "$SOCK4" \| sed -n 's/.snapshots=$[0-9]$.*/\1/p')
		429	set +e
		430	{ printf 'printf "quic-%%s\\n" reattached\n'; sleep 2; printf '\034'; } \| \
		431	timeout 30 "$MUX" "quic://127.0.0.1:$QPORT" --key "$QKEY" \
		432	--quic-idle-ms 15000 > "$OUT.qc" 2>&1
		433	RC=$?
		434	set -e
		435	[ "$RC" -eq 0 ] \|\| {
		436	echo "e2e FAIL: quic:// reattach exited $RC (want 0)"; cat "$OUT.qc"; exit 1;
		437	}
		438	# The earlier marker is still on the grid this client was handed, which is
		439	# the session having survived the detach rather than a new shell.
		440	grep -q "quic-attach-ok" "$OUT.qc" \|\| {
		441	echo "e2e FAIL: quic:// reattach did not land in the existing session"
		442	cat "$OUT.qc"; exit 1;
		443	}
		444	grep -q "quic-reattached" "$OUT.qc" \|\| {
		445	echo "e2e FAIL: quic:// reattach could not run a command"; cat "$OUT.qc"; exit 1;
		446	}
		447	SNAPS_RA2=$("$MUXD" stats --sock "$SOCK4" \| sed -n 's/.snapshots=$[0-9]$.*/\1/p')
		448	[ "$((SNAPS_RA2 - SNAPS_RA))" -eq 1 ] \|\| {
		449	echo "e2e FAIL: reattach served $((SNAPS_RA2 - SNAPS_RA)) snapshots (want exactly 1)"
		450	exit 1;
		451	}
		452	# ...and the slot it held is free again now that it has gone.
		453	CLIENTS=$("$MUXD" stats --sock "$SOCK4" \| sed -n 's/.clients=$[0-9]$.*/\1/p')
		454	[ "$CLIENTS" = "0" ] \|\| {
		455	echo "e2e FAIL: $CLIENTS client slots still held after a clean detach (want 0)"; exit 1;
		456	}
		457
417	# 2. The key is checked, and a wrong one is refused loudly rather than	458	# 2. The key is checked, and a wrong one is refused loudly rather than
418	# retried forever. Nothing was ever established, so the reconnect loop	459	# retried forever. Nothing was ever established, so the reconnect loop
419	# must not engage — that is the never-established gate, over QUIC.	460	# must not engage — that is the never-established gate, over QUIC.
@@ -476,7 +517,7 @@ SNAPS_BEFORE=$("$MUXD" stats --sock "$SOCK4" \| sed -n 's/.snapshots=$[0-9]$.
476	[ -n "$SNAPS_BEFORE" ] \|\| { echo "e2e FAIL: could not read snapshots before the tear"; exit 1; }	517	[ -n "$SNAPS_BEFORE" ] \|\| { echo "e2e FAIL: could not read snapshots before the tear"; exit 1; }
477		518
478	set +e	519	set +e
479	{ printf 'printf "quic-%%s\\n" pre-tear\n'; sleep 12; printf '\034'; } \| \	520	{ printf 'printf "quic-%%s\\n" pre-tear\n'; sleep 9; printf '\034'; } \| \
480	timeout 40 "$MUX" "quic://127.0.0.1:$QPORT" --key "$QKEY" \	521	timeout 40 "$MUX" "quic://127.0.0.1:$QPORT" --key "$QKEY" \
481	--quic-idle-ms 1500 > "$OUT.qr" 2>&1 &	522	--quic-idle-ms 1500 > "$OUT.qr" 2>&1 &
482	QRPID=$!	523	QRPID=$!
@@ -489,7 +530,9 @@ wait_for "$OUT.qr" "quic-pre-tear" 20 \|\| {
489	# Held well past the 1500ms idle timeout, so the client cannot mistake it	530	# Held well past the 1500ms idle timeout, so the client cannot mistake it
490	# for a slow moment.	531	# for a slow moment.
491	kill -STOP "$D4PID"	532	kill -STOP "$D4PID"
492	sleep 4	533	# Twice the 1500ms idle timeout: long enough that the client cannot mistake
		534	# it for a slow moment, short enough not to pad the suite.
		535	sleep 3
493	kill -CONT "$D4PID"	536	kill -CONT "$D4PID"
494		537
495	set +e	538	set +e
@@ -515,8 +558,79 @@ SNAPS_AFTER=$("$MUXD" stats --sock "$SOCK4" \| sed -n 's/.snapshots=$[0-9]$.*
515	exit 1;	558	exit 1;
516	}	559	}
517		560
		561
		562	# 5. The daemon is killed outright and started again on the SAME paths — unix
		563	# socket and UDP port both. Two things are under test. The client must
		564	# resume into a session that no longer exists, which can only be a
		565	# snapshot under a new epoch (the stale-seq fence): a delta here would
		566	# mean the daemon honoured a seq belonging to content that is gone. And
		567	# the restarted daemon must be able to rebind the UDP port at all — there
		568	# is no SO_REUSEADDR any more, so anything lingering from the killed
		569	# process would show up as a bind failure rather than as silent sharing.
		570	set +e
		571	{ printf 'printf "quic-%%s\\n" pre-restart\n'; sleep 3; \
		572	printf 'printf "quic-%%s\\n" post-restart\n'; sleep 3; printf '\034'; } \| \
		573	timeout 40 "$MUX" "quic://127.0.0.1:$QPORT" --key "$QKEY" \
		574	--quic-idle-ms 1500 > "$OUT.qk" 2>&1 &
		575	QKPID=$!
		576	set -e
		577	wait_for "$OUT.qk" "quic-pre-restart" 20 \|\| {
		578	echo "e2e FAIL: quic restart client never got its pre-restart marker"
		579	cat "$OUT.qk"; exit 1;
		580	}
		581
		582	kill -9 "$D4PID"
		583	D4PID=""
		584	sleep 0.5
		585	"$MUXD" run --sock "$SOCK4" --shell /bin/sh \
		586	--quic "127.0.0.1:$QPORT" --key "$QKEY" --quic-idle-ms 15000 > "$OUT.q" 2>&1 &
		587	D4PID=$!
		588	i=0
		589	while [ ! -S "$SOCK4" ] && [ "$i" -lt 50 ]; do sleep 0.1; i=$((i+1)); done
		590	[ -S "$SOCK4" ] \|\| {
		591	echo "e2e FAIL: restarted --quic daemon never rebound its session socket"
		592	cat "$OUT.q"; exit 1;
		593	}
		594	# The UDP port really came back, and to THIS daemon. Without SO_REUSEADDR a
		595	# bind that collided would have failed loudly instead.
		596	grep -qi " $QHEX " /proc/net/udp \|\| {
		597	echo "e2e FAIL: restarted daemon did not rebind udp 127.0.0.1:$QPORT"
		598	cat "$OUT.q"; exit 1;
		599	}
		600	kill -0 "$D4PID" \|\| { echo "e2e FAIL: restarted --quic daemon died"; cat "$OUT.q"; exit 1; }
		601
		602	set +e
		603	wait "$QKPID"
		604	RC=$?
		605	set -e
		606	[ "$RC" -eq 0 ] \|\| {
		607	echo "e2e FAIL: quic client exited $RC across a daemon restart (want 0)"
		608	cat "$OUT.qk"; exit 1;
		609	}
		610	grep -q "quic-post-restart" "$OUT.qk" \|\| {
		611	echo "e2e FAIL: quic client did not resume into the restarted daemon"
		612	cat "$OUT.qk"; exit 1;
		613	}
		614	# A new daemon means a new shell, so the old marker cannot be in its grid.
		615	# If it were, we would be looking at a client that resumed off a seq
		616	# belonging to a session that no longer exists.
		617	if "$MUXD" dump --sock "$SOCK4" \| grep -q "quic-pre-restart"; then
		618	echo "e2e FAIL: restarted daemon's grid still holds the pre-restart marker"
		619	exit 1
		620	fi
		621	# And the counter, because markers are blind to how a resume was served: the
		622	# reconnecting client is this daemon's only client, so a snapshot in its
		623	# stats is proof it was resynced from scratch rather than handed a delta off
		624	# a stale seq.
		625	SNAPS_NQ=$("$MUXD" stats --sock "$SOCK4" \| sed -n 's/.snapshots=$[0-9]$.*/\1/p')
		626	[ -n "$SNAPS_NQ" ] \|\| { echo "e2e FAIL: could not read the restarted daemon's counters"; exit 1; }
		627	[ "$SNAPS_NQ" -ge 1 ] \|\| {
		628	echo "e2e FAIL: restarted daemon served no snapshot ($SNAPS_NQ); the stale seq was honoured"
		629	exit 1;
		630	}
		631
518	kill "$D4PID" 2>/dev/null \|\| true	632	kill "$D4PID" 2>/dev/null \|\| true
519	D4PID=""	633	D4PID=""
520	rm -f "$OUT.q" "$OUT.qc" "$OUT.qr" "$OUT.qa" "$QKEY" "$QKEY.bad" "$QKEY.wrong"	634	rm -f "$OUT.q" "$OUT.qc" "$OUT.qr" "$OUT.qa" "$OUT.qk" "$QKEY" "$QKEY.bad" "$QKEY.wrong"
521		635
522	echo "e2e OK"	636	echo "e2e OK"