test/e2e_02_predict.sh
Ref: Size: 39.6 KiB History
# shellcheck shell=sh
# e2e_02_predict.sh — sourced by test/e2e.sh after e2e_lib.sh. Scenarios run in
# the order they stand in; see the lib's header for what this file may
# assume and what it must register.
# M9 prediction: each scenario needs a session whose LINE DISCIPLINE it
# controls, so they cannot share the long-lived /bin/sh daemon.
SOCK5="${TMPDIR:-/tmp}/muxd-e2e-pred-$$.sock"
defer_sock "$SOCK5"
SOCK6="${TMPDIR:-/tmp}/muxd-e2e-pw-$$.sock"
defer_sock "$SOCK6"
SOCK7="${TMPDIR:-/tmp}/muxd-e2e-raw-$$.sock"
defer_sock "$SOCK7"
PWSH="${TMPDIR:-/tmp}/mux-e2e-pw-$$.sh"
defer_rm "$PWSH"
# The delayed link is a PIPELINE, and `--via CMD` is argv words the client
# execs with no shell anywhere: the pipeline has to live in a script `mux`
# runs, not in a string `mux` would have had to parse.
DVIA5="${TMPDIR:-/tmp}/mux-e2e-dvia5-$$.sh"
defer_rm "$DVIA5"
DVIA7="${TMPDIR:-/tmp}/mux-e2e-dvia7-$$.sh"
defer_rm "$DVIA7"
# DELAY_MS is read from the environment the client was given, so the delay
# stays each scenario's to set.
cat > "$DVIA5" <<EOF
#!/bin/sh
$DELAYPIPE | $MUX d proxy --sock $SOCK5 | $DELAYPIPE
EOF
cat > "$DVIA7" <<EOF
#!/bin/sh
$DELAYPIPE | $MUX d proxy --sock $SOCK7 | $DELAYPIPE
EOF
chmod +x "$DVIA5" "$DVIA7"
# M12 pty scenarios. Each needs a daemon whose grid size it owns: tp2
# resizes the grid twice and tp1's session is a scrollback-generating
# wrapper, so neither can share the long-lived /bin/sh daemon.
SOCK12="${TMPDIR:-/tmp}/muxd-e2e-tp2-$$.sock"
defer_sock "$SOCK12"
SOCK13="${TMPDIR:-/tmp}/muxd-e2e-tp1-$$.sock"
defer_sock "$SOCK13"
# tp1's session: a wrapper that fills the scrollback and then becomes cat.
TP1SH="${TMPDIR:-/tmp}/mux-e2e-tp1-$$.sh"
defer_rm "$TP1SH"
# ---- M9: prediction ---------------------------------------------------
#
# Every scenario here asserts COUNTERS, because the screen cannot tell a
# predicted glyph from an echoed one — that is the entire point of the
# feature — so a rendering test would pass just as happily with prediction
# switched off.
#
# The session shell is /bin/cat, not /bin/sh. An interactive bash prompt is
# icanon=0/echo=0 (readline echoes for itself), which is the ADAPTIVE tier,
# not the always-predict one; cat is a genuinely canonical reader and the
# only way to exercise `.always` here. That distinction cost a milestone's
# worth of confusion to learn and is why it is written down twice.
# Delay per direction. The round trip is twice this, and every assertion
# below about "before the daemon could have answered" is measured against
# it. Chosen against three constraints at once, which is why it is not a
# round number picked for looks:
#
# - Larger than delaypipe's built-in default of 150, and far enough from
# it to be told apart. The snapshot below is taken 450ms after the
# keystroke: the echo cannot have returned over this 600ms round trip,
# but WOULD have over the 300ms one the default gives. A DELAY_MS lookup
# that silently stopped working — leaving every scenario here measuring
# a path twice as fast as it claimed — fails that assertion rather than
# passing quietly.
# - Small enough that the round trip stays clear of the overlay's 1000ms
# expiry bound. This is a real ceiling, not a test artefact: delaypipe
# delays each chunk serially, so a burst's later keystrokes queue behind
# the earlier ones and age while they wait. At 400ms each way the burst
# scenario went to confirmed=1 — the predictions were expiring before
# their echo could arrive, exactly as the overlay is designed to do.
# - Leaving margin on both sides, so neither is a race under load.
PDELAY=300
# Own file, not the client capture: the daemon's fd survives the client's truncation and would write into the replayed stream.
start_daemon "$SOCK5" "$OUT.p1.d" "prediction daemon never bound" --shell /bin/cat
D5PID=$DPID
# 1. Line mode: the glyph is on screen before the round trip could have
# delivered it.
set +e
{ sleep 2; printf 'z'; sleep 4; printf '\034\034'; } | \
DELAY_MS="$PDELAY" MUX_PREDICT_STATS=1 timeout 40 "$MUX" \
--via "$DVIA5" \
> "$OUT.p1" 2> "$OUT.p1.err" &
P1PID=$!
defer_kill "$P1PID"
set -e
# Snapshot the client output 450ms after the keystroke: well
# before this 600ms round trip could bring the pty own echo back, and well
# AFTER the 300ms round trip delaypipe default of 150 would have given.
sleep 2.45
cp "$OUT.p1" "$OUT.p1.early" 2>/dev/null || true
set +e
wait "$P1PID"
RC=$?
set -e
rc0 "line-mode prediction client exited $RC" "$OUT.p1" "$OUT.p1.err"
# The underlined glyph, in the snapshot taken before the echo could arrive.
# This is the whole claim of the milestone, as an effect rather than a
# counter: the character was on the screen while it was still in flight.
grep -q "$(printf '\033\[4mz')" "$OUT.p1.early" || {
echo "e2e FAIL: no predicted glyph 450ms after the keystroke (RTT is $((PDELAY * 2))ms)"
echo "--- early snapshot ---"; cat -v "$OUT.p1.early"; exit 1;
}
# ...and the daemon's own answer was NOT there yet, which is what makes the
# line above mean anything. A delta paints a row with ECH before its
# content; the prediction never does.
if grep -q "$(printf '\033\[80X\033\[0mz')" "$OUT.p1.early"; then
echo "e2e FAIL: the echo arrived within 450ms, so the path is not $((PDELAY * 2))ms;"
echo " either DELAY_MS was ignored (default is 150) or delaypipe is not delaying"
exit 1
fi
want_stat "$OUT.p1.err" contradicted 0 "line mode"
CONF=$(predict_stat "$OUT.p1.err" confirmed)
[ -n "$CONF" ] && [ "$CONF" -ge 1 ] || {
echo "e2e FAIL: line mode: confirmed=$CONF, want >=1"; grep "^predict " "$OUT.p1.err"; exit 1;
}
# The session really received the keystroke: cat echoed it into the grid.
"$MUX" d dump --sock "$SOCK5" | grep -q "z" || {
echo "e2e FAIL: line mode: daemon grid never saw the keystroke"; exit 1;
}
assert_converged "$OUT.p1" "$SOCK5" "line-mode prediction"
# 2. A burst outrunning the round trip. This is reconcile v2's reason to
# exist: under the old rule every character after the first was judged by
# a frame built before it was typed, read as a contradiction, and the
# whole queue was flushed once per round trip. If anyone regresses the
# three-way judgment, this scenario is what says so.
set +e
{ sleep 2; for c in b u r s t; do printf '%s' "$c"; sleep 0.25; done; \
sleep 4; printf '\034\034'; } | \
DELAY_MS="$PDELAY" MUX_PREDICT_STATS=1 timeout 40 "$MUX" \
--via "$DVIA5" \
> "$OUT.pb" 2> "$OUT.pb.err"
RC=$?
set -e
rc0 "burst client exited $RC" "$OUT.pb" "$OUT.pb.err"
# Five keystrokes typed at 250ms into a 600ms round trip: three are always
# outstanding at once (at t=600, when the first is confirmed, the keys typed
# at 0, 250 and 500 are all in flight). `made` is asserted too, so two
# keystrokes arriving in one read (which would be suppressed as a multi-byte
# chunk, not predicted) fails loudly instead of quietly weakening the test.
#
# The interval is 250 and not 200 because of a bound that has to be COMPUTED
# rather than eyeballed. delaypipe delivers one chunk per DELAY_MS serially
# in EACH direction, so keystrokes closer together than DELAY_MS queue, and
# each one ages (DELAY_MS - interval) longer than the one before it:
#
# age at which keystroke k is confirmed = 2*DELAY_MS + k*(DELAY_MS - interval)
#
# At interval=200 the fifth keystroke lands at 600 + 4*100 = 1000ms, which is
# exactly predict.expire_after_ms: a coin flip, not a margin. It failed 4 runs
# in 20 on an idle machine. At 250 it is 600 + 4*50 = 800ms, measured at
# 798-799 across 12 runs. The same formula puts the SECOND keystroke at
# 1000ms when DELAY_MS=400, which is the confirmed=1 collapse noted above.
want_stat "$OUT.pb.err" made 5 "burst"
# `confirmed` is a LIVENESS counter and the only timing-dependent one here:
# the fifth confirmation can still be in flight when the detach byte lands,
# which is why it is a floor rather than an equality. Observed 1 run in 3
# under load (review runs at 715f71b, 2026-08-09), and decisions.md records
# the same counter coming up one short on the M9 WAN burst (made=55
# confirmed=54 abandoned=1, the mode-transition churn flush).
#
# `contradicted` and `expired` stay EXACT. They are the correctness claims —
# a prediction judged wrong, or one that aged out unjudged, is a defect
# whenever it happens, and nothing about timing makes either acceptable.
want_stat_ge "$OUT.pb.err" confirmed 4 "burst"
# `expired` first: an expiry runs through the same abandonAll tail as a
# refutation and increments BOTH counters, so contradicted >= expired always.
# Asserting contradicted first reports an age-out as "a prediction judged
# wrong" — a different defect with a different cause, and the reason this
# flake was first investigated as a reconcile bug.
want_stat "$OUT.pb.err" expired 0 "burst"
want_stat "$OUT.pb.err" contradicted 0 "burst"
"$MUX" d dump --sock "$SOCK5" | grep -q "burst" || {
echo "e2e FAIL: burst: daemon grid does not hold the typed text"; exit 1;
}
assert_converged "$OUT.pb" "$SOCK5" "burst"
softkill "$D5PID" || true
D5PID=""
# 3. A password prompt. The session is canonical with echo OFF from the
# moment it starts, so the whole run sits in the tier that predicts
# nothing at all.
cat > "$PWSH" <<'PWEOF'
#!/bin/sh
stty -echo
printf 'pw-ready\n'
read secret
printf 'pw-len-%s\n' "${#secret}"
# Blocks forever rather than exiting: the shell exiting would end the
# session, and the daemon would be gone before its grid could be read.
read _hold
PWEOF
chmod +x "$PWSH"
start_daemon "$SOCK6" "$OUT.pw.d" "password daemon never bound" --shell "$PWSH"
D6PID=$DPID
# One character per write, and NOT as a single `printf hunter2`. A whole
# word in one write reaches the client as a multi-byte chunk, which is
# refused for being multi-byte before the tier is ever consulted — so the
# scenario would pass with the password tier completely broken. It did:
# mapping echo-off canonical to the always-predict tier survived this
# scenario until the typing was split up.
set +e
{ sleep 2; for c in h u n t e r 2; do printf '%s' "$c"; sleep 0.2; done; \
printf '\n'; sleep 2; printf '\034\034'; } | \
MUX_PREDICT_STATS=1 timeout 40 "$MUX" --sock "$SOCK6" > "$OUT.pw" 2> "$OUT.pw.err"
RC=$?
set -e
rc0 "password client exited $RC" "$OUT.pw" "$OUT.pw.err"
# Nothing was shown, and nothing was even attempted: echo-off canonical is
# the tier where a prediction must never be MADE, not merely never painted,
# because a made prediction sits in a buffer the overlay paints from.
want_stat "$OUT.pw.err" made 0 "password"
want_stat "$OUT.pw.err" displayed 0 "password"
# The effect, not the counter: the secret appears nowhere in the bytes the
# client wrote — either stream. The stats line cannot carry a secret, but
# checking both costs one word and the claim is about everything the client
# emitted, not about the half of it that happens to be the screen.
if grep -q "hunter2" "$OUT.pw" "$OUT.pw.err"; then
echo "e2e FAIL: the password appears in the client's terminal output"; exit 1
fi
# ...and the run was not vacuous: the shell really received all seven
# characters, so the absence above is prediction declining rather than
# nothing having been typed.
"$MUX" d dump --sock "$SOCK6" | grep -q "pw-len-7" || {
echo "e2e FAIL: the password never reached the shell; the absence above proves nothing"
"$MUX" d dump --sock "$SOCK6"; exit 1;
}
assert_converged "$OUT.pw" "$SOCK6" "password"
softkill "$D6PID" || true
D6PID=""
# 4. Raw mode: display is earned, then lost to a keystroke the application
# swallows. rawmode echoes like an editor in insert mode until it is sent
# 0x00, after which it consumes input and prints nothing.
#
# Through delaypipe, like leg 1, and for the same reason leg 4b below is
# NOT: the overlay measures its own round trip and hides everything on a
# path faster than `predict.local_below_ms`. This leg is about the
# earn/lose policy, which only a remote path lets through to the screen.
# Keys are 1s apart so each confirm (600ms) lands before the next key —
# at 0.5s `c` would find only `a` confirmed and stay hidden.
start_daemon "$SOCK7" "$OUT.rw.d" "rawmode daemon never bound" --shell "$RAWMODE"
D7PID=$DPID
set +e
{ sleep 2; for c in a b c; do printf '%s' "$c"; sleep 1; done; \
printf '\000'; sleep 1; printf 'j'; sleep 2.5; printf '\034\034'; } | \
DELAY_MS="$PDELAY" MUX_PREDICT_STATS=1 timeout 40 "$MUX" \
--via "$DVIA7" \
> "$OUT.rw" 2> "$OUT.rw.err"
RC=$?
set -e
rc0 "rawmode client exited $RC" "$OUT.rw" "$OUT.rw.err"
# Four printable keystrokes made predictions; the 0x00 that switched modes
# is not printable and was refused.
want_stat "$OUT.rw.err" made 4 "raw mode"
want_stat "$OUT.rw.err" suppressed 1 "raw mode"
# a and b are invisible while promotion is earned; c is the first painted,
# and j is painted because by then display had been earned.
want_stat "$OUT.rw.err" displayed 2 "raw mode"
want_stat "$OUT.rw.err" confirmed 3 "raw mode"
# j was swallowed: no frame ever answered the cell it was drawn in, so the
# expiry bound retired it rather than leaving a phantom glyph on screen for
# the rest of the session.
want_stat "$OUT.rw.err" expired 1 "raw mode"
want_stat "$OUT.rw.err" contradicted 1 "raw mode"
want_stat "$OUT.rw.err" abandoned 1 "raw mode"
# Every prediction accounted for: made = confirmed + abandoned + pending,
# and the queue is empty at exit, so nothing survived unexplained.
RW_MADE=$(predict_stat "$OUT.rw.err" made)
RW_CONF=$(predict_stat "$OUT.rw.err" confirmed)
RW_ABND=$(predict_stat "$OUT.rw.err" abandoned)
[ "$((RW_MADE - RW_CONF - RW_ABND))" -eq 0 ] || {
echo "e2e FAIL: raw mode: $((RW_MADE - RW_CONF - RW_ABND)) predictions unaccounted for"
grep "^predict " "$OUT.rw.err"; exit 1;
}
# The counters say `j` expired; this says the screen agrees. An expiry that
# retired the prediction in the book but left its underlined glyph painted is
# exactly the phantom the styled half of this check exists to catch.
assert_converged "$OUT.rw" "$SOCK7" "raw mode"
# 4b. The same keystrokes on the bare socket: a local daemon answers in
# about a millisecond, and that is the one path on which a prediction
# can only ever be SEEN when it is wrong. Display is still earned (the
# counters say so) and nothing is painted — `local` is where `c` and
# `j` went instead of `displayed`. `j` still expires: the queue keeps
# judging while hidden, which is how the gate would lift again on a
# box that got slow. The screen check is the point: no phantom `j`,
# because there was never an underlined one to leave behind.
#
# A fresh daemon: the rawmode above has taken its 0x00 and echoes
# nothing now, and the leg needs the insert-mode half back.
softkill "$D7PID" || true
wait "$D7PID" 2>/dev/null || true
rm -f "$SOCK7"
start_daemon "$SOCK7" "$OUT.rwl.d" "rawmode local daemon never bound" --shell "$RAWMODE"
D7PID=$DPID
set +e
{ sleep 2; for c in a b c; do printf '%s' "$c"; sleep 0.5; done; \
printf '\000'; sleep 0.5; printf 'j'; sleep 2.5; printf '\034\034'; } | \
MUX_PREDICT_STATS=1 timeout 40 "$MUX" --sock "$SOCK7" > "$OUT.rwl" 2> "$OUT.rwl.err"
RC=$?
set -e
rc0 "rawmode local client exited $RC" "$OUT.rwl" "$OUT.rwl.err"
want_stat "$OUT.rwl.err" made 4 "raw mode local"
want_stat "$OUT.rwl.err" confirmed 3 "raw mode local"
want_stat "$OUT.rwl.err" displayed 0 "raw mode local"
want_stat "$OUT.rwl.err" local 2 "raw mode local"
want_stat "$OUT.rwl.err" expired 1 "raw mode local"
grep -q "$(printf '\033\[4m')" "$OUT.rwl" && {
echo "e2e FAIL: raw mode local: an underlined prediction reached the screen"
cat -v "$OUT.rwl"; exit 1;
}
assert_converged "$OUT.rwl" "$SOCK7" "raw mode local"
softkill "$D7PID" || true
D7PID=""
# 5. A transport torn down with predictions outstanding. The overlay must
# come back empty: what was queued was predicted against a connection
# that no longer exists.
start_daemon "$SOCK5" "$OUT.pr.d" "reconnect daemon never bound" --shell /bin/cat
D5PID=$DPID
set +e
{ sleep 2; printf 'p'; sleep 5; printf 'q'; sleep 4; printf '\034\034'; } | \
DELAY_MS="$PDELAY" MUX_PREDICT_STATS=1 timeout 60 "$MUX" \
--via "$DVIA5" \
> "$OUT.pr" 2> "$OUT.pr.err" &
PRPID=$!
defer_kill "$PRPID"
set -e
# Kill the transport a tenth of a second after the keystroke, while its
# prediction is still outstanding — the round trip is 600ms, so it cannot
# have been judged yet. By comm+pid: a pattern kill on the socket path
# would take out the delayed-link script that names it too.
sleep 2.1
PP=$(proxy_pid "$SOCK5")
[ -n "$PP" ] || {
echo "e2e FAIL: no proxy to tear down"
cat "$OUT.pr" "$OUT.pr.err" 2>/dev/null; exit 1; }
kill -9 "$PP"
set +e
wait "$PRPID"
RC=$?
set -e
rc0 "reconnect client exited $RC" "$OUT.pr" "$OUT.pr.err"
# The queue is empty at exit, by attribution rather than by inspection:
# nothing predicted is still outstanding, and the flush that dropped the
# in-flight prediction was not counted as anybody being wrong.
PR_MADE=$(predict_stat "$OUT.pr.err" made)
PR_CONF=$(predict_stat "$OUT.pr.err" confirmed)
PR_ABND=$(predict_stat "$OUT.pr.err" abandoned)
[ -n "$PR_MADE" ] || {
echo "e2e FAIL: reconnect: no predict stats"
cat "$OUT.pr" "$OUT.pr.err" 2>/dev/null; exit 1; }
[ "$((PR_MADE - PR_CONF - PR_ABND))" -eq 0 ] || {
echo "e2e FAIL: reconnect: $((PR_MADE - PR_CONF - PR_ABND)) predictions outlived the tear"
grep "^predict " "$OUT.pr.err"; exit 1;
}
# And it was the reconnect that dropped it, not the expiry bound catching it
# a second later. Without this the scenario cannot tell the two apart: the
# resync after a reconnect flushes as well, so the outcome looks identical
# either way and removing the reconnect's own flush passes unnoticed. It
# did, until this line.
want_stat "$OUT.pr.err" expired 0 "reconnect"
[ -n "$PR_ABND" ] && [ "$PR_ABND" -ge 1 ] || {
echo "e2e FAIL: reconnect: nothing was abandoned ($PR_ABND), so no prediction was"
echo " outstanding when the transport died and the tear proves nothing"
grep "^predict " "$OUT.pr.err"; exit 1;
}
# The session survived and the client resumed into it: the keystroke typed
# after the tear reached the shell.
"$MUX" d dump --sock "$SOCK5" | grep -q "q" || {
echo "e2e FAIL: reconnect: the post-tear keystroke never reached the session"
"$MUX" d dump --sock "$SOCK5"; exit 1;
}
assert_converged "$OUT.pr" "$SOCK5" "reconnect flush"
softkill "$D5PID" || true
D5PID=""
# ---- M12: ptyclient fixture controls ----------------------------------
# Before any scenario trusts the fixture, prove both directions: a
# roundtrip over plain /bin/cat (the pty line discipline's own echo
# answers — no mux anywhere, so a failure here is the FIXTURE'S), and an
# expect that cannot match, which must time out, exit nonzero, and say
# what it did see. A check that cannot fail proves nothing.
PCLOG="$OUT.pc.log"
set +e
timeout 30 "$PTYCLIENT" --cols 80 --rows 24 --out "$OUT.pc" --err "$OUT.pc.err" -- /bin/cat > "$PCLOG" 2>&1 <<'EOF'
send hello\n
expect hello 10000
send \x04
waitexit 10000
EOF
RC=$?
set -e
rc0 "ptyclient roundtrip over cat exited $RC:" "$PCLOG"
grep -q "hello" "$OUT.pc" || {
echo "e2e FAIL: ptyclient capture missing the pty echo"; cat -v "$OUT.pc"; exit 1; }
# The must-fail leg. 500ms: nothing is being waited FOR — the needle never
# arrives by construction — so the deadline only bounds the control's cost.
set +e
timeout 30 "$PTYCLIENT" --cols 80 --rows 24 --out "$OUT.pc2" --err "$OUT.pc2.err" -- /bin/cat > "$PCLOG.2" 2>&1 <<'EOF'
expect never-going-to-match 500
EOF
RC=$?
set -e
# The literal, not merely nonzero: 3 is EXIT_TIMEOUT, and the tp1/tp2
# scenarios read that code to tell a missed needle apart from a client that
# died or a script the fixture refused. A bare -ne 0 would pass just as
# happily on a usage error that never ran the script at all.
[ "$RC" -eq 3 ] || {
echo "e2e FAIL: ptyclient expect control exited $RC, want 3 (EXIT_TIMEOUT)"; cat "$PCLOG.2"; exit 1; }
grep -q "did not arrive" "$PCLOG.2" || {
echo "e2e FAIL: ptyclient timeout fired but never said what it saw"; cat "$PCLOG.2"; exit 1; }
# The stderr split, end to end: the client's predict stats must land in the
# .err sibling and NOT in the capture the convergence machinery diffs. Only
# a child that writes to BOTH streams can tell a working split from a child
# that simply never wrote to stderr.
set +e
timeout 30 "$PTYCLIENT" --cols 80 --rows 24 --out "$OUT.pc3" --err "$OUT.pc3.err" -- \
/bin/sh -c 'printf pc-stdout; printf pc-stderr 1>&2' > "$PCLOG.3" 2>&1 <<'EOF'
expect pc-stdout 10000
waitexit 10000
EOF
RC=$?
set -e
rc0 "ptyclient stderr-split leg exited $RC:" "$PCLOG.3"
grep -q "pc-stdout" "$OUT.pc3" || {
echo "e2e FAIL: ptyclient capture missing the child's stdout"; cat -v "$OUT.pc3"; exit 1; }
! grep -q "pc-stderr" "$OUT.pc3" || {
echo "e2e FAIL: the child's stderr leaked into the capture"; cat -v "$OUT.pc3"; exit 1; }
grep -q "pc-stderr" "$OUT.pc3.err" || {
echo "e2e FAIL: the child's stderr reached neither the capture nor --err"
cat -v "$OUT.pc3.err"; exit 1; }
rm_swept "$OUT.pc" "$OUT.pc.err" "$OUT.pc2" "$OUT.pc2.err" "$OUT.pc3" "$OUT.pc3.err" \
"$PCLOG" "$PCLOG.2" "$PCLOG.3"
ok "ptyclient controls: pty echo roundtrips, impossible expect fails loudly, stderr stays off the capture"
# ---- M12 tp2: resize under a real tty (campaign row 7's branch) --------
# Two clients, two convergence points, because convergence is a scenario's
# LAST act after detach.
#
# tp2a is the suite's first attach at a size that is not the non-tty 80x24
# default: it pins the size a real tty reports reaching the daemon, and
# convergence measured against that size rather than the default. It does
# NOT exercise the snapshot resize prefix — an attach applies the
# attacher's size to the grid BEFORE the answering resync, so the prefix
# always equals the size the replica already has and the client's
# resize-on-prefix guard never fires. Verified by probe, not by reading.
#
# tp2b is the leg that catches row 7 (the prefix ignored), and it only
# catches it because of WHERE the damage can show. DELTA STREAMING
# bypasses the replica: those rows go straight to the tty, absolutely
# addressed and clipped to the tty, so a stale replica cannot spoil them
# and a wrong row is healed by the next delta that re-addresses it. The
# damage therefore has to survive to a FULL REPAINT, which is the paint
# that reads the replica back out and clips at min(replica, tty) — still
# on screen by then, and in a region the stale geometry cannot reach.
# Hence: fill the screen, put a 95-wide row on it, then winch UP to
# 100x30. With the prefix ignored the replica stays 90x28 and the repaint
# answering the resize reproduces the OLD geometry.
start_daemon "$SOCK12" "$OUT.tp2.d" "tp2 daemon never bound" --shell /bin/sh
D12PID=$DPID
# Two rules govern every script below, and breaking either one produces a
# capture that is honestly SHORT rather than wrong — a divergence whose
# diff blames the product for a scenario's mistake:
#
# A marker must not appear in the command line that produces it. The
# shell echoes what is typed, so `echo tp2-claim` paints the needle
# before it has run anything; the expect then matches the echo and the
# script detaches while the OUTPUT is still in flight. `printf 'tp2-%s'
# claim` keeps the literal off the typed line, so a hit is proof the
# shell ran it — the same argument-splitting trick the M5 scenario uses.
#
# The LAST thing expected must be the prompt, not the output. A shell
# writes its result and its next prompt as two separate writes, so a
# script that detaches on the result can stop one frame short of the
# grid the daemon holds. PS1 is set to a marker here purely so that
# prompt is assertable: it is the quiesce signal that would otherwise
# have to be a sleep.
set +e
hostroom pty2a
XDG_STATE_HOME="$HOSTROOM" timeout 40 "$PTYCLIENT" --cols 100 --rows 30 --out "$OUT.tp2a" --err "$OUT.tp2a.err" -- \
"$MUX" --sock "$SOCK12" > "$OUT.tp2a.log" 2>&1 <<'EOF'
expect \x1b[?1049h 10000
send PS1=tp2rdy@\n
# Twice, and the fixture's cursor is what makes that meaningful: the first
# hit is the ECHO of the assignment line, the second is the prompt the
# shell painted from it. Only the second proves PS1 took effect.
#
# Counting sentinel occurrences is safe HERE and nowhere else in tp2: this
# grid is nearly empty, so nothing scrolls, no repaint splits across
# deltas, and the two hits keep the order they were painted in. tp2b runs
# on a full screen and cannot assume any of that — see the hazard note
# there before copying this shape.
expect tp2rdy@ 10000
expect tp2rdy@ 10000
send printf 'tp2-%s\\n' claim\n
expect tp2-claim 10000
expect tp2rdy@ 10000
settle 500 15000
send \x1c\x1c
waitexit 10000
EOF
RC=$?
set -e
rc0 "tp2a ptyclient exited $RC:" "$OUT.tp2a.log"
assert_converged_pty "$OUT.tp2a" "$SOCK12" "pty attach at 100x30" 100 30
# Resize UP, not down: a replica LARGER than the grid is invisible (the
# paint clips at min(replica, tty) either way), so only a replica left
# too SMALL has anywhere to show. `seq 1 60` fills the screen so growing
# to 30 rows has to pull history down rather than append blanks, and the
# 95-wide row is wider than the stale 90 and narrower than the new 100.
#
# PS1 is still tp2rdy@ from tp2a: same daemon, same shell, and the
# assignment outlived the client that typed it. Note where the prompt
# expects sit — the sentinel is re-emitted whenever the prompt ROW is
# repainted, including by the echo of a command typed at it, so each
# trailing prompt is consumed BEFORE the resize. That leaves the expect
# after `resize` satisfiable only by the answering snapshot's repaint.
#
# THE HAZARD, and it is worth stating precisely because three shapes of
# this scenario died to it: on a FULL screen the output scrolls, and the
# daemon can flush the new prompt row in one delta and the scrolled rows
# in a LATER one. Prompt-row sentinels therefore have NO reliable order
# relative to the output rows around them, and counting their occurrences
# fails in both directions. Too late: the sentinel's bytes arrive before
# the output needle that precedes it in the script, the needle's match
# consumes the cursor past them, and the next sentinel expect waits on a
# shell that has gone idle — a hang. Too early: a post-resize sentinel
# matches bytes from BEFORE the resize and the client detaches mid-repaint,
# leaving a capture in the old geometry. Measured under load, that second
# one is the dangerous one: it produces a diff byte-identical to row 7's
# — missing rows and a still-wrapped wide row — so a healthy build fails
# wearing the bug's own signature.
#
# The immune forms, and the only ones used below. UNIQUE OUTPUT MARKERS
# (`fill-done`, `00007`) for progress: each is printed once, by a command
# whose typed line cannot contain it, so its arrival is a fact about the
# session rather than about paint order. Occurrence-counting of a
# REPAINTED row — which is what a prompt sentinel is — is not a
# substitute, and is what the three earlier shapes of this leg died to.
#
# `expect 00007` earns its place as more than a marker: its arrival proves
# the daemon processed the wide row while the grid was still 90 columns,
# which is what puts the damage on the screen the resize will repaint from.
#
# After the resize the leg asserts TWO different things, and neither one
# implies the other:
# the 91-zero WITNESS proves the reflowed paint ARRIVED. 91 contiguous
# zeros cannot exist at width 90 — the row is painted wrapped there — so
# this needle is structurally incapable of firing early, whatever order
# the rows came in. It is also not the load risk an extra deadline looks
# like: this needle went 150/150 under 16-way load, at the same load that
# broke the two prompt-sentinel shapes 9 and 15 failures in 150.
# `settle` then proves nothing further is IN FLIGHT. The witness alone
# would permit a detach mid-repaint, with the rest of the screen still
# on its way.
# Dropping the witness and keeping only settle looks equivalent and is
# not: settle succeeds on 500ms of silence measured from its own start and
# never requires that a single byte arrived, so a healthy build whose
# repaint has not BEGUN inside that window satisfies it having observed
# nothing at all — leaving a pre-resize capture that diffs exactly like
# row 7. It would also be used outside its own precondition, which is that
# the session is already idle; straight after a resize the answering
# snapshot has not been generated yet.
set +e
hostroom pty2b
XDG_STATE_HOME="$HOSTROOM" timeout 40 "$PTYCLIENT" --cols 90 --rows 28 --out "$OUT.tp2b" --err "$OUT.tp2b.err" -- \
"$MUX" --sock "$SOCK12" > "$OUT.tp2b.log" 2>&1 <<'EOF'
expect \x1b[?1049h 10000
send seq 1 60; printf 'fill-%s\\n' done\n
expect fill-done 10000
send printf '%095d\\n' 7\n
expect 00007 10000
resize 100 30
expect 0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 10000
settle 500 15000
send \x1c\x1c
waitexit 10000
EOF
RC=$?
set -e
rc0 "tp2b ptyclient exited $RC:" "$OUT.tp2b.log"
# Graded against the mutation (row 7: the prefix resize commented out) in
# this exact shape: 10 targeted runs, 10 red, plus a red full-suite run. What
# dies is the WITNESS, above — a replica still 90 wide cannot paint 91
# contiguous zeros, so the leg fails on the fixture timeout at that verb
# ("expect ... did not arrive within 10000ms", exit 3) and the RC check
# reports it in ~10s.
#
# This assertion is the backstop rather than the tripwire, and it is kept
# because it sees what the witness does not: before the witness existed
# the same build reached here and diverged with both halves of the stale
# geometry in one diff — a 30-row grid coming back short its top rows,
# because a 28-row replica has only 28 rows to repaint from, and the
# 95-wide row still WRAPPED at 90 into "...0" + "00007" while the daemon
# had rejoined it into one row at 100.
assert_converged_pty "$OUT.tp2b" "$SOCK12" "pty resize mid-session" 100 30
rm_swept "$OUT.tp2a" "$OUT.tp2a.err" "$OUT.tp2a.log" \
"$OUT.tp2b" "$OUT.tp2b.err" "$OUT.tp2b.log" "$OUT.tp2.d"
# Closed here like every other per-scenario daemon, not left to the trap:
# tp1 runs below and would otherwise share the box with a daemon nobody is
# watching, holding a session whose shell is still alive.
softkill "$D12PID" || true
D12PID=""
ok "a pty client resizes: snapshot prefix applied, winch follows the tty"
# ---- M12 tp1: reconnect while scrolled (row 18 + the 412f38f pin) ------
# The session fills its scrollback BY ITSELF and then execs cat: no
# pre-tear typing means no pre-tear predictions, so the predict counters at
# exit belong entirely to the one post-resync keystroke — which is what
# lets them isolate the 412f38f fix. cat leaves the pty canonical and
# echoing, the `.always` tier, so a real keystroke predicts.
cat > "$TP1SH" <<'EOF'
#!/bin/sh
seq 1 100
exec /bin/cat
EOF
chmod +x "$TP1SH"
start_daemon "$SOCK13" "$OUT.tp1.d" "tp1 daemon never bound" --shell "$TP1SH"
D13PID=$DPID
# Attach only after seq finished: the client must be served a snapshot of
# the TAIL, so the scrollback page it fetches later is content it has never
# been sent. Observed at 80x24: the attach snapshot carries rows 78..100 and
# the session goes quiet there (cat prints no prompt), history_rows is 77,
# and page 1 is rows 54..77. Every needle below is chosen against those
# numbers, which is why this wait is part of the scenario and not a nicety.
wait_grid "$SOCK13" "100" "tp1 session never finished seq"
# MUX_PREDICT_STATS is exported into the FIXTURE's environment, not spelled
# as a `VAR=x cmd` prefix on the client: the fixture execve()s its child
# with an absolute path and no PATH search (src/server/pty.zig), so `env` would
# have to be an absolute path too, and the child inherits this environment
# anyway. Getting that wrong costs a client that exits 127 before the first
# verb, which the fixture reports as "closed the pty before ... matched".
#
# EXACTLY ONE keystroke, and that is the load-bearing choice in this script.
# offerKeystroke refuses any stdin chunk that is not one byte — an escape
# sequence, a multi-byte character, or a paste — and counts it suppressed.
# Consecutive `send`s have no barrier between them, so the client's next
# read can pick up two of them at once and predict neither: measured at
# idle, a three-keystroke version of this leg scored made=3 on one run and
# made=2/suppressed=1 on the next. Under load all three could coalesce,
# which would read as made=0 — the mutation's own signature. One send
# cannot coalesce with anything, so the counters below are exact.
#
# suppressed=0 therefore says more than "the keystroke predicted", and the
# rest is worth writing down because a future edit to the input path could
# take it away silently: Ctrl-\ is answered and returned on before
# offerKeystroke is ever reached (client.zig:874), so the detach chord
# cannot land in the counter; and Shift+PageUp has to arrive as ONE read,
# because the whole-chunk mem.eql at client.zig:882 is what recognizes it —
# split across two reads it matches nothing, falls through to the keystroke
# path, and is both counted suppressed and forwarded to the shell as input.
hostroom predict1
XDG_STATE_HOME="$HOSTROOM" MUX_PREDICT_STATS=1 timeout 60 "$PTYCLIENT" --cols 80 --rows 24 \
--out "$OUT.tp1" --err "$OUT.tp1.err" -- \
"$MUX" --via "$MUX d proxy --sock $SOCK13" > "$OUT.tp1.log" 2>&1 <<'EOF' &
expect \x1b[0m100 10000
send \x1b[5;2~
expect 60 10000
expect reconnecting 20000
expect \x1b[0m100 20000
send t
expect \x1b[80X\x1b[0mt 20000
settle 500 15000
send \x1c\x1c
waitexit 15000
EOF
TP1PID=$!
defer_kill "$TP1PID"
# The tear goes between verb 3 (the scroll view is on screen) and verb 4;
# the fixture's "done 3" line is the barrier. Every needle above is unique
# to its phase, which is what makes each expect sound under the no-counting
# rule:
# the first "100" is the attach snapshot's last row, and it is spelled
# with the SGR reset the paint puts in front of every row because a bare
# "100" is no longer unique to it: the client asks its own terminal for
# mouse reporting with `\x1b[?1000h`, which lands in this capture before
# any row does. Measured, not feared — that is the collision that made
# this scenario fail when the wheel landed;
# "60" is in the history page (54..77) and in NOTHING before it. The
# snapshot's rows are 78..100, and the escapes around them cannot spell
# it either — but that is a fact about THIS session, not about the paint
# code: both paint paths end with a CUP carrying the cursor's real
# column (client.zig:968, :991), and a cursor at column 60 would put a
# "60" in the stream. Here it cannot be: the session is cat in canonical
# mode with one keystroke typed after the tear, so that column is 1 or 2
# for the whole scenario, and the row half of a CUP tops out at 24;
# "reconnecting" is the banner, painted once, on the tear;
# the SECOND "\x1b[0m100" can only be the post-resync repaint: the scroll view
# holds 54..77, and the banner carries no digits but its own position;
# the last needle is STRUCTURAL, and it has to be. In canonical mode the
# only thing a single keystroke produces is the line discipline's echo of
# the very glyph the prediction just painted, so no content needle can
# tell the daemon's answer from the client's own guess. `\x1b[80X` can:
# it is emitted by exactly one paint path in the client, the one that
# paints a DELTA (client.zig, paintDeltaClipped), so its arrival means a
# frame came back. The prediction paints the same glyph underlined and
# without an erase ("\x1b[24;1H\x1b[4mt\x1b[0m"), and the resync repaint
# erases the whole screen with \x1b[2J instead — neither can forge this.
# The two post-tear deadlines are 20000 because a reconnect is allowed to
# retry: the daemon may not have reaped the dead client's slot yet, and the
# client tolerates that for reconnect_grace_ms (5s) with a backoff that
# reaches 2s. That is a budget for a documented retry loop, not a deadline
# tuned until a flake stopped.
wait_for "$OUT.tp1.log" "done 3" 20 || {
echo "e2e FAIL: tp1 never reached the scroll view:"; cat "$OUT.tp1.log"; exit 1; }
TP1PROXY=$(proxy_pid "$SOCK13")
[ -n "$TP1PROXY" ] || { echo "e2e FAIL: tp1: no proxy to tear"; exit 1; }
kill -9 "$TP1PROXY"
# The daemon must still be SERVING, which is what `kill -0` cannot tell us:
# it is our own backgrounded child, so a dead one is a zombie until the
# shell reaps it, and signalling a zombie succeeds. A dump answers only if
# the process is alive and its accept loop is running.
"$MUX" d dump --sock "$SOCK13" >/dev/null 2>&1 || {
echo "e2e FAIL: tp1 tear killed the daemon, not the proxy"; exit 1; }
set +e
wait "$TP1PID"
RC=$?
TP1PID=""
set -e
[ "$RC" -eq 0 ] || {
echo "e2e FAIL: tp1 ptyclient exited $RC:"; cat "$OUT.tp1.log"
cat -v "$OUT.tp1.err" 2>/dev/null; exit 1; }
# Verb 5 is row 18's catch: a resync that fails to leave the scroll view
# suppresses every live paint, so the second "100" never arrives.
#
# The counters are the 412f38f pin, and they are EXACT rather than floors.
# One keystroke in the whole scenario, typed after the resync onto a blank
# cell at a live cursor, in a session whose termios never moves: nothing
# here is timing-dependent, so every number is a fact.
# made=1, suppressed=0 is the fix working. With it reverted the overlay
# still believes it is scrolled and predictAt refuses on that bit alone —
# made=0, suppressed=1, with nothing else in the run to hide it.
# displayed=1 says the glyph reached the SCREEN, which `made` does not:
# the overlay queues while unconfident and paints nothing, so a client
# that lost the `.always` tier would still count the prediction.
# confirmed=1 and contradicted=0 say the daemon's own echo agreed with
# the guess, which is what makes the prediction a speedup and not a lie.
want_stat "$OUT.tp1.err" made 1 "pty scroll reconnect"
want_stat "$OUT.tp1.err" displayed 1 "pty scroll reconnect"
want_stat "$OUT.tp1.err" confirmed 1 "pty scroll reconnect"
want_stat "$OUT.tp1.err" contradicted 0 "pty scroll reconnect"
want_stat "$OUT.tp1.err" suppressed 0 "pty scroll reconnect"
assert_converged_pty "$OUT.tp1" "$SOCK13" "pty scroll reconnect" 80 24
# The doctored control, extended to a pty capture: this capture's byte shape
# (alt screen, banner paints, a history page) exists nowhere else in the
# corpus, so it earns its own cannot-fail check.
#
# It CANNOT be the plain append the non-pty control uses. render replays
# only up to the LAST alt-screen exit, and a tty client's capture ends with
# one — so appended bytes land after the grid under test and change nothing.
# Measured: the doctored capture rendered byte-identical to the original,
# i.e. a control that could never fire. Dropping that trailing 8-byte exit
# puts the append back INSIDE the alt screen (a stream with no alt-exit is
# replayed whole), which corrupts exactly one row of the real grid — row 10,
# "87" in this session. The tail is asserted rather than assumed, because if
# the client's teardown ever stops ending there, the truncation would silently
# go back to being a no-op appended after the grid.
[ "$(tail -c 8 "$OUT.tp1" | od -An -tx1 | tr -d ' \n')" = "1b5b3f313034396c" ] || {
echo "e2e FAIL: tp1 capture does not end with the alt-screen exit;"
echo " the doctored control below would not be doctoring the grid"
tail -c 16 "$OUT.tp1" | cat -v; exit 1; }
head -c "$(( $(wc -c < "$OUT.tp1") - 8 ))" "$OUT.tp1" > "$OUT.tp1.doc"
printf '\033[10;1Hpty-doctor-glyphs' >> "$OUT.tp1.doc"
if converged_quiet "$OUT.tp1.doc" "$SOCK13"; then
echo "e2e FAIL: convergence control did not fire on a doctored pty capture"; exit 1
fi
rm_swept "$OUT.tp1.doc" "$OUT.tp1.doc.render" "$OUT.tp1.doc.dump" \
"$OUT.tp1.doc.render.n" "$OUT.tp1.doc.dump.n" "$OUT.tp1.doc.diff" \
"$OUT.tp1.doc.rvt" "$OUT.tp1.doc.dvt"
softkill "$D13PID" || true
D13PID=""
rm_swept "$OUT.tp1" "$OUT.tp1.err" "$OUT.tp1.log" "$OUT.tp1.d" "$TP1SH"
ok "reconnect while scrolled: view restored, prediction resumed"