test/bench.sh

Ref: main worktree-web-client-spec   Size: 3.2 KiB   History

#!/bin/sh
# M4 bytes-on-wire measurement: a typing-heavy workload, then the delta
# bytes actually sent per keystroke, against an absolute bound.
#
# The bound used to be a ratio of delta bytes to the full-snapshot
# equivalent. Cells on the wire (2026-09-04) made a snapshot of a mostly
# blank screen tiny — 5644 bytes of VT became 86 bytes of cells — so the
# same delta traffic went from 1% to 43% of it against a 50% gate: the
# ratio had started measuring the denominator. A keystroke's cost is a
# thing to bound on its own.
set -eu
MUX="$1"
SOCK="${TMPDIR:-/tmp}/muxd-bench-$$.sock"
# Attaches record wall tiles; keep them out of the developer's real wall file.
XDG_STATE_HOME="${TMPDIR:-/tmp}/muxd-bench-$$-state"; export XDG_STATE_HOME

cleanup() { kill "$DPID" 2>/dev/null || true; rm -rf "$SOCK" "$XDG_STATE_HOME"; }
trap cleanup EXIT INT TERM

"$MUX" d start --sock "$SOCK" --shell /bin/sh &
DPID=$!
i=0
while [ ! -S "$SOCK" ] && [ "$i" -lt 50 ]; do sleep 0.1; i=$((i+1)); done
[ -S "$SOCK" ] || { echo "bench FAIL: socket never appeared"; exit 1; }

# Steady-state typing: KEYS single characters with small gaps (no newlines,
# so no scroll; this is the workload the handoff's done-criterion names).
KEYS=120
{
    sleep 0.5
    j=0
    while [ "$j" -lt "$KEYS" ]; do printf 'x'; sleep 0.05; j=$((j+1)); done
    sleep 0.5
    printf '\034\034'
} | "$MUX" --sock "$SOCK" > /dev/null

STATS="$("$MUX" d stats --sock "$SOCK")"
echo "$STATS"

DELTA=$(echo "$STATS" | sed -n 's/.*[^_]delta_bytes=\([0-9]*\).*/\1/p')
SNAPBYTES=$(echo "$STATS" | sed -n 's/.*[^_]snapshot_bytes=\([0-9]*\).*/\1/p')
DELTAS=$(echo "$STATS" | sed -n 's/.*[^_]deltas=\([0-9]*\).*/\1/p')
SNAPS=$(echo "$STATS" | sed -n 's/.*snapshots=\([0-9]*\).*/\1/p')
[ -n "$DELTA" ] && [ -n "$SNAPBYTES" ] && [ -n "$DELTAS" ] && [ -n "$SNAPS" ] || {
    echo "bench FAIL: could not parse stats"; exit 1;
}

# A delta path that sent nothing would score a perfect 0 per keystroke. The
# workload must actually have produced deltas, and exactly one snapshot
# (the attach) — a mid-run resync means the differ fell back, and the
# figure would be measuring the fallback rather than the deltas.
[ "$DELTAS" -gt 0 ] || {
    echo "bench FAIL: no deltas sent; the delta path is dead, not efficient"
    exit 1
}
[ "$SNAPS" -eq 1 ] || {
    echo "bench FAIL: expected exactly 1 snapshot (the attach), got $SNAPS"
    exit 1
}
# A keystroke changes one row, and a delta re-sends that row whole: the
# 18-byte delta header, the 6-byte row header, and the row's cells, which
# on this workload grow to the prompt plus KEYS characters. Measured
# 2026-09-04 at 64 bytes per keystroke on the VT wire and 65 on cells.
# The bound is what a wrong differ would cost: re-sending the whole
# viewport per keystroke is about 340 bytes on a blank 80x24 screen (23
# empty rows at 8 bytes each plus the typed row), so 160 fails that and
# leaves the honest figure room to drift without a false alarm.
PER_KEY=$(( DELTA / KEYS ))
echo "delta bytes: $DELTA over $KEYS keystrokes = $PER_KEY per keystroke   attach snapshot: $SNAPBYTES bytes"
if [ "$PER_KEY" -ge 160 ]; then
    echo "bench FAIL: a keystroke costs $PER_KEY bytes, a row's worth is under 160 (M4 kill criterion)"
    exit 1
fi
echo "bench OK"