test/vm.sh
Ref: Size: 12.4 KiB History
#!/bin/sh
# test/vm.sh — the `make vm` gate: real user journeys against a real box.
#
# The e2e suite's remotes are ssh SHIMS — scripts that log argv and run mux
# under a second HOME on this machine. Twice (2026-08-29, both askpass) a
# shim leg went green about a mechanism that never ran. This gate is where
# those assumptions get audited: a real sshd, a real network hop, a real
# Ubuntu, and mux spelled exactly as a user spells it. It runs beside
# `make ci`, never inside it — ci stays hermetic and offline.
#
# The fixture is ONE long-lived VM, like a real user's server (the spec:
# docs/superpowers/specs/2026-09-01-e2e-dedupe-and-vm-gate-design.md).
# Hermeticity comes from the scorched-earth reset below, not from
# re-creating the VM; VM lifecycle is a human action, never harness code.
#
# Every claim about the VM is asserted by a separate ssh reading /proc, the
# daemon's grid dump, or pgrep — never by trusting the daemon's own report
# over the connection under test ("ask the OS about the OS").
set -eu
MUX="$1" # static musl mux: the client here AND the binary installed there
PTYCLIENT="$2" # the real-pty client fixture, driving mux the way hands do
VM="${MUX_VM:-ubuntu@10.78.5.3}"
# BatchMode so a broken auth fails in seconds instead of prompting a harness
# nobody sits in front of; accept-new so the FIRST run records the box's
# host key in the user's own known_hosts — mux's entry dial spawns plain
# `ssh $VM`, which reads that file, so the trust decision has to land where
# plain ssh will find it.
SSH_OPTS="-o BatchMode=yes -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new"
vssh() {
# shellcheck disable=SC2086 # SSH_OPTS is words on purpose
timeout 30 ssh $SSH_OPTS "$VM" "$@"
}
OK_COUNT=0
ok() {
OK_COUNT=$((OK_COUNT + 1))
echo "vm OK ($OK_COUNT): $1"
}
# ---- preflight: refuse loudly, never skip -----------------------------
# A gate that can quietly not run is the one failure a green tree cannot
# show — the same stance e2e.sh takes on nvim and curl.
set +e
vssh true 2>/tmp/vm_preflight.err
RC=$?
set -e
[ "$RC" -eq 0 ] || {
echo "vm FAIL: cannot ssh to $VM (rc $RC):"
sed 's/^/ /' /tmp/vm_preflight.err
echo " The fixture is the eitri VM 'mux-e2e' on host charizard."
echo " If it is gone: recreate it PINNED to charizard (default"
echo " placement lands on the arm64 Mac) and re-read its underlay"
echo " IP. If auth broke: re-sign against ~/.config/mux-vm/vm_ca."
exit 1; }
ARCH=$(vssh 'uname -m')
[ "$ARCH" = "x86_64" ] || {
echo "vm FAIL: $VM is $ARCH, and this gate pushes an x86_64-musl build."
echo " The fleet's default placement is the arm64 Mac; recreate"
echo " the VM pinned to host charizard."
exit 1; }
# ---- the local side is a hermetic laptop ------------------------------
# Every XDG dir mux reads, isolated (state, runtime, config, cache — the
# 2026-08-29 demo leaked a QUIC cache entry through the one rig that forgot
# cache). Short root because a unix socket path caps at 107 bytes.
SCRATCH=$(mktemp -d /tmp/muxvm.XXXXXX)
export XDG_STATE_HOME="$SCRATCH/st" XDG_RUNTIME_DIR="$SCRATCH/rt"
export XDG_CONFIG_HOME="$SCRATCH/cf" XDG_CACHE_HOME="$SCRATCH/ca"
mkdir -p "$XDG_STATE_HOME" "$XDG_RUNTIME_DIR" "$XDG_CONFIG_HOME" "$XDG_CACHE_HOME"
trap 'rm -rf "$SCRATCH"' EXIT
# ---- scorched earth on the VM -----------------------------------------
# The reset IS the hermeticity: kill every mux, remove the binary and all
# state, so each run starts from the mux-less box a new user has. The two
# standing arrangements it (re)makes:
# linger — without it logind may tear down /run/user/1000 (the daemon's
# socket dir) between ssh sessions and reap the daemon with it;
# a real server user runs `enable-linger` once and so do we.
# PATH — mux's dials run `ssh $VM 'mux ...'`, a non-interactive bash
# whose PATH lacks ~/.local/bin; the line goes ABOVE bashrc's
# interactivity return, where Debian's ssh-aware bash reads it.
vssh 'sh -s' <<'RESET'
set -eu
pkill -x mux 2>/dev/null || true
sleep 0.3
pkill -9 -x mux 2>/dev/null || true
rm -rf ~/.local/bin/mux ~/.local/state/mux ~/.cache/mux ~/.config/mux
rm -rf "${XDG_RUNTIME_DIR:-/run/user/$(id -u)}"/mux*
sudo -n loginctl enable-linger "$(id -un)"
grep -q '^export PATH="$HOME/.local/bin' ~/.bashrc 2>/dev/null || \
sed -i '1i export PATH="$HOME/.local/bin:$PATH"' ~/.bashrc
mkdir -p ~/.local/bin
RESET
ok "preflight: $VM answers, x86_64, reset to a mux-less box"
# ---- journey 1: install like a user -----------------------------------
set +e
vssh 'command -v mux' >/dev/null 2>&1
RC=$?
set -e
[ "$RC" -ne 0 ] || {
echo "vm FAIL: install: the box still answers 'command -v mux' after the reset"
exit 1; }
scp -q $SSH_OPTS "$MUX" "$VM":.local/bin/mux
WANT_VER=$("$MUX" --version)
GOT_VER=$(vssh 'mux --version')
[ "$WANT_VER" = "$GOT_VER" ] || {
echo "vm FAIL: install: the box answers '$GOT_VER', this build is '$WANT_VER'"
exit 1; }
ok "install: scp to ~/.local/bin and the box answers '$GOT_VER'"
# ---- journey 2: the entry dial ----------------------------------------
# `mux $VM` is the whole product path: plain ssh, the ONE `endpoint
# --start` an asked dial is allowed, the daemon auto-starting on the box,
# a session painting back here. The marker's $$ expands in the REMOTE
# shell, so the pid it paints is a pid only the VM's /proc can confirm —
# which is the assert no shim can fake.
set +e
timeout 90 "$PTYCLIENT" --cols 80 --rows 24 \
--out "$SCRATCH/j2.cap" --err "$SCRATCH/j2.err" -- "$MUX" "$VM" \
> "$SCRATCH/j2.log" 2>&1 <<'EOF'
expect \x1b[?1049h 30000
settle 700 20000
send echo vmj2=$$\n
expect vmj2= 20000
settle 600 15000
send \x1cd
waitexit 15000
EOF
RC=$?
set -e
[ "$RC" -eq 0 ] || {
echo "vm FAIL: entry dial: ptyclient exited $RC (did ssh or the remote start refuse?):"
cat "$SCRATCH/j2.log"; cat "$SCRATCH/j2.err" 2>/dev/null; exit 1; }
RPID=$(vssh 'mux d dump' | sed -n 's/.*vmj2=\([0-9][0-9]*\).*/\1/p' | head -1)
[ -n "$RPID" ] || {
echo "vm FAIL: entry dial: the marker never reached the REMOTE daemon's grid; it holds:"
vssh 'mux d dump' | sed 's/^/ /'
exit 1; }
# Read on the VM, and the VM is Linux — /proc is the right spelling THERE
# whatever this script is run from, so this one stays out of the e2e_lib.sh
# oracle (whose helpers answer about the local box).
vssh "test -d /proc/$RPID" || {
echo "vm FAIL: entry dial: the grid names shell pid $RPID but the VM has no such process"
exit 1; }
ok "entry dial: mux $VM auto-started a daemon and shell $RPID runs on the box"
DPID=$(vssh 'pgrep -x mux' | head -1)
NPROCS=$(vssh 'pgrep -cx mux')
[ "$NPROCS" = "1" ] || {
echo "vm FAIL: entry dial: after detach the box runs $NPROCS mux processes, want 1 (the daemon):"
vssh 'ps -o pid,args -C mux' | sed 's/^/ /'
exit 1; }
# Also read on the VM, and for the same reason: the box under test is a
# Linux VM, so its own /proc is the oracle here.
EXE=$(vssh "readlink /proc/$DPID/exe")
[ "$EXE" = "/home/ubuntu/.local/bin/mux" ] || {
echo "vm FAIL: entry dial: daemon $DPID execs '$EXE', not the installed image"
exit 1; }
# Reattach: the snapshot must carry the pre-detach marker — session state
# lived on the box, not in the client that left.
set +e
timeout 90 "$PTYCLIENT" --cols 80 --rows 24 \
--out "$SCRATCH/j2b.cap" --err "$SCRATCH/j2b.err" -- "$MUX" "$VM" \
> "$SCRATCH/j2b.log" 2>&1 <<'EOF'
expect vmj2= 30000
settle 600 15000
send \x1cd
waitexit 15000
EOF
RC=$?
set -e
[ "$RC" -eq 0 ] || {
echo "vm FAIL: reattach: ptyclient exited $RC, or the snapshot lost the marker:"
cat "$SCRATCH/j2b.log"; cat "$SCRATCH/j2b.err" 2>/dev/null; exit 1; }
ok "reattach: the snapshot still paints the first visit's marker"
# ---- journey 3: a stop stays stopped ----------------------------------
# The wall polls `mux d endpoint` — the bare read verb — once a second per
# host. The bug this pins (found on a live box): a poll that spelled
# --start undid every remote `mux d stop` within a second. The entry dial
# above already recorded $VM in the scratch hosts file, so a bare `mux`
# here is a user's wall watching that box.
vssh 'mux d stop'
_i=0
while [ "$_i" -lt 50 ]; do
[ "$(vssh 'pgrep -cx mux' || true)" = "0" ] && break
sleep 0.2; _i=$((_i + 1))
done
[ "$(vssh 'pgrep -cx mux' || true)" = "0" ] || {
echo "vm FAIL: stop: mux processes survive 'mux d stop' on the box:"
vssh 'ps -o pid,args -C mux' | sed 's/^/ /'
exit 1; }
grep -q "$VM" "$XDG_STATE_HOME/mux/hosts" || {
echo "vm FAIL: stop: the entry dial never recorded $VM in the hosts file; it holds:"
sed 's/^/ /' "$XDG_STATE_HOME/mux/hosts" 2>/dev/null || echo " (no file)"
exit 1; }
# Hold the wall open past a pipe-link poll cycle (10s), then leave. The
# empty wall opens the picker by itself; Esc closes it to the one-line
# text and the quit chord ends the wall.
set +e
timeout 90 "$PTYCLIENT" --cols 80 --rows 24 \
--out "$SCRATCH/j3.cap" --err "$SCRATCH/j3.err" -- "$MUX" \
> "$SCRATCH/j3.log" 2>&1 <<'EOF'
expect \x1b[?1049h 30000
settle 12000 20000
send \x1b
settle 400 5000
send \x1c\x1c
waitexit 15000
EOF
RC=$?
set -e
[ "$RC" -eq 0 ] || {
echo "vm FAIL: stop: the watching wall exited $RC:"
cat "$SCRATCH/j3.log"; cat "$SCRATCH/j3.err" 2>/dev/null; exit 1; }
[ "$(vssh 'pgrep -cx mux' || true)" = "0" ] || {
echo "vm FAIL: stop: the wall's polling RESURRECTED a daemon on the box:"
vssh 'ps -o pid,args -C mux' | sed 's/^/ /'
exit 1; }
grep -q 'unreachable' "$SCRATCH/j3.cap" || {
echo "vm FAIL: stop: the wall never painted the stopped box as unreachable"
exit 1; }
ok "stop stays stopped: 12s of wall polling started nothing on the box"
# ---- journey 4: remote upgrade, same pid across the exec --------------
# The daemon is started the way a shell-integration user starts it, so the
# manifest has OSC 133 marks to carry — the one piece of session state
# nothing can rebuild after the exec.
vssh 'MUX_SHELL_INTEGRATION=1 mux d start -d'
_i=0
while [ "$_i" -lt 50 ]; do
vssh 'mux d endpoint' >/dev/null 2>&1 && break
sleep 0.2; _i=$((_i + 1))
done
set +e
timeout 90 "$PTYCLIENT" --cols 80 --rows 24 \
--out "$SCRATCH/j4.cap" --err "$SCRATCH/j4.err" -- "$MUX" "$VM" \
> "$SCRATCH/j4.log" 2>&1 <<'EOF'
expect \x1b[?1049h 30000
settle 700 20000
send echo vmj4=ok\n
expect vmj4= 20000
settle 600 15000
send \x1cd
waitexit 15000
EOF
RC=$?
set -e
[ "$RC" -eq 0 ] || {
echo "vm FAIL: upgrade: the pre-upgrade attach exited $RC:"
cat "$SCRATCH/j4.log"; cat "$SCRATCH/j4.err" 2>/dev/null; exit 1; }
vssh 'mux d dump' | grep -q 'vmj4=ok' || {
echo "vm FAIL: upgrade: the pre-upgrade marker never reached the box's grid"
exit 1; }
PID_BEFORE=$(vssh 'pgrep -x mux' | head -1)
set +e
timeout 120 "$MUX" d upgrade "$VM" --allow-same-version > "$SCRATCH/j4.up" 2>&1
RC=$?
set -e
[ "$RC" -eq 0 ] || {
echo "vm FAIL: upgrade: 'mux d upgrade $VM' exited $RC:"
sed 's/^/ /' "$SCRATCH/j4.up"; exit 1; }
# The exec keeps the pid: same process, new image. Poll for the served
# state first — the exec is not instant.
_i=0
while [ "$_i" -lt 50 ]; do
vssh 'mux d endpoint' >/dev/null 2>&1 && break
sleep 0.2; _i=$((_i + 1))
done
PID_AFTER=$(vssh 'pgrep -x mux' | head -1)
[ -n "$PID_BEFORE" ] && [ "$PID_BEFORE" = "$PID_AFTER" ] || {
echo "vm FAIL: upgrade: daemon pid changed across the exec ($PID_BEFORE -> $PID_AFTER)"
exit 1; }
vssh 'mux d dump' | grep -q 'vmj4=ok' || {
echo "vm FAIL: upgrade: the session's grid did not survive the exec"
exit 1; }
ok "upgrade: same pid $PID_AFTER across the exec, session grid intact"
# marks survived the manifest: the upgraded daemon still knows real exit
# codes, which only OSC 133 state carried across the exec can.
MRUN=$(vssh 'mux a run --timeout 8000 true')
echo "$MRUN" | grep -q '"mechanism": *"marks"' || {
echo "vm FAIL: upgrade: mux a does not answer under marks after the exec; it said:"
echo " $MRUN"
exit 1; }
ok "upgrade: mux a still answers mechanism=marks after the exec"
# ---- the pin ----------------------------------------------------------
# A literal, e2e.sh-style: adding a journey means editing this by hand.
[ "$OK_COUNT" = "7" ] || {
echo "vm FAIL: $OK_COUNT checkpoints ran, the pin says 7 —"
echo " a journey was added (update the pin) or silently lost"
exit 1
}
echo "vm OK ($OK_COUNT journey checkpoints against $VM)"