a73x

5b5bf1d0

test: the e2e pins ask the OS through named oracle helpers

a73x   2026-09-03 15:10

Commit message
test: the e2e pins ask the OS through named oracle helpers

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SakwJEwD9dXBoRP5kWbemW
test/e2e.sh
Old New
@@ -105,6 +105,12 @@ command -v python3 > /dev/null 2>&1 || {
105 E2E_DIR=$(dirname "$0") 105 E2E_DIR=$(dirname "$0")
106 . "$E2E_DIR/e2e_lib.sh" 106 . "$E2E_DIR/e2e_lib.sh"
107 107
108 # Before any group, and in the E2E_ONLY path too: every pin below asks the
109 # OS through the lib's oracle helpers, and a helper that has stopped
110 # answering makes those pins agree with anything. One scenario here says
111 # the questions still reach the OS, so a later failure is the product's.
112 oracle_selftest
113
108 E2E_GROUPS='01_boot 02_predict 03_side 04_handoff 05_session 06_web 07_wallcli 08_mouse 09_hosts 10_agent 11_select 12_panes 13_birth 14_upgrade 15_askpass 16_push' 114 E2E_GROUPS='01_boot 02_predict 03_side 04_handoff 05_session 06_web 07_wallcli 08_mouse 09_hosts 10_agent 11_select 12_panes 13_birth 14_upgrade 15_askpass 16_push'
109 # The groups that cannot stand alone, and what they need. E2E_ONLY refuses 115 # The groups that cannot stand alone, and what they need. E2E_ONLY refuses
110 # these by name rather than running them: a group whose fixtures are 116 # these by name rather than running them: a group whose fixtures are
@@ -170,8 +176,8 @@ done
170 # one of those and adds a convergence point would be pinning a fact every 176 # one of those and adds a convergence point would be pinning a fact every
171 # leg above already establishes. 177 # leg above already establishes.
172 178
173 [ "$OK_COUNT" = "112" ] || { 179 [ "$OK_COUNT" = "113" ] || {
174 echo "e2e FAIL: $OK_COUNT scenario checkpoints ran, the pin says 112 —" 180 echo "e2e FAIL: $OK_COUNT scenario checkpoints ran, the pin says 113 —"
175 echo " a scenario was added (update the pin) or silently lost" 181 echo " a scenario was added (update the pin) or silently lost"
176 exit 1 182 exit 1
177 } 183 }
test/e2e_01_boot.sh
Old New
@@ -54,15 +54,15 @@ ok "--version on both binaries"
54 KEYOUT=$("$MUX" d keygen) 54 KEYOUT=$("$MUX" d keygen)
55 [ "$KEYOUT" = "$XDG_CONFIG_HOME/mux/key" ] || { 55 [ "$KEYOUT" = "$XDG_CONFIG_HOME/mux/key" ] || {
56 echo "e2e FAIL: keygen printed '$KEYOUT'"; exit 1; } 56 echo "e2e FAIL: keygen printed '$KEYOUT'"; exit 1; }
57 PERMS=$(stat -c %a "$KEYOUT") 57 PERMS=$(file_mode "$KEYOUT")
58 [ "$PERMS" = "600" ] || { echo "e2e FAIL: keygen perms $PERMS, want 600"; exit 1; } 58 [ "$PERMS" = "600" ] || { echo "e2e FAIL: keygen perms $PERMS, want 600"; exit 1; }
59 DPERMS=$(stat -c %a "$XDG_CONFIG_HOME/mux") 59 DPERMS=$(file_mode "$XDG_CONFIG_HOME/mux")
60 [ "$DPERMS" = "700" ] || { echo "e2e FAIL: key dir perms $DPERMS, want 700"; exit 1; } 60 [ "$DPERMS" = "700" ] || { echo "e2e FAIL: key dir perms $DPERMS, want 700"; exit 1; }
61 SUM1=$(sha256sum "$KEYOUT") 61 SUM1=$(sha256_of "$KEYOUT")
62 if "$MUX" d keygen > /dev/null 2>&1; then 62 if "$MUX" d keygen > /dev/null 2>&1; then
63 echo "e2e FAIL: second keygen did not refuse"; exit 1 63 echo "e2e FAIL: second keygen did not refuse"; exit 1
64 fi 64 fi
65 SUM2=$(sha256sum "$KEYOUT") 65 SUM2=$(sha256_of "$KEYOUT")
66 [ "$SUM1" = "$SUM2" ] || { echo "e2e FAIL: refused keygen still changed the key"; exit 1; } 66 [ "$SUM1" = "$SUM2" ] || { echo "e2e FAIL: refused keygen still changed the key"; exit 1; }
67 ok "keygen creates once, 0600 in a 0700 dir, refuses twice" 67 ok "keygen creates once, 0600 in a 0700 dir, refuses twice"
68 68
@@ -532,13 +532,14 @@ start_daemon "$SOCK4" "$OUT.d4.d" "--quic daemon never bound its session socket"
532 --quic "127.0.0.1:$QPORT" --key "$QKEY" --quic-idle-ms 3000 532 --quic "127.0.0.1:$QPORT" --key "$QKEY" --quic-idle-ms 3000
533 D4PID=$DPID 533 D4PID=$DPID
534 534
535 # The UDP port is actually held. /proc/net/udp rather than ss or lsof: it is 535 # The UDP port is actually held. Asked through the lib's udp_local_bound
536 # always there on the platform this daemon runs on, and needs no privileges. 536 # rather than ss or lsof: the kernel's own table is always there and needs
537 # 127.0.0.1 is 0100007F in the little-endian hex the file uses. 537 # no privileges, and the spelling of the question is the oracle's business.
538 # 127.0.0.1 is 0100007F in the little-endian hex that table uses.
538 QHEX=$(printf '0100007F:%04X' "$QPORT") 539 QHEX=$(printf '0100007F:%04X' "$QPORT")
539 grep -qi " $QHEX " /proc/net/udp || { 540 udp_local_bound "$QHEX" || {
540 echo "e2e FAIL: no UDP socket bound at 127.0.0.1:$QPORT ($QHEX)" 541 echo "e2e FAIL: no UDP socket bound at 127.0.0.1:$QPORT ($QHEX)"
541 grep -i "0100007F" /proc/net/udp || true 542 udp_table | grep -i "0100007F" || true
542 exit 1 543 exit 1
543 } 544 }
544 545
@@ -783,8 +784,9 @@ D4PID=$DPID
783 # here rather than one, since either alone still passes on an accident. 784 # here rather than one, since either alone still passes on an accident.
784 # 785 #
785 # One expression, used by the wait and the verdict: two spellings of the 786 # One expression, used by the wait and the verdict: two spellings of the
786 # same match are two matches waiting to drift apart. 787 # same match are two matches waiting to drift apart. It is the lib's
787 udp_local_bound() { awk -v h="$1" '$2==h{f=1} END{exit !f}' /proc/net/udp; } 788 # udp_local_bound now, so the local-address-only rule this block bought is
789 # also what the M8 bind check above asks, and neither can drift again.
788 i=0 790 i=0
789 # 50 x 0.1s, the same 5s every daemon wait in this file allows — and scaled 791 # 50 x 0.1s, the same 5s every daemon wait in this file allows — and scaled
790 # by TIME_SCALE like every other budget, because under test/coverage.sh the 792 # by TIME_SCALE like every other budget, because under test/coverage.sh the
@@ -796,8 +798,8 @@ done
796 udp_local_bound "$QHEX" || { 798 udp_local_bound "$QHEX" || {
797 echo "e2e FAIL: restarted daemon did not rebind udp 127.0.0.1:$QPORT ($QHEX) in $(( 5 * TIME_SCALE ))s" 799 echo "e2e FAIL: restarted daemon did not rebind udp 127.0.0.1:$QPORT ($QHEX) in $(( 5 * TIME_SCALE ))s"
798 echo " daemon pid $D4PID: $(ps -o stat=,comm= -p "$D4PID" 2>/dev/null || echo gone)" 800 echo " daemon pid $D4PID: $(ps -o stat=,comm= -p "$D4PID" 2>/dev/null || echo gone)"
799 echo " /proc/net/udp lines mentioning the port:" 801 echo " kernel UDP table lines mentioning the port:"
800 grep -i " $QHEX " /proc/net/udp || echo " (none)" 802 udp_table | grep -i "$QHEX" || echo " (none)"
801 cat "$OUT.q"; exit 1; 803 cat "$OUT.q"; exit 1;
802 } 804 }
803 kill -0 "$D4PID" || { echo "e2e FAIL: restarted --quic daemon died"; cat "$OUT.q"; exit 1; } 805 kill -0 "$D4PID" || { echo "e2e FAIL: restarted --quic daemon died"; cat "$OUT.q"; exit 1; }
@@ -901,7 +903,7 @@ kill -0 "$SPID" || { echo "e2e FAIL: started daemon not alive"; exit 1; }
901 # way in 03_side), so this holds today by construction — which is exactly 903 # way in 03_side), so this holds today by construction — which is exactly
902 # why it is asserted here: a later split of the two callers would leave 904 # why it is asserted here: a later split of the two callers would leave
903 # `d start` free to hand execve the /proc link and name its daemon `exe`. 905 # `d start` free to hand execve the /proc link and name its daemon `exe`.
904 SCOMM=$(cat "/proc/$SPID/comm") 906 SCOMM=$(pid_comm "$SPID")
905 [ "$SCOMM" = "mux" ] || { 907 [ "$SCOMM" = "mux" ] || {
906 echo "e2e FAIL: the daemon mux d start -d brought up has comm '$SCOMM', want 'mux'" 908 echo "e2e FAIL: the daemon mux d start -d brought up has comm '$SCOMM', want 'mux'"
907 exit 1; } 909 exit 1; }
@@ -923,7 +925,7 @@ assert_converged "$OUT.s8" "$SOCK8" "started daemon"
923 # than the key is: it carries whatever the daemon says about its own start. 925 # than the key is: it carries whatever the daemon says about its own start.
924 MUXLOG="$XDG_STATE_HOME/mux/muxd.log" 926 MUXLOG="$XDG_STATE_HOME/mux/muxd.log"
925 [ -f "$MUXLOG" ] || { echo "e2e FAIL: no daemon log at $MUXLOG"; exit 1; } 927 [ -f "$MUXLOG" ] || { echo "e2e FAIL: no daemon log at $MUXLOG"; exit 1; }
926 LPERMS=$(stat -c %a "$MUXLOG") 928 LPERMS=$(file_mode "$MUXLOG")
927 [ "$LPERMS" = "600" ] || { echo "e2e FAIL: daemon log perms $LPERMS, want 600"; exit 1; } 929 [ "$LPERMS" = "600" ] || { echo "e2e FAIL: daemon log perms $LPERMS, want 600"; exit 1; }
928 930
929 # Rerun: silent no-op beyond the already-running line, exit 0, same daemon. 931 # Rerun: silent no-op beyond the already-running line, exit 0, same daemon.
@@ -941,12 +943,12 @@ grep -q "already running on $SOCK8 (stop it first with \`mux d stop --sock $SOCK
941 # leave that line where it is and grow the file. 943 # leave that line where it is and grow the file.
942 SEED="a live daemon was writing here" 944 SEED="a live daemon was writing here"
943 echo "$SEED" >> "$MUXLOG" 945 echo "$SEED" >> "$MUXLOG"
944 SEEDED=$(stat -c %s "$MUXLOG") 946 SEEDED=$(file_size "$MUXLOG")
945 "$MUX" d start -d --sock "$SOCK8T" 2> "$OUT.start" 947 "$MUX" d start -d --sock "$SOCK8T" 2> "$OUT.start"
946 TPID=$(sed -n 's/.* pid=\([0-9]*\).*/\1/p' "$OUT.start") 948 TPID=$(sed -n 's/.* pid=\([0-9]*\).*/\1/p' "$OUT.start")
947 defer_kill "$TPID" 949 defer_kill "$TPID"
948 [ -n "$TPID" ] || { echo "e2e FAIL: append spawn reported no pid"; exit 1; } 950 [ -n "$TPID" ] || { echo "e2e FAIL: append spawn reported no pid"; exit 1; }
949 GREW=$(stat -c %s "$MUXLOG") 951 GREW=$(file_size "$MUXLOG")
950 [ "$GREW" -ge "$SEEDED" ] || { 952 [ "$GREW" -ge "$SEEDED" ] || {
951 echo "e2e FAIL: log shrank on spawn ($SEEDED -> $GREW) — a second daemon zeroed the first's" 953 echo "e2e FAIL: log shrank on spawn ($SEEDED -> $GREW) — a second daemon zeroed the first's"
952 exit 1; } 954 exit 1; }
@@ -1039,9 +1041,10 @@ set -e
1039 echo "e2e FAIL: mux d endpoint --sock ... --start exited $ESRC (want 0)" 1041 echo "e2e FAIL: mux d endpoint --sock ... --start exited $ESRC (want 0)"
1040 cat "$OUT.esa.err"; exit 1; } 1042 cat "$OUT.esa.err"; exit 1; }
1041 # (a) The named socket is bound, and bound by a `mux` — asked of the OS by 1043 # (a) The named socket is bound, and bound by a `mux` — asked of the OS by
1042 # INODE, not by a name. A `mux` running anywhere would satisfy a name check 1044 # INODE, not by a name, which is what pid_holds_unix_sock does. A `mux`
1043 # while binding something else entirely; the listening inode for this path 1045 # running anywhere would satisfy a name check while binding something else
1044 # out of /proc/net/unix, found among that pid's open fds, cannot. 1046 # entirely; the kernel's listening inode for this path, found among that
1047 # pid's own open fds, cannot.
1045 ESPID=$(sed -n 's/.* pid=\([0-9]*\).*/\1/p' "$OUT.esa.err" | head -1) 1048 ESPID=$(sed -n 's/.* pid=\([0-9]*\).*/\1/p' "$OUT.esa.err" | head -1)
1046 defer_kill "$ESPID" 1049 defer_kill "$ESPID"
1047 [ -n "$ESPID" ] || { 1050 [ -n "$ESPID" ] || {
@@ -1049,14 +1052,12 @@ defer_kill "$ESPID"
1049 [ -S "$ESOWN" ] || { 1052 [ -S "$ESOWN" ] || {
1050 echo "e2e FAIL: endpoint --start bound nothing at the socket it was given" 1053 echo "e2e FAIL: endpoint --start bound nothing at the socket it was given"
1051 ls -la "$ESRUN"; exit 1; } 1054 ls -la "$ESRUN"; exit 1; }
1052 ESINO=$(awk -v p="$ESOWN" '$NF == p {print $7}' /proc/net/unix | head -1) 1055 pid_holds_unix_sock "$ESPID" "$ESOWN" || {
1053 [ -n "$ESINO" ] || {
1054 echo "e2e FAIL: the kernel lists no listening socket at $ESOWN"; exit 1; }
1055 readlink /proc/"$ESPID"/fd/* 2>/dev/null | grep -qx "socket:\[$ESINO\]" || {
1056 echo "e2e FAIL: pid $ESPID does not hold the socket at $ESOWN — the daemon" 1056 echo "e2e FAIL: pid $ESPID does not hold the socket at $ESOWN — the daemon"
1057 echo " that answered is not the daemon --start reported starting" 1057 echo " that answered is not the daemon --start reported starting,"
1058 echo " or the kernel lists no listening socket at that path at all"
1058 exit 1; } 1059 exit 1; }
1059 ESCOMM=$(cat "/proc/$ESPID/comm") 1060 ESCOMM=$(pid_comm "$ESPID")
1060 [ "$ESCOMM" = "mux" ] || { 1061 [ "$ESCOMM" = "mux" ] || {
1061 echo "e2e FAIL: the socket at $ESOWN is served by comm '$ESCOMM', want 'mux'"; exit 1; } 1062 echo "e2e FAIL: the socket at $ESOWN is served by comm '$ESCOMM', want 'mux'"; exit 1; }
1062 # (b) ...and the announce came back on stdout, first line, in the grammar 1063 # (b) ...and the announce came back on stdout, first line, in the grammar
@@ -1149,12 +1150,13 @@ HFMARK="$OUT.hf.sent"; defer_rm "$HFMARK"
1149 # connect() succeeds into the listen backlog, so the peer's marker below 1150 # connect() succeeds into the listen backlog, so the peer's marker below
1150 # says the byte was written and NOT that the daemon has it. What the leg 1151 # says the byte was written and NOT that the daemon has it. What the leg
1151 # needs is an ACCEPTED half frame — an unaccepted one parks nothing — and 1152 # needs is an ACCEPTED half frame — an unaccepted one parks nothing — and
1152 # the accept is a socket appearing in /proc. Ask the OS about the OS. 1153 # the accept is one more fd appearing in the daemon's table. Ask the OS
1154 # about the OS.
1153 HFDPID=$(real_pid "$DHFPID") 1155 HFDPID=$(real_pid "$DHFPID")
1154 HFFD0=$(find "/proc/$HFDPID/fd" -mindepth 1 2>/dev/null | wc -l) 1156 HFFD0=$(pid_fd_count "$HFDPID")
1155 [ "$HFFD0" -gt 0 ] || { 1157 [ "$HFFD0" -gt 0 ] || {
1156 echo "e2e FAIL: half-frame: cannot read /proc/$HFDPID/fd, so the accept" 1158 echo "e2e FAIL: half-frame: the OS will not list pid $HFDPID's open fds,"
1157 echo " below could not be witnessed"; exit 1; } 1159 echo " so the accept below could not be witnessed"; exit 1; }
1158 python3 - "$SOCKHF" "$HFMARK" <<'EOF' & 1160 python3 - "$SOCKHF" "$HFMARK" <<'EOF' &
1159 import socket, sys, time 1161 import socket, sys, time
1160 s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) 1162 s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
@@ -1175,7 +1177,7 @@ wait_for "$HFMARK" sent 5 || {
1175 # cannot speak for. Nothing else connects in this window, so one more fd is 1177 # cannot speak for. Nothing else connects in this window, so one more fd is
1176 # this peer's. 1178 # this peer's.
1177 _hfi=0 1179 _hfi=0
1178 while [ "$(find "/proc/$HFDPID/fd" -mindepth 1 2>/dev/null | wc -l)" -le "$HFFD0" ]; do 1180 while [ "$(pid_fd_count "$HFDPID")" -le "$HFFD0" ]; do
1179 _hfi=$((_hfi + 1)) 1181 _hfi=$((_hfi + 1))
1180 [ "$_hfi" -lt $(( 100 * TIME_SCALE )) ] || { 1182 [ "$_hfi" -lt $(( 100 * TIME_SCALE )) ] || {
1181 echo "e2e FAIL: half-frame: the daemon never accepted the peer, so the" 1183 echo "e2e FAIL: half-frame: the daemon never accepted the peer, so the"
test/e2e_03_side.sh
Old New
@@ -193,23 +193,24 @@ defer_kill "$PAPID"
193 # agent code in it and passed. The exec names the running image, so the link 193 # agent code in it and passed. The exec names the running image, so the link
194 # below can only ever be that image — and this reads the link rather 194 # below can only ever be that image — and this reads the link rather
195 # than the argv, because argv is what a wrong spawn would still get right. 195 # than the argv, because argv is what a wrong spawn would still get right.
196 # readlink -f on both sides: /proc/PID/exe is fully resolved, so a logical 196 # Fully resolved on both sides: pid_exe answers with the resolved image, so
197 # path reached through a symlinked directory would fail a correct spawn. 197 # a logical path reached through a symlinked directory would fail a correct
198 PAEXE=$(readlink -f "/proc/$PAPID/exe") 198 # spawn unless this side resolves too.
199 PAEXE=$(pid_exe "$PAPID")
199 PAWANT=$(readlink -f "$MUX_ELF") 200 PAWANT=$(readlink -f "$MUX_ELF")
200 [ "$PAEXE" = "$PAWANT" ] || { 201 [ "$PAEXE" = "$PAWANT" ] || {
201 echo "e2e FAIL: the auto-started daemon is running $PAEXE, not the build's $PAWANT" 202 echo "e2e FAIL: the auto-started daemon is running $PAEXE, not the build's $PAWANT"
202 echo " (an auto-start that resolves a NAME grades whatever is installed)" 203 echo " (an auto-start that resolves a NAME grades whatever is installed)"
203 exit 1; } 204 exit 1; }
204 # And the name the kernel gives it, which the link cannot see: `comm` comes 205 # And the OS's own name for the process, which the image path cannot see:
205 # from the basename of the FILENAME handed to execve, so exec'ing the 206 # `comm` comes from the basename of the FILENAME handed to execve, so
206 # /proc/self/exe link itself leaves every daemon on the box called `exe` — 207 # exec'ing the self-exe link itself leaves every daemon on the box called
207 # invisible to `pgrep mux`, `killall mux`, `ps -o comm` and systemd's 208 # `exe` — invisible to `pgrep mux`, `killall mux`, `ps -o comm` and
208 # MainPID name, with only the args still saying `mux d start`. 209 # systemd's MainPID name, with only the args still saying `mux d start`.
209 PACOMM=$(cat "/proc/$PAPID/comm") 210 PACOMM=$(pid_comm "$PAPID")
210 [ "$PACOMM" = "mux" ] || { 211 [ "$PACOMM" = "mux" ] || {
211 echo "e2e FAIL: the auto-started daemon's comm is '$PACOMM', want 'mux'" 212 echo "e2e FAIL: the auto-started daemon's comm is '$PACOMM', want 'mux'"
212 echo " (exec the RESOLVED path; the /proc link names the process after itself)" 213 echo " (exec the RESOLVED path; the self-exe link names the process after itself)"
213 exit 1; } 214 exit 1; }
214 # And the argv, asked of the kernel for the same reason: the attach ASKS 215 # And the argv, asked of the kernel for the same reason: the attach ASKS
215 # for a daemon in words — `d start -d --sock PATH`, nothing else — and the 216 # for a daemon in words — `d start -d --sock PATH`, nothing else — and the
@@ -219,7 +220,7 @@ PACOMM=$(cat "/proc/$PAPID/comm")
219 # what this is watching for is an ADDITION. `-d` is absent by the same 220 # what this is watching for is an ADDITION. `-d` is absent by the same
220 # rule the fork obeys — the child is the foreground daemon, and a `-d` 221 # rule the fork obeys — the child is the foreground daemon, and a `-d`
221 # still in its line would fork again. 222 # still in its line would fork again.
222 PACMD=$(tr '\0' ' ' < "/proc/$PAPID/cmdline") 223 PACMD=$(pid_args "$PAPID")
223 [ "$PACMD" = "mux d start --sock $SOCK15 " ] || { 224 [ "$PACMD" = "mux d start --sock $SOCK15 " ] || {
224 echo "e2e FAIL: the auto-started daemon's argv is [$PACMD]," 225 echo "e2e FAIL: the auto-started daemon's argv is [$PACMD],"
225 echo " want [mux d start --sock $SOCK15 ]" 226 echo " want [mux d start --sock $SOCK15 ]"
test/e2e_04_handoff.sh
Old New
@@ -88,7 +88,11 @@ mkdir -p "$SSHIM_DIR" "$HRUN" "$HRUN2"
88 ssh_shim_head "$SSHIM_DIR/ssh" 88 ssh_shim_head "$SSHIM_DIR/ssh"
89 cat >> "$SSHIM_DIR/ssh" <<'SHIM' 89 cat >> "$SSHIM_DIR/ssh" <<'SHIM'
90 echo $$ >> "${SSHIM_PIDLOG:?}" 90 echo $$ >> "${SSHIM_PIDLOG:?}"
91 printf '%s %s\n' "$(cat /proc/$$/comm)" "$(cat /proc/$PPID/comm)" >> "${SSHIM_COMMLOG:?}" 91 # ps rather than the lib's pid_comm: this shim is written to disk and run
92 # as its own `sh` by the client under test, so nothing the suite sourced
93 # is in scope here. `ps -o comm=` is the same answer the oracle gives and
94 # is spelled the same way on every OS the suite could run on.
95 printf '%s %s\n' "$(ps -o comm= -p $$)" "$(ps -o comm= -p $PPID)" >> "${SSHIM_COMMLOG:?}"
92 shift 96 shift
93 printf '%s\n' "$*" >> "${SSHIM_ARGLOG:?}" 97 printf '%s\n' "$*" >> "${SSHIM_ARGLOG:?}"
94 exec /bin/sh -c "$*" 98 exec /bin/sh -c "$*"
@@ -199,23 +203,23 @@ kill -0 "$HAPID" 2>/dev/null || {
199 # The cache: present, private (it holds the key), and naming a port that is 203 # The cache: present, private (it holds the key), and naming a port that is
200 # really being listened on. The last one is the assertion that matters — 204 # really being listened on. The last one is the assertion that matters —
201 # a remembered number proves nothing, a remembered number that maps to the 205 # a remembered number proves nothing, a remembered number that maps to the
202 # daemon's actual UDP socket proves the announce was true. /proc/net/udp 206 # daemon's actual UDP socket proves the announce was true. Asked through
203 # for the same reasons the M8 block uses it, and 00000000 because the lazy 207 # udp_local_bound for the same reasons the M8 block uses it, and 00000000
204 # bind takes 0.0.0.0. 208 # because the lazy bind takes 0.0.0.0.
205 [ -f "$HCACHE" ] || { 209 [ -f "$HCACHE" ] || {
206 echo "e2e FAIL: cold handoff left no cache at $HCACHE" 210 echo "e2e FAIL: cold handoff left no cache at $HCACHE"
207 ls -la "$XDG_CACHE_HOME/mux/hosts" 2>/dev/null; exit 1; } 211 ls -la "$XDG_CACHE_HOME/mux/hosts" 2>/dev/null; exit 1; }
208 HCPERM=$(stat -c %a "$HCACHE") 212 HCPERM=$(file_mode "$HCACHE")
209 [ "$HCPERM" = "600" ] || { 213 [ "$HCPERM" = "600" ] || {
210 echo "e2e FAIL: handoff cache perms $HCPERM, want 600 (the file holds a key)"; exit 1; } 214 echo "e2e FAIL: handoff cache perms $HCPERM, want 600 (the file holds a key)"; exit 1; }
211 HCDPERM=$(stat -c %a "$XDG_CACHE_HOME/mux/hosts") 215 HCDPERM=$(file_mode "$XDG_CACHE_HOME/mux/hosts")
212 [ "$HCDPERM" = "700" ] || { 216 [ "$HCDPERM" = "700" ] || {
213 echo "e2e FAIL: handoff cache dir perms $HCDPERM, want 700"; exit 1; } 217 echo "e2e FAIL: handoff cache dir perms $HCDPERM, want 700"; exit 1; }
214 HPORT=$(sed -n 's/^endpoint \([0-9][0-9]*\) [0-9a-f]*$/\1/p' "$HCACHE") 218 HPORT=$(sed -n 's/^endpoint \([0-9][0-9]*\) [0-9a-f]*$/\1/p' "$HCACHE")
215 [ -n "$HPORT" ] || { 219 [ -n "$HPORT" ] || {
216 echo "e2e FAIL: the cache is not an announce line:"; cat -v "$HCACHE"; exit 1; } 220 echo "e2e FAIL: the cache is not an announce line:"; cat -v "$HCACHE"; exit 1; }
217 HHEXUDP=$(printf '00000000:%04X' "$HPORT") 221 HHEXUDP=$(printf '00000000:%04X' "$HPORT")
218 grep -qi " $HHEXUDP " /proc/net/udp || { 222 udp_local_bound "$HHEXUDP" || {
219 echo "e2e FAIL: the cached port $HPORT names no UDP listener ($HHEXUDP);" 223 echo "e2e FAIL: the cached port $HPORT names no UDP listener ($HHEXUDP);"
220 echo " the announce the client believed was not the daemon's" 224 echo " the announce the client believed was not the daemon's"
221 exit 1; } 225 exit 1; }
@@ -479,7 +483,7 @@ HWPORT=$(sed -n 's/^endpoint \([0-9][0-9]*\) [0-9a-f]*$/\1/p' "$HCACHE_D")
479 [ "$HWPORT" = "$HQPORT" ] || { 483 [ "$HWPORT" = "$HQPORT" ] || {
480 echo "e2e FAIL: the cache at $HCACHE_D names port '$HWPORT', want the silent $HQPORT" 484 echo "e2e FAIL: the cache at $HCACHE_D names port '$HWPORT', want the silent $HQPORT"
481 cat -v "$HCACHE_D"; exit 1; } 485 cat -v "$HCACHE_D"; exit 1; }
482 HWMODE=$(stat -c %a "$HCACHE_D") 486 HWMODE=$(file_mode "$HCACHE_D")
483 [ "$HWMODE" = "600" ] || { 487 [ "$HWMODE" = "600" ] || {
484 echo "e2e FAIL: the cache at $HCACHE_D is mode $HWMODE; readCache refuses" 488 echo "e2e FAIL: the cache at $HCACHE_D is mode $HWMODE; readCache refuses"
485 echo " anything looser, so this leg would attach COLD" 489 echo " anything looser, so this leg would attach COLD"
test/e2e_lib.sh
Old New
@@ -300,7 +300,7 @@ hardkill() {
300 # wrapper writes no coverage.db at any --output-interval, so a clean 300 # wrapper writes no coverage.db at any --output-interval, so a clean
301 # exit is the only exit that keeps the data. Then wait for it, because 301 # exit is the only exit that keeps the data. Then wait for it, because
302 # killing both at once is the same as never killing the child at all. 302 # killing both at once is the same as never killing the child at all.
303 for _c in $(ps -o pid= --ppid "$1" 2>/dev/null); do 303 for _c in $(pid_children "$1"); do
304 kill -9 "$_c" 2>/dev/null || true 304 kill -9 "$_c" 2>/dev/null || true
305 done 305 done
306 _i=0 306 _i=0
@@ -321,7 +321,7 @@ hardkill() {
321 # cmdline is what a resumed-argv check would otherwise read. 321 # cmdline is what a resumed-argv check would otherwise read.
322 real_pid() { 322 real_pid() {
323 if [ "$(ps -o comm= -p "$1" 2>/dev/null)" = kcov ]; then 323 if [ "$(ps -o comm= -p "$1" 2>/dev/null)" = kcov ]; then
324 ps -o pid= --ppid "$1" 2>/dev/null | head -1 | tr -d ' ' 324 pid_children "$1" | head -1
325 else 325 else
326 echo "$1" 326 echo "$1"
327 fi 327 fi
@@ -330,7 +330,7 @@ real_pid() {
330 softkill() { 330 softkill() {
331 if [ "$(ps -o comm= -p "$1" 2>/dev/null)" = kcov ]; then 331 if [ "$(ps -o comm= -p "$1" 2>/dev/null)" = kcov ]; then
332 _rc=1 332 _rc=1
333 for _c in $(ps -o pid= --ppid "$1" 2>/dev/null); do 333 for _c in $(pid_children "$1"); do
334 kill "$_c" 2>/dev/null && _rc=0 334 kill "$_c" 2>/dev/null && _rc=0
335 done 335 done
336 return $_rc 336 return $_rc
@@ -338,6 +338,14 @@ softkill() {
338 kill "$1" 2>/dev/null 338 kill "$1" 2>/dev/null
339 } 339 }
340 340
341 # ---- the OS oracle ------------------------------------------------------
342 # The helpers themselves are in os_oracle.sh, sourced here where they used
343 # to be written out. They moved because soak.sh reads two of them as well,
344 # and its copies were a second spelling of a question this repo means to
345 # ask in one place. That file is trap-free on purpose: the registry and the
346 # EXIT trap below are this file's, and it must not bring a second of either.
347 . "$(dirname "$0")/os_oracle.sh"
348
341 # Poll until nothing answers on a socket path (2s). Keyed off the daemon's 349 # Poll until nothing answers on a socket path (2s). Keyed off the daemon's
342 # own liveness rather than a fixed sleep, same reasoning as wait_for. 350 # own liveness rather than a fixed sleep, same reasoning as wait_for.
343 wait_gone() { 351 wait_gone() {
@@ -1315,3 +1323,137 @@ cleanup() {
1315 fi 1323 fi
1316 } 1324 }
1317 trap cleanup EXIT INT TERM 1325 trap cleanup EXIT INT TERM
1326
1327 # ---- the oracle's own pin -----------------------------------------------
1328 # The runner calls this once, before the first group. Every helper above is
1329 # a question a pin asks INSTEAD of asking the daemon, so a helper that
1330 # quietly stopped answering would not fail a pin loudly — it would make the
1331 # pin agree with anything, and the suite would go green having tested
1332 # nothing. This asks each helper something the box can independently
1333 # confirm.
1334 #
1335 # Off-origin on every dimension the helpers could accidentally hold
1336 # constant, because a fixture that pins N=1 or offset=0 is blind to N and
1337 # to the offset (CLAUDE.md): the subject is a child that is not pid 1, it
1338 # holds MORE than the three fds any fixture would, it has an argv worth
1339 # losing, and it has a child of its own. The socket arm is asked in both
1340 # directions — a listener that HOLDS the path and a shell that does not —
1341 # because a `pid_holds_unix_sock` that answered yes to everything would
1342 # pass the one pin that reads it just as happily as a correct one.
1343 #
1344 # The `sh -c` body ends in `:` on purpose. With nothing after it, a shell
1345 # execs the last command in place rather than forking it — bash measured
1346 # doing exactly that — and the subject would be `sleep`, wearing sleep's
1347 # comm, exe and argv and having no child at all. That is a different
1348 # process from the one this means to ask about.
1349 oracle_selftest() {
1350 _osock="$OUT.oracle.sock"
1351 rm -f "$_osock"
1352 printf 'hello' > "$OUT.oracle.probe"
1353 # python3 rather than sh: nothing in POSIX sh can bind a unix socket,
1354 # and the runner has already refused a box without python3.
1355 python3 -c 'import socket,sys,time
1356 s = socket.socket(socket.AF_UNIX); s.bind(sys.argv[1]); s.listen(1); time.sleep(300)' \
1357 "$_osock" &
1358 _olpid=$!
1359 sh -c 'exec 5>/dev/null 6>/dev/null; sleep 300; :' &
1360 _opid=$!
1361 # Registered, not merely killed at the bottom: an assertion below that
1362 # fails exits through the trap, and an unregistered `sleep 300` would
1363 # outlive the run by five minutes — ten of them under `make soak`.
1364 defer_kill "$_olpid" "$_opid"
1365 _oi=0
1366 while [ ! -S "$_osock" ] && [ "$_oi" -lt $(( 50 * TIME_SCALE )) ]; do
1367 sleep 0.1; _oi=$((_oi + 1))
1368 done
1369 [ -S "$_osock" ] || {
1370 echo "e2e FAIL: oracle: the listener bound no socket at $_osock"; exit 1; }
1371
1372 pid_alive "$_opid" || {
1373 echo "e2e FAIL: oracle: pid_alive says a live shell is dead"; exit 1; }
1374 [ "$(pid_comm "$_opid")" = sh ] || {
1375 echo "e2e FAIL: oracle: pid_comm of an sh is '$(pid_comm "$_opid")'"; exit 1; }
1376 pid_args "$_opid" | grep -q 'sleep 300' || {
1377 echo "e2e FAIL: oracle: pid_args lost the argv: [$(pid_args "$_opid")]"; exit 1; }
1378 [ "$(pid_fd_count "$_opid")" -ge 5 ] || {
1379 echo "e2e FAIL: oracle: pid_fd_count is $(pid_fd_count "$_opid") for a shell"
1380 echo " holding fd 5 and fd 6 on top of the usual three"; exit 1; }
1381 [ "$(pid_exe "$_opid")" = "$(readlink -f "$(command -v sh)")" ] || {
1382 echo "e2e FAIL: oracle: pid_exe is '$(pid_exe "$_opid")', want the resolved"
1383 echo " $(readlink -f "$(command -v sh)")"; exit 1; }
1384 # head -1 because this is a pid to kill as well as an answer to check,
1385 # and the registry and kill(1) below both take one word.
1386 _okid=$(pid_children "$_opid" | head -1)
1387 [ -n "$_okid" ] || {
1388 echo "e2e FAIL: oracle: pid_children names no child of a shell running sleep"
1389 exit 1; }
1390 defer_kill "$_okid"
1391 # Through a variable and a case rather than straight into test(1): a
1392 # helper that answered with nothing would otherwise abort the run with
1393 # a bare `[: -gt: unary operator expected` and no e2e FAIL line saying
1394 # which helper went quiet.
1395 _orss=$(pid_rss_kb "$_opid")
1396 case "$_orss" in ''|*[!0-9]*) _orss=0 ;; esac
1397 [ "$_orss" -gt 0 ] || {
1398 echo "e2e FAIL: oracle: pid_rss_kb is '$(pid_rss_kb "$_opid")'"; exit 1; }
1399 pid_holds_unix_sock "$_olpid" "$_osock" || {
1400 echo "e2e FAIL: oracle: pid_holds_unix_sock cannot see $_osock among the"
1401 echo " fds of the pid $_olpid that bound it"; exit 1; }
1402 # The `socket` KIND off a real fd: the listener bound one, so its count
1403 # is at least one. Only that KIND is asserted here. `carrier` and
1404 # `ptymaster` would each need a process holding an anonymous memory file
1405 # or a pty master, which is a daemon — and e2e_14_upgrade already asks
1406 # both of a real one, where a wrong answer fails that pin.
1407 _ofdk=$(pid_holds_fd_kind "$_olpid" socket)
1408 case "$_ofdk" in ''|*[!0-9]*) _ofdk=0 ;; esac
1409 [ "$_ofdk" -ge 1 ] || {
1410 echo "e2e FAIL: oracle: pid_holds_fd_kind says $_olpid holds"
1411 echo " '$(pid_holds_fd_kind "$_olpid" socket)' fds of kind socket, and it"
1412 echo " bound $_osock"; exit 1; }
1413 # `! cmd || {}` rather than `cmd && {}`, for the reason spelled out
1414 # below: an AND-list whose left side fails is a failing command under
1415 # `set -e`, so the passing case would end the run.
1416 ! pid_holds_fd_kind "$_opid" nosuchkind 2>/dev/null || {
1417 echo "e2e FAIL: oracle: pid_holds_fd_kind accepted a kind it does not know,"
1418 echo " so a typo in a group file would count zero and pass"; exit 1; }
1419 ! pid_holds_unix_sock "$_opid" "$_osock" || {
1420 echo "e2e FAIL: oracle: pid_holds_unix_sock says the shell $_opid holds"
1421 echo " $_osock, which only $_olpid ever opened"; exit 1; }
1422 # `! cmd || {}` rather than `cmd && {}`, for wait_sock's reason: an
1423 # AND-list whose left side fails is a failing command under `set -e`,
1424 # so the passing case would end the run.
1425 ! udp_local_bound 'ZZZZZZZZ:ZZZZ' || {
1426 echo "e2e FAIL: oracle: udp_local_bound found a socket at an address no"
1427 echo " kernel can spell"; exit 1; }
1428 [ -n "$(udp_table)" ] || {
1429 echo "e2e FAIL: oracle: udp_table is empty, so a bind diagnostic would"
1430 echo " print nothing"; exit 1; }
1431 # Against what the probe file IS, never against a second spelling of the
1432 # same command: comparing file_mode with `stat -c %a` would agree with
1433 # itself on any OS and would have to be ported alongside the helper. A
1434 # chmod the kernel performed, five bytes written, and the published
1435 # SHA-256 of "hello" are answers no arm of the oracle can define away.
1436 chmod 640 "$OUT.oracle.probe"
1437 [ "$(file_mode "$OUT.oracle.probe")" = "640" ] || {
1438 echo "e2e FAIL: oracle: file_mode says '$(file_mode "$OUT.oracle.probe")' for a"
1439 echo " file the kernel just chmod'd to 640"; exit 1; }
1440 [ "$(file_size "$OUT.oracle.probe")" = "5" ] || {
1441 echo "e2e FAIL: oracle: file_size says '$(file_size "$OUT.oracle.probe")' for five bytes"
1442 exit 1; }
1443 [ "$(sha256_of "$OUT.oracle.probe")" = \
1444 "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824" ] || {
1445 echo "e2e FAIL: oracle: sha256_of hashes 'hello' to"
1446 echo " '$(sha256_of "$OUT.oracle.probe")'"; exit 1; }
1447
1448 kill "$_olpid" "$_opid" "$_okid" 2>/dev/null || true
1449 wait "$_olpid" 2>/dev/null || true
1450 wait "$_opid" 2>/dev/null || true
1451 _oi=0
1452 while pid_alive "$_opid" && [ "$_oi" -lt $(( 50 * TIME_SCALE )) ]; do
1453 sleep 0.05; _oi=$((_oi + 1))
1454 done
1455 ! pid_alive "$_opid" || {
1456 echo "e2e FAIL: oracle: pid_alive says a killed shell still lives"; exit 1; }
1457 rm -f "$_osock" "$OUT.oracle.probe"
1458 ok "oracle: the OS answers the helpers by name"
1459 }
test/os_oracle.sh
Old New
@@ -0,0 +1,65 @@
1 # shellcheck shell=sh
2 # os_oracle.sh — the OS oracle, sourced and never run.
3 #
4 # Sourced by test/e2e_lib.sh, where these helpers used to live, and by
5 # test/soak.sh, which reads two of them between runs. It is deliberately
6 # trap-free and state-free: both readers bring a cleanup registry and an
7 # EXIT trap of their own, and a second trap here would replace one of
8 # theirs. soak.sh used to carry copies of the two it needs, and a copy is a
9 # second spelling to port the day a second OS arrives.
10 #
11 # "Ask the OS about the OS, not the daemon" (CLAUDE.md). Every pin that
12 # reads a pid, an fd table, a bound UDP port or a file's mode asks through
13 # these names, so the SPELLING of the question lives in one place per OS
14 # and the question itself lives in the group file. The Linux arm is /proc,
15 # GNU stat and sha256sum; a second OS adds a `case "$(uname)"` arm here and
16 # nothing in a group file changes. Each prints its answer on stdout and
17 # returns 1 when the OS will not say, so a caller may treat "no answer"
18 # and "the wrong answer" as the same failure.
19 #
20 # oracle_selftest in e2e_lib.sh is these helpers' own pin. A helper that
21 # stopped answering — a missing binary, a /proc the sandbox will not show —
22 # would otherwise turn every pin that reads it into a check that passes
23 # without running, which is the one failure a green tree cannot show.
24 pid_alive() { kill -0 "$1" 2>/dev/null || [ -d "/proc/$1" ]; }
25 pid_exe() { readlink -f "/proc/$1/exe" 2>/dev/null; }
26 pid_comm() { cat "/proc/$1/comm" 2>/dev/null; }
27 pid_args() { tr '\0' ' ' < "/proc/$1/cmdline" 2>/dev/null; }
28 pid_children() { ps -o pid= --ppid "$1" 2>/dev/null | tr -d ' '; }
29 pid_fd_count() { find "/proc/$1/fd" -mindepth 1 2>/dev/null | wc -l | tr -d ' '; }
30 pid_fd_targets() { readlink "/proc/$1"/fd/* 2>/dev/null; }
31 # pid_holds_fd_kind PID KIND — how many of PID's open fds are of KIND, which
32 # is one of `socket`, `carrier` (the upgrade manifest's anonymous memory
33 # file) or `ptymaster`. Prints the count. These are the three KINDS an
34 # upgrade pin asks about, and naming them is the point: a second OS's arm
35 # counts the same three things under whatever that OS calls them, and the
36 # group file keeps asking the same question. `|| true` because grep -c
37 # prints 0 AND exits 1 when nothing matches, and callers run under `set -e`.
38 pid_holds_fd_kind() {
39 case "$2" in
40 socket) _fdk='socket:' ;;
41 carrier) _fdk='memfd:' ;;
42 ptymaster) _fdk='/dev/ptmx' ;;
43 *) echo "pid_holds_fd_kind: no such fd kind '$2'" >&2; return 1 ;;
44 esac
45 pid_fd_targets "$1" | grep -c -F "$_fdk" || true
46 }
47 # By INODE, never by name: a process running the right program while
48 # binding something else entirely would satisfy a name check. The kernel's
49 # listening inode for PATH, found among that pid's open fds, cannot.
50 pid_holds_unix_sock() {
51 _ino=$(awk -v p="$2" '$NF == p {print $7}' /proc/net/unix | head -1)
52 [ -n "$_ino" ] && pid_fd_targets "$1" | grep -qx "socket:\[$_ino\]"
53 }
54 pid_rss_kb() { awk '/VmRSS/{print $2}' "/proc/$1/status" 2>/dev/null || echo 0; }
55 # Matched on the LOCAL address ($2) rather than anywhere on the line: the
56 # remote address of a client dialling this port is the same hex, and a
57 # whole-line match read a reconnecting client as a bound listener.
58 udp_local_bound() { awk -v h="$1" '$2==h{f=1} END{exit !f}' /proc/net/udp; }
59 udp_table() { cat /proc/net/udp 2>/dev/null; }
60 file_mode() { stat -c %a "$1"; }
61 file_size() { stat -c %s "$1"; }
62 sha256_of() { sha256sum "$1" | cut -d' ' -f1; }
63 # GNU timeout is a binary here; a box without one names it gtimeout, and a
64 # group file keeps spelling `timeout` either way.
65 command -v timeout >/dev/null 2>&1 || timeout() { gtimeout "$@"; }