test/os_oracle.sh
Ref: Size: 19.4 KiB History
# shellcheck shell=sh
# os_oracle.sh — the OS oracle, sourced and never run.
#
# Sourced by test/e2e_lib.sh, where these helpers used to live, by
# test/oracle_selftest.sh, which is their own pin, and by test/soak.sh,
# which reads two of them between runs. It is deliberately trap-free and
# state-free: its readers bring a cleanup registry and an EXIT trap of
# their own, and a second trap here would replace one of theirs. soak.sh
# used to carry copies of the two it needs, and a copy is a second spelling
# to port the day a second OS arrives.
#
# "Ask the OS about the OS, not the daemon" (CLAUDE.md). Every pin that
# reads a pid, an fd table, a bound UDP port or a file's mode asks through
# these names, so the SPELLING of the question lives in one place per OS
# and the question itself lives in the group file. Each prints its answer
# on stdout and returns 1 when the OS will not say, so a caller may treat
# "no answer" and "the wrong answer" as the same failure.
#
# There are two arms below and a group file must never learn which one it
# is on. That is why udp_local_bound still takes the Linux hex spelling on
# Darwin and converts it here, and why now_ms and real_path exist at all:
# `date +%s%N` and `readlink -f` are GNU, and a group file that spelled
# either would be a Linux group file.
#
# oracle_selftest (test/oracle_selftest.sh) is these helpers' own pin. A
# helper that stopped answering — a missing binary, a /proc the sandbox
# will not show — would otherwise turn every pin that reads it into a check
# that passes without running, which is the one failure a green tree cannot
# show.
case "$(uname)" in
Darwin)
# No /proc here, so lsof is the fd oracle and ps answers the process
# questions. Both read the kernel through libproc, so this arm still
# asks the OS about the OS and never a daemon. lsof's -F mode prints
# one FIELD per line — `f` an fd, `t` its type, `n` its name — which is
# what the helpers below parse, and -a ANDs its selection flags where
# lsof would otherwise OR them.
# _fd_rows PID — one `TYPE NAME` line per NUMBERED fd. lsof reports the
# cwd, the executable and the root directory through the same stream
# with non-numeric fd ids (`fcwd`, `ftxt`, `frtd`), and no question
# below asks about those. The TYPE has to come along because an lsof
# name alone does not say what an fd is: a unix socket and a regular
# file both read as a path, where a /proc symlink target says which.
_fd_rows() {
lsof -p "$1" -Ftn 2>/dev/null | awk '
/^f/ { fd = substr($0, 2); ty = "" }
/^t/ { ty = substr($0, 2) }
/^n/ { if (fd ~ /^[0-9]+$/) print ty " " substr($0, 2) }'
}
# No `[ -d /proc/$1 ]` behind it, because there is no /proc — which
# leaves one behavioural difference between the arms: kill(2) on a pid
# this uid does not own answers EPERM, so a live process belonging to
# somebody else reads as dead here where Linux's directory check would
# still see it. Every pid this suite asks about is one it started.
pid_alive() { kill -0 "$1" 2>/dev/null; }
# The FIRST txt vnode is the executable; the ones after it are dyld and
# the dylibs it mapped. Two things this answers that a caller has to
# know (both measured on macOS 26.6, lsof 4.91): lsof canonicalizes, so
# a binary opened as /tmp/x reads /private/tmp/x, which is exactly what
# real_path answers for the same path; and a process that exec'd
# /bin/sh reports /bin/bash, because Apple ships /bin/sh as a stub in
# front of the bash image. No binary mux runs is such a stub, so the
# pins that read this get back the path they asked about — but the
# oracle's own self-test asks about a `sleep` for that reason.
# Through a variable so that no answer is a FAILURE and not an empty
# success: the pipeline's exit status is head's, which is 0 for a pid
# that is gone, and this file's header promises a 1 when the OS will not
# say. Linux's readlink already answers that way.
pid_exe() {
_pe=$(lsof -p "$1" -a -d txt -Fn 2>/dev/null | sed -n 's/^n//p' | head -1)
[ -n "$_pe" ] || return 1
printf '%s\n' "$_pe"
}
# ps prints the full path for a binary outside the system directories
# and the bare name for one inside them, so the last path component is
# taken either way. Through a variable rather than basename(1): an
# empty answer must be no answer, and `basename ""` prints a line.
pid_comm() {
_pc=$(ps -o comm= -p "$1" 2>/dev/null) || return 1
[ -n "$_pc" ] || return 1
printf '%s\n' "${_pc##*/}"
}
pid_args() { ps -o args= -p "$1" 2>/dev/null; }
pid_children() { pgrep -P "$1" 2>/dev/null; }
pid_fd_count() { lsof -p "$1" -Ff 2>/dev/null | grep -c '^f[0-9]' || true; }
pid_fd_targets() { _fd_rows "$1"; }
pid_holds_fd_kind() {
case "$2" in
# Every socket family, which is what the Linux arm's `socket:`
# prefix counts. PIPE is not among them, because `pipe:[N]` is
# not on that side either.
socket) _fd_rows "$1" | grep -cE '^(unix|IPv4|IPv6|sock) ' || true ;;
# The upgrade manifest is an unlinked mkstemp file here, not a
# memfd (server_os_macos.anonFd). lsof still prints the path it
# had before the unlink, canonicalized and with no marker of
# its own, so the NAME cannot tell a carrier from an ordinary
# open file — the suite's own captures live under /tmp with the
# same `mux-` prefix and would be counted as carriers. What
# does tell them apart is that a carrier's path names nothing
# any more, which is the whole of what "anonymous" means here.
carrier)
_fd_rows "$1" | sed -n 's/^REG //p' | {
_fdn=0
while IFS= read -r _fdp; do
[ -e "$_fdp" ] || _fdn=$((_fdn + 1))
done
echo "$_fdn"
} ;;
ptymaster) _fd_rows "$1" | grep -c '^CHR /dev/ptmx$' || true ;;
*) echo "pid_holds_fd_kind: no such fd kind '$2'" >&2; return 1 ;;
esac
}
# What `server_os_macos.anonFd` makes, in one word, so a test that has to
# OPEN a carrier of the local shape can ask for it without asking which
# OS it is on. `unlinked` is an unlinked mkstemp file.
carrier_kind() { echo unlinked; }
# By PATH, where the Linux arm matches the listening inode: lsof answers
# a unix socket's bound path and no inode, so there is no inode to
# match. It is still this pid's own fd table being read and not the
# filesystem, so a process that merely runs the right program does not
# satisfy it; what it cannot rule out, and Linux can, is a second
# socket bound at the same path. -F because a path holds regex
# metacharacters and this is a literal comparison.
pid_holds_unix_sock() { _fd_rows "$1" | grep -qxF "unix $2"; }
pid_rss_kb() { ps -o rss= -p "$1" 2>/dev/null | tr -d ' '; }
# _udp_dotted HEX — the Linux argument spelling turned into the dotted
# form lsof prints. Anything that is not eight hex digits, a colon and
# four more is no address at all and gets no answer, which is what the
# self-test asks for when it hands this a spelling no kernel can hold.
# Shell arithmetic reads a 0x constant by POSIX, so this forks nothing:
# udp_local_bound is polled in a loop and an interpreter start per tick
# would be most of the wait.
_udp_dotted() {
# Eight hex digits, a colon and four more. Asked as "no byte outside
# the alphabet, one colon, and these two lengths" rather than as one
# bracket pattern built from a variable, because a case pattern that
# came out of an expansion is not treated as a pattern by every
# shell — zsh needs an option set for it, and this file is sourced
# by whatever the caller runs.
case "$1" in
*[!0-9A-Fa-f:]* | *:*:* | *[!0-9A-Fa-f]) return 1 ;;
esac
_uh=${1%:*}
_ut=${1#*:}
[ "${#_uh}" -eq 8 ] && [ "${#_ut}" -eq 4 ] || return 1
_up=$(( 0x$_ut ))
# A wildcard bind reads `*` to lsof and not 0.0.0.0, and that is the
# spelling the handoff group hands over for the daemon's lazy bind.
if [ "$_uh" = 00000000 ]; then echo "*:$_up"; return 0; fi
# /proc/net/udp writes the local address as the host-order u32 in
# hex, which on a little-endian box puts the last octet first, so
# the four bytes come back out in reverse.
_u1=${_uh%??????}
_ur=${_uh#??}; _u2=${_ur%????}
_ur=${_uh#????}; _u3=${_ur%??}
_u4=${_uh#??????}
echo "$(( 0x$_u4 )).$(( 0x$_u3 )).$(( 0x$_u2 )).$(( 0x$_u1 )):$_up"
}
# Matched on the LOCAL address for the Linux arm's reason: lsof prints
# a connected socket as LOCAL->REMOTE, so a client dialling this port
# answers a match that reads anywhere on the line. The arrow and
# everything after it goes before the comparison.
udp_local_bound() {
_ua=$(_udp_dotted "$1") || return 1
lsof -iUDP -P -n -Fn 2>/dev/null | sed -n 's/^n//p' | sed 's/->.*$//' |
grep -qxF "$_ua"
}
udp_table() { lsof -iUDP -P -n 2>/dev/null; }
# tcp_listeners PORT — how many sockets LISTEN on 127.0.0.1:PORT, as a
# bare decimal. Asked with lsof's own state filter so a connected client
# on the same port is not counted, and matched on the local address
# for udp_local_bound's reason. The web group wants exactly one, because
# SO_REUSEPORT once let two hubs bind the same port.
tcp_listeners() {
lsof -iTCP@127.0.0.1:"$1" -sTCP:LISTEN -P -n -Fn 2>/dev/null |
sed -n 's/^n//p' | grep -cx "127.0.0.1:$1"
}
file_mode() { stat -f %Lp "$1"; }
file_size() { stat -f %z "$1"; }
sha256_of() { shasum -a 256 "$1" | cut -d' ' -f1; }
# perl, and not GNU coreutils under their g-names. A pristine macOS has
# no Homebrew on it and this arm has to answer there — test/mac.sh runs
# the oracle INSIDE a freshly cloned guest, and installing a package to
# answer one question would make that guest something other than the
# bare Mac the gate is about. perl ships with macOS and needs nothing.
#
# Cwd::realpath refuses a path whose last component does not exist,
# where `readlink -f` resolves it — and e2e_14_upgrade canonicalizes the
# name of a candidate binary it has not written yet. So the whole path
# is tried first and the parent alone second, which is GNU's rule: every
# component but the last must exist.
#
# The parent is checked with -d and not merely for a defined answer,
# because Cwd::realpath is not the same function on every perl. On 5.42
# with Cwd 3.94 it hands BACK the spelling of a directory that is not
# there — realpath("/nope") is "/nope" — so a `defined` test alone let
# real_path answer a path whose parent does not exist, where readlink -f
# answers nothing with rc 1. oracle_selftest pins both branches, because
# only the perl arm can get this wrong and only on some perls.
real_path() {
perl -e '
use Cwd ();
my $p = $ARGV[0];
my $r = Cwd::realpath($p);
unless (defined $r) {
my ($d, $b) = $p =~ m{^(.*)/([^/]*)$} ? ($1 eq "" ? "/" : $1, $2) : (".", $p);
my $rd = Cwd::realpath($d);
exit 1 unless defined $rd && -d $rd;
$r = $rd eq "/" ? "/$b" : "$rd/$b";
}
print "$r\n";' "$1"
}
# macOS 26's /bin/date does answer %N and older ones do not, so spelling
# the system date here would make the suite's clock depend on how new
# the OS is — and its failure on an older one is a literal N inside an
# arithmetic expansion partway through a run. Time::HiRes is core perl
# and answers on every version.
#
# It costs 4.5ms a call against gdate's 2.1 and python3's 21.6 (measured
# 2026-09-03). That is inside the noise of every bracket in the e2e,
# whose tightest budget is 1500ms. It is NOT inside the noise of the
# throughput gate's 10ms ceiling around a 6ms leg, so a `make throughput`
# run on a Mac will need SOLO_MAX_MS raised — which that gate already
# takes from the environment for exactly this kind of reason.
#
# No backslash anywhere in the snippet, hence -l rather than a printed
# "\n": this text is TYPED into a session by the throughput and
# agent-mute legs, and the pty fixture reads a backslash as the start of
# an escape rather than as a byte to send.
now_ms_snippet() {
echo 'now_ms() { perl -MTime::HiRes -le "print int(Time::HiRes::time() * 1000)"; }'
}
;;
*)
pid_alive() { kill -0 "$1" 2>/dev/null || [ -d "/proc/$1" ]; }
pid_exe() { readlink -f "/proc/$1/exe" 2>/dev/null; }
pid_comm() { cat "/proc/$1/comm" 2>/dev/null; }
# The words with ONE space between them and none at the end. Every
# entry in /proc/PID/cmdline is NUL-TERMINATED, so a straight
# translation leaves a trailing separator that the Darwin arm's
# `ps -o args=` does not, and the one caller that compares the whole
# argv as a string then has to spell a space it can only have learned
# from Linux. Trimmed here so both arms answer the same shape.
pid_args() { tr '\0' ' ' < "/proc/$1/cmdline" 2>/dev/null | sed 's/ *$//'; }
pid_children() { ps -o pid= --ppid "$1" 2>/dev/null | tr -d ' '; }
pid_fd_count() { find "/proc/$1/fd" -mindepth 1 2>/dev/null | wc -l | tr -d ' '; }
pid_fd_targets() { readlink "/proc/$1"/fd/* 2>/dev/null; }
# pid_holds_fd_kind PID KIND — how many of PID's open fds are of KIND,
# which is one of `socket`, `carrier` (the upgrade manifest's anonymous
# memory file) or `ptymaster`. Prints the count. These are the three
# KINDS an upgrade pin asks about, and naming them is the point: the
# other arm counts the same three things under whatever that OS calls
# them, and the group file keeps asking the same question. `|| true`
# because grep -c prints 0 AND exits 1 when nothing matches, and
# callers run under `set -e`.
pid_holds_fd_kind() {
case "$2" in
socket) _fdk='socket:' ;;
carrier) _fdk='memfd:' ;;
ptymaster) _fdk='/dev/ptmx' ;;
*) echo "pid_holds_fd_kind: no such fd kind '$2'" >&2; return 1 ;;
esac
pid_fd_targets "$1" | grep -c -F "$_fdk" || true
}
# What `server_os_linux.anonFd` makes; see the Darwin arm's carrier_kind.
carrier_kind() { echo memfd; }
# By INODE, never by name: a process running the right program while
# binding something else entirely would satisfy a name check. The
# kernel's listening inode for PATH, found among that pid's open fds,
# cannot.
pid_holds_unix_sock() {
_ino=$(awk -v p="$2" '$NF == p {print $7}' /proc/net/unix | head -1)
[ -n "$_ino" ] && pid_fd_targets "$1" | grep -qx "socket:\[$_ino\]"
}
pid_rss_kb() { awk '/VmRSS/{print $2}' "/proc/$1/status" 2>/dev/null || echo 0; }
# Matched on the LOCAL address ($2) rather than anywhere on the line:
# the remote address of a client dialling this port is the same hex,
# and a whole-line match read a reconnecting client as a bound
# listener.
udp_local_bound() { awk -v h="$1" '$2==h{f=1} END{exit !f}' /proc/net/udp; }
udp_table() { cat /proc/net/udp 2>/dev/null; }
# 0A is LISTEN in /proc/net/tcp, and 0100007F the little-endian hex the
# file spells 127.0.0.1 in. /proc rather than ss or lsof, for the udp
# arm's reason: always present, no privileges.
tcp_listeners() {
awk -v a="$(printf '0100007F:%04X' "$1")" \
'$2 == a && $4 == "0A" { n++ } END { print n+0 }' /proc/net/tcp
}
file_mode() { stat -c %a "$1"; }
file_size() { stat -c %s "$1"; }
sha256_of() { sha256sum "$1" | cut -d' ' -f1; }
real_path() { readlink -f "$1"; }
now_ms_snippet() { echo 'now_ms() { echo $(( $(date +%s%N) / 1000000 )); }'; }
;;
esac
# Each arm above defines now_ms_snippet — the TEXT of a now_ms — and this is
# where the harness's own now_ms comes from. The throughput legs and the
# agent-mute leg time themselves from INSIDE a mux session, so their clock
# has to be typed into the bytes they send rather than called, and a group
# file that spelled `date +%s%N` into a session would be a Linux group file
# however OS-blind the rest of it is.
#
# Defined THROUGH the snippet rather than beside it so the clock this shell
# reads and the clock it types into another shell are the same text and
# cannot drift into two answers on one box.
eval "$(now_ms_snippet)"
# GNU timeout is a binary on Linux. A Mac has it only if somebody installed
# coreutils, where it answers to gtimeout, and a pristine one has neither —
# so the last resort is perl, which every macOS ships. A group file keeps
# spelling `timeout` through all three.
#
# The perl arm answers GNU's EXIT contract for the five cases this harness
# asks, and oracle_selftest pins all five: 0 for a command that returns in
# time, 124 when the budget expired, the child's own status when it exited,
# 127 for a command that is not there, and 128+N for one a signal took. A
# harness that read a timeout as a pass, or a real failure as a timeout,
# would grade the wrong thing — `refuse()` and half the e2e's daemon legs
# turn on telling 124 from 1.
#
# It is not GNU timeout in three ways, each measured and none of them
# reached by a caller here: a command that exists but is not executable
# answers 127 where GNU answers 126; a signal sent to the WRAPPER is not
# relayed to the child, where GNU forwards TERM, INT and HUP; and the child
# is not put in a process group, so GNU's kill-the-group on expiry becomes
# kill-the-child and a grandchild outlives the budget.
#
# Almost every caller in this tree runs `timeout <seconds> CMD` in the
# foreground over a single mux or ptyclient child, with no flags, and none
# of the three differences can reach one. The exceptions, and why they are
# still safe: e2e_01_boot.sh and e2e_10_agent.sh each background one
# timeout-wrapped client, and both reap it with `wait` on the wrapper's own
# pid rather than by signalling it — the agent group's `defer_kill` on that
# pid runs only when the leg has already failed, and on this arm it would
# leave the child to finish its own budget, which is a slower cleanup and
# not a different verdict; e2e_05_session.sh, e2e_06_web.sh and
# e2e_13_birth.sh wrap a wsclient, which is one child like the others; and
# test/vm.sh, test/mac.sh and test/xos.sh wrap ssh, but those three drivers
# run on the Linux side of a gate, where `timeout` is the GNU binary and
# this arm is never defined.
if command -v timeout >/dev/null 2>&1; then
:
elif command -v gtimeout >/dev/null 2>&1; then
timeout() { gtimeout "$@"; }
else
timeout() {
_tsecs=$1; shift
# waitpid is restarted around the handler rather than trusted once:
# perl defers a signal to a safe point and the wait returns EINTR
# when it does, and a status read from that return is not the
# child's.
perl -e '
use POSIX ();
use Time::HiRes ();
my $secs = shift @ARGV;
my $pid = fork();
defined $pid or exit 125;
unless ($pid) { exec { $ARGV[0] } @ARGV; exit 127; }
my $fired = 0;
$SIG{ALRM} = sub { $fired = 1; kill "TERM", $pid };
Time::HiRes::alarm($secs);
my $r;
do { $r = waitpid($pid, 0) } while ($r == -1 && $! == POSIX::EINTR());
my $st = $?;
Time::HiRes::alarm(0);
exit 124 if $fired;
exit(($st & 127) ? 128 + ($st & 127) : $st >> 8);' "$_tsecs" "$@"
}
fi