test/oracle_selftest.sh
Ref: Size: 22.8 KiB History
# shellcheck shell=sh
# oracle_selftest.sh — the OS oracle's own pin, sourced and also runnable.
#
# It lives beside test/os_oracle.sh rather than inside test/e2e_lib.sh so
# that a new OS arm can be graded on a box where nothing else in the
# harness works yet:
#
# sh test/oracle_selftest.sh
#
# needs a shell, python3 and the oracle's own tools, and answers ORACLE_OK
# or an `e2e FAIL:` line. Sourced into the suite it is the same function,
# called by test/e2e.sh with the lib's registry, counter and time scale
# already up. It is one function either way, never a copy: a self-test with
# two spellings would be the second one that goes stale.
#
# Trap-free when sourced, for os_oracle.sh's reason: e2e_lib.sh arms the
# EXIT trap this suite cleans up through, and a second one here would
# replace it. The standalone arm at the bottom arms its own, because on
# that path there is no other.
#
# The runner calls this once, before the first group. Every helper in
# os_oracle.sh is a question a pin asks INSTEAD of asking the daemon, so a
# helper that quietly stopped answering would not fail a pin loudly — it
# would make the pin agree with anything, and the suite would go green
# having tested nothing. This asks each helper something the box can
# independently confirm.
#
# Off-origin on every dimension the helpers could accidentally hold
# constant, because a fixture that pins N=1 or offset=0 is blind to N and
# to the offset (CLAUDE.md): the subject is a child that is not pid 1, it
# holds MORE than the three fds any fixture would, it has an argv worth
# losing, and it has a child of its own. The socket arm is asked in both
# directions — a listener that HOLDS the path and a shell that does not —
# because a `pid_holds_unix_sock` that answered yes to everything would
# pass the one pin that reads it just as happily as a correct one.
#
# The `sh -c` body ends in `:` on purpose. With nothing after it, a shell
# execs the last command in place rather than forking it — bash measured
# doing exactly that — and the subject would be `sleep`, wearing sleep's
# comm, exe and argv and having no child at all. That is a different
# process from the one this means to ask about.
oracle_selftest() {
_osock="$OUT.oracle.sock"
_oport_f="$OUT.oracle.port"
rm -f "$_osock" "$_oport_f"
printf 'hello' > "$OUT.oracle.probe"
# python3 rather than sh: nothing in POSIX sh can bind a unix socket,
# and the runner has already refused a box without python3.
#
# It holds one fd of every KIND a pin asks about, so that each of them
# can be asked in BOTH directions. Before this the listener held only a
# unix socket, and `carrier`, `ptymaster` and `udp_local_bound` were
# asked nothing but negatives anywhere in the tree — an arm that
# answered 0 and false forever passed every gate on both OSes, which is
# a check that never runs rather than a check that fails.
#
# The carrier is opened in the local shape, which carrier_kind names:
# a memfd on Linux and an unlinked mkstemp file on Darwin, matching what
# server_os_linux.anonFd and server_os_macos.anonFd each make. Asking
# the oracle rather than reading `uname` here keeps every "which OS"
# answer in the one file that is allowed to know.
#
# The UDP port is EPHEMERAL and reported back rather than picked here:
# a number this file chose would be a number some other process on the
# box may already hold, and the pin would fail for a reason that is
# nothing to do with the helper.
python3 -c 'import os, socket, sys, tempfile, time
s = socket.socket(socket.AF_UNIX); s.bind(sys.argv[1]); s.listen(1)
u = socket.socket(socket.AF_INET, socket.SOCK_DGRAM); u.bind(("127.0.0.1", 0))
if sys.argv[2] == "memfd":
carrier = os.memfd_create("mux-upgrade")
else:
carrier, path = tempfile.mkstemp(prefix="mux-upgrade-", dir="/tmp")
os.unlink(path)
master, slave = os.openpty()
# Last, and to a temporary name renamed into place, so that the file
# existing means every fd above is open and not that some of them are.
with open(sys.argv[3] + ".part", "w") as f:
f.write("%d" % u.getsockname()[1])
os.rename(sys.argv[3] + ".part", sys.argv[3])
time.sleep(300)' \
"$_osock" "$(carrier_kind)" "$_oport_f" &
_olpid=$!
sh -c 'exec 5>/dev/null 6>/dev/null; sleep 300; :' &
_opid=$!
# Registered, not merely killed at the bottom: an assertion below that
# fails exits through the trap, and an unregistered `sleep 300` would
# outlive the run by five minutes — ten of them under `make soak`.
defer_kill "$_olpid" "$_opid"
_oi=0
while [ ! -f "$_oport_f" ] && [ "$_oi" -lt $(( 50 * TIME_SCALE )) ]; do
sleep 0.1; _oi=$((_oi + 1))
done
[ -f "$_oport_f" ] || {
echo "e2e FAIL: oracle: the listener never reported its port, so one of"
echo " the fds the pins below read was never opened"; exit 1; }
[ -S "$_osock" ] || {
echo "e2e FAIL: oracle: the listener bound no socket at $_osock"; exit 1; }
_oport=$(cat "$_oport_f")
# The subject is only the process this means to ask about once it has
# opened fds 5 and 6 and forked `sleep`; until then `pid_fd_count` and
# `pid_children` would be graded against a shell still starting up, and
# the self-test would fail for a reason that is nothing to do with the
# helpers. A child is the LAST of those steps, so waiting for one waits
# for all of them.
wait_until 50 "oracle: the subject shell never forked its sleep" \
'[ -n "$(pid_children "$_opid")" ]'
pid_alive "$_opid" || {
echo "e2e FAIL: oracle: pid_alive says a live shell is dead"; exit 1; }
# `sh` on both OSes, measured: Linux reads /proc/PID/comm, which is the
# last component of the path execve was handed, and Darwin's ps prints
# the bare name for a binary under /bin. It is only the IMAGE behind
# that name that differs between them, which is pid_exe's business
# below and not this line's.
[ "$(pid_comm "$_opid")" = sh ] || {
echo "e2e FAIL: oracle: pid_comm of an sh is '$(pid_comm "$_opid")'"; exit 1; }
pid_args "$_opid" | grep -q 'sleep 300' || {
echo "e2e FAIL: oracle: pid_args lost the argv: [$(pid_args "$_opid")]"; exit 1; }
# The SHAPE too, because the one caller that compares a whole argv does
# it with `=`: one space between words and none at the end. Linux reads
# a NUL-terminated /proc entry and Darwin reads ps, and the two agreed
# on every word and disagreed on the last byte until this was pinned.
case "$(pid_args "$_opid")" in
*' ') echo "e2e FAIL: oracle: pid_args ends in a space: [$(pid_args "$_opid")]"; exit 1 ;;
*' '*) echo "e2e FAIL: oracle: pid_args doubles a space: [$(pid_args "$_opid")]"; exit 1 ;;
esac
[ "$(pid_fd_count "$_opid")" -ge 5 ] || {
echo "e2e FAIL: oracle: pid_fd_count is $(pid_fd_count "$_opid") for a shell"
echo " holding fd 5 and fd 6 on top of the usual three"; exit 1; }
# head -1 because this is a pid to kill as well as an answer to check,
# and the registry and kill(1) below both take one word.
_okid=$(pid_children "$_opid" | head -1)
[ -n "$_okid" ] || {
echo "e2e FAIL: oracle: pid_children names no child of a shell running sleep"
exit 1; }
defer_kill "$_okid"
# pid_exe is asked about the CHILD and not the shell, and the shell is
# the reason: macOS ships /bin/sh as a stub in front of the bash image,
# so the running image of a process this file started as `sh` is
# /bin/bash there — measured, and true of a copy of /bin/sh as well, so
# it is the image and not the name that differs. `sleep` is an ordinary
# binary on both OSes and its own path is what either arm answers, so
# the assertion stays exact rather than being weakened to fit. Through
# real_path on both sides because a distribution that puts /bin on a
# symlink to /usr/bin, and a Darwin lsof that canonicalizes /tmp to
# /private/tmp, both answer the resolved path and not the spelling.
_oexe=$(pid_exe "$_okid") || _oexe=''
_owant=$(real_path "$(command -v sleep)") || _owant=''
# The non-empty guard is the point of the first test: with `=` alone, a
# box where lsof answered nothing AND perl answered nothing would compare
# "" with "" and read ORACLE_OK — two dead helpers agreeing.
[ -n "$_owant" ] && [ "$_oexe" = "$_owant" ] || {
echo "e2e FAIL: oracle: pid_exe of the subject's sleep is '$_oexe',"
echo " want the resolved '$_owant'"; exit 1; }
# Through a variable and a case rather than straight into test(1): a
# helper that answered with nothing would otherwise abort the run with
# a bare `[: -gt: unary operator expected` and no e2e FAIL line saying
# which helper went quiet.
_orss=$(pid_rss_kb "$_opid")
case "$_orss" in ''|*[!0-9]*) _orss=0 ;; esac
[ "$_orss" -gt 0 ] || {
echo "e2e FAIL: oracle: pid_rss_kb is '$(pid_rss_kb "$_opid")'"; exit 1; }
pid_holds_unix_sock "$_olpid" "$_osock" || {
echo "e2e FAIL: oracle: pid_holds_unix_sock cannot see $_osock among the"
echo " fds of the pid $_olpid that bound it"; exit 1; }
# Every KIND, off a real fd, in both directions: the listener holds one
# of each and the subject shell holds none of the two that are not a
# socket. Both halves matter. e2e_14_upgrade asks `carrier` and
# `ptymaster` too, but every one of its call sites asserts ZERO — so an
# arm that answered 0 forever would pass that pin as happily as a
# correct one, and on Darwin the carrier body is a heuristic of its own
# (an unlinked regular file) rather than a name the kernel hands over.
for _ok in socket carrier ptymaster; do
_ofdk=$(pid_holds_fd_kind "$_olpid" "$_ok")
case "$_ofdk" in ''|*[!0-9]*) _ofdk=0 ;; esac
[ "$_ofdk" -ge 1 ] || {
echo "e2e FAIL: oracle: pid_holds_fd_kind says $_olpid holds"
echo " '$(pid_holds_fd_kind "$_olpid" "$_ok")' fds of kind $_ok, and it"
echo " opened one of each before it reported its port"; exit 1; }
done
# The subject shell opened two /dev/null fds and forked a sleep, and
# nothing else. A helper that counted any open file as a carrier, or any
# character device as a pty master, would answer here.
for _ok in carrier ptymaster; do
_ofdk=$(pid_holds_fd_kind "$_opid" "$_ok")
case "$_ofdk" in ''|*[!0-9]*) _ofdk=-1 ;; esac
[ "$_ofdk" -eq 0 ] || {
echo "e2e FAIL: oracle: pid_holds_fd_kind says the shell $_opid holds"
echo " '$(pid_holds_fd_kind "$_opid" "$_ok")' fds of kind $_ok, and it"
echo " opened nothing but /dev/null"; exit 1; }
done
# `! cmd || {}` rather than `cmd && {}`, for the reason spelled out
# below: an AND-list whose left side fails is a failing command under
# `set -e`, so the passing case would end the run.
! pid_holds_fd_kind "$_opid" nosuchkind 2>/dev/null || {
echo "e2e FAIL: oracle: pid_holds_fd_kind accepted a kind it does not know,"
echo " so a typo in a group file would count zero and pass"; exit 1; }
! pid_holds_unix_sock "$_opid" "$_osock" || {
echo "e2e FAIL: oracle: pid_holds_unix_sock says the shell $_opid holds"
echo " $_osock, which only $_olpid ever opened"; exit 1; }
# The port the listener really bound, asked in the ONE spelling both
# arms take: /proc/net/udp's local address, which is the host-order u32
# in hex — 0100007F for 127.0.0.1 on a little-endian box, the same
# constant e2e_01_boot and e2e_04_handoff already print. The Darwin arm
# converts it to lsof's dotted form, and until this line nothing in the
# tree asked that conversion for an address that exists.
_oudp=$(printf '0100007F:%04X' "$_oport")
udp_local_bound "$_oudp" || {
echo "e2e FAIL: oracle: udp_local_bound cannot see the socket the listener"
echo " bound at 127.0.0.1:$_oport ($_oudp)"
udp_table | head -20; exit 1; }
# `! cmd || {}` rather than `cmd && {}`, for wait_sock's reason: an
# AND-list whose left side fails is a failing command under `set -e`,
# so the passing case would end the run.
#
# The SAME port at a different local address. This is the one that pins
# the local-address rule both arms are built around: a helper matching
# the port anywhere on its line — the remote half of a connected
# socket's row included — answers yes here.
! udp_local_bound "$(printf '0200007F:%04X' "$_oport")" || {
echo "e2e FAIL: oracle: udp_local_bound reads the socket bound at"
echo " 127.0.0.1:$_oport as bound at 127.0.0.2 as well, so it is"
echo " matching the port and not the local address"; exit 1; }
! udp_local_bound 'ZZZZZZZZ:ZZZZ' || {
echo "e2e FAIL: oracle: udp_local_bound found a socket at an address no"
echo " kernel can spell"; exit 1; }
[ -n "$(udp_table)" ] || {
echo "e2e FAIL: oracle: udp_table is empty, so a bind diagnostic would"
echo " print nothing"; exit 1; }
# Against what the probe file IS, never against a second spelling of the
# same command: comparing file_mode with `stat -c %a` would agree with
# itself on any OS and would have to be ported alongside the helper. A
# chmod the kernel performed, five bytes written, and the published
# SHA-256 of "hello" are answers no arm of the oracle can define away.
chmod 640 "$OUT.oracle.probe"
[ "$(file_mode "$OUT.oracle.probe")" = "640" ] || {
echo "e2e FAIL: oracle: file_mode says '$(file_mode "$OUT.oracle.probe")' for a"
echo " file the kernel just chmod'd to 640"; exit 1; }
[ "$(file_size "$OUT.oracle.probe")" = "5" ] || {
echo "e2e FAIL: oracle: file_size says '$(file_size "$OUT.oracle.probe")' for five bytes"
exit 1; }
[ "$(sha256_of "$OUT.oracle.probe")" = \
"2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824" ] || {
echo "e2e FAIL: oracle: sha256_of hashes 'hello' to"
echo " '$(sha256_of "$OUT.oracle.probe")'"; exit 1; }
# real_path's two branches, against the rule `readlink -f` states and
# both arms owe: every component but the LAST must exist.
#
# The unwritten leaf first. e2e_14_upgrade canonicalizes the name of a
# candidate binary before anything writes one, so an arm that refused a
# path it cannot stat would fail that leg — which is what a bare
# Cwd::realpath does.
_orpl=$(real_path "$OUT.oracle.notyet") || _orpl=''
case "$_orpl" in
/*.oracle.notyet) ;;
*) echo "e2e FAIL: oracle: real_path answers '$_orpl' for a name whose parent"
echo " exists and whose last component does not, where it owes an"
echo " absolute path ending in that name"; exit 1 ;;
esac
# And the missing PARENT, which must be no answer at all. This is the
# half only one arm gets for free: readlink -f refuses it, while
# Cwd::realpath on some perls hands back the spelling of a directory that
# is not there, so the Darwin arm has to check that the parent is a
# directory rather than that the call answered.
# The status kept through `||` and not read after the fact: an
# assignment whose command substitution fails IS a failing command, and
# under `set -e` this pin would end the run with no line at all rather
# than grade anything. stderr goes to /dev/null rather than into the
# value, because the contract is about what reaches STDOUT and a traced
# run would otherwise capture its own xtrace.
_orpmrc=0
_orpm=$(real_path "$OUT.oracle.nodir/x" 2>/dev/null) || _orpmrc=$?
[ "$_orpmrc" -ne 0 ] && [ -z "$_orpm" ] || {
echo "e2e FAIL: oracle: real_path answers '$_orpm' (rc $_orpmrc) for a path"
echo " whose PARENT does not exist, where it owes nothing and rc 1"
exit 1; }
# The `timeout` a group file spells, which is GNU's binary here, gtimeout
# on a Mac that has coreutils and perl's alarm on one that does not.
# Nothing else committed calls it on that last arm, so without these five
# lines the bare-Mac fallback ships unrun — and every one of these codes
# is one a caller reads: refuse() and half the daemon legs turn on
# telling 124 from 1.
#
# 0.3s for the expiry, because that is the only case that has to wait for
# the alarm and every arm takes a fractional budget; the whole block
# costs about a third of a second.
_otimeout_is() {
_otw=$1; shift
_otrc=0
# A redirected subshell that ENDS IN `exit`, and both halves matter.
# GNU timeout re-raises the signal that killed its child so as to
# report 128+N, and a shell whose foreground command died of a signal
# prints "Terminated" to its own stderr — a line that is not this
# suite's and would land in the middle of an e2e log. The subshell
# moves that report inside the redirection; the trailing `exit` stops
# the shell exec'ing timeout in place of the subshell, which would
# leave the signal to be reported by the shell outside it after all.
( timeout "$@"; exit $? ) >/dev/null 2>&1 || _otrc=$?
[ "$_otrc" -eq "$_otw" ] || {
echo "e2e FAIL: oracle: timeout $* exited $_otrc, want $_otw"
echo " (124 the budget expiring, 127 a command that is not there,"
echo " 128+N a signal, anything else the child's own status)"
exit 1; }
}
_otimeout_is 0 5 true
_otimeout_is 124 0.3 sleep 5
_otimeout_is 7 5 sh -c 'exit 7'
_otimeout_is 127 5 /no/such/binary-for-the-oracle
_otimeout_is 143 5 sh -c 'kill -TERM $$'
# The clock, asked of the CALENDAR and then of itself. Thirteen digits is
# what epoch milliseconds have been since 2001 and will be until 2286, so
# it separates milliseconds from the seconds and the nanoseconds either
# side of them — the one mistake a `date +%s%N` bracket that lost or kept
# a /1000000 would make. Then a real second has to pass for the delta to
# be a second: a clock stuck at a constant would satisfy the digit count
# and nothing else.
#
# A band and not a number, and a generous one, because the sleep and two
# process starts are inside it and this runs on loaded boxes and under
# test/coverage.sh's tracing.
# Captured with its stderr and with the failure swallowed, then graded
# out of the variable: a now_ms whose clock is not installed exits 127,
# and an assignment that fails ends the run under `set -e` with the
# shell's own "not found" and no line saying which helper it was.
_onow=$(now_ms 2>&1) || _onow="$_onow (exit $?)"
_onowsaid="$_onow"
case "$_onow" in ''|*[!0-9]*) _onow='' ;; esac
[ "${#_onow}" -eq 13 ] || {
echo "e2e FAIL: oracle: now_ms answers '$_onowsaid', which is not the 13"
echo " digits epoch MILLISECONDS have — seconds or nanoseconds"
echo " would make every timing bracket read it wrong"; exit 1; }
sleep 1
_oelapsed=$(( $(now_ms) - _onow ))
[ "$_oelapsed" -ge 900 ] && [ "$_oelapsed" -le $(( 3000 * TIME_SCALE )) ] || {
echo "e2e FAIL: oracle: now_ms moved ${_oelapsed}ms across a 1s sleep"; exit 1; }
# And the TEXT of that clock, run in a shell this one did not configure,
# which is exactly how the throughput legs and the agent-mute leg use it.
# On Darwin it is a perl one-liner, so a Mac whose perl has lost
# Time::HiRes fails HERE rather than inside a $(( )) partway through a
# group file, which is where an unpinned snippet would have failed.
_osnip=$(sh -c "$(now_ms_snippet); now_ms" 2>&1 || true)
_osnipsaid="$_osnip"
case "$_osnip" in ''|*[!0-9]*) _osnip='' ;; esac
[ "${#_osnip}" -eq 13 ] || {
echo "e2e FAIL: oracle: now_ms_snippet run in a fresh shell answers"
echo " '$_osnipsaid'"
echo " rather than 13 digits of epoch milliseconds"; exit 1; }
kill "$_olpid" "$_opid" "$_okid" 2>/dev/null || true
wait "$_olpid" 2>/dev/null || true
wait "$_opid" 2>/dev/null || true
_oi=0
while pid_alive "$_opid" && [ "$_oi" -lt $(( 50 * TIME_SCALE )) ]; do
sleep 0.05; _oi=$((_oi + 1))
done
! pid_alive "$_opid" || {
echo "e2e FAIL: oracle: pid_alive says a killed shell still lives"; exit 1; }
rm -f "$_osock" "$_oport_f" "$OUT.oracle.probe" "$OUT.oracle.notyet"
ok "oracle: the OS answers the helpers by name"
# One fixed word, on top of ok()'s sentence: a port is graded over ssh
# and the run that grades it has one line to look for.
echo "ORACLE_OK"
}
# ---- the standalone arm -------------------------------------------------
# `$0` is this file only when it was RUN. Sourced from e2e_lib.sh it is the
# runner's path, and sourced by hand it is the shell's, so neither takes
# this branch and neither gets a second EXIT trap.
#
# The four names oracle_selftest reads out of the suite are spelled here in
# their smallest honest form. They are not a copy of the lib's: standalone
# there is no cleanup registry to share, no scenario count to keep and no
# traced run to scale timeouts for, so each is the whole of that duty on
# this path rather than a reimplementation of the suite's.
case "$0" in
*oracle_selftest.sh)
set -e
. "$(dirname "$0")/os_oracle.sh"
: "${TIME_SCALE:=1}"
: "${OUT:=${TMPDIR:-/tmp}/mux-oracle-$$}"
_SELF_KILL=""
defer_kill() {
for _dk in "$@"; do
if [ -n "$_dk" ]; then _SELF_KILL="$_dk $_SELF_KILL"; fi
done
return 0
}
ok() { echo "e2e OK: $1"; }
wait_until() {
_i=0
while [ "$_i" -lt $(( $1 * TIME_SCALE )) ]; do
# First command of an AND-OR list, which set -e exempts: a
# predicate that is false on this tick is the normal case.
eval "$3" && return 0
sleep 0.1; _i=$((_i + 1))
done
echo "e2e FAIL: $2"; exit 1
}
# Children first, and through the oracle's own pid_children: the subject
# is a shell whose `sleep 300` is registered only after the assertions
# that come before it, so a run that fails early leaves a grandchild
# that no list names. It inherits this script's stdout, and an ssh
# reading that pipe waits on the fd rather than on the process — so the
# leak does not merely outlive the run, it hangs whoever asked for it,
# which is exactly how a port gets graded from another box.
trap 'for _k in $_SELF_KILL; do
for _kc in $(pid_children "$_k" 2>/dev/null); do
kill "$_kc" 2>/dev/null || true
done
kill "$_k" 2>/dev/null || true
done
rm -f "$OUT".oracle.*' EXIT INT TERM
oracle_selftest
;;
esac