test/coverage.sh

Ref: main worktree-web-client-spec   Size: 9.9 KiB   History

#!/bin/sh
# Line coverage for the e2e suite. kcov wraps the four user binaries, the
# suite runs UNCHANGED, and the per-process databases are merged into one
# report.
#
# kcov is a ptrace tracer, not an instrumentation pass, so nothing here
# changes how anything is built: the report describes the same Debug
# artifacts `zig build e2e` grades, not a special coverage build. That is
# the whole reason kcov was chosen over -fprofile-instr-generate, which Zig
# 0.15 does not expose at all.
#
# Two consequences of ptrace are load-bearing, and both are why this file
# exists instead of a one-line kcov invocation:
#
#   - kcov cannot nest. It sets PTRACE_O_TRACEFORK, so every descendant of
#     a traced process is ALREADY traced, and an inner kcov's TRACEME is
#     refused with EPERM. `mux --via` spawns a daemon, which is exactly that
#     shape. MUX_KCOV_ACTIVE makes a nested shim exec the real binary bare:
#     those lines are attributed to whichever ancestor is traced, or lost,
#     but the suite never dies.
#   - the pid the suite holds becomes kcov's, not the daemon's. SIGKILL is
#     the one signal a wrapper cannot forward, so a leg that kill -9s a
#     daemon would leave the real one alive holding its socket. e2e.sh's
#     hardkill answers that; see the comment there.
#   - kcov waits for the LAST traced descendant, and `mux d start -d`
#     daemonises: the launcher forks, the parent exits, and the daemon
#     reparents to init. Traced, that daemon outlives the scenario, so kcov
#     never exits and the suite's `wait` on it never returns — a hang at the
#     auto-start leg that reads as a slow run, and one that ignores SIGTERM,
#     so `timeout` will not free it either. So the shim reads the WORD that
#     forks: `mux d start -d` runs bare, and the plain `mux d start` a
#     scenario runs in the foreground is traced like anything else. That is
#     where server.zig's e2e lines come from, and widening the case back to
#     every `start` loses all of them for a gate that still goes green;
#     narrowing it to every `start` hangs the suite at the first `-d` leg.
#     The cost is the detached daemons' lines. `endpoint` is the same shape
#     one hop out and stays bare whole: it spawns the daemon it announces,
#     and the ssh handoff client then kill()s its ssh child and WAITS on it
#     — which under the shim is kcov, holding on for that daemon. The
#     handoff legs stalled there with attaches=0.
#
#     Not --exit-first-process, which is kcov's own answer to daemons: it
#     reparents the tracer, and then the real process is no longer a CHILD of
#     the pid the suite holds — which is precisely what hardkill needs it to
#     be. It trades this hang for a dead hardkill and a hang one leg later.
#
# A SIGKILLed wrapper writes nothing. kcov commits its database on clean exit
# only — measured across --collect-only, --output-interval=1000 and full
# report mode, all three yield no coverage.db when the wrapper is SIGKILLed.
# That is why e2e.sh's hardkill signals the child and then WAITS for the
# wrapper rather than killing both at once, and why the daemons this suite
# tears down with SIGKILL would otherwise contribute nothing at all.

set -eu
MUX="$1"; RAWMODE="$2"; DELAYPIPE="$3"; RENDER="$4"; PTYCLIENT="$5"
WSCLIENT="$6"

command -v kcov > /dev/null 2>&1 || {
    echo "coverage FAIL: this target needs kcov (pacman -S kcov)"
    echo "               It is a ptrace tracer, so no rebuild is required —"
    echo "               it reads DWARF from the binaries already built."
    exit 1
}

E2E="$(dirname "$0")/e2e.sh"
SRC="$(cd "$(dirname "$0")/../src" && pwd)"
OUT="${COV_OUT:-$(cd "$(dirname "$0")/.." && pwd)/zig-out/coverage}"
SHIM="$OUT/shim"
RAW="$OUT/raw"
rm -rf "$OUT"
mkdir -p "$SHIM" "$RAW"

# ONE shim, named `mux`, writing to a $$-suffixed directory per process:
# kcov cannot have two live processes share one output dir, and this suite
# runs a dozen daemons and hundreds of clients. They are merged below.
#
# COV_WRAP names which MODES to trace, and the shim tests the mode word
# ($1) exactly rather than matching anywhere in argv — a session named "d"
# would otherwise run its client traced in silence. The client mode is NOT
# in the default set, and that is a limitation rather than a preference: a
# traced client that auto-starts a daemon can never exit. The daemon is now
# an exec of THIS image from a fork of the client (spawn.zig) rather than of
# a sibling on PATH, and ptrace follows an exec, so it
# is traced whenever the client is, stays traced after it reparents to
# init, and kcov waits for its last traced descendant — a process designed
# never to end. The auto-start and `--via` legs wedge on this, and a
# SIGTERM will not free it. --exit-first-process is kcov's answer and does
# fix this one, but it reparents the tracer, and then the real process is
# no longer a child of the pid the suite holds — which breaks hardkill and
# moves the hang one leg later. Daemons carry server.zig, so tracing them
# and not the client is the side of the trade worth taking.
#
# Set COV_WRAP="client" to measure the client over a slice that avoids
# those legs (E2E_STOP_AFTER pairs with it), and merge the two runs by hand.
WRAP="${COV_WRAP:-d a web}"
# Validated in THIS shell, before anything is generated. The obvious spelling
# puts the check inside the case below, where `exit 1` only leaves the
# command substitution: the message prints, the suite runs anyway, and the
# whole thing exits 0 — a FAIL that does not fail.
for w in $WRAP; do
    case "$w" in
        d|a|web|client) ;;
        *) echo "coverage FAIL: COV_WRAP names unknown mode '$w'"
           echo "               known: d a web client"
           exit 1 ;;
    esac
done

real=$(cd "$(dirname "$MUX")" && pwd)/$(basename "$MUX")
# `client` is spelled as the empty mode inside the shim: the client is the
# mode you reach by typing no letter at all.
TRACED=""
for w in $WRAP; do
    case "$w" in client) TRACED="$TRACED :" ;; *) TRACED="$TRACED $w" ;; esac
done
cat > "$SHIM/mux" <<SHIM_EOF
#!/bin/sh
[ -n "\$MUX_KCOV_ACTIVE" ] && exec "$real" "\$@"
case "\${1:-}" in d|a|web) _m=\$1; _v=\${2:-} ;; *) _m=: ; _v=\${1:-} ;; esac
case " $TRACED " in *" \$_m "*) ;; *) exec "$real" "\$@" ;; esac
case "\$_m \$_v" in
"d endpoint") exec "$real" "\$@" ;;
"d start") case "\${3:-}" in -d|--detach) exec "$real" "\$@" ;; esac ;;
esac
MUX_KCOV_ACTIVE=1; export MUX_KCOV_ACTIVE
exec kcov --collect-only "--include-path=$SRC" "$RAW/mux.\$\$" "$real" "\$@"
SHIM_EOF
chmod +x "$SHIM/mux"

# e2e.sh prepends `dirname "$MUX"` to PATH, which is $SHIM here — so the
# `mux` an ssh shim runs by name is wrapped like the one the suite spells.
# Scenario boundaries, stamped by e2e.sh's ok(). kcov writes a database when
# a traced process EXITS, so a database's mtime and this file together say
# which scenario was running when it was written — which is the only
# per-scenario attribution available from a single run. It is exact for the
# short-lived clients and coarse for the dozen daemons that outlive several
# scenarios and land wherever they were killed.
OK_LOG="$OUT/scenarios.tsv"
export E2E_OK_LOG="$OK_LOG"

# Tracing costs roughly half again in wall clock, and several legs assert on a
# reconnect or a handoff healing inside a real-time budget. Those budgets are
# generous at native speed and merely tight under a tracer, so the suite is
# told to wait longer — not to demand less. See e2e.sh's TIME_SCALE note.
export E2E_TIME_SCALE="${COV_TIME_SCALE:-4}"

# The upgrade legs copy a candidate binary and watch it exec; a shim cannot be
# exec'd into. The real path goes beside the shim.
MUX_ELF="$MUX"; export MUX_ELF
set +e
"$E2E" "$SHIM/mux" "$RAWMODE" "$DELAYPIPE" "$RENDER" "$PTYCLIENT" "$WSCLIENT"
E2E_RC=$?
set -e

# A failing suite still collected everything up to the failure, and throwing
# that away would make the run you most want to look at the one that produces
# nothing — so the report below is still built. But the status is NOT
# swallowed: this exited 0 on a suite that failed at scenario 13, which is a
# green `make coverage` reporting on a third of the suite. It is re-raised at
# the bottom, after the report exists.
[ "$E2E_RC" -eq 0 ] || echo "coverage NOTE: e2e exited $E2E_RC; the report below covers the run up to that point"

DIRS=$(find "$RAW" -mindepth 1 -maxdepth 1 -type d | wc -l)
[ "$DIRS" -gt 0 ] || { echo "coverage FAIL: no kcov databases in $RAW"; exit 1; }
kcov --merge --include-path="$SRC" "$OUT/merged" "$RAW"/* > /dev/null 2>&1

JSON="$OUT/merged/kcov-merged/coverage.json"
[ -f "$JSON" ] || { echo "coverage FAIL: merge produced no $JSON"; exit 1; }

# Sorted by UNCOVERED lines, not by percentage: the question this report
# answers is "what is worth a test", and 0% of a 24-line file is not the
# same size of gap as 54% of server.zig.
echo
echo "coverage: $DIRS processes traced, e2e exited $E2E_RC"
echo
printf '%-22s %8s %13s %10s\n' file cov covered/total uncovered
awk -F'"' '/"file":/ {
    n = split($4, p, "/")
    printf "%-22s %7s%% %6d/%-6d %10d\n", p[n], $8, $12, $16, $16 - $12
}' "$JSON" | sort -k4 -rn
# The totals live at the top level of the same file, past the files array;
# anchored on the key names, so quoting of the values does not matter.
awk -F'[:,]' '
    /^  "percent_covered"/ { gsub(/[ "]/, "", $2); pct = $2 }
    /^  "covered_lines"/   { gsub(/[ "]/, "", $2); cov = $2 }
    /^  "total_lines"/     { gsub(/[ "]/, "", $2); tot = $2 }
    END { printf "\nTOTAL %s%%  %s/%s lines\n", pct, cov, tot }
' "$JSON"
echo
echo "html: file://$OUT/merged/kcov-merged/index.html"
echo "per-process databases: $RAW (mtimes attribute to $OK_LOG)"

# The suite's verdict, re-raised now that the report is written. A coverage
# run whose suite failed has measured part of a suite, and saying so is the
# difference between a number and a misleading number.
if [ "$E2E_RC" -ne 0 ]; then
    echo
    echo "coverage FAIL: e2e exited $E2E_RC — the figures above describe an"
    echo "               incomplete run, not this suite's coverage."
    exit "$E2E_RC"
fi