internal/smoke/scenario.go
Ref: Size: 16.9 KiB History
package smoke
import (
"context"
"errors"
"fmt"
"regexp"
"strings"
"time"
"github.com/a73x/eitri/internal/server/api/client"
)
// bootedPattern / panickedPattern classify a guest's serial console text. They
// mirror the boot-gate's grep -aiE serial-console patterns exactly.
var (
bootedPattern = regexp.MustCompile(`(?i)Welcome to.*Ubuntu|login:|Reached target.*Multi-User`)
panickedPattern = regexp.MustCompile(`(?i)Kernel panic|Cannot open root`)
errPollTimeout = errors.New("poll timeout")
)
// classifySerial inspects a guest's serial console text and reports whether it
// shows evidence of a successful userspace boot and/or a kernel panic /
// root-mount failure. It is pure — no I/O — so the boot-proof classification
// logic is fully unit-testable.
func classifySerial(text string) (booted, panicked bool) {
return bootedPattern.MatchString(text), panickedPattern.MatchString(text)
}
// proveBoot watches a VM's serial console until it shows a userspace boot
// (login prompt), a panic/root-mount failure (immediate FAIL), or the deadline
// passes. phase names which boot is being proven in the failure messages
// ("first boot", "after power cycle"). A console that is not there yet is just
// an empty iteration — the deadline is what decides.
func proveBoot(ctx context.Context, tail *consoleTail, now func() time.Time, sleep func(time.Duration), phase string) error {
err := pollLoop(ctx, now, sleep, 180*time.Second, 6*time.Second, func() (bool, error) {
booted, panicked := classifySerial(tail.text())
if panicked {
return false, fmt.Errorf("FAIL: guest panic / root-mount failure on the console (%s)", phase)
}
return booted, nil
})
if errors.Is(err, errPollTimeout) {
return fmt.Errorf("FAIL: no userspace boot evidence on the console within 180s (%s)%s", phase, tail.why())
}
return err
}
// vmAPI is the subset of the shared API client the scenario needs. Declaring
// it lets tests supply a fake instead of a real HTTP-backed *client.Client.
type vmAPI interface {
ListHosts(ctx context.Context) ([]client.Host, error)
CreateVM(ctx context.Context, req client.CreateVMRequest) (client.CreateVMResponse, error)
ListVMs(ctx context.Context) ([]client.VM, error)
DeleteVM(ctx context.Context, id string) error
PatchVM(ctx context.Context, id, powerState string) error
CreateExposure(ctx context.Context, vmID string, guestPort, hostPort int64, protocol string) (client.Exposure, error)
DeleteExposure(ctx context.Context, id string) error
CreateVolumeClaim(ctx context.Context, name string, sizeGB int64) (client.VolumeClaim, error)
DeleteVolumeClaim(ctx context.Context, id string) error
}
// getVM finds the VM with the given id in the current listing. The bool
// return reports whether it was present. There is deliberately no GET-one
// endpoint, so this list-filter is the smoke's lookup.
func getVM(ctx context.Context, c vmAPI, id string) (client.VM, bool, error) {
vms, err := c.ListVMs(ctx)
if err != nil {
return client.VM{}, false, err
}
for _, vm := range vms {
if vm.ID == id {
return vm, true, nil
}
}
return client.VM{}, false, nil
}
// proveHostRelease refuses a host whose agent is not running the release this
// run is proving. want == "" is no expectation at all, which is every run but a
// ship's: a boot gate proves binaries built from a working tree and their
// stamped version is "dev".
//
// It exists because a VM proves whatever agent it lands on, and every leg of
// this scenario — create, boot, gate SSH, published port, reap — is satisfied
// just as well by the release before this one. A ship that rolls the plane and
// fails to converge the fleet therefore looked, from the smoke, exactly like a
// ship that worked; twice it exited green with the hosts a release behind. What
// a mismatch here means is never that the VM is broken: it is that the fleet
// did not converge, so the message says so and says how to finish it.
//
// It guards this scenario's VM only. The MCP leg's VM places itself through the
// plane's own default placement and is deliberately left unguarded — out of
// scope here, and pinning its host would cost that leg the one thing it proves
// about placement, which is that the plane's default choice works at all.
func proveHostRelease(h client.Host, want string) error {
if want == "" || h.AgentVersion == want {
return nil
}
reported := h.AgentVersion
if reported == "" {
reported = "no version at all"
}
return fmt.Errorf("FAIL: host %s is running agent %s, not %s — the plane is serving %s, "+
"so the fleet did not converge. A VM placed here would prove the release before "+
"this one. Converge the fleet and resume: scripts/ship.sh --from 8",
h.Name, reported, want, want)
}
// placeableHost picks the host this run will place its VM on: the first one
// that is connected.
//
// A dark host is skipped rather than judged. The ship has already dealt with it
// one stage earlier — it cannot take an upgrade offer while it is disconnected,
// so stage 8 names it, warns, and carries on — and a host that cannot take an
// offer cannot take a VM either. Letting it reach proveHostRelease meant a
// machine that was merely switched off could fail a release two other hosts had
// converged and were ready to prove, with no retry that could ever pass while it
// stayed off.
//
// This is deliberately not "search for a host on the release". Every connected
// host is still judged by proveHostRelease, so a fleet that is up and behind
// fails exactly as it did — which is the #33 defect this whole check exists for.
// The only hosts skipped are the ones nobody could have converged.
func placeableHost(hosts []client.Host) (client.Host, error) {
for _, h := range hosts {
if h.Online {
return h, nil
}
}
names := make([]string, 0, len(hosts))
for _, h := range hosts {
names = append(names, h.Name)
}
return client.Host{}, fmt.Errorf("FAIL: no connected host to place a VM on — every host is dark (%s). "+
"The plane is up, so this is the fleet, not the release: bring a host back and resume "+
"with scripts/ship.sh --from 8", strings.Join(names, ", "))
}
// gateHooks bundles the optional SSH-CA gate steps. nil means "skip the gate".
type gateHooks struct {
register func(ctx context.Context) error // upload the smoke user CA to the tenant (before create)
exec func(ctx context.Context, vmName string) error // reach the guest through the gate (after boot)
// run reaches a guest the same way exec does and hands back what the
// command printed. It is how a leg proves something INSIDE a guest — the
// volume leg's marker — rather than proving the gate itself.
run func(ctx context.Context, vmName, cmd string) (string, error)
}
// mcpLeg is the remote-MCP proof: one full cycle driven entirely through the
// control plane's /mcp endpoint with a bearer PAT. nil means "skip it".
type mcpLeg func(ctx context.Context, vmName string) error
// pollLoop calls attempt repeatedly (with sleep between calls) until attempt
// reports done, returns a non-nil error, or the deadline (now()+timeout) is
// reached, in which case it returns errPollTimeout. now and sleep are
// injected so callers can drive the deadline logic with a virtual clock.
func pollLoop(ctx context.Context, now func() time.Time, sleep func(time.Duration), timeout, interval time.Duration, attempt func() (done bool, err error)) error {
deadline := now().Add(timeout)
for {
if err := ctx.Err(); err != nil {
return err
}
done, err := attempt()
if err != nil {
return err
}
if done {
return nil
}
if !now().Before(deadline) {
return errPollTimeout
}
sleep(interval)
}
}
// runScenario drives the full register -> create -> ready -> boot-proof ->
// gate-exec -> published-port -> reap sequence against c (the API), dialConsole
// (the serial console the boot proofs watch), readPubKey (the local SSH key
// source), and dialBanner (the published-port transport). vmName is the
// pre-generated name for the throwaway VM. gate, when non-nil, registers the
// smoke's user CA with the tenant before create (the guest bakes its trusted
// CAs at boot, so registration MUST happen first) and proves gate SSH access —
// plus the cloud-init merge this VM's tenant document exercises — after the
// boot-proof; a hard gate, so a failure there fails the scenario.
// mcp, when non-nil, drives a second VM's whole life through the remote MCP
// endpoint — that leg owns its own VM, created after it registers its own CA
// with the tenant, so the guest trusts the certificates eitri presents.
// expectAgentVersion, when non-empty, is the release the host the VM lands on
// must be running (see proveHostRelease); empty skips that check.
// now/sleep are the injected clock so the poll deadlines are unit-testable
// without real waiting. On success it returns the human-readable COMPLETE
// line; on any failure it returns a descriptive error.
func runScenario(ctx context.Context, vmName string, c vmAPI, dialConsole consoleDialer, gate *gateHooks, mcp mcpLeg, expectAgentVersion string, now func() time.Time, sleep func(time.Duration), readPubKey func() string, dialBanner bannerFunc) (string, error) {
if gate != nil {
if err := gate.register(ctx); err != nil {
return "", fmt.Errorf("register smoke user CA: %w", err)
}
}
hostList, err := c.ListHosts(ctx)
if err != nil {
return "", fmt.Errorf("list hosts: %w", err)
}
if len(hostList) == 0 {
return "", errors.New("no hosts available")
}
host, err := placeableHost(hostList)
if err != nil {
return "", err
}
// Before anything is created: there is no reason to boot-prove a VM on an
// agent this run was never meant to prove. The list is still not searched
// for a host that does match — a CONNECTED fleet where only some hosts
// converged is itself the defect, and shopping among those would hide
// exactly the bug this check exists to expose.
if err := proveHostRelease(host, expectAgentVersion); err != nil {
return "", err
}
hostID := host.ID
sshKey := readPubKey()
start := now()
// The VM carries a tenant cloud-init (see byoCloudInit) so this one guest is
// also the guest the merge proof needs: the gate leg below reads back both
// eitri's grown root and the tenant's own marker files. No extra boot buys
// that, and a VM created with an empty document would never exercise the
// merge at all.
created, err := c.CreateVM(ctx, client.CreateVMRequest{HostID: hostID, Name: vmName, SSHAuthorizedKey: sshKey, CloudInit: byoCloudInit})
if err != nil {
return "", fmt.Errorf("create vm: %w", err)
}
vmID := created.ID
var lastPhase string
err = pollLoop(ctx, now, sleep, 600*time.Second, 5*time.Second, func() (bool, error) {
vm, _, err := getVM(ctx, c, vmID)
if err != nil {
return false, fmt.Errorf("poll vm ready: %w", err)
}
lastPhase = vm.Phase
return vm.Phase == "ready" && vm.AssignedIP != "", nil
})
if err != nil {
if errors.Is(err, errPollTimeout) {
return "", fmt.Errorf("FAIL: VM not ready within 600s (phase=%s)", lastPhase)
}
return "", err
}
coldStart := now().Sub(start)
// The boot proof watches the guest's own serial console through the control
// plane, exactly as an operator watching the VM come up in a browser does:
// the host replays its console backlog on attach, so the whole boot is
// there to read however late the attach lands.
boot := attachConsole(ctx, dialConsole, vmID)
bootErr := proveBoot(ctx, boot, now, sleep, "first boot")
boot.close()
if bootErr != nil {
return "", bootErr
}
gateOK := false
exposureOK := false
if gate != nil {
if err := gate.exec(ctx, vmName); err != nil {
return "", err
}
gateOK = true
}
// Publish the guest's own sshd on its host and read the banner back
// through the listener. The address comes from the exposure record itself —
// the grant names where it is published, so the leg dials what the fleet
// told it to dial rather than an address assembled on the side.
if err := proveExposure(ctx, c, vmID, now, sleep, dialBanner); err != nil {
return "", err
}
exposureOK = true
// Remote MCP: the same fleet, driven by a bearer PAT over HTTP with no
// local install and no uploaded CA. It creates and destroys its own VM,
// and it is where the UDP published port is proven — the proof needs a
// listener inside the guest, and this is the leg that can start one.
mcpOK := false
if mcp != nil {
if err := mcp(ctx, vmName+"-mcp"); err != nil {
return "", err
}
mcpOK = true
}
// Power cycle: prove the guest comes BACK. A first boot runs on the
// kernel's in-memory partition table; only a stop→start proves the
// on-disk GPT survived growpart. The sector-0 regression hid exactly
// here — every first boot green, every persistent VM lost at its first
// reboot.
//
// The console is attached BEFORE the cycle starts and held across it, so
// the second boot is watched from a stream that was already open when the
// guest went down. The mark below is what keeps the proof honest: console
// history survives a restart, so only bytes that arrive after the start
// command can answer for the boot that command triggers.
reboot := attachConsole(ctx, dialConsole, vmID)
defer reboot.close()
if err := c.PatchVM(ctx, vmID, "stopped"); err != nil {
return "", fmt.Errorf("patch vm stopped: %w", err)
}
err = pollLoop(ctx, now, sleep, 120*time.Second, 5*time.Second, func() (bool, error) {
vm, present, err := getVM(ctx, c, vmID)
if err != nil {
return false, fmt.Errorf("poll vm stopped: %w", err)
}
return present && vm.ActualPower == "stopped", nil
})
if err != nil {
if errors.Is(err, errPollTimeout) {
return "", errors.New("FAIL: VM did not power off within 120s of the power cycle's stop")
}
return "", err
}
reboot.mark()
if err := c.PatchVM(ctx, vmID, "running"); err != nil {
return "", fmt.Errorf("patch vm running: %w", err)
}
err = pollLoop(ctx, now, sleep, 120*time.Second, 5*time.Second, func() (bool, error) {
vm, present, err := getVM(ctx, c, vmID)
if err != nil {
return false, fmt.Errorf("poll vm restarted: %w", err)
}
return present && vm.ActualPower == "running", nil
})
if err != nil {
if errors.Is(err, errPollTimeout) {
return "", errors.New("FAIL: VM did not power on within 120s of the power cycle's start")
}
return "", err
}
rebootErr := proveBoot(ctx, reboot, now, sleep, "after power cycle")
// The tail's last use, so it is detached HERE rather than left to the
// deferred close: the volume leg below runs for minutes after this VM is
// reaped, and a tail still open would spend them re-dialing a console that
// no longer has a VM behind it. close is idempotent, so the defer above
// stays as the backstop for the early returns between attach and here.
reboot.close()
if rebootErr != nil {
return "", rebootErr
}
if gate != nil {
if err := gate.exec(ctx, vmName); err != nil {
return "", fmt.Errorf("gate SSH after power cycle: %w", err)
}
}
if err := c.DeleteVM(ctx, vmID); err != nil {
return "", fmt.Errorf("delete vm: %w", err)
}
// The reap window must outlast the agent's tombstone grace — the quarantine
// a deleted VM sits in before the agent destroys it. That grace defaults to
// five minutes, and a fielded plane legitimately runs the default, so the
// window is that plus margin for teardown and the sync ack. A healthy reap
// still ends the poll the tick it lands; only a broken one waits this out.
err = pollLoop(ctx, now, sleep, 7*time.Minute, 5*time.Second, func() (bool, error) {
_, present, err := getVM(ctx, c, vmID)
if err != nil {
return false, fmt.Errorf("poll vm reaped: %w", err)
}
return !present, nil
})
if err != nil {
if errors.Is(err, errPollTimeout) {
return "", errors.New("FAIL: VM not hard-deleted within 7m of tombstone")
}
return "", err
}
// Durable storage: a claim that outlives the VM it was attached to. It runs
// on its own pair of VMs — the proof needs one guest to die while another
// picks the volume up — and only where the gate is configured, because the
// marker is written and read INSIDE the guests.
//
// It runs here, after this scenario's own VM is reaped, so the smoke never
// asks the host for more than two guests at once. Its VMs are serial by
// nature (the second cannot exist until the first is destroyed), so the
// host sees one smoke VM at a time from this point on.
volumeOK := false
if gate != nil {
if err := proveVolume(ctx, c, hostID, vmName, gate, readPubKey, now, sleep); err != nil {
return "", err
}
volumeOK = true
}
msg := fmt.Sprintf("SMOKE COMPLETE — booted under UEFI, cold_start=%ds, reboot: ok, reaped OK", int64(coldStart.Seconds()))
if gateOK {
// One clause per thing the gate leg proved: reaching the guest with a
// CA-signed certificate, and the tenant cloud-init the guest booted from
// leaving eitri's own write_files and runcmd intact (see gateExec).
msg += ", gate SSH: ok, BYO cloud-init merge: ok"
}
if exposureOK {
msg += ", exposed port: ok"
}
if volumeOK {
msg += ", volume outlived its VM: ok"
}
if mcpOK {
msg += ", remote MCP: ok, published UDP port: ok"
}
return msg, nil
}