internal/smoke/scenario_test.go
Ref: Size: 36.2 KiB History
package smoke
import (
"context"
"errors"
"fmt"
"io"
"strings"
"testing"
"time"
"github.com/a73x/eitri/internal/server/api/client"
)
// --- classifySerial -------------------------------------------------------
func TestClassifySerialBooted(t *testing.T) {
text := "[ 5.123456] Ubuntu 22.04.3 LTS ubuntu-vm ttyS0\n\nubuntu-vm login: "
booted, panicked := classifySerial(text)
if !booted {
t.Error("booted = false, want true")
}
if panicked {
t.Error("panicked = true, want false")
}
}
func TestClassifySerialPanic(t *testing.T) {
text := "[ 2.345678] Kernel panic - not syncing: VFS: Unable to mount root fs on unknown-block(0,0)"
booted, panicked := classifySerial(text)
if booted {
t.Error("booted = true, want false")
}
if !panicked {
t.Error("panicked = false, want true")
}
}
func TestClassifySerialStillBooting(t *testing.T) {
text := "[ 0.123456] Booting Linux on physical CPU 0x0\n[ 0.234567] Linux version 6.5.0"
booted, panicked := classifySerial(text)
if booted {
t.Error("booted = true, want false")
}
if panicked {
t.Error("panicked = true, want false")
}
}
// --- fakes for runScenario -------------------------------------------------
// fakeClock is a controllable now()/sleep() pair: sleep advances the virtual
// clock instead of waiting, so deadline logic runs at test speed. It yields for
// a moment as it does so, because what the boot proofs poll is filled by a real
// goroutine reading a real stream — a poll loop that never gave up the
// processor would spend its whole virtual deadline before that goroutine ran.
type fakeClock struct{ t time.Time }
func (c *fakeClock) now() time.Time { return c.t }
func (c *fakeClock) sleep(d time.Duration) {
c.t = c.t.Add(d)
time.Sleep(time.Millisecond)
}
// testAPI implements vmAPI by delegating to per-test closures. patchVMFunc,
// createExposureFunc, deleteExposureFunc, createVolumeClaimFunc, and
// deleteVolumeClaimFunc default to a benign answer when nil, so a test whose
// subject is elsewhere need not supply them.
type testAPI struct {
listHostsFunc func(ctx context.Context) ([]client.Host, error)
createVMFunc func(ctx context.Context, req client.CreateVMRequest) (client.CreateVMResponse, error)
listVMsFunc func(ctx context.Context) ([]client.VM, error)
deleteVMFunc func(ctx context.Context, id string) error
patchVMFunc func(ctx context.Context, id, powerState string) error
createExposureFunc func(ctx context.Context, vmID string, guestPort, hostPort int64, protocol string) (client.Exposure, error)
deleteExposureFunc func(ctx context.Context, id string) error
createVolumeClaimFunc func(ctx context.Context, name string, sizeGB int64) (client.VolumeClaim, error)
deleteVolumeClaimFunc func(ctx context.Context, id string) error
}
func (a *testAPI) ListHosts(ctx context.Context) ([]client.Host, error) {
return a.listHostsFunc(ctx)
}
func (a *testAPI) CreateVM(ctx context.Context, req client.CreateVMRequest) (client.CreateVMResponse, error) {
return a.createVMFunc(ctx, req)
}
func (a *testAPI) ListVMs(ctx context.Context) ([]client.VM, error) {
return a.listVMsFunc(ctx)
}
func (a *testAPI) DeleteVM(ctx context.Context, id string) error { return a.deleteVMFunc(ctx, id) }
func (a *testAPI) PatchVM(ctx context.Context, id, powerState string) error {
if a.patchVMFunc == nil {
return nil
}
return a.patchVMFunc(ctx, id, powerState)
}
func (a *testAPI) CreateExposure(ctx context.Context, vmID string, guestPort, hostPort int64, protocol string) (client.Exposure, error) {
if a.createExposureFunc == nil {
return client.Exposure{ID: "x-fake", HostPort: 30080, HostAddr: "192.168.0.190", Protocol: protocol}, nil
}
return a.createExposureFunc(ctx, vmID, guestPort, hostPort, protocol)
}
func (a *testAPI) DeleteExposure(ctx context.Context, id string) error {
if a.deleteExposureFunc == nil {
return nil
}
return a.deleteExposureFunc(ctx, id)
}
func (a *testAPI) CreateVolumeClaim(ctx context.Context, name string, sizeGB int64) (client.VolumeClaim, error) {
if a.createVolumeClaimFunc == nil {
return client.VolumeClaim{ID: "claim-fake", Name: name, SizeGB: sizeGB, Status: "pending"}, nil
}
return a.createVolumeClaimFunc(ctx, name, sizeGB)
}
func (a *testAPI) DeleteVolumeClaim(ctx context.Context, id string) error {
if a.deleteVolumeClaimFunc == nil {
return nil
}
return a.deleteVolumeClaimFunc(ctx, id)
}
func noopReadPubKey() string { return "ssh-ed25519 AAAAfake test@smoke" }
// bootedGuest is what a guest that reached userspace has on its console.
const bootedGuest = "Ubuntu 24.04 LTS ubuntu-vm login: "
// powerCycleConsole gives api a console and the power hooks that make one
// scripted power cycle observable: attaching replays the console's history (as
// a host does), the stop waits until the tail attached for the reboot proof has
// taken that history — so the proof's mark has something to discard — and the
// start puts afterStart on the stream. It wraps whatever power hook api already
// has, so the fake keeps actuating its own power state.
func powerCycleConsole(t *testing.T, api *testAPI, history, afterStart string) *fakeConsole {
t.Helper()
console := newFakeConsole(history)
actuate := api.patchVMFunc
api.patchVMFunc = func(ctx context.Context, id, powerState string) error {
if actuate != nil {
if err := actuate(ctx, id, powerState); err != nil {
return err
}
}
switch powerState {
case "stopped":
console.waitReplayed(t, 2) // the reboot proof's own attach
case "running":
console.write(t, afterStart)
}
return nil
}
return console
}
// okBanner is the banner a converged listener answers with, for tests whose
// subject is not the exposure leg.
func okBanner(ctx context.Context, addr string) (string, error) { return "SSH-2.0-Test\r\n", nil }
// --- runScenario: success path ---------------------------------------------
func TestRunScenarioSuccess(t *testing.T) {
listVMsCalls := 0
power := "running" // actual power the fake reports; PatchVM moves it
deleted := false
api := &testAPI{
listHostsFunc: func(ctx context.Context) ([]client.Host, error) {
return []client.Host{{ID: "host-1", Online: true, UplinkAddr: "192.168.0.190"}}, nil
},
createVMFunc: func(ctx context.Context, req client.CreateVMRequest) (client.CreateVMResponse, error) {
if req.HostID != "host-1" {
t.Errorf("CreateVM HostID = %q, want host-1", req.HostID)
}
return client.CreateVMResponse{ID: "vm-1"}, nil
},
listVMsFunc: func(ctx context.Context) ([]client.VM, error) {
listVMsCalls++
switch {
case deleted:
// Reap poll: absent from the first check.
return nil, nil
case listVMsCalls < 3:
return []client.VM{{ID: "vm-1", Phase: "booting"}}, nil
default:
return []client.VM{{ID: "vm-1", Phase: "ready", AssignedIP: "10.0.0.9", ActualPower: power}}, nil
}
},
patchVMFunc: func(ctx context.Context, id, powerState string) error {
if id != "vm-1" {
t.Errorf("patchVM id = %q, want vm-1", id)
}
power = powerState // the fake actuates instantly
return nil
},
deleteVMFunc: func(ctx context.Context, id string) error {
if id != "vm-1" {
t.Errorf("deleteVM id = %q, want vm-1", id)
}
deleted = true
return nil
},
}
// The console replays the first boot on every attach — including the one
// the reboot proof makes — and the guest prints its second boot when the
// start command lands.
console := powerCycleConsole(t, api, bootedGuest, "[ 0.9] Booting Linux\nubuntu-vm login: ")
clock := &fakeClock{t: time.Unix(0, 0)}
msg, err := runScenario(context.Background(), "smoke-test", api, console.dial, nil, nil, "", clock.now, clock.sleep, noopReadPubKey, okBanner)
if err != nil {
t.Fatalf("runScenario: %v", err)
}
if !strings.Contains(msg, "SMOKE COMPLETE") || !strings.Contains(msg, "reaped OK") {
t.Errorf("message = %q, want SMOKE COMPLETE ... reaped OK", msg)
}
if !strings.Contains(msg, "cold_start=") {
t.Errorf("message = %q, want cold_start=", msg)
}
if !strings.Contains(msg, "reboot: ok") {
t.Errorf("message = %q, want reboot: ok", msg)
}
if !strings.Contains(msg, "exposed port: ok") {
t.Errorf("message = %q, want exposed port: ok", msg)
}
// One attach for the first boot, a second held across the power cycle.
if console.attachCount() < 2 {
t.Errorf("console attaches = %d, want the reboot proof to attach its own stream", console.attachCount())
}
}
// TestRunScenarioRebootDeathFails pins the regression that motivated the
// power-cycle leg, and with it the freshness rule the console proof rests on: a
// VM whose second boot never reaches userspace (stale on-disk GPT → initramfs
// emergency mode) must FAIL the smoke. The console here replays the FIRST
// boot's login prompt to the stream watching the restart — exactly what a host
// does — so a proof that counted replayed history would call this dead VM
// green.
func TestRunScenarioRebootDeathFails(t *testing.T) {
power := "running"
api := &testAPI{
listHostsFunc: func(ctx context.Context) ([]client.Host, error) {
return []client.Host{{ID: "host-1", Online: true, UplinkAddr: "192.168.0.190"}}, nil
},
createVMFunc: func(ctx context.Context, req client.CreateVMRequest) (client.CreateVMResponse, error) {
return client.CreateVMResponse{ID: "vm-1"}, nil
},
listVMsFunc: func(ctx context.Context) ([]client.VM, error) {
return []client.VM{{ID: "vm-1", Phase: "ready", AssignedIP: "10.0.0.9", ActualPower: power}}, nil
},
patchVMFunc: func(ctx context.Context, id, powerState string) error {
power = powerState
return nil
},
deleteVMFunc: func(ctx context.Context, id string) error {
t.Fatal("DeleteVM must not be called when the reboot proof failed")
return nil
},
}
// After the restart: an emergency shell, never a login prompt.
console := powerCycleConsole(t, api, bootedGuest, "Press Enter for system maintenance")
clock := &fakeClock{t: time.Unix(0, 0)}
_, err := runScenario(context.Background(), "smoke-test", api, console.dial, nil, nil, "", clock.now, clock.sleep, noopReadPubKey, okBanner)
if err == nil {
t.Fatal("runScenario: want error for a VM that never came back, got nil")
}
if !strings.Contains(err.Error(), "power cycle") {
t.Errorf("error = %q, want it to mention the power cycle", err.Error())
}
}
// --- runScenario: published port ----------------------------------------
// TestRunScenarioExposureFailureFails pins the published-port leg as a hard
// gate: a host whose listener answers with something that is not an sshd fails
// the scenario, and the VM is never reaped over the failure.
func TestRunScenarioExposureFailureFails(t *testing.T) {
var calls []string
api := happyPathAPI(t, &calls)
api.deleteVMFunc = func(ctx context.Context, id string) error {
t.Fatal("DeleteVM must not be called when the published port failed")
return nil
}
revoked := ""
api.deleteExposureFunc = func(ctx context.Context, id string) error { revoked = id; return nil }
badBanner := func(ctx context.Context, addr string) (string, error) { return "HTTP/1.1 400\r\n", nil }
clock := &fakeClock{t: time.Unix(0, 0)}
_, err := runScenario(context.Background(), "smoke-test", api, newFakeConsole(bootedGuest).dial, nil, nil, "", clock.now, clock.sleep, noopReadPubKey, badBanner)
if err == nil {
t.Fatal("runScenario: want error, got nil")
}
if !strings.Contains(err.Error(), "SSH-2.0") {
t.Errorf("error = %q, want it to name the banner it wanted", err.Error())
}
if revoked != "x-fake" {
t.Errorf("revoked exposure = %q, want the failed leg to revoke its grant", revoked)
}
}
// --- runScenario: serial panic -----------------------------------------
func TestRunScenarioSerialPanicFails(t *testing.T) {
api := &testAPI{
listHostsFunc: func(ctx context.Context) ([]client.Host, error) {
return []client.Host{{ID: "host-1", Online: true, UplinkAddr: "192.168.0.190"}}, nil
},
createVMFunc: func(ctx context.Context, req client.CreateVMRequest) (client.CreateVMResponse, error) {
return client.CreateVMResponse{ID: "vm-1"}, nil
},
listVMsFunc: func(ctx context.Context) ([]client.VM, error) {
return []client.VM{{ID: "vm-1", Phase: "ready", AssignedIP: "10.0.0.9"}}, nil
},
deleteVMFunc: func(ctx context.Context, id string) error {
t.Fatal("DeleteVM should not be called after a panic")
return nil
},
}
console := newFakeConsole("Kernel panic - not syncing: VFS: Unable to mount root fs")
clock := &fakeClock{t: time.Unix(0, 0)}
_, err := runScenario(context.Background(), "smoke-test", api, console.dial, nil, nil, "", clock.now, clock.sleep, noopReadPubKey, okBanner)
if err == nil {
t.Fatal("runScenario: want error, got nil")
}
if !strings.Contains(err.Error(), "panic") {
t.Errorf("error = %q, want it to mention panic", err.Error())
}
}
// --- runScenario: never-ready timeout ---------------------------------
func TestRunScenarioNeverReadyTimesOut(t *testing.T) {
api := &testAPI{
listHostsFunc: func(ctx context.Context) ([]client.Host, error) {
return []client.Host{{ID: "host-1", Online: true, UplinkAddr: "192.168.0.190"}}, nil
},
createVMFunc: func(ctx context.Context, req client.CreateVMRequest) (client.CreateVMResponse, error) {
return client.CreateVMResponse{ID: "vm-1"}, nil
},
listVMsFunc: func(ctx context.Context) ([]client.VM, error) {
return []client.VM{{ID: "vm-1", Phase: "booting"}}, nil
},
deleteVMFunc: func(ctx context.Context, id string) error {
t.Fatal("DeleteVM should not be called when the VM never becomes ready")
return nil
},
}
attached := 0
console := func(ctx context.Context, vmID string) (io.ReadWriteCloser, error) {
attached++
return nil, errors.New("console should not be attached before the VM is ready")
}
clock := &fakeClock{t: time.Unix(0, 0)}
_, err := runScenario(context.Background(), "smoke-test", api, console, nil, nil, "", clock.now, clock.sleep, noopReadPubKey, okBanner)
if err == nil {
t.Fatal("runScenario: want error, got nil")
}
if attached != 0 {
t.Errorf("console attaches = %d, want none before the VM is ready", attached)
}
if !strings.Contains(err.Error(), "FAIL: VM not ready within 600s") {
t.Errorf("error = %q, want FAIL: VM not ready within 600s ...", err.Error())
}
if !strings.Contains(err.Error(), "phase=booting") {
t.Errorf("error = %q, want it to include phase=booting", err.Error())
}
}
// --- runScenario: never-reaped timeout ---------------------------------
func TestRunScenarioNeverReapedTimesOut(t *testing.T) {
power := "running"
api := &testAPI{
listHostsFunc: func(ctx context.Context) ([]client.Host, error) {
return []client.Host{{ID: "host-1", Online: true, UplinkAddr: "192.168.0.190"}}, nil
},
createVMFunc: func(ctx context.Context, req client.CreateVMRequest) (client.CreateVMResponse, error) {
return client.CreateVMResponse{ID: "vm-1"}, nil
},
listVMsFunc: func(ctx context.Context) ([]client.VM, error) {
// Always present, even after delete — models a stuck reap.
return []client.VM{{ID: "vm-1", Phase: "ready", AssignedIP: "10.0.0.9", ActualPower: power}}, nil
},
patchVMFunc: func(ctx context.Context, id, powerState string) error {
power = powerState
return nil
},
deleteVMFunc: func(ctx context.Context, id string) error { return nil },
}
console := powerCycleConsole(t, api, bootedGuest, "ubuntu-vm login: ")
clock := &fakeClock{t: time.Unix(0, 0)}
_, err := runScenario(context.Background(), "smoke-test", api, console.dial, nil, nil, "", clock.now, clock.sleep, noopReadPubKey, okBanner)
if err == nil {
t.Fatal("runScenario: want error, got nil")
}
if !strings.Contains(err.Error(), "FAIL: VM not hard-deleted within 7m") {
t.Errorf("error = %q, want FAIL: VM not hard-deleted within 7m ...", err.Error())
}
}
// --- runScenario: no hosts ------------------------------------------------
func TestRunScenarioNoHosts(t *testing.T) {
api := &testAPI{
listHostsFunc: func(ctx context.Context) ([]client.Host, error) { return nil, nil },
createVMFunc: func(ctx context.Context, req client.CreateVMRequest) (client.CreateVMResponse, error) {
t.Fatal("CreateVM should not be called with no hosts")
return client.CreateVMResponse{}, nil
},
listVMsFunc: func(ctx context.Context) ([]client.VM, error) {
t.Fatal("ListVMs should not be called with no hosts")
return nil, nil
},
deleteVMFunc: func(ctx context.Context, id string) error {
t.Fatal("DeleteVM should not be called with no hosts")
return nil
},
}
clock := &fakeClock{t: time.Unix(0, 0)}
_, err := runScenario(context.Background(), "smoke-test", api, nil, nil, nil, "", clock.now, clock.sleep, noopReadPubKey, okBanner)
if err == nil {
t.Fatal("runScenario: want error, got nil")
}
}
// --- runScenario: gate hooks -----------------------------------------------
// happyPathAPI returns a testAPI that succeeds all the way through reap,
// tracking call order in calls (a shared slice each hook also appends to). It
// keeps a registry rather than one hard-coded VM, because the legs that run
// under a gate (the volume leg) create VMs of their own alongside the
// scenario's: each one boots after a couple of polls and vanishes when deleted.
func happyPathAPI(t *testing.T, calls *[]string) *testAPI {
t.Helper()
type fakeVM struct {
polls int
power string
}
vms := map[string]*fakeVM{}
created := 0
return &testAPI{
listHostsFunc: func(ctx context.Context) ([]client.Host, error) {
return []client.Host{{ID: "host-1", Online: true, UplinkAddr: "192.168.0.190"}}, nil
},
createVMFunc: func(ctx context.Context, req client.CreateVMRequest) (client.CreateVMResponse, error) {
*calls = append(*calls, "createVM")
created++
id := fmt.Sprintf("vm-%d", created)
vms[id] = &fakeVM{power: "running"}
return client.CreateVMResponse{ID: id}, nil
},
listVMsFunc: func(ctx context.Context) ([]client.VM, error) {
var out []client.VM
for id, vm := range vms {
vm.polls++
if vm.polls < 3 {
out = append(out, client.VM{ID: id, Phase: "booting"})
continue
}
out = append(out, client.VM{ID: id, Phase: "ready", AssignedIP: "10.0.0.9", ActualPower: vm.power})
}
return out, nil
},
patchVMFunc: func(ctx context.Context, id, powerState string) error {
if vm, ok := vms[id]; ok {
vm.power = powerState
}
return nil
},
deleteVMFunc: func(ctx context.Context, id string) error {
delete(vms, id)
return nil
},
}
}
// okGateRun answers the volume leg the way a working fleet does: the write
// command prints nothing, and the read hands back the marker that was written.
func okGateRun(ctx context.Context, vmName, cmd string) (string, error) {
if strings.Contains(cmd, "cat /mnt/marker") {
return volumeMarker + "\n", nil
}
return "", nil
}
func TestRunScenarioGateRegistersBeforeCreateAndExecsAfterBoot(t *testing.T) {
var calls []string
api := happyPathAPI(t, &calls)
gate := &gateHooks{
register: func(ctx context.Context) error {
calls = append(calls, "register")
return nil
},
exec: func(ctx context.Context, vmName string) error {
if vmName != "smoke-test" {
t.Errorf("gate.exec vmName = %q, want smoke-test", vmName)
}
calls = append(calls, "exec")
return nil
},
run: okGateRun,
}
clock := &fakeClock{t: time.Unix(0, 0)}
msg, err := runScenario(context.Background(), "smoke-test", api, powerCycleConsole(t, api, bootedGuest, "ubuntu-vm login: ").dial, gate, nil, "", clock.now, clock.sleep, noopReadPubKey, okBanner)
if err != nil {
t.Fatalf("runScenario: %v", err)
}
if !strings.Contains(msg, "gate SSH: ok") {
t.Errorf("message = %q, want it to mention gate SSH: ok", msg)
}
// The gate leg is also where the cloud-init merge is proven, and a run that
// proved it says so — a report that only says "gate SSH: ok" leaves a reader
// to guess whether the tenant document was exercised at all.
if !strings.Contains(msg, "BYO cloud-init merge: ok") {
t.Errorf("message = %q, want it to mention BYO cloud-init merge: ok", msg)
}
if !strings.Contains(msg, "volume outlived its VM: ok") {
t.Errorf("message = %q, want it to mention the volume leg", msg)
}
// exec appears twice: once after the first boot proof, once after the
// power-cycle proof — SSH through the gate must survive a reboot too. The
// two creates at the end are the volume leg's own pair of VMs: it only
// runs where the gate can reach inside a guest, and it waits until this
// scenario's VM is reaped so the host is never asked for a third.
want := []string{"register", "createVM", "exec", "exec", "createVM", "createVM"}
if len(calls) != len(want) {
t.Fatalf("call order = %v, want %v", calls, want)
}
for i, c := range want {
if calls[i] != c {
t.Errorf("call order = %v, want %v", calls, want)
break
}
}
}
func TestRunScenarioCreatesItsVMWithATenantCloudInit(t *testing.T) {
// The scenario's own VM carries the tenant document, so the gate leg proves
// the cloud-init merge on a guest that already exists rather than on a
// second boot added for the purpose.
var calls []string
api := happyPathAPI(t, &calls)
var got client.CreateVMRequest
create := api.createVMFunc
api.createVMFunc = func(ctx context.Context, req client.CreateVMRequest) (client.CreateVMResponse, error) {
got = req
return create(ctx, req)
}
clock := &fakeClock{t: time.Unix(0, 0)}
if _, err := runScenario(context.Background(), "smoke-test", api, powerCycleConsole(t, api, bootedGuest, "ubuntu-vm login: ").dial, nil, nil, "", clock.now, clock.sleep, noopReadPubKey, okBanner); err != nil {
t.Fatalf("runScenario: %v", err)
}
if got.CloudInit != byoCloudInit {
t.Errorf("CreateVM CloudInit = %q, want the tenant document the merge proof reads back", got.CloudInit)
}
}
func TestRunScenarioGateRegisterErrorAbortsBeforeCreate(t *testing.T) {
api := &testAPI{
listHostsFunc: func(ctx context.Context) ([]client.Host, error) {
t.Fatal("ListHosts should not be called when register fails")
return nil, nil
},
createVMFunc: func(ctx context.Context, req client.CreateVMRequest) (client.CreateVMResponse, error) {
t.Fatal("CreateVM should not be called when register fails")
return client.CreateVMResponse{}, nil
},
listVMsFunc: func(ctx context.Context) ([]client.VM, error) {
t.Fatal("ListVMs should not be called when register fails")
return nil, nil
},
deleteVMFunc: func(ctx context.Context, id string) error {
t.Fatal("DeleteVM should not be called when register fails")
return nil
},
}
gate := &gateHooks{
register: func(ctx context.Context) error { return errors.New("upload boom") },
exec: func(ctx context.Context, vmName string) error {
t.Fatal("exec should not be called when register fails")
return nil
},
}
clock := &fakeClock{t: time.Unix(0, 0)}
_, err := runScenario(context.Background(), "smoke-test", api, nil, gate, nil, "", clock.now, clock.sleep, noopReadPubKey, okBanner)
if err == nil {
t.Fatal("runScenario: want error, got nil")
}
if !strings.Contains(err.Error(), "register smoke user CA") {
t.Errorf("error = %q, want it to mention register smoke user CA", err.Error())
}
}
func TestRunScenarioGateExecErrorFails(t *testing.T) {
var calls []string
api := happyPathAPI(t, &calls)
api.deleteVMFunc = func(ctx context.Context, id string) error {
t.Fatal("DeleteVM should not be called when gate exec fails")
return nil
}
gate := &gateHooks{
register: func(ctx context.Context) error { return nil },
exec: func(ctx context.Context, vmName string) error {
return errors.New("FAIL: gate SSH boom")
},
}
clock := &fakeClock{t: time.Unix(0, 0)}
_, err := runScenario(context.Background(), "smoke-test", api, powerCycleConsole(t, api, bootedGuest, "ubuntu-vm login: ").dial, gate, nil, "", clock.now, clock.sleep, noopReadPubKey, okBanner)
if err == nil {
t.Fatal("runScenario: want error, got nil")
}
if !strings.Contains(err.Error(), "gate SSH boom") {
t.Errorf("error = %q, want it to mention gate SSH boom", err.Error())
}
}
// --- pollLoop ---------------------------------------------------------
func TestPollLoopReturnsErrPollTimeoutAtDeadline(t *testing.T) {
clock := &fakeClock{t: time.Unix(0, 0)}
calls := 0
err := pollLoop(context.Background(), clock.now, clock.sleep, 10*time.Second, 5*time.Second, func() (bool, error) {
calls++
return false, nil
})
if !errors.Is(err, errPollTimeout) {
t.Errorf("err = %v, want errPollTimeout", err)
}
if calls == 0 {
t.Error("attempt was never called")
}
}
func TestPollLoopPropagatesAttemptError(t *testing.T) {
wantErr := errors.New("boom")
clock := &fakeClock{t: time.Unix(0, 0)}
err := pollLoop(context.Background(), clock.now, clock.sleep, 10*time.Second, 5*time.Second, func() (bool, error) {
return false, wantErr
})
if !errors.Is(err, wantErr) {
t.Errorf("err = %v, want %v", err, wantErr)
}
}
// TestGetVMFiltersById pins that getVM discriminates by ID within a listing
// that contains other VMs — on the live fleet the list always does (eitri-dev
// at minimum), so a match-first regression would poll the wrong VM's
// phase/IP during the ready wait.
func TestGetVMFiltersById(t *testing.T) {
api := &testAPI{listVMsFunc: func(ctx context.Context) ([]client.VM, error) {
return []client.VM{
{ID: "vm-1", Phase: "creating"},
{ID: "vm-2", Phase: "ready", AssignedIP: "10.77.1.9"},
}, nil
}}
vm, present, err := getVM(context.Background(), api, "vm-2")
if err != nil || !present {
t.Fatalf("getVM(vm-2) = present %v, err %v; want present", present, err)
}
if vm.ID != "vm-2" || vm.Phase != "ready" || vm.AssignedIP != "10.77.1.9" {
t.Errorf("getVM(vm-2) returned the wrong row: %+v", vm)
}
if _, present, err := getVM(context.Background(), api, "vm-3"); err != nil || present {
t.Errorf("getVM(vm-3) against a non-empty list = present %v, err %v; want absent", present, err)
}
}
// TestRunScenarioRunsTheMCPLegOnItsOwnVM pins where the remote-MCP leg sits and
// that it owns a separate VM: it must not reuse the scenario's, whose CA set
// was baked before the MCP leg registered the CA it delegates with.
func TestRunScenarioRunsTheMCPLegOnItsOwnVM(t *testing.T) {
var calls []string
api := happyPathAPI(t, &calls)
mcpVM := ""
mcp := func(_ context.Context, vmName string) error {
mcpVM = vmName
calls = append(calls, "mcp")
return nil
}
clock := &fakeClock{t: time.Unix(0, 0)}
msg, err := runScenario(context.Background(), "smoke-test", api, powerCycleConsole(t, api, bootedGuest, "ubuntu-vm login: ").dial, nil, mcp, "", clock.now, clock.sleep, noopReadPubKey, okBanner)
if err != nil {
t.Fatalf("runScenario: %v", err)
}
if !strings.Contains(msg, "remote MCP: ok") {
t.Errorf("message = %q, want it to mention remote MCP: ok", msg)
}
if mcpVM == "smoke-test" || mcpVM == "" {
t.Errorf("MCP leg drove vm %q, want a VM of its own", mcpVM)
}
}
// TestRunScenarioFailsWhenTheMCPLegFails: the remote endpoint is a hard gate,
// not an advisory check.
func TestRunScenarioFailsWhenTheMCPLegFails(t *testing.T) {
var calls []string
api := happyPathAPI(t, &calls)
mcp := func(context.Context, string) error { return errors.New("FAIL: register refused") }
clock := &fakeClock{t: time.Unix(0, 0)}
_, err := runScenario(context.Background(), "smoke-test", api, powerCycleConsole(t, api, bootedGuest, "ubuntu-vm login: ").dial, nil, mcp, "", clock.now, clock.sleep, noopReadPubKey, okBanner)
if err == nil || !strings.Contains(err.Error(), "register refused") {
t.Fatalf("runScenario err = %v, want the MCP leg's failure to fail the gate", err)
}
}
// --- runScenario: the host must be running the release ----------------------
// TestRunScenarioRefusesAHostBehindTheRelease pins the half of the #33 story
// that lives in the smoke. A ship rolls the plane, converges the fleet, and
// then proves the release here — but the proof is only ever as good as the
// agent the VM lands on, and every leg of this scenario is satisfied just as
// well by the release before this one. Twice a ship reached this point with an
// unconverged fleet and exited green. With an expectation set, it cannot: the
// mismatch fails the run before a VM is created, and says the fleet is what
// went wrong.
func TestRunScenarioRefusesAHostBehindTheRelease(t *testing.T) {
api := &testAPI{
listHostsFunc: func(ctx context.Context) ([]client.Host, error) {
return []client.Host{{ID: "host-1", Name: "onyx", Online: true, AgentVersion: "v0.0.6"}}, nil
},
createVMFunc: func(ctx context.Context, req client.CreateVMRequest) (client.CreateVMResponse, error) {
t.Fatal("CreateVM must not be called on a host that is not running the release")
return client.CreateVMResponse{}, nil
},
listVMsFunc: func(ctx context.Context) ([]client.VM, error) {
t.Fatal("ListVMs must not be called on a host that is not running the release")
return nil, nil
},
deleteVMFunc: func(ctx context.Context, id string) error {
t.Fatal("DeleteVM must not be called on a host that is not running the release")
return nil
},
}
clock := &fakeClock{t: time.Unix(0, 0)}
_, err := runScenario(context.Background(), "smoke-test", api, nil, nil, nil, "v0.0.7", clock.now, clock.sleep, noopReadPubKey, okBanner)
if err == nil {
t.Fatal("runScenario: want a failure on a host a release behind, got nil")
}
// Both versions, or the reader cannot tell which side is stale.
for _, want := range []string{"onyx", "v0.0.6", "v0.0.7", "did not converge", "--from 8"} {
if !strings.Contains(err.Error(), want) {
t.Errorf("error = %q, want it to mention %q", err.Error(), want)
}
}
}
// TestRunScenarioAcceptsTheHostRunningTheRelease: the expectation met is the
// ordinary case, and it must be invisible — the same one ListHosts call, the
// same COMPLETE line.
func TestRunScenarioAcceptsTheHostRunningTheRelease(t *testing.T) {
var calls []string
api := happyPathAPI(t, &calls)
listHostsCalls := 0
api.listHostsFunc = func(ctx context.Context) ([]client.Host, error) {
listHostsCalls++
return []client.Host{{ID: "host-1", Name: "onyx", Online: true, UplinkAddr: "192.168.0.190", AgentVersion: "v0.0.7"}}, nil
}
clock := &fakeClock{t: time.Unix(0, 0)}
msg, err := runScenario(context.Background(), "smoke-test", api, powerCycleConsole(t, api, bootedGuest, "ubuntu-vm login: ").dial, nil, nil, "v0.0.7", clock.now, clock.sleep, noopReadPubKey, okBanner)
if err != nil {
t.Fatalf("runScenario: %v", err)
}
if !strings.Contains(msg, "SMOKE COMPLETE") {
t.Errorf("message = %q, want SMOKE COMPLETE", msg)
}
if listHostsCalls != 1 {
t.Errorf("ListHosts calls = %d, want 1 — the check reads the listing the scenario already made", listHostsCalls)
}
}
// TestRunScenarioWithoutAnExpectationJudgesNoVersion: no expectation is no
// check and no extra traffic. This is every run but a ship's — a boot gate
// proves binaries built from a working tree, whose stamped version is "dev".
func TestRunScenarioWithoutAnExpectationJudgesNoVersion(t *testing.T) {
var calls []string
api := happyPathAPI(t, &calls)
listHostsCalls := 0
api.listHostsFunc = func(ctx context.Context) ([]client.Host, error) {
listHostsCalls++
return []client.Host{{ID: "host-1", Name: "onyx", Online: true, UplinkAddr: "192.168.0.190", AgentVersion: "dev"}}, nil
}
clock := &fakeClock{t: time.Unix(0, 0)}
msg, err := runScenario(context.Background(), "smoke-test", api, powerCycleConsole(t, api, bootedGuest, "ubuntu-vm login: ").dial, nil, nil, "", clock.now, clock.sleep, noopReadPubKey, okBanner)
if err != nil {
t.Fatalf("runScenario: %v", err)
}
if !strings.Contains(msg, "SMOKE COMPLETE") {
t.Errorf("message = %q, want SMOKE COMPLETE", msg)
}
if listHostsCalls != 1 {
t.Errorf("ListHosts calls = %d, want 1 — an expectation-free run asks nothing extra", listHostsCalls)
}
}
// TestProveHostReleaseNamesAnAgentTooOldToNameItself: an agent from before
// versions were reported says nothing at all, and "" in the middle of a
// sentence reads as a bug in the message rather than a fact about the host.
func TestProveHostReleaseNamesAnAgentTooOldToNameItself(t *testing.T) {
err := proveHostRelease(client.Host{ID: "host-1", Name: "onyx"}, "v0.0.7")
if err == nil {
t.Fatal("proveHostRelease: want a failure for a host reporting no version, got nil")
}
if !strings.Contains(err.Error(), "no version at all") {
t.Errorf("error = %q, want it to say the host reports no version at all", err.Error())
}
}
// TestRunScenarioSkipsADarkHostAndProvesOnAConnectedOne is the other half of
// the #33 story, learned on the first prod ship of v0.0.8: a host that is
// simply switched off must not fail a release that two connected hosts have
// converged and are ready to prove. The ship already said its piece about that
// host one stage earlier — it cannot take an upgrade offer while it is dark —
// and no retry could ever pass while it stayed off.
func TestRunScenarioSkipsADarkHostAndProvesOnAConnectedOne(t *testing.T) {
var calls []string
api := happyPathAPI(t, &calls)
var placedOn string
api.listHostsFunc = func(ctx context.Context) ([]client.Host, error) {
return []client.Host{
// Dark, never reported a version: exactly onyx on 2026-09-05.
{ID: "host-dark", Name: "onyx", Online: false, AgentVersion: ""},
{ID: "host-up", Name: "charizard", Online: true, AgentVersion: "v0.0.8"},
}, nil
}
create := api.createVMFunc
api.createVMFunc = func(ctx context.Context, req client.CreateVMRequest) (client.CreateVMResponse, error) {
placedOn = req.HostID
return create(ctx, req)
}
clock := &fakeClock{t: time.Unix(0, 0)}
msg, err := runScenario(context.Background(), "smoke-test", api, powerCycleConsole(t, api, bootedGuest, "ubuntu-vm login: ").dial, nil, nil, "v0.0.8", clock.now, clock.sleep, noopReadPubKey, okBanner)
if err != nil {
t.Fatalf("runScenario: want the dark host skipped, got %v", err)
}
if placedOn != "host-up" {
t.Errorf("placed on %q, want the connected host host-up", placedOn)
}
if !strings.Contains(msg, "SMOKE COMPLETE") {
t.Errorf("message = %q, want a complete run", msg)
}
}
// TestRunScenarioStillRefusesAConnectedHostBehindTheRelease guards the line the
// fix above must not cross. Skipping dark hosts is not "shop for a host on the
// release": a host that is UP and behind is the #33 defect itself, and it still
// fails even when a converged host sits beside it in the same listing.
func TestRunScenarioStillRefusesAConnectedHostBehindTheRelease(t *testing.T) {
api := &testAPI{
listHostsFunc: func(ctx context.Context) ([]client.Host, error) {
return []client.Host{
{ID: "host-old", Name: "onyx", Online: true, AgentVersion: "v0.0.7"},
{ID: "host-up", Name: "charizard", Online: true, AgentVersion: "v0.0.8"},
}, nil
},
createVMFunc: func(ctx context.Context, req client.CreateVMRequest) (client.CreateVMResponse, error) {
t.Fatal("CreateVM must not be called while a connected host is behind the release")
return client.CreateVMResponse{}, nil
},
}
clock := &fakeClock{t: time.Unix(0, 0)}
_, err := runScenario(context.Background(), "smoke-test", api, nil, nil, nil, "v0.0.8", clock.now, clock.sleep, noopReadPubKey, okBanner)
if err == nil {
t.Fatal("runScenario: want a failure on a connected host a release behind, got nil")
}
for _, want := range []string{"onyx", "v0.0.7", "v0.0.8", "did not converge"} {
if !strings.Contains(err.Error(), want) {
t.Errorf("error = %q, want it to mention %q", err.Error(), want)
}
}
}
// TestRunScenarioFailsWhenEveryHostIsDark: skipping dark hosts cannot become
// skipping the proof. With nobody left to place on, the run fails — and says
// this is the fleet rather than the release, because a reader who has just been
// told the plane is serving the tag deserves to know which half is at fault.
func TestRunScenarioFailsWhenEveryHostIsDark(t *testing.T) {
api := &testAPI{
listHostsFunc: func(ctx context.Context) ([]client.Host, error) {
return []client.Host{
{ID: "host-a", Name: "onyx", Online: false, AgentVersion: "v0.0.8"},
{ID: "host-b", Name: "Squirtle.local", Online: false, AgentVersion: "v0.0.8"},
}, nil
},
createVMFunc: func(ctx context.Context, req client.CreateVMRequest) (client.CreateVMResponse, error) {
t.Fatal("CreateVM must not be called when no host is connected")
return client.CreateVMResponse{}, nil
},
}
clock := &fakeClock{t: time.Unix(0, 0)}
_, err := runScenario(context.Background(), "smoke-test", api, nil, nil, nil, "v0.0.8", clock.now, clock.sleep, noopReadPubKey, okBanner)
if err == nil {
t.Fatal("runScenario: want a failure when every host is dark, got nil")
}
for _, want := range []string{"onyx", "Squirtle.local", "dark", "--from 8"} {
if !strings.Contains(err.Error(), want) {
t.Errorf("error = %q, want it to mention %q", err.Error(), want)
}
}
}