internal/agent/vfkit/vfkit_test.go
Ref: Size: 32.2 KiB History
package vfkit
import (
"context"
"errors"
"fmt"
"net"
"net/http"
"os"
"path/filepath"
"strconv"
"strings"
"syscall"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
agentexec "github.com/a73x/eitri/internal/agent/exec"
"github.com/a73x/eitri/internal/agent/state"
)
// testSpec is the VM every test provisions unless it needs otherwise.
func testSpec() state.VMSpec {
return state.VMSpec{VMID: "vm-1", Name: "web", VCPUs: 2, MemMB: 2048, DiskGB: 10}
}
func newTestProv(t *testing.T, run agentexec.Runner) *Provisioner {
t.Helper()
st, err := state.Open(t.TempDir())
require.NoError(t, err)
p := New(st, "vfkit", run)
// Never let a test read the real host's lease database.
p.leasesPath = filepath.Join(t.TempDir(), "dhcpd_leases")
return p
}
// isPermanent reports whether err carries the marker reconcile terminal-fails on.
func isPermanent(err error) bool {
var perm interface{ Permanent() bool }
return errors.As(err, &perm) && perm.Permanent()
}
func TestBuildArgsIsTheVfkitContract(t *testing.T) {
p := newTestProv(t, nil)
vmDir := p.st.VMDir("vm-1")
args := p.buildArgs(testSpec(), true)
assert.Equal(t, []string{
"--restful-uri", "unix://" + filepath.Join(vmDir, "vfkit.sock"),
"--cpus", "2",
"--memory", "2048",
"--bootloader", "efi,variable-store=" + filepath.Join(vmDir, "efi-vars.fd") + ",create",
"--device", "virtio-blk,path=" + filepath.Join(vmDir, "disk.raw"),
"--device", "virtio-blk,path=" + filepath.Join(vmDir, "seed.iso"),
"--device", "virtio-net,nat,mac=" + state.MAC("vm-1"),
"--device", "virtio-serial,pty",
"--device", "virtio-rng",
}, args)
}
func TestBuildArgsPutsTheRootDiskFirst(t *testing.T) {
p := newTestProv(t, nil)
args := p.buildArgs(testSpec(), false)
var blk []string
for i, a := range args {
if strings.HasPrefix(a, "virtio-blk,") {
blk = append(blk, a)
require.Equal(t, "--device", args[i-1])
}
}
require.Len(t, blk, 2)
// vfkit maps block devices to /dev/vd* by argument position, so the guest
// boots from whichever comes first. The seed overtaking the root disk would
// be silent until the guest failed to boot.
assert.Contains(t, blk[0], "disk.raw", "the root disk must be the first block device")
assert.Contains(t, blk[1], "seed.iso")
}
// TestDisksAppendVolumesAfterSeedInOrder pins the rest of the guest ABI: a
// volume never displaces the root disk or the seed, and the spec's order is the
// device order, so the first volume a tenant attached is /dev/vdc on every boot.
func TestDisksAppendVolumesAfterSeedInOrder(t *testing.T) {
p := newTestProv(t, nil)
st := p.st
spec := testSpec()
spec.VolumeIDs = []string{"vb", "va"}
disks := p.disks(spec)
require.Len(t, disks, 4)
assert.Equal(t, st.DiskPath("vm-1"), disks[0].Path)
assert.Equal(t, st.SeedPath("vm-1"), disks[1].Path)
assert.Equal(t, st.VolumePath("vb"), disks[2].Path, "spec order, not sorted: the first volume is /dev/vdc")
assert.Equal(t, st.VolumePath("va"), disks[3].Path)
assert.False(t, disks[2].ReadOnly, "a volume is the guest's to write to")
assert.False(t, disks[3].ReadOnly)
args := p.buildArgs(spec, false)
joined := strings.Join(args, " ")
assert.Contains(t, joined, "virtio-blk,path="+st.VolumePath("vb"))
assert.Less(t, strings.Index(joined, st.SeedPath("vm-1")), strings.Index(joined, st.VolumePath("vb")),
"vfkit maps block devices by argument position, so the volumes come last")
}
func TestBuildArgsCreatesTheVariableStoreOnlyWhenThereIsNone(t *testing.T) {
p := newTestProv(t, nil)
assert.Contains(t, strings.Join(p.buildArgs(testSpec(), true), " "), "efi-vars.fd,create")
// A second boot must NOT recreate it: the store is the guest's NVRAM, and
// re-initialising it throws away the boot entry the guest wrote there.
assert.NotContains(t, strings.Join(p.buildArgs(testSpec(), false), " "), ",create")
}
func TestPreflightRefusesAHostWithoutVfkit(t *testing.T) {
p := newTestProv(t, nil)
p.lookPath = func(string) (string, error) { return "", errors.New("executable file not found in $PATH") }
err := p.Preflight(context.Background())
require.Error(t, err)
assert.True(t, isPermanent(err), "no retry installs a signed binary — the refusal must be terminal")
assert.Contains(t, err.Error(), "brew install vfkit", "the refusal must name the fix")
}
func TestPreflightPassesWhenVfkitResolves(t *testing.T) {
p := newTestProv(t, nil)
p.lookPath = func(file string) (string, error) { return "/opt/homebrew/bin/" + file, nil }
assert.NoError(t, p.Preflight(context.Background()))
}
func TestPreflightRefusesAStateDirectoryTooDeepForASocket(t *testing.T) {
deep := filepath.Join(t.TempDir(), strings.Repeat("d", maxSocketPath))
require.NoError(t, os.MkdirAll(deep, 0o700))
st, err := state.Open(deep)
require.NoError(t, err)
p := New(st, "vfkit", nil)
p.lookPath = func(file string) (string, error) { return file, nil }
// macOS caps a unix socket path at 104 bytes. Left to vfkit, this surfaces
// as a URI complaint after the image download; here it is the host's own
// verdict, before anything is spent.
err = p.Preflight(context.Background())
require.Error(t, err)
assert.True(t, isPermanent(err), "a state directory does not get shorter on retry")
assert.Contains(t, err.Error(), "--state-dir", "the refusal must name the fix")
}
func TestPreflightAcceptsTheDefaultStateDirectory(t *testing.T) {
// Not t.TempDir(): its paths carry the test's own name and are long enough
// to trip the guard — which is a fair demonstration that the guard bites.
dir, err := os.MkdirTemp("", "e")
require.NoError(t, err)
t.Cleanup(func() { _ = os.RemoveAll(dir) })
st, err := state.Open(dir)
require.NoError(t, err)
p := New(st, "vfkit", nil)
p.lookPath = func(file string) (string, error) { return file, nil }
assert.NoError(t, p.Preflight(context.Background()))
// The shipped default is /var/lib/eitri-agent; a VM's socket under it is
// 68 bytes, so the guard must not be so tight that normal use trips it.
assert.Less(t, len("/var/lib/eitri-agent/vms/"+strings.Repeat("0", vmIDLen)+"/vfkit.sock"), sunPathBytes,
"the shipped default --state-dir must leave room for a VM's control socket inside sockaddr_un")
}
// sunPathBytes is macOS's sockaddr_un.sun_path, terminator included. It is a
// literal here because it is the OS's number, not eitri's: maxSocketPath only
// restates it, so a Preflight boundary asserted against maxSocketPath moves
// whenever the constant does while the real bind keeps failing at 104.
const sunPathBytes = 104
// provWithSocketPathLen builds a Provisioner whose state directory is padded so
// that a VM's control socket path is exactly want bytes long.
func provWithSocketPathLen(t *testing.T, want int) *Provisioner {
t.Helper()
root, err := os.MkdirTemp("", "e")
require.NoError(t, err)
t.Cleanup(func() { _ = os.RemoveAll(root) })
probe, err := state.Open(root)
require.NoError(t, err)
base := len(New(probe, "vfkit", nil).sockPath(strings.Repeat("0", vmIDLen)))
pad := want - base - 1 // -1 for the separator filepath.Join adds
require.Positive(t, pad, "TMPDIR is already longer than the target path; run with a shorter TMPDIR")
dir := filepath.Join(root, strings.Repeat("d", pad))
require.NoError(t, os.MkdirAll(dir, 0o700))
st, err := state.Open(dir)
require.NoError(t, err)
p := New(st, "vfkit", nil)
p.lookPath = func(file string) (string, error) { return file, nil }
require.Len(t, p.sockPath(strings.Repeat("0", vmIDLen)), want, "harness must hit the target length exactly")
return p
}
// TestPreflightAcceptsTheLongestSocketPathThatBinds and its sibling below pin
// the guard on the byte, from both sides. Between them they hold maxSocketPath
// to macOS's number rather than to whatever it currently says.
func TestPreflightAcceptsTheLongestSocketPathThatBinds(t *testing.T) {
p := provWithSocketPathLen(t, sunPathBytes-1)
assert.NoError(t, p.Preflight(context.Background()),
"103 bytes plus the NUL terminator is exactly sockaddr_un.sun_path (104), so this path binds — refusing it turns a "+
"working --state-dir away and sends an operator hunting for a shorter one that was never needed")
}
func TestPreflightRefusesASocketPathOneByteOverSunPath(t *testing.T) {
p := provWithSocketPathLen(t, sunPathBytes)
err := p.Preflight(context.Background())
require.Error(t, err,
"104 bytes does not fit sockaddr_un.sun_path, which is 104 INCLUDING the terminator: macOS refuses the bind, and "+
"raising maxSocketPath does not raise sun_path — it only moves the refusal to vfkit, after the image download, "+
"as a sentence about URIs")
assert.True(t, isPermanent(err), "a state directory does not get shorter on retry")
assert.Contains(t, err.Error(), "--state-dir", "the refusal must name the fix")
}
// fakeRunner records the commands it is asked to run and answers from a script
// of per-command results, so disk preparation is testable without a Mac.
type fakeRunner struct {
calls [][]string
// fail returns an error for the given argv, or nil to let the call succeed.
fail func(argv []string) error
}
func (f *fakeRunner) run(_ context.Context, name string, args ...string) (string, error) {
argv := append([]string{name}, args...)
f.calls = append(f.calls, argv)
if f.fail != nil {
if err := f.fail(argv); err != nil {
return "", err
}
}
// A real cp produces the destination; tests that inspect the disk need it.
if len(argv) >= 2 {
src, dst := argv[len(argv)-2], argv[len(argv)-1]
if data, err := os.ReadFile(src); err == nil {
_ = os.WriteFile(dst, data, 0o600)
}
}
return "", nil
}
func writeBaseImage(t *testing.T, size int) string {
t.Helper()
path := filepath.Join(t.TempDir(), "base.raw")
require.NoError(t, os.WriteFile(path, make([]byte, size), 0o600))
return path
}
func TestPrepareRootDiskClonesTheBaseImageAndGrowsIt(t *testing.T) {
r := &fakeRunner{}
p := newTestProv(t, r.run)
base := writeBaseImage(t, 4096)
require.NoError(t, p.PrepareRootDisk(context.Background(), testSpec(), base))
require.Len(t, r.calls, 1, "an APFS clone is one command, not a copy loop")
assert.Equal(t, []string{"cp", "-c", base, p.st.DiskPath("vm-1") + ".partial"}, r.calls[0])
st, err := os.Stat(p.st.DiskPath("vm-1"))
require.NoError(t, err, "the disk must be renamed into place, not left as .partial")
assert.Equal(t, int64(10)<<30, st.Size())
_, err = os.Stat(p.st.DiskPath("vm-1") + ".partial")
assert.True(t, os.IsNotExist(err), "no temp file may survive a successful prepare")
}
func TestPrepareRootDiskFallsBackToAPlainCopyOffAPFS(t *testing.T) {
r := &fakeRunner{fail: func(argv []string) error {
if len(argv) > 1 && argv[1] == "-c" {
return errors.New("cp: clonefile failed: Operation not supported")
}
return nil
}}
p := newTestProv(t, r.run)
base := writeBaseImage(t, 4096)
// `cp -c` errors rather than degrading when the filesystem cannot clone, so
// a state directory on a non-APFS volume must still produce a disk.
require.NoError(t, p.PrepareRootDisk(context.Background(), testSpec(), base))
require.Len(t, r.calls, 2)
assert.Equal(t, "-c", r.calls[0][1])
assert.Equal(t, []string{"cp", base, p.st.DiskPath("vm-1") + ".partial"}, r.calls[1])
st, err := os.Stat(p.st.DiskPath("vm-1"))
require.NoError(t, err)
assert.Equal(t, int64(10)<<30, st.Size())
}
func TestPrepareRootDiskReportsACopyThatFailedBothWays(t *testing.T) {
// A real cp writes what it can before it runs out of space, so the failure
// leaves a partial file behind — which is the only state in which the
// cleanup that follows it does any work.
r := &fakeRunner{fail: func(argv []string) error {
require.NoError(t, os.WriteFile(argv[len(argv)-1], make([]byte, 512), 0o600))
return errors.New("No space left on device")
}}
p := newTestProv(t, r.run)
err := p.PrepareRootDisk(context.Background(), testSpec(), writeBaseImage(t, 4096))
require.Error(t, err)
assert.False(t, isPermanent(err), "a full disk is retryable — the operator can free space")
_, statErr := os.Stat(p.st.DiskPath("vm-1"))
assert.True(t, os.IsNotExist(statErr), "a failed prepare must leave no disk behind")
// The temp file matters as much as the final path: it sits on the same
// filesystem the copy just filled, and a create that retries every tick
// would otherwise strand a fresh multi-gigabyte carcass each time.
_, statErr = os.Stat(p.st.DiskPath("vm-1") + ".partial")
assert.True(t, os.IsNotExist(statErr), "a failed prepare must leave no temp file behind")
}
func TestPrepareRootDiskRefusesToShrinkTheBaseImage(t *testing.T) {
r := &fakeRunner{}
p := newTestProv(t, r.run)
spec := testSpec()
spec.DiskGB = 1
// Growing is a truncate to an EXACT size, so a target under the base image
// would chop the guest filesystem rather than fitting it.
err := p.PrepareRootDisk(context.Background(), spec, writeBaseImage(t, 2<<30))
require.Error(t, err)
assert.True(t, isPermanent(err), "a spec that cannot hold its image is not fixed by retrying")
assert.Empty(t, r.calls, "nothing may be copied for a disk that cannot be built")
}
func TestPrepareRootDiskRejectsAnOutOfRangeSize(t *testing.T) {
base := writeBaseImage(t, 4096)
for _, gb := range []int64{0, -1, maxDiskGB + 1} {
t.Run(strconv.FormatInt(gb, 10), func(t *testing.T) {
r := &fakeRunner{}
p := newTestProv(t, r.run)
spec := testSpec()
spec.DiskGB = gb
err := p.PrepareRootDisk(context.Background(), spec, base)
require.Error(t, err)
assert.True(t, isPermanent(err))
// The range check must run BEFORE the byte computation: DiskGB<<30
// wraps to a small positive number for sizes past 2^33 GiB, and a
// wrapped value would sail through the shrink guard.
assert.Empty(t, r.calls)
})
}
}
// fakePumps records the serial-pump lifecycle calls the provisioner drives.
type fakePumps struct{ ensured, stopped []string }
func (f *fakePumps) Ensure(vmID string) { f.ensured = append(f.ensured, vmID) }
func (f *fakePumps) Stop(vmID string) { f.stopped = append(f.stopped, vmID) }
// argRecorder writes a fake vfkit that records its argv and then sleeps, so
// Boot's whole chain — stale-socket cleanup, argument assembly, spawn, pidfile,
// pump — is observable without a hypervisor.
func argRecorder(t *testing.T) (bin, argsFile string) {
t.Helper()
dir := t.TempDir()
bin, argsFile = filepath.Join(dir, "fake-vfkit"), filepath.Join(dir, "argv")
script := fmt.Sprintf("#!/bin/sh\nprintf '%%s\\n' \"$@\" > %s\nexec sleep 30\n", argsFile)
require.NoError(t, os.WriteFile(bin, []byte(script), 0o700))
return bin, argsFile
}
func TestBootSpawnsVfkitAndTracksIt(t *testing.T) {
bin, argsFile := argRecorder(t)
p := newTestProv(t, nil)
p.bin = bin
pumps := &fakePumps{}
p.Pumps = pumps
require.NoError(t, os.MkdirAll(p.st.VMDir("vm-1"), 0o700))
t.Cleanup(func() { _ = p.kill("vm-1") })
require.NoError(t, p.Boot(context.Background(), "vm-1", testSpec()))
assert.True(t, p.Running("vm-1"), "a booted VM must be tracked by its pidfile")
assert.Equal(t, []string{"vm-1"}, pumps.ensured,
"the pump must start at power-on or the boot log is lost before anyone attaches")
var argv []byte
require.Eventually(t, func() bool {
var err error
argv, err = os.ReadFile(argsFile)
return err == nil && len(argv) > 0
}, 5*time.Second, 20*time.Millisecond, "the fake vfkit never recorded its arguments")
assert.Contains(t, string(argv), "efi-vars.fd,create",
"the first boot of a VM must create its EFI variable store")
_, err := os.Stat(p.logPath("vm-1"))
assert.NoError(t, err, "vfkit's own diagnostics must land in a log, not the void")
}
func TestBootClearsAStaleSocket(t *testing.T) {
bin, _ := argRecorder(t)
p := newTestProv(t, nil)
p.bin = bin
require.NoError(t, os.MkdirAll(p.st.VMDir("vm-1"), 0o700))
// vfkit removes its socket on a clean exit only, so a crash or a host reboot
// leaves one that the next bind would trip over.
require.NoError(t, os.WriteFile(p.sockPath("vm-1"), []byte("stale"), 0o600))
t.Cleanup(func() { _ = p.kill("vm-1") })
require.NoError(t, p.Boot(context.Background(), "vm-1", testSpec()))
data, err := os.ReadFile(p.sockPath("vm-1"))
if err == nil {
assert.NotEqual(t, "stale", string(data), "the stale socket must be gone before vfkit binds")
}
}
func TestBootReportsAVfkitThatCannotStart(t *testing.T) {
p := newTestProv(t, nil)
p.bin = filepath.Join(t.TempDir(), "not-installed")
require.NoError(t, os.MkdirAll(p.st.VMDir("vm-1"), 0o700))
err := p.Boot(context.Background(), "vm-1", testSpec())
require.Error(t, err)
assert.Contains(t, err.Error(), "vfkit start")
assert.False(t, p.Running("vm-1"))
}
func TestRunningIsFalseForAVMThatWasNeverBooted(t *testing.T) {
p := newTestProv(t, nil)
assert.False(t, p.Running("vm-1"))
}
func TestRunningIsFalseAfterTheProcessIsGone(t *testing.T) {
p := newTestProv(t, nil)
require.NoError(t, os.MkdirAll(p.st.VMDir("vm-1"), 0o700))
// A pid that cannot be running: the kernel rejects it outright.
require.NoError(t, os.WriteFile(p.pidPath("vm-1"), []byte("2147483647"), 0o600))
assert.False(t, p.Running("vm-1"))
}
// serveREST starts an HTTP server on vmID's vfkit socket path and returns the
// requests it received.
func serveREST(t *testing.T, p *Provisioner, vmID string, h http.HandlerFunc) *[]*http.Request {
t.Helper()
require.NoError(t, os.MkdirAll(p.st.VMDir(vmID), 0o700))
ln, err := net.Listen("unix", p.sockPath(vmID))
require.NoError(t, err)
var got []*http.Request
srv := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
got = append(got, r)
h(w, r)
})}
go func() { _ = srv.Serve(ln) }()
t.Cleanup(func() { _ = srv.Close() })
return &got
}
func TestShutdownAsksVfkitForAGracefulStop(t *testing.T) {
p := newTestProv(t, nil)
var body string
got := serveREST(t, p, "vm-1", func(w http.ResponseWriter, r *http.Request) {
b := make([]byte, 64)
n, _ := r.Body.Read(b)
body = string(b[:n])
w.WriteHeader(http.StatusAccepted) // what vfkit answers
})
require.NoError(t, p.Shutdown(context.Background(), "vm-1"))
require.Len(t, *got, 1)
assert.Equal(t, http.MethodPost, (*got)[0].Method)
assert.Equal(t, "/vm/state", (*got)[0].URL.Path)
// "Stop" is the ACPI power-down; "HardStop" would cut the power instead,
// which is Destroy's job, not Shutdown's.
assert.JSONEq(t, `{"state":"Stop"}`, body)
}
// TestShutdownFallsBackToSigtermWhenVfkitRefuses pins the fallback itself: a
// vfkit that answers 500 has not stopped the guest, so the signal must go to
// the process named by the pidfile. Without it an unhealthy vfkit means
// Shutdown quietly does nothing and the guest runs on until someone destroys
// it — a stop that reports success and leaves the VM up.
func TestShutdownFallsBackToSigtermWhenVfkitRefuses(t *testing.T) {
p := newTestProv(t, nil)
serveREST(t, p, "vm-1", func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusInternalServerError)
})
writePidfile(t, p, "vm-1", "4321\n"+p.bootID()+"\n")
var sent []struct {
pid int
sig syscall.Signal
}
p.signal = func(pid int, sig syscall.Signal) error {
sent = append(sent, struct {
pid int
sig syscall.Signal
}{pid, sig})
return nil
}
require.NoError(t, p.Shutdown(context.Background(), "vm-1"))
require.Len(t, sent, 1, "a refused stop must reach the process, exactly once")
assert.Equal(t, 4321, sent[0].pid, "the signal goes to the pid on the VM's own pidfile")
// SIGTERM, not SIGKILL: this is still the graceful stop, and the guest is
// owed the chance to flush. Cutting the power is Destroy's job.
assert.Equal(t, syscall.SIGTERM, sent[0].sig)
}
// TestShutdownTreatsAGoneProcessAsStopped pins the tolerance the fallback
// needs: a vfkit that has already exited leaves a stale pidfile, and the signal
// comes back ESRCH. That is the outcome Shutdown wanted — the guest is not
// running — so it must not be reported as a failure, or reconcile would retry a
// stop forever on a VM that stopped.
func TestShutdownTreatsAGoneProcessAsStopped(t *testing.T) {
p := newTestProv(t, nil)
serveREST(t, p, "vm-1", func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusInternalServerError)
})
writePidfile(t, p, "vm-1", "4321\n"+p.bootID()+"\n")
p.signal = func(int, syscall.Signal) error { return syscall.ESRCH }
assert.NoError(t, p.Shutdown(context.Background(), "vm-1"), "a process that is already gone is a stop that already happened")
}
// TestShutdownTakesTheConsoleDownWithTheGuest pins the second half of the
// pump's lifetime — it lasts exactly as long as the guest is powered on. The
// ring is replayed to every new viewer, so a pump left running across a stop
// answers for a guest that is not there. cloudhv carries the same test.
func TestShutdownTakesTheConsoleDownWithTheGuest(t *testing.T) {
p := newTestProv(t, nil)
pumps := &fakePumps{}
p.Pumps = pumps
serveREST(t, p, "vm-1", func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusAccepted)
})
require.NoError(t, p.Shutdown(context.Background(), "vm-1"))
assert.Equal(t, []string{"vm-1"}, pumps.stopped)
}
// TestAFailedShutdownLeavesTheConsoleUp is its complement: a SIGTERM the kernel
// refused proves nothing about the guest, which is very likely still running —
// and a running guest keeps its console.
func TestAFailedShutdownLeavesTheConsoleUp(t *testing.T) {
p := newTestProv(t, nil)
pumps := &fakePumps{}
p.Pumps = pumps
p.signal = func(int, syscall.Signal) error { return syscall.EPERM }
writePidfile(t, p, "vm-1", "4321\n"+p.bootID()+"\n")
// No REST socket: the graceful stop is unreachable and the fallback refused.
require.Error(t, p.Shutdown(context.Background(), "vm-1"))
assert.Empty(t, pumps.stopped, "a shutdown that did not happen must not take the console")
}
func TestShutdownOfAVMWithNoProcessIsNotAnError(t *testing.T) {
p := newTestProv(t, nil)
// No socket, no pidfile: reconcile calls Shutdown on every tick past a stop
// request, and a VM that is already down must not fail the pass.
assert.NoError(t, p.Shutdown(context.Background(), "vm-1"))
}
func TestDestroyKillsTheProcessAndStopsThePump(t *testing.T) {
bin, _ := argRecorder(t)
p := newTestProv(t, nil)
p.bin = bin
pumps := &fakePumps{}
p.Pumps = pumps
require.NoError(t, os.MkdirAll(p.st.VMDir("vm-1"), 0o700))
require.NoError(t, p.Boot(context.Background(), "vm-1", testSpec()))
pid := p.ownedPID("vm-1")
require.NotZero(t, pid)
// nil is the promise reconcile deletes the VM's record on the strength of,
// so it may only follow a kill that actually happened.
require.NoError(t, p.Destroy(context.Background(), "vm-1"))
assert.Equal(t, []string{"vm-1"}, pumps.stopped)
assert.Eventually(t, func() bool { return syscall.Kill(pid, 0) != nil },
5*time.Second, 20*time.Millisecond, "the vfkit process outlived Destroy")
_, err := os.Stat(p.pidPath("vm-1"))
assert.True(t, os.IsNotExist(err), "the pidfile must not outlive the process it names")
}
func TestDestroyOfAVMThatWasNeverBootedIsNotAnError(t *testing.T) {
p := newTestProv(t, nil)
// Destroy is called on every tick past a VM's grace until it returns nil.
assert.NoError(t, p.Destroy(context.Background(), "vm-1"))
assert.NoError(t, p.Destroy(context.Background(), "vm-1"))
}
// writePidfile puts a pidfile in place without booting anything, so the paths
// that read one can be driven to any state a real host can be in.
func writePidfile(t *testing.T, p *Provisioner, vmID, contents string) {
t.Helper()
require.NoError(t, os.MkdirAll(p.st.VMDir(vmID), 0o700))
require.NoError(t, os.WriteFile(p.pidPath(vmID), []byte(contents), 0o600))
}
func TestDestroyReportsAKillTheKernelRefused(t *testing.T) {
p := newTestProv(t, nil)
p.signal = func(int, syscall.Signal) error { return syscall.EPERM }
writePidfile(t, p, "vm-1", "4321\n"+p.bootID()+"\n")
err := p.Destroy(context.Background(), "vm-1")
// A refused SIGKILL has not proved the process gone. reconcile deletes the
// VM's whole directory the moment Destroy answers nil, so answering nil
// here would take the pidfile — the only record of a live vfkit still
// holding an unlinked disk and a vmnet attachment — with it.
require.Error(t, err)
assert.Contains(t, err.Error(), "SIGKILL")
_, statErr := os.Stat(p.pidPath("vm-1"))
assert.NoError(t, statErr, "the record of a process we could not kill must survive")
}
func TestDestroyTreatsAProcessThatIsAlreadyGoneAsDone(t *testing.T) {
p := newTestProv(t, nil)
p.signal = func(int, syscall.Signal) error { return syscall.ESRCH }
writePidfile(t, p, "vm-1", "4321\n"+p.bootID()+"\n")
// ESRCH is the answer that proves the kill's whole purpose is served.
require.NoError(t, p.Destroy(context.Background(), "vm-1"))
_, statErr := os.Stat(p.pidPath("vm-1"))
assert.True(t, os.IsNotExist(statErr))
}
func TestBootRecordsTheHostBootAlongsideThePID(t *testing.T) {
bin, _ := argRecorder(t)
p := newTestProv(t, nil)
p.bin = bin
p.bootID = func() string { return "boot-a" }
require.NoError(t, os.MkdirAll(p.st.VMDir("vm-1"), 0o700))
t.Cleanup(func() { _ = p.kill("vm-1") })
require.NoError(t, p.Boot(context.Background(), "vm-1", testSpec()))
raw, err := os.ReadFile(p.pidPath("vm-1"))
require.NoError(t, err)
assert.Contains(t, string(raw), "boot-a", "a pid without the boot it belongs to is not evidence of anything")
}
func TestAPidFromAnEarlierBootIsNotOurs(t *testing.T) {
p := newTestProv(t, nil)
p.bootID = func() string { return "boot-b" }
var signalled []int
p.signal = func(pid int, _ syscall.Signal) error { signalled = append(signalled, pid); return nil }
// A pidfile written before the last reboot. macOS recycles pids out of a
// small space and this backend's host is a laptop, so the number now names
// whatever the user happens to be running.
writePidfile(t, p, "vm-1", "4321\nboot-a\n")
assert.False(t, p.Running("vm-1"), "a VM whose process died with the host is not running")
assert.NoError(t, p.Shutdown(context.Background(), "vm-1"))
assert.NoError(t, p.Destroy(context.Background(), "vm-1"))
assert.Empty(t, signalled, "SIGTERM and SIGKILL went to a pid this agent never started")
_, statErr := os.Stat(p.pidPath("vm-1"))
assert.True(t, os.IsNotExist(statErr), "nothing of ours is left, so the record must go")
}
func TestAPidfileWithoutABootIDIsStillOurs(t *testing.T) {
p := newTestProv(t, nil)
p.bootID = func() string { return "boot-a" }
var signalled []syscall.Signal
p.signal = func(_ int, sig syscall.Signal) error { signalled = append(signalled, sig); return nil }
// The format an agent that predates the boot id wrote. Refusing it would be
// worse than trusting it: reconcile would read the VM as lost and boot a
// second vfkit onto the same disk image.
writePidfile(t, p, "vm-1", "4321\n")
assert.True(t, p.Running("vm-1"))
require.NoError(t, p.Destroy(context.Background(), "vm-1"))
assert.Contains(t, signalled, syscall.SIGKILL)
}
// TestBootedVMSurvivesCtxCancellation pins the VM-lifetime contract: the vfkit
// process must NOT die when the context passed to Boot is cancelled. The
// agent's root context is cancelled on every graceful agent stop, and guests
// are meant to outlive the agent (that is why Boot uses Setsid and why its
// context parameter is deliberately unused). Killing a VM is the exclusive job
// of Shutdown/Destroy. cloudhv carries the same test for the same reason.
func TestBootedVMSurvivesCtxCancellation(t *testing.T) {
bin, _ := argRecorder(t)
p := newTestProv(t, nil)
p.bin = bin
require.NoError(t, os.MkdirAll(p.st.VMDir("vm-1"), 0o700))
t.Cleanup(func() { _ = p.Destroy(context.Background(), "vm-1") })
ctx, cancel := context.WithCancel(context.Background())
require.NoError(t, p.Boot(ctx, "vm-1", testSpec()))
require.True(t, p.Running("vm-1"), "the process must be alive right after Boot")
cancel()
// Poll rather than sleep once: exec.CommandContext's SIGKILL lands and is
// reaped within milliseconds, so a regression fails immediately and 300ms
// is orders of magnitude of margin.
deadline := time.Now().Add(300 * time.Millisecond)
for time.Now().Before(deadline) {
require.True(t, p.Running("vm-1"),
"the vfkit process died with the context passed to Boot — a guest's lifetime is not the agent's")
time.Sleep(50 * time.Millisecond)
}
}
// varStoreRecorder writes a fake vfkit that initialises the EFI variable store
// the way the real one does, then records its argv and sleeps. Creating the
// store FIRST means argv's arrival proves the store exists, so a second Boot
// can be sequenced against it without a second wait.
func varStoreRecorder(t *testing.T, varStore string) (bin, argsFile string) {
t.Helper()
dir := t.TempDir()
bin, argsFile = filepath.Join(dir, "fake-vfkit"), filepath.Join(dir, "argv")
script := fmt.Sprintf("#!/bin/sh\ntouch %s\nprintf '%%s\\n' \"$@\" > %s\nexec sleep 30\n", varStore, argsFile)
require.NoError(t, os.WriteFile(bin, []byte(script), 0o700))
return bin, argsFile
}
func TestASecondBootKeepsTheVariableStoreTheGuestWrote(t *testing.T) {
p := newTestProv(t, nil)
require.NoError(t, os.MkdirAll(p.st.VMDir("vm-1"), 0o700))
bin, argsFile := varStoreRecorder(t, p.varStore("vm-1"))
p.bin = bin
t.Cleanup(func() { _ = p.kill("vm-1") })
require.NoError(t, p.Boot(context.Background(), "vm-1", testSpec()))
assert.Contains(t, waitForArgv(t, argsFile), "efi-vars.fd,create",
"the first boot of a VM must initialise its EFI variable store")
require.NoError(t, p.kill("vm-1"))
require.NoError(t, os.Remove(argsFile))
require.NoError(t, p.Boot(context.Background(), "vm-1", testSpec()))
// The store is the guest's NVRAM and holds the boot entry Ubuntu writes on
// first boot. Re-initialising it every launch throws that away and leaves
// the guest booting only by the removable-media fallback path.
assert.NotContains(t, waitForArgv(t, argsFile), ",create",
"a boot that already has a variable store must not recreate it")
}
func TestBootRebuildsAVariableStoreItCannotStat(t *testing.T) {
p := newTestProv(t, nil)
require.NoError(t, os.MkdirAll(p.st.VMDir("vm-1"), 0o700))
bin, argsFile := varStoreRecorder(t, p.varStore("vm-1"))
p.bin = bin
t.Cleanup(func() { _ = p.kill("vm-1") })
// A store whose stat fails for a reason that is not ENOENT — here a symlink
// loop, on a real host an EACCES from a state dir whose ownership moved.
// Matching ENOENT alone read every one of those as "the store is there" and
// launched vfkit against NVRAM it could not open, on every retry forever.
require.NoError(t, os.Symlink(p.varStore("vm-1"), p.varStore("vm-1")))
require.NoError(t, p.Boot(context.Background(), "vm-1", testSpec()))
assert.Contains(t, waitForArgv(t, argsFile), ",create")
}
// waitForArgv returns the argv the fake vfkit recorded, once it has.
func waitForArgv(t *testing.T, argsFile string) string {
t.Helper()
var argv []byte
require.Eventually(t, func() bool {
var err error
argv, err = os.ReadFile(argsFile)
return err == nil && len(argv) > 0
}, 5*time.Second, 20*time.Millisecond, "the fake vfkit never recorded its arguments")
return string(argv)
}
func TestShutdownDialsTheVMsOwnSocket(t *testing.T) {
p := newTestProv(t, nil)
first := serveREST(t, p, "vm-1", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusAccepted) })
second := serveREST(t, p, "vm-2", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusAccepted) })
// One client serves every VM on the host, and the URL's host — the only
// thing an idle connection is pooled under — is the same placeholder for
// all of them. A pooled connection would answer the wrong VM's socket.
require.NoError(t, p.Shutdown(context.Background(), "vm-1"))
require.NoError(t, p.Shutdown(context.Background(), "vm-2"))
assert.Len(t, *first, 1)
assert.Len(t, *second, 1)
}
func TestShutdownDoesNotLeakAConnectionPerCall(t *testing.T) {
p := newTestProv(t, nil)
serveREST(t, p, "vm-1", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusAccepted) })
require.NoError(t, p.Shutdown(context.Background(), "vm-1"))
assertNoGoroutineGrowth(t, func() {
for range restCallCount {
require.NoError(t, p.Shutdown(context.Background(), "vm-1"))
}
})
}
// TestFailureReasonQuotesVfkitLog pins the wiring, not the tail logic (that is
// hyperlog's own test): FailureReason must read the SAME file Boot redirects
// vfkit's stdout and stderr to.
func TestFailureReasonQuotesVfkitLog(t *testing.T) {
p := newTestProv(t, nil)
require.Empty(t, p.FailureReason("vm1"), "no log yet means nothing to add")
require.NoError(t, os.MkdirAll(p.st.VMDir("vm1"), 0o700))
require.NoError(t, os.WriteFile(p.logPath("vm1"),
[]byte("vfkit starting\nvirtual machine failed to start: unsupported guest\n"), 0o600))
assert.Equal(t, "virtual machine failed to start: unsupported guest", p.FailureReason("vm1"))
}