internal/agent/cloudhv/cloudhv_test.go
Ref: Size: 34.2 KiB History
package cloudhv
import (
"context"
"errors"
"fmt"
"net"
"net/http"
"os"
"path/filepath"
"strings"
"sync/atomic"
"syscall"
"testing"
"time"
"github.com/a73x/eitri/internal/agent/permanent"
"github.com/a73x/eitri/internal/agent/pidfile"
"github.com/a73x/eitri/internal/agent/state"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestMACDeterministicAndLocallyAdministered(t *testing.T) {
m1, m2 := state.MAC("vm-abc"), state.MAC("vm-abc")
assert.Equal(t, m1, m2)
assert.NotEqual(t, m1, state.MAC("vm-def"))
assert.True(t, strings.HasPrefix(m1, "52:54:00:"), "QEMU/KVM locally-administered OUI")
}
func TestBuildArgs(t *testing.T) {
st, _ := state.Open(t.TempDir())
p := New(st, "/usr/bin/cloud-hypervisor", "/usr/share/ch/hypervisor-fw", nil, newFakeNet())
spec := state.VMSpec{VMID: "vm1", VCPUs: 2, MemMB: 2048}
args := p.buildArgs(spec)
joined := strings.Join(args, " ")
assert.Contains(t, joined, "--api-socket "+st.SocketPath("vm1"))
assert.Contains(t, joined, "--kernel /usr/share/ch/hypervisor-fw")
assert.Contains(t, joined, "boot=2")
assert.Contains(t, joined, "size=2048M")
assert.Contains(t, joined, st.DiskPath("vm1"))
assert.Contains(t, joined, st.SeedPath("vm1"))
assert.Contains(t, joined, "tap=eit-vm1,mac="+state.MAC("vm1"))
assert.Equal(t, 1, strings.Count(joined, "--net"), "a guest with no named network has one NIC")
}
// TestBuildArgsGivesANetworkedGuestTwoNICsNATFirst pins the guest ABI: the NAT
// NIC's --net comes first, so it is eth0 in every guest whether or not the VM
// also sits on the operator's LAN. The second NIC is the named one, on its own
// tap, with its own deterministic MAC — the one the host's snoop listens for
// and the one the seed's second netplan stanza matches.
func TestBuildArgsGivesANetworkedGuestTwoNICsNATFirst(t *testing.T) {
st, err := state.Open(t.TempDir())
require.NoError(t, err)
p := New(st, "ch", "fw", nil, newFakeNet())
args := p.buildArgs(state.VMSpec{VMID: "vm1", Network: "lan", VCPUs: 1, MemMB: 512})
joined := strings.Join(args, " ")
nat := "tap=eit-vm1,mac=" + state.MAC("vm1")
named := "tap=eil-vm1,mac=" + state.NetMAC("vm1")
assert.Contains(t, joined, nat)
assert.Contains(t, joined, named)
assert.Equal(t, 2, strings.Count(joined, "--net"), "two NICs, no more")
assert.Less(t, strings.Index(joined, nat), strings.Index(joined, named),
"NIC order is the guest ABI: eth0 is the NAT NIC")
}
// TestBuildArgsDeclaresRawImageType pins image_type=raw on BOTH disk entries.
// Left to autodetection, CH treats "raw" as a guess and DISABLES SECTOR 0
// WRITES — so the first boot's growpart rewrites the partition table only in
// memory, and the VM lands in initramfs emergency mode at its first power
// cycle (on-disk GPT still image-sized, filesystem inside already grown).
func TestBuildArgsDeclaresRawImageType(t *testing.T) {
st, err := state.Open(t.TempDir())
require.NoError(t, err)
p := New(st, "ch", "fw", nil, newFakeNet())
args := p.buildArgs(state.VMSpec{VMID: "vm1", VCPUs: 1, MemMB: 512, VolumeIDs: []string{"vb"}})
joined := strings.Join(args, " ")
assert.Contains(t, joined, st.DiskPath("vm1")+",image_type=raw")
assert.Contains(t, joined, st.SeedPath("vm1")+",image_type=raw,readonly=on")
// A volume is a raw file like the others, and writable: the sector-0 trap
// costs a volume its partition table just as surely as it costs the root disk.
assert.Contains(t, joined, "path="+st.VolumePath("vb")+",image_type=raw")
assert.NotContains(t, joined, st.VolumePath("vb")+",image_type=raw,readonly=on")
}
// TestDisksPutsRootFirstAndSeedReadOnly pins the attachment order: index 0
// must be the root disk. Both cloud-hypervisor and vfkit assign /dev/vda to
// the first --disk/volume argument, so a reorder here silently swaps which
// device the guest boots from.
func TestDisksPutsRootFirstAndSeedReadOnly(t *testing.T) {
st, err := state.Open(t.TempDir())
if err != nil {
t.Fatalf("state.Open: %v", err)
}
p := New(st, "cloud-hypervisor", "/fw/CLOUDHV.fd", nil, newFakeNet())
disks := p.disks(state.VMSpec{VMID: "vm-1"})
if len(disks) != 2 {
t.Fatalf("want root + seed, got %d disks: %v", len(disks), disks)
}
if disks[0].Path != st.DiskPath("vm-1") {
t.Errorf("index 0 must be the root disk, got %q", disks[0].Path)
}
if disks[0].ReadOnly {
t.Error("root disk must be writable")
}
if disks[1].Path != st.SeedPath("vm-1") {
t.Errorf("index 1 must be the seed, got %q", disks[1].Path)
}
if !disks[1].ReadOnly {
t.Error("seed must be read-only")
}
}
// TestDisksAppendVolumesAfterSeedInOrder pins the rest of the guest ABI: a
// volume never displaces the root disk or the seed, and the spec's order is the
// device order, so the first volume a tenant attached is /dev/vdc on every boot.
func TestDisksAppendVolumesAfterSeedInOrder(t *testing.T) {
st, err := state.Open(t.TempDir())
require.NoError(t, err)
p := New(st, "cloud-hypervisor", "/fw/CLOUDHV.fd", nil, newFakeNet())
disks := p.disks(state.VMSpec{VMID: "vm-1", VolumeIDs: []string{"vb", "va"}})
require.Len(t, disks, 4)
assert.Equal(t, st.DiskPath("vm-1"), disks[0].Path)
assert.Equal(t, st.SeedPath("vm-1"), disks[1].Path)
assert.Equal(t, st.VolumePath("vb"), disks[2].Path, "spec order, not sorted: the first volume is /dev/vdc")
assert.Equal(t, st.VolumePath("va"), disks[3].Path)
assert.False(t, disks[2].ReadOnly, "a volume is the guest's to write to")
assert.False(t, disks[3].ReadOnly)
}
func TestBuildArgsUsesSerialSocket(t *testing.T) {
st, err := state.Open(t.TempDir())
require.NoError(t, err)
p := New(st, "ch", "fw", nil, newFakeNet()) // chBin/firmware placeholders fine: args only
args := p.buildArgs(state.VMSpec{VMID: "vm1", VCPUs: 1, MemMB: 512, DiskGB: 5})
joined := strings.Join(args, " ")
assert.Contains(t, joined, "--serial socket="+st.SerialSocketPath("vm1"))
assert.NotContains(t, joined, "--serial file=", "serial must be a socket now — the pump owns the log")
}
// pumpRecorder records Ensure/Stop calls through the consumer-owned hook.
type pumpRecorder struct{ ensured, stopped []string }
func (r *pumpRecorder) Ensure(vmID string) { r.ensured = append(r.ensured, vmID) }
func (r *pumpRecorder) Stop(vmID string) { r.stopped = append(r.stopped, vmID) }
func TestDestroyStopsPumpAndRemovesSerialSocket(t *testing.T) {
st, err := state.Open(t.TempDir())
require.NoError(t, err)
rec := &pumpRecorder{}
p := New(st, "ch", "fw", nil, newFakeNet())
p.Pumps = rec
// No CH process running: Destroy on an unknown VM must still be clean —
// and must still stop the pump + remove the socket path.
require.NoError(t, os.MkdirAll(st.VMDir("vm1"), 0o755))
require.NoError(t, os.WriteFile(st.SerialSocketPath("vm1"), nil, 0o644))
_ = p.Destroy(context.Background(), "vm1")
assert.Equal(t, []string{"vm1"}, rec.stopped)
_, statErr := os.Stat(st.SerialSocketPath("vm1"))
assert.True(t, os.IsNotExist(statErr), "stale serial socket must be removed")
}
// TestBootEnsuresPumpAndShutdownStopsIt pins both ends of the pump's lifetime,
// which is exactly the guest's power-on time. Boot must Ensure the pump right
// after CH starts (a regression here = silently dead consoles fleet-wide, since
// reconcile tests fake the whole Provisioner), and Shutdown must stop it: the
// pump's ring is replayed to every new viewer, so one left running across a stop
// hands a dead boot's login prompt to whoever attaches next — including the
// smoke's power-cycle boot proof.
func TestBootEnsuresPumpAndShutdownStopsIt(t *testing.T) {
st, err := state.Open(t.TempDir())
require.NoError(t, err)
vmID := "vm-pump-hook"
require.NoError(t, st.SaveVM(state.Record{Spec: state.VMSpec{VMID: vmID}}))
// Fake cloud-hypervisor: ignores its CLI args and sleeps (same pattern as
// TestBootedVMSurvivesCtxCancellation).
fakeCH := filepath.Join(t.TempDir(), "fake-ch")
require.NoError(t, os.WriteFile(fakeCH, []byte("#!/bin/sh\nexec sleep 60\n"), 0o755))
rec := &pumpRecorder{}
p := New(st, fakeCH, "fw", nil, newFakeNet())
p.Pumps = rec
require.NoError(t, p.Boot(context.Background(), vmID, state.VMSpec{VMID: vmID, VCPUs: 1, MemMB: 128}))
t.Cleanup(func() { _ = p.Destroy(context.Background(), vmID) })
assert.Equal(t, []string{vmID}, rec.ensured, "Boot must attach the serial pump")
// No API socket is listening, so Shutdown falls back to SIGTERM — either
// path powers the guest off, and a powered-off guest has no console.
require.NoError(t, p.Shutdown(context.Background(), vmID))
assert.Equal(t, []string{vmID}, rec.stopped, "Shutdown must take the console down with the guest")
}
// TestAFailedShutdownLeavesTheConsoleUp is the other side of that invariant.
// SIGTERM refused by the kernel proves nothing about the guest, which is very
// likely still running — and a running guest keeps its console, or the operator
// loses the console at exactly the moment the VM stops answering.
func TestAFailedShutdownLeavesTheConsoleUp(t *testing.T) {
st, err := state.Open(t.TempDir())
require.NoError(t, err)
vmID := "vm-stubborn"
require.NoError(t, os.MkdirAll(st.VMDir(vmID), 0o755))
rec := &pumpRecorder{}
p := New(st, "ch", "fw", nil, newFakeNet())
p.Pumps = rec
require.NoError(t, pidfile.Write(p.pidPath(vmID), 4242, p.bootID()))
p.signal = func(int, syscall.Signal) error { return syscall.EPERM }
// No API socket: the power button is unreachable and the SIGTERM fallback
// is refused.
require.Error(t, p.Shutdown(context.Background(), vmID))
assert.Empty(t, rec.stopped, "a shutdown that did not happen must not take the console")
}
// sparseFile creates a sparse file of the given size and returns its path.
// Sparse: no real disk space is consumed regardless of the nominal size.
func sparseFile(t *testing.T, size int64) string {
t.Helper()
path := filepath.Join(t.TempDir(), "base.raw")
f, err := os.Create(path)
require.NoError(t, err)
require.NoError(t, f.Truncate(size))
require.NoError(t, f.Close())
return path
}
func TestPrepareRootDiskUsesReflinkAndResizes(t *testing.T) {
var cmds []string
run := func(ctx context.Context, name string, args ...string) (string, error) {
cmds = append(cmds, name+" "+strings.Join(args, " "))
if name == "cp" { // simulate the copy so the atomic rename has a file to move
_ = os.WriteFile(args[len(args)-1], []byte("disk"), 0o600)
}
return "", nil
}
st, _ := state.Open(t.TempDir())
p := New(st, "ch", "fw", run, newFakeNet())
base := sparseFile(t, 1<<20) // 1 MiB base, well under the 10G target
require.NoError(t, p.PrepareRootDisk(context.Background(),
state.VMSpec{VMID: "vm1", DiskGB: 10}, base))
joined := strings.Join(cmds, "\n")
// Artifacts are built at a .partial sibling then renamed into place atomically.
partial := st.DiskPath("vm1") + ".partial"
// reflink=auto: instant on XFS/btrfs, silent full-copy fallback on ext4 (spec)
assert.Contains(t, joined, "cp --reflink=auto "+base+" "+partial)
assert.Contains(t, joined, "truncate -s 10G "+partial)
assert.FileExists(t, st.DiskPath("vm1"), "disk.raw must be renamed into place")
assert.NoFileExists(t, partial, "temp must not survive a successful prepare")
}
// TestPrepareRootDiskRefusesToShrinkBaseImage pins the never-shrink guard:
// truncate -s sets an EXACT size, so a DiskGB smaller than the base image
// would silently corrupt the guest filesystem. PrepareRootDisk must refuse
// before running any command.
func TestPrepareRootDiskRefusesToShrinkBaseImage(t *testing.T) {
var cmds []string
run := func(ctx context.Context, name string, args ...string) (string, error) {
cmds = append(cmds, name+" "+strings.Join(args, " "))
return "", nil
}
st, _ := state.Open(t.TempDir())
p := New(st, "ch", "fw", run, newFakeNet())
base := sparseFile(t, 2<<30) // sparse 2 GiB base
err := p.PrepareRootDisk(context.Background(),
state.VMSpec{VMID: "vm1", DiskGB: 1}, base)
require.Error(t, err, "shrinking below the base image must be rejected")
assert.Contains(t, err.Error(), "smaller than base image")
assert.Empty(t, cmds, "no command may run once the shrink is detected")
}
// TestPrepareRootDiskFailsOnMissingBaseImage: a stat failure on the base image is
// a real error (cp would fail anyway) and must surface before any command.
func TestPrepareRootDiskFailsOnMissingBaseImage(t *testing.T) {
var cmds []string
run := func(ctx context.Context, name string, args ...string) (string, error) {
cmds = append(cmds, name+" "+strings.Join(args, " "))
return "", nil
}
st, _ := state.Open(t.TempDir())
p := New(st, "ch", "fw", run, newFakeNet())
err := p.PrepareRootDisk(context.Background(),
state.VMSpec{VMID: "vm1", DiskGB: 10}, "/nonexistent/base.raw")
require.Error(t, err)
assert.Empty(t, cmds)
}
// TestBootedVMSurvivesCtxCancellation pins the VM-lifetime contract: the
// cloud-hypervisor process must NOT die when the context passed to Boot is
// cancelled. The agent's root context is cancelled on every graceful agent
// stop (SIGINT/SIGTERM in main), and VMs are meant to survive agent restarts
// (that is why Boot uses Setsid). Killing the VM is exclusively the job of
// the reconcile Shutdown/Kill path.
func TestBootedVMSurvivesCtxCancellation(t *testing.T) {
st, err := state.Open(t.TempDir())
require.NoError(t, err)
vmID := "vm-survive-test"
// Ensure the VM directory exists (Boot writes ch.log + pidfile inside it).
require.NoError(t, st.SaveVM(state.Record{Spec: state.VMSpec{VMID: vmID}}))
// Fake cloud-hypervisor: ignores its CLI args and sleeps. exec replaces the
// shell, so the pidfile PID is the sleep itself and the cleanup Destroy
// reaps it directly (no orphaned child if the shell wouldn't exec its tail).
fakeCH := filepath.Join(t.TempDir(), "fake-ch")
require.NoError(t, os.WriteFile(fakeCH, []byte("#!/bin/sh\nexec sleep 60\n"), 0o755))
p := New(st, fakeCH, "fw", nil, newFakeNet())
ctx, cancel := context.WithCancel(context.Background())
require.NoError(t, p.Boot(ctx, vmID, state.VMSpec{VMID: vmID, VCPUs: 1, MemMB: 128}))
t.Cleanup(func() { _ = p.Destroy(context.Background(), vmID) })
require.True(t, p.Running(vmID), "process must be alive right after Boot")
cancel()
// The process must still be alive well after cancellation. Poll instead of
// a single sleep so a kill-on-cancel regression fails fast and reliably
// (SIGKILL from exec.CommandContext lands and is reaped within
// milliseconds, so 300ms is orders-of-magnitude margin).
deadline := time.Now().Add(300 * time.Millisecond)
for time.Now().Before(deadline) {
require.True(t, p.Running(vmID),
"cloud-hypervisor process died after ctx cancellation — VM lifetime must not be tied to the agent's context")
time.Sleep(50 * time.Millisecond)
}
}
// TestShutdownFallsBackToSIGTERMOn500 verifies fix 3: a non-2xx HTTP response
// from the cloud-hypervisor socket is treated as failure and the SIGTERM
// fallback path is taken.
//
// Setup: a unix-socket HTTP server serving 500 is bound at the VM's SocketPath.
// No PID file exists, so sigterm() is a no-op returning nil.
// Expected: Shutdown returns nil (fallback succeeded) AND the server observed
// the incoming request (proving the API was actually called before falling back).
func TestShutdownFallsBackToSIGTERMOn500(t *testing.T) {
st, err := state.Open(t.TempDir())
require.NoError(t, err)
vmID := "vm-shutdown-test"
// Ensure VM directory exists (SocketPath lives inside it).
require.NoError(t, st.SaveVM(state.Record{Spec: state.VMSpec{VMID: vmID}}))
sockPath := st.SocketPath(vmID)
// Bind a Unix socket serving HTTP 500 at the VM's socket path.
ln, err := net.Listen("unix", sockPath)
require.NoError(t, err)
defer ln.Close()
var requestSeen atomic.Bool
srv := &http.Server{
Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
requestSeen.Store(true)
w.WriteHeader(http.StatusInternalServerError)
}),
}
go srv.Serve(ln) //nolint:errcheck
defer srv.Close()
p := New(st, "ch", "fw", nil, newFakeNet())
// No pidfile → sigterm fallback is a no-op returning nil.
err = p.Shutdown(context.Background(), vmID)
assert.NoError(t, err, "Shutdown must not error when SIGTERM fallback has no pidfile")
assert.True(t, requestSeen.Load(), "Shutdown must attempt the CH API before falling back")
}
// TestShutdownSucceedsOn204 verifies that a 2xx response is treated as success
// (no fallback to SIGTERM).
func TestShutdownSucceedsOn204(t *testing.T) {
st, err := state.Open(t.TempDir())
require.NoError(t, err)
vmID := "vm-shutdown-ok"
require.NoError(t, st.SaveVM(state.Record{Spec: state.VMSpec{VMID: vmID}}))
sockPath := st.SocketPath(vmID)
ln, err := net.Listen("unix", sockPath)
require.NoError(t, err)
defer ln.Close()
srv := &http.Server{
Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusNoContent)
}),
}
go srv.Serve(ln) //nolint:errcheck
defer srv.Close()
p := New(st, "ch", "fw", nil, newFakeNet())
assert.NoError(t, p.Shutdown(context.Background(), vmID))
}
// TestPrepareRootDiskShrinkGuardEdgeCases pins the guard's arithmetic: an exact
// fit is allowed (truncate to same size is a no-op), and an absurd DiskGB
// that would overflow a byte computation (DiskGB<<30) must still be caught —
// 2^34+10 wraps to a small positive byte count if computed naively.
func TestPrepareRootDiskShrinkGuardEdgeCases(t *testing.T) {
newP := func(t *testing.T, cmds *[]string) *Provisioner {
run := func(ctx context.Context, name string, args ...string) (string, error) {
*cmds = append(*cmds, name)
if name == "cp" { // simulate the copy so the atomic rename has a file to move
_ = os.WriteFile(args[len(args)-1], []byte("disk"), 0o600)
}
return "", nil
}
st, _ := state.Open(t.TempDir())
return New(st, "ch", "fw", run, newFakeNet())
}
t.Run("exact fit is allowed", func(t *testing.T) {
var cmds []string
p := newP(t, &cmds)
base := sparseFile(t, 2<<30) // exactly 2 GiB
require.NoError(t, p.PrepareRootDisk(context.Background(),
state.VMSpec{VMID: "vm1", DiskGB: 2}, base))
assert.NotEmpty(t, cmds)
})
t.Run("overflow-sized disk_gb does not bypass the guard", func(t *testing.T) {
var cmds []string
p := newP(t, &cmds)
base := sparseFile(t, 2<<30)
// (1<<34)+10 << 30 wraps to +10 GiB... no: ((1<<34)+10)*2^30 mod 2^64
// wraps to 10 GiB-ish positive — either way the request is absurd and
// must not run cp/truncate with a nonsense size. DiskGB=2^34+10 > any
// real disk; the guard must reject or the size math must be exact.
err := p.PrepareRootDisk(context.Background(),
state.VMSpec{VMID: "vm1", DiskGB: (1 << 34) + 10}, base)
if err == nil {
// Accepting it is only sound if the target genuinely covers the
// base, which it does mathematically (2^34+10 GiB >> 2 GiB) — but
// then truncate would run with a size beyond off_t. Reject instead.
t.Fatal("absurd disk_gb accepted; overflow in the guard arithmetic")
}
assert.Empty(t, cmds, "no command may run for an absurd disk_gb")
})
}
// TestDiskGuardErrorsArePermanent pins that the never-shrink and range
// guards mark their errors with the reconcile-consumed Permanent() marker:
// no retry can ever fix a disk_gb below the base image.
func TestDiskGuardErrorsArePermanent(t *testing.T) {
st, _ := state.Open(t.TempDir())
p := New(st, "ch", "fw", func(ctx context.Context, name string, args ...string) (string, error) {
return "", nil
}, newFakeNet())
isPermanent := func(err error) bool {
var m interface{ Permanent() bool }
return errors.As(err, &m) && m.Permanent()
}
base := sparseFile(t, 2<<30)
err := p.PrepareRootDisk(context.Background(), state.VMSpec{VMID: "v", DiskGB: 1}, base)
require.Error(t, err)
assert.True(t, isPermanent(err), "shrink guard error must be permanent")
err = p.PrepareRootDisk(context.Background(), state.VMSpec{VMID: "v", DiskGB: (1 << 34) + 10}, base)
require.Error(t, err)
assert.True(t, isPermanent(err), "range guard error must be permanent")
err = p.PrepareRootDisk(context.Background(), state.VMSpec{VMID: "v", DiskGB: 10}, "/nonexistent/base.raw")
require.Error(t, err)
assert.False(t, isPermanent(err), "stat failure may be transient (NFS blip, cache re-fetch)")
}
// TestBootstrapDest pins the --ch-bin → install-destination mapping: a fresh
// host's bare name lands on the systemd default $PATH (not the agent's cwd,
// where launch-time lookup would never find it); an already-installed bare
// name resolves to itself so Ensure no-ops; an explicit path is its own
// destination.
func TestBootstrapDest(t *testing.T) {
t.Run("explicit path is its own destination", func(t *testing.T) {
if got := BootstrapDest("/opt/ch/cloud-hypervisor"); got != "/opt/ch/cloud-hypervisor" {
t.Errorf("got %q", got)
}
})
t.Run("bare name not on PATH installs to /usr/local/bin", func(t *testing.T) {
t.Setenv("PATH", t.TempDir())
if got := BootstrapDest("cloud-hypervisor"); got != "/usr/local/bin/cloud-hypervisor" {
t.Errorf("got %q", got)
}
})
t.Run("bare name on PATH resolves to the installed binary", func(t *testing.T) {
dir := t.TempDir()
bin := filepath.Join(dir, "cloud-hypervisor")
if err := os.WriteFile(bin, []byte("#!/bin/sh\n"), 0o755); err != nil {
t.Fatal(err)
}
t.Setenv("PATH", dir)
if got := BootstrapDest("cloud-hypervisor"); got != bin {
t.Errorf("got %q, want %q", got, bin)
}
})
}
// fakeNet is the host networking a VM attaches to. It mirrors netenv: sticky
// addresses keyed by vmID, idempotent create/delete, and a tap name derived
// from the id — enough to drive argument assembly and lifecycle without a
// bridge on the test machine.
type fakeNet struct {
reserved map[string]string
taps map[string]bool
reserveErr error
tapErr error
delErr error
// tapNetwork records the network CreateTap was passed, so a test can see
// that the spec's request reached the networking layer intact without a
// bridge on the test machine. discovered stands in for what a snoop heard.
tapNetwork map[string]string
tapIP map[string]string
discovered map[string]string
}
func newFakeNet() *fakeNet {
return &fakeNet{
reserved: map[string]string{}, taps: map[string]bool{},
tapNetwork: map[string]string{}, tapIP: map[string]string{},
discovered: map[string]string{},
}
}
func (f *fakeNet) ReserveIP(vmID string) (string, error) {
if f.reserveErr != nil {
return "", f.reserveErr
}
if ip, ok := f.reserved[vmID]; ok {
return ip, nil
}
ip := fmt.Sprintf("10.77.1.%d", len(f.reserved)+2)
f.reserved[vmID] = ip
return ip, nil
}
func (f *fakeNet) Address(vmID string) string { return f.reserved[vmID] }
func (f *fakeNet) NetworkAddress(vmID string) string { return f.discovered[vmID] }
func (f *fakeNet) CreateTap(_ context.Context, vmID, ip, network string) error {
f.tapNetwork[vmID] = network
f.tapIP[vmID] = ip
if f.tapErr != nil {
return f.tapErr
}
f.taps[vmID] = true
return nil
}
func (f *fakeNet) DeleteTap(_ context.Context, vmID string) error {
if f.delErr != nil {
return f.delErr
}
delete(f.reserved, vmID)
delete(f.taps, vmID)
delete(f.discovered, vmID)
return nil
}
func (f *fakeNet) TapName(vmID string) string { return "eit-" + shortID(vmID) }
func (f *fakeNet) NetTapName(vmID string) string { return "eil-" + shortID(vmID) }
func shortID(vmID string) string {
if len(vmID) > 8 {
return vmID[:8]
}
return vmID
}
// TestBootAttachesTheNetworkBeforeLaunching pins the fold: the --net argument
// names a device that exists, because Boot reserved the address and created the
// tap itself. Nothing above this package sequences that any more.
func TestBootAttachesTheNetworkBeforeLaunching(t *testing.T) {
st, err := state.Open(t.TempDir())
require.NoError(t, err)
vmID := "vm-attach"
require.NoError(t, st.SaveVM(state.Record{Spec: state.VMSpec{VMID: vmID}}))
fakeCH := filepath.Join(t.TempDir(), "fake-ch")
require.NoError(t, os.WriteFile(fakeCH, []byte("#!/bin/sh\nexec sleep 60\n"), 0o755))
fn := newFakeNet()
p := New(st, fakeCH, "fw", nil, fn)
require.NoError(t, p.Boot(context.Background(), vmID, state.VMSpec{VMID: vmID, VCPUs: 1, MemMB: 128}))
t.Cleanup(func() { _ = p.Destroy(context.Background(), vmID) })
assert.True(t, fn.taps[vmID], "Boot must create the VM's tap")
assert.Equal(t, "10.77.1.2", p.Address(vmID), "Boot must reserve the VM's address")
}
// TestBootNetworkedVMReservesAndPassesTheNetwork pins what this package owes
// the networking layer for a guest that asked for a named network: the address
// is still reserved (the NAT NIC is unconditional), and the network name
// reaches CreateTap unaltered so the second NIC can be built.
func TestBootNetworkedVMReservesAndPassesTheNetwork(t *testing.T) {
st, err := state.Open(t.TempDir())
require.NoError(t, err)
vmID := "vm-lan"
spec := state.VMSpec{VMID: vmID, Network: "lan", VCPUs: 1, MemMB: 128}
require.NoError(t, st.SaveVM(state.Record{Spec: spec}))
fakeCH := filepath.Join(t.TempDir(), "fake-ch")
require.NoError(t, os.WriteFile(fakeCH, []byte("#!/bin/sh\nexec sleep 60\n"), 0o755))
fn := newFakeNet()
p := New(st, fakeCH, "fw", nil, fn)
require.NoError(t, p.Boot(context.Background(), vmID, spec))
t.Cleanup(func() { _ = p.Destroy(context.Background(), vmID) })
assert.Equal(t, "lan", fn.tapNetwork[vmID])
assert.Equal(t, "10.77.1.2", fn.tapIP[vmID],
"a networked guest is on the NAT underlay too — it has an address before it boots")
assert.Equal(t, "10.77.1.2", p.Address(vmID))
assert.Empty(t, p.NetworkAddress(vmID), "and nothing on the named NIC until its guest asks")
assert.True(t, fn.taps[vmID], "the tap is still Boot's to create")
}
// TestNetworkAddressIsPolledFromTheNetworkingLayer pins the second address's
// only route upward: whatever the host discovered, unaltered and not confused
// with the reservation.
func TestNetworkAddressIsPolledFromTheNetworkingLayer(t *testing.T) {
st, err := state.Open(t.TempDir())
require.NoError(t, err)
fn := newFakeNet()
p := New(st, "ch", "fw", nil, fn)
assert.Empty(t, p.NetworkAddress("vm-lan"))
fn.discovered["vm-lan"] = "192.168.0.42"
assert.Equal(t, "192.168.0.42", p.NetworkAddress("vm-lan"))
assert.Empty(t, p.Address("vm-lan"), "which is never mistaken for the NAT address")
}
// TestBootNATVMPassesNoNetwork is the counterpart, and the regression guard for
// every host running today: an unnamed spec asks for no second NIC and gets the
// address this host allocated.
func TestBootNATVMPassesNoNetwork(t *testing.T) {
st, err := state.Open(t.TempDir())
require.NoError(t, err)
vmID := "vm-nat"
spec := state.VMSpec{VMID: vmID, VCPUs: 1, MemMB: 128}
require.NoError(t, st.SaveVM(state.Record{Spec: spec}))
fakeCH := filepath.Join(t.TempDir(), "fake-ch")
require.NoError(t, os.WriteFile(fakeCH, []byte("#!/bin/sh\nexec sleep 60\n"), 0o755))
fn := newFakeNet()
p := New(st, fakeCH, "fw", nil, fn)
require.NoError(t, p.Boot(context.Background(), vmID, spec))
t.Cleanup(func() { _ = p.Destroy(context.Background(), vmID) })
assert.Empty(t, fn.tapNetwork[vmID])
assert.Equal(t, "10.77.1.2", fn.tapIP[vmID], "the allocated address is what the tap pins")
}
// TestBootKeepsTheAddressTheVMAlreadyHolds pins that a reboot is not a renumber:
// the reservation the VM already holds is what Boot re-attaches it to.
func TestBootKeepsTheAddressTheVMAlreadyHolds(t *testing.T) {
st, err := state.Open(t.TempDir())
require.NoError(t, err)
vmID := "vm-sticky"
// Boot opens ch.log under st.VMDir(vmID); SaveVM is what creates that dir.
require.NoError(t, st.SaveVM(state.Record{Spec: state.VMSpec{VMID: vmID}}))
fakeCH := filepath.Join(t.TempDir(), "fake-ch")
require.NoError(t, os.WriteFile(fakeCH, []byte("#!/bin/sh\nexec sleep 60\n"), 0o755))
fn := newFakeNet()
fn.reserved[vmID] = "10.77.1.44"
p := New(st, fakeCH, "fw", nil, fn)
require.NoError(t, p.Boot(context.Background(), vmID, state.VMSpec{VMID: vmID, VCPUs: 1, MemMB: 128}))
t.Cleanup(func() { _ = p.Destroy(context.Background(), vmID) })
assert.Equal(t, "10.77.1.44", p.Address(vmID))
}
// TestBootFailsBeforeLaunchWhenTheNetworkRefuses pins that no cloud-hypervisor
// process is started for a VM that has no network — and that the network's own
// error reaches the caller, not an illegible hypervisor failure for the same
// root cause.
func TestBootFailsBeforeLaunchWhenTheNetworkRefuses(t *testing.T) {
st, err := state.Open(t.TempDir())
require.NoError(t, err)
vmID := "vm-nonet"
fn := newFakeNet()
fn.tapErr = errors.New("link eit-vm-nonet exists but is not a TAP device")
// A chBin that would fail loudly if it were ever reached.
p := New(st, "/nonexistent/cloud-hypervisor", "fw", nil, fn)
err = p.Boot(context.Background(), vmID, state.VMSpec{VMID: vmID, VCPUs: 1, MemMB: 128})
require.Error(t, err)
assert.Contains(t, err.Error(), "not a TAP device")
assert.False(t, p.Running(vmID), "no hypervisor may be launched without a network")
}
// TestBootPreservesThePermanenceMarkerFromTheNetwork pins that wrapping the
// attach error keeps reconcile's Permanent() fast-fail working: a tap name
// collision must still terminal-fail in one attempt rather than burn the budget.
func TestBootPreservesThePermanenceMarkerFromTheNetwork(t *testing.T) {
st, err := state.Open(t.TempDir())
require.NoError(t, err)
fn := newFakeNet()
fn.tapErr = permanent.Errorf("name collision")
p := New(st, "/nonexistent/cloud-hypervisor", "fw", nil, fn)
err = p.Boot(context.Background(), "vm-perm", state.VMSpec{VMID: "vm-perm", VCPUs: 1, MemMB: 128})
require.Error(t, err)
var perm interface{ Permanent() bool }
require.True(t, errors.As(err, &perm), "the attach error must stay unwrappable")
assert.True(t, perm.Permanent())
}
// TestDestroyReleasesTheNetworkAndRetriesOnFailure pins the teardown contract:
// Destroy releases the address, and a failure to release is the error reconcile
// sees (it keeps the record and retries next tick).
func TestDestroyReleasesTheNetworkAndRetriesOnFailure(t *testing.T) {
st, err := state.Open(t.TempDir())
require.NoError(t, err)
vmID := "vm-teardown"
require.NoError(t, os.MkdirAll(st.VMDir(vmID), 0o755))
fn := newFakeNet()
_, err = fn.ReserveIP(vmID)
require.NoError(t, err)
p := New(st, "ch", "fw", nil, fn)
fn.delErr = errors.New("ip link del: context deadline exceeded")
require.Error(t, p.Destroy(context.Background(), vmID),
"a failed release must reach reconcile so the record is kept")
fn.delErr = nil
require.NoError(t, p.Destroy(context.Background(), vmID))
assert.Empty(t, p.Address(vmID), "destroy releases the VM's address")
}
// TestDestroyReportsAKillTheKernelRefused pins the other half of the teardown
// contract. A nil from Destroy is the backend's promise that nothing is left to
// reap, and reconcile deletes the VM's record and its whole directory on the
// strength of it. If SIGKILL was refused, the guest is still running — and the
// pidfile is the only handle anything has on it.
func TestDestroyReportsAKillTheKernelRefused(t *testing.T) {
st, err := state.Open(t.TempDir())
require.NoError(t, err)
vmID := "vm-refused"
require.NoError(t, os.MkdirAll(st.VMDir(vmID), 0o755))
p := New(st, "ch", "fw", nil, newFakeNet())
require.NoError(t, pidfile.Write(p.pidPath(vmID), 4242, p.bootID()))
p.signal = func(int, syscall.Signal) error { return syscall.EPERM }
err = p.Destroy(context.Background(), vmID)
require.Error(t, err, "a refused kill must reach reconcile, or the record is deleted under a live guest")
assert.Contains(t, err.Error(), "SIGKILL")
_, statErr := os.Stat(p.pidPath(vmID))
assert.NoError(t, statErr, "the pidfile is the only handle left on that process")
}
func TestDestroyOfAProcessAlreadyGoneIsDone(t *testing.T) {
st, err := state.Open(t.TempDir())
require.NoError(t, err)
vmID := "vm-gone"
require.NoError(t, os.MkdirAll(st.VMDir(vmID), 0o755))
p := New(st, "ch", "fw", nil, newFakeNet())
require.NoError(t, pidfile.Write(p.pidPath(vmID), 4242, p.bootID()))
p.signal = func(int, syscall.Signal) error { return syscall.ESRCH }
// ESRCH is the answer that proves the guest gone, which is exactly what
// reconcile needs to hear before it deletes the record.
require.NoError(t, p.Destroy(context.Background(), vmID))
_, statErr := os.Stat(p.pidPath(vmID))
assert.True(t, os.IsNotExist(statErr))
}
// TestAPidFromAnEarlierBootIsNeverSignalled is the reboot guard. The state
// directory outlives the host and Linux recycles pids, so a pidfile written
// before the last boot names whatever now holds that number — and this agent
// runs as root, so nothing would refuse the signal on its behalf.
func TestAPidFromAnEarlierBootIsNeverSignalled(t *testing.T) {
st, err := state.Open(t.TempDir())
require.NoError(t, err)
vmID := "vm-rebooted"
require.NoError(t, os.MkdirAll(st.VMDir(vmID), 0o755))
p := New(st, "ch", "fw", nil, newFakeNet())
require.NoError(t, pidfile.Write(p.pidPath(vmID), 4242, "boot-before"))
p.bootID = func() string { return "boot-after" }
var signalled []syscall.Signal
p.signal = func(_ int, sig syscall.Signal) error {
signalled = append(signalled, sig)
return nil
}
assert.False(t, p.Running(vmID), "a process from a previous boot is not this VM")
require.NoError(t, p.Shutdown(context.Background(), vmID))
require.NoError(t, p.Destroy(context.Background(), vmID))
assert.Empty(t, signalled, "reapVM reaches Shutdown before any boot-ID reasoning — the refusal has to be here")
}
// TestAPidfileWithoutABootIDIsStillOurs covers the rolling upgrade: every VM
// running under the agent being replaced has a pidfile in the old format.
// Refusing them would read every live guest as lost and boot a SECOND
// hypervisor onto its disk.
func TestAPidfileWithoutABootIDIsStillOurs(t *testing.T) {
st, err := state.Open(t.TempDir())
require.NoError(t, err)
vmID := "vm-upgraded"
require.NoError(t, os.MkdirAll(st.VMDir(vmID), 0o755))
p := New(st, "ch", "fw", nil, newFakeNet())
require.NoError(t, os.WriteFile(p.pidPath(vmID), []byte("4242\n"), 0o600))
p.signal = func(int, syscall.Signal) error { return nil }
assert.True(t, p.Running(vmID), "an agent upgrade must not orphan the guests it inherits")
}
// TestFailureReasonQuotesCHLog pins the wiring, not the tail logic (that is
// hyperlog's own test): FailureReason must read the SAME file Boot redirects
// cloud-hypervisor's stdout and stderr to. A backend quoting the wrong file
// reports nothing forever and looks exactly like a backend with nothing to say.
func TestFailureReasonQuotesCHLog(t *testing.T) {
st, err := state.Open(t.TempDir())
require.NoError(t, err)
p := New(st, "ch", "fw", nil, newFakeNet())
require.Empty(t, p.FailureReason("vm1"), "no log yet means nothing to add")
require.NoError(t, os.MkdirAll(st.VMDir("vm1"), 0o700))
require.NoError(t, os.WriteFile(filepath.Join(st.VMDir("vm1"), "ch.log"),
[]byte("cloud-hypervisor booting\nError: VmBoot(DeviceManager(Kernel))\n"), 0o600))
assert.Equal(t, "Error: VmBoot(DeviceManager(Kernel))", p.FailureReason("vm1"))
}