d019c3b7 — eitri

docs/architecture.md

internal/agent/cloudhv/cloudhv.go

Old		New
@@ -32,32 +32,48 @@ type PumpHooks interface {
32	Stop(vmID string)	32	Stop(vmID string)
33	}	33	}
34		34
		35	// Network is the host networking this backend's guests sit on: a Linux bridge
		36	// reached through a per-VM tap. Consumer-owned (R5) and satisfied by
		37	// netenv.Net — declaring it here rather than importing netenv keeps the driver
		38	// testable against a fake and keeps the package graph free of a
		39	// cloudhv → netenv edge.
		40	//
		41	// Every method is per-VM. Host-wide setup (the bridge, NAT, the DHCP
		42	// responder) is the composition root's job and never appears here: it happens
		43	// once at agent start, not once per guest.
		44	type Network interface {
		45	// ReserveIP returns this VM's sticky address, recording its DHCP
		46	// reservation.
		47	ReserveIP(vmID string) (string, error)
		48	// Address returns the address already reserved for vmID, or "" if none.
		49	// It never allocates.
		50	Address(vmID string) string
		51	// CreateTap creates the VM's tap device, enslaves it to the bridge and
		52	// pins ip as the guest's DHCP reservation. Idempotent.
		53	CreateTap(ctx context.Context, vmID, ip string) error
		54	// DeleteTap removes the reservation and the tap. Idempotent.
		55	DeleteTap(ctx context.Context, vmID string) error
		56	// TapName is the device name that goes in cloud-hypervisor's --net argument.
		57	TapName(vmID string) string
		58	}
		59
35	// Provisioner manages cloud-hypervisor processes for all VMs on this host.	60	// Provisioner manages cloud-hypervisor processes for all VMs on this host.
36	type Provisioner struct {	61	type Provisioner struct {
37	st *state.Store	62	st *state.Store
38	chBin string // path to cloud-hypervisor binary	63	chBin string // path to cloud-hypervisor binary
39	firmware string // path to CLOUDHV.fd (UEFI firmware)	64	firmware string // path to CLOUDHV.fd (UEFI firmware)
40	run agentexec.Runner	65	run agentexec.Runner
		66	net Network
41		67
42	// Pumps receives serial-pump lifecycle calls at Boot/Kill. nil = no-op.	68	// Pumps receives serial-pump lifecycle calls at Boot/Kill. nil = no-op.
43	Pumps PumpHooks	69	Pumps PumpHooks
44	}	70	}
45		71
46	// New constructs a Provisioner. run may be nil when only pure methods	72	// New constructs a Provisioner. run may be nil when only pure methods
47	// (buildArgs, MAC) are needed.	73	// (buildArgs) are needed; net must not be, since a VM's network attachment is
48	func New(st state.Store, chBin, firmware string, run agentexec.Runner) Provisioner {	74	// part of booting it.
49	return &Provisioner{st: st, chBin: chBin, firmware: firmware, run: run}	75	func New(st state.Store, chBin, firmware string, run agentexec.Runner, net Network) Provisioner {
50	}	76	return &Provisioner{st: st, chBin: chBin, firmware: firmware, run: run, net: net}
51
52	// RealRunner is the production one-shot command runner the host-touching agent
53	// packages (netenv, imagecache, cloudhv, syncclient) run with. It spawns
54	// name+args, waits, and returns their combined stdout/stderr. It lives here
55	// because cloudhv is the data plane's one sanctioned os/exec site (arch R6/R7);
56	// every other agent package receives it as an injected agentexec.Runner and so
57	// never imports os/exec itself.
58	func RealRunner(ctx context.Context, name string, args ...string) (string, error) {
59	out, err := exec.CommandContext(ctx, name, args...).CombinedOutput()
60	return string(out), err
61	}	77	}
62		78
63	// BootstrapDest maps a --ch-bin value to the filesystem path bootstrap may	79	// BootstrapDest maps a --ch-bin value to the filesystem path bootstrap may
@@ -78,31 +94,47 @@ func BootstrapDest(chBin string) string {
78	return filepath.Join("/usr/local/bin", chBin)	94	return filepath.Join("/usr/local/bin", chBin)
79	}	95	}
80		96
		97	// disks returns the VM's block devices in attachment order: the root disk
		98	// first, then the read-only cloud-init seed. User-attached volumes append here.
		99	func (p *Provisioner) disks(spec state.VMSpec) []state.Disk {
		100	return []state.Disk{
		101	{Path: p.st.DiskPath(spec.VMID)},
		102	{Path: p.st.SeedPath(spec.VMID), ReadOnly: true},
		103	}
		104	}
		105
81	// buildArgs returns the cloud-hypervisor command-line arguments for spec.	106	// buildArgs returns the cloud-hypervisor command-line arguments for spec.
82	// The result is deterministic given the same spec so it can be unit-tested	107	// The result is deterministic given the same spec so it can be unit-tested
83	// without spawning a process.	108	// without spawning a process.
84	func (p *Provisioner) buildArgs(spec state.VMSpec) []string {	109	func (p *Provisioner) buildArgs(spec state.VMSpec) []string {
85	vmID := spec.VMID	110	vmID := spec.VMID
86	tap := state.TapName(vmID)	111	tap := p.net.TapName(vmID)
87	mac := state.MAC(vmID)	112	mac := state.MAC(vmID)
88		113
89	return []string{	114	args := []string{
90	"--api-socket", p.st.SocketPath(vmID),	115	"--api-socket", p.st.SocketPath(vmID),
91	"--kernel", p.firmware,	116	"--kernel", p.firmware,
92	"--cpus", fmt.Sprintf("boot=%d", spec.VCPUs),	117	"--cpus", fmt.Sprintf("boot=%d", spec.VCPUs),
93	"--memory", fmt.Sprintf("size=%dM", spec.MemMB),	118	"--memory", fmt.Sprintf("size=%dM", spec.MemMB),
94	// image_type=raw is load-bearing, not decoration: autodetected raw
95	// makes CH DISABLE SECTOR 0 WRITES, so the first boot's growpart
96	// rewrites the partition table only in memory and the guest dies in
97	// initramfs at its first power cycle. Declared raw keeps the GPT
98	// writable. (Autodetection is also deprecated in CH v53.)
99	"--disk",	119	"--disk",
100	fmt.Sprintf("path=%s,image_type=raw", p.st.DiskPath(vmID)),	120	}
101	fmt.Sprintf("path=%s,image_type=raw,readonly=on", p.st.SeedPath(vmID)),	121	// image_type=raw is load-bearing, not decoration: autodetected raw makes
		122	// CH DISABLE SECTOR 0 WRITES, so the first boot's growpart rewrites the
		123	// partition table only in memory and the guest dies in initramfs at its
		124	// first power cycle. Declared raw keeps the GPT writable. (Autodetection
		125	// is also deprecated in CH v53.)
		126	for _, d := range p.disks(spec) {
		127	if d.ReadOnly {
		128	args = append(args, fmt.Sprintf("path=%s,image_type=raw,readonly=on", d.Path))
		129	continue
		130	}
		131	args = append(args, fmt.Sprintf("path=%s,image_type=raw", d.Path))
		132	}
		133	return append(args,
102	"--net", fmt.Sprintf("tap=%s,mac=%s", tap, mac),	134	"--net", fmt.Sprintf("tap=%s,mac=%s", tap, mac),
103	"--serial", fmt.Sprintf("socket=%s", p.st.SerialSocketPath(vmID)),	135	"--serial", fmt.Sprintf("socket=%s", p.st.SerialSocketPath(vmID)),
104	"--console", "off",	136	"--console", "off",
105	}	137	)
106	}	138	}
107		139
108	// maxDiskGB caps a VM disk at 1 PiB (2^20 GiB) — far beyond any real host,	140	// maxDiskGB caps a VM disk at 1 PiB (2^20 GiB) — far beyond any real host,
@@ -122,18 +154,18 @@ func permanentf(format string, args ...any) error {
122	return permanentError{err: fmt.Errorf(format, args...)}	154	return permanentError{err: fmt.Errorf(format, args...)}
123	}	155	}
124		156
125	// PrepareDisk creates the VM disk by making a reflink copy of basePath	157	// PrepareRootDisk creates the VM's root disk by making a reflink copy of
126	// (instant on XFS/btrfs; silent full-copy fallback on ext4) and then	158	// basePath (instant on XFS/btrfs; silent full-copy fallback on ext4) and then
127	// truncating it to spec.DiskGB gigabytes.	159	// truncating it to spec.DiskGB gigabytes.
128	//	160	//
129	// truncate -s sets an EXACT size, so a target smaller than the base image	161	// truncate -s sets an EXACT size, so a target smaller than the base image
130	// would silently chop the guest filesystem. PrepareDisk refuses to shrink:	162	// would silently chop the guest filesystem. PrepareRootDisk refuses to
131	// spec.DiskGB must be in [1, maxDiskGB] and cover the base image. The range	163	// shrink: spec.DiskGB must be in [1, maxDiskGB] and cover the base image. The
132	// check runs first so the byte computation is overflow-safe (a naive	164	// range check runs first so the byte computation is overflow-safe (a naive
133	// DiskGB<<30 wraps to a small positive value for e.g. 2^34+10, bypassing the	165	// DiskGB<<30 wraps to a small positive value for e.g. 2^34+10, bypassing the
134	// guard). Pinned by TestPrepareDiskRefusesToShrinkBaseImage and	166	// guard). Pinned by TestPrepareRootDiskRefusesToShrinkBaseImage and
135	// TestPrepareDiskShrinkGuardEdgeCases.	167	// TestPrepareRootDiskShrinkGuardEdgeCases.
136	func (p *Provisioner) PrepareDisk(ctx context.Context, spec state.VMSpec, basePath string) error {	168	func (p *Provisioner) PrepareRootDisk(ctx context.Context, spec state.VMSpec, basePath string) error {
137	base, err := os.Stat(basePath)	169	base, err := os.Stat(basePath)
138	if err != nil {	170	if err != nil {
139	return fmt.Errorf("stat base image %s: %w", basePath, err)	171	return fmt.Errorf("stat base image %s: %w", basePath, err)
@@ -183,12 +215,23 @@ func (p *Provisioner) pidPath(vmID string) string {
183	// its own session (Setsid) so it survives an agent restart. A goroutine calls	215	// its own session (Setsid) so it survives an agent restart. A goroutine calls
184	// cmd.Wait to reap the child when it exits.	216	// cmd.Wait to reap the child when it exits.
185	//	217	//
186	// The ctx parameter is deliberately NOT wired to the process: the VM's	218	// The ctx parameter bounds the network attach only. It is deliberately NOT
187	// lifetime must not be tied to the agent's (exec.CommandContext SIGKILLs the	219	// wired to the process: the VM's lifetime must not be tied to the agent's
188	// child on ctx cancel, which would hard-power-off every VM on a graceful	220	// (exec.CommandContext SIGKILLs the child on ctx cancel, which would
189	// agent stop). Stopping a VM is exclusively the job of Shutdown/Kill, driven	221	// hard-power-off every VM on a graceful agent stop). Stopping a VM is
190	// by the reconcile loop. Pinned by TestBootedVMSurvivesCtxCancellation.	222	// exclusively the job of Shutdown/Destroy, driven by the reconcile loop. Pinned
191	func (p *Provisioner) Boot(_ context.Context, vmID string, spec state.VMSpec) error {	223	// by TestBootedVMSurvivesCtxCancellation.
		224	func (p *Provisioner) Boot(ctx context.Context, vmID string, spec state.VMSpec) error {
		225	// Attach the network FIRST: cloud-hypervisor takes the tap as a launch
		226	// argument, so the device has to exist before the process does. A failure
		227	// here surfaces the networking layer's own message — letting the launch
		228	// fail instead yields an illegible cloud-hypervisor error for the same
		229	// root cause — and %w keeps a Permanent() marker (a tap name collision)
		230	// unwrappable, so reconcile still terminal-fails it in one attempt.
		231	if err := p.attachNet(ctx, vmID); err != nil {
		232	return fmt.Errorf("attach network %s: %w", vmID, err)
		233	}
		234
192	// Remove stale sockets from a previous run. CH does NOT unlink a	235	// Remove stale sockets from a previous run. CH does NOT unlink a
193	// pre-existing socket path before binding (it removes it only on clean	236	// pre-existing socket path before binding (it removes it only on clean
194	// exit), so after a CH crash, SIGKILL, or host reboot a stale socket	237	// exit), so after a CH crash, SIGKILL, or host reboot a stale socket
@@ -218,7 +261,7 @@ func (p *Provisioner) Boot(_ context.Context, vmID string, spec state.VMSpec) er
218	// Close the log fd in the parent; the child has its own copy.	261	// Close the log fd in the parent; the child has its own copy.
219	_ = chLog.Close()	262	_ = chLog.Close()
220		263
221	// Write PID file so Running/Shutdown/Kill can find the process later.	264	// Write PID file so Running/Shutdown/Destroy can find the process later.
222	pidData := []byte(strconv.Itoa(cmd.Process.Pid))	265	pidData := []byte(strconv.Itoa(cmd.Process.Pid))
223	if err := os.WriteFile(p.pidPath(vmID), pidData, 0o600); err != nil {	266	if err := os.WriteFile(p.pidPath(vmID), pidData, 0o600); err != nil {
224	// Best effort — kill the orphan if we can't track it, and Wait to reap it	267	// Best effort — kill the orphan if we can't track it, and Wait to reap it
@@ -243,6 +286,18 @@ func (p *Provisioner) Boot(_ context.Context, vmID string, spec state.VMSpec) er
243	return nil	286	return nil
244	}	287	}
245		288
		289	// attachNet gives the VM its address and its tap, in that order — the tap
		290	// carries the DHCP reservation, so the address has to be known first.
		291	// Idempotent: both halves tolerate a re-run, which is what makes a Boot retry
		292	// and a restart-after-host-reboot the same code path.
		293	func (p *Provisioner) attachNet(ctx context.Context, vmID string) error {
		294	ip, err := p.net.ReserveIP(vmID)
		295	if err != nil {
		296	return err
		297	}
		298	return p.net.CreateTap(ctx, vmID, ip)
		299	}
		300
246	// readPID reads the PID file for vmID and returns the PID, or 0 on error.	301	// readPID reads the PID file for vmID and returns the PID, or 0 on error.
247	func (p *Provisioner) readPID(vmID string) int {	302	func (p *Provisioner) readPID(vmID string) int {
248	raw, err := os.ReadFile(p.pidPath(vmID))	303	raw, err := os.ReadFile(p.pidPath(vmID))
@@ -270,6 +325,12 @@ func (p *Provisioner) Running(vmID string) bool {
270	return syscall.Kill(pid, 0) == nil	325	return syscall.Kill(pid, 0) == nil
271	}	326	}
272		327
		328	// Address returns the address reserved for vmID on the bridge, or "" when the
		329	// VM has none. This backend allocates before the guest boots, so the answer is
		330	// available the instant Boot returns — reconcile polls it either way, because a
		331	// backend whose host OS hands out addresses cannot answer that early.
		332	func (p *Provisioner) Address(vmID string) string { return p.net.Address(vmID) }
		333
273	// socketClient returns an *http.Client whose transport dials over the VM's	334	// socketClient returns an *http.Client whose transport dials over the VM's
274	// Unix socket.	335	// Unix socket.
275	func (p Provisioner) socketClient(vmID string) http.Client {	336	func (p Provisioner) socketClient(vmID string) http.Client {
@@ -320,13 +381,31 @@ func (p *Provisioner) sigterm(vmID string) error {
320	return nil	381	return nil
321	}	382	}
322		383
323	// Kill sends SIGKILL to the cloud-hypervisor process for vmID, stops its	384	// Destroy stops the VM and releases every host resource it holds: the
324	// serial pump, and removes the PID file and serial socket.	385	// cloud-hypervisor process, its serial pump and socket, and its tap device with
325	func (p *Provisioner) Kill(ctx context.Context, vmID string) error {	386	// the DHCP reservation the tap carries.
326	// Pump teardown + serial-socket removal come BEFORE the SIGKILL error	387	//
327	// return: a pump leaked past a failed Kill would dial a deleted path	388	// The kill's error is deliberately not propagated. SIGKILL against a pidfile is
328	// forever. ch.sock is deliberately left in place — the next Boot clears	389	// idempotent, and the failures it can report (the process is gone, or is not
329	// a stale API socket itself, and reap's DeleteVM removes the whole VM dir.	390	// ours) are not ones a retry fixes. Releasing the tap IS retryable — `ip link
		391	// del` under an expired pass context fails, and an orphaned eit-XXXXXXXX device
		392	// has nothing left to reap it (the agent's EnsureBridge does not sweep orphan
		393	// taps) — so that is the error that reaches reconcile and keeps the VM's record
		394	// for another tick.
		395	func (p *Provisioner) Destroy(ctx context.Context, vmID string) error {
		396	p.kill(vmID)
		397	return p.net.DeleteTap(ctx, vmID)
		398	}
		399
		400	// kill SIGKILLs the cloud-hypervisor process for vmID, stops its serial pump,
		401	// and removes the PID file and serial socket. It takes no context: killing a
		402	// process by pidfile is a syscall, and pretending otherwise made callers think
		403	// a cancelled context could skip a teardown.
		404	func (p *Provisioner) kill(vmID string) {
		405	// Pump teardown + serial-socket removal come BEFORE the SIGKILL: a pump
		406	// leaked past a failed kill would dial a deleted path forever. ch.sock is
		407	// deliberately left in place — the next Boot clears a stale API socket
		408	// itself, and reap's DeleteVM removes the whole VM dir.
330	if p.Pumps != nil {	409	if p.Pumps != nil {
331	p.Pumps.Stop(vmID)	410	p.Pumps.Stop(vmID)
332	}	411	}
@@ -334,11 +413,11 @@ func (p *Provisioner) Kill(ctx context.Context, vmID string) error {
334	pid := p.readPID(vmID)	413	pid := p.readPID(vmID)
335	if pid != 0 {	414	if pid != 0 {
336	if err := syscall.Kill(pid, syscall.SIGKILL); err != nil && err != syscall.ESRCH {	415	if err := syscall.Kill(pid, syscall.SIGKILL); err != nil && err != syscall.ESRCH {
337	return fmt.Errorf("SIGKILL %s (pid %d): %w", vmID, pid, err)	416	// Leave the pidfile: a SIGKILL that failed with anything but
		417	// ESRCH has not proved the process gone, and deleting its only
		418	// record would claim that it is.
		419	return
338	}	420	}
339	}	421	}
340	// pidfile removal stays LAST: a failed SIGKILL must leave the pidfile so
341	// a Kill retry can find the process again.
342	_ = os.Remove(p.pidPath(vmID))	422	_ = os.Remove(p.pidPath(vmID))
343	return nil
344	}	423	}

internal/agent/cloudhv/cloudhv_test.go

Old		New
@@ -3,6 +3,7 @@ package cloudhv
3	import (	3	import (
4	"context"	4	"context"
5	"errors"	5	"errors"
		6	"fmt"
6	"net"	7	"net"
7	"net/http"	8	"net/http"
8	"os"	9	"os"
@@ -26,7 +27,7 @@ func TestMACDeterministicAndLocallyAdministered(t *testing.T) {
26		27
27	func TestBuildArgs(t *testing.T) {	28	func TestBuildArgs(t *testing.T) {
28	st, _ := state.Open(t.TempDir())	29	st, _ := state.Open(t.TempDir())
29	p := New(st, "/usr/bin/cloud-hypervisor", "/usr/share/ch/hypervisor-fw", nil)	30	p := New(st, "/usr/bin/cloud-hypervisor", "/usr/share/ch/hypervisor-fw", nil, newFakeNet())
30	spec := state.VMSpec{VMID: "vm1", VCPUs: 2, MemMB: 2048}	31	spec := state.VMSpec{VMID: "vm1", VCPUs: 2, MemMB: 2048}
31	args := p.buildArgs(spec)	32	args := p.buildArgs(spec)
32	joined := strings.Join(args, " ")	33	joined := strings.Join(args, " ")
@@ -47,17 +48,47 @@ func TestBuildArgs(t *testing.T) {
47	func TestBuildArgsDeclaresRawImageType(t *testing.T) {	48	func TestBuildArgsDeclaresRawImageType(t *testing.T) {
48	st, err := state.Open(t.TempDir())	49	st, err := state.Open(t.TempDir())
49	require.NoError(t, err)	50	require.NoError(t, err)
50	p := New(st, "ch", "fw", nil)	51	p := New(st, "ch", "fw", nil, newFakeNet())
51	args := p.buildArgs(state.VMSpec{VMID: "vm1", VCPUs: 1, MemMB: 512})	52	args := p.buildArgs(state.VMSpec{VMID: "vm1", VCPUs: 1, MemMB: 512})
52	joined := strings.Join(args, " ")	53	joined := strings.Join(args, " ")
53	assert.Contains(t, joined, st.DiskPath("vm1")+",image_type=raw")	54	assert.Contains(t, joined, st.DiskPath("vm1")+",image_type=raw")
54	assert.Contains(t, joined, st.SeedPath("vm1")+",image_type=raw,readonly=on")	55	assert.Contains(t, joined, st.SeedPath("vm1")+",image_type=raw,readonly=on")
55	}	56	}
56		57
		58	// TestDisksPutsRootFirstAndSeedReadOnly pins the attachment order: index 0
		59	// must be the root disk. Both cloud-hypervisor and vfkit assign /dev/vda to
		60	// the first --disk/volume argument, so a reorder here silently swaps which
		61	// device the guest boots from.
		62	func TestDisksPutsRootFirstAndSeedReadOnly(t *testing.T) {
		63	st, err := state.Open(t.TempDir())
		64	if err != nil {
		65	t.Fatalf("state.Open: %v", err)
		66	}
		67	p := New(st, "cloud-hypervisor", "/fw/CLOUDHV.fd", nil, newFakeNet())
		68
		69	disks := p.disks(state.VMSpec{VMID: "vm-1"})
		70
		71	if len(disks) != 2 {
		72	t.Fatalf("want root + seed, got %d disks: %v", len(disks), disks)
		73	}
		74	if disks[0].Path != st.DiskPath("vm-1") {
		75	t.Errorf("index 0 must be the root disk, got %q", disks[0].Path)
		76	}
		77	if disks[0].ReadOnly {
		78	t.Error("root disk must be writable")
		79	}
		80	if disks[1].Path != st.SeedPath("vm-1") {
		81	t.Errorf("index 1 must be the seed, got %q", disks[1].Path)
		82	}
		83	if !disks[1].ReadOnly {
		84	t.Error("seed must be read-only")
		85	}
		86	}
		87
57	func TestBuildArgsUsesSerialSocket(t *testing.T) {	88	func TestBuildArgsUsesSerialSocket(t *testing.T) {
58	st, err := state.Open(t.TempDir())	89	st, err := state.Open(t.TempDir())
59	require.NoError(t, err)	90	require.NoError(t, err)
60	p := New(st, "ch", "fw", nil) // chBin/firmware placeholders fine: args only	91	p := New(st, "ch", "fw", nil, newFakeNet()) // chBin/firmware placeholders fine: args only
61	args := p.buildArgs(state.VMSpec{VMID: "vm1", VCPUs: 1, MemMB: 512, DiskGB: 5})	92	args := p.buildArgs(state.VMSpec{VMID: "vm1", VCPUs: 1, MemMB: 512, DiskGB: 5})
62	joined := strings.Join(args, " ")	93	joined := strings.Join(args, " ")
63	assert.Contains(t, joined, "--serial socket="+st.SerialSocketPath("vm1"))	94	assert.Contains(t, joined, "--serial socket="+st.SerialSocketPath("vm1"))
@@ -70,17 +101,17 @@ type pumpRecorder struct{ ensured, stopped []string }
70	func (r *pumpRecorder) Ensure(vmID string) { r.ensured = append(r.ensured, vmID) }	101	func (r *pumpRecorder) Ensure(vmID string) { r.ensured = append(r.ensured, vmID) }
71	func (r *pumpRecorder) Stop(vmID string) { r.stopped = append(r.stopped, vmID) }	102	func (r *pumpRecorder) Stop(vmID string) { r.stopped = append(r.stopped, vmID) }
72		103
73	func TestKillStopsPumpAndRemovesSerialSocket(t *testing.T) {	104	func TestDestroyStopsPumpAndRemovesSerialSocket(t *testing.T) {
74	st, err := state.Open(t.TempDir())	105	st, err := state.Open(t.TempDir())
75	require.NoError(t, err)	106	require.NoError(t, err)
76	rec := &pumpRecorder{}	107	rec := &pumpRecorder{}
77	p := New(st, "ch", "fw", nil)	108	p := New(st, "ch", "fw", nil, newFakeNet())
78	p.Pumps = rec	109	p.Pumps = rec
79	// No CH process running: Kill on an unknown VM must still be clean —	110	// No CH process running: Destroy on an unknown VM must still be clean —
80	// and must still stop the pump + remove the socket path.	111	// and must still stop the pump + remove the socket path.
81	require.NoError(t, os.MkdirAll(st.VMDir("vm1"), 0o755))	112	require.NoError(t, os.MkdirAll(st.VMDir("vm1"), 0o755))
82	require.NoError(t, os.WriteFile(st.SerialSocketPath("vm1"), nil, 0o644))	113	require.NoError(t, os.WriteFile(st.SerialSocketPath("vm1"), nil, 0o644))
83	_ = p.Kill(context.Background(), "vm1")	114	_ = p.Destroy(context.Background(), "vm1")
84	assert.Equal(t, []string{"vm1"}, rec.stopped)	115	assert.Equal(t, []string{"vm1"}, rec.stopped)
85	_, statErr := os.Stat(st.SerialSocketPath("vm1"))	116	_, statErr := os.Stat(st.SerialSocketPath("vm1"))
86	assert.True(t, os.IsNotExist(statErr), "stale serial socket must be removed")	117	assert.True(t, os.IsNotExist(statErr), "stale serial socket must be removed")
@@ -90,7 +121,7 @@ func TestKillStopsPumpAndRemovesSerialSocket(t *testing.T) {
90	// the serial pump right after CH starts (a regression here = silently dead	121	// the serial pump right after CH starts (a regression here = silently dead
91	// consoles fleet-wide, since reconcile tests fake the whole Provisioner). It	122	// consoles fleet-wide, since reconcile tests fake the whole Provisioner). It
92	// also pins the deliberate asymmetry: Shutdown must NOT stop the pump — the	123	// also pins the deliberate asymmetry: Shutdown must NOT stop the pump — the
93	// pump survives VM stop/start (it reconnects to the fresh socket); only Kill	124	// pump survives VM stop/start (it reconnects to the fresh socket); only Destroy
94	// (VM destroyed) tears it down.	125	// (VM destroyed) tears it down.
95	func TestBootEnsuresPump(t *testing.T) {	126	func TestBootEnsuresPump(t *testing.T) {
96	st, err := state.Open(t.TempDir())	127	st, err := state.Open(t.TempDir())
@@ -105,10 +136,10 @@ func TestBootEnsuresPump(t *testing.T) {
105	require.NoError(t, os.WriteFile(fakeCH, []byte("#!/bin/sh\nexec sleep 60\n"), 0o755))	136	require.NoError(t, os.WriteFile(fakeCH, []byte("#!/bin/sh\nexec sleep 60\n"), 0o755))
106		137
107	rec := &pumpRecorder{}	138	rec := &pumpRecorder{}
108	p := New(st, fakeCH, "fw", nil)	139	p := New(st, fakeCH, "fw", nil, newFakeNet())
109	p.Pumps = rec	140	p.Pumps = rec
110	require.NoError(t, p.Boot(context.Background(), vmID, state.VMSpec{VMID: vmID, VCPUs: 1, MemMB: 128}))	141	require.NoError(t, p.Boot(context.Background(), vmID, state.VMSpec{VMID: vmID, VCPUs: 1, MemMB: 128}))
111	t.Cleanup(func() { _ = p.Kill(context.Background(), vmID) })	142	t.Cleanup(func() { _ = p.Destroy(context.Background(), vmID) })
112	assert.Equal(t, []string{vmID}, rec.ensured, "Boot must attach the serial pump")	143	assert.Equal(t, []string{vmID}, rec.ensured, "Boot must attach the serial pump")
113		144
114	// No API socket is listening, so Shutdown falls back to SIGTERM — either	145	// No API socket is listening, so Shutdown falls back to SIGTERM — either
@@ -129,7 +160,7 @@ func sparseFile(t *testing.T, size int64) string {
129	return path	160	return path
130	}	161	}
131		162
132	func TestPrepareDiskUsesReflinkAndResizes(t *testing.T) {	163	func TestPrepareRootDiskUsesReflinkAndResizes(t *testing.T) {
133	var cmds []string	164	var cmds []string
134	run := func(ctx context.Context, name string, args ...string) (string, error) {	165	run := func(ctx context.Context, name string, args ...string) (string, error) {
135	cmds = append(cmds, name+" "+strings.Join(args, " "))	166	cmds = append(cmds, name+" "+strings.Join(args, " "))
@@ -139,9 +170,9 @@ func TestPrepareDiskUsesReflinkAndResizes(t *testing.T) {
139	return "", nil	170	return "", nil
140	}	171	}
141	st, _ := state.Open(t.TempDir())	172	st, _ := state.Open(t.TempDir())
142	p := New(st, "ch", "fw", run)	173	p := New(st, "ch", "fw", run, newFakeNet())
143	base := sparseFile(t, 1<<20) // 1 MiB base, well under the 10G target	174	base := sparseFile(t, 1<<20) // 1 MiB base, well under the 10G target
144	require.NoError(t, p.PrepareDisk(context.Background(),	175	require.NoError(t, p.PrepareRootDisk(context.Background(),
145	state.VMSpec{VMID: "vm1", DiskGB: 10}, base))	176	state.VMSpec{VMID: "vm1", DiskGB: 10}, base))
146	joined := strings.Join(cmds, "\n")	177	joined := strings.Join(cmds, "\n")
147	// Artifacts are built at a .partial sibling then renamed into place atomically.	178	// Artifacts are built at a .partial sibling then renamed into place atomically.
@@ -153,37 +184,37 @@ func TestPrepareDiskUsesReflinkAndResizes(t *testing.T) {
153	assert.NoFileExists(t, partial, "temp must not survive a successful prepare")	184	assert.NoFileExists(t, partial, "temp must not survive a successful prepare")
154	}	185	}
155		186
156	// TestPrepareDiskRefusesToShrinkBaseImage pins the never-shrink guard:	187	// TestPrepareRootDiskRefusesToShrinkBaseImage pins the never-shrink guard:
157	// truncate -s sets an EXACT size, so a DiskGB smaller than the base image	188	// truncate -s sets an EXACT size, so a DiskGB smaller than the base image
158	// would silently corrupt the guest filesystem. PrepareDisk must refuse	189	// would silently corrupt the guest filesystem. PrepareRootDisk must refuse
159	// before running any command.	190	// before running any command.
160	func TestPrepareDiskRefusesToShrinkBaseImage(t *testing.T) {	191	func TestPrepareRootDiskRefusesToShrinkBaseImage(t *testing.T) {
161	var cmds []string	192	var cmds []string
162	run := func(ctx context.Context, name string, args ...string) (string, error) {	193	run := func(ctx context.Context, name string, args ...string) (string, error) {
163	cmds = append(cmds, name+" "+strings.Join(args, " "))	194	cmds = append(cmds, name+" "+strings.Join(args, " "))
164	return "", nil	195	return "", nil
165	}	196	}
166	st, _ := state.Open(t.TempDir())	197	st, _ := state.Open(t.TempDir())
167	p := New(st, "ch", "fw", run)	198	p := New(st, "ch", "fw", run, newFakeNet())
168	base := sparseFile(t, 2<<30) // sparse 2 GiB base	199	base := sparseFile(t, 2<<30) // sparse 2 GiB base
169	err := p.PrepareDisk(context.Background(),	200	err := p.PrepareRootDisk(context.Background(),
170	state.VMSpec{VMID: "vm1", DiskGB: 1}, base)	201	state.VMSpec{VMID: "vm1", DiskGB: 1}, base)
171	require.Error(t, err, "shrinking below the base image must be rejected")	202	require.Error(t, err, "shrinking below the base image must be rejected")
172	assert.Contains(t, err.Error(), "smaller than base image")	203	assert.Contains(t, err.Error(), "smaller than base image")
173	assert.Empty(t, cmds, "no command may run once the shrink is detected")	204	assert.Empty(t, cmds, "no command may run once the shrink is detected")
174	}	205	}
175		206
176	// TestPrepareDiskFailsOnMissingBaseImage: a stat failure on the base image is	207	// TestPrepareRootDiskFailsOnMissingBaseImage: a stat failure on the base image is
177	// a real error (cp would fail anyway) and must surface before any command.	208	// a real error (cp would fail anyway) and must surface before any command.
178	func TestPrepareDiskFailsOnMissingBaseImage(t *testing.T) {	209	func TestPrepareRootDiskFailsOnMissingBaseImage(t *testing.T) {
179	var cmds []string	210	var cmds []string
180	run := func(ctx context.Context, name string, args ...string) (string, error) {	211	run := func(ctx context.Context, name string, args ...string) (string, error) {
181	cmds = append(cmds, name+" "+strings.Join(args, " "))	212	cmds = append(cmds, name+" "+strings.Join(args, " "))
182	return "", nil	213	return "", nil
183	}	214	}
184	st, _ := state.Open(t.TempDir())	215	st, _ := state.Open(t.TempDir())
185	p := New(st, "ch", "fw", run)	216	p := New(st, "ch", "fw", run, newFakeNet())
186	err := p.PrepareDisk(context.Background(),	217	err := p.PrepareRootDisk(context.Background(),
187	state.VMSpec{VMID: "vm1", DiskGB: 10}, "/nonexistent/base.raw")	218	state.VMSpec{VMID: "vm1", DiskGB: 10}, "/nonexistent/base.raw")
188	require.Error(t, err)	219	require.Error(t, err)
189	assert.Empty(t, cmds)	220	assert.Empty(t, cmds)
@@ -204,15 +235,15 @@ func TestBootedVMSurvivesCtxCancellation(t *testing.T) {
204	require.NoError(t, st.SaveVM(state.Record{Spec: state.VMSpec{VMID: vmID}}))	235	require.NoError(t, st.SaveVM(state.Record{Spec: state.VMSpec{VMID: vmID}}))
205		236
206	// Fake cloud-hypervisor: ignores its CLI args and sleeps. exec replaces the	237	// Fake cloud-hypervisor: ignores its CLI args and sleeps. exec replaces the
207	// shell, so the pidfile PID is the sleep itself and the cleanup Kill reaps	238	// shell, so the pidfile PID is the sleep itself and the cleanup Destroy
208	// it directly (no orphaned child if the shell wouldn't exec its tail).	239	// reaps it directly (no orphaned child if the shell wouldn't exec its tail).
209	fakeCH := filepath.Join(t.TempDir(), "fake-ch")	240	fakeCH := filepath.Join(t.TempDir(), "fake-ch")
210	require.NoError(t, os.WriteFile(fakeCH, []byte("#!/bin/sh\nexec sleep 60\n"), 0o755))	241	require.NoError(t, os.WriteFile(fakeCH, []byte("#!/bin/sh\nexec sleep 60\n"), 0o755))
211		242
212	p := New(st, fakeCH, "fw", nil)	243	p := New(st, fakeCH, "fw", nil, newFakeNet())
213	ctx, cancel := context.WithCancel(context.Background())	244	ctx, cancel := context.WithCancel(context.Background())
214	require.NoError(t, p.Boot(ctx, vmID, state.VMSpec{VMID: vmID, VCPUs: 1, MemMB: 128}))	245	require.NoError(t, p.Boot(ctx, vmID, state.VMSpec{VMID: vmID, VCPUs: 1, MemMB: 128}))
215	t.Cleanup(func() { _ = p.Kill(context.Background(), vmID) })	246	t.Cleanup(func() { _ = p.Destroy(context.Background(), vmID) })
216	require.True(t, p.Running(vmID), "process must be alive right after Boot")	247	require.True(t, p.Running(vmID), "process must be alive right after Boot")
217		248
218	cancel()	249	cancel()
@@ -262,7 +293,7 @@ func TestShutdownFallsBackToSIGTERMOn500(t *testing.T) {
262	go srv.Serve(ln) //nolint:errcheck	293	go srv.Serve(ln) //nolint:errcheck
263	defer srv.Close()	294	defer srv.Close()
264		295
265	p := New(st, "ch", "fw", nil)	296	p := New(st, "ch", "fw", nil, newFakeNet())
266	// No pidfile → sigterm fallback is a no-op returning nil.	297	// No pidfile → sigterm fallback is a no-op returning nil.
267	err = p.Shutdown(context.Background(), vmID)	298	err = p.Shutdown(context.Background(), vmID)
268	assert.NoError(t, err, "Shutdown must not error when SIGTERM fallback has no pidfile")	299	assert.NoError(t, err, "Shutdown must not error when SIGTERM fallback has no pidfile")
@@ -291,15 +322,15 @@ func TestShutdownSucceedsOn204(t *testing.T) {
291	go srv.Serve(ln) //nolint:errcheck	322	go srv.Serve(ln) //nolint:errcheck
292	defer srv.Close()	323	defer srv.Close()
293		324
294	p := New(st, "ch", "fw", nil)	325	p := New(st, "ch", "fw", nil, newFakeNet())
295	assert.NoError(t, p.Shutdown(context.Background(), vmID))	326	assert.NoError(t, p.Shutdown(context.Background(), vmID))
296	}	327	}
297		328
298	// TestPrepareDiskShrinkGuardEdgeCases pins the guard's arithmetic: an exact	329	// TestPrepareRootDiskShrinkGuardEdgeCases pins the guard's arithmetic: an exact
299	// fit is allowed (truncate to same size is a no-op), and an absurd DiskGB	330	// fit is allowed (truncate to same size is a no-op), and an absurd DiskGB
300	// that would overflow a byte computation (DiskGB<<30) must still be caught —	331	// that would overflow a byte computation (DiskGB<<30) must still be caught —
301	// 2^34+10 wraps to a small positive byte count if computed naively.	332	// 2^34+10 wraps to a small positive byte count if computed naively.
302	func TestPrepareDiskShrinkGuardEdgeCases(t *testing.T) {	333	func TestPrepareRootDiskShrinkGuardEdgeCases(t *testing.T) {
303	newP := func(t testing.T, cmds []string) *Provisioner {	334	newP := func(t testing.T, cmds []string) *Provisioner {
304	run := func(ctx context.Context, name string, args ...string) (string, error) {	335	run := func(ctx context.Context, name string, args ...string) (string, error) {
305	cmds = append(cmds, name)	336	cmds = append(cmds, name)
@@ -309,14 +340,14 @@ func TestPrepareDiskShrinkGuardEdgeCases(t *testing.T) {
309	return "", nil	340	return "", nil
310	}	341	}
311	st, _ := state.Open(t.TempDir())	342	st, _ := state.Open(t.TempDir())
312	return New(st, "ch", "fw", run)	343	return New(st, "ch", "fw", run, newFakeNet())
313	}	344	}
314		345
315	t.Run("exact fit is allowed", func(t *testing.T) {	346	t.Run("exact fit is allowed", func(t *testing.T) {
316	var cmds []string	347	var cmds []string
317	p := newP(t, &cmds)	348	p := newP(t, &cmds)
318	base := sparseFile(t, 2<<30) // exactly 2 GiB	349	base := sparseFile(t, 2<<30) // exactly 2 GiB
319	require.NoError(t, p.PrepareDisk(context.Background(),	350	require.NoError(t, p.PrepareRootDisk(context.Background(),
320	state.VMSpec{VMID: "vm1", DiskGB: 2}, base))	351	state.VMSpec{VMID: "vm1", DiskGB: 2}, base))
321	assert.NotEmpty(t, cmds)	352	assert.NotEmpty(t, cmds)
322	})	353	})
@@ -329,7 +360,7 @@ func TestPrepareDiskShrinkGuardEdgeCases(t *testing.T) {
329	// wraps to 10 GiB-ish positive — either way the request is absurd and	360	// wraps to 10 GiB-ish positive — either way the request is absurd and
330	// must not run cp/truncate with a nonsense size. DiskGB=2^34+10 > any	361	// must not run cp/truncate with a nonsense size. DiskGB=2^34+10 > any
331	// real disk; the guard must reject or the size math must be exact.	362	// real disk; the guard must reject or the size math must be exact.
332	err := p.PrepareDisk(context.Background(),	363	err := p.PrepareRootDisk(context.Background(),
333	state.VMSpec{VMID: "vm1", DiskGB: (1 << 34) + 10}, base)	364	state.VMSpec{VMID: "vm1", DiskGB: (1 << 34) + 10}, base)
334	if err == nil {	365	if err == nil {
335	// Accepting it is only sound if the target genuinely covers the	366	// Accepting it is only sound if the target genuinely covers the
@@ -348,22 +379,22 @@ func TestDiskGuardErrorsArePermanent(t *testing.T) {
348	st, _ := state.Open(t.TempDir())	379	st, _ := state.Open(t.TempDir())
349	p := New(st, "ch", "fw", func(ctx context.Context, name string, args ...string) (string, error) {	380	p := New(st, "ch", "fw", func(ctx context.Context, name string, args ...string) (string, error) {
350	return "", nil	381	return "", nil
351	})	382	}, newFakeNet())
352	isPermanent := func(err error) bool {	383	isPermanent := func(err error) bool {
353	var m interface{ Permanent() bool }	384	var m interface{ Permanent() bool }
354	return errors.As(err, &m) && m.Permanent()	385	return errors.As(err, &m) && m.Permanent()
355	}	386	}
356		387
357	base := sparseFile(t, 2<<30)	388	base := sparseFile(t, 2<<30)
358	err := p.PrepareDisk(context.Background(), state.VMSpec{VMID: "v", DiskGB: 1}, base)	389	err := p.PrepareRootDisk(context.Background(), state.VMSpec{VMID: "v", DiskGB: 1}, base)
359	require.Error(t, err)	390	require.Error(t, err)
360	assert.True(t, isPermanent(err), "shrink guard error must be permanent")	391	assert.True(t, isPermanent(err), "shrink guard error must be permanent")
361		392
362	err = p.PrepareDisk(context.Background(), state.VMSpec{VMID: "v", DiskGB: (1 << 34) + 10}, base)	393	err = p.PrepareRootDisk(context.Background(), state.VMSpec{VMID: "v", DiskGB: (1 << 34) + 10}, base)
363	require.Error(t, err)	394	require.Error(t, err)
364	assert.True(t, isPermanent(err), "range guard error must be permanent")	395	assert.True(t, isPermanent(err), "range guard error must be permanent")
365		396
366	err = p.PrepareDisk(context.Background(), state.VMSpec{VMID: "v", DiskGB: 10}, "/nonexistent/base.raw")	397	err = p.PrepareRootDisk(context.Background(), state.VMSpec{VMID: "v", DiskGB: 10}, "/nonexistent/base.raw")
367	require.Error(t, err)	398	require.Error(t, err)
368	assert.False(t, isPermanent(err), "stat failure may be transient (NFS blip, cache re-fetch)")	399	assert.False(t, isPermanent(err), "stat failure may be transient (NFS blip, cache re-fetch)")
369	}	400	}
@@ -397,3 +428,160 @@ func TestBootstrapDest(t *testing.T) {
397	}	428	}
398	})	429	})
399	}	430	}
		431
		432	// fakeNet is the host networking a VM attaches to. It mirrors netenv: sticky
		433	// addresses keyed by vmID, idempotent create/delete, and a tap name derived
		434	// from the id — enough to drive argument assembly and lifecycle without a
		435	// bridge on the test machine.
		436	type fakeNet struct {
		437	reserved map[string]string
		438	taps map[string]bool
		439	reserveErr error
		440	tapErr error
		441	delErr error
		442	}
		443
		444	func newFakeNet() *fakeNet {
		445	return &fakeNet{reserved: map[string]string{}, taps: map[string]bool{}}
		446	}
		447
		448	func (f *fakeNet) ReserveIP(vmID string) (string, error) {
		449	if f.reserveErr != nil {
		450	return "", f.reserveErr
		451	}
		452	if ip, ok := f.reserved[vmID]; ok {
		453	return ip, nil
		454	}
		455	ip := fmt.Sprintf("10.77.1.%d", len(f.reserved)+2)
		456	f.reserved[vmID] = ip
		457	return ip, nil
		458	}
		459
		460	func (f *fakeNet) Address(vmID string) string { return f.reserved[vmID] }
		461
		462	func (f *fakeNet) CreateTap(_ context.Context, vmID, _ string) error {
		463	if f.tapErr != nil {
		464	return f.tapErr
		465	}
		466	f.taps[vmID] = true
		467	return nil
		468	}
		469
		470	func (f *fakeNet) DeleteTap(_ context.Context, vmID string) error {
		471	if f.delErr != nil {
		472	return f.delErr
		473	}
		474	delete(f.reserved, vmID)
		475	delete(f.taps, vmID)
		476	return nil
		477	}
		478
		479	func (f *fakeNet) TapName(vmID string) string {
		480	if len(vmID) > 8 {
		481	vmID = vmID[:8]
		482	}
		483	return "eit-" + vmID
		484	}
		485
		486	// TestBootAttachesTheNetworkBeforeLaunching pins the fold: the --net argument
		487	// names a device that exists, because Boot reserved the address and created the
		488	// tap itself. Nothing above this package sequences that any more.
		489	func TestBootAttachesTheNetworkBeforeLaunching(t *testing.T) {
		490	st, err := state.Open(t.TempDir())
		491	require.NoError(t, err)
		492	vmID := "vm-attach"
		493	require.NoError(t, st.SaveVM(state.Record{Spec: state.VMSpec{VMID: vmID}}))
		494
		495	fakeCH := filepath.Join(t.TempDir(), "fake-ch")
		496	require.NoError(t, os.WriteFile(fakeCH, []byte("#!/bin/sh\nexec sleep 60\n"), 0o755))
		497
		498	fn := newFakeNet()
		499	p := New(st, fakeCH, "fw", nil, fn)
		500	require.NoError(t, p.Boot(context.Background(), vmID, state.VMSpec{VMID: vmID, VCPUs: 1, MemMB: 128}))
		501	t.Cleanup(func() { _ = p.Destroy(context.Background(), vmID) })
		502
		503	assert.True(t, fn.taps[vmID], "Boot must create the VM's tap")
		504	assert.Equal(t, "10.77.1.2", p.Address(vmID), "Boot must reserve the VM's address")
		505	}
		506
		507	// TestBootKeepsTheAddressTheVMAlreadyHolds pins that a reboot is not a renumber:
		508	// the reservation the VM already holds is what Boot re-attaches it to.
		509	func TestBootKeepsTheAddressTheVMAlreadyHolds(t *testing.T) {
		510	st, err := state.Open(t.TempDir())
		511	require.NoError(t, err)
		512	vmID := "vm-sticky"
		513	// Boot opens ch.log under st.VMDir(vmID); SaveVM is what creates that dir.
		514	require.NoError(t, st.SaveVM(state.Record{Spec: state.VMSpec{VMID: vmID}}))
		515
		516	fakeCH := filepath.Join(t.TempDir(), "fake-ch")
		517	require.NoError(t, os.WriteFile(fakeCH, []byte("#!/bin/sh\nexec sleep 60\n"), 0o755))
		518
		519	fn := newFakeNet()
		520	fn.reserved[vmID] = "10.77.1.44"
		521	p := New(st, fakeCH, "fw", nil, fn)
		522	require.NoError(t, p.Boot(context.Background(), vmID, state.VMSpec{VMID: vmID, VCPUs: 1, MemMB: 128}))
		523	t.Cleanup(func() { _ = p.Destroy(context.Background(), vmID) })
		524
		525	assert.Equal(t, "10.77.1.44", p.Address(vmID))
		526	}
		527
		528	// TestBootFailsBeforeLaunchWhenTheNetworkRefuses pins that no cloud-hypervisor
		529	// process is started for a VM that has no network — and that the network's own
		530	// error reaches the caller, not an illegible hypervisor failure for the same
		531	// root cause.
		532	func TestBootFailsBeforeLaunchWhenTheNetworkRefuses(t *testing.T) {
		533	st, err := state.Open(t.TempDir())
		534	require.NoError(t, err)
		535	vmID := "vm-nonet"
		536
		537	fn := newFakeNet()
		538	fn.tapErr = errors.New("link eit-vm-nonet exists but is not a TAP device")
		539	// A chBin that would fail loudly if it were ever reached.
		540	p := New(st, "/nonexistent/cloud-hypervisor", "fw", nil, fn)
		541
		542	err = p.Boot(context.Background(), vmID, state.VMSpec{VMID: vmID, VCPUs: 1, MemMB: 128})
		543	require.Error(t, err)
		544	assert.Contains(t, err.Error(), "not a TAP device")
		545	assert.False(t, p.Running(vmID), "no hypervisor may be launched without a network")
		546	}
		547
		548	// TestBootPreservesThePermanenceMarkerFromTheNetwork pins that wrapping the
		549	// attach error keeps reconcile's Permanent() fast-fail working: a tap name
		550	// collision must still terminal-fail in one attempt rather than burn the budget.
		551	func TestBootPreservesThePermanenceMarkerFromTheNetwork(t *testing.T) {
		552	st, err := state.Open(t.TempDir())
		553	require.NoError(t, err)
		554
		555	fn := newFakeNet()
		556	fn.tapErr = permanentError{err: errors.New("name collision")}
		557	p := New(st, "/nonexistent/cloud-hypervisor", "fw", nil, fn)
		558
		559	err = p.Boot(context.Background(), "vm-perm", state.VMSpec{VMID: "vm-perm", VCPUs: 1, MemMB: 128})
		560	require.Error(t, err)
		561	var perm interface{ Permanent() bool }
		562	require.True(t, errors.As(err, &perm), "the attach error must stay unwrappable")
		563	assert.True(t, perm.Permanent())
		564	}
		565
		566	// TestDestroyReleasesTheNetworkAndRetriesOnFailure pins the teardown contract:
		567	// Destroy releases the address, and a failure to release is the error reconcile
		568	// sees (it keeps the record and retries next tick).
		569	func TestDestroyReleasesTheNetworkAndRetriesOnFailure(t *testing.T) {
		570	st, err := state.Open(t.TempDir())
		571	require.NoError(t, err)
		572	vmID := "vm-teardown"
		573	require.NoError(t, os.MkdirAll(st.VMDir(vmID), 0o755))
		574
		575	fn := newFakeNet()
		576	_, err = fn.ReserveIP(vmID)
		577	require.NoError(t, err)
		578	p := New(st, "ch", "fw", nil, fn)
		579
		580	fn.delErr = errors.New("ip link del: context deadline exceeded")
		581	require.Error(t, p.Destroy(context.Background(), vmID),
		582	"a failed release must reach reconcile so the record is kept")
		583
		584	fn.delErr = nil
		585	require.NoError(t, p.Destroy(context.Background(), vmID))
		586	assert.Empty(t, p.Address(vmID), "destroy releases the VM's address")
		587	}

internal/agent/cloudhv/console.go

@@ @@ -0,0 +1,21 @@ @@
+package cloudhv
+import (
+	"io"
+	"net"
+)
+// ConsoleSource opens a VM's guest console by dialling the unix socket
+// cloud-hypervisor serves the serial line on (--serial socket=…). It satisfies
+// serialpump.ConsoleSource; the func it wraps resolves a VM's socket path
+// (production: state.Store.SerialSocketPath).
+//
+// The socket serves ONE client at a time — a new connection kicks the old one —
+// so the pump must be its only client. Do not connect socat or similar.
+type ConsoleSource func(vmID string) string
+// Open dials vmID's serial socket. It fails while the VM is down or the socket
+// has not yet been created; the pump's reopen loop is the retry.
+func (s ConsoleSource) Open(vmID string) (io.ReadWriteCloser, error) {
+	return net.Dial("unix", s(vmID))
+}

internal/agent/cloudhv/console_test.go

@@ @@ -0,0 +1,48 @@ @@
+package cloudhv
+import (
+	"net"
+	"path/filepath"
+	"testing"
+	"github.com/stretchr/testify/require"
+)
+// TestConsoleSourceOpensTheSerialSocket pins that ConsoleSource dials the
+// socket cloud-hypervisor serves the serial line on, and that the resulting
+// stream is readable — the shape serialpump.ConsoleSource.Open requires.
+func TestConsoleSourceOpensTheSerialSocket(t *testing.T) {
+	dir := t.TempDir()
+	sock := filepath.Join(dir, "serial.sock")
+	ln, err := net.Listen("unix", sock)
+	require.NoError(t, err)
+	defer ln.Close()
+	go func() {
+		c, aerr := ln.Accept()
+		if aerr == nil {
+			_, _ = c.Write([]byte("boot"))
+			c.Close()
+		}
+	}()
+	src := ConsoleSource(func(vmID string) string { return sock })
+	rwc, err := src.Open("vm-1")
+	require.NoError(t, err)
+	defer rwc.Close()
+	buf := make([]byte, 4)
+	_, err = rwc.Read(buf)
+	require.NoError(t, err)
+	require.Equal(t, "boot", string(buf))
+}
+// TestConsoleSourceErrorsWhenSocketAbsent pins that Open surfaces the dial
+// error rather than swallowing it — the pump's reopen loop depends on seeing
+// a real error to drive its backoff.
+func TestConsoleSourceErrorsWhenSocketAbsent(t *testing.T) {
+	src := ConsoleSource(func(vmID string) string {
+		return filepath.Join(t.TempDir(), "missing.sock")
+	})
+	_, err := src.Open("vm-1")
+	require.Error(t, err)
+}

internal/agent/dhcp/dhcp.go

Old		New
@@ -96,8 +96,9 @@ func (s *Server) lookup(mac net.HardwareAddr) (net.IP, bool) {
96	return ip, ok	96	return ip, ok
97	}	97	}
98		98
99	// LookupForTest exposes reservation lookup for tests in other packages.	99	// Lookup returns the reserved IP for mac. It never allocates — it is the read
100	func (s *Server) LookupForTest(mac net.HardwareAddr) (net.IP, bool) { return s.lookup(mac) }	100	// side of the table, used to answer "what address does this guest have?".
		101	func (s *Server) Lookup(mac net.HardwareAddr) (net.IP, bool) { return s.lookup(mac) }
101		102
102	// buildReply produces the DHCP response for req, or (nil, nil) when req's MAC	103	// buildReply produces the DHCP response for req, or (nil, nil) when req's MAC
103	// has no reservation (fail-closed) or is a message type we do not serve. It	104	// has no reservation (fail-closed) or is a message type we do not serve. It

internal/agent/dhcp/dhcp_test.go

Old		New
@@ -52,6 +52,20 @@ func TestReserveSkipsPreexistingReservation(t *testing.T) {
52	assert.Equal(t, "10.77.1.3", ip.String(), "an externally-set reservation is treated as used")	52	assert.Equal(t, "10.77.1.3", ip.String(), "an externally-set reservation is treated as used")
53	}	53	}
54		54
		55	// TestReserveHandsBackAPreloadedReservation pins the handoff the startup replay
		56	// depends on: an address preloaded into the table is what Reserve returns for
		57	// its MAC, so a surviving guest keeps its address across an agent restart
		58	// instead of being renumbered on its next boot.
		59	func TestReserveHandsBackAPreloadedReservation(t *testing.T) {
		60	s := newTestServer()
		61	survivor, _ := net.ParseMAC("52:54:00:cc:cc:cc")
		62	s.SetReservation(survivor, net.ParseIP("10.77.1.55"))
		63
		64	ip, err := s.Reserve(survivor)
		65	require.NoError(t, err)
		66	assert.Equal(t, "10.77.1.55", ip.String(), "a preloaded reservation is never renumbered")
		67	}
		68
55	func mustMAC(t *testing.T, s string) net.HardwareAddr {	69	func mustMAC(t *testing.T, s string) net.HardwareAddr {
56	t.Helper()	70	t.Helper()
57	m, err := net.ParseMAC(s)	71	m, err := net.ParseMAC(s)

internal/agent/hostinfo/hostinfo.go

Old		New
@@ -7,6 +7,7 @@ package hostinfo
7	import (	7	import (
8	"context"	8	"context"
9	"os"	9	"os"
		10	"runtime"
10	"strconv"	11	"strconv"
11	"strings"	12	"strings"
12	"syscall"	13	"syscall"
@@ -76,6 +77,23 @@ func Metrics(stateDir string) *pb.HostMetrics {
76	return m	77	return m
77	}	78	}
78		79
		80	// BootID returns an opaque token that changes across host reboots. Reconcile
		81	// compares it only for equality, to detect that VMs were lost to a reboot — so
		82	// any per-boot-unique value satisfies the contract, and the source is
		83	// per-platform.
		84	func BootID() string { return readBootID() }
		85
		86	// Capacity returns the host's TOTAL capacity: total disk at stateDir, total
		87	// memory, and CPU count. The server computes allocated/available by subtracting
		88	// the sum of live VM specs, so these must be totals, not free space.
		89	func Capacity(stateDir string) *pb.Capacity {
		90	return &pb.Capacity{
		91	Vcpus: int64(runtime.NumCPU()),
		92	MemMb: totalMemMB(),
		93	DiskGb: totalDiskGB(stateDir),
		94	}
		95	}
		96
79	// memAvailableMB prefers /proc/meminfo MemAvailable (accounts for reclaimable	97	// memAvailableMB prefers /proc/meminfo MemAvailable (accounts for reclaimable
80	// cache — the honest headroom number); falls back to Sysinfo Free+Buffer, which	98	// cache — the honest headroom number); falls back to Sysinfo Free+Buffer, which
81	// undercounts cache, when meminfo is unreadable.	99	// undercounts cache, when meminfo is unreadable.

internal/agent/hostinfo/hostinfo_linux.go

@@ @@ -0,0 +1,37 @@ @@
+//go:build linux
+package hostinfo
+import (
+	"os"
+	"strings"
+	"syscall"
+)
+// bootIDPath is the kernel's per-boot UUID. Seam for tests.
+var bootIDPath = "/proc/sys/kernel/random/boot_id"
+func readBootID() string {
+	b, err := os.ReadFile(bootIDPath)
+	if err != nil {
+		return ""
+	}
+	return strings.TrimSpace(string(b))
+}
+func totalMemMB() int64 {
+	var si syscall.Sysinfo_t
+	if err := sysinfoFn(&si); err != nil {
+		return 0
+	}
+	return int64(si.Totalram) * int64(si.Unit) / mb
+}
+func totalDiskGB(stateDir string) int64 {
+	var fs syscall.Statfs_t
+	if err := statfsFn(stateDir, &fs); err != nil {
+		return 0
+	}
+	// Bsize is int64 on Linux and uint32 on Darwin — convert, never multiply raw.
+	return int64(fs.Blocks) * int64(fs.Bsize) / gb
+}

internal/agent/hostinfo/hostinfo_test.go

@@ @@ -2,8 +2,10 @@ package hostinfo @@
+import (
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"runtime"
+	"syscall"
+	"testing"
@@ @@ -81,6 +83,112 @@ func TestFactsBestEffortOnMissingFiles(t *testing.T) { @@
+	assert.Empty(t, f.GetCpuModel())
+// TestBootIDIsStableWithinABoot guards the contract reconcile relies on: a
+// single boot must always yield the same token, whatever the source.
+func TestBootIDIsStableWithinABoot(t *testing.T) {
+	a := BootID()
+	b := BootID()
+	if a == "" {
+		t.Skip("no boot identity available in this environment")
+	}
+	if a != b {
+		t.Errorf("BootID must be stable within a boot: %q != %q", a, b)
+	}
+}
+// TestBootIDReadsSeam exercises the bootIDPath seam directly (rather than
+// relying on the real kernel file being present/stable in CI) and asserts the
+// trailing newline the kernel appends is trimmed.
+func TestBootIDReadsSeam(t *testing.T) {
+	old := bootIDPath
+	defer func() { bootIDPath = old }()
+	bootIDPath = writeFixture(t, "boot_id", "1b2c3d4e-0000-1111-2222-333344445555\n")
+	assert.Equal(t, "1b2c3d4e-0000-1111-2222-333344445555", BootID())
+}
+// TestCapacityReportsTotals guards the contract the server relies on:
+// Capacity must report host TOTALS (never free space), since the server
+// derives available capacity by subtracting live VM specs from these numbers.
+func TestCapacityReportsTotals(t *testing.T) {
+	c := Capacity(t.TempDir())
+	if c.GetVcpus() < 1 {
+		t.Errorf("vcpus must be at least 1, got %d", c.GetVcpus())
+	}
+	if c.GetMemMb() < 1 {
+		t.Errorf("mem_mb must be positive, got %d", c.GetMemMb())
+	}
+	if c.GetDiskGb() < 0 {
+		t.Errorf("disk_gb must not be negative, got %d", c.GetDiskGb())
+	}
+}
+// TestCapacityExactArithmetic pins TOTALS (not free space): a regression that
+// swaps in Bavail/Bfree or free-memory fields must fail this test, since the
+// server derives available capacity by subtracting live VM specs from
+// whatever Capacity reports.
+func TestCapacityExactArithmetic(t *testing.T) {
+	oldSys, oldStat := sysinfoFn, statfsFn
+	defer func() { sysinfoFn, statfsFn = oldSys, oldStat }()
+	sysinfoFn = func(si *syscall.Sysinfo_t) error {
+		*si = syscall.Sysinfo_t{
+			Totalram: 8 * 1024 * 1024 * 1024, // 8 GiB
+			Unit:     1,
+		}
+		return nil
+	}
+	statfsFn = func(_ string, fs *syscall.Statfs_t) error {
+		*fs = syscall.Statfs_t{
+			Bsize:  4096,
+			Blocks: 26214400, // 100 GiB total
+		}
+		return nil
+	}
+	c := Capacity("/whatever")
+	assert.Equal(t, int64(runtime.NumCPU()), c.GetVcpus())
+	assert.Equal(t, int64(8192), c.GetMemMb())
+	assert.Equal(t, int64(100), c.GetDiskGb())
+}
+// TestCapacityBestEffortOnSyscallFailure covers the package's best-effort
+// contract: a failing probe zeroes its own dimension rather than erroring or
+// blocking, and the two probes fail independently of each other.
+func TestCapacityBestEffortOnSyscallFailure(t *testing.T) {
+	oldSys, oldStat := sysinfoFn, statfsFn
+	defer func() { sysinfoFn, statfsFn = oldSys, oldStat }()
+	sysinfoFn = func(*syscall.Sysinfo_t) error { return errors.New("boom") }
+	statfsFn = func(_ string, fs *syscall.Statfs_t) error {
+		*fs = syscall.Statfs_t{Bsize: 4096, Blocks: 26214400}
+		return nil
+	}
+	c := Capacity("/whatever")
+	assert.Equal(t, int64(0), c.GetMemMb())
+	assert.Equal(t, int64(100), c.GetDiskGb())
+	sysinfoFn = func(si *syscall.Sysinfo_t) error {
+		*si = syscall.Sysinfo_t{Totalram: 8 * 1024 * 1024 * 1024, Unit: 1}
+		return nil
+	}
+	statfsFn = func(_ string, _ *syscall.Statfs_t) error { return errors.New("boom") }
+	c = Capacity("/whatever")
+	assert.Equal(t, int64(8192), c.GetMemMb())
+	assert.Equal(t, int64(0), c.GetDiskGb())
+}
+// TestBootIDMissingFile covers readBootID's error path directly (rather than
+// only via the trimmed-happy-path seam test): an unreadable boot_id must
+// yield "", per the package's best-effort contract, not a panic or error.
+func TestBootIDMissingFile(t *testing.T) {
+	old := bootIDPath
+	defer func() { bootIDPath = old }()
+	bootIDPath = filepath.Join(t.TempDir(), "does-not-exist")
+	assert.Empty(t, BootID())
+}
+func TestMetricsComputes(t *testing.T) {
+	oldSys, oldStat, oldMem := sysinfoFn, statfsFn, memInfoPath
+	defer func() { sysinfoFn, statfsFn, memInfoPath = oldSys, oldStat, oldMem }()

internal/agent/imagecache/imagecache.go

Old		New
@@ -4,7 +4,7 @@
4	// file's mtime, and after every successful Ensure the oldest .raw images	4	// file's mtime, and after every successful Ensure the oldest .raw images
5	// beyond MaxBytes are evicted (never the one just ensured).	5	// beyond MaxBytes are evicted (never the one just ensured).
6	//	6	//
7	// Eviction is safe for running VMs: PrepareDisk copies (reflink) the base, so	7	// Eviction is safe for running VMs: PrepareRootDisk copies (reflink) the base, so
8	// nothing references it after create. With concurrent per-VM creates there is a	8	// nothing references it after create. With concurrent per-VM creates there is a
9	// narrow window where one VM's evict can remove a base another VM just resolved	9	// narrow window where one VM's evict can remove a base another VM just resolved
10	// but has not yet copied — it requires the two in-flight images to exceed	10	// but has not yet copied — it requires the two in-flight images to exceed

internal/agent/netenv/netenv.go

Old		New
@@ -87,11 +87,24 @@ func (e tapConflictError) Permanent() bool { return true }
87	// Gateway returns the host-side IP (.1) on the bridge, as a bare address string.	87	// Gateway returns the host-side IP (.1) on the bridge, as a bare address string.
88	func (n *Net) Gateway() string { return n.cidr.Masked().Addr().Next().String() }	88	func (n *Net) Gateway() string { return n.cidr.Masked().Addr().Next().String() }
89		89
		90	// TapName returns the TAP device name for vmID: "eit-" plus the first 8
		91	// characters of the id, giving 12 characters, safely below the 15-char
		92	// IFNAMSIZ limit. It is a method rather than a package function because it is
		93	// part of what a host networking backend answers for — a backend with no tap
		94	// device has no such name, and nothing above the platform line may assume one.
		95	func (n *Net) TapName(vmID string) string {
		96	prefix := vmID
		97	if len(prefix) > 8 {
		98	prefix = prefix[:8]
		99	}
		100	return "eit-" + prefix
		101	}
		102
90	// ReserveIP returns a sticky IP for vmID, allocated from the bridge CIDR and	103	// ReserveIP returns a sticky IP for vmID, allocated from the bridge CIDR and
91	// recorded as the guest's DHCP reservation (keyed by the VM's deterministic	104	// recorded as the guest's DHCP reservation (keyed by the VM's deterministic
92	// MAC). Idempotent: the same VM gets the same address across reboots and agent	105	// MAC). Idempotent: the same VM gets the same address across reboots and agent
93	// restarts. This is the reconcile addressing seam — the DHCP server owns the	106	// restarts. The table lives in memory, so an address survives an agent restart
94	// used-address set, so no caller-supplied used-set is needed.	107	// only by way of the startup replay that rebuilds it from durable records.
95	func (n *Net) ReserveIP(vmID string) (string, error) {	108	func (n *Net) ReserveIP(vmID string) (string, error) {
96	mac, err := net.ParseMAC(state.MAC(vmID))	109	mac, err := net.ParseMAC(state.MAC(vmID))
97	if err != nil {	110	if err != nil {
@@ -104,6 +117,21 @@ func (n *Net) ReserveIP(vmID string) (string, error) {
104	return ip.String(), nil	117	return ip.String(), nil
105	}	118	}
106		119
		120	// Address returns the address currently reserved for vmID, or "" when the VM
		121	// holds no reservation. Unlike ReserveIP it never allocates: it is the polled
		122	// read of what this host believes the guest's address is.
		123	func (n *Net) Address(vmID string) string {
		124	mac, err := net.ParseMAC(state.MAC(vmID))
		125	if err != nil {
		126	return ""
		127	}
		128	ip, ok := n.dhcp.Lookup(mac)
		129	if !ok {
		130	return ""
		131	}
		132	return ip.String()
		133	}
		134
107	// tolerated reports whether err carries one of the given substrings in either	135	// tolerated reports whether err carries one of the given substrings in either
108	// the command stdout or the error message. iproute2 puts the same condition in	136	// the command stdout or the error message. iproute2 puts the same condition in
109	// different streams across versions, so both are checked.	137	// different streams across versions, so both are checked.
@@ -233,7 +261,7 @@ func (n *Net) AddReservation(vmID, ip string) {
233	// the previous attempt's tap must not mask the retry's real error. Pinned by	261	// the previous attempt's tap must not mask the retry's real error. Pinned by
234	// TestCreateTapIdempotentWhenTapExists.	262	// TestCreateTapIdempotentWhenTapExists.
235	func (n *Net) CreateTap(ctx context.Context, vmID, ip string) error {	263	func (n *Net) CreateTap(ctx context.Context, vmID, ip string) error {
236	tap := state.TapName(vmID)	264	tap := n.TapName(vmID)
237	if _, err := n.run(ctx, "ip", "link", "show", "dev", tap); err != nil {	265	if _, err := n.run(ctx, "ip", "link", "show", "dev", tap); err != nil {
238	if _, err := n.best(ctx, "ip", "tuntap", "add", "dev", tap, "mode", "tap"); err != nil {	266	if _, err := n.best(ctx, "ip", "tuntap", "add", "dev", tap, "mode", "tap"); err != nil {
239	return err	267	return err
@@ -257,7 +285,7 @@ func (n *Net) CreateTap(ctx context.Context, vmID, ip string) error {
257	// DeleteTap removes the VM's DHCP reservation and TAP device. Idempotent:	285	// DeleteTap removes the VM's DHCP reservation and TAP device. Idempotent:
258	// a missing device is tolerated.	286	// a missing device is tolerated.
259	func (n *Net) DeleteTap(ctx context.Context, vmID string) error {	287	func (n *Net) DeleteTap(ctx context.Context, vmID string) error {
260	tap := state.TapName(vmID)	288	tap := n.TapName(vmID)
261	if mac, err := net.ParseMAC(state.MAC(vmID)); err == nil {	289	if mac, err := net.ParseMAC(state.MAC(vmID)); err == nil {
262	n.dhcp.RemoveReservation(mac)	290	n.dhcp.RemoveReservation(mac)
263	}	291	}

internal/agent/netenv/netenv_test.go

Old		New
@@ -359,7 +359,7 @@ func TestCreateTapAddsReservationAndTapCommands(t *testing.T) {
359	assert.Contains(t, all, "ip link set eit-vm-abc12 master eitri0")	359	assert.Contains(t, all, "ip link set eit-vm-abc12 master eitri0")
360		360
361	mac, _ := net.ParseMAC(stateMAC("vm-abc12345"))	361	mac, _ := net.ParseMAC(stateMAC("vm-abc12345"))
362	ip, ok := n.dhcp.LookupForTest(mac)	362	ip, ok := n.dhcp.Lookup(mac)
363	require.True(t, ok)	363	require.True(t, ok)
364	assert.Equal(t, "10.77.1.7", ip.String())	364	assert.Equal(t, "10.77.1.7", ip.String())
365	}	365	}
@@ -374,7 +374,7 @@ func TestReserveIPAllocatesAndRecordsReservation(t *testing.T) {
374	assert.Equal(t, "10.77.1.2", ip)	374	assert.Equal(t, "10.77.1.2", ip)
375		375
376	mac, _ := net.ParseMAC(stateMAC("vm-alpha"))	376	mac, _ := net.ParseMAC(stateMAC("vm-alpha"))
377	got, ok := n.dhcp.LookupForTest(mac)	377	got, ok := n.dhcp.Lookup(mac)
378	require.True(t, ok, "ReserveIP must record the DHCP reservation")	378	require.True(t, ok, "ReserveIP must record the DHCP reservation")
379	assert.Equal(t, "10.77.1.2", got.String())	379	assert.Equal(t, "10.77.1.2", got.String())
380		380
@@ -396,6 +396,58 @@ func TestDeleteTapRemovesReservationAndTap(t *testing.T) {
396		396
397	assert.Contains(t, joinCalls(calls), "ip link del eit-vm-abc12")	397	assert.Contains(t, joinCalls(calls), "ip link del eit-vm-abc12")
398	mac, _ := net.ParseMAC(stateMAC("vm-abc12345"))	398	mac, _ := net.ParseMAC(stateMAC("vm-abc12345"))
399	_, ok := n.dhcp.LookupForTest(mac)	399	_, ok := n.dhcp.Lookup(mac)
400	assert.False(t, ok, "reservation must be gone after DeleteTap")	400	assert.False(t, ok, "reservation must be gone after DeleteTap")
401	}	401	}
		402
		403	func TestAddressReportsTheReservationWithoutAllocating(t *testing.T) {
		404	noop := func(_ context.Context, _ string, _ ...string) (string, error) { return "", nil }
		405	n, err := New(noop, "10.77.1.0/24")
		406	require.NoError(t, err)
		407
		408	assert.Empty(t, n.Address("vm-alpha"), "an unreserved VM has no address")
		409
		410	ip, err := n.ReserveIP("vm-alpha")
		411	require.NoError(t, err)
		412	assert.Equal(t, ip, n.Address("vm-alpha"))
		413
		414	assert.Empty(t, n.Address("vm-beta"),
		415	"Address must never allocate — only ReserveIP does")
		416
		417	beta, err := n.ReserveIP("vm-beta")
		418	require.NoError(t, err)
		419	assert.Equal(t, "10.77.1.3", beta,
		420	"the read above must not have consumed an address")
		421	}
		422
		423	func TestReserveIPKeepsAPreloadedAddress(t *testing.T) {
		424	noop := func(_ context.Context, _ string, _ ...string) (string, error) { return "", nil }
		425	n, err := New(noop, "10.77.1.0/24")
		426	require.NoError(t, err)
		427
		428	// The reservation table is in-memory and preloaded at agent startup from
		429	// durable records. A VM that survived a restart must get the address its
		430	// guest already holds back, not the next free one.
		431	n.AddReservation("vm-survivor", "10.77.1.55")
		432
		433	ip, err := n.ReserveIP("vm-survivor")
		434	require.NoError(t, err)
		435	assert.Equal(t, "10.77.1.55", ip)
		436	assert.Equal(t, "10.77.1.55", n.Address("vm-survivor"))
		437
		438	// A preloaded address is held against every other VM too: one address
		439	// served to two guests breaks connectivity for both.
		440	other, err := n.ReserveIP("vm-new")
		441	require.NoError(t, err)
		442	assert.NotEqual(t, ip, other)
		443	}
		444
		445	func TestTapNameIsTruncatedBelowIFNAMSIZ(t *testing.T) {
		446	noop := func(_ context.Context, _ string, _ ...string) (string, error) { return "", nil }
		447	n, err := New(noop, "10.77.1.0/24")
		448	require.NoError(t, err)
		449
		450	assert.Equal(t, "eit-vm1", n.TapName("vm1"))
		451	assert.Equal(t, "eit-abcdefgh", n.TapName("abcdefghijklmnop"))
		452	assert.LessOrEqual(t, len(n.TapName("abcdefghijklmnop")), 15, "IFNAMSIZ is 16 including NUL")
		453	}

internal/agent/reconcile/reconcile.go

Old		New
@@ -17,14 +17,14 @@
17	//	17	//
18	// The "exists" definition deserves a comment:	18	// The "exists" definition deserves a comment:
19	// rec.BootID is the sole completion witness. create() sets it to the current	19	// rec.BootID is the sole completion witness. create() sets it to the current
20	// host boot ID only after every side effect (image, tap, disk, seed, boot) has	20	// host boot ID only after every side effect (image, disk, seed, boot) has
21	// succeeded, so rec.BootID != "" means — and only means — a create finished.	21	// succeeded, so rec.BootID != "" means — and only means — a create finished.
22	// Disk presence is deliberately NOT consulted: create() writes disk.raw in the	22	// Disk presence is deliberately NOT consulted: create() writes disk.raw in the
23	// middle of the sequence, so a create that fails after the disk is written but	23	// middle of the sequence, so a create that fails after the disk is written but
24	// before boot leaves a disk on a still-empty BootID; treating that as "exists"	24	// before boot leaves a disk on a still-empty BootID; treating that as "exists"
25	// would divert the retry to converge() (which rebuilds neither disk nor seed)	25	// would divert the retry to converge() (which rebuilds neither disk nor seed)
26	// and strand the VM. Belt-and-suspenders: PrepareDisk and seed.Build both write	26	// and strand the VM. Belt-and-suspenders: PrepareRootDisk and seed.Build both
27	// via a temp file + rename, so a killed create can never leave a torn artifact.	27	// write via a temp file + rename, so a killed create can never leave a torn artifact.
28	package reconcile	28	package reconcile
29		29
30	import (	30	import (
@@ -44,30 +44,56 @@ import (
44	"github.com/a73x/eitri/internal/pb"	44	"github.com/a73x/eitri/internal/pb"
45	)	45	)
46		46
47	// Provisioner is implemented by the cloud-hypervisor backend (cloudhv.Provisioner).	47	// Provisioner is one VM's whole lifecycle on this host, networking included.
		48	// Implemented by the cloud-hypervisor backend (cloudhv.Provisioner) and by
		49	// inert.Provisioner on a host that runs no guests.
		50	//
		51	// Networking is part of this seam rather than beside it because a VM's network
		52	// attachment is not separately lifecycled from the VM: cloud-hypervisor takes
		53	// its tap as a launch argument and vfkit takes --device virtio-net, so on both
		54	// backends "attach the NIC" is a step inside "start the VM". The verbs that
		55	// would make it a seam of its own — create a tap, reserve an address — are host
		56	// mechanism and do not generalize. What generalizes is the data: the
		57	// deterministic MAC, the host's subnet, and a VM's current address.
48	type Provisioner interface {	58	type Provisioner interface {
49	PrepareDisk(ctx context.Context, spec state.VMSpec, basePath string) error	59	// PrepareRootDisk materialises the VM's root disk from a base image
		60	// (clone + grow). It is root-disk-only by contract: reconcile's rebuild
		61	// path calls create again, so routing a user volume through it would
		62	// destroy data that is meant to outlive the VM.
		63	PrepareRootDisk(ctx context.Context, spec state.VMSpec, basePath string) error
		64
		65	// Boot starts the VM and attaches it to the host network. The address the
		66	// guest ends up with is the backend's business — state.MAC is the
		67	// stickiness key on a backend that allocates addresses itself, and the host
		68	// OS decides on one that does not — so Boot is told nothing about it and
		69	// reports it through Address rather than returning it.
50	Boot(ctx context.Context, vmID string, spec state.VMSpec) error	70	Boot(ctx context.Context, vmID string, spec state.VMSpec) error
		71
51	Shutdown(ctx context.Context, vmID string) error	72	Shutdown(ctx context.Context, vmID string) error
52	Kill(ctx context.Context, vmID string) error	73
		74	// Destroy stops the VM and releases every host resource it holds — its
		75	// process and its network attachment, address reservation included.
		76	// Idempotent: it is called on every tick past a VM's grace until it
		77	// returns nil. A non-nil error KEEPS the VM's record so a later tick
		78	// retries; returning nil is the backend's promise that nothing is left to
		79	// reap, and reconcile deletes the record on the strength of it.
		80	Destroy(ctx context.Context, vmID string) error
		81
53	Running(vmID string) bool	82	Running(vmID string) bool
54	}
55		83
56	// NetEnv is implemented by the host networking layer (netenv.Net).	84	// Address returns the VM's current guest address, or "" when the backend
57	type NetEnv interface {	85	// does not know one (never booted, or gone). It is POLLED rather than
58	CreateTap(ctx context.Context, vmID, ip string) error	86	// returned by Boot: where the host OS's own DHCP server hands out the
59	DeleteTap(ctx context.Context, vmID string) error	87	// address, it does not exist until the guest has booted and asked for one,
60	// ReserveIP returns a sticky IP for vmID from this host's network,	88	// and Boot must not block inside the create slot waiting for that. Empty
61	// recording its DHCP reservation. The host networking layer owns the	89	// means "no answer", never "no address" — see noteAddress.
62	// used-address set, so allocation needs no caller-supplied used-set.	90	Address(vmID string) string
63	ReserveIP(vmID string) (string, error)
64	}	91	}
65		92
66	// Engine is the reconcile loop. All fields must be set before calling Step.	93	// Engine is the reconcile loop. All fields must be set before calling Step.
67	type Engine struct {	94	type Engine struct {
68	St *state.Store	95	St *state.Store
69	Prov Provisioner	96	Prov Provisioner
70	Net NetEnv
71		97
72	// Images resolves an image URL+sha256 to a local base-image path, fetching	98	// Images resolves an image URL+sha256 to a local base-image path, fetching
73	// if necessary. Returns the path to the raw base image.	99	// if necessary. Returns the path to the raw base image.
@@ -297,9 +323,9 @@ func (e Engine) fenceReport(currentEpoch uint64) pb.ActualStateReport {
297	// It is the entire unit of work a per-VM worker runs, and it touches no other	323	// It is the entire unit of work a per-VM worker runs, and it touches no other
298	// VM's record. The only shared state reconcile itself owns is the compute	324	// VM's record. The only shared state reconcile itself owns is the compute
299	// ledger, serialized under Engine.mu (see admit). The pass also reaches shared	325	// ledger, serialized under Engine.mu (see admit). The pass also reaches shared
300	// subsystems it does NOT own — the image cache (Images), the DHCP reservation	326	// subsystems it does NOT own — the image cache (Images), and the host backend
301	// table (Net.CreateTap/DeleteTap), the serial-pump manager (Prov.Boot/Kill) —	327	// (Prov.Boot/Destroy), which owns both the guest's network attachment and its
302	// each of which carries its own locking.	328	// serial pump — each of which carries its own locking.
303	//	329	//
304	// The bool reports whether the pass RAN. A pass that cannot read this VM's own	330	// The bool reports whether the pass RAN. A pass that cannot read this VM's own
305	// record cannot tell an absent VM from a live one, so it changes nothing and	331	// record cannot tell an absent VM from a live one, so it changes nothing and
@@ -374,8 +400,8 @@ func (e Engine) ackDestroyed(rep pb.ActualStateReport, tombstoned map[string]b
374	// split out so a single VM's teardown is expressible on its own.	400	// split out so a single VM's teardown is expressible on its own.
375	func (e Engine) reapVM(ctx context.Context, id string, rec state.Record, isTombstoned bool, res vmResult) {	401	func (e Engine) reapVM(ctx context.Context, id string, rec state.Record, isTombstoned bool, res vmResult) {
376	// Being reaped (absent from desired or tombstoned): free its compute	402	// Being reaped (absent from desired or tombstoned): free its compute
377	// from the admission ledger so a new VM can use it. Idempotent; the IP	403	// from the admission ledger so a new VM can use it. Idempotent; the
378	// reservation is released on destroy by DeleteTap.	404	// address is released by the backend's Destroy.
379	e.releaseCompute(id)	405	e.releaseCompute(id)
380		406
381	now := e.Now()	407	now := e.Now()
@@ -401,21 +427,17 @@ func (e *Engine) reapVM(ctx context.Context, id string, rec state.Record, isTomb
401	grace := e.graceFor(rec)	427	grace := e.graceFor(rec)
402		428
403	if now.Sub(*rec.QuarantinedAt) >= grace {	429	if now.Sub(*rec.QuarantinedAt) >= grace {
404	// Grace expired: destroy the VM.	430	// Grace expired: destroy the VM. Destroy stops the guest AND releases
		431	// its host network resources; on failure the record is KEPT so a later
		432	// tick retries, because deleting it would orphan whatever the backend
		433	// still holds with nothing left to reap it by.
		434	//
405	// NOTE: this may run with an expired pass ctx (watchdog). Safe today	435	// NOTE: this may run with an expired pass ctx (watchdog). Safe today
406	// because cloudhv.Kill ignores ctx (SIGKILL via pidfile) and	436	// because the cloud-hypervisor backend's kill ignores ctx (SIGKILL via
407	// Shutdown falls back to a ctx-free SIGTERM — a future Provisioner	437	// pidfile) — a backend that honors ctx throughout would skip the
408	// that honors ctx here would skip the destroy until a later tick,	438	// destroy until a later tick, which the level-triggered loop tolerates
409	// which the level-triggered loop tolerates but delays.	439	// but delays.
410	_ = e.Prov.Kill(ctx, id)	440	if err := e.Prov.Destroy(ctx, id); err != nil {
411	if err := e.Net.DeleteTap(ctx, id); err != nil {
412	// TAP deletion failed — e.g. DeleteTap runs `ip link del` under
413	// ctx and the ctx expired during a SIGTERM shutdown. KEEP the
414	// record so a later tick retries; deleting it here would orphan
415	// the eit-XXXXXXXX interface with nothing left to reap it (agent
416	// restart's EnsureBridge does not sweep orphan taps). Kill already
417	// stopped the guest, so re-entering here next tick (grace still
418	// expired) just retries DeleteTap idempotently until it succeeds.
419	return	441	return
420	}	442	}
421	_ = e.St.DeleteVM(id)	443	_ = e.St.DeleteVM(id)
@@ -432,9 +454,9 @@ func (e *Engine) reapVM(ctx context.Context, id string, rec state.Record, isTomb
432	//	454	//
433	// exists = record present AND create completed. rec.BootID is the sole	455	// exists = record present AND create completed. rec.BootID is the sole
434	// completion witness: create() writes it only after every side effect (image,	456	// completion witness: create() writes it only after every side effect (image,
435	// tap, disk, seed, boot) has succeeded. Disk presence is NOT a witness —	457	// disk, seed, boot) has succeeded. Disk presence is NOT a witness —
436	// create() writes disk.raw mid-sequence, so a create that fails after	458	// create() writes disk.raw mid-sequence, so a create that fails after
437	// PrepareDisk but before boot leaves a disk on a still-empty BootID. Treating	459	// PrepareRootDisk but before boot leaves a disk on a still-empty BootID. Treating
438	// that disk as "exists" would divert the retry to converge(), which never	460	// that disk as "exists" would divert the retry to converge(), which never
439	// rebuilds the disk or seed, and the VM would never recover.	461	// rebuilds the disk or seed, and the VM would never recover.
440	func (e Engine) reconcileVM(ctx context.Context, d pb.VMDesired, rec state.Record, ok bool, res *vmResult) {	462	func (e Engine) reconcileVM(ctx context.Context, d pb.VMDesired, rec state.Record, ok bool, res *vmResult) {
@@ -455,23 +477,22 @@ func (e *Engine) graceFor(rec state.Record) time.Duration {
455	}	477	}
456		478
457	// admit is the single serialized admission gate: it checks compute quota and,	479	// admit is the single serialized admission gate: it checks compute quota and,
458	// if admitted, reserves the VM's IP. Returns quotaMsg non-empty for a	480	// if admitted, commits the VM's compute to the ledger. It returns a non-empty
459	// NON-TERMINAL quota refusal (nothing is committed, no IP reserved). Otherwise	481	// reason for a NON-TERMINAL quota refusal, in which case nothing is committed.
460	// it commits the VM's compute to the ledger BEFORE reserving the IP, so an IP	482	//
461	// reservation failure still counts this VM against a same-tick sibling's cap	483	// Committing here — before any of the create's side effects run — is what makes
462	// (parity with the old failCreate publish) — err carries the reservation error.	484	// a same-tick sibling count a VM whose create later fails.
463	func (e *Engine) admit(vmID string, spec state.VMSpec) (ip, quotaMsg string, err error) {	485	func (e *Engine) admit(vmID string, spec state.VMSpec) string {
464	e.mu.Lock()	486	e.mu.Lock()
465	defer e.mu.Unlock()	487	defer e.mu.Unlock()
466	if e.committed == nil {	488	if e.committed == nil {
467	e.committed = map[string]state.VMSpec{}	489	e.committed = map[string]state.VMSpec{}
468	}	490	}
469	if msg := e.quotaCheckLocked(vmID, spec); msg != "" {	491	if msg := e.quotaCheckLocked(vmID, spec); msg != "" {
470	return "", msg, nil	492	return msg
471	}	493	}
472	e.committed[vmID] = spec	494	e.committed[vmID] = spec
473	ip, err = e.Net.ReserveIP(vmID)	495	return ""
474	return ip, "", err
475	}	496	}
476		497
477	// acquireCreateSlot takes one of the host's create slots, returning the release	498	// acquireCreateSlot takes one of the host's create slots, returning the release
@@ -509,14 +530,28 @@ func (e *Engine) note(vmID string, spec state.VMSpec) {
509	e.committed[vmID] = spec	530	e.committed[vmID] = spec
510	}	531	}
511		532
512	// releaseCompute frees a VM's compute from the ledger (at quarantine). The IP	533	// releaseCompute frees a VM's compute from the ledger (at quarantine). The VM's
513	// reservation is released separately, on destroy, by DeleteTap.	534	// address is released separately, on destroy, by the backend.
514	func (e *Engine) releaseCompute(vmID string) {	535	func (e *Engine) releaseCompute(vmID string) {
515	e.mu.Lock()	536	e.mu.Lock()
516	defer e.mu.Unlock()	537	defer e.mu.Unlock()
517	delete(e.committed, vmID)	538	delete(e.committed, vmID)
518	}	539	}
519		540
		541	// noteAddress folds the backend's current answer for this VM's address into
		542	// rec, reporting whether it changed. An empty answer never clears a known
		543	// address: "I don't know yet" is not "it has none", and blanking rec.IP would
		544	// break the guest SSH tunnel (syncclient refuses an empty address) for a VM
		545	// that is perfectly reachable.
		546	func (e Engine) noteAddress(rec state.Record) bool {
		547	ip := e.Prov.Address(rec.Spec.VMID)
		548	if ip == "" \|\| ip == rec.IP {
		549	return false
		550	}
		551	rec.IP = ip
		552	return true
		553	}
		554
520	// SeedLedger rebuilds the compute ledger from persisted records at startup so	555	// SeedLedger rebuilds the compute ledger from persisted records at startup so
521	// the first step accounts for every surviving VM. Quarantined records are	556	// the first step accounts for every surviving VM. Quarantined records are
522	// excluded (their guests are stopped; their compute is free).	557	// excluded (their guests are stopped; their compute is free).
@@ -563,8 +598,9 @@ func (e *Engine) quotaCheckLocked(vmID string, spec state.VMSpec) string {
563	}	598	}
564		599
565	// create attempts to create a new VM from desired state d. rec is this VM's own	600	// create attempts to create a new VM from desired state d. rec is this VM's own
566	// prior record (retry budget, existing IP) and ok reports whether one exists.	601	// prior record (retry budget, last known address) and ok reports whether one
567	// Quota and IP come from the serialized admission ledger (see admit).	602	// exists. Quota comes from the serialized admission ledger (see admit); the
		603	// address comes from the backend, at boot.
568	func (e Engine) create(ctx context.Context, d pb.VMDesired, rec state.Record, ok bool, res *vmResult) {	604	func (e Engine) create(ctx context.Context, d pb.VMDesired, rec state.Record, ok bool, res *vmResult) {
569	// Defensive: never start an attempt (which would burn retry budget) on a	605	// Defensive: never start an attempt (which would burn retry budget) on a
570	// context that is already dead. UNREACHABLE today — a pass context is a	606	// context that is already dead. UNREACHABLE today — a pass context is a
@@ -594,25 +630,17 @@ func (e Engine) create(ctx context.Context, d pb.VMDesired, rec state.Record,
594	return	630	return
595	}	631	}
596		632
597	// Serialized admission: quota then a sticky IP, under one lock. A quota	633	// Serialized admission: compute quota under one lock. A refusal is
598	// refusal is NON-TERMINAL — it returns before touching CreateAttempts, so	634	// NON-TERMINAL — it returns before touching CreateAttempts, so once room
599	// once room frees the next tick retries and boots. An IP reservation	635	// frees the next tick retries and boots.
600	// failure DOES spend an attempt (via failCreate); admit has already
601	// committed this VM's compute, so a same-tick sibling counts it.
602	rec.Spec = spec	636	rec.Spec = spec
603	ip, quotaMsg, err := e.admit(d.VmId, spec)	637	if quotaMsg := e.admit(d.VmId, spec); quotaMsg != "" {
604	if quotaMsg != "" {
605	res.report(d.VmId, rec.IP, "stopped", "failed", quotaMsg)	638	res.report(d.VmId, rec.IP, "stopped", "failed", quotaMsg)
606	return	639	return
607	}	640	}
608	rec.CreateAttempts++	641	rec.CreateAttempts++
609	rec.CreatedAt = e.Now()	642	rec.CreatedAt = e.Now()
610	rec.LastError = "" // clear for this attempt	643	rec.LastError = "" // clear for this attempt
611	if err != nil {
612	e.failCreate(ctx, rec, err, res)
613	return
614	}
615	rec.IP = ip
616		644
617	// Record BEFORE side effects so a crash is recoverable.	645	// Record BEFORE side effects so a crash is recoverable.
618	if err := e.St.SaveVM(rec); err != nil {	646	if err := e.St.SaveVM(rec); err != nil {
@@ -644,20 +672,14 @@ func (e Engine) create(ctx context.Context, d pb.VMDesired, rec state.Record,
644	return	672	return
645	}	673	}
646		674
647	// Create tap device.	675	// Prepare root disk.
648	if err := e.Net.CreateTap(ctx, d.VmId, rec.IP); err != nil {	676	if err := e.Prov.PrepareRootDisk(ctx, rec.Spec, basePath); err != nil {
649	e.failCreate(ctx, rec, err, res)
650	return
651	}
652
653	// Prepare disk.
654	if err := e.Prov.PrepareDisk(ctx, rec.Spec, basePath); err != nil {
655	e.failCreate(ctx, rec, err, res)	677	e.failCreate(ctx, rec, err, res)
656	return	678	return
657	}	679	}
658		680
659	// Build cloud-init seed ISO. The guest DHCPs its address from the agent's	681	// Build cloud-init seed ISO. The guest gets its address from the host
660	// reservation, so no IP/gateway is baked into the seed.	682	// network, so no IP or gateway is baked into the seed.
661	if err := e.Seed(e.St.SeedPath(d.VmId), seed.Params{	683	if err := e.Seed(e.St.SeedPath(d.VmId), seed.Params{
662	Hostname: d.Name,	684	Hostname: d.Name,
663	InstanceID: d.VmId,	685	InstanceID: d.VmId,
@@ -671,12 +693,17 @@ func (e Engine) create(ctx context.Context, d pb.VMDesired, rec state.Record,
671	return	693	return
672	}	694	}
673		695
674	// Boot if desired running.	696	// Boot if desired running. The backend attaches the network inside Boot, so
		697	// ask it for the address the moment Boot returns: a backend that knows it
		698	// immediately records it in this same pass — which is what keeps the create
		699	// report row carrying the address, as it always has — and one that learns
		700	// it later fills it in on a converge poll.
675	if d.PowerState == "running" {	701	if d.PowerState == "running" {
676	if err := e.Prov.Boot(ctx, d.VmId, rec.Spec); err != nil {	702	if err := e.Prov.Boot(ctx, d.VmId, rec.Spec); err != nil {
677	e.failCreate(ctx, rec, err, res)	703	e.failCreate(ctx, rec, err, res)
678	return	704	return
679	}	705	}
		706	e.noteAddress(&rec)
680	}	707	}
681		708
682	// Success: record completion.	709	// Success: record completion.
@@ -706,7 +733,7 @@ func permanent(err error) bool {
706	// the VM: the attempt is refunded so a wedged pass can never drive a healthy VM	733	// the VM: the attempt is refunded so a wedged pass can never drive a healthy VM
707	// to terminal failed (see TestWatchdogExpiryDoesNotBurnCreateAttempts). A	734	// to terminal failed (see TestWatchdogExpiryDoesNotBurnCreateAttempts). A
708	// Permanent() error spends the whole budget at once — retrying a permanent	735	// Permanent() error spends the whole budget at once — retrying a permanent
709	// misconfiguration only wastes tap/image work across three ticks (pinned by	736	// misconfiguration only wastes image/disk work across three ticks (pinned by
710	// TestPermanentCreateErrorFailsTerminallyInOneAttempt). Ordering: the ctx	737	// TestPermanentCreateErrorFailsTerminallyInOneAttempt). Ordering: the ctx
711	// refund wins over permanence — a permanent error surfacing under an expired	738	// refund wins over permanence — a permanent error surfacing under an expired
712	// ctx is refunded this tick and, being deterministic, terminal-fails on the	739	// ctx is refunded this tick and, being deterministic, terminal-fails on the
@@ -747,6 +774,14 @@ func (e Engine) converge(ctx context.Context, d pb.VMDesired, rec state.Record
747	running := e.Prov.Running(d.VmId)	774	running := e.Prov.Running(d.VmId)
748	bootID := e.BootID()	775	bootID := e.BootID()
749		776
		777	// The address is a polled fact — a backend whose host OS hands it out only
		778	// learns it once the guest has asked. Persist a first (or changed) answer
		779	// straight away so a tick that takes no other action still records it. On a
		780	// backend that allocates before boot this never fires after the first pass.
		781	if e.noteAddress(&rec) {
		782	_ = e.St.SaveVM(rec)
		783	}
		784
750	// lost = boot ID changed OR process died without a recorded stop request.	785	// lost = boot ID changed OR process died without a recorded stop request.
751	// A deliberately stopped VM has StopRequested=true, so !running && StopRequested is NOT lost.	786	// A deliberately stopped VM has StopRequested=true, so !running && StopRequested is NOT lost.
752	lost := rec.BootID != bootID \|\| (!running && !rec.StopRequested)	787	lost := rec.BootID != bootID \|\| (!running && !rec.StopRequested)
@@ -763,18 +798,14 @@ func (e Engine) converge(ctx context.Context, d pb.VMDesired, rec state.Record
763		798
764	// Persistent lost VM.	799	// Persistent lost VM.
765	if d.PowerState == "running" {	800	if d.PowerState == "running" {
766	// Restart: tap dies on reboot, recreate it. A CreateTap failure	801	// Restart: the backend re-attaches the VM to the host network as
767	// (e.g. a foreign device squatting on the name) is reported with	802	// part of Boot, which is what rebuilds a tap that did not survive
768	// ITS message — letting Boot fail instead yields an illegible	803	// the host reboot.
769	// cloud-hypervisor error for the same root cause.
770	if err := e.Net.CreateTap(ctx, d.VmId, rec.IP); err != nil {
771	e.failConverge(rec, err, res)
772	return
773	}
774	if err := e.Prov.Boot(ctx, d.VmId, rec.Spec); err != nil {	804	if err := e.Prov.Boot(ctx, d.VmId, rec.Spec); err != nil {
775	e.failConverge(rec, err, res)	805	e.failConverge(rec, err, res)
776	return	806	return
777	}	807	}
		808	e.noteAddress(&rec)
778	rec.BootID = bootID	809	rec.BootID = bootID
779	rec.StopRequested = false	810	rec.StopRequested = false
780	rec.LastError = "" // Fix 3: clear stale error on successful restart	811	rec.LastError = "" // Fix 3: clear stale error on successful restart
@@ -792,11 +823,12 @@ func (e Engine) converge(ctx context.Context, d pb.VMDesired, rec state.Record
792		823
793	// Not lost: drive power state.	824	// Not lost: drive power state.
794	if d.PowerState == "running" && !running {	825	if d.PowerState == "running" && !running {
795	// Start the VM.	826	// Start the VM. Boot re-attaches the network, idempotently.
796	if err := e.Prov.Boot(ctx, d.VmId, rec.Spec); err != nil {	827	if err := e.Prov.Boot(ctx, d.VmId, rec.Spec); err != nil {
797	e.failConverge(rec, err, res)	828	e.failConverge(rec, err, res)
798	return	829	return
799	}	830	}
		831	e.noteAddress(&rec)
800	rec.StopRequested = false	832	rec.StopRequested = false
801	rec.LastError = "" // Fix 3: clear stale error on successful boot	833	rec.LastError = "" // Fix 3: clear stale error on successful boot
802	_ = e.St.SaveVM(rec)	834	_ = e.St.SaveVM(rec)

internal/agent/reconcile/reconcile_test.go

Old		New
@@ -23,21 +23,41 @@ import (
23	// goroutines at once, so every method takes mu. Test-goroutine reads of the	23	// goroutines at once, so every method takes mu. Test-goroutine reads of the
24	// recorded slices are deliberately unguarded: they happen after f.step()'s	24	// recorded slices are deliberately unguarded: they happen after f.step()'s
25	// waitIdle, which establishes happens-before against every worker's last write.	25	// waitIdle, which establishes happens-before against every worker's last write.
		26	//
		27	// It is also a real addressing backend — Boot allocates a sticky address the
		28	// way netenv/dhcp does, Address reports it, Destroy releases it. Reconcile's
		29	// addressing is only observable through the seam, so the fake has to own it for
		30	// these tests to mean anything.
26	type fakeProv struct {	31	type fakeProv struct {
27	mu sync.Mutex	32	mu sync.Mutex
28	running map[string]bool	33	running map[string]bool
29	prepCalls int // total PrepareDisk invocations, including failed ones	34	prepCalls int // total PrepareRootDisk invocations, including failed ones
30	prepared []string	35	prepared []string
31	booted []string	36	booted []string
32	shutdown []string	37	shutdown []string
33	killed []string	38	destroyed []string
34	prepErr error	39	prepErr error
35	bootErr error // one-shot: consumed and cleared on first Boot call	40	bootErr error // one-shot: consumed and cleared on first Boot call
		41	destroyErr error // sticky: every Destroy fails until it is cleared
		42
		43	cidr string
		44	addrs map[string]string // vmID -> ip (sticky, mirrors the DHCP table)
		45
		46	// lateAddress switches the fake to the shape a host-run DHCP server has:
		47	// Boot attaches the NIC but assigns nothing, and the address exists only
		48	// once the guest has booted and asked for one (answerAddress).
		49	lateAddress bool
		50	}
		51
		52	func newFakeProv() *fakeProv {
		53	return &fakeProv{
		54	running: map[string]bool{},
		55	addrs: map[string]string{},
		56	cidr: "10.77.1.0/24",
		57	}
36	}	58	}
37		59
38	func newFakeProv() *fakeProv { return &fakeProv{running: map[string]bool{}} }	60	func (f *fakeProv) PrepareRootDisk(_ context.Context, s state.VMSpec, _ string) error {
39
40	func (f *fakeProv) PrepareDisk(_ context.Context, s state.VMSpec, _ string) error {
41	f.mu.Lock()	61	f.mu.Lock()
42	defer f.mu.Unlock()	62	defer f.mu.Unlock()
43	f.prepCalls++ // counted BEFORE the error short-circuit: total invocations	63	f.prepCalls++ // counted BEFORE the error short-circuit: total invocations
@@ -47,6 +67,7 @@ func (f *fakeProv) PrepareDisk(_ context.Context, s state.VMSpec, _ string) erro
47	f.prepared = append(f.prepared, s.VMID)	67	f.prepared = append(f.prepared, s.VMID)
48	return nil	68	return nil
49	}	69	}
		70
50	func (f *fakeProv) Boot(_ context.Context, id string, _ state.VMSpec) error {	71	func (f *fakeProv) Boot(_ context.Context, id string, _ state.VMSpec) error {
51	f.mu.Lock()	72	f.mu.Lock()
52	defer f.mu.Unlock()	73	defer f.mu.Unlock()
@@ -55,88 +76,89 @@ func (f *fakeProv) Boot(_ context.Context, id string, _ state.VMSpec) error {
55	f.bootErr = nil // one-shot: clear after first use	76	f.bootErr = nil // one-shot: clear after first use
56	return err	77	return err
57	}	78	}
		79	if !f.lateAddress {
		80	if err := f.attachLocked(id); err != nil {
		81	return err
		82	}
		83	}
58	f.booted = append(f.booted, id)	84	f.booted = append(f.booted, id)
59	f.running[id] = true	85	f.running[id] = true
60	return nil	86	return nil
61	}	87	}
62	func (f *fakeProv) Shutdown(_ context.Context, id string) error {	88
63	f.mu.Lock()	89	// attachLocked gives id a sticky address, mirroring netenv: a VM that already
64	defer f.mu.Unlock()	90	// holds one keeps it, otherwise one is allocated over the set already handed
65	f.shutdown = append(f.shutdown, id)	91	// out. Serialized by the fake's own lock, exactly as the real backend's DHCP
66	f.running[id] = false	92	// table is — which is what makes concurrent creates collision-free.
		93	func (f *fakeProv) attachLocked(id string) error {
		94	if _, ok := f.addrs[id]; ok {
		95	return nil
		96	}
		97	used := make([]string, 0, len(f.addrs))
		98	for _, ip := range f.addrs {
		99	used = append(used, ip)
		100	}
		101	ip, err := ipalloc.Alloc(f.cidr, used)
		102	if err != nil {
		103	return err
		104	}
		105	f.addrs[id] = ip
67	return nil	106	return nil
68	}	107	}
69	func (f *fakeProv) Kill(_ context.Context, id string) error {	108
		109	func (f *fakeProv) Address(id string) string {
70	f.mu.Lock()	110	f.mu.Lock()
71	defer f.mu.Unlock()	111	defer f.mu.Unlock()
72	f.killed = append(f.killed, id)	112	return f.addrs[id]
73	f.running[id] = false
74	return nil
75	}	113	}
76	func (f *fakeProv) Running(id string) bool {	114
		115	// answerAddress starts answering for id: the guest has now booted and asked the
		116	// host's DHCP server, seconds after Boot returned. Only meaningful under
		117	// lateAddress.
		118	func (f *fakeProv) answerAddress(id string) {
77	f.mu.Lock()	119	f.mu.Lock()
78	defer f.mu.Unlock()	120	defer f.mu.Unlock()
79	return f.running[id]	121	_ = f.attachLocked(id)
80	}	122	}
81		123
82	// fakeNet records host-networking calls. Guarded for the same reason as	124	// forgetAddresses drops the backend's in-memory reservations the way an agent
83	// fakeProv: concurrent per-VM workers. ReserveIP is additionally called under	125	// restart does, leaving it unable to answer Address for a VM that is still
84	// Engine.mu (the admission gate), so its allocation stays serialized either way.	126	// running.
85	type fakeNet struct {	127	func (f *fakeProv) forgetAddresses() {
86	mu sync.Mutex	128	f.mu.Lock()
87	taps []string	129	defer f.mu.Unlock()
88	deleted []string	130	f.addrs = map[string]string{}
89	cidr string
90	reserved map[string]string // vmID -> ip (sticky, mirrors dhcp)
91	reserveErr error // one-shot: consumed and cleared on first ReserveIP
92	}	131	}
93		132
94	func (f *fakeNet) CreateTap(_ context.Context, vmID, ip string) error {	133	func (f *fakeProv) Shutdown(_ context.Context, id string) error {
95	f.mu.Lock()	134	f.mu.Lock()
96	defer f.mu.Unlock()	135	defer f.mu.Unlock()
97	f.taps = append(f.taps, vmID)	136	f.shutdown = append(f.shutdown, id)
		137	f.running[id] = false
98	return nil	138	return nil
99	}	139	}
100	func (f *fakeNet) DeleteTap(_ context.Context, vmID string) error {	140
		141	func (f *fakeProv) Destroy(_ context.Context, id string) error {
101	f.mu.Lock()	142	f.mu.Lock()
102	defer f.mu.Unlock()	143	defer f.mu.Unlock()
103	f.deleted = append(f.deleted, vmID)	144	f.destroyed = append(f.destroyed, id) // every attempt, failed ones included
		145	if f.destroyErr != nil {
		146	return f.destroyErr // released nothing: the backend still holds it all
		147	}
		148	f.running[id] = false
		149	delete(f.addrs, id)
104	return nil	150	return nil
105	}	151	}
106		152
107	// ReserveIP mirrors netenv/dhcp: sticky per VM, allocating distinct addresses	153	func (f *fakeProv) Running(id string) bool {
108	// from the CIDR over the set of already-reserved ones, so reconcile tests
109	// exercise identical addressing through the seam.
110	func (f *fakeNet) ReserveIP(vmID string) (string, error) {
111	f.mu.Lock()	154	f.mu.Lock()
112	defer f.mu.Unlock()	155	defer f.mu.Unlock()
113	if f.reserveErr != nil {	156	return f.running[id]
114	err := f.reserveErr
115	f.reserveErr = nil // one-shot
116	return "", err
117	}
118	if f.reserved == nil {
119	f.reserved = map[string]string{}
120	}
121	if ip, ok := f.reserved[vmID]; ok {
122	return ip, nil // sticky
123	}
124	used := make([]string, 0, len(f.reserved))
125	for _, ip := range f.reserved {
126	used = append(used, ip)
127	}
128	ip, err := ipalloc.Alloc(f.cidr, used)
129	if err != nil {
130	return "", err
131	}
132	f.reserved[vmID] = ip
133	return ip, nil
134	}	157	}
135		158
136	type fixture struct {	159	type fixture struct {
137	eng *Engine	160	eng *Engine
138	prov *fakeProv	161	prov *fakeProv
139	net *fakeNet
140	st *state.Store	162	st *state.Store
141	now time.Time	163	now time.Time
142	boot string	164	boot string
@@ -146,11 +168,10 @@ func setup(t testing.T) fixture {
146	t.Helper()	168	t.Helper()
147	st, err := state.Open(t.TempDir())	169	st, err := state.Open(t.TempDir())
148	require.NoError(t, err)	170	require.NoError(t, err)
149	f := &fixture{st: st, prov: newFakeProv(), net: &fakeNet{cidr: "10.77.1.0/24"}, now: time.Unix(1_700_000_000, 0), boot: "boot-1"}	171	f := &fixture{st: st, prov: newFakeProv(), now: time.Unix(1_700_000_000, 0), boot: "boot-1"}
150	f.eng = &Engine{	172	f.eng = &Engine{
151	St: st,	173	St: st,
152	Prov: f.prov,	174	Prov: f.prov,
153	Net: f.net,
154	Images: func(ctx context.Context, url, sha string) (string, error) {	175	Images: func(ctx context.Context, url, sha string) (string, error) {
155	return "/cache/" + sha + ".raw", nil	176	return "/cache/" + sha + ".raw", nil
156	},	177	},
@@ -229,7 +250,7 @@ func TestCreateAllocatesIPPreparesAndBoots(t *testing.T) {
229	}	250	}
230		251
231	// TestIncompleteCreateWithDiskPresentIsRetried pins the completion-witness fix:	252	// TestIncompleteCreateWithDiskPresentIsRetried pins the completion-witness fix:
232	// a create that failed after PrepareDisk wrote disk.raw but before boot leaves a	253	// a create that failed after PrepareRootDisk wrote disk.raw but before boot leaves a
233	// record with an empty BootID AND a disk file on disk. The witness is BootID, not	254	// record with an empty BootID AND a disk file on disk. The witness is BootID, not
234	// disk presence — so the next tick must re-run create() (rebuilding disk + seed),	255	// disk presence — so the next tick must re-run create() (rebuilding disk + seed),
235	// NOT divert to converge() (which rebuilds neither and would strand the VM).	256	// NOT divert to converge() (which rebuilds neither and would strand the VM).
@@ -252,7 +273,7 @@ func TestIncompleteCreateWithDiskPresentIsRetried(t *testing.T) {
252		273
253	rep := f.step(snap(1, vm(id)))	274	rep := f.step(snap(1, vm(id)))
254		275
255	assert.Equal(t, []string{id}, f.prov.prepared, "must re-run create (PrepareDisk), not converge")	276	assert.Equal(t, []string{id}, f.prov.prepared, "must re-run create (PrepareRootDisk), not converge")
256	assert.Equal(t, []string{id}, f.prov.booted)	277	assert.Equal(t, []string{id}, f.prov.booted)
257	av := findVM(rep, id)	278	av := findVM(rep, id)
258	require.NotNil(t, av)	279	require.NotNil(t, av)
@@ -279,7 +300,7 @@ func TestCreateSurvivesRecordLoadFailure(t *testing.T) {
279		300
280	// TestUnreadableRecordDoesNotRecreateTheVM pins the consequence of Store.Get	301	// TestUnreadableRecordDoesNotRecreateTheVM pins the consequence of Store.Get
281	// telling an unreadable record apart from an absent one. A live VM whose record	302	// telling an unreadable record apart from an absent one. A live VM whose record
282	// cannot be read must NOT be re-created: create() runs PrepareDisk over the disk	303	// cannot be read must NOT be re-created: create() runs PrepareRootDisk over the disk
283	// its guest is running from and boots a second cloud-hypervisor over the first,	304	// its guest is running from and boots a second cloud-hypervisor over the first,
284	// and it is the only path here that corrupts state instead of retrying. The pass	305	// and it is the only path here that corrupts state instead of retrying. The pass
285	// is skipped instead, and the VM keeps its last-known row until the store	306	// is skipped instead, and the VM keeps its last-known row until the store
@@ -302,15 +323,15 @@ func TestUnreadableRecordDoesNotRecreateTheVM(t *testing.T) {
302	}	323	}
303		324
304	// TestSameTickSiblingCountsFailedCreateAgainstQuota pins that a VM whose create	325	// TestSameTickSiblingCountsFailedCreateAgainstQuota pins that a VM whose create
305	// fails early (here: IP allocation) still has its Spec committed to the	326	// fails (here: at boot) still has its Spec committed to the
306	// admission ledger, so a SAME-TICK sibling's quota check counts it. Both VMs are	327	// admission ledger, so a SAME-TICK sibling's quota check counts it. Both VMs are
307	// identical (2 vCPU) under a 3-vCPU cap, so whichever is processed first fails IP	328	// identical (2 vCPU) under a 3-vCPU cap, so whichever is processed first fails to
308	// allocation and the other must be quota-blocked — NEITHER boots. Before the fix	329	// boot and the other must be quota-blocked — NEITHER boots. Before the fix
309	// the failed VM was invisible to the sibling, which wrongly booted.	330	// the failed VM was invisible to the sibling, which wrongly booted.
310	func TestSameTickSiblingCountsFailedCreateAgainstQuota(t *testing.T) {	331	func TestSameTickSiblingCountsFailedCreateAgainstQuota(t *testing.T) {
311	f := setup(t)	332	f := setup(t)
312	f.eng.MaxVCPUs = 3	333	f.eng.MaxVCPUs = 3
313	f.net.reserveErr = assert.AnError	334	f.prov.bootErr = assert.AnError
314	twoVCPU := func(v *pb.VMDesired) { v.Vcpus = 2 }	335	twoVCPU := func(v *pb.VMDesired) { v.Vcpus = 2 }
315		336
316	rep := f.step(snap(1, vm("vm1", twoVCPU), vm("vm2", twoVCPU)))	337	rep := f.step(snap(1, vm("vm1", twoVCPU), vm("vm2", twoVCPU)))
@@ -330,7 +351,7 @@ func TestFenceRefusesLowerEpochWithoutActing(t *testing.T) {
330	f.step(snap(5, vm("vm1")))	351	f.step(snap(5, vm("vm1")))
331	rep := f.step(snap(3)) // restore signature: vm1 missing, lower epoch	352	rep := f.step(snap(3)) // restore signature: vm1 missing, lower epoch
332	assert.True(t, rep.FenceViolation)	353	assert.True(t, rep.FenceViolation)
333	assert.Empty(t, f.prov.killed, "fenced snapshot must trigger no destroys")	354	assert.Empty(t, f.prov.destroyed, "fenced snapshot must trigger no destroys")
334	assert.Empty(t, f.prov.shutdown)	355	assert.Empty(t, f.prov.shutdown)
335	assert.Equal(t, uint64(5), rep.LastSeenEpoch)	356	assert.Equal(t, uint64(5), rep.LastSeenEpoch)
336	}	357	}
@@ -404,12 +425,12 @@ func TestTombstoneQuarantinesThenDestroysAfterGrace(t *testing.T) {
404		425
405	f.now = f.now.Add(6 * time.Minute) // past TombstoneGrace	426	f.now = f.now.Add(6 * time.Minute) // past TombstoneGrace
406	rep = f.step(snap(2, tombstoned(vm("vm1"))))	427	rep = f.step(snap(2, tombstoned(vm("vm1"))))
407	assert.Equal(t, []string{"vm1"}, f.prov.killed)	428	assert.Equal(t, []string{"vm1"}, f.prov.destroyed)
408	assert.Contains(t, rep.Destroyed, "vm1", "destroy ack after grace")	429	assert.Contains(t, rep.Destroyed, "vm1", "destroy ack after grace")
409	recs, _ := f.st.LoadVMs()	430	recs, _ := f.st.LoadVMs()
410	assert.NotContains(t, recs, "vm1")	431	assert.NotContains(t, recs, "vm1")
411	// Fix 6: tap must be cleaned up on destroy	432	// Destroy is the whole teardown: the guest AND its network resources.
412	assert.Contains(t, f.net.deleted, "vm1", "tap cleaned up on destroy")	433	assert.Empty(t, f.prov.Address("vm1"), "destroy releases the VM's address")
413	}	434	}
414		435
415	func TestVanishedWithoutTombstoneGetsLongGrace(t *testing.T) {	436	func TestVanishedWithoutTombstoneGetsLongGrace(t *testing.T) {
@@ -419,12 +440,12 @@ func TestVanishedWithoutTombstoneGetsLongGrace(t *testing.T) {
419	f.step(snap(2))	440	f.step(snap(2))
420	f.now = f.now.Add(30 * time.Minute)	441	f.now = f.now.Add(30 * time.Minute)
421	rep := f.step(snap(2))	442	rep := f.step(snap(2))
422	assert.Empty(t, f.prov.killed, "vanished VMs get the full VanishGrace (1h)")	443	assert.Empty(t, f.prov.destroyed, "vanished VMs get the full VanishGrace (1h)")
423	require.Len(t, rep.Quarantined, 1)	444	require.Len(t, rep.Quarantined, 1)
424		445
425	f.now = f.now.Add(31 * time.Minute)	446	f.now = f.now.Add(31 * time.Minute)
426	f.step(snap(2))	447	f.step(snap(2))
427	assert.Equal(t, []string{"vm1"}, f.prov.killed)	448	assert.Equal(t, []string{"vm1"}, f.prov.destroyed)
428	}	449	}
429		450
430	func TestDestroyedIsLevelTriggeredForUnknownTombstones(t *testing.T) {	451	func TestDestroyedIsLevelTriggeredForUnknownTombstones(t *testing.T) {
@@ -449,7 +470,7 @@ func TestUndeleteClearsQuarantineSoNextDeleteGetsFullGrace(t *testing.T) {
449	// much later, delete again: must get a FRESH grace window, not instant kill	470	// much later, delete again: must get a FRESH grace window, not instant kill
450	f.now = f.now.Add(24 * time.Hour)	471	f.now = f.now.Add(24 * time.Hour)
451	rep := f.step(snap(4, tombstoned(vm("vm1"))))	472	rep := f.step(snap(4, tombstoned(vm("vm1"))))
452	assert.Empty(t, f.prov.killed, "fresh quarantine window required after un-delete")	473	assert.Empty(t, f.prov.destroyed, "fresh quarantine window required after un-delete")
453	require.Len(t, rep.Quarantined, 1)	474	require.Len(t, rep.Quarantined, 1)
454	recs, _ := f.st.LoadVMs()	475	recs, _ := f.st.LoadVMs()
455	require.Contains(t, recs, "vm1")	476	require.Contains(t, recs, "vm1")
@@ -608,7 +629,7 @@ func TestPermanentCreateErrorFailsTerminallyInOneAttempt(t *testing.T) {
608	assert.Equal(t, "failed", row.Phase, "permanent error must be terminal on attempt 1")	629	assert.Equal(t, "failed", row.Phase, "permanent error must be terminal on attempt 1")
609	assert.Contains(t, row.LastError, "smaller than base image")	630	assert.Contains(t, row.LastError, "smaller than base image")
610		631
611	require.Equal(t, 1, f.prov.prepCalls, "exactly one PrepareDisk invocation")	632	require.Equal(t, 1, f.prov.prepCalls, "exactly one PrepareRootDisk invocation")
612	rep = f.step(snap(1, vm("vm1")))	633	rep = f.step(snap(1, vm("vm1")))
613	row = findVM(rep, "vm1")	634	row = findVM(rep, "vm1")
614	require.NotNil(t, row)	635	require.NotNil(t, row)
@@ -617,9 +638,9 @@ func TestPermanentCreateErrorFailsTerminallyInOneAttempt(t *testing.T) {
617	}	638	}
618		639
619	// TestTwoVMsCreatedInOneStepGetDistinctIPs pins that when a single Step creates	640	// TestTwoVMsCreatedInOneStepGetDistinctIPs pins that when a single Step creates
620	// two VMs, each gets a distinct address. Admission is serialized, so the second	641	// two VMs, each gets a distinct address. The backend owns the used-address set
621	// create's ReserveIP allocates over the reservation table the first just wrote	642	// under its own lock (netenv's DHCP table does; the fake mirrors it), so two
622	// and picks a different address. The reconcile loop's converge order is	643	// concurrent creates cannot collide. The reconcile loop's converge order is
623	// randomized, so this must hold regardless of which VM is created first.	644	// randomized, so this must hold regardless of which VM is created first.
624	func TestTwoVMsCreatedInOneStepGetDistinctIPs(t *testing.T) {	645	func TestTwoVMsCreatedInOneStepGetDistinctIPs(t *testing.T) {
625	f := setup(t)	646	f := setup(t)
@@ -677,3 +698,100 @@ func TestTransientCreateErrorStillRetries(t *testing.T) {
677	assert.Equal(t, "failed", row.Phase, "budget spent -> terminal failed")	698	assert.Equal(t, "failed", row.Phase, "budget spent -> terminal failed")
678	assert.Equal(t, f.eng.MaxCreateAttempts, f.prov.prepCalls, "one invocation per budgeted attempt")	699	assert.Equal(t, f.eng.MaxCreateAttempts, f.prov.prepCalls, "one invocation per budgeted attempt")
679	}	700	}
		701
		702	// TestAddressDiscoveredAfterBootIsPersisted pins the reason per-VM networking
		703	// lives behind the Provisioner seam at all: on a backend whose host OS hands out
		704	// the address, it does not exist when Boot returns. The VM is healthy without
		705	// one, and the address arrives on a later converge poll — no boot, no state
		706	// change, nothing else for that tick to do but notice it.
		707	func TestAddressDiscoveredAfterBootIsPersisted(t *testing.T) {
		708	f := setup(t)
		709	f.prov.lateAddress = true
		710
		711	rep := f.step(snap(1, vm("vm1")))
		712	av := findVM(rep, "vm1")
		713	require.NotNil(t, av)
		714	require.Equal(t, []string{"vm1"}, f.prov.booted)
		715	assert.Empty(t, av.Ip, "the guest has not asked for an address yet")
		716	assert.Equal(t, "running", av.Power, "no address is not a failure")
		717	assert.Equal(t, "ready", av.Phase)
		718	recs, _ := f.st.LoadVMs()
		719	require.Contains(t, recs, "vm1")
		720	assert.Empty(t, recs["vm1"].IP)
		721
		722	f.prov.answerAddress("vm1") // guest booted and DHCP'd
		723
		724	rep = f.step(snap(2, vm("vm1")))
		725	av = findVM(rep, "vm1")
		726	require.NotNil(t, av)
		727	assert.Equal(t, "10.77.1.2", av.Ip, "a converge poll must pick the address up")
		728	recs, _ = f.st.LoadVMs()
		729	assert.Equal(t, "10.77.1.2", recs["vm1"].IP, "and persist it, not just report it")
		730	}
		731
		732	// TestEmptyAddressAnswerKeepsTheKnownOne pins noteAddress's empty guard: "I have
		733	// no answer" is not "it has no address". Blanking rec.IP would cut the guest SSH
		734	// tunnel to a VM that is reachable and running.
		735	func TestEmptyAddressAnswerKeepsTheKnownOne(t *testing.T) {
		736	f := setup(t)
		737	f.step(snap(1, vm("vm1")))
		738	recs, _ := f.st.LoadVMs()
		739	require.Equal(t, "10.77.1.2", recs["vm1"].IP, "precondition: address is known")
		740
		741	f.prov.forgetAddresses() // backend can no longer answer for a live VM
		742
		743	rep := f.step(snap(2, vm("vm1")))
		744	av := findVM(rep, "vm1")
		745	require.NotNil(t, av)
		746	assert.Equal(t, "10.77.1.2", av.Ip, "an empty answer must not clear a known address")
		747	recs, _ = f.st.LoadVMs()
		748	assert.Equal(t, "10.77.1.2", recs["vm1"].IP)
		749	}
		750
		751	// TestRestartRebootsAPersistentVMAndRepublishesItsAddress pins that a host
		752	// reboot puts a persistent VM back on its feet: reconcile boots it again, and
		753	// whatever address the backend then reports is the one that reaches both the
		754	// report and the durable record.
		755	func TestRestartRebootsAPersistentVMAndRepublishesItsAddress(t *testing.T) {
		756	f := setup(t)
		757	f.step(snap(1, vm("vm1", persistent)))
		758	recs, _ := f.st.LoadVMs()
		759	require.Equal(t, "10.77.1.2", recs["vm1"].IP)
		760
		761	f.boot = "boot-2" // host rebooted; the agent and its backend restarted with it
		762	f.prov.running["vm1"] = false
		763	f.prov.booted = nil
		764
		765	rep := f.step(snap(2, vm("vm1", persistent)))
		766	require.Equal(t, []string{"vm1"}, f.prov.booted)
		767	assert.Equal(t, "10.77.1.2", f.prov.Address("vm1"))
		768	assert.Equal(t, "10.77.1.2", findVM(rep, "vm1").GetIp())
		769	recs, _ = f.st.LoadVMs()
		770	assert.Equal(t, "10.77.1.2", recs["vm1"].IP)
		771	}
		772
		773	// TestFailedDestroyKeepsTheRecordForRetry pins the anti-orphan invariant: only
		774	// Destroy returning nil promises the backend has released the VM's process and
		775	// its address, and only then may the record go. Deleting it on a failure would
		776	// leave host resources with nothing left to reap them by, so the record stays
		777	// and the level-triggered loop retries.
		778	func TestFailedDestroyKeepsTheRecordForRetry(t *testing.T) {
		779	f := setup(t)
		780	f.step(snap(1, vm("vm1")))
		781	f.step(snap(2, tombstoned(vm("vm1"))))
		782	f.prov.destroyErr = assert.AnError
		783
		784	f.now = f.now.Add(6 * time.Minute) // past TombstoneGrace
		785	rep := f.step(snap(2, tombstoned(vm("vm1"))))
		786	require.Equal(t, []string{"vm1"}, f.prov.destroyed, "precondition: destroy was attempted")
		787	recs, _ := f.st.LoadVMs()
		788	assert.Contains(t, recs, "vm1", "a failed destroy must keep the record")
		789	assert.NotContains(t, rep.Destroyed, "vm1", "nothing to ack while the backend still holds it")
		790
		791	f.prov.destroyErr = nil
		792	rep = f.step(snap(2, tombstoned(vm("vm1"))))
		793	assert.Equal(t, []string{"vm1", "vm1"}, f.prov.destroyed, "the next tick retries the destroy")
		794	recs, _ = f.st.LoadVMs()
		795	assert.NotContains(t, recs, "vm1")
		796	assert.Contains(t, rep.Destroyed, "vm1")
		797	}

internal/agent/reconcile/worker.go

Old		New
@@ -18,9 +18,9 @@ import (
18	// out, never across a reconcile pass, so deliver and collect never wait on	18	// out, never across a reconcile pass, so deliver and collect never wait on
19	// slow work — that is what protects the heartbeat;	19	// slow work — that is what protects the heartbeat;
20	// - anything shared BETWEEN VMs is guarded where it lives: the admission	20	// - anything shared BETWEEN VMs is guarded where it lives: the admission
21	// ledger under Engine.mu (see admit), the DHCP reservation table under the	21	// ledger under Engine.mu (see admit), whatever the host backend shares
22	// dhcp server's own lock, and the state store by one file per VM written	22	// between VMs, under the backend's own locking, and the state store by one
23	// via temp-file rename.	23	// file per VM written via temp-file rename.
24	type manager struct {	24	type manager struct {
25	eng *Engine	25	eng *Engine
26		26
@@ -77,12 +77,12 @@ func (m *manager) deliver(id string, a assignment) {
77	// pass owns that VM, and dropping it from the map would let the next deliver for	77	// pass owns that VM, and dropping it from the map would let the next deliver for
78	// the same id spawn a SECOND worker — two goroutines reconciling one VM at once,	78	// the same id spawn a SECOND worker — two goroutines reconciling one VM at once,
79	// which is exactly the invariant this layer exists to provide. A concurrent	79	// which is exactly the invariant this layer exists to provide. A concurrent
80	// Kill/DeleteTap/DeleteVM against a Boot/CreateTap/SaveVM can orphan a	80	// Destroy/DeleteVM against a Boot/SaveVM can orphan a hypervisor process with
81	// cloud-hypervisor process with no record left to find it by, and nothing	81	// no record left to find it by, and nothing self-heals from that. Reaping is
82	// self-heals from that. Reaping is level-triggered, so deferring a busy worker	82	// level-triggered, so deferring a busy worker to a later tick costs nothing.
83	// to a later tick costs nothing. This is reachable in ordinary operation: a	83	// This is reachable in ordinary operation: a transiently unreadable record makes
84	// transiently unreadable record makes LoadVMs skip a live VM (it continues past	84	// LoadVMs skip a live VM (it continues past read errors), dropping it out of
85	// read errors), dropping it out of live for one tick.	85	// live for one tick.
86	func (m *manager) reapAbsent(live map[string]assignment) {	86	func (m *manager) reapAbsent(live map[string]assignment) {
87	m.mu.Lock()	87	m.mu.Lock()
88	defer m.mu.Unlock()	88	defer m.mu.Unlock()

internal/agent/run/cli.go

Old		New
@@ -15,25 +15,34 @@ import (
15	"fmt"	15	"fmt"
16	"log/slog"	16	"log/slog"
17	"os"	17	"os"
		18	"os/exec"
18	"os/signal"	19	"os/signal"
19	"runtime"	20	"runtime"
20	"syscall"	21	"syscall"
21	"time"	22	"time"
22		23
23	"github.com/a73x/eitri/internal/agent/bootstrap"
24	"github.com/a73x/eitri/internal/agent/cloudhv"
25	"github.com/a73x/eitri/internal/agent/enrollclient"	24	"github.com/a73x/eitri/internal/agent/enrollclient"
		25	"github.com/a73x/eitri/internal/agent/hostinfo"
26	"github.com/a73x/eitri/internal/agent/imagecache"	26	"github.com/a73x/eitri/internal/agent/imagecache"
27	"github.com/a73x/eitri/internal/agent/netenv"
28	"github.com/a73x/eitri/internal/agent/reconcile"	27	"github.com/a73x/eitri/internal/agent/reconcile"
29	"github.com/a73x/eitri/internal/agent/seed"	28	"github.com/a73x/eitri/internal/agent/seed"
30	"github.com/a73x/eitri/internal/agent/serialpump"
31	"github.com/a73x/eitri/internal/agent/state"	29	"github.com/a73x/eitri/internal/agent/state"
32	"github.com/a73x/eitri/internal/agent/syncclient"	30	"github.com/a73x/eitri/internal/agent/syncclient"
33	"github.com/a73x/eitri/internal/covsnap"	31	"github.com/a73x/eitri/internal/covsnap"
34	"github.com/a73x/eitri/internal/joinblob"	32	"github.com/a73x/eitri/internal/joinblob"
35	)	33	)
36		34
		35	// hostRunner is the production one-shot command runner injected into the
		36	// host-touching agent packages (netenv, imagecache, cloudhv, syncclient). It
		37	// spawns name+args, waits, and returns their combined stdout/stderr. It lives
		38	// in the composition root because it is a wiring value: constructing the
		39	// concrete dependency is what a root is for, and keeping it here means no
		40	// platform's wiring has to reach into another platform's provisioner for it.
		41	func hostRunner(ctx context.Context, name string, args ...string) (string, error) {
		42	out, err := exec.CommandContext(ctx, name, args...).CombinedOutput()
		43	return string(out), err
		44	}
		45
37	// Config carries serve's wiring, replacing a long positional list.	46	// Config carries serve's wiring, replacing a long positional list.
38	type Config struct {	47	type Config struct {
39	StateDir, CHBin, Firmware string	48	StateDir, CHBin, Firmware string
@@ -190,53 +199,22 @@ func serve(st *state.Store, cfg Config) error {
190	// from the live agent without bouncing the process.	199	// from the live agent without bouncing the process.
191	covsnap.Install(ctx)	200	covsnap.Install(ctx)
192		201
193	net, err := netenv.New(cloudhv.RealRunner, id.BridgeCIDR)	202	plat, err := newPlatform(ctx, cfg, st, id.BridgeCIDR)
194	if err != nil {	203	if err != nil {
195	return fmt.Errorf("netenv init: %w", err)	204	return err
196	}
197
198	if err := net.EnsureBridge(ctx); err != nil {
199	return fmt.Errorf("ensure bridge: %w", err)
200	}
201
202	// Bootstrap cloud-hypervisor and its UEFI firmware before anything tries
203	// to launch a VM: a bare host that just joined has neither, and the agent
204	// is useless without at least the hypervisor binary. BootstrapDest maps
205	// the --ch-bin value (usually a bare $PATH name) to a real install path.
206	bs := &bootstrap.Bootstrapper{CHPath: cloudhv.BootstrapDest(cfg.CHBin), FirmwarePath: cfg.Firmware, ManifestURL: cfg.BootstrapURL}
207	if err := bs.Ensure(ctx); err != nil {
208	return fmt.Errorf("bootstrap runtime: %w", err)
209	}	205	}
210		206	prov, pumps := plat.Prov, plat.Pumps
211	prov := cloudhv.New(st, cfg.CHBin, cfg.Firmware, cloudhv.RealRunner)
212
213	// Serial console pumps: one per running VM, started at Boot (cloudhv hook)
214	// and reattached here for VMs that survived an agent restart (CH runs in
215	// its own process group; the pump reconnects to the still-listening
216	// serial socket).
217	pumps := serialpump.NewManager(st.SerialSocketPath, st.SerialLogPath)
218	prov.Pumps = pumps
219	if recs, err := st.LoadVMs(); err == nil {	207	if recs, err := st.LoadVMs(); err == nil {
220	for _, rec := range recs {	208	for _, rec := range recs {
221	if rec.IP != "" {
222	net.AddReservation(rec.Spec.VMID, rec.IP)
223	}
224	if prov.Running(rec.Spec.VMID) {	209	if prov.Running(rec.Spec.VMID) {
225	pumps.Ensure(rec.Spec.VMID)	210	pumps.Ensure(rec.Spec.VMID)
226	}	211	}
227	}	212	}
228	} else {	213	} else {
229	slog.Warn("state load failed; surviving VMs' consoles will be silent AND their DHCP reservations are not rebuilt (they may fail to renew until reconcile recreates them)", "err", err)	214	slog.Warn("state load failed; surviving VMs' consoles will be silent until reconcile restarts them", "err", err)
230	}
231
232	// Start the embedded DHCP responder AFTER reservations are rebuilt from
233	// state, so a surviving guest's renewal is never answered from an empty
234	// table (fail-closed + reservation preload close the gap).
235	if err := net.StartDHCP(ctx); err != nil {
236	return fmt.Errorf("start dhcp: %w", err)
237	}	215	}
238		216
239	cache := imagecache.New(st.ImagesDir(), cloudhv.RealRunner)	217	cache := imagecache.New(st.ImagesDir(), hostRunner)
240	// Clamp before shifting: GB<<30 overflows int64 for absurd flag values —	218	// Clamp before shifting: GB<<30 overflows int64 for absurd flag values —
241	// same trap class cloudhv's maxDiskGB comment documents. 1 PiB is beyond	219	// same trap class cloudhv's maxDiskGB comment documents. 1 PiB is beyond
242	// any real cache; anything above disables eviction just like 0 would.	220	// any real cache; anything above disables eviction just like 0 would.
@@ -254,10 +232,9 @@ func serve(st *state.Store, cfg Config) error {
254	engine := &reconcile.Engine{	232	engine := &reconcile.Engine{
255	St: st,	233	St: st,
256	Prov: prov,	234	Prov: prov,
257	Net: net,
258	Images: cache.Ensure,	235	Images: cache.Ensure,
259	Seed: seed.Build,	236	Seed: seed.Build,
260	BootID: syncclient.HostBootID,	237	BootID: hostinfo.BootID,
261	Now: time.Now,	238	Now: time.Now,
262	TombstoneGrace: cfg.TombstoneGrace,	239	TombstoneGrace: cfg.TombstoneGrace,
263	VanishGrace: cfg.VanishGrace,	240	VanishGrace: cfg.VanishGrace,
@@ -288,7 +265,7 @@ func serve(st *state.Store, cfg Config) error {
288	St: st,	265	St: st,
289	Identity: id,	266	Identity: id,
290	StateDir: cfg.StateDir,	267	StateDir: cfg.StateDir,
291	Runner: cloudhv.RealRunner,	268	Runner: hostRunner,
292	Console: pumps,	269	Console: pumps,
293	MaxVCPUs: cfg.MaxVCPUs,	270	MaxVCPUs: cfg.MaxVCPUs,
294	MaxMemMB: cfg.MaxMemMB,	271	MaxMemMB: cfg.MaxMemMB,

internal/agent/run/cli_test.go

Old		New
@@ -1,6 +1,7 @@
1	package run	1	package run
2		2
3	import (	3	import (
		4	"context"
4	"strings"	5	"strings"
5	"testing"	6	"testing"
6	"time"	7	"time"
@@ -10,6 +11,26 @@ import (
10	"github.com/stretchr/testify/require"	11	"github.com/stretchr/testify/require"
11	)	12	)
12		13
		14	// TestHostRunnerReturnsCombinedOutput pins that stdout and stderr both land in
		15	// hostRunner's single return string: injected consumers (netenv, imagecache,
		16	// cloudhv, syncclient) diagnose failures from that one string, so losing
		17	// either stream would blind them.
		18	func TestHostRunnerReturnsCombinedOutput(t *testing.T) {
		19	out, err := hostRunner(context.Background(), "sh", "-c", "echo out; echo err 1>&2")
		20	require.NoError(t, err)
		21	assert.Contains(t, out, "out")
		22	assert.Contains(t, out, "err")
		23	}
		24
		25	// TestHostRunnerReturnsErrorAndOutputOnFailure pins that a non-zero exit
		26	// surfaces both a non-nil error AND the output collected up to that point —
		27	// callers need the output alongside the error to diagnose what went wrong.
		28	func TestHostRunnerReturnsErrorAndOutputOnFailure(t *testing.T) {
		29	out, err := hostRunner(context.Background(), "sh", "-c", "echo boom 1>&2; exit 3")
		30	require.Error(t, err)
		31	assert.Contains(t, out, "boom")
		32	}
		33
13	// TestParseConfigDefaults pins every flag default: this binary runs the fleet,	34	// TestParseConfigDefaults pins every flag default: this binary runs the fleet,
14	// so a silent change to any default is a production behavior change.	35	// so a silent change to any default is a production behavior change.
15	func TestParseConfigDefaults(t *testing.T) {	36	func TestParseConfigDefaults(t *testing.T) {

internal/agent/run/reservations.go

@@ @@ -0,0 +1,37 @@ @@
+package run
+import (
+	"log/slog"
+	"github.com/a73x/eitri/internal/agent/state"
+)
+// reservations is the slice of a host's networking the startup replay needs:
+// pinning one guest's address. Consumer-owned and minimal (arch R5) so a
+// platform whose networking is nothing like a bridge and a DHCP responder can
+// still be handed the surviving guests' addresses.
+type reservations interface {
+	AddReservation(vmID, ip string)
+}
+// replayReservations rebuilds net's reservation table from durable records. A
+// platform's wiring must call it BEFORE its DHCP responder starts, so a
+// surviving guest's renewal is never answered from an empty table (fail-closed
+// plus this preload close the gap). It is also the only thing that carries an
+// address across an agent restart: the table is in memory, so a guest whose
+// reservation is not replayed is renumbered at its next boot. A record carrying
+// no address has nothing to pin. A failed load warns and returns rather than
+// aborting startup — a host that renumbers its guests still beats one that
+// refuses to come up.
+func replayReservations(st *state.Store, net reservations) {
+	recs, err := st.LoadVMs()
+	if err != nil {
+		slog.Warn("state load failed; surviving guests' DHCP reservations are not rebuilt (they may fail to renew until reconcile reboots them)", "err", err)
+		return
+	}
+	for _, rec := range recs {
+		if rec.IP != "" {
+			net.AddReservation(rec.Spec.VMID, rec.IP)
+		}
+	}
+}

internal/agent/run/reservations_test.go

@@ @@ -0,0 +1,94 @@ @@
+package run
+import (
+	"bytes"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"testing"
+	"github.com/a73x/eitri/internal/agent/state"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+// fakeReservations records what the replay pinned. The real table lives behind
+// a DHCP responder on a host bridge; the replay only has to decide what to pin.
+type fakeReservations struct{ pinned map[string]string }
+func newFakeReservations() *fakeReservations {
+	return &fakeReservations{pinned: map[string]string{}}
+}
+func (f *fakeReservations) AddReservation(vmID, ip string) { f.pinned[vmID] = ip }
+// storeWithRecords opens a store in a temp dir and persists recs, returning the
+// store and its directory.
+func storeWithRecords(t *testing.T, recs ...state.Record) (*state.Store, string) {
+	t.Helper()
+	dir := t.TempDir()
+	st, err := state.Open(dir)
+	require.NoError(t, err)
+	for _, rec := range recs {
+		require.NoError(t, st.SaveVM(rec))
+	}
+	return st, dir
+}
+// captureLogs redirects the default logger into a buffer for the duration of
+// the test — replayReservations reports a state-load failure the only way a
+// caller-less side effect can, by logging it.
+func captureLogs(t *testing.T) *bytes.Buffer {
+	t.Helper()
+	var logs bytes.Buffer
+	prev := slog.Default()
+	slog.SetDefault(slog.New(slog.NewTextHandler(&logs, nil)))
+	t.Cleanup(func() { slog.SetDefault(prev) })
+	return &logs
+}
+// TestReplayReservationsPinsEveryRecordedAddress is the whole point of the
+// replay: a guest that survived the agent restart keeps its address, so the
+// responder cannot hand it to a VM created afterwards.
+func TestReplayReservationsPinsEveryRecordedAddress(t *testing.T) {
+	st, _ := storeWithRecords(t,
+		state.Record{Spec: state.VMSpec{VMID: "vm-a"}, IP: "10.77.1.5"},
+		state.Record{Spec: state.VMSpec{VMID: "vm-b"}, IP: "10.77.1.6"},
+	)
+	table := newFakeReservations()
+	replayReservations(st, table)
+	assert.Equal(t, map[string]string{"vm-a": "10.77.1.5", "vm-b": "10.77.1.6"}, table.pinned)
+}
+// TestReplayReservationsSkipsRecordWithoutAddress pins that a record with no
+// recorded address contributes nothing: there is no address to protect, and
+// pinning an empty one would claim a reservation the guest does not hold.
+func TestReplayReservationsSkipsRecordWithoutAddress(t *testing.T) {
+	st, _ := storeWithRecords(t,
+		state.Record{Spec: state.VMSpec{VMID: "vm-addressed"}, IP: "10.77.1.7"},
+		state.Record{Spec: state.VMSpec{VMID: "vm-unaddressed"}},
+	)
+	table := newFakeReservations()
+	replayReservations(st, table)
+	assert.Equal(t, map[string]string{"vm-addressed": "10.77.1.7"}, table.pinned)
+}
+// TestReplayReservationsSurvivesStateLoadFailure pins the fail-closed
+// behaviour: an unreadable state directory warns and returns, leaving the
+// table empty rather than aborting the agent's startup.
+func TestReplayReservationsSurvivesStateLoadFailure(t *testing.T) {
+	st, dir := storeWithRecords(t, state.Record{Spec: state.VMSpec{VMID: "vm-a"}, IP: "10.77.1.5"})
+	require.NoError(t, os.RemoveAll(filepath.Join(dir, "vms")))
+	logs := captureLogs(t)
+	table := newFakeReservations()
+	replayReservations(st, table)
+	assert.Empty(t, table.pinned)
+	assert.Contains(t, logs.String(), "level=WARN")
+	assert.Contains(t, logs.String(), "reservations are not rebuilt")
+}

internal/agent/run/wire_linux.go

@@ @@ -0,0 +1,79 @@ @@
+//go:build linux
+package run
+import (
+	"context"
+	"fmt"
+	"github.com/a73x/eitri/internal/agent/bootstrap"
+	"github.com/a73x/eitri/internal/agent/cloudhv"
+	"github.com/a73x/eitri/internal/agent/netenv"
+	"github.com/a73x/eitri/internal/agent/reconcile"
+	"github.com/a73x/eitri/internal/agent/serialpump"
+	"github.com/a73x/eitri/internal/agent/state"
+)
+// platform is this host's backend pair. It is a struct, not an interface:
+// nothing consumes the bundle as a type, so the two seams stay independent — a
+// different VMM is a change in this file alone. serve() is the contract's only
+// consumer: a sibling wire_*.go must produce this same struct shape from a
+// newPlatform with this same signature, and the fields must satisfy what
+// serve() actually calls on them — Prov is reconcile.Provisioner; Pumps needs
+// Ensure, StopAll, and the console Attach the sync client uses to service
+// server-opened streams.
+//
+// Host networking does not appear here. It is per-VM state the provisioner
+// attaches inside Boot, and host-wide setup that this file performs once at
+// startup — neither is anything the platform-neutral serve() can sequence.
+//
+// Pumps is the console seam already wired to its lifecycle: the platform knows
+// both how its console is exposed and which provisioner drives the pump, so it
+// hands back a Manager rather than a raw source. One pump runs per running VM,
+// started at Boot (cloudhv hook); CH runs in its own process group, so a pump
+// surviving an agent restart reconnects to the still-listening serial socket
+// rather than needing to be relaunched.
+type platform struct {
+	Prov  reconcile.Provisioner
+	Pumps *serialpump.Manager
+}
+// newPlatform builds the Linux backend: cloud-hypervisor over a bridge and tap,
+// with the guest console on cloud-hypervisor's serial socket. bridgeCIDR is the
+// server-assigned CIDR from the enrolled identity (state.Store.Identity), not a
+// Config flag; a second platform's newPlatform must source it the same way. A
+// bare host that just joined has neither the hypervisor nor its UEFI firmware,
+// so newPlatform installs them before anything can try to launch a VM.
+func newPlatform(ctx context.Context, cfg Config, st *state.Store, bridgeCIDR string) (platform, error) {
+	net, err := netenv.New(hostRunner, bridgeCIDR)
+	if err != nil {
+		return platform{}, fmt.Errorf("netenv init: %w", err)
+	}
+	if err := net.EnsureBridge(ctx); err != nil {
+		return platform{}, fmt.Errorf("ensure bridge: %w", err)
+	}
+	// Must precede StartDHCP: the responder may not answer a surviving guest's
+	// renewal from an empty table.
+	replayReservations(st, net)
+	if err := net.StartDHCP(ctx); err != nil {
+		return platform{}, fmt.Errorf("start dhcp: %w", err)
+	}
+	// BootstrapDest maps the --ch-bin value (usually a bare $PATH name) to a
+	// real install path.
+	bs := &bootstrap.Bootstrapper{
+		CHPath:       cloudhv.BootstrapDest(cfg.CHBin),
+		FirmwarePath: cfg.Firmware,
+		ManifestURL:  cfg.BootstrapURL,
+	}
+	if err := bs.Ensure(ctx); err != nil {
+		return platform{}, fmt.Errorf("bootstrap runtime: %w", err)
+	}
+	prov := cloudhv.New(st, cfg.CHBin, cfg.Firmware, hostRunner, net)
+	pumps := serialpump.NewManager(cloudhv.ConsoleSource(st.SerialSocketPath), st.SerialLogPath)
+	prov.Pumps = pumps
+	return platform{Prov: prov, Pumps: pumps}, nil
+}

internal/agent/serialpump/serialpump.go

Old		New
@@ -1,15 +1,16 @@
1	// Package serialpump owns the durability of VM serial consoles. With	1	// Package serialpump owns the durability of VM serial consoles. A backend's
2	// cloud-hypervisor's --serial socket= mode the serial line is a unix socket	2	// ConsoleSource opens the raw byte stream (cloud-hypervisor serves it on a
3	// serving ONE client at a time (a new connection kicks the old one), and	3	// unix socket, one client at a time — a new connection kicks the old one —
4	// depending on CH version, output written while no client is connected is	4	// and depending on CH version, output written while no client is connected is
5	// dropped (older) or buffered in a bounded replay ring (current). The pump is	5	// dropped (older) or buffered in a bounded replay ring (current)); the pump is
6	// that one client, always: a supervised goroutine per running VM dials the	6	// that one client, always: a supervised goroutine per running VM opens the
7	// socket, drains continuously into a bounded in-memory ring (backlog for new	7	// stream, drains continuously into a bounded in-memory ring (backlog for new
8	// viewers) and a capped on-disk serial.log (survives agent restart, unbounded	8	// viewers) and a capped on-disk serial.log (survives agent restart, unbounded
9	// history), fans live bytes out to attached console viewers, and forwards	9	// history), fans live bytes out to attached console viewers, and forwards
10	// viewer input back to the socket. A slow viewer is dropped, never allowed to	10	// viewer input back to the stream. A slow viewer is dropped, never allowed to
11	// stall the drain. Do NOT connect other clients (socat etc.) to the serial	11	// stall the drain. For cloud-hypervisor's socket shape specifically, do NOT
12	// socket — they would steal the line from the pump.	12	// connect other clients (socat etc.) — they would steal the line from the
		13	// pump.
13	package serialpump	14	package serialpump
14		15
15	import (	16	import (
@@ -18,39 +19,49 @@ import (
18	"fmt"	19	"fmt"
19	"io"	20	"io"
20	"log/slog"	21	"log/slog"
21	"net"
22	"os"	22	"os"
23	"sync"	23	"sync"
24	"time"	24	"time"
25	)	25	)
26		26
		27	// ConsoleSource opens the guest console for vmID. Implementations are
		28	// per-backend: cloud-hypervisor serves the serial line on a unix socket, vfkit
		29	// hands out a PTY. Open is called inside the pump's reconnect loop, so it must
		30	// be cheap and safely retryable, and it may fail while the VM is down. A nil
		31	// error means a usable, non-nil stream — the pump closes it unconditionally
		32	// when the drain ends.
		33	type ConsoleSource interface {
		34	Open(vmID string) (io.ReadWriteCloser, error)
		35	}
		36
27	const (	37	const (
28	defaultRingMax = 256 << 10 // 256 KiB — enough for a boot log	38	defaultRingMax = 256 << 10 // 256 KiB — enough for a boot log
29	defaultLogMax = 4 << 20 // 4 MiB on disk, then rotate once to .old	39	defaultLogMax = 4 << 20 // 4 MiB on disk, then rotate once to .old
30	viewerDepth = 64 // live-tail channel depth before a viewer is dropped	40	viewerDepth = 64 // live-tail channel depth before a viewer is dropped
31	)	41	)
32		42
33	// Manager runs one Pump per VM. Paths are injected so the package stays a leaf	43	// Manager runs one Pump per VM. The console source and log path are injected so
34	// (no dependency on the agent's state store).	44	// the package stays a leaf (no dependency on the agent's state store, and no
		45	// knowledge of how any backend exposes its console).
35	type Manager struct {	46	type Manager struct {
36	socketPath func(vmID string) string	47	src ConsoleSource
37	logPath func(vmID string) string	48	logPath func(vmID string) string
38	ringMax int	49	ringMax int
39	logMax int64	50	logMax int64
40		51
41	mu sync.Mutex	52	mu sync.Mutex
42	pumps map[string]*pump	53	pumps map[string]*pump
43	}	54	}
44		55
45	// NewManager returns a Manager resolving each VM's serial socket and on-disk	56	// NewManager returns a Manager opening each VM's console through src and
46	// log through the given path funcs.	57	// resolving its on-disk log through logPath.
47	func NewManager(socketPath, logPath func(vmID string) string) *Manager {	58	func NewManager(src ConsoleSource, logPath func(vmID string) string) *Manager {
48	return &Manager{	59	return &Manager{
49	socketPath: socketPath,	60	src: src,
50	logPath: logPath,	61	logPath: logPath,
51	ringMax: defaultRingMax,	62	ringMax: defaultRingMax,
52	logMax: defaultLogMax,	63	logMax: defaultLogMax,
53	pumps: map[string]*pump{},	64	pumps: map[string]*pump{},
54	}	65	}
55	}	66	}
56		67
@@ -71,7 +82,8 @@ func (m *Manager) Ensure(vmID string) {
71	return	82	return
72	}	83	}
73	p := &pump{	84	p := &pump{
74	socket: m.socketPath(vmID),	85	vmID: vmID,
		86	src: m.src,
75	logPath: m.logPath(vmID),	87	logPath: m.logPath(vmID),
76	ringMax: m.ringMax,	88	ringMax: m.ringMax,
77	logMax: m.logMax,	89	logMax: m.logMax,
@@ -128,17 +140,18 @@ func (m *Manager) Attach(ctx context.Context, vmID string, rw io.ReadWriter, onR
128	return p.attach(ctx, rw)	140	return p.attach(ctx, rw)
129	}	141	}
130		142
131	// pump drains one VM's serial socket. It reconnects forever (CH restarts on VM	143	// pump drains one VM's console. It reopens forever (the VMM restarts on VM
132	// stop/start; the socket may not exist yet at boot) until stop() is called.	144	// stop/start; the console may not exist yet at boot) until stop() is called.
133	type pump struct {	145	type pump struct {
134	socket string	146	vmID string
		147	src ConsoleSource
135	logPath string	148	logPath string
136	ringMax int	149	ringMax int
137	logMax int64	150	logMax int64
138		151
139	mu sync.Mutex	152	mu sync.Mutex
140	ring []byte	153	ring []byte
141	conn net.Conn // current socket conn; input writes go here	154	conn io.ReadWriteCloser // current console stream; input writes go here
142	viewers map[int]chan []byte	155	viewers map[int]chan []byte
143	nextID int	156	nextID int
144	logF *os.File	157	logF *os.File
@@ -177,7 +190,7 @@ func (p *pump) run() {
177	return	190	return
178	default:	191	default:
179	}	192	}
180	conn, err := net.Dial("unix", p.socket)	193	conn, err := p.src.Open(p.vmID)
181	if err != nil {	194	if err != nil {
182	select {	195	select {
183	case <-p.done:	196	case <-p.done:
@@ -198,7 +211,7 @@ func (p *pump) run() {
198	p.mu.Lock()	211	p.mu.Lock()
199	select {	212	select {
200	case <-p.done:	213	case <-p.done:
201	// stop() ran between Dial and here: it closed p.conn (nil at that	214	// stop() ran between Open and here: it closed p.conn (nil at that
202	// point) but not THIS conn — do not resurrect the pump.	215	// point) but not THIS conn — do not resurrect the pump.
203	p.mu.Unlock()	216	p.mu.Unlock()
204	conn.Close()	217	conn.Close()
@@ -217,7 +230,7 @@ func (p *pump) run() {
217	}	230	}
218	}	231	}
219		232
220	func (p *pump) drain(conn net.Conn) {	233	func (p *pump) drain(conn io.Reader) {
221	buf := make([]byte, 4096)	234	buf := make([]byte, 4096)
222	for {	235	for {
223	n, err := conn.Read(buf)	236	n, err := conn.Read(buf)
@@ -352,8 +365,8 @@ func (p *pump) subscribe() (backlog []byte, ch chan []byte, cancel func(), err e
352	}	365	}
353		366
354	// writeInput forwards viewer keystrokes to the guest. Dropped silently when	367	// writeInput forwards viewer keystrokes to the guest. Dropped silently when
355	// the socket is not currently connected (VM stopped): the serial line simply	368	// the console stream is not currently connected (VM stopped): the serial line
356	// isn't there, exactly like typing into an unplugged terminal.	369	// simply isn't there, exactly like typing into an unplugged terminal.
357	func (p *pump) writeInput(b []byte) {	370	func (p *pump) writeInput(b []byte) {
358	p.mu.Lock()	371	p.mu.Lock()
359	conn := p.conn	372	conn := p.conn

internal/agent/serialpump/serialpump_test.go

Old		New
@@ -6,6 +6,7 @@ import (
6	"net"	6	"net"
7	"os"	7	"os"
8	"path/filepath"	8	"path/filepath"
		9	"sync"
9	"testing"	10	"testing"
10	"time"	11	"time"
11		12
@@ -50,10 +51,18 @@ func (f fakeCH) conn(t testing.T) net.Conn {
50	}	51	}
51	}	52	}
52		53
		54	// testSource dials the unix socket path returned by socketPath, standing in
		55	// for a backend's ConsoleSource (e.g. cloudhv.ConsoleSource) in these tests.
		56	type testSource func(vmID string) string
		57
		58	func (s testSource) Open(vmID string) (io.ReadWriteCloser, error) {
		59	return net.Dial("unix", s(vmID))
		60	}
		61
53	func newTestManager(t testing.T, dir string) Manager {	62	func newTestManager(t testing.T, dir string) Manager {
54	t.Helper()	63	t.Helper()
55	m := NewManager(	64	m := NewManager(
56	func(vmID string) string { return filepath.Join(dir, vmID+".serial.sock") },	65	testSource(func(vmID string) string { return filepath.Join(dir, vmID+".serial.sock") }),
57	func(vmID string) string { return filepath.Join(dir, vmID+".serial.log") },	66	func(vmID string) string { return filepath.Join(dir, vmID+".serial.log") },
58	)	67	)
59	t.Cleanup(m.StopAll)	68	t.Cleanup(m.StopAll)
@@ -347,3 +356,52 @@ func TestStopUnknownVMIsNoop(t *testing.T) {
347	m := newTestManager(t, t.TempDir())	356	m := newTestManager(t, t.TempDir())
348	m.Stop("never-started") // must not panic	357	m.Stop("never-started") // must not panic
349	}	358	}
		359
		360	// fakeSource hands out an in-memory console stream and records how many
		361	// times Open was called, pinning that the pump reconnects through the seam.
		362	type fakeSource struct {
		363	mu sync.Mutex
		364	opens int
		365	conn io.ReadWriteCloser
		366	err error
		367	}
		368
		369	func (f *fakeSource) Open(vmID string) (io.ReadWriteCloser, error) {
		370	f.mu.Lock()
		371	defer f.mu.Unlock()
		372	f.opens++
		373	if f.err != nil {
		374	return nil, f.err
		375	}
		376	return f.conn, nil
		377	}
		378
		379	func (f *fakeSource) count() int {
		380	f.mu.Lock()
		381	defer f.mu.Unlock()
		382	return f.opens
		383	}
		384
		385	func TestManagerOpensConsoleThroughSource(t *testing.T) {
		386	guest, host := net.Pipe() // host end stands in for any ReadWriteCloser
		387	src := &fakeSource{conn: host}
		388	m := NewManager(src, func(vmID string) string {
		389	return filepath.Join(t.TempDir(), "serial.log")
		390	})
		391	defer m.StopAll()
		392
		393	m.Ensure("vm-1")
		394	go func() { _, _ = guest.Write([]byte("hello console")) }()
		395
		396	deadline := time.After(2 * time.Second)
		397	for {
		398	if src.count() > 0 {
		399	return
		400	}
		401	select {
		402	case <-deadline:
		403	t.Fatal("pump never called ConsoleSource.Open")
		404	case <-time.After(10 * time.Millisecond):
		405	}
		406	}
		407	}

internal/agent/state/state.go

Old		New
@@ -20,6 +20,14 @@ type VMSpec struct {
20	Persistent bool	20	Persistent bool
21	}	21	}
22		22
		23	// Disk is one block device attached to a VM, in attachment order. Index 0 is
		24	// the root disk (/dev/vda) — both cloud-hypervisor and vfkit order by argument
		25	// position and treat the first as root.
		26	type Disk struct {
		27	Path string
		28	ReadOnly bool
		29	}
		30
23	type Record struct {	31	type Record struct {
24	Spec VMSpec	32	Spec VMSpec
25	IP string	33	IP string
@@ -78,17 +86,6 @@ func (s *Store) SerialLogPath(vmID string) string {
78	return filepath.Join(s.VMDir(vmID), "serial.log")	86	return filepath.Join(s.VMDir(vmID), "serial.log")
79	}	87	}
80		88
81	// TapName returns the TAP device name for a given vmID. The name is truncated
82	// to the first 8 characters of the vmID, giving "eit-XXXXXXXX" (12 chars),
83	// which is safely below the 15-char IFNAMSIZ limit.
84	func TapName(vmID string) string {
85	prefix := vmID
86	if len(prefix) > 8 {
87	prefix = prefix[:8]
88	}
89	return "eit-" + prefix
90	}
91
92	// MAC returns a deterministic, locally-administered MAC address for vmID.	89	// MAC returns a deterministic, locally-administered MAC address for vmID.
93	// It uses the QEMU/KVM OUI prefix 52:54:00 and derives the last three octets	90	// It uses the QEMU/KVM OUI prefix 52:54:00 and derives the last three octets
94	// from SHA-256(vmID). Shared by the hypervisor backend (guest NIC address) and	91	// from SHA-256(vmID). Shared by the hypervisor backend (guest NIC address) and
@@ -166,7 +163,7 @@ func (s *Store) LoadVMs() (map[string]Record, error) {
166	// Callers MUST distinguish the two. "No record" means this host does not run	163	// Callers MUST distinguish the two. "No record" means this host does not run
167	// this VM, so the agent's reconcile loop answers it by creating the VM — which	164	// this VM, so the agent's reconcile loop answers it by creating the VM — which
168	// rebuilds disk.raw and boots cloud-hypervisor. Letting an unreadable record	165	// rebuilds disk.raw and boots cloud-hypervisor. Letting an unreadable record
169	// read as absent would put a LIVE VM down that path: PrepareDisk over the disk	166	// read as absent would put a LIVE VM down that path: PrepareRootDisk over the disk
170	// its guest is running from, and a second hypervisor whose pidfile orphans the	167	// its guest is running from, and a second hypervisor whose pidfile orphans the
171	// first. Anything that cannot be observed must be retried, never assumed empty.	168	// first. Anything that cannot be observed must be retried, never assumed empty.
172	//	169	//

internal/agent/syncclient/client.go

Old		New
@@ -13,7 +13,6 @@ import (
13	"strings"	13	"strings"
14	"sync"	14	"sync"
15	"sync/atomic"	15	"sync/atomic"
16	"syscall"
17	"time"	16	"time"
18		17
19	agentexec "github.com/a73x/eitri/internal/agent/exec"	18	agentexec "github.com/a73x/eitri/internal/agent/exec"
@@ -27,42 +26,9 @@ import (
27	"github.com/quic-go/quic-go"	26	"github.com/quic-go/quic-go"
28	)	27	)
29		28
30	// HostBootID reads /proc/sys/kernel/random/boot_id and returns the trimmed value.
31	func HostBootID() string {
32	b, err := os.ReadFile("/proc/sys/kernel/random/boot_id")
33	if err != nil {
34	return ""
35	}
36	return strings.TrimSpace(string(b))
37	}
38
39	// capacity returns the host's TOTAL capacity: total disk at stateDir, total mem,
40	// and CPU count. The server computes allocated/available by subtracting the sum
41	// of live VM specs, so capacity must be totals (not free) for the math to cohere.
42	// computeCapacity is the raw-capacity source, indirected through a var so tests	29	// computeCapacity is the raw-capacity source, indirected through a var so tests
43	// can count how often the (memoized) computation actually runs.	30	// can count how often the (memoized) computation actually runs.
44	var computeCapacity = capacity	31	var computeCapacity = hostinfo.Capacity
45
46	func capacity(stateDir string) *pb.Capacity {
47	var fs syscall.Statfs_t
48	var diskGB int64
49	if err := syscall.Statfs(stateDir, &fs); err == nil {
50	// Total blocks * block size → bytes → GB
51	diskGB = int64(fs.Blocks) * fs.Bsize / (1024 * 1024 * 1024)
52	}
53
54	var info syscall.Sysinfo_t
55	var memMB int64
56	if err := syscall.Sysinfo(&info); err == nil {
57	memMB = int64(info.Totalram) * int64(info.Unit) / (1024 * 1024)
58	}
59
60	return &pb.Capacity{
61	Vcpus: int64(runtime.NumCPU()),
62	MemMb: memMB,
63	DiskGb: diskGB,
64	}
65	}
66		32
67	// Console bridges server-opened console streams to a VM's serial pump	33	// Console bridges server-opened console streams to a VM's serial pump
68	// (consumer-owned; the concrete implementation is *serialpump.Manager).	34	// (consumer-owned; the concrete implementation is *serialpump.Manager).

internal/agent/syncclient/client_test.go

Old		New
@@ -26,23 +26,16 @@ import (
26	"github.com/stretchr/testify/require"	26	"github.com/stretchr/testify/require"
27	)	27	)
28		28
29	// noopProv / noopNet satisfy the reconcile interfaces with no side effects so we	29	// noopProv satisfies the reconcile provisioner seam with no side effects so we
30	// can drive a real Client.Run against a real syncsvc server over QUIC loopback.	30	// can drive a real Client.Run against a real syncsvc server over QUIC loopback.
31	type noopProv struct{}	31	type noopProv struct{}
32		32
33	func (noopProv) PrepareDisk(context.Context, state.VMSpec, string) error { return nil }	33	func (noopProv) PrepareRootDisk(context.Context, state.VMSpec, string) error { return nil }
34	func (noopProv) Boot(context.Context, string, state.VMSpec) error { return nil }	34	func (noopProv) Boot(context.Context, string, state.VMSpec) error { return nil }
35	func (noopProv) Shutdown(context.Context, string) error { return nil }	35	func (noopProv) Shutdown(context.Context, string) error { return nil }
36	func (noopProv) Kill(context.Context, string) error { return nil }	36	func (noopProv) Destroy(context.Context, string) error { return nil }
37	func (noopProv) Running(string) bool { return false }	37	func (noopProv) Running(string) bool { return false }
38		38	func (noopProv) Address(string) string { return "10.77.1.2" }
39	type noopNet struct{}
40
41	func (noopNet) CreateTap(context.Context, string, string) error { return nil }
42	func (noopNet) DeleteTap(context.Context, string) error { return nil }
43	func (noopNet) ReserveIP(string) (string, error) {
44	return "10.77.1.2", nil
45	}
46		39
47	// testQUICIdle is the deliberately-short idle timeout the test listeners use so	40	// testQUICIdle is the deliberately-short idle timeout the test listeners use so
48	// reconnect/drop cases don't wait out the production SyncMaxIdleTimeout.	41	// reconnect/drop cases don't wait out the production SyncMaxIdleTimeout.
@@ -221,7 +214,7 @@ func newClient(t testing.T, addr, fp, hostID, cred string) Client {
221	}	214	}
222	require.NoError(t, agentSt.SaveIdentity(id))	215	require.NoError(t, agentSt.SaveIdentity(id))
223	engine := &reconcile.Engine{	216	engine := &reconcile.Engine{
224	St: agentSt, Prov: noopProv{}, Net: noopNet{},	217	St: agentSt, Prov: noopProv{},
225	Images: func(context.Context, string, string) (string, error) { return "/x.raw", nil },	218	Images: func(context.Context, string, string) (string, error) { return "/x.raw", nil },
226	Seed: func(string, seed.Params) error { return nil },	219	Seed: func(string, seed.Params) error { return nil },
227	// CIDR/grace not exercised by these tests.	220	// CIDR/grace not exercised by these tests.

internal/arch/arch_test.go

Old		New
@@ -212,14 +212,19 @@ func TestServerNeverShellsOut(t *testing.T) {
212		212
213	// R6: all external process execution in the data plane funnels through	213	// R6: all external process execution in the data plane funnels through
214	// agent/exec.Runner, which keeps the host-touching packages mockable and	214	// agent/exec.Runner, which keeps the host-touching packages mockable and
215	// auditable. The sole exception is agent/cloudhv, which launches the	215	// auditable. Two roles are exempt. Provisioner packages launch the long-lived
216	// long-lived cloud-hypervisor process directly (exec.Command) rather than	216	// VMM process directly (exec.Command) rather than through the one-shot Runner
217	// through the one-shot Runner. Every other agent package must use Runner and	217	// — starting processes on the host is what a provisioner IS, so each backend
218	// must not reach os/exec — directly or through an internal wrapper (reaching	218	// is sanctioned for the same reason rather than re-argued. The composition
219	// it via the sanctioned cloudhv is fine).	219	// root constructs the concrete Runner it injects. Every other agent package
220	func TestOnlyCloudhvImportsOsExecInDataPlane(t *testing.T) {	220	// must use Runner and must not reach os/exec — directly or through an internal
		221	// wrapper (reaching it via a sanctioned package is fine).
		222	func TestOnlyProvisionersAndRootImportOsExecInDataPlane(t *testing.T) {
221	g := directImports(t)	223	g := directImports(t)
222	allowed := map[string]bool{module + "/internal/agent/cloudhv": true}	224	allowed := map[string]bool{
		225	module + "/internal/agent/cloudhv": true, // provisioner: spawns cloud-hypervisor
		226	module + "/internal/agent/run": true, // composition root: builds hostRunner
		227	}
223	for pkg, offenders := range execViolations(g, module, "internal/agent/", allowed) {	228	for pkg, offenders := range execViolations(g, module, "internal/agent/", allowed) {
224	for _, o := range offenders {	229	for _, o := range offenders {
225	t.Errorf("data-plane package %s reaches os/exec via %s — use agent/exec.Runner instead", short(pkg), short(o))	230	t.Errorf("data-plane package %s reaches os/exec via %s — use agent/exec.Runner instead", short(pkg), short(o))
@@ -228,8 +233,10 @@ func TestOnlyCloudhvImportsOsExecInDataPlane(t *testing.T) {
228	}	233	}
229		234
230	// R7: external process execution anywhere in internal/ is confined to a	235	// R7: external process execution anywhere in internal/ is confined to a
231	// sanctioned allowlist — cloudhv (the data plane's one exec funnel, R6's	236	// sanctioned allowlist — cloudhv (a provisioner: spawning the VMM directly is
232	// narrower story), cli (interactive ssh must be the real OpenSSH client), shape	237	// what a provisioner is, R6's narrower story), agent/run (the agent
		238	// composition root, which builds the Runner it injects into every other
		239	// package), cli (interactive ssh must be the real OpenSSH client), shape
233	// (architecture tooling that shells out to `go list`, the same introspection	240	// (architecture tooling that shells out to `go list`, the same introspection
234	// internal/arch's own tests do), and smoke (the deploy boot-gate harness, an	241	// internal/arch's own tests do), and smoke (the deploy boot-gate harness, an
235	// external test rig that shells out to ssh, pkill, and `go tool covdata`).	242	// external test rig that shells out to ssh, pkill, and `go tool covdata`).
@@ -238,7 +245,8 @@ func TestOnlyCloudhvImportsOsExecInDataPlane(t *testing.T) {
238	func TestExecIsConfinedToSanctionedPackages(t *testing.T) {	245	func TestExecIsConfinedToSanctionedPackages(t *testing.T) {
239	g := directImports(t)	246	g := directImports(t)
240	allowed := map[string]bool{	247	allowed := map[string]bool{
241	module + "/internal/agent/cloudhv": true, // the data plane's one sanctioned exec funnel (R6's story)	248	module + "/internal/agent/cloudhv": true, // provisioner: spawns the VMM directly (R6's story)
		249	module + "/internal/agent/run": true, // agent composition root: builds the injected Runner
242	module + "/internal/cli": true, // interactive sessions must be the real OpenSSH client	250	module + "/internal/cli": true, // interactive sessions must be the real OpenSSH client
243	module + "/internal/shape": true, // architecture tooling: shells `go list -json` to build the module graph — the same shell-out internal/arch's own tests make	251	module + "/internal/shape": true, // architecture tooling: shells `go list -json` to build the module graph — the same shell-out internal/arch's own tests make
244	module + "/internal/smoke": true, // deploy boot-gate harness: shells out to ssh, pkill, and `go tool covdata` against the live fleet	252	module + "/internal/smoke": true, // deploy boot-gate harness: shells out to ssh, pkill, and `go tool covdata` against the live fleet

internal/arch/execwalk_test.go

Old		New
@@ -37,7 +37,7 @@ bridge IP (`assigned_ip`) via the agent.
37	\| R2 \| No `internal/server` package shells out—the server is pure control plane. Checked transitively: an internal wrapper around `os/exec` cannot smuggle a shell-out in. \| `internal/arch` `TestServerNeverShellsOut` (transitive) + `depguard` `server-no-exec` (direct, fast in-editor). \|	37	\| R2 \| No `internal/server` package shells out—the server is pure control plane. Checked transitively: an internal wrapper around `os/exec` cannot smuggle a shell-out in. \| `internal/arch` `TestServerNeverShellsOut` (transitive) + `depguard` `server-no-exec` (direct, fast in-editor). \|
38	\| R3 \| The wire contract (`pb`, `transport`) imports no other internal package, so a heavy dependency can't leak across the boundary into both binaries. \| `internal/arch` `TestWireContractIsLeaf`. Behavior pinned by `transport` round-trip contract tests. \|	38	\| R3 \| The wire contract (`pb`, `transport`) imports no other internal package, so a heavy dependency can't leak across the boundary into both binaries. \| `internal/arch` `TestWireContractIsLeaf`. Behavior pinned by `transport` round-trip contract tests. \|
39	\| R4 \| Pure domain packages (`agent/state`, `agent/seed`, `agent/ipalloc`, `server/registry`) don't depend on the transport stack (HTTP/QUIC/`transport`). `server/store` may use `transport` (cert helpers) but not HTTP/QUIC. \| `internal/arch` `TestDomainDoesNotImportTransportStack` + `depguard` `domain-no-transport`. \|	39	\| R4 \| Pure domain packages (`agent/state`, `agent/seed`, `agent/ipalloc`, `server/registry`) don't depend on the transport stack (HTTP/QUIC/`transport`). `server/store` may use `transport` (cert helpers) but not HTTP/QUIC. \| `internal/arch` `TestDomainDoesNotImportTransportStack` + `depguard` `domain-no-transport`. \|
40	\| R5 \| The reconcile boundary interfaces (`Provisioner`, `NetEnv`) stay consumer-owned and small; the addressing seam (`NetEnv.ReserveIP`) is where a future central allocator plugs in. \| Convention (below) + `ireturn` allow-list keeps the seams' interface returns honest. \|	40	\| R5 \| The reconcile boundary interface (`Provisioner`) stays consumer-owned and small. A VM's network attachment lives inside it, because on every backend we ship the NIC is a launch argument rather than a separately-lifecycled resource; what crosses the seam is data (the VM's address) and not mechanism (taps, reservations). \| Convention (below) + `ireturn` allow-list keeps the seam's interface returns honest. \|
41	\| R6 \| All external process execution in the data plane funnels through `agent/exec.Runner`. The sole exception is `agent/cloudhv`, which launches the long-lived cloud-hypervisor process directly. Checked transitively (reaching `os/exec` via the sanctioned `cloudhv` is fine). \| `internal/arch` `TestOnlyCloudhvImportsOsExecInDataPlane` (transitive). \|	41	\| R6 \| All external process execution in the data plane funnels through `agent/exec.Runner`. The sole exception is `agent/cloudhv`, which launches the long-lived cloud-hypervisor process directly. Checked transitively (reaching `os/exec` via the sanctioned `cloudhv` is fine). \| `internal/arch` `TestOnlyCloudhvImportsOsExecInDataPlane` (transitive). \|
42	\| R13 \| The OIDC issuer and the relying party stay separate binaries. `eitri-server` is a pure relying party: the bundled issuer (`internal/oidcprovider`) is importable only by its own binary `cmd/eitri-oidc`, and the `go-oidc` verifier module only by `internal/server/api` (the RP). A server import of the issuer would silently rebuild the embedded-IdP coupling; `go-oidc` anywhere but the RP means a second relying party is being hand-rolled. \| `internal/arch` `TestIssuerAndRelyingPartyAreSeparate`. \|	42	\| R13 \| The OIDC issuer and the relying party stay separate binaries. `eitri-server` is a pure relying party: the bundled issuer (`internal/oidcprovider`) is importable only by its own binary `cmd/eitri-oidc`, and the `go-oidc` verifier module only by `internal/server/api` (the RP). A server import of the issuer would silently rebuild the embedded-IdP coupling; `go-oidc` anywhere but the RP means a second relying party is being hand-rolled. \| `internal/arch` `TestIssuerAndRelyingPartyAreSeparate`. \|
43		43
@@ -52,20 +52,21 @@ why the invariants hold:
52		52
53	1. Consumer-side interfaces only. An interface is declared by the package	53	1. Consumer-side interfaces only. An interface is declared by the package
54	that uses it, not the one that implements it. `reconcile` owns	54	that uses it, not the one that implements it. `reconcile` owns
55	`Provisioner` and `NetEnv`; `cloudhv`/`netenv` implement them. Keep them	55	`Provisioner`; `cloudhv` implements it. `cloudhv` in turn owns `Network`,
56	minimal (`Provisioner` is 5 methods). This is what lets the cloud-hypervisor	56	which `netenv` implements. Keep them minimal (`Provisioner` is 6 methods).
57	backend and the IP allocator evolve without touching the reconcile loop.	57	This is what lets a host backend evolve — a second VMM, a different way of
		58	addressing guests — without touching the reconcile loop.
58	2. Dependency injection via struct + function fields, no DI framework.	59	2. Dependency injection via struct + function fields, no DI framework.
59	`reconcile.Engine` is the template: collaborators as interface fields	60	`reconcile.Engine` is the template: collaborators as interface fields
60	(`Prov`, `Net`), pure side effects as func fields (`Images`, `Seed`,	61	(`Prov`), pure side effects as func fields (`Images`, `Seed`, `BootID`,
61	`BootID`, `Now`). Injected time (`Now func() time.Time`) is the sanctioned	62	`Now`). Injected time (`Now func() time.Time`) is the sanctioned way to make
62	way to make decision logic testable—don't call `time.Now()` directly in	63	decision logic testable—don't call `time.Now()` directly in reconcile/store
63	reconcile/store decision paths.	64	decision paths.
64	3. No mutable global state. Constructors (`store.Open`, `hub.New`,	65	3. No mutable global state. Constructors (`store.Open`, `hub.New`,
65	`syncsvc.New`) own all state. Immutable package vars (compiled regexps) are	66	`syncsvc.New`) own all state. Immutable package vars (compiled regexps) are
66	fine.	67	fine.
67	4. Mock only true external dependencies. Tests use hand-written fakes for	68	4. Mock only true external dependencies. Tests use hand-written fakes for
68	the boundary interfaces (`Provisioner`, `NetEnv`) and the	69	the boundary interfaces (`Provisioner`, `cloudhv.Network`) and the
69	`exec.Runner`. The SQLite store is used for real in tests, not mocked. Don't	70	`exec.Runner`. The SQLite store is used for real in tests, not mocked. Don't
70	introduce a mocking framework—hand-written fakes keep tests honest about	71	introduce a mocking framework—hand-written fakes keep tests honest about
71	real behavior.	72	real behavior.

Old		New
@@ -10,9 +10,9 @@ import (
10	// TestExecViolationsDetectsWrappers pins the transitive os/exec detector on a	10	// TestExecViolationsDetectsWrappers pins the transitive os/exec detector on a
11	// fake graph: a control-plane package must be flagged whether it imports	11	// fake graph: a control-plane package must be flagged whether it imports
12	// os/exec directly OR reaches it through an internal wrapper package (the	12	// os/exec directly OR reaches it through an internal wrapper package (the
13	// loophole a direct-import check misses). Allowed packages (cloudhv for R6)	13	// loophole a direct-import check misses). Allowed packages (provisioners and
14	// are sanctioned exec users: reaching os/exec via them is fine, and they are	14	// the composition root, per R6) are sanctioned exec users: reaching os/exec
15	// themselves exempt.	15	// via them is fine, and they are themselves exempt.
16	func TestExecViolationsDetectsWrappers(t *testing.T) {	16	func TestExecViolationsDetectsWrappers(t *testing.T) {
17	const m = "github.com/a73x/eitri"	17	const m = "github.com/a73x/eitri"
18	graph := map[string][]string{	18	graph := map[string][]string{
@@ -58,7 +58,7 @@ func TestExecViolationsDetectsWrappers(t *testing.T) {
58	// exec user) is reachable ONLY through cloudhv, so it is sanctioned by	58	// exec user) is reachable ONLY through cloudhv, so it is sanctioned by
59	// extension — code in the plane can only reach it via cloudhv's API.	59	// extension — code in the plane can only reach it via cloudhv's API.
60	if len(v) != 0 {	60	if len(v) != 0 {
61	t.Errorf("no agent violations expected (cloudhv and its subtree sanctioned), got %v", v)	61	t.Errorf("no agent violations expected (allowed packages and their subtrees are exempt), got %v", v)
62	}	62	}
63	})	63	})
64	}	64	}
@@ -67,7 +67,8 @@ func TestExecViolationsDetectsWrappers(t *testing.T) {
67	// given prefix, the set of packages through which os/exec becomes reachable:	67	// given prefix, the set of packages through which os/exec becomes reachable:
68	// the package itself (direct import) and/or any transitively-reached internal	68	// the package itself (direct import) and/or any transitively-reached internal
69	// package that imports os/exec. Walking never descends into (or flags)	69	// package that imports os/exec. Walking never descends into (or flags)
70	// packages in allowed — their exec use is sanctioned (R6: agent/cloudhv).	70	// packages in allowed — their exec use is sanctioned (provisioners and the
		71	// composition root, per R6).
71	//	72	//
72	// This closes the wrapper loophole: a direct-import check misses an internal	73	// This closes the wrapper loophole: a direct-import check misses an internal
73	// package that wraps os/exec and is imported by the guarded plane.	74	// package that wraps os/exec and is imported by the guarded plane.