a73x

internal/agent/cloudhv/cloudhv.go

Ref:   Size: 22.0 KiB   History 

// Package cloudhv manages one cloud-hypervisor process per VM.
// It handles disk preparation (reflink copy + resize), argument assembly,
// process lifecycle (boot / running / shutdown / kill), and the
// cloud-hypervisor HTTP API over a Unix socket.
package cloudhv

import (
	"context"
	"fmt"
	"net"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"syscall"
	"time"

	agentexec "github.com/a73x/eitri/internal/agent/exec"
	"github.com/a73x/eitri/internal/agent/hostinfo"
	"github.com/a73x/eitri/internal/agent/hyperlog"
	"github.com/a73x/eitri/internal/agent/permanent"
	"github.com/a73x/eitri/internal/agent/pidfile"
	"github.com/a73x/eitri/internal/agent/state"
)

// chLogMode is the file mode for the cloud-hypervisor diagnostic log.
const chLogMode = 0o600

// PumpHooks is the serial-console pump lifecycle the provisioner drives
// (consumer-owned; the concrete implementation is *serialpump.Manager, wired
// by main — cloudhv must not import serialpump). nil disables the hooks.
type PumpHooks interface {
	Ensure(vmID string)
	Stop(vmID string)
}

// Network is the host networking this backend's guests sit on: a Linux bridge
// reached through a per-VM tap. Consumer-owned (R5) and satisfied by
// netenv.Net — declaring it here rather than importing netenv keeps the driver
// testable against a fake and keeps the package graph free of a
// cloudhv → netenv edge.
//
// Every method is per-VM. Host-wide setup (the bridge, NAT, the DHCP
// responder) is the composition root's job and never appears here: it happens
// once at agent start, not once per guest.
type Network interface {
	// ReserveIP returns this VM's sticky address on the NAT underlay,
	// recording its DHCP reservation. Every guest has one, whatever else its
	// spec asks for.
	ReserveIP(vmID string) (string, error)
	// Address returns the NAT address already reserved for vmID, or "" if none.
	// It never allocates.
	Address(vmID string) string
	// NetworkAddress returns the address the site's own DHCP server granted
	// this VM's named-network NIC, or "" — for a guest with no such NIC, and
	// for one whose guest has not finished asking yet. The host learns it by
	// watching, so unlike Address it is not known at boot.
	NetworkAddress(vmID string) string
	// CreateTap creates the VM's tap on the NAT bridge and pins ip as its
	// reservation. When network is non-empty it ALSO creates a second tap on
	// that named network's bridge — additional, never instead. Idempotent.
	CreateTap(ctx context.Context, vmID, ip, network string) error
	// DeleteTap removes the reservation and both taps. Idempotent.
	DeleteTap(ctx context.Context, vmID string) error
	// TapName is the device name of the NAT NIC's tap — the first --net
	// argument, and therefore the guest's eth0.
	TapName(vmID string) string
	// NetTapName is the device name of the named-network NIC's tap, used only
	// by a VM whose spec asked for one.
	NetTapName(vmID string) string
}

// Provisioner manages cloud-hypervisor processes for all VMs on this host.
type Provisioner struct {
	st       *state.Store
	chBin    string // path to cloud-hypervisor binary
	firmware string // path to CLOUDHV.fd (UEFI firmware)
	run      agentexec.Runner
	net      Network

	// bootID identifies the host boot a VM's process was started in. It is the
	// same reader reconcile's own boot-ID guard uses, so the two agree by
	// construction. Injected so a test can move the host to a later boot
	// without rebooting the box running it.
	bootID func() string

	// signal delivers a signal to a pid. Injected because the branch that
	// matters — a kill the kernel refuses — cannot otherwise be reached without
	// depending on what the box running the tests is allowed to signal.
	signal func(pid int, sig syscall.Signal) error

	// Pumps receives serial-pump lifecycle calls at Boot/Shutdown/kill — a VM's
	// pump lives exactly as long as its guest is powered on. nil = no-op.
	Pumps PumpHooks
}

// New constructs a Provisioner. run may be nil when only pure methods
// (buildArgs) are needed; net must not be, since a VM's network attachment is
// part of booting it.
func New(st *state.Store, chBin, firmware string, run agentexec.Runner, net Network) *Provisioner {
	return &Provisioner{
		st: st, chBin: chBin, firmware: firmware, run: run, net: net,
		bootID: hostinfo.BootID,
		signal: syscall.Kill,
	}
}

// BootstrapDest maps a --ch-bin value to the filesystem path bootstrap may
// install the binary at. The two are different vocabularies: --ch-bin is
// usually a bare command name resolved on $PATH at launch, which names no
// install destination — writing to it literally would drop the binary in the
// agent's working directory and every launch would still miss it. A bare name
// resolves via $PATH when already installed (so bootstrap sees it and
// no-ops), else lands in /usr/local/bin, which the systemd unit's default
// $PATH includes. An explicit path is its own destination.
func BootstrapDest(chBin string) string {
	if strings.ContainsRune(chBin, os.PathSeparator) {
		return chBin
	}
	if p, err := exec.LookPath(chBin); err == nil {
		return p
	}
	return filepath.Join("/usr/local/bin", chBin)
}

// disks returns the VM's block devices in attachment order: the root disk, the
// read-only cloud-init seed, then every volume in spec order — so the first
// volume the tenant attached is /dev/vdc, and stays /dev/vdc across reboots.
// Volumes are writable: the guest owns what is on them.
func (p *Provisioner) disks(spec state.VMSpec) []state.Disk {
	out := []state.Disk{
		{Path: p.st.DiskPath(spec.VMID)},
		{Path: p.st.SeedPath(spec.VMID), ReadOnly: true},
	}
	for _, id := range spec.VolumeIDs {
		out = append(out, state.Disk{Path: p.st.VolumePath(id)})
	}
	return out
}

// buildArgs returns the cloud-hypervisor command-line arguments for spec.
// The result is deterministic given the same spec so it can be unit-tested
// without spawning a process.
func (p *Provisioner) buildArgs(spec state.VMSpec) []string {
	vmID := spec.VMID
	tap := p.net.TapName(vmID)
	mac := state.MAC(vmID)

	args := []string{
		"--api-socket", p.st.SocketPath(vmID),
		"--kernel", p.firmware,
		"--cpus", fmt.Sprintf("boot=%d", spec.VCPUs),
		"--memory", fmt.Sprintf("size=%dM", spec.MemMB),
		"--disk",
	}
	// image_type=raw is load-bearing, not decoration: autodetected raw makes
	// CH DISABLE SECTOR 0 WRITES, so the first boot's growpart rewrites the
	// partition table only in memory and the guest dies in initramfs at its
	// first power cycle. Declared raw keeps the GPT writable. (Autodetection
	// is also deprecated in CH v53.)
	for _, d := range p.disks(spec) {
		if d.ReadOnly {
			args = append(args, fmt.Sprintf("path=%s,image_type=raw,readonly=on", d.Path))
			continue
		}
		args = append(args, fmt.Sprintf("path=%s,image_type=raw", d.Path))
	}
	// NIC ORDER IS THE GUEST ABI. The NAT NIC is first and therefore eth0: it
	// is the management fabric every guest has, the address the gate splices
	// to, and what the seed's primary netplan stanza expects to find. A guest
	// that also asked for a named network gets that NIC second — same MAC
	// determinism, its own tap, its own DHCP server answering it.
	args = append(args, "--net", fmt.Sprintf("tap=%s,mac=%s", tap, mac))
	if spec.Network != "" {
		args = append(args, "--net",
			fmt.Sprintf("tap=%s,mac=%s", p.net.NetTapName(vmID), state.NetMAC(vmID)))
	}
	return append(args,
		"--serial", fmt.Sprintf("socket=%s", p.st.SerialSocketPath(vmID)),
		"--console", "off",
	)
}

// maxDiskGB caps a VM disk at 1 PiB (2^20 GiB) — far beyond any real host,
// and small enough that DiskGB<<30 can never overflow int64 (2^50 max).
const maxDiskGB = 1 << 20

// Preflight passes unconditionally: this backend exists only on a host that
// runs guests, and the one thing it needs beyond itself — the
// cloud-hypervisor binary — is installed by the agent's bootstrap step at
// start, long before any create. Re-checking it per create would trade a
// clear startup failure for a per-VM one.
func (p *Provisioner) Preflight(_ context.Context) error { return nil }

// PrepareRootDisk creates the VM's root disk by making a reflink copy of
// basePath (instant on XFS/btrfs; silent full-copy fallback on ext4) and then
// truncating it to spec.DiskGB gigabytes.
//
// truncate -s sets an EXACT size, so a target smaller than the base image
// would silently chop the guest filesystem. PrepareRootDisk refuses to
// shrink: spec.DiskGB must be in [1, maxDiskGB] and cover the base image. The
// range check runs first so the byte computation is overflow-safe (a naive
// DiskGB<<30 wraps to a small positive value for e.g. 2^34+10, bypassing the
// guard). Pinned by TestPrepareRootDiskRefusesToShrinkBaseImage and
// TestPrepareRootDiskShrinkGuardEdgeCases.
func (p *Provisioner) PrepareRootDisk(ctx context.Context, spec state.VMSpec, basePath string) error {
	base, err := os.Stat(basePath)
	if err != nil {
		return fmt.Errorf("stat base image %s: %w", basePath, err)
	}
	if spec.DiskGB < 1 || spec.DiskGB > maxDiskGB {
		return permanent.Errorf("disk_gb %d out of range [1, %d]", spec.DiskGB, int64(maxDiskGB))
	}
	if targetBytes := spec.DiskGB << 30; targetBytes < base.Size() {
		return permanent.Errorf("disk_gb %d (%d bytes) is smaller than base image %s (%d bytes) — shrinking would corrupt the guest",
			spec.DiskGB, targetBytes, basePath, base.Size())
	}
	diskPath := p.st.DiskPath(spec.VMID)
	// Ensure VM directory exists.
	if err := os.MkdirAll(p.st.VMDir(spec.VMID), 0o700); err != nil {
		return fmt.Errorf("mkdir %s: %w", p.st.VMDir(spec.VMID), err)
	}
	// Build into a sibling temp file and rename into place, so a create that is
	// killed (ctx cancel on agent stop, ENOSPC) mid-copy/truncate can never leave
	// a torn disk.raw at the final path — the reconcile "exists" gate keys off
	// rec.BootID, not disk presence, but an atomic artifact keeps the on-disk
	// state honest for debugging and any future disk-aware code. The temp is a
	// sibling (same dir/filesystem) so cp keeps its reflink and rename is atomic.
	tmpPath := diskPath + ".partial"
	_ = os.Remove(tmpPath) // clear any leftover from an earlier interrupted create
	if _, err := p.run(ctx, "cp", "--reflink=auto", basePath, tmpPath); err != nil {
		_ = os.Remove(tmpPath)
		return fmt.Errorf("cp --reflink=auto %s %s: %w", basePath, tmpPath, err)
	}
	sizeArg := fmt.Sprintf("%dG", spec.DiskGB)
	if _, err := p.run(ctx, "truncate", "-s", sizeArg, tmpPath); err != nil {
		_ = os.Remove(tmpPath)
		return fmt.Errorf("truncate -s %s %s: %w", sizeArg, tmpPath, err)
	}
	if err := os.Rename(tmpPath, diskPath); err != nil {
		_ = os.Remove(tmpPath)
		return fmt.Errorf("rename %s -> %s: %w", tmpPath, diskPath, err)
	}
	return nil
}

// pidPath returns the path to the PID file for vmID.
func (p *Provisioner) pidPath(vmID string) string {
	return filepath.Join(p.st.VMDir(vmID), "ch.pid")
}

// Boot spawns a cloud-hypervisor process for spec. The process is placed in
// its own session (Setsid) so it survives an agent restart. A goroutine calls
// cmd.Wait to reap the child when it exits.
//
// The ctx parameter bounds the network attach only. It is deliberately NOT
// wired to the process: the VM's lifetime must not be tied to the agent's
// (exec.CommandContext SIGKILLs the child on ctx cancel, which would
// hard-power-off every VM on a graceful agent stop). Stopping a VM is
// exclusively the job of Shutdown/Destroy, driven by the reconcile loop. Pinned
// by TestBootedVMSurvivesCtxCancellation.
func (p *Provisioner) Boot(ctx context.Context, vmID string, spec state.VMSpec) error {
	// Attach the network FIRST: cloud-hypervisor takes the tap as a launch
	// argument, so the device has to exist before the process does. A failure
	// here surfaces the networking layer's own message — letting the launch
	// fail instead yields an illegible cloud-hypervisor error for the same
	// root cause — and %w keeps a Permanent() marker (a tap name collision)
	// unwrappable, so reconcile still terminal-fails it in one attempt.
	if err := p.attachNet(ctx, spec); err != nil {
		return fmt.Errorf("attach network %s: %w", vmID, err)
	}

	// Remove stale sockets from a previous run. CH does NOT unlink a
	// pre-existing socket path before binding (it removes it only on clean
	// exit), so after a CH crash, SIGKILL, or host reboot a stale socket
	// would make the next boot fail to bind.
	_ = os.Remove(p.st.SocketPath(vmID))
	_ = os.Remove(p.st.SerialSocketPath(vmID))

	args := p.buildArgs(spec)
	cmd := exec.Command(p.chBin, args...)
	cmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true}

	// Guest console output (serial) goes to a unix socket (--serial socket=…)
	// that the serialpump drains into serial.log.
	// CH's own diagnostic output (startup errors, API logs) goes to ch.log.
	chLog, err := os.OpenFile(p.logPath(vmID), os.O_CREATE|os.O_APPEND|os.O_WRONLY, chLogMode)
	if err != nil {
		return fmt.Errorf("open ch.log %s: %w", vmID, err)
	}
	cmd.Stdout = chLog
	cmd.Stderr = chLog

	if err := cmd.Start(); err != nil {
		_ = chLog.Close()
		return fmt.Errorf("cloud-hypervisor start %s: %w", vmID, err)
	}
	// Close the log fd in the parent; the child has its own copy.
	_ = chLog.Close()

	// Record the process so Running/Shutdown/Destroy can find it later — with
	// the host boot it belongs to, because a pid alone stops meaning anything
	// the moment the host reboots (see ownedPID).
	if err := pidfile.Write(p.pidPath(vmID), cmd.Process.Pid, p.bootID()); err != nil {
		// Best effort — kill the orphan if we can't track it, and Wait to reap it
		// (the async reaper below is not started on this path, so without Wait the
		// killed child would linger as a zombie for the agent's whole lifetime).
		_ = cmd.Process.Kill()
		_ = cmd.Wait()
		return fmt.Errorf("track cloud-hypervisor %s: %w", vmID, err)
	}

	// Reap child asynchronously; ignore exit error (VM may be killed intentionally).
	go func() { _ = cmd.Wait() }()

	// Attach the serial pump NOW: the pump must be the socket's one client
	// from as close to power-on as possible so the boot log lands in the
	// ring/log (older CH drops unconsumed serial output entirely; current CH
	// only buffers a bounded amount).
	if p.Pumps != nil {
		p.Pumps.Ensure(vmID)
	}

	return nil
}

// attachNet gives the VM its address and its tap, in that order — the tap
// carries the DHCP reservation, so the address has to be known first.
// Idempotent: both halves tolerate a re-run, which is what makes a Boot retry
// and a restart-after-host-reboot the same code path.
//
// It takes the whole spec rather than the id because whether the VM gets a
// second NIC is the spec's to say: the network name the control plane sent
// rides here and nowhere else. The reservation is unconditional either way —
// every guest is on the NAT underlay.
func (p *Provisioner) attachNet(ctx context.Context, spec state.VMSpec) error {
	ip, err := p.net.ReserveIP(spec.VMID)
	if err != nil {
		return err
	}
	return p.net.CreateTap(ctx, spec.VMID, ip, spec.Network)
}

// ownedPID is the pid of the cloud-hypervisor process THIS agent started for
// vmID, or 0 when there is none it may signal. The rule — and the reason a pid
// alone is not evidence after a reboot — lives in internal/agent/pidfile,
// shared with the other backend so the two cannot disagree about whose process
// they are about to kill.
func (p *Provisioner) ownedPID(vmID string) int {
	return pidfile.Owned(p.pidPath(vmID), p.bootID())
}

// Running reports whether the cloud-hypervisor process for vmID is still alive,
// by pidfile and signal 0.
//
// PID-liveness within this host boot. A pidfile written before the last boot
// reports not-running outright (see ownedPID), so the recycled-pid
// false-positive this used to concede is gone; within one boot, a pid this
// agent wrote cannot have been recycled while its process is alive. reconcile's
// own boot-ID check still decides what a lost VM means — this only decides
// which process, if any, is ours to ask about.
// logPath is where cloud-hypervisor's own diagnostic output is kept (startup
// errors, API logs) — distinct from serial.log, which carries the GUEST's
// console.
func (p *Provisioner) logPath(vmID string) string {
	return filepath.Join(p.st.VMDir(vmID), "ch.log")
}

// FailureReason quotes the last thing cloud-hypervisor said. See
// reconcile.Provisioner.
func (p *Provisioner) FailureReason(vmID string) string { return hyperlog.Reason(p.logPath(vmID)) }

func (p *Provisioner) Running(vmID string) bool {
	pid := p.ownedPID(vmID)
	if pid == 0 {
		return false
	}
	return p.signal(pid, 0) == nil
}

// Address returns the address reserved for vmID on the bridge, or "" when the
// VM has none. This backend allocates before the guest boots, so the answer is
// available the instant Boot returns — reconcile polls it either way, because a
// backend whose host OS hands out addresses cannot answer that early.
func (p *Provisioner) Address(vmID string) string { return p.net.Address(vmID) }

// NetworkAddress returns what the site's DHCP server granted this VM's
// named-network NIC, or "" when it has none or has not been heard from yet.
// Unlike Address it is genuinely polled: the host discovers it by watching the
// guest's own exchange, which cannot have happened before the guest boots.
func (p *Provisioner) NetworkAddress(vmID string) string { return p.net.NetworkAddress(vmID) }

// socketClient returns an *http.Client whose transport dials over the VM's
// Unix socket.
func (p *Provisioner) socketClient(vmID string) *http.Client {
	sockPath := p.st.SocketPath(vmID)
	return &http.Client{
		Timeout: 5 * time.Second,
		Transport: &http.Transport{
			DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
				return (&net.Dialer{}).DialContext(ctx, "unix", sockPath)
			},
		},
	}
}

// Shutdown powers the guest off and takes its serial console with it: a
// powered-off guest has no console to replay. The pump's ring is 256 KiB of
// backlog replayed to every new viewer, so a pump left running across a stop
// would answer for a guest that is not there — the next viewer, or the next
// boot proof, reads the dead boot's login prompt as the live one's.
//
// A shutdown that FAILED leaves the pump alone: the guest may well still be
// running, and a running guest keeps its console. On the way back up Boot
// re-Ensures the pump, which opens the fresh socket with an empty ring.
//
// Console history is not lost with the ring. serial.log is opened O_APPEND at
// a path fixed per VM, so the next pump continues the same file (rotating once
// at 4 MiB) — what a stop costs is the tail of the guest's shutdown sequence,
// which is written after the power button is pressed and has no pump left to
// drain it.
func (p *Provisioner) Shutdown(ctx context.Context, vmID string) error {
	if err := p.powerOff(ctx, vmID); err != nil {
		return err
	}
	if p.Pumps != nil {
		p.Pumps.Stop(vmID)
	}
	return nil
}

// powerOff requests a clean shutdown via the cloud-hypervisor power-button API.
// Falls back to SIGTERM via the PID file if the API call fails or returns a
// non-2xx status (e.g. 404/500 when CH is unhealthy or the VM is not running).
func (p *Provisioner) powerOff(ctx context.Context, vmID string) error {
	client := p.socketClient(vmID)
	req, err := http.NewRequestWithContext(ctx, http.MethodPut,
		"http://localhost/api/v1/vm.power-button", nil)
	if err != nil {
		return p.sigterm(vmID)
	}
	resp, err := client.Do(req)
	if err != nil {
		return p.sigterm(vmID)
	}
	resp.Body.Close()
	// Treat any non-2xx response as a failure and fall through to SIGTERM.
	// A 404 means the VM is not in a running state; a 500 means CH is unhealthy.
	// Either way the power-button did not trigger a shutdown.
	if resp.StatusCode >= 300 {
		return p.sigterm(vmID)
	}
	return nil
}

// sigterm sends SIGTERM to the process identified by vmID's PID file.
func (p *Provisioner) sigterm(vmID string) error {
	pid := p.ownedPID(vmID)
	if pid == 0 {
		return nil // already gone
	}
	if err := p.signal(pid, syscall.SIGTERM); err != nil && err != syscall.ESRCH {
		return fmt.Errorf("SIGTERM %s (pid %d): %w", vmID, pid, err)
	}
	return nil
}

// Destroy stops the VM and releases every host resource it holds: the
// cloud-hypervisor process, its serial pump and socket, and its tap device with
// the DHCP reservation the tap carries.
//
// Both failures reach reconcile, because returning nil is a promise that
// nothing is left to reap and reconcile deletes the VM's record on the strength
// of it. Releasing the tap is the retryable one — `ip link del` under an expired
// pass context fails, and an orphaned eit-XXXXXXXX device has nothing left to
// reap it, since EnsureBridge does not sweep them. A kill the kernel refused is
// rarer and worse: the record and the VM directory would go while the guest
// kept running against unlinked files, leaving a hypervisor on the host that
// nothing on it or in the fleet can name.
func (p *Provisioner) Destroy(ctx context.Context, vmID string) error {
	killErr := p.kill(vmID)
	if err := p.net.DeleteTap(ctx, vmID); err != nil {
		return err
	}
	return killErr
}

// kill SIGKILLs the cloud-hypervisor process for vmID, stops its serial pump,
// and removes the PID file and serial socket. It takes no context: killing a
// process by pidfile is a syscall, and pretending otherwise made callers think
// a cancelled context could skip a teardown.
func (p *Provisioner) kill(vmID string) error {
	// Pump teardown + serial-socket removal come BEFORE the SIGKILL: a pump
	// leaked past a failed kill would dial a deleted path forever. ch.sock is
	// deliberately left in place — the next Boot clears a stale API socket
	// itself, and reap's DeleteVM removes the whole VM dir.
	if p.Pumps != nil {
		p.Pumps.Stop(vmID)
	}
	_ = os.Remove(p.st.SerialSocketPath(vmID))
	pid := p.ownedPID(vmID)
	if pid != 0 {
		if err := p.signal(pid, syscall.SIGKILL); err != nil && err != syscall.ESRCH {
			// Keep the pidfile AND say so. A SIGKILL that failed with anything
			// but ESRCH has not proved the process gone, so the record is the
			// only handle left on it — and an error is what stops reconcile
			// deleting the directory that record lives in.
			return fmt.Errorf("SIGKILL %s (pid %d): %w", vmID, pid, err)
		}
	}
	_ = os.Remove(p.pidPath(vmID))
	return nil
}