a73x

internal/agent/vfkit/vfkit.go

Ref:   Size: 19.8 KiB   History 

// Package vfkit manages one vfkit process per VM: the macOS backend, where
// vfkit is the signed helper that drives Apple's Virtualization.framework.
// It is cloudhv's opposite number and deliberately its mirror image —
// argument assembly, a pidfile, a graceful stop over a unix socket with a
// SIGTERM fallback — because the two backends answer the same seam and a
// reader who knows one should recognize the other.
//
// vfkit rather than a helper of our own: Virtualization.framework is reachable
// only from Objective-C or Swift, so SOMETHING has to stand between the agent
// and the framework. vfkit is that binary already written, Apache-2.0, signed
// with com.apple.security.virtualization, and shipped through Homebrew — and
// it takes the shape the agent already spawns, one process per VM that outlives
// its parent. Writing our own would buy a second thing to sign and notarize.
//
// The package carries no build tag, so Linux CI proves its behavior even
// though only wire_darwin.go builds it into a binary. Nothing here is
// Darwin-only in Go terms: the platform lives in the strings (vfkit's
// arguments, the lease database's path), not in the syscalls.
package vfkit

import (
	"bytes"
	"context"
	"fmt"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"syscall"

	agentexec "github.com/a73x/eitri/internal/agent/exec"
	"github.com/a73x/eitri/internal/agent/hostinfo"
	"github.com/a73x/eitri/internal/agent/hyperlog"
	"github.com/a73x/eitri/internal/agent/permanent"
	"github.com/a73x/eitri/internal/agent/pidfile"
	"github.com/a73x/eitri/internal/agent/state"
)

// logMode is the file mode for vfkit's diagnostic log.
const logMode = 0o600

// PumpHooks is the serial-console pump lifecycle the provisioner drives
// (consumer-owned; the concrete implementation is *serialpump.Manager, wired
// by the composition root — vfkit must not import serialpump).
type PumpHooks interface {
	Ensure(vmID string)
	Stop(vmID string)
}

// Provisioner manages vfkit processes for all VMs on this host.
//
// There is no Network seam here, and its absence is the design. On Linux the
// agent owns addressing outright — it builds the bridge, creates a tap per
// guest and answers DHCP itself — so cloudhv needs a collaborator to do that
// per VM. macOS keeps that machinery for itself: vmnet's NAT and its bootpd
// hand a guest its address once the guest asks, and nothing the agent can
// install changes the answer. So there is no per-VM host networking to
// lifecycle, only a lease to read (see Address).
type Provisioner struct {
	st  *state.Store
	bin string // vfkit binary: a path, or a bare name resolved on $PATH
	run agentexec.Runner

	lookPath func(file string) (string, error)

	leasesPath string

	// bootID identifies the host boot a VM's process was started in. It is the
	// same reader reconcile's own boot-ID guard uses, so the two agree by
	// construction. Injected so a test can move the host to a later boot
	// without rebooting the box running it.
	bootID func() string

	signal func(pid int, sig syscall.Signal) error

	// rest is the one client every vfkit REST call on this host goes through.
	// One, deliberately: see newRESTClient.
	rest *http.Client

	Pumps PumpHooks
}

// defaultLeasesPath is where macOS's bootpd records the addresses it has
// handed out. It is the whole of this backend's address knowledge.
const defaultLeasesPath = "/var/db/dhcpd_leases"

// New constructs a Provisioner. run may be nil when only pure methods
// (buildArgs) are needed.
func New(st *state.Store, bin string, run agentexec.Runner) *Provisioner {
	return &Provisioner{
		st:         st,
		bin:        bin,
		run:        run,
		lookPath:   exec.LookPath,
		leasesPath: defaultLeasesPath,
		bootID:     hostinfo.BootID,
		signal:     syscall.Kill,
		rest:       newRESTClient(),
	}
}

// Preflight refuses a host that has no vfkit. The one thing this backend needs
// beyond itself cannot be installed by the agent the way cloud-hypervisor can:
// vfkit must carry Apple's virtualization entitlement, and an entitlement
// survives only a signature we cannot produce, so a downloaded copy is not a
// working copy. Homebrew's is signed; that is the install path, and naming it
// here is the whole point of answering before the image fetch rather than
// after it.
//
// The refusal is permanent because the retry budget cannot fix it: three
// attempts against a Mac with no vfkit produce the same sentence three times.
// Installing vfkit does not revive the failed VM — create it again.
// It also refuses a state directory too deep to hold a VM's control socket.
// That is a property of the host's configuration, not of any one VM, and it is
// checked here for the same reason: the alternative is discovering it from
// vfkit, after the image download, in a sentence about URIs.
func (p *Provisioner) Preflight(_ context.Context) error {
	if _, err := p.lookPath(p.bin); err != nil {
		return permanent.Errorf("vfkit not found on this host (looked for %q): install it with `brew install vfkit` — "+
			"the macOS backend runs guests through it, and it must be Apple-entitled, so the agent cannot install it itself", p.bin)
	}
	if sock := p.sockPath(strings.Repeat("0", vmIDLen)); len(sock) > maxSocketPath {
		return permanent.Errorf("state directory is too deep for macOS: a VM's control socket would be %q, %d bytes against the %d-byte limit — "+
			"run the agent with a shorter --state-dir", sock, len(sock), maxSocketPath)
	}
	return nil
}

const (
	// maxSocketPath is macOS's cap on a unix socket path: sockaddr_un.sun_path
	// is 104 bytes and the last one is the terminator. Linux allows 108, so a
	// path that works in a test on Linux can still be refused on the platform
	// this backend runs on — which is why it is checked rather than assumed.
	maxSocketPath = 103

	// vmIDLen is how long a VM id is (32 hex characters, server-minted).
	// Preflight has to measure a VM's socket path before there is a VM, so it
	// measures one of the right shape.
	vmIDLen = 32
)

// vmFile returns a path inside the VM's directory. vfkit's per-VM artifacts
// are named here rather than in state.Store because they are this backend's
// business: the store holds what every backend has (the disk, the seed, the
// serial log), and a second backend's private files do not belong in a type
// both of them import.
func (p *Provisioner) vmFile(vmID, name string) string {
	return filepath.Join(p.st.VMDir(vmID), name)
}

func (p *Provisioner) pidPath(vmID string) string  { return p.vmFile(vmID, "vfkit.pid") }
func (p *Provisioner) sockPath(vmID string) string { return p.vmFile(vmID, "vfkit.sock") }
func (p *Provisioner) varStore(vmID string) string { return p.vmFile(vmID, "efi-vars.fd") }
func (p *Provisioner) logPath(vmID string) string  { return p.vmFile(vmID, "vfkit.log") }

// FailureReason quotes the last thing vfkit said. See
// reconcile.Provisioner.
func (p *Provisioner) FailureReason(vmID string) string { return hyperlog.Reason(p.logPath(vmID)) }

// SocketPath is the VM's vfkit REST socket, exported so the composition root
// can hand ConsoleSource the same path Boot writes.
func (p *Provisioner) SocketPath(vmID string) string { return p.sockPath(vmID) }

// disks returns the VM's block devices in attachment order: the root disk, the
// cloud-init seed, then every volume in spec order — so the first volume the
// tenant attached is /dev/vdc, and stays /dev/vdc across reboots. Every one of
// these files must already exist when vfkit starts — it opens them while
// parsing its arguments, before any VM is built — which they do: reconcile
// prepares the disk, writes the seed and materialises the volumes before Boot.
//
// vfkit's virtio-blk has no read-only option
// — unlike cloud-hypervisor's — so state.Disk.ReadOnly is dropped here rather
// than honored. Nothing rests on it: the seed is a per-VM file, and cloud-init
// mounts it read-only from inside the guest regardless.
func (p *Provisioner) disks(spec state.VMSpec) []state.Disk {
	out := []state.Disk{
		{Path: p.st.DiskPath(spec.VMID)},
		{Path: p.st.SeedPath(spec.VMID), ReadOnly: true},
	}
	for _, id := range spec.VolumeIDs {
		out = append(out, state.Disk{Path: p.st.VolumePath(id)})
	}
	return out
}

func (p *Provisioner) buildArgs(spec state.VMSpec, createVarStore bool) []string {
	vmID := spec.VMID

	// The EFI variable store is the guest's NVRAM, and `create` initializes a
	// fresh one — which is why it is conditional. Ubuntu writes its boot entry
	// there on first boot; recreating the store every launch would throw that
	// away each time and leave the guest booting only by the removable-media
	// fallback path. Create it exactly once, on the boot that has no store yet.
	bootloader := "efi,variable-store=" + p.varStore(vmID)
	if createVarStore {
		bootloader += ",create"
	}

	args := []string{
		// The REST socket is how Shutdown asks for an ACPI power-down and how
		// the console finds its PTY. Without it vfkit exposes neither.
		"--restful-uri", "unix://" + p.sockPath(vmID),
		"--cpus", strconv.FormatInt(spec.VCPUs, 10),
		"--memory", strconv.FormatInt(spec.MemMB, 10),
		"--bootloader", bootloader,
	}
	for _, d := range p.disks(spec) {
		args = append(args, "--device", "virtio-blk,path="+d.Path)
	}
	return append(args,
		// nat is Apple's vmnet-shared network: the host NATs the guest out and
		// its bootpd answers the guest's DHCP. mac is the stickiness key — it
		// is what makes the lease, and therefore the address, the same one
		// across a rebuild (see Address).
		"--device", "virtio-net,nat,mac="+state.MAC(vmID),
		// pty, not logFilePath: the console has to carry keystrokes back to the
		// guest, and a log file is one-way. The pump opens the far end.
		"--device", "virtio-serial,pty",
		// cloud-hypervisor gives a guest virtio-rng without being asked; vfkit
		// does not, and a guest short of entropy stalls in early boot.
		"--device", "virtio-rng",
	)
}

// maxDiskGB caps a VM disk at 1 PiB (2^20 GiB) — far beyond any real host,
// and small enough that DiskGB<<30 can never overflow int64 (2^50 max).
const maxDiskGB = 1 << 20

// PrepareRootDisk creates the VM's root disk by cloning basePath on APFS and
// growing it to spec.DiskGB gigabytes.
//
// The shrink guard is cloudhv's, restated rather than shared: truncating to an
// EXACT size means a target below the base image would chop the guest
// filesystem, and that is true of any backend that grows a cloned image. The
// duplication is deliberate for now — hoisting the policy into reconcile is
// the right fix, and is a change to the shared engine, not to this backend.
func (p *Provisioner) PrepareRootDisk(ctx context.Context, spec state.VMSpec, basePath string) error {
	base, err := os.Stat(basePath)
	if err != nil {
		return fmt.Errorf("stat base image %s: %w", basePath, err)
	}
	if spec.DiskGB < 1 || spec.DiskGB > maxDiskGB {
		return permanent.Errorf("disk_gb %d out of range [1, %d]", spec.DiskGB, int64(maxDiskGB))
	}
	targetBytes := spec.DiskGB << 30
	if targetBytes < base.Size() {
		return permanent.Errorf("disk_gb %d (%d bytes) is smaller than base image %s (%d bytes) — shrinking would corrupt the guest",
			spec.DiskGB, targetBytes, basePath, base.Size())
	}
	if err := os.MkdirAll(p.st.VMDir(spec.VMID), 0o700); err != nil {
		return fmt.Errorf("mkdir %s: %w", p.st.VMDir(spec.VMID), err)
	}
	// Build into a sibling temp file and rename into place so a create killed
	// mid-copy can never leave a torn disk.raw at the final path. Sibling, so
	// the clone stays on one filesystem and the rename is atomic.
	diskPath := p.st.DiskPath(spec.VMID)
	tmpPath := diskPath + ".partial"
	_ = os.Remove(tmpPath)
	if err := p.clone(ctx, basePath, tmpPath); err != nil {
		_ = os.Remove(tmpPath)
		return err
	}
	// Grown in-process rather than by truncate(1): macOS does not ship one.
	// os.Truncate grows sparsely, exactly as GNU truncate does on Linux.
	if err := os.Truncate(tmpPath, targetBytes); err != nil {
		_ = os.Remove(tmpPath)
		return fmt.Errorf("grow %s to %dG: %w", tmpPath, spec.DiskGB, err)
	}
	if err := os.Rename(tmpPath, diskPath); err != nil {
		_ = os.Remove(tmpPath)
		return fmt.Errorf("rename %s -> %s: %w", tmpPath, diskPath, err)
	}
	return nil
}

// clone copies src to dst, sharing blocks where the filesystem can. `cp -c`
// is APFS's clonefile — the reflink analog — but unlike `cp --reflink=auto` it
// FAILS rather than degrading when the filesystem cannot clone, so the fallback
// is spelled out here. A Mac's boot volume is APFS; a state directory on an
// external HFS+ disk is the case that needs the second attempt.
func (p *Provisioner) clone(ctx context.Context, src, dst string) error {
	if _, err := p.run(ctx, "cp", "-c", src, dst); err == nil {
		return nil
	}
	_ = os.Remove(dst) // a failed clone may have left a partial file in the way
	if _, err := p.run(ctx, "cp", src, dst); err != nil {
		return fmt.Errorf("cp %s %s: %w", src, dst, err)
	}
	return nil
}

// Boot spawns a vfkit process for spec. The process is placed in its own
// session (Setsid) so it survives an agent restart, and ctx is deliberately NOT
// wired to it: tying a guest's lifetime to the agent's would power off every VM
// on a graceful agent stop. Stopping a VM is Shutdown/Destroy's job alone.
func (p *Provisioner) Boot(_ context.Context, vmID string, spec state.VMSpec) error {
	_ = os.Remove(p.sockPath(vmID))

	// Anything short of a successful stat means "initialise a store". Matching
	// ENOENT alone read every other stat failure — EACCES on a state dir whose
	// ownership moved, EIO on a sick disk — as proof the store is there, which
	// is the one thing a failed stat cannot establish. Contents are not judged
	// and cannot be: a truncated store stats perfectly well, and the firmware is
	// the only reader that would know. A VM whose NVRAM is unreadable is rebuilt
	// under a new id and a new directory, which is where it gets a fresh one.
	_, err := os.Stat(p.varStore(vmID))
	args := p.buildArgs(spec, err != nil)

	cmd := exec.Command(p.bin, args...)
	cmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true}

	// Guest console output goes to the PTY the pump drains. vfkit's own
	// diagnostics — startup errors, the framework's complaints — go here.
	vfLog, err := os.OpenFile(p.logPath(vmID), os.O_CREATE|os.O_APPEND|os.O_WRONLY, logMode)
	if err != nil {
		return fmt.Errorf("open vfkit.log %s: %w", vmID, err)
	}
	cmd.Stdout = vfLog
	cmd.Stderr = vfLog

	if err := cmd.Start(); err != nil {
		_ = vfLog.Close()
		return fmt.Errorf("vfkit start %s: %w", vmID, err)
	}
	_ = vfLog.Close() // the child holds its own copy

	if err := pidfile.Write(p.pidPath(vmID), cmd.Process.Pid, p.bootID()); err != nil {
		// Best effort: kill the orphan we can no longer track, and Wait to reap
		// it — the async reaper below is not started on this path.
		_ = cmd.Process.Kill()
		_ = cmd.Wait()
		return fmt.Errorf("track vfkit %s: %w", vmID, err)
	}

	go func() { _ = cmd.Wait() }() // reap; exit status is not ours to judge

	// Start the pump now so the boot log lands in the ring from as close to
	// power-on as possible. Its first few Opens will fail — the PTY does not
	// exist until vfkit has built the VM — and the pump's reconnect loop is
	// exactly the retry for that.
	if p.Pumps != nil {
		p.Pumps.Ensure(vmID)
	}
	return nil
}

// ownedPID is the pid of the vfkit process THIS agent started for vmID, or 0
// when there is none it may signal. The rule — and the reason a pid alone is
// not evidence after a reboot — lives in internal/agent/pidfile, shared with
// the other backend so the two cannot disagree about whose process they are
// about to kill.
func (p *Provisioner) ownedPID(vmID string) int {
	return pidfile.Owned(p.pidPath(vmID), p.bootID())
}

// Running reports whether the vfkit process for vmID is still alive, by
// pidfile and signal 0.
//
// PID-liveness within this host boot: a pidfile from an earlier boot reports
// not-running outright (see ownedPID), and within one boot a pid this agent
// wrote cannot have been recycled while its process is alive. reconcile's
// boot-ID check remains the authoritative reboot guard for the VM's record.
func (p *Provisioner) Running(vmID string) bool {
	pid := p.ownedPID(vmID)
	if pid == 0 {
		return false
	}
	return p.signal(pid, 0) == nil
}

// Shutdown powers the guest off and takes its serial console with it: a
// powered-off guest has no console to replay, and a pump left running across a
// stop would hand its backlog — a dead boot — to the next viewer as if it were
// the live one. cloudhv holds the same invariant and states the reasoning in
// full. A shutdown that FAILED leaves the pump alone: the guest may still be
// running, and a running guest keeps its console.
func (p *Provisioner) Shutdown(ctx context.Context, vmID string) error {
	if err := p.powerOff(ctx, vmID); err != nil {
		return err
	}
	if p.Pumps != nil {
		p.Pumps.Stop(vmID)
	}
	return nil
}

// powerOff asks vfkit for a graceful stop — the framework's ACPI power-down,
// the same request cloud-hypervisor's power-button API makes. Falls back to
// SIGTERM when the socket is gone or refuses, which covers a vfkit that is
// unhealthy or a VM that is not running.
func (p *Provisioner) powerOff(ctx context.Context, vmID string) error {
	body := bytes.NewReader([]byte(`{"state":"Stop"}`))
	req, err := http.NewRequestWithContext(withSocket(ctx, p.sockPath(vmID)), http.MethodPost, "http://vfkit/vm/state", body)
	if err != nil {
		return p.sigterm(vmID)
	}
	req.Header.Set("Content-Type", "application/json")
	resp, err := p.rest.Do(req)
	if err != nil {
		return p.sigterm(vmID)
	}
	resp.Body.Close()
	// vfkit answers 202 Accepted. Anything past 2xx means the stop did not
	// happen — a 400 for a VM in the wrong state, a 500 from the framework.
	if resp.StatusCode >= 300 {
		return p.sigterm(vmID)
	}
	return nil
}

// sigterm sends SIGTERM to the process identified by vmID's PID file.
func (p *Provisioner) sigterm(vmID string) error {
	pid := p.ownedPID(vmID)
	if pid == 0 {
		return nil
	}
	if err := p.signal(pid, syscall.SIGTERM); err != nil && err != syscall.ESRCH {
		return fmt.Errorf("SIGTERM %s (pid %d): %w", vmID, pid, err)
	}
	return nil
}

// Destroy stops the VM and releases every host resource it holds: the vfkit
// process, its serial pump, its REST socket.
//
// The kill's error is propagated, exactly as cloudhv propagates the failure to
// delete a VM's tap, and for the same reason: nil is the promise reconcile
// deletes the record on the strength of, and the record is the only thing on
// this host that names the process. A SIGKILL the kernel refused has not proved
// the process gone, so answering nil would strand a live vfkit holding an
// unlinked disk, a vmnet attachment and a DHCP lease, with nothing left to find
// it by. A non-nil answer keeps the record and the next tick tries again.
//
// This backend has less to release than cloudhv does: it created no host
// networking, because vmnet's NAT is host-wide and outlives every guest. The
// guest's DHCP lease outlives it too, in a root-owned file bootpd ages out on
// its own; because the lease is keyed on the VM's deterministic MAC, a rebuilt
// VM reclaims the same address rather than leaking a new one.
func (p *Provisioner) Destroy(_ context.Context, vmID string) error {
	return p.kill(vmID)
}

// kill SIGKILLs the vfkit process for vmID, stops its serial pump and removes
// the pidfile and REST socket. It takes no context: killing a process by
// pidfile is a syscall, and pretending otherwise made callers think a cancelled
// context could skip a teardown.
func (p *Provisioner) kill(vmID string) error {
	// Pump teardown first: a pump leaked past a failed kill would reopen a PTY
	// whose far end is gone, forever.
	if p.Pumps != nil {
		p.Pumps.Stop(vmID)
	}
	pid := p.ownedPID(vmID)
	if pid != 0 {
		if err := p.signal(pid, syscall.SIGKILL); err != nil && err != syscall.ESRCH {
			return fmt.Errorf("SIGKILL %s (pid %d): %w", vmID, pid, err)
		}
	}
	_ = os.Remove(p.pidPath(vmID))
	_ = os.Remove(p.sockPath(vmID))
	return nil
}