internal/agent/netenv/netenv.go

Ref: main v0.0.5   Size: 18.4 KiB   History

// Package netenv manages the host side of VM networking: bridge eitri0 with
// the host as .1 gateway, per-VM taps, and NAT for outbound internet. The
// bridge is a pure masqueraded underlay; inbound admin reachability is via the
// server's SSH-CA jump gate (internal/server/sshgate), not any guest overlay.
//
// A VM whose spec names one of the host's configured networks gets a SECOND
// tap on top of all that, attached to a bridge the operator already owns (see
// named.go). None of the machinery above touches it — the site's own DHCP
// server addresses that NIC and its own gateway routes it — and none of it is
// withheld from the guest either: every VM keeps its NAT attachment, its
// reservation and its place on eitri0 beside its siblings.
package netenv

import (
	"context"
	"errors"
	"fmt"
	"net"
	"net/netip"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/a73x/eitri/internal/agent/dhcp"
	"github.com/a73x/eitri/internal/agent/exec"
	"github.com/a73x/eitri/internal/agent/permanent"
	"github.com/a73x/eitri/internal/agent/state"
)

// Bridge is the name of the Linux bridge device created by EnsureBridge.
const Bridge = "eitri0"

// Net holds the runner and the bridge CIDR for this agent instance.
type Net struct {
	run  exec.Runner
	cidr netip.Prefix
	// isTap reports whether an existing link is a tun/tap device. Injectable
	// for tests; production checks /sys/class/net/<name>/tun_flags, which
	// exists only for tun/tap links — no error-string parsing.
	isTap func(name string) bool
	// ifaces enumerates this host's interfaces for the uplink-collision check.
	// Injectable for the same reason isTap is: the production reader touches
	// the kernel, and the logic above it should be testable without one.
	ifaces func() ([]Iface, error)
	dhcp   *dhcp.Server

	// networks maps a configured network name to its operator-owned bridge.
	// Fixed at construction: it is the agent's command line, and changing it
	// is a restart.
	networks map[string]string
	// isBridge reports whether a link is a bridge. Injectable like isTap;
	// production reads /sys/class/net/<name>/bridge, a directory that exists
	// exactly for bridge devices.
	isBridge func(name string) bool
	// listen watches a tap for the DHCP ACK addressed to a guest's MAC.
	// Injectable like isTap, and for a second reason: the production snoop is
	// a Linux packet socket, so the default is chosen per platform.
	listen func(ctx context.Context, ifname string, mac net.HardwareAddr, found func(ip string)) error

	mu sync.Mutex
	// attached holds one entry per VM that has a second NIC on a named network,
	// and holding an entry is what being attached means: it is created when the
	// NIC is (CreateTap, or AdoptNetwork replaying one that survived a restart)
	// and deleted once, whole, when the NIC goes (DeleteTap). Everything the
	// attachment accumulates — the address the site granted it, the goroutine
	// watching for the next one — lives and dies with that single entry, so
	// there is no state left over to speak for a VM this map has forgotten. In
	// memory, rebuilt at startup by the replay, exactly like the DHCP
	// reservation table.
	attached map[string]*attachment
}

// attachment is one VM's named-network NIC as this agent holds it. Which
// network that is does not appear here: the record the control plane placed is
// what says so, and the agent re-reads it on every replay, so a copy in memory
// would only be a second answer to a question already settled. What is here is
// what nothing else can answer — the address the site handed this NIC, and the
// goroutine listening for the next one. Both arrive after the attachment does,
// and both end with it, because the entry does.
type attachment struct {
	// discovered is the last address the site's DHCP server granted this NIC.
	// Never the NAT address: that one this host allocates, and it lives in the
	// reservation table. Empty until an ACK is snooped or a restart replays one.
	discovered string
	// stopSnoop ends the goroutine watching this NIC's tap, and its presence is
	// what says one is running: nil for a VM adopted with no tap to watch, and
	// nil again after a snoop that would not start (so a retry re-arms).
	stopSnoop context.CancelFunc
}

// guestDNS is handed to guests as their DHCP DNS servers (option 6). Public
// resolvers reachable via the eitri0 masquerade — matches the pre-DHCP static
// netplan, and avoids handing guests a host-loopback stub resolver.
var guestDNS = []net.IP{net.IPv4(1, 1, 1, 1), net.IPv4(9, 9, 9, 9)}

// New constructs a Net. cidr must be a valid IPv4 prefix (e.g. "10.77.1.0/24").
// networks maps a named guest network to the operator-owned bridge backing it;
// nil or empty means this host serves only the NAT underlay. The names are not
// checked against the host's links here — VerifyNetworks does that, so a
// caller decides whether a missing bridge stops the agent.
func New(run exec.Runner, cidr string, networks map[string]string) (*Net, error) {
	p, err := netip.ParsePrefix(cidr)
	if err != nil {
		return nil, err
	}
	if !p.Addr().Is4() {
		return nil, fmt.Errorf("bridge CIDR must be IPv4, got %s", cidr)
	}
	n := &Net{
		run: run, cidr: p, isTap: sysfsIsTap, ifaces: hostIfaces,
		networks: networks, isBridge: sysfsIsBridge, listen: defaultListen,
		attached: map[string]*attachment{},
	}
	gw := net.ParseIP(n.Gateway())
	mask := net.CIDRMask(p.Bits(), 32)
	n.dhcp = dhcp.NewServer(Bridge, n.cidr.String(), gw, mask, guestDNS, 12*time.Hour)
	return n, nil
}

// sysfsIsTap reports whether name is an L2 TAP link: /sys/class/net/<name>/
// tun_flags exists exactly for tun-driver links, and bit 0x0002 (IFF_TAP)
// distinguishes a TAP from an L3 TUN (which cannot join a bridge and would
// fail later with a raw RTNETLINK error).
func sysfsIsTap(name string) bool { return sysfsIsTapAt("/sys/class/net", name) }

// sysfsIsTapAt is sysfsIsTap with an injectable sysfs root (for tests).
func sysfsIsTapAt(root, name string) bool {
	raw, err := os.ReadFile(filepath.Join(root, name, "tun_flags"))
	if err != nil {
		return false
	}
	flags, err := strconv.ParseUint(strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(string(raw)), "0x")), 16, 64)
	if err != nil {
		return false
	}
	const iffTap = 0x0002
	return flags&iffTap != 0
}

// Gateway returns the host-side IP (.1) on the bridge, as a bare address string.
func (n *Net) Gateway() string { return n.cidr.Masked().Addr().Next().String() }

// TapName returns the TAP device name for vmID: "eit-" plus the first 8
// characters of the id, giving 12 characters, safely below the 15-char
// IFNAMSIZ limit. It is a method rather than a package function because it is
// part of what a host networking backend answers for — a backend with no tap
// device has no such name, and nothing above the platform line may assume one.
func (n *Net) TapName(vmID string) string { return tapName("eit-", vmID) }

// netTapPrefix begins the name of every named-network tap, and is the one place
// that shape is written down. The nftables rule in named.go matches devices by
// it, so a rename that missed the rule would leave the bridge judging a name no
// tap has any more — hence one home, and both readers derived from it.
const netTapPrefix = "eil-"

// NetTapName returns the device name of the VM's SECOND tap, the one on a
// named host network: "eil-" plus the same 8 characters, so the two taps of
// one guest sort together and neither can be mistaken for the other in
// `ip link`. The "l" is just that marker — the second-NIC tap, distinct from
// "eit-" — not short for anything the console says; live devices already wear
// the prefix, so it stays even though the word it once stood for did not.
// Same 12-character budget as TapName.
func (n *Net) NetTapName(vmID string) string { return tapName(netTapPrefix, vmID) }

// tapName is the shared truncation both device names owe their IFNAMSIZ safety
// to: one rule, so a second NIC can never be the one that overruns it.
func tapName(prefix, vmID string) string {
	if len(vmID) > 8 {
		vmID = vmID[:8]
	}
	return prefix + vmID
}

// ReserveIP returns a sticky IP for vmID, allocated from the bridge CIDR and
// recorded as the guest's DHCP reservation (keyed by the VM's deterministic
// MAC). Idempotent: the same VM gets the same address across reboots and agent
// restarts. The table lives in memory, so an address survives an agent restart
// only by way of the startup replay that rebuilds it from durable records.
//
// Every VM gets one, whatever else its spec asks for: the NAT NIC is the
// management fabric — the gate's splice target, the guests' way to each other,
// the egress — and a guest that also sits on the operator's LAN is a guest with
// two NICs, not a guest that gave this one up.
func (n *Net) ReserveIP(vmID string) (string, error) {
	mac, err := net.ParseMAC(state.MAC(vmID))
	if err != nil {
		return "", err
	}
	ip, err := n.dhcp.Reserve(mac)
	if err != nil {
		return "", err
	}
	return ip.String(), nil
}

// Address returns the address currently reserved for vmID on the NAT underlay,
// or "" when the VM holds no reservation. Unlike ReserveIP it never allocates:
// it is the polled read of what this host believes the guest's address is.
//
// It is unaffected by a named network. A guest's second NIC has its own address
// and its own question (NetworkAddress); this one is the address everything
// eitri does with a VM goes through, and it is known before the guest boots.
func (n *Net) Address(vmID string) string {
	mac, err := net.ParseMAC(state.MAC(vmID))
	if err != nil {
		return ""
	}
	ip, ok := n.dhcp.Lookup(mac)
	if !ok {
		return ""
	}
	return ip.String()
}

// tolerated reports whether err carries one of the given substrings in either
// the command stdout or the error message. iproute2 puts the same condition in
// different streams across versions, so both are checked.
func tolerated(out string, err error, substrs ...string) bool {
	msg := err.Error()
	for _, s := range substrs {
		if strings.Contains(out, s) || strings.Contains(msg, s) {
			return true
		}
	}
	return false
}

// best runs a command, tolerating "already exists" / "File exists" errors so
// that EnsureBridge is idempotent (called on every agent start).
func (n *Net) best(ctx context.Context, name string, args ...string) (string, error) {
	out, err := n.run(ctx, name, args...)
	if err != nil {
		if tolerated(out, err, "File exists", "already exists") {
			return out, nil
		}
		return out, fmt.Errorf("%s %s: %w", name, strings.Join(args, " "), err)
	}
	return out, nil
}

// EnsureBridge creates and configures the eitri0 Linux bridge, enables IP
// forwarding, and installs an nftables NAT rule that masquerades VM outbound
// traffic on every interface.
//
// On a host that serves named networks it also programs the bridge-family rule
// those networks owe the operator (blockGuestDHCP). A host with none issues no
// bridge-family command at all: its ruleset is what it always was.
//
// The function is safe to call on every agent restart: the nft chain is
// flushed before the masquerade rule is added, so rules never accumulate
// across restarts. The same holds for the bridge-family chain.
func (n *Net) EnsureBridge(ctx context.Context) error {
	gw := n.Gateway()
	bits := n.cidr.Bits()
	cidrStr := n.cidr.Masked().String()

	// Idempotency by genuinely-idempotent operations, NOT by parsing kernel
	// error strings: iproute2 message wording varies across versions (a
	// duplicate address says "Address already assigned." on some, "File exists"
	// on others), so a string allow-list is fragile and silently strands a
	// restarting agent. Instead: existence-check the link, and use `ip addr
	// replace` (add-or-update, exit 0 whether or not the address is present).

	// Bridge link — create only if absent.
	if _, err := n.run(ctx, "ip", "link", "show", "dev", Bridge); err != nil {
		if _, err := n.run(ctx, "ip", "link", "add", Bridge, "type", "bridge"); err != nil {
			return fmt.Errorf("ip link add %s: %w", Bridge, err)
		}
	}

	// Gateway address — `replace` is idempotent.
	if _, err := n.run(ctx, "ip", "addr", "replace",
		fmt.Sprintf("%s/%d", gw, bits), "dev", Bridge); err != nil {
		return fmt.Errorf("ip addr replace %s/%d dev %s: %w", gw, bits, Bridge, err)
	}

	// Post-condition: the gateway must actually be on the bridge now. Catches a
	// genuine failure (e.g. the address ended up elsewhere) with an actionable
	// error instead of mysterious VM-networking failures later.
	if out, err := n.run(ctx, "ip", "-o", "addr", "show", "dev", Bridge); err != nil ||
		!strings.Contains(out, gw+"/") {
		return fmt.Errorf("gateway IP %s not present on %s after setup", gw, Bridge)
	}

	// Remaining steps are idempotent by nature: `ip link set up` is a no-op if
	// already up; `sysctl -w` just sets the value; nft `add table`/`add chain`
	// are no-ops if the object already exists (only `create` errors).
	idempotentSteps := [][]string{
		{"ip", "link", "set", Bridge, "up"},
		{"sysctl", "-w", "net.ipv4.ip_forward=1"},
		{"nft", "add", "table", "ip", "eitri"},
		{"nft", "add", "chain", "ip", "eitri", "postrouting",
			"{ type nat hook postrouting priority srcnat ; }"},
	}
	for _, s := range idempotentSteps {
		if _, err := n.run(ctx, s[0], s[1:]...); err != nil {
			return fmt.Errorf("%s %s: %w", s[0], strings.Join(s[1:], " "), err)
		}
	}

	// Flush the chain before adding the rule so that repeated agent restarts do
	// not accumulate duplicate masquerade rules in the kernel ruleset.
	if _, err := n.run(ctx, "nft", "flush", "chain", "ip", "eitri", "postrouting"); err != nil {
		return fmt.Errorf("nft flush chain ip eitri postrouting: %w", err)
	}

	// Plain masquerade rule: no interface exclusions.
	ruleArgs := []string{"add", "rule", "ip", "eitri", "postrouting",
		"ip", "saddr", cidrStr, "masquerade"}

	if _, err := n.run(ctx, "nft", ruleArgs...); err != nil {
		return fmt.Errorf("nft add rule: %w", err)
	}

	return n.blockGuestDHCP(ctx)
}

// StartDHCP begins serving reserved leases on the bridge. Call after
// EnsureBridge (the interface must exist) and after reservations are preloaded.
func (n *Net) StartDHCP(ctx context.Context) error { return n.dhcp.Start(ctx) }

// AddReservation pins vmID's deterministic MAC to ip in the DHCP table. Used at
// startup to rebuild the (in-memory) table from durable records, and by
// CreateTap. A bad ip is ignored (the record is malformed; nothing to serve).
func (n *Net) AddReservation(vmID, ip string) {
	mac, err := net.ParseMAC(state.MAC(vmID))
	if err != nil {
		return
	}
	parsed := net.ParseIP(ip)
	if parsed == nil {
		return
	}
	n.dhcp.SetReservation(mac, parsed)
}

// CreateTap creates the VM's TAP device, attaches it to eitri0, and adds the
// guest's DHCP reservation (MAC(vmID) -> ip). vmID is the reconcile identity;
// ip is the agent-allocated address.
//
// Idempotent by existence check, not error-string parsing (same rationale as
// EnsureBridge): re-running `ip tuntap add` on a live tap fails with
// "ioctl(TUNSETIFF): Device or resource busy" — a message the tolerated()
// allow-list can never chase across iproute2 versions. A create-retry hitting
// the previous attempt's tap must not mask the retry's real error. Pinned by
// TestCreateTapIdempotentWhenTapExists.
//
// network, when non-empty, adds a SECOND tap on the operator's own bridge (see
// createNamedTap) after this one — the guest keeps everything above and gains a
// NIC on the named network. It is validated before anything is created, so a
// network this agent does not serve leaves the host exactly as it found it.
func (n *Net) CreateTap(ctx context.Context, vmID, ip, network string) error {
	if network != "" {
		if _, ok := n.networks[network]; !ok {
			// The control plane only places a network on a host that advertised
			// it, so getting here means this agent restarted without the flag.
			// Permanent, and refused before the NAT tap exists: converge must
			// fail the VM legibly, never boot it with the NIC it asked for
			// missing — a guest silently reachable on the private underlay
			// alone is a guest nothing that was talking to it can find.
			return permanent.Errorf("VM wants network %q but this agent is not configured with it; "+
				"restart the agent with --host-network %s=<bridge> (its NIC is never silently dropped)",
				network, network)
		}
	}
	tap := n.TapName(vmID)
	if _, err := n.run(ctx, "ip", "link", "show", "dev", tap); err != nil {
		if _, err := n.best(ctx, "ip", "tuntap", "add", "dev", tap, "mode", "tap"); err != nil {
			return err
		}
	} else if !n.isTap(tap) {
		// Name collision or stale device: silently enslaving a non-tap would
		// surface later as an illegible cloud-hypervisor failure. Fail loudly
		// here instead (pinned by TestCreateTapRejectsNonTapDevice).
		return permanent.Errorf("link %s exists but is not a TAP device — name collision or stale interface; remove it or rename the VM", tap)
	}
	if _, err := n.best(ctx, "ip", "link", "set", tap, "master", Bridge); err != nil {
		return err
	}
	if _, err := n.best(ctx, "ip", "link", "set", tap, "up"); err != nil {
		return err
	}
	n.AddReservation(vmID, ip)
	if network == "" {
		return nil
	}
	return n.createNamedTap(ctx, vmID, network)
}

// DeleteTap removes the VM's DHCP reservation and BOTH of its TAP devices, plus
// the snoop and discovered address a named NIC leaves behind. Idempotent: a
// missing device is tolerated.
//
// The named tap is deleted whether or not this agent remembers the VM having
// one. The memory of it is in-memory state rebuilt from records at startup, and
// a tap outliving its guest holds a port on the operator's bridge — so the
// device, not the bookkeeping, is what gets asked.
//
// Both deletes are attempted and their errors joined, for the same reason: an
// early return on the NAT tap would leave the named one on the operator's
// bridge, alive, with nothing left that remembers to ask again.
func (n *Net) DeleteTap(ctx context.Context, vmID string) error {
	n.mu.Lock()
	var cancel context.CancelFunc
	if a, live := n.attached[vmID]; live {
		cancel = a.stopSnoop
	}
	delete(n.attached, vmID)
	n.mu.Unlock()
	if cancel != nil {
		cancel()
	}
	if mac, err := net.ParseMAC(state.MAC(vmID)); err == nil {
		n.dhcp.RemoveReservation(mac)
	}
	return errors.Join(
		n.deleteLink(ctx, n.TapName(vmID)),
		n.deleteLink(ctx, n.NetTapName(vmID)),
	)
}

// deleteLink removes one device, tolerating its absence — the shape both taps
// are torn down in.
func (n *Net) deleteLink(ctx context.Context, dev string) error {
	out, err := n.run(ctx, "ip", "link", "del", dev)
	if err != nil {
		if tolerated(out, err, "Cannot find device") {
			return nil
		}
		return fmt.Errorf("ip link del %s: %w", dev, err)
	}
	return nil
}