internal/agent/seed/seed.go
Ref: Size: 23.8 KiB History
// Package seed builds the cloud-init NoCloud config-drive ISO (label CIDATA).
package seed
import (
"fmt"
"os"
"strings"
"github.com/a73x/eitri/internal/cloudinit"
"github.com/a73x/eitri/internal/guest"
diskfs "github.com/diskfs/go-diskfs"
"github.com/diskfs/go-diskfs/disk"
"github.com/diskfs/go-diskfs/filesystem"
"github.com/diskfs/go-diskfs/filesystem/iso9660"
)
// Params holds the cloud-init configuration for a single VM.
type Params struct {
Hostname string
SSHAuthorizedKey string
UserData string // verbatim if set; default generated otherwise
InstanceID string // used as cloud-init instance-id; falls back to Hostname when empty
// SSHUserCAAuthorizedKey, when non-empty, is the tenant's user-CA set in
// authorized_keys form — one canonical CA per line (from the agent joining the
// VMSpec.ssh_user_ca_authorized_keys set). Its presence
// makes the seed inject an sshd drop-in (TrustedUserCAKeys) so the guest
// trusts CA-signed user certs minted by the jump gate. Empty = no injection.
// Exempt from the newline check: it is embedded as a YAML block scalar (like
// UserData), and MarshalAuthorizedKey output carries a trailing newline.
SSHUserCAAuthorizedKey string
// SSHHostKeyPEM / SSHHostCert, when non-empty, are the VM's persistent
// ed25519 host private key (OpenSSH PEM) and its eitri-CA-signed host cert
// (authorized_keys form). Their presence makes the seed hand them to
// cloud-init via its native ssh_keys map (ed25519_private / ed25519_certificate),
// which installs them at /etc/ssh/ssh_host_ed25519_key(+-cert.pub); the drop-in
// then points sshd at the cert (HostCertificate) so clients can verify the VM
// via `@cert-authority`. Providing ssh_keys makes cloud-init install ONLY the
// ed25519 host key; with ssh_deletekeys at its default (true) the image's baked
// rsa/ecdsa host keys are deleted and NOT regenerated — so each VM ends up with
// a single per-VM, CA-certified ed25519 host key and no shared image keys.
// (Add ssh_genkeytypes if rsa/ecdsa host keys are ever needed too.) Empty = no host-cert
// injection (TOFU as before). Both are embedded as YAML block scalars (PEM is
// multi-line), so they are exempt from the newline check like UserData / the
// CA key.
SSHHostKeyPEM string
SSHHostCert string
// MAC and NetworkMAC are the guest's NICs, in the order the hypervisor
// attaches them: MAC is the NAT NIC every guest has, NetworkMAC the second
// one a guest on a named host network gets. NetworkMAC empty — the usual
// case — writes the single-NIC network-config verbatim as it has always
// been; non-empty requires MAC, because a two-NIC netplan matches both by
// hardware address and a stanza with nothing to match would claim the wrong
// link or none.
MAC string
NetworkMAC string
}
// validateParams checks that fields embedded into YAML do not contain newlines
// or carriage returns, which would allow YAML injection into cloud-init documents.
// UserData is deliberately exempt: it is a verbatim multi-line document.
func validateParams(p Params) error {
for _, f := range []struct {
name string
value string
}{
{"Hostname", p.Hostname},
{"InstanceID", p.InstanceID},
{"SSHAuthorizedKey", p.SSHAuthorizedKey},
{"MAC", p.MAC},
{"NetworkMAC", p.NetworkMAC},
} {
if strings.ContainsAny(f.value, "\n\r") {
return fmt.Errorf("seed: %s must not contain newline or carriage return", f.name)
}
}
// A second NIC with no first one to match is a netplan that claims the
// wrong link or no link at all — refuse to build it rather than ship a
// guest whose management NIC never comes up.
if p.NetworkMAC != "" && p.MAC == "" {
return fmt.Errorf("seed: NetworkMAC given without MAC — a two-NIC network-config matches both by hardware address")
}
return nil
}
// Guest paths the seed installs the eitri SSH material into. cloud-init's
// ssh_keys map installs the eitri-CA-signed ed25519 host key at the default
// path so the VM presents it; sshd is pointed at the cert via the drop-in.
const (
userCAPath = "/etc/ssh/eitri_user_ca.pub"
hostKeyPath = "/etc/ssh/ssh_host_ed25519_key"
hostCertPath = "/etc/ssh/ssh_host_ed25519_key-cert.pub"
dropInPath = "/etc/ssh/sshd_config.d/eitri-ca.conf"
)
// sshdDropIn builds the eitri sshd drop-in content: a TrustedUserCAKeys line
// when the user CA is injected (so the guest trusts CA-signed USER certs), and
// HostKey + HostCertificate lines when a per-VM HOST cert is injected (so the
// guest presents a CA-signed host key clients verify via `@cert-authority`).
// Either or both may be present. A DROP-IN under sshd_config.d — sshd_config
// itself is never edited in place. Trailing newlines keep it a well-formed conf.
func sshdDropIn(p Params) string {
var b strings.Builder
if p.SSHUserCAAuthorizedKey != "" {
b.WriteString("TrustedUserCAKeys " + userCAPath + "\n")
}
if p.SSHHostKeyPEM != "" {
b.WriteString("HostKey " + hostKeyPath + "\n")
b.WriteString("HostCertificate " + hostCertPath + "\n")
}
return b.String()
}
// growRuncmd grows the root partition to fill disk_gb and resizes the
// filesystem, online, on the mounted root — eitri's side of the contract
// (eitri delivers disk_gb to the guest; the guest owns everything above it).
// It lives in eitri's own cloud-config part, not the default user-data, so it
// applies even when a tenant brings their own cloud-init — including one with a
// `runcmd` of its own, which without the merge directive (see mergeHow) would
// replace this.
//
// The image ships a GPT sized for its built-in ~3.5G disk; the agent extends the
// raw file to disk_gb, which strands the GPT's backup header mid-device. The
// image's own cloud-init growpart is therefore turned OFF (see eitriConfigDoc):
// its sfdisk resizer treats sfdisk's non-zero exit on the "GPT backup not at the
// end / PMBR size mismatch" auto-correction as a failure and REVERTS the grow,
// so the root never grows and cloud-init reports degraded. This runcmd does the
// same sfdisk write but ignores that exit — the write persists — then completes
// the grow online: partx -u makes the kernel pick up the new partition size
// without unmounting, and resize2fs grows the mounted root filesystem.
//
// `printf ',+\n' | sfdisk -N 1` keeps partition 1's start and grows it to fill
// the disk; --no-reread/--no-tell-kernel avoid the BLKRRPART that fails on a
// mounted root (partx -u supplies the online update instead).
const growRuncmd = ` - ["sh", "-c", "printf ',+\\n' | sfdisk --no-reread --no-tell-kernel -N 1 /dev/vda; partx -u /dev/vda; resize2fs /dev/vda1"]` + "\n"
// chronyDropInPath is the guest file the clock config lands in. The image's
// chrony reads /etc/chrony/conf.d via a confdir directive, so a drop-in there
// overrides the shipped defaults without editing chrony.conf — the same
// drop-in idiom as the sshd config (see dropInPath).
const chronyDropInPath = "/etc/chrony/conf.d/eitri-clock.conf"
// chronyDropIn keeps the guest's clock true across a host suspend. A guest's
// vCPUs freeze with the host that runs them, so wall time stands still for the
// length of the suspend and nothing inside the guest observes the gap: the
// guest simply wakes up hours behind, with its own monotonic clock agreeing
// that no time passed. Only an NTP correction can find it.
//
// The image's chrony ships `makestep 1 3` — step the clock for the first three
// updates after chronyd starts, and slew every correction after that. Slewing
// is capped at maxslewrate, so an hour of suspend takes the better part of a day
// to walk back. A negative limit disables the limit: step at any update, which
// is the only setting that recovers a suspend at all.
//
// chrony's manual prefers stepping only at boot, to spare programs that assume
// time advances monotonically. eitri takes the other side of that trade. A step
// moves CLOCK_REALTIME and leaves CLOCK_MONOTONIC alone, so the programs that
// care are already insulated, and the alternative is worse than a jump: SSH
// user certs carry a 30-minute validity window, so a guest whose clock is
// further behind than that rejects every certificate the gate mints and is
// unreachable until the slew finishes.
//
// The sources are the image's own (chrony's sources.d carries the Ubuntu NTP
// pools), reached over the host's NAT like any other guest egress. Nothing here
// adds a server: the drop-in changes how a correction is applied, not where
// time comes from.
const chronyDropIn = `# Step the clock at any update, not just the first few after boot: this guest's
# host can suspend, and the guest cannot observe the gap on its own.
makestep 1 -1
`
// chronyRuncmd applies the drop-in to the chronyd that is already running from
// the image's own config: cloud-init writes guest files after chrony.service
// has started, so without this the setting would first take effect a boot late
// — on the boot after the one that needs it. try-restart does nothing on an
// image that does not run chrony, and the redirect and `:` keep that image from
// failing the runcmd and reporting cloud-init degraded.
const chronyRuncmd = ` - ["sh", "-c", "systemctl try-restart chrony.service 2>/dev/null || :"]` + "\n"
// mergeHow is the merge directive eitri's cloud-config carries, and the only
// thing that makes the rest of it survive a tenant's own cloud-init.
//
// cloud-init's cloud-config part handler folds the parts of one data source
// into a single buffer, one at a time, merging each part into what the earlier
// parts built. The mergers it uses are the ones named by the part being merged
// IN — a `merge_how` key in that part's document, or a `Merge-Type` MIME header
// — and a part that names neither falls back to `dict(replace)+list()+str()`:
// dict keys REPLACED, lists NOT appended. That fallback is the defect. A tenant
// document carrying `write_files:` or `runcmd:` and eitri's document carrying
// its own would leave the guest with whichever was folded in last and nothing
// of the other — no /etc/ssh/eitri_user_ca.pub and no CA-signed certificate
// accepted, or the image's ~3.5G root still sitting on a 10G disk.
//
// So eitri's document names its own mergers and is the LAST part appended (see
// composeUserData): `list(prepend)` puts eitri's list items in front of the
// tenant's instead of discarding them; `dict(recurse_array,no_replace)` merges
// mappings key by key, recursing into a list a key holds rather than treating
// it as one opaque value, and never overwriting a key the tenant set; `str()`
// leaves a scalar the tenant set alone. eitri's entries are ADDED to the
// tenant's, and every scalar the tenant chose still wins — `growpart: mode`
// included, which is what the userData comment below promises them.
//
// prepend rather than append because the order is load-bearing: the grow
// runcmd has to run before whatever the tenant put in runcmd, so their commands
// see the disk they asked for and not the image's.
//
// This is the ONE layer that reads merge_how. The handler pops the key out of
// each part as it folds it in and writes the stripped result, so the merge that
// happens afterwards — user-data against vendor-data — never sees a merge_how
// from either. That is why eitri's config is a part of user-data and there is
// no vendor-data in the seed at all.
const mergeHow = `merge_how: "list(prepend)+dict(recurse_array,no_replace)+str()"` + "\n"
// eitriConfigDoc renders eitri's own #cloud-config for a seed — the part
// appended after the tenant's document (see composeUserData). It always carries
// the disk-grow (growpart disabled + the online grow runcmd, see growRuncmd) and
// the clock drop-in (see chronyDropIn); it additionally carries the eitri
// user-CA trust drop-in when the CA key is set and the per-VM host key + cert
// when those are set. Its first line is the merge directive (see mergeHow) that
// keeps all of it from being discarded by a tenant's own write_files or runcmd.
//
// The keys, cert, and drop-in conf are embedded as YAML block scalars (`|`), so
// their bytes appear verbatim in the guest files. Block scalars are
// injection-safe: every indented line is literal content. The sshd reload is
// gated behind `sshd -t` so a malformed config can never lock anyone out.
func eitriConfigDoc(p Params) string {
var b strings.Builder
b.WriteString("#cloud-config\n")
b.WriteString(mergeHow)
// Disable the image's growpart: its sfdisk resizer reverts on this disk (see
// growRuncmd). Quoted so YAML reads the string "off", not the boolean false.
b.WriteString("growpart:\n mode: \"off\"\n")
if p.SSHHostKeyPEM != "" {
// Hand the ed25519 host key + cert to cloud-init's native ssh_keys map.
// cc_ssh installs ed25519_private/ed25519_certificate at the default paths
// (/etc/ssh/ssh_host_ed25519_key(+-cert.pub)). With ssh_deletekeys at its
// default (true), cc_ssh deletes the image's baked host keys and, because a
// ssh_keys map is present, installs ONLY our ed25519 key — no rsa/ecdsa are
// regenerated. Result: one per-VM CA-certified ed25519 host key, no shared
// image keys (the intended tidy-up). This suits eitri's cert-only access.
// ssh_keys is a top-level key, only emitted when we inject a host key.
b.WriteString("ssh_keys:\n")
writeBlockScalar(&b, " ", "ed25519_private", p.SSHHostKeyPEM)
writeBlockScalar(&b, " ", "ed25519_certificate", p.SSHHostCert)
}
dropIn := sshdDropIn(p)
// write_files is unconditional: the clock drop-in is always written.
b.WriteString("write_files:\n")
if p.SSHUserCAAuthorizedKey != "" {
writeFileBlock(&b, userCAPath, p.SSHUserCAAuthorizedKey)
}
if dropIn != "" {
writeFileBlock(&b, dropInPath, dropIn)
}
writeFileBlock(&b, chronyDropInPath, chronyDropIn)
b.WriteString("runcmd:\n")
b.WriteString(growRuncmd)
if dropIn != "" {
// -t gates the reload: a bad config won't reload, so no lockout.
b.WriteString(" - [\"sh\", \"-c\", \"sshd -t && systemctl reload sshd\"]\n")
}
b.WriteString(chronyRuncmd)
return b.String()
}
// writeBlockScalar appends a `key: |` mapping entry (at the given indent) whose
// value is a YAML literal block scalar, so multi-line content (e.g. a PEM) lands
// verbatim. Value lines are indented two spaces past the key. Used for the
// cloud-init ssh_keys map (ed25519_private / ed25519_certificate).
func writeBlockScalar(b *strings.Builder, indent, key, content string) {
b.WriteString(indent + key + ": |\n")
for line := range strings.SplitSeq(strings.TrimRight(content, "\n"), "\n") {
b.WriteString(indent + " " + line + "\n")
}
}
// writeFileBlock appends a write_files entry (mode 0644) whose content is the
// given bytes, rendered as a YAML literal block scalar so the bytes land in the
// guest file verbatim. Used for the public user-CA key and the sshd drop-in —
// both public trust material, so 0644 is correct.
func writeFileBlock(b *strings.Builder, path, content string) {
b.WriteString(" - path: " + path + "\n")
b.WriteString(" permissions: '0644'\n")
b.WriteString(" content: |\n")
for line := range strings.SplitSeq(strings.TrimRight(content, "\n"), "\n") {
b.WriteString(" " + line + "\n")
}
}
// userData returns the cloud-config string. If p.UserData is non-empty it is
// returned verbatim (advanced users own their user-data). Otherwise a sensible
// default is generated with an SSH key and a default ubuntu user.
//
// The default does NOT configure growpart or disk_setup: growing the root to
// disk_gb is eitri's job and lives in eitri's own cloud-config part (see
// eitriConfigDoc/growRuncmd), which applies to BYO user-data too. Setting
// growpart here would also win over that part's `growpart: off` — a scalar the
// tenant's document set is never overwritten (see mergeHow) — re-arming the
// image growpart that reverts on this disk.
func userData(p Params) string {
if p.UserData != "" {
return p.UserData
}
// Render ssh_authorized_keys only when a key is supplied. Emitting the key
// with no value produces a null list item ("- ") that cloud-init rejects
// (users.0.ssh_authorized_keys.0: None is not of type 'string') and runs
// degraded, so omit the whole mapping when SSHAuthorizedKey is empty.
sshKeys := ""
if p.SSHAuthorizedKey != "" {
sshKeys = fmt.Sprintf("\n ssh_authorized_keys:\n - %s", p.SSHAuthorizedKey)
}
return fmt.Sprintf(`#cloud-config
hostname: %s
users:
- name: %s
sudo: ALL=(ALL) NOPASSWD:ALL
shell: /bin/bash%s
`, p.Hostname, guest.LoginUser, sshKeys)
}
// composeUserData builds the guest's whole user-data: the tenant's document
// (or eitri's default when they brought none) first, then eitri's own
// cloud-config appended as the last part of a MIME multipart archive. Both
// halves reach the guest, and the tenant's bytes are never edited.
//
// One mechanism, not two. eitri's config used to ride in a separate vendor-data
// file, which cloud-init merges against user-data with mergers that no document
// can influence — first-wins per key, so a tenant list quietly replaced eitri's
// (see mergeHow). Inside ONE data source the parts are folded together under the
// mergers the incoming part names, so this is the only arrangement in which
// eitri's merge directive is read at all.
//
// It can fail: a jinja-templated or gzipped tenant document cannot be given an
// honest MIME part header, so there is nowhere to put eitri's config. Refusing
// to build the seed is the point — the guest that would boot instead is one
// with no CA trust file, reachable by nobody and silent about why.
func composeUserData(p Params) (string, error) {
out, err := cloudinit.AppendCloudConfig(userData(p), eitriConfigDoc(p))
if err != nil {
return "", fmt.Errorf("seed: %w", err)
}
return out, nil
}
// metaData returns the cloud-init meta-data content.
// InstanceID is used when set; falls back to Hostname for backward compatibility.
func metaData(p Params) string {
id := p.InstanceID
if id == "" {
id = p.Hostname
}
return fmt.Sprintf("instance-id: %s\nlocal-hostname: %s\n", id, p.Hostname)
}
// networkConfig returns a netplan v2 network-config that DHCPs on the primary
// NIC. The address is served by whichever DHCP server the host runs — the
// agent's own responder on Linux, macOS's bootpd under vmnet — so nothing about
// addressing is baked into the guest image. DNS comes from the same place.
//
// dhcp-identifier: mac makes the guest identify itself by its hardware address
// instead of the DUID systemd-networkd would otherwise invent. It is load-
// bearing on macOS and free on Linux. eitri picks a VM's MAC deterministically
// and that is the key everything about addressing hangs off: the agent's
// responder looks up reservations by it, and on macOS it is the only handle we
// have on a lease we did not grant. A guest that identifies as a DUID is a
// guest whose lease is filed under a name the host never chose — bootpd records
// it under the identifier, so the address becomes unfindable, and worse, an
// earlier MAC-keyed offer can linger and be read as current. Linux is unharmed
// because our responder keys on the packet's hardware address either way.
//
// A guest with a second NIC on a named host network gets a second stanza, and
// then BOTH are matched by MAC rather than by name: two NICs cannot share one
// name glob (netplan refuses a link claimed by two definitions), and the MACs
// are the only thing the host and the guest already agree on for each NIC
// independently. Route metrics decide the default route explicitly — 100 for
// the NAT NIC, 200 for the named one — so egress is deterministic and a LAN
// outage costs reachability, not the guest's outbound.
func networkConfig(p Params) string {
if p.NetworkMAC == "" {
return `network:
version: 2
ethernets:
primary:
match:
name: "en*"
dhcp4: true
dhcp-identifier: mac
dhcp4-overrides:
route-metric: 100
`
}
return fmt.Sprintf(`network:
version: 2
ethernets:
primary:
match:
macaddress: "%s"
dhcp4: true
dhcp-identifier: mac
dhcp4-overrides:
route-metric: 100
net1:
match:
macaddress: "%s"
dhcp4: true
dhcp-identifier: mac
dhcp4-overrides:
route-metric: 200
`, p.MAC, p.NetworkMAC)
}
// Build creates a cloud-init NoCloud seed ISO at outPath.
// The ISO uses volume label "cidata" as required by cloud-init's NoCloud source.
// Files written: /user-data, /meta-data, /network-config. eitri's own config is
// the last part of /user-data rather than a /vendor-data file (see
// composeUserData), so a seed carries three files and no fourth.
func Build(outPath string, p Params) error {
if err := validateParams(p); err != nil {
return err
}
// Composed before anything is created, so a tenant document eitri cannot
// compose with fails without leaving an artifact behind.
userDataDoc, err := composeUserData(p)
if err != nil {
return err
}
// 1 MiB covers the text files plus slack for ISO9660 structures.
isoSize := int64(1 * 1024 * 1024)
// ISO9660 requires 2048-byte logical block size; diskfs.SectorSize512 (default) would fail.
const isoSectorSize diskfs.SectorSize = 2048
// Workspace dir for iso9660 staging; cleaned up after Finalize writes to outPath.
workDir, err := os.MkdirTemp("", "eitri-seed-")
if err != nil {
return fmt.Errorf("seed: create workspace: %w", err)
}
defer os.RemoveAll(workDir)
// Build into a sibling temp file and rename into place, so a Build that fails
// or is killed part-way (MkdirTemp, filesystem create, the write loop, or
// Finalize) can never leave a partial/empty seed.iso at the final path. The
// reconcile "exists" gate keys off rec.BootID rather than seed presence, but
// an atomic artifact keeps on-disk state honest. The deferred guard below
// closes the image (if not already closed) and drops the temp on any error
// path; on success we close explicitly before the rename so the ISO is fully
// flushed, and the guard is disarmed.
tmpPath := outPath + ".partial"
_ = os.Remove(tmpPath)
d, err := diskfs.Create(tmpPath, isoSize, isoSectorSize)
if err != nil {
return fmt.Errorf("seed: create disk image: %w", err)
}
// success and closed track how far Build got. While d is still open, the
// guard closes it before removing the temp file; once Close has been
// attempted (success or fail), it must not be called again — the guard
// then just removes the temp file, matching the two distinct cleanups the
// manual version used at the close-failure and rename-failure sites.
success := false
closed := false
defer func() {
if success {
return
}
if !closed {
d.Close()
}
_ = os.Remove(tmpPath)
}()
fsi, err := d.CreateFilesystem(disk.FilesystemSpec{
Partition: 0,
FSType: filesystem.TypeISO9660,
WorkDir: workDir,
})
if err != nil {
return fmt.Errorf("seed: create iso9660 filesystem: %w", err)
}
fs, ok := fsi.(*iso9660.FileSystem)
if !ok {
return fmt.Errorf("seed: unexpected filesystem type %T", fsi)
}
// No vendor-data: eitri's own config rides as the last part of user-data,
// which is the only place cloud-init reads a merge directive from (see
// composeUserData).
files := map[string]string{
"/user-data": userDataDoc,
"/meta-data": metaData(p),
"/network-config": networkConfig(p),
}
for name, content := range files {
f, err := fs.OpenFile(name, os.O_CREATE|os.O_RDWR)
if err != nil {
return fmt.Errorf("seed: open %s: %w", name, err)
}
if _, err := f.Write([]byte(content)); err != nil {
return fmt.Errorf("seed: write %s: %w", name, err)
}
}
// RockRidge extensions preserve lowercase names and hyphens (e.g. "user-data").
// Without RockRidge, ISO9660 level-1 would mangle "user-data" → "USER_DAT".
if err := fs.Finalize(iso9660.FinalizeOptions{
VolumeIdentifier: "cidata",
RockRidge: true,
}); err != nil {
return fmt.Errorf("seed: finalize iso: %w", err)
}
closeErr := d.Close()
closed = true
if closeErr != nil {
return fmt.Errorf("seed: close iso: %w", closeErr)
}
if err := os.Rename(tmpPath, outPath); err != nil {
return fmt.Errorf("seed: rename %s -> %s: %w", tmpPath, outPath, err)
}
success = true
return nil
}