scripts/devhost.sh

Ref: main v0.0.5   Size: 16.1 KiB   History

#!/usr/bin/env bash
#
# The branch gate's fleet: a nested KVM host, defined here rather than
# hand-built, so a wedged dev host is a three-minute rebuild instead of a
# debugging session.
#
#   scripts/devhost.sh create  [name]   build it from nothing
#   scripts/devhost.sh recycle [name]   destroy, then create — the whole point
#   scripts/devhost.sh destroy [name]
#   scripts/devhost.sh address [name]   print its AGENT_HOSTS line
#
# The host runs on the dev machine rather than the workstation:
#
#   DEVHOST_REMOTE=ubuntu@192.168.0.193:2222 scripts/devhost.sh create
#
# With DEVHOST_REMOTE set the script copies itself and the seed there and
# re-execs under sudo, so every hypervisor call — libvirt, qemu-img, apt —
# happens on the machine that owns the guest, and the workstation stays free of
# both root and libvirt. Output streams back; the exit status is the remote's.
# Unset, the same four verbs build the host on this machine against the libvirt
# 'default' network.
#
# The gate's fleet has no coupling to any hosted thing and dies the moment
# `make deploy` is not running. Nested guests get hardware virtualization
# through --cpu host-passthrough, so the machine's CPU must expose nested KVM
# (kvm_amd nested=1 / kvm_intel nested=1). What nested CANNOT prove is bridged
# networking and real-hardware quirks — that coverage moved to stg, which gates
# pre-release tags on real metal.
#
# On the dev machine the guest is a macvtap child of the LAN uplink
# (DEVHOST_UPLINK, default enp4s0): the LAN's router leases it an address and
# the workstation dials it directly. macvtap isolates a guest from its own
# host by design, so the dev machine cannot reach the guest it runs — which
# costs nothing here, because every gate connection comes from the workstation
# anyway. It does mean the address is discovered after boot rather than
# assigned: the guest agent reports it, and this script waits for it.
#
# The seed installs nothing eitri needs: cloud-hypervisor, the guest firmware
# and the agent all arrive through `make deploy` and the agent's own bootstrap,
# so a cold first deploy exercises that path rather than skipping it.
#
# Disks live in the libvirt system pool, which is root-owned — the qemu-img and
# rm calls below use sudo for that and nothing else.
set -euo pipefail

# The pool dir and the 'default' network live on the SYSTEM daemon; a bare
# virsh from a user shell talks to qemu:///session and sees neither.
export LIBVIRT_DEFAULT_URI="${LIBVIRT_DEFAULT_URI:-qemu:///system}"

REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
SCRIPT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/$(basename "${BASH_SOURCE[0]}")"

# Pinned guest image. Bump deliberately: take the sha from the SHA256SUMS file
# beside the image in the same dated directory.
IMAGE_URL="${DEVHOST_IMAGE_URL:-https://cloud-images.ubuntu.com/noble/20260801/noble-server-cloudimg-amd64.img}"
IMAGE_SHA256="${DEVHOST_IMAGE_SHA256:-0533b0655c32e68b31d792ecd6ccfca95abdbc536c4446874fe0513bd4140ffe}"
OS_VARIANT="${DEVHOST_OS_VARIANT:-ubuntu24.04}"

# Sized so the agent's admission logic runs against a cap BELOW the machine's
# real size, the same shape a bare-metal host runs — which keeps the
# admission path honestly covered. The caps themselves go in deploy.env's
# AGENT_EXTRA_FLAGS; see the line this script prints.
MEM_MB="${DEVHOST_MEM_MB:-16384}"
VCPUS="${DEVHOST_VCPUS:-8}"
DISK_GB="${DEVHOST_DISK_GB:-120}"

NETWORK="${DEVHOST_NETWORK:-default}"
POOL_DIR="${DEVHOST_POOL_DIR:-/var/lib/libvirt/images}"
CACHE_DIR="${DEVHOST_CACHE_DIR:-$HOME/.cache/eitri/devhost}"
SSH_KEY="${DEVHOST_SSH_KEY:-$HOME/.ssh/id_ed25519.pub}"
SSH_PORT="${DEVHOST_SSH_PORT:-22}"
SEED_TEMPLATE="${DEVHOST_SEED:-$REPO_ROOT/scripts/devhost.cloud-init.yaml}"

# Where the host is built. REMOTE empty means this machine. MANAGED is set only
# on the copy that lands on the dev machine, and says two things: this machine
# is ours to prepare, and something upstream is waiting to print the result.
REMOTE="${DEVHOST_REMOTE:-}"
MANAGED="${DEVHOST_MANAGED:-}"

# A non-empty uplink switches the guest onto macvtap over that interface; empty
# keeps it on the libvirt network. Driving a remote implies the former, because
# a dev machine is a LAN citizen and its guest should be one too.
UPLINK="${DEVHOST_UPLINK:-}"
if [[ -n "$REMOTE" && -z "$UPLINK" ]]; then
	UPLINK=enp4s0
fi

# The key the guest must admit is the workstation's, wherever the guest is
# built: the driving side reads the file and hands the material over, so the
# dev machine never needs a key of its own.
SSH_PUBKEY="${DEVHOST_SSH_PUBKEY:-}"

bold() { printf '\n\033[1;36m==> %s\033[0m\n' "$*"; }
die() { printf 'devhost: %s\n' "$*" >&2; exit 1; }

VERB="${1:-}"
NAME="${2:-eitri-dev1}"

case "$VERB" in
create | recycle | destroy | address) ;;
*)
	sed -n '2,21p' "$0" | sed 's/^#\{1,2\} \{0,1\}//' >&2
	exit 1
	;;
esac

RTARGET=""
RPORT=22
if [[ -n "$REMOTE" ]]; then
	RTARGET="${REMOTE%:*}"
	if [[ "$REMOTE" == *:* ]]; then
		RPORT="${REMOTE##*:}"
	fi
	[[ "$RTARGET" == *@* ]] || die "DEVHOST_REMOTE must read user@host[:port] (got '$REMOTE')"
fi

# Where to look when a host will not answer. Named early because half the
# failure paths below want to say it.
CONSOLE_HINT="virsh console $NAME"
if [[ -n "$REMOTE" ]]; then
	CONSOLE_HINT="ssh -p $RPORT $RTARGET sudo virsh console $NAME"
fi

# The MAC is derived from the name, so a recycled host comes back the same
# machine to the router and to libvirt. Locally-administered QEMU prefix plus
# three bytes of the name's digest: stable per name, and distinct enough not to
# collide with anything else on the segment.
mac_tail="$(printf '%s' "$NAME" | sha256sum | cut -c1-6 | sed 's/../&:/g; s/:$//')"
MAC="${DEVHOST_MAC:-52:54:00:$mac_tail}"
DISK="$POOL_DIR/$NAME.qcow2"

# On the libvirt network the address comes from the name too, via a reservation
# this script owns, so AGENT_HOSTS never has to be edited. The index is the
# trailing digits of the name ("eitri-dev2" -> 2 -> .51). On the LAN uplink the
# router owns DHCP, so the address is whatever it leased and is discovered.
IP="${DEVHOST_IP:-}"
if [[ -z "$UPLINK" && -z "$IP" ]]; then
	index="$(printf '%s' "$NAME" | grep -oE '[0-9]+$' || echo 1)"
	IP="192.168.122.$((49 + index))"
fi

need() { command -v "$1" >/dev/null 2>&1 || die "$1 is not installed"; }

# The dev machine gets the hypervisor toolchain the first time it is asked for
# it, and nothing at all on every run after that. Only ever on a machine this
# script was copied to: the workstation's package set is the workstation's own
# business.
ensure_prereqs() {
	[[ -n "$MANAGED" ]] || return 0

	missing=()
	for cmd in virsh virt-install qemu-img curl; do
		command -v "$cmd" >/dev/null 2>&1 || missing+=("$cmd")
	done
	if ((${#missing[@]} > 0)); then
		# Named rather than pulled in by recommendation, and every one of them
		# is used: the emulator, qemu-img, the daemon, virsh, virt-install. A
		# guest on the LAN uplink needs no NAT network, so nothing here has to
		# drag in dnsmasq or the bridge tooling to serve one.
		bold "Installing the hypervisor toolchain (missing: ${missing[*]})"
		sudo -n env DEBIAN_FRONTEND=noninteractive apt-get update -qq
		sudo -n env DEBIAN_FRONTEND=noninteractive apt-get install -y \
			--no-install-recommends qemu-system-x86 qemu-utils \
			libvirt-daemon-system libvirt-clients virtinst curl
	fi
	if systemctl is-active --quiet libvirtd; then
		echo "libvirtd is running"
	else
		bold "Starting libvirtd"
		sudo -n systemctl enable --now libvirtd
	fi
}

# The script is its own payload. It and the seed go to the dev machine's mktemp
# dir and are re-exec'd there as root, which is what lets a single ssh session
# do libvirt, the pool and apt without the driving side holding any privilege.
# The remote's stdout is teed so the address it discovers can be read back;
# its stderr passes straight through untouched.
run_remote() {
	if [[ "$VERB" == create || "$VERB" == recycle ]]; then
		[[ -f "$SSH_KEY" ]] || die "no public key at $SSH_KEY (set DEVHOST_SSH_KEY)"
		SSH_PUBKEY="$(cat "$SSH_KEY")"
	fi

	# ServerAliveInterval, because a create is ten quiet minutes of image
	# fetching and first boot on the far end, and a session that times out
	# during them abandons a guest that is already running.
	ssh_opts=(-o BatchMode=yes -o ConnectTimeout=10 -o ServerAliveInterval=30)
	tmp="$(ssh -p "$RPORT" "${ssh_opts[@]}" "$RTARGET" 'mktemp -d /tmp/devhost.XXXXXX')" ||
		die "$REMOTE did not answer ssh"
	scp -P "$RPORT" -q "${ssh_opts[@]}" "$SCRIPT" "$SEED_TEMPLATE" "$RTARGET:$tmp/" ||
		die "could not copy the script to $REMOTE"

	envs=(
		"DEVHOST_REMOTE="
		"DEVHOST_MANAGED=1"
		"DEVHOST_SEED=$tmp/$(basename "$SEED_TEMPLATE")"
		"DEVHOST_UPLINK=$UPLINK"
		"DEVHOST_MAC=$MAC"
		"DEVHOST_MEM_MB=$MEM_MB"
		"DEVHOST_VCPUS=$VCPUS"
		"DEVHOST_DISK_GB=$DISK_GB"
		"DEVHOST_IMAGE_URL=$IMAGE_URL"
		"DEVHOST_IMAGE_SHA256=$IMAGE_SHA256"
		"DEVHOST_OS_VARIANT=$OS_VARIANT"
		"DEVHOST_POOL_DIR=$POOL_DIR"
	)
	if [[ -n "$SSH_PUBKEY" ]]; then
		envs+=("DEVHOST_SSH_PUBKEY=$SSH_PUBKEY")
	fi
	if [[ -n "$IP" ]]; then
		envs+=("DEVHOST_IP=$IP")
	fi
	# The image cache belongs to whichever machine fetches, so it is forwarded
	# only when someone has said where they want it.
	if [[ -n "${DEVHOST_CACHE_DIR:-}" ]]; then
		envs+=("DEVHOST_CACHE_DIR=$DEVHOST_CACHE_DIR")
	fi

	cmd="sudo -n env"
	for assignment in "${envs[@]}"; do
		cmd+=" $(printf '%q' "$assignment")"
	done
	cmd+=" bash $tmp/$(basename "$SCRIPT") $(printf '%q %q' "$VERB" "$NAME")"
	# The temp dir goes whatever the run did, and the run's status survives it.
	cmd="$cmd; rc=\$?; rm -rf $tmp; exit \$rc"

	bold "Driving $VERB of $NAME on $RTARGET"
	out="$(mktemp)"
	trap 'rm -f "$out"' EXIT
	if ssh -p "$RPORT" "${ssh_opts[@]}" "$RTARGET" "$cmd" | tee "$out"; then
		rc=0
	else
		rc=$?
	fi
	[[ $rc -eq 0 ]] || exit "$rc"

	case "$VERB" in
	create | recycle | address)
		IP="$(sed -n 's/^devhost-address: //p' "$out" | tail -n1)"
		[[ -n "$IP" ]] || die "$RTARGET never reported an address for $NAME"
		;;
	esac
	rm -f "$out"
	trap - EXIT

	case "$VERB" in
	create | recycle)
		wait_for_ssh
		print_address
		;;
	address) print_address ;;
	esac
}

destroy_host() {
	bold "Destroying $NAME"
	if command -v virsh >/dev/null 2>&1; then
		virsh destroy "$NAME" >/dev/null 2>&1 || true
		# --remove-all-storage takes the root disk and the cloud-init seed ISO
		# with the domain; the explicit rm is the belt for a domain that never
		# defined.
		virsh undefine "$NAME" --nvram --remove-all-storage >/dev/null 2>&1 || true
	fi
	sudo rm -f "$DISK"
	if [[ -n "$UPLINK" ]]; then
		echo "$NAME is gone"
	else
		# The DHCP reservation deliberately survives: it is what makes the
		# address stable across a recycle, and create re-adds it either way.
		echo "$NAME is gone (its reservation at $IP is kept)"
	fi
}

create_host() {
	ensure_prereqs
	need virsh
	need virt-install
	need qemu-img
	[[ -e /dev/kvm ]] || die "/dev/kvm is missing — nested guests need hardware virtualization"
	if [[ -z "$SSH_PUBKEY" ]]; then
		[[ -f "$SSH_KEY" ]] || die "no public key at $SSH_KEY (set DEVHOST_SSH_KEY)"
		SSH_PUBKEY="$(cat "$SSH_KEY")"
	fi
	if [[ -n "$UPLINK" ]]; then
		ip link show "$UPLINK" >/dev/null 2>&1 ||
			die "no interface '$UPLINK' on this machine (set DEVHOST_UPLINK)"
	else
		virsh net-info "$NETWORK" >/dev/null 2>&1 || die "libvirt network '$NETWORK' does not exist"
		# grep consumes all input on purpose: under pipefail, grep -q's early exit
		# EPIPEs the writer and fails the pipeline even when the match succeeded.
		virsh net-info "$NETWORK" | grep 'Active:.*yes' >/dev/null || die "libvirt network '$NETWORK' is not running (virsh net-start $NETWORK)"
	fi
	if virsh dominfo "$NAME" >/dev/null 2>&1; then
		die "$NAME already exists — 'recycle' rebuilds it, 'destroy' removes it"
	fi

	bold "Fetching the guest image"
	mkdir -p "$CACHE_DIR"
	cached="$CACHE_DIR/$(basename "$IMAGE_URL")"
	if [[ ! -f "$cached" ]]; then
		curl -fsSL -o "$cached.part" "$IMAGE_URL"
		mv "$cached.part" "$cached"
	fi
	echo "$IMAGE_SHA256  $cached" | sha256sum -c - >/dev/null || {
		rm -f "$cached"
		die "image sha mismatch — cache purged, re-run"
	}
	echo "image ok: $cached"

	if [[ -n "$UPLINK" ]]; then
		netarg="type=direct,source=$UPLINK,source_mode=bridge,model=virtio,mac=$MAC"
		bold "Putting $NAME on the LAN through $UPLINK ($MAC)"
	else
		netarg="network=$NETWORK,mac=$MAC,model=virtio"
		bold "Pinning $NAME to $IP on the '$NETWORK' network"
		virsh net-update "$NETWORK" delete ip-dhcp-host "<host mac='$MAC'/>" \
			--live --config >/dev/null 2>&1 || true
		virsh net-update "$NETWORK" add ip-dhcp-host \
			"<host mac='$MAC' name='$NAME' ip='$IP'/>" --live --config >/dev/null
	fi

	bold "Creating $NAME ($VCPUS vCPU, $((MEM_MB / 1024)) GB, $DISK_GB GB)"
	sudo qemu-img convert -O qcow2 "$cached" "$DISK"
	sudo qemu-img resize "$DISK" "${DISK_GB}G" >/dev/null

	seed="$(mktemp)"
	trap 'rm -f "$seed"' EXIT
	template="$(cat "$SEED_TEMPLATE")"
	printf '%s\n' "${template//'${SSH_AUTHORIZED_KEY}'/$SSH_PUBKEY}" >"$seed"

	virt-install \
		--name "$NAME" \
		--memory "$MEM_MB" \
		--vcpus "$VCPUS" \
		--cpu host-passthrough \
		--disk "path=$DISK,format=qcow2,bus=virtio" \
		--network "$netarg" \
		--os-variant "$OS_VARIANT" \
		--cloud-init "user-data=$seed" \
		--graphics none \
		--noautoconsole \
		--import
	rm -f "$seed"
	trap - EXIT

	if [[ -n "$UPLINK" ]]; then
		# Discovery waits on the guest agent, which the seed installs on first
		# boot. DEVHOST_IP is the way past it for a guest the router already has
		# a reservation for: the address is known before the machine exists.
		if [[ -z "$IP" ]]; then
			IP="$(await_address)"
		fi
		# macvtap keeps this machine off its own guest, so the ssh proof belongs
		# to whoever drives this script. Report the address and let them make it.
		printf 'devhost-address: %s\n' "$IP"
		if [[ -z "$MANAGED" ]]; then
			print_address
		fi
	else
		wait_for_ssh
		print_address
	fi
}

# The guest's address, from the first source that knows it. Under macvtap the
# host has no path to its own guest, so the ARP table never learns it and
# libvirt owns no lease to read — the guest agent's virtio channel is the one
# source that answers. The other two are kept for a host whose guest shares its
# L2, where they answer sooner.
domain_ipv4() {
	for src in agent arp; do
		found="$(virsh domifaddr "$NAME" --source "$src" 2>/dev/null |
			awk '{
				for (i = 1; i < NF; i++)
					if ($i == "ipv4") {
						split($(i + 1), a, "/")
						if (a[1] !~ /^(127\.|169\.254\.)/ && !ip) ip = a[1]
					}
			} END { if (ip) print ip }' || true)"
		if [[ -n "$found" ]]; then
			printf '%s' "$found"
			return 0
		fi
	done
	ip -4 neigh show 2>/dev/null |
		awk -v mac="$MAC" 'BEGIN { mac = tolower(mac) }
			tolower($0) ~ mac && !ip { ip = $1 } END { if (ip) print ip }' || true
}

await_address() {
	bold "Waiting for $NAME to report the address the LAN leased it" >&2
	for _ in $(seq 1 90); do
		found="$(domain_ipv4)"
		if [[ -n "$found" ]]; then
			printf '%s' "$found"
			return 0
		fi
		sleep 5
	done
	die "$NAME never reported an address ($CONSOLE_HINT)"
}

wait_for_ssh() {
	# A rebuilt host presents a new host key at an address ssh already knows,
	# which otherwise fails the first deploy with a warning that looks like an
	# attack rather than a recycle.
	ssh-keygen -R "$IP" >/dev/null 2>&1 || true

	bold "Waiting for ssh on $IP"
	for _ in $(seq 1 60); do
		if ssh -p "$SSH_PORT" -o BatchMode=yes -o StrictHostKeyChecking=accept-new \
			-o ConnectTimeout=5 "ubuntu@$IP" true 2>/dev/null; then
			return 0
		fi
		sleep 5
	done
	die "$NAME did not answer ssh at $IP within 5 minutes ($CONSOLE_HINT)"
}

print_address() {
	cat <<-EOF

		$NAME is up. Point the branch gate at it in ~/eitri-deploy/deploy.env:

		  AGENT_HOSTS="ubuntu@$IP"
		  AGENT_EXTRA_FLAGS="--max-vcpus $((VCPUS - 2)) --max-mem-mb $((MEM_MB - 4096)) --max-disk-gb $((DISK_GB - 30))"

		then run: make deploy
	EOF
}

if [[ -n "$REMOTE" && -z "$MANAGED" ]]; then
	run_remote
	exit 0
fi

case "$VERB" in
create) create_host ;;
destroy) destroy_host ;;
recycle)
	destroy_host
	create_host
	;;
address)
	if [[ -n "$UPLINK" ]]; then
		if [[ -z "$IP" ]]; then
			IP="$(domain_ipv4)"
		fi
		[[ -n "$IP" ]] || die "$NAME has no address to report ($CONSOLE_HINT)"
		printf 'devhost-address: %s\n' "$IP"
		if [[ -z "$MANAGED" ]]; then
			print_address
		fi
	else
		print_address
	fi
	;;
esac