857a4547

feat(agent): run fleet agents under systemd

a73x 2026-07-26 13:06

Commit message

feat(agent): run fleet agents under systemd

scripts/eitri-agent.service: Restart=on-failure revives a crashed agent;
KillMode=process keeps guests (cloud-hypervisor processes in the unit's
cgroup) alive across agent stops — the control-group default would tear
down every running VM. Flags ride /etc/default/eitri-agent, and agent
self-upgrades re-exec in place keeping the PID, so systemd sees one
uninterrupted service.

Deploy ships and converges the unit on every roll — flags, a GOCOVERDIR
drop-in for the coverage boot-gate — and drives the agent with
systemctl; logs live in journald (journalctl -u eitri-agent). The stop
path handles both the unit and a pre-unit setsid agent, so a host adopts
systemd on its first roll.

README.md

Old		New
@@ -75,7 +75,9 @@ make build # binaries into ./bin
75	Enrolling a real host. The server mints a single-paste join blob; on the host,	75	Enrolling a real host. The server mints a single-paste join blob; on the host,
76	`eitri-agent join <blob>` enrolls it (posting to `/api/v1/enroll`), pins the	76	`eitri-agent join <blob>` enrolls it (posting to `/api/v1/enroll`), pins the
77	server certificate from the blob, and persists its identity. From then on the	77	server certificate from the blob, and persists its identity. From then on the
78	agent runs the reconcile + sync loop against the fleet.	78	agent runs the reconcile + sync loop against the fleet. Run it under systemd
		79	with `scripts/eitri-agent.service` — `Restart=on-failure` revives a crashed
		80	agent, and its `KillMode=process` keeps running VMs alive across agent stops.
79		81
80	Guests boot from cloud images (the default is Ubuntu resolute) via UEFI firmware	82	Guests boot from cloud images (the default is Ubuntu resolute) via UEFI firmware
81	(`CLOUDHV.fd`) shipped to each host, so the guest owns its kernel and any	83	(`CLOUDHV.fd`) shipped to each host, so the guest owns its kernel and any

scripts/deploy.env.example

Old		New
@@ -17,7 +17,8 @@ ADMIN_TOKEN_FILE="$HOME/eitri-deploy/admin-token" # optional: enables API verif
17	AGENT_HOSTS="ubuntu@192.168.0.193:2222"	17	AGENT_HOSTS="ubuntu@192.168.0.193:2222"
18	AGENT_BIN="/usr/local/bin/eitri-agent" # install destination on each host	18	AGENT_BIN="/usr/local/bin/eitri-agent" # install destination on each host
19	AGENT_STATE_DIR="/var/lib/eitri-agent"	19	AGENT_STATE_DIR="/var/lib/eitri-agent"
20	AGENT_LOG="/var/lib/eitri-agent/agent.log"	20	# Agent logs live in journald once the systemd unit is adopted:
		21	# journalctl -u eitri-agent
21		22
22	# Agent launch flags shared by all hosts.	23	# Agent launch flags shared by all hosts.
23	CH_BIN="/usr/local/bin/cloud-hypervisor"	24	CH_BIN="/usr/local/bin/cloud-hypervisor"

scripts/deploy.sh

Old		New
@@ -26,7 +26,7 @@ fi
26	source "$ENV_FILE"	26	source "$ENV_FILE"
27		27
28	: "${SERVER_BIN:?set in $ENV_FILE}" "${SERVER_CONFIG:?}" "${SERVER_LOG:?}" "${SERVER_URL:?}"	28	: "${SERVER_BIN:?set in $ENV_FILE}" "${SERVER_CONFIG:?}" "${SERVER_LOG:?}" "${SERVER_URL:?}"
29	: "${AGENT_HOSTS:?}" "${AGENT_BIN:?}" "${AGENT_STATE_DIR:?}" "${AGENT_LOG:?}"	29	: "${AGENT_HOSTS:?}" "${AGENT_BIN:?}" "${AGENT_STATE_DIR:?}"
30	: "${CH_BIN:?}" "${FIRMWARE:?}"	30	: "${CH_BIN:?}" "${FIRMWARE:?}"
31		31
32	REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"	32	REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
@@ -120,6 +120,7 @@ for entry in $AGENT_HOSTS; do
120	port="${entry##*:}"; [[ "$port" == "$entry" ]] && port=22	120	port="${entry##*:}"; [[ "$port" == "$entry" ]] && port=22
121	bold "Rolling agent -> $userhost (port $port)"	121	bold "Rolling agent -> $userhost (port $port)"
122	scp -q -P "$port" -o ConnectTimeout=10 bin/eitri-agent "$userhost:/tmp/eitri-agent-new"	122	scp -q -P "$port" -o ConnectTimeout=10 bin/eitri-agent "$userhost:/tmp/eitri-agent-new"
		123	scp -q -P "$port" -o ConnectTimeout=10 scripts/eitri-agent.service "$userhost:/tmp/eitri-agent.service"
123	# Optionally ship the guest firmware (e.g. CLOUDHV.fd) so a firmware change	124	# Optionally ship the guest firmware (e.g. CLOUDHV.fd) so a firmware change
124	# rolls WITH the deploy instead of needing a manual per-host install. When	125	# rolls WITH the deploy instead of needing a manual per-host install. When
125	# FIRMWARE_SRC is unset, hosts keep whatever firmware is already at $FIRMWARE.	126	# FIRMWARE_SRC is unset, hosts keep whatever firmware is already at $FIRMWARE.
@@ -131,17 +132,32 @@ for entry in $AGENT_HOSTS; do
131	# Unquoted heredoc: local vars expand here; \$(...) runs on the remote.	132	# Unquoted heredoc: local vars expand here; \$(...) runs on the remote.
132	ssh -p "$port" -o BatchMode=yes "$userhost" bash -s <<REMOTE	133	ssh -p "$port" -o BatchMode=yes "$userhost" bash -s <<REMOTE
133	set -e	134	set -e
		135	# Stop whichever agent shape is running: the systemd unit if adopted, plus any
		136	# pre-unit setsid agent. SIGTERM is a "clean" signal to systemd, so a
		137	# unit-managed agent is not resurrected by Restart=on-failure mid-roll.
		138	sudo systemctl stop eitri-agent 2>/dev/null \|\| true
134	sudo kill -TERM \$(pgrep -x eitri-agent) 2>/dev/null \|\| true	139	sudo kill -TERM \$(pgrep -x eitri-agent) 2>/dev/null \|\| true
135	for _ in \$(seq 1 20); do pgrep -x eitri-agent >/dev/null \|\| break; sleep 0.5; done	140	for _ in \$(seq 1 20); do pgrep -x eitri-agent >/dev/null \|\| break; sleep 0.5; done
136	if pgrep -x eitri-agent >/dev/null; then echo "agent did not stop on \$(hostname)" >&2; exit 1; fi	141	if pgrep -x eitri-agent >/dev/null; then echo "agent did not stop on \$(hostname)" >&2; exit 1; fi
137	sudo install -m 0755 /tmp/eitri-agent-new "$AGENT_BIN"	142	sudo install -m 0755 /tmp/eitri-agent-new "$AGENT_BIN"
138	rm -f /tmp/eitri-agent-new	143	rm -f /tmp/eitri-agent-new
139	$fw_install	144	$fw_install
		145	# The agent runs under systemd. Deploy converges the unit, its flags, and the
		146	# coverage env on every roll so hosts cannot drift from the repo's unit file;
		147	# agent logs live in journald (journalctl -u eitri-agent).
		148	sudo install -m 0644 /tmp/eitri-agent.service /etc/systemd/system/eitri-agent.service
		149	rm -f /tmp/eitri-agent.service
		150	echo "EITRI_AGENT_FLAGS=\"$agent_flags\"" \| sudo tee /etc/default/eitri-agent >/dev/null
140	sudo rm -rf "$AGENT_GOCOVERDIR"; sudo mkdir -p "$AGENT_GOCOVERDIR"	151	sudo rm -rf "$AGENT_GOCOVERDIR"; sudo mkdir -p "$AGENT_GOCOVERDIR"
141	sudo bash -c "GOCOVERDIR=$AGENT_GOCOVERDIR setsid $AGENT_BIN $agent_flags </dev/null >>$AGENT_LOG 2>&1 &"	152	sudo mkdir -p /etc/systemd/system/eitri-agent.service.d
		153	printf '[Service]\nEnvironment=GOCOVERDIR=%s\n' "$AGENT_GOCOVERDIR" \| sudo tee /etc/systemd/system/eitri-agent.service.d/coverage.conf >/dev/null
		154	sudo systemctl daemon-reload
		155	sudo systemctl enable eitri-agent >/dev/null 2>&1
		156	sudo systemctl start eitri-agent
142	sleep 2	157	sleep 2
143	pgrep -x eitri-agent >/dev/null \|\| { echo "agent FAILED to start on \$(hostname) (see $AGENT_LOG)" >&2; exit 1; }	158	sudo systemctl is-active --quiet eitri-agent \|\| { echo "agent FAILED to start on \$(hostname) (journalctl -u eitri-agent)" >&2; exit 1; }
144	echo "agent up: \$(sha256sum $AGENT_BIN \| cut -c1-12) on \$(hostname)"	159	pgrep -x eitri-agent >/dev/null \|\| { echo "agent process missing on \$(hostname)" >&2; exit 1; }
		160	echo "agent up: \$(sha256sum $AGENT_BIN \| cut -c1-12) on \$(hostname) (systemd)"
145	REMOTE	161	REMOTE
146	done	162	done
147		163

scripts/eitri-agent.service

@@ @@ -0,0 +1,38 @@ @@
+# eitri-agent systemd unit. Install on each fleet host:
+#
+#   cp eitri-agent.service /etc/systemd/system/
+#   systemctl daemon-reload && systemctl enable --now eitri-agent
+#
+# Restart=on-failure is the reviver for a crashed agent. Agent self-upgrades
+# never need it: the agent re-execs in place, keeping its PID, so systemd sees
+# one uninterrupted service. If an upgraded binary crashes, the restart loop
+# stops at the start limit and the previous binary sits beside it as
+# eitri-agent.prev for manual recovery.
+[Unit]
+Description=eitri agent — reconciles this host's VMs against the fleet control plane
+Documentation=https://eitri.sh
+After=network-online.target
+Wants=network-online.target
+StartLimitIntervalSec=300
+StartLimitBurst=10
+[Service]
+# Flags (resource caps, paths) go in /etc/default/eitri-agent as
+# EITRI_AGENT_FLAGS="--max-vcpus 8 ..."; the defaults suit a standard install
+# (state under /var/lib/eitri-agent, cloud-hypervisor on PATH, firmware at
+# /usr/share/eitri/CLOUDHV.fd).
+EnvironmentFile=-/etc/default/eitri-agent
+ExecStart=/usr/local/bin/eitri-agent $EITRI_AGENT_FLAGS
+Restart=on-failure
+RestartSec=5
+# KillMode=process is LOAD-BEARING: guests are cloud-hypervisor processes in
+# this unit's cgroup, and the default control-group kill would tear down every
+# running VM on a mere agent stop. Only the agent itself may be signalled —
+# VMs survive agent restarts by design.
+KillMode=process
+# The agent manages KVM guests, bridges, and taps — it needs root.
+User=root
+[Install]
+WantedBy=multi-user.target