72a48183 — eitri

Makefile

cmd/eitri-agent/main.go

Old		New
@@ -11,7 +11,6 @@ import (
11	"os"	11	"os"
12	"os/exec"	12	"os/exec"
13	"os/signal"	13	"os/signal"
14	"path/filepath"
15	"runtime"	14	"runtime"
16	"syscall"	15	"syscall"
17	"time"	16	"time"
@@ -42,7 +41,7 @@ type agentConfig struct {
42	func main() {	41	func main() {
43	stateDir := flag.String("state-dir", "/var/lib/eitri-agent", "agent state directory")	42	stateDir := flag.String("state-dir", "/var/lib/eitri-agent", "agent state directory")
44	chBin := flag.String("ch-bin", "cloud-hypervisor", "path to cloud-hypervisor binary")	43	chBin := flag.String("ch-bin", "cloud-hypervisor", "path to cloud-hypervisor binary")
45	firmware := flag.String("firmware", "/usr/share/eitri/hypervisor-fw", "path to hypervisor-fw")	44	firmware := flag.String("firmware", "/usr/share/eitri/CLOUDHV.fd", "path to CH UEFI firmware (CLOUDHV.fd)")
46	tombstoneGrace := flag.Duration("tombstone-grace", 5*time.Minute, "quarantine period for tombstoned VMs before destroy")	45	tombstoneGrace := flag.Duration("tombstone-grace", 5*time.Minute, "quarantine period for tombstoned VMs before destroy")
47	vanishGrace := flag.Duration("vanish-grace", time.Hour, "quarantine period for VMs that vanished without a tombstone")	46	vanishGrace := flag.Duration("vanish-grace", time.Hour, "quarantine period for VMs that vanished without a tombstone")
48	stepTimeout := flag.Duration("step-timeout", 15*time.Minute, "watchdog bound on one WHOLE reconcile step — all VMs, summed (0 disables); keep above the 10m image-download timeout")	47	stepTimeout := flag.Duration("step-timeout", 15*time.Minute, "watchdog bound on one WHOLE reconcile step — all VMs, summed (0 disables); keep above the 10m image-download timeout")
@@ -169,9 +168,7 @@ func runAgent(st *state.Store, cfg agentConfig) {
169	// and reattached here for VMs that survived an agent restart (CH runs in	168	// and reattached here for VMs that survived an agent restart (CH runs in
170	// its own process group; the pump reconnects to the still-listening	169	// its own process group; the pump reconnects to the still-listening
171	// serial socket).	170	// serial socket).
172	pumps := serialpump.NewManager(st.SerialSocketPath, func(vmID string) string {	171	pumps := serialpump.NewManager(st.SerialSocketPath, st.SerialLogPath)
173	return filepath.Join(st.VMDir(vmID), "serial.log")
174	})
175	prov.Pumps = pumps	172	prov.Pumps = pumps
176	if recs, err := st.LoadVMs(); err == nil {	173	if recs, err := st.LoadVMs(); err == nil {
177	for _, rec := range recs {	174	for _, rec := range recs {
@@ -211,10 +208,6 @@ func runAgent(st *state.Store, cfg agentConfig) {
211	MaxDiskGB: cfg.MaxDiskGB,	208	MaxDiskGB: cfg.MaxDiskGB,
212	}	209	}
213		210
214	// Compile-time interface satisfaction checks.
215	var _ reconcile.Provisioner = prov
216	var _ reconcile.NetEnv = net
217
218	client := &syncclient.Client{	211	client := &syncclient.Client{
219	Engine: engine,	212	Engine: engine,
220	St: st,	213	St: st,
@@ -228,4 +221,12 @@ func runAgent(st *state.Store, cfg agentConfig) {
228		221
229	slog.Info("agent started", "host_id", id.HostID, "bridge_cidr", id.BridgeCIDR)	222	slog.Info("agent started", "host_id", id.HostID, "bridge_cidr", id.BridgeCIDR)
230	client.Run(ctx)	223	client.Run(ctx)
		224
		225	// client.Run blocks until ctx is cancelled (SIGINT/SIGTERM) and only then
		226	// returns, so the reconcile/sync loop is done driving VMs and no further
		227	// Ensure/Stop calls are expected to reach pumps (an unjoined session
		228	// worker could in principle still be mid-Engine.Step, but it has nothing
		229	// left to drive once client.Run has returned). Tear down every serial
		230	// console pump here, at the very end of agent shutdown.
		231	pumps.StopAll()
231	}	232	}

cmd/eitri-server/main.go

Old		New
@@ -5,75 +5,60 @@ package main
5	import (	5	import (
6	"context"	6	"context"
7	"encoding/json"	7	"encoding/json"
		8	"errors"
8	"flag"	9	"flag"
9	"log/slog"	10	"log/slog"
10	"net"
11	"net/http"	11	"net/http"
12	"os"	12	"os"
		13	"os/signal"
13	"regexp"	14	"regexp"
		15	"syscall"
14	"time"	16	"time"
15		17
16	"github.com/a73x/eitri/internal/joinblob"	18	"github.com/a73x/eitri/internal/joinblob"
17	"github.com/a73x/eitri/internal/server/api"	19	"github.com/a73x/eitri/internal/server/api"
		20	serverconfig "github.com/a73x/eitri/internal/server/config"
18	"github.com/a73x/eitri/internal/server/health"	21	"github.com/a73x/eitri/internal/server/health"
19	"github.com/a73x/eitri/internal/server/hub"	22	"github.com/a73x/eitri/internal/server/hub"
20	"github.com/a73x/eitri/internal/server/registry"	23	"github.com/a73x/eitri/internal/server/registry"
21	"github.com/a73x/eitri/internal/server/sshca"
22	"github.com/a73x/eitri/internal/server/sshgate"
23	"github.com/a73x/eitri/internal/server/store"	24	"github.com/a73x/eitri/internal/server/store"
24	"github.com/a73x/eitri/internal/server/syncsvc"	25	"github.com/a73x/eitri/internal/server/syncsvc"
25	"github.com/a73x/eitri/internal/server/web"	26	"github.com/a73x/eitri/internal/server/web"
26	"github.com/a73x/eitri/internal/transport"	27	"github.com/a73x/eitri/internal/transport"
27	"github.com/quic-go/quic-go"	28	"github.com/quic-go/quic-go"
28	"golang.org/x/crypto/ssh"
29	)	29	)
30		30
31	// defaultImageSHARe matches a valid lowercase hex SHA-256 digest. Kept	31	// defaultImageSHARe matches a valid lowercase hex SHA-256 digest. Kept local
32	// local (not internal/agent/imagecache.ValidSHA256): R1 forbids the	32	// to the control plane rather than shared with the agent's imagecache: R1
33	// control plane importing the data plane, even transitively.	33	// forbids the control plane importing the data plane, even transitively.
34	var defaultImageSHARe = regexp.MustCompile(`^[a-f0-9]{64}$`)	34	var defaultImageSHARe = regexp.MustCompile(`^[a-f0-9]{64}$`)
35		35
36	type config struct {	36	// parseDurationCfg parses raw (a config duration string) for the knob named
37	HTTPListen string `json:"http_listen"`	37	// name (the JSON field label echoed in error logs). Empty raw keeps def. A
38	QUICListen string `json:"quic_listen"`	38	// parse failure is fatal. valid, when non-nil, is the knob's range rule; a
39	DBPath string `json:"db_path"`	39	// value failing it is fatal, with rule (e.g. ">= 0", "> 0") spelling the rule
40	AdminToken string `json:"admin_token"`	40	// in the log line. A nil valid skips the range check.
41	HostSecret string `json:"host_secret"`	41	func parseDurationCfg(name, raw string, def time.Duration, valid func(time.Duration) bool, rule string) time.Duration {
42	CIDRPool string `json:"cidr_pool"`	42	if raw == "" {
43	DefaultImageURL string `json:"default_image_url"`	43	return def
44	DefaultImageSHA string `json:"default_image_sha256"`	44	}
45	AdvertiseHTTP string `json:"advertise_http"`	45	d, err := time.ParseDuration(raw)
46	AdvertiseQUIC string `json:"advertise_quic"`	46	if err != nil {
47	// CredentialMaxAge optionally bounds host credential age (Go duration,	47	slog.Error(name+" invalid", "err", err)
48	// e.g. "2160h" for 90 days). Empty/zero disables — revocation via	48	os.Exit(1)
49	// POST /api/v1/hosts/{id}/revoke-credential is the primary mechanism;	49	}
50	// max-age forces periodic re-enrollment and is opt-in defense-in-depth.	50	if valid != nil && !valid(d) {
51	CredentialMaxAge string `json:"credential_max_age"`	51	slog.Error(name+" must be "+rule, "value", raw)
52	// AuditRetention bounds the audit_log age (Go duration; default "2160h" =	52	os.Exit(1)
53	// 90 days; "0" disables pruning). Pruned at startup and daily.	53	}
54	AuditRetention string `json:"audit_retention"`	54	return d
55	// SSHCAKey is the path to the persistent SSH user CA private key
56	// (auto-created 0600 if absent, handled like AdminToken — never logged).
57	// The CA signs the short-lived certs the jump gate accepts.
58	SSHCAKey string `json:"ssh_ca_key"`
59	// SSHHostKey is the path to the gate's persistent SSH host key
60	// (auto-created 0600 if absent, never regenerated on restart so users
61	// don't see host-key-changed warnings).
62	SSHHostKey string `json:"ssh_host_key"`
63	// SSHListen is the jump-gate listen address. Empty ⇒ gate is OFF (no
64	// key material is loaded and no listener is started).
65	SSHListen string `json:"ssh_listen"`
66	// SSHGateDomain is the hostname clients dial the gate as (the principal put
67	// on the gate's signed HOST certificate). Empty ⇒ derived from SSHListen's
68	// host part; if that is also empty (e.g. ":2222") it falls back to
69	// "localhost". It must match the host in EITRI_GATE so `@cert-authority`
70	// verification accepts the presented host cert.
71	SSHGateDomain string `json:"ssh_gate_domain"`
72	// SSHCertTTL bounds minted user-cert validity (Go duration; default 10m).
73	// Set server-side; client-requested validity is never honored.
74	SSHCertTTL string `json:"ssh_cert_ttl"`
75	}	55	}
76		56
		57	// config is the on-disk JSON schema, shared with the integration harness
		58	// (which renders this file for the server subprocess it launches) via
		59	// internal/server/config so the compiler keeps the two in agreement.
		60	type config = serverconfig.Config
		61
77	func main() {	62	func main() {
78	cfgPath := flag.String("config", "/etc/eitri/server.json", "config file")	63	cfgPath := flag.String("config", "/etc/eitri/server.json", "config file")
79	flag.Parse()	64	flag.Parse()
@@ -97,8 +82,8 @@ func main() {
97	}	82	}
98	// [carry-forward] Fail fast on a malformed default image digest rather	83	// [carry-forward] Fail fast on a malformed default image digest rather
99	// than letting every mint silently propagate a bad hash. A local regex	84	// than letting every mint silently propagate a bad hash. A local regex
100	// (not internal/agent/imagecache.ValidSHA256) because R1 forbids the	85	// (not the agent's imagecache) because R1 forbids the control plane
101	// control plane importing the data plane, even transitively.	86	// importing the data plane, even transitively.
102	if cfg.DefaultImageSHA != "" && !defaultImageSHARe.MatchString(cfg.DefaultImageSHA) {	87	if cfg.DefaultImageSHA != "" && !defaultImageSHARe.MatchString(cfg.DefaultImageSHA) {
103	slog.Error("default_image_sha256 malformed (want 64 lowercase hex chars)", "value", cfg.DefaultImageSHA)	88	slog.Error("default_image_sha256 malformed (want 64 lowercase hex chars)", "value", cfg.DefaultImageSHA)
104	os.Exit(1)	89	os.Exit(1)
@@ -145,20 +130,10 @@ func main() {
145	warnIfRenewalDue()	130	warnIfRenewalDue()
146		131
147	// Audit retention: bound the append-only log (default 90 days, 0 disables).	132	// Audit retention: bound the append-only log (default 90 days, 0 disables).
148	auditRetention := 90 * 24 * time.Hour	133	// A negative value is almost certainly a typo — refuse rather than silently
149	if cfg.AuditRetention != "" {	134	// keeping the audit log forever ("0" is the explicit disable spelling).
150	auditRetention, err = time.ParseDuration(cfg.AuditRetention)	135	auditRetention := parseDurationCfg("audit_retention", cfg.AuditRetention, 9024time.Hour,
151	if err != nil {	136	func(d time.Duration) bool { return d >= 0 }, ">= 0")
152	slog.Error("audit_retention invalid", "err", err)
153	os.Exit(1)
154	}
155	if auditRetention < 0 {
156	// Almost certainly a typo — refuse rather than silently keeping
157	// the audit log forever ("0" is the explicit disable spelling).
158	slog.Error("audit_retention must be >= 0", "value", cfg.AuditRetention)
159	os.Exit(1)
160	}
161	}
162	pruneAudit := func() {	137	pruneAudit := func() {
163	if auditRetention <= 0 {	138	if auditRetention <= 0 {
164	return	139	return
@@ -186,40 +161,10 @@ func main() {
186	os.Exit(1)	161	os.Exit(1)
187	}	162	}
188		163
189	// SSH jump gate (§B): OFF unless ssh_listen is set. When enabled, load or	164	// SSH jump gate (§B): nil when ssh_listen is unset (gate OFF); see
190	// create the persistent user CA + gate host key (0600, never logged) and	165	// setupSSHGate. The listener itself is started below, once syncsvc.Service
191	// resolve the cert TTL now so a misconfig fails fast at startup.	166	// (the tunnel dialer) exists.
192	var sshGate *sshca.CA	167	sshGate := setupSSHGate(cfg)
193	sshCertTTL := 10 * time.Minute
194	if cfg.SSHListen != "" {
195	if cfg.SSHCAKey == "" \|\| cfg.SSHHostKey == "" {
196	slog.Error("ssh_ca_key and ssh_host_key are required when ssh_listen is set")
197	os.Exit(1)
198	}
199	if cfg.SSHCertTTL != "" {
200	sshCertTTL, err = time.ParseDuration(cfg.SSHCertTTL)
201	if err != nil {
202	slog.Error("ssh_cert_ttl invalid", "err", err)
203	os.Exit(1)
204	}
205	if sshCertTTL <= 0 {
206	slog.Error("ssh_cert_ttl must be > 0", "value", cfg.SSHCertTTL)
207	os.Exit(1)
208	}
209	}
210	sshGate, err = sshca.New(cfg.SSHCAKey, cfg.SSHHostKey)
211	if err != nil {
212	slog.Error("ssh ca", "err", err)
213	os.Exit(1)
214	}
215	// Log the CA identity operators pin in known_hosts / inject into VMs.
216	// Only the public key is ever logged (private material never is).
217	slog.Info("ssh jump gate configured", "listen", cfg.SSHListen,
218	"cert_ttl", sshCertTTL,
219	"user_ca", string(sshGate.UserCAAuthorizedKey()))
220	// The gate listener itself is started below, once syncsvc.Service (the
221	// tunnel dialer) exists.
222	}
223		168
224	reg := registry.New(time.Now)	169	reg := registry.New(time.Now)
225	h := hub.New()	170	h := hub.New()
@@ -237,112 +182,26 @@ func main() {
237	os.Exit(1)	182	os.Exit(1)
238	}	183	}
239	lis, err := quic.ListenAddr(cfg.QUICListen, tlsConf,	184	lis, err := quic.ListenAddr(cfg.QUICListen, tlsConf,
240	&quic.Config{KeepAlivePeriod: 15 * time.Second, MaxIdleTimeout: 30 * time.Second})	185	// Shared with the agent dialer via transport so the two ends can't drift.
		186	transport.SyncQUICConfig())
241	if err != nil {	187	if err != nil {
242	slog.Error("quic listen", "err", err)	188	slog.Error("quic listen", "err", err)
243	os.Exit(1)	189	os.Exit(1)
244	}	190	}
245	var maxCredAge time.Duration	191	maxCredAge := parseDurationCfg("credential_max_age", cfg.CredentialMaxAge, 0, nil, "")
246	if cfg.CredentialMaxAge != "" {	192	// SSH cert minters: no-op when the gate is off, so the endpoint 404s.
247	maxCredAge, err = time.ParseDuration(cfg.CredentialMaxAge)	193	sshGate.wireAPI(a)
248	if err != nil {
249	slog.Error("credential_max_age invalid", "err", err)
250	os.Exit(1)
251	}
252	}
253	// SSH cert minter: when the jump gate is enabled, the API mints short-lived
254	// user certs signed by the persistent user CA (POST /api/v1/ssh-certs).
255	// Left nil when the gate is off, so the endpoint 404s.
256	if sshGate != nil {
257	a.SetCertMinter(api.NewMinter(sshGate.UserCA(), sshCertTTL))
258	// Per-VM host certs: sign a persistent host key + cert at each VM create,
259	// so VMs present verifiable host keys (clients accept via @cert-authority).
260	a.SetHostCertMinter(api.NewHostMinter(sshGate.UserCA()))
261	// Publish the CA public key so clients can pin `@cert-authority` for host
262	// verification of both the gate and every VM.
263	a.SetSSHCAAuthorizedKey(string(sshGate.UserCAAuthorizedKey()))
264	}
265		194
266	svc := syncsvc.New(st, reg, h, []byte(cfg.HostSecret), maxCredAge)	195	svc := syncsvc.New(st, reg, h, []byte(cfg.HostSecret), maxCredAge)
267	// When the jump gate is enabled, advertise the user-CA public key in every	196	// Advertise the user-CA public key in desired-VM snapshots (no-op when off).
268	// desired-VM snapshot so guests inject it as an sshd TrustedUserCAKeys	197	sshGate.wireSync(svc)
269	// drop-in and trust CA-signed certs. Off (nil gate) => no injection.
270	if sshGate != nil {
271	svc.SetSSHUserCAKey(string(sshGate.UserCAAuthorizedKey()))
272	}
273	// Console broker: the API bridges browser WebSockets to agent console	198	// Console broker: the API bridges browser WebSockets to agent console
274	// streams over the live sync connections the service tracks.	199	// streams over the live sync connections the service tracks.
275	a.SetConsoleDialer(svc)	200	a.SetConsoleDialer(svc)
276		201
277	// SSH jump gate listener: when enabled, front `ssh -J gate ubuntu@<vm>` with	202	// SSH jump gate listener (no-op when off); fatal on a failed bind, like
278	// the hardened bastion. It resolves VM names against the store, tunnels port	203	// QUIC/HTTP below — a dead gate must not run silently.
279	// 22 through the sync connection (svc.OpenTCP), and trusts only certs signed	204	sshGate.startListener(st, svc)
280	// by the user CA. A failed bind is fatal (like QUIC/HTTP below): a dead gate
281	// must not run silently.
282	if sshGate != nil {
283	// The gate host cert's principal is the name clients dial. Prefer the
284	// configured domain; else the host part of ssh_listen; else "localhost".
285	gateDomain := cfg.SSHGateDomain
286	if gateDomain == "" {
287	if h, _, err := net.SplitHostPort(cfg.SSHListen); err == nil {
288	gateDomain = h
289	}
290	}
291	if gateDomain == "" {
292	gateDomain = "localhost"
293	}
294	slog.Info("ssh gate host cert", "principal", gateDomain)
295	// resolve maps a VM name to its host/VM IDs; unknown or tombstoned ⇒ ok=false.
296	resolve := func(name string) (hostID, vmID string, ok bool) {
297	vm, err := st.VMByName(name)
298	if err != nil {
299	return "", "", false
300	}
301	return vm.HostID, vm.ID, true
302	}
303	// v1 single-admin: any CA-signed cert reaches any VM; per-user ownership is Task/Slice per §5/§9.
304	authorize := func(principal, vmID string) bool { return true }
305	// Sign a long-lived HOST cert for the gate's own host key and present THAT
306	// (via a cert signer) instead of the bare key, so a client verifying with
307	// `@cert-authority` accepts the gate on first connect — no TOFU window.
308	gateCert, err := sshca.SignHostCert(sshGate.UserCA(), sshGate.HostKey().PublicKey(),
309	[]string{gateDomain}, "eitri-gate", time.Now(), sshca.HostCertTTL)
310	if err != nil {
311	slog.Error("sign gate host cert", "err", err)
312	os.Exit(1)
313	}
314	gateHostSigner, err := ssh.NewCertSigner(gateCert, sshGate.HostKey())
315	if err != nil {
316	slog.Error("gate host cert signer", "err", err)
317	os.Exit(1)
318	}
319	// isRevoked gates every cert auth against the revocation list. Fail-CLOSED
320	// for the single connection on a DB error: a store hiccup rejects THAT
321	// login (returns revoked=true) rather than fail-open (which would let a
322	// possibly-revoked cert through) or fail-the-whole-gate (which a global
323	// close would amount to, DoSing every login on any transient error).
324	isRevoked := func(serial uint64) bool {
325	revoked, err := st.IsSSHCertRevoked(serial)
326	if err != nil {
327	slog.Error("ssh cert revocation lookup failed; rejecting connection", "err", err)
328	return true
329	}
330	return revoked
331	}
332	gate := sshgate.New(gateHostSigner, sshGate.UserCA().PublicKey(), resolve, authorize, svc.OpenTCP, isRevoked)
333	ln, err := net.Listen("tcp", cfg.SSHListen)
334	if err != nil {
335	slog.Error("ssh gate listen", "err", err)
336	os.Exit(1)
337	}
338	go func() {
339	slog.Info("ssh jump gate listening", "addr", cfg.SSHListen)
340	if err := gate.Serve(ln); err != nil {
341	slog.Error("ssh gate serve", "err", err)
342	os.Exit(1)
343	}
344	}()
345	}
346		205
347	go func() {	206	go func() {
348	slog.Info("quic listening", "addr", cfg.QUICListen)	207	slog.Info("quic listening", "addr", cfg.QUICListen)
@@ -370,9 +229,28 @@ func main() {
370	))	229	))
371	root.Handle("/", web.Handler())	230	root.Handle("/", web.Handler())
372		231
373	slog.Info("http listening", "addr", cfg.HTTPListen)	232	// Graceful shutdown: SIGINT/SIGTERM stops accepting, drains in-flight
374	if err := http.ListenAndServe(cfg.HTTPListen, root); err != nil {	233	// requests, then closes the API — stopping the SSE snapshot-hub goroutine and
375	slog.Error("http serve", "err", err)	234	// releasing its notifier subscription. The QUIC listener and background worker
376	os.Exit(1)	235	// die with the process.
		236	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
		237	defer stop()
		238
		239	srv := &http.Server{Addr: cfg.HTTPListen, Handler: root}
		240	go func() {
		241	slog.Info("http listening", "addr", cfg.HTTPListen)
		242	if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
		243	slog.Error("http serve", "err", err)
		244	os.Exit(1)
		245	}
		246	}()
		247
		248	<-ctx.Done()
		249	slog.Info("shutting down")
		250	shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
		251	defer cancel()
		252	if err := srv.Shutdown(shutdownCtx); err != nil {
		253	slog.Warn("http graceful shutdown", "err", err)
377	}	254	}
		255	a.Close()
378	}	256	}

cmd/eitri-server/sshgate.go

@@ @@ -0,0 +1,154 @@ @@
+package main
+import (
+	"log/slog"
+	"net"
+	"os"
+	"time"
+	"github.com/a73x/eitri/internal/server/api"
+	"github.com/a73x/eitri/internal/server/sshca"
+	"github.com/a73x/eitri/internal/server/sshgate"
+	"github.com/a73x/eitri/internal/server/store"
+	"github.com/a73x/eitri/internal/server/syncsvc"
+	"golang.org/x/crypto/ssh"
+)
+// sshGateSetup carries the jump-gate state from config-time setup to the later
+// wiring points in startup (API cert minters, sync snapshot, gate listener).
+// A nil *sshGateSetup means the gate is OFF: every method is a no-op on a nil
+// receiver, so main holds one value instead of repeating `!= nil` guards.
+type sshGateSetup struct {
+	ca      *sshca.CA
+	certTTL time.Duration
+	listen  string // cfg.SSHListen
+	domain  string // cfg.SSHGateDomain
+}
+// setupSSHGate wires the SSH jump gate (§B): OFF unless ssh_listen is set
+// (returns nil). When enabled, load or create the persistent user CA + gate
+// host key (0600, never logged) and resolve the cert TTL now so a misconfig
+// fails fast at startup.
+func setupSSHGate(cfg config) *sshGateSetup {
+	if cfg.SSHListen == "" {
+		return nil
+	}
+	if cfg.SSHCAKey == "" || cfg.SSHHostKey == "" {
+		slog.Error("ssh_ca_key and ssh_host_key are required when ssh_listen is set")
+		os.Exit(1)
+	}
+	sshCertTTL := parseDurationCfg("ssh_cert_ttl", cfg.SSHCertTTL, 10*time.Minute,
+		func(d time.Duration) bool { return d > 0 }, "> 0")
+	sshGate, err := sshca.New(cfg.SSHCAKey, cfg.SSHHostKey)
+	if err != nil {
+		slog.Error("ssh ca", "err", err)
+		os.Exit(1)
+	}
+	// Log the CA identity operators pin in known_hosts / inject into VMs.
+	// Only the *public* key is ever logged (private material never is).
+	slog.Info("ssh jump gate configured", "listen", cfg.SSHListen,
+		"cert_ttl", sshCertTTL,
+		"user_ca", string(sshGate.UserCAAuthorizedKey()))
+	// The gate listener itself is started later (startListener), once
+	// syncsvc.Service (the tunnel dialer) exists.
+	return &sshGateSetup{ca: sshGate, certTTL: sshCertTTL, listen: cfg.SSHListen, domain: cfg.SSHGateDomain}
+}
+// wireAPI installs the SSH cert minter: when the jump gate is enabled, the API
+// mints short-lived user certs signed by the persistent user CA
+// (POST /api/v1/ssh-certs). Left nil when the gate is off, so the endpoint 404s.
+func (g *sshGateSetup) wireAPI(a *api.API) {
+	if g == nil {
+		return
+	}
+	a.SetCertMinter(api.NewMinter(g.ca.UserCA(), g.certTTL))
+	// Per-VM host certs: sign a persistent host key + cert at each VM create,
+	// so VMs present verifiable host keys (clients accept via @cert-authority).
+	a.SetHostCertMinter(api.NewHostMinter(g.ca.UserCA()))
+	// Publish the CA public key so clients can pin `@cert-authority` for host
+	// verification of both the gate and every VM.
+	a.SetSSHCAAuthorizedKey(string(g.ca.UserCAAuthorizedKey()))
+}
+// wireSync: when the jump gate is enabled, advertise the user-CA public key in
+// every desired-VM snapshot so guests inject it as an sshd TrustedUserCAKeys
+// drop-in and trust CA-signed certs. Off (nil gate) => no injection.
+func (g *sshGateSetup) wireSync(svc *syncsvc.Service) {
+	if g == nil {
+		return
+	}
+	svc.SetSSHUserCAKey(string(g.ca.UserCAAuthorizedKey()))
+}
+// startListener starts the SSH jump gate listener: when enabled, front
+// `ssh -J gate ubuntu@<vm>` with the hardened bastion. It resolves VM names
+// against the store, tunnels port 22 through the sync connection (svc.OpenTCP),
+// and trusts only certs signed by the user CA. A failed bind is fatal (like
+// QUIC/HTTP): a dead gate must not run silently.
+func (g *sshGateSetup) startListener(st *store.Store, svc *syncsvc.Service) {
+	if g == nil {
+		return
+	}
+	// The gate host cert's principal is the name clients dial. Prefer the
+	// configured domain; else the host part of ssh_listen; else "localhost".
+	gateDomain := g.domain
+	if gateDomain == "" {
+		if h, _, err := net.SplitHostPort(g.listen); err == nil {
+			gateDomain = h
+		}
+	}
+	if gateDomain == "" {
+		gateDomain = "localhost"
+	}
+	slog.Info("ssh gate host cert", "principal", gateDomain)
+	// resolve maps a VM name to its host/VM IDs; unknown or tombstoned ⇒ ok=false.
+	resolve := func(name string) (hostID, vmID string, ok bool) {
+		vm, err := st.VMByName(name)
+		if err != nil {
+			return "", "", false
+		}
+		return vm.HostID, vm.ID, true
+	}
+	// v1 single-admin: any CA-signed cert reaches any VM; per-user ownership is Task/Slice per §5/§9.
+	authorize := func(principal, vmID string) bool { return true }
+	// Sign a long-lived HOST cert for the gate's own host key and present THAT
+	// (via a cert signer) instead of the bare key, so a client verifying with
+	// `@cert-authority` accepts the gate on first connect — no TOFU window.
+	gateCert, err := sshca.SignHostCert(g.ca.UserCA(), g.ca.HostKey().PublicKey(),
+		[]string{gateDomain}, "eitri-gate", time.Now(), sshca.HostCertTTL)
+	if err != nil {
+		slog.Error("sign gate host cert", "err", err)
+		os.Exit(1)
+	}
+	gateHostSigner, err := ssh.NewCertSigner(gateCert, g.ca.HostKey())
+	if err != nil {
+		slog.Error("gate host cert signer", "err", err)
+		os.Exit(1)
+	}
+	// isRevoked gates every cert auth against the revocation list. Fail-CLOSED
+	// for the single connection on a DB error: a store hiccup rejects THAT
+	// login (returns revoked=true) rather than fail-open (which would let a
+	// possibly-revoked cert through) or fail-the-whole-gate (which a global
+	// close would amount to, DoSing every login on any transient error).
+	isRevoked := func(serial uint64) bool {
+		revoked, err := st.IsSSHCertRevoked(serial)
+		if err != nil {
+			slog.Error("ssh cert revocation lookup failed; rejecting connection", "err", err)
+			return true
+		}
+		return revoked
+	}
+	gate := sshgate.New(gateHostSigner, g.ca.UserCA().PublicKey(), resolve, authorize, svc.OpenTCP, isRevoked)
+	ln, err := net.Listen("tcp", g.listen)
+	if err != nil {
+		slog.Error("ssh gate listen", "err", err)
+		os.Exit(1)
+	}
+	go func() {
+		slog.Info("ssh jump gate listening", "addr", g.listen)
+		if err := gate.Serve(ln); err != nil {
+			slog.Error("ssh gate serve", "err", err)
+			os.Exit(1)
+		}
+	}()
+}

docs/architecture.md

docs/ethos.md

@@ @@ -0,0 +1,47 @@ @@
+# eitri design ethos
+*Something to fall back on when a design debate stalls.*
+## North star
+eitri is a **general-purpose cloud for quasi-cattle** — user-administered VMs with an on-demand lifecycle (throwaway → long-lived webserver).
+> **The guest owns the guest; eitri owns the fleet.**
+That line is the tie-breaker for almost everything.
+## Principles
+Each is tagged with a real fork it settled, so it stays concrete rather than aspirational.
+. **The guest/fleet boundary is the VM's edge.** Inside — kernel, packages, filesystem, in-VM config — is the user's; don't manage it. Outside — placement, lifecycle, access seam, quotas, identity — is eitri's; own it fully.
+   *→ guests own their kernel (UEFI + guest GRUB, not pinned by us); cloud-init injection stays minimal.*
+. **When unsure, do what a real cloud does — and distrust anything the clouds abandoned.** EC2/GCE/Linode/DO already paid for these lessons.
+   *→ UEFI + guest GRUB, not provider-pinned external kernels (which AWS→PV-GRUB, DigitalOcean, and Linode all migrated away from).*
+. **Composable, agnostic seams — BYO everything (CNI-style).** We care that an interface is satisfied, not what satisfies it.
+   *→ BYO image = just a disk; network-provider seam; no hardcoded fabric, image, or kernel.*
+. **Design for the most demanding workload in the union, not the average.** The long-lived pet's needs dominate; serve it and the throwaway case is free.
+   *→ "kernel security updates must take effect" (the webserver) drove UEFI over direct kernel boot.*
+. **Prefer the design that removes machinery.** Ballooning scope — extra wire fields, validation ladders, lifecycle plumbing — is a smell that you're on the wrong side of a boundary or fighting the ecosystem.
+   *→ a firmware swap beat direct kernel boot + a wire contract + a validation ladder + kernel-as-control-plane-input.*
+. **Defer freely, foreclose never.** Ship the minimum; keep seams open so deferred features slot in without a rewrite.
+   *→ multi-tenancy deferred but design-forward; direct kernel boot kept as a back-pocket fast-ephemeral profile.*
+. **Prove it on the stack; evidence over assertion.** Boot the VM, capture the serial.
+   *→ the "modern images panic under CH firmware (EFI/Secure-Boot/TPM, CH #7356)" comment was flat wrong — only caught by reproducing it.*
+## The "um and ahh" checklist
+When a design debate stalls, walk this out loud. The first item that bites usually decides it.
+. Guest's business or fleet's? (guest → hands off)
+. What does EC2/GCE do here? (default to it; stop if it's something they abandoned)
+. Does this remove machinery or add it? (prefer removal)
+. Does it serve the long-lived pet? (design for that; throwaway comes free)
+. Are we deferring (fine) or foreclosing (stop)?
+. Have we booted it, or are we guessing?

docs/openapi.json

Old		New
@@ -170,6 +170,13 @@
170	"id": {	170	"id": {
171	"type": "string"	171	"type": "string"
172	},	172	},
		173	"last_seen": {
		174	"format": "date-time",
		175	"type": [
		176	"string",
		177	"null"
		178	]
		179	},
173	"name": {	180	"name": {
174	"type": "string"	181	"type": "string"
175	},	182	},
@@ -182,6 +189,18 @@
182	"provisioner": {	189	"provisioner": {
183	"type": "string"	190	"type": "string"
184	},	191	},
		192	"seconds_since_last_seen": {
		193	"type": [
		194	"integer",
		195	"null"
		196	]
		197	},
		198	"sessions": {
		199	"type": "integer"
		200	},
		201	"stale": {
		202	"type": "boolean"
		203	},
185	"status": {	204	"status": {
186	"type": "string"	205	"type": "string"
187	}	206	}
@@ -197,6 +216,8 @@
197	"online",	216	"online",
198	"os",	217	"os",
199	"provisioner",	218	"provisioner",
		219	"sessions",
		220	"stale",
200	"status"	221	"status"
201	],	222	],
202	"type": "object"	223	"type": "object"

docs/shape.html

Old		New
@@ -84,6 +84,7 @@
84	"imports": [	84	"imports": [
85	"internal/joinblob",	85	"internal/joinblob",
86	"internal/server/api",	86	"internal/server/api",
		87	"internal/server/config",
87	"internal/server/health",	88	"internal/server/health",
88	"internal/server/hub",	89	"internal/server/hub",
89	"internal/server/registry",	90	"internal/server/registry",
@@ -217,6 +218,12 @@
217	"imports": []	218	"imports": []
218	},	219	},
219	{	220	{
		221	"importPath": "internal/random",
		222	"plane": "wire",
		223	"synopsis": "Package random provides small cryptographically-random helpers shared across the control plane, CLIs, and the integration harness — a leaf package so a CLI or test binary can reuse them without importing heavier deps (e.g.",
		224	"imports": []
		225	},
		226	{
220	"importPath": "internal/server/api",	227	"importPath": "internal/server/api",
221	"plane": "control",	228	"plane": "control",
222	"synopsis": "Package api implements the admin REST API and the unauthenticated enrollment endpoint.",	229	"synopsis": "Package api implements the admin REST API and the unauthenticated enrollment endpoint.",
@@ -224,6 +231,7 @@
224	"internal/cloudinit",	231	"internal/cloudinit",
225	"internal/joinblob",	232	"internal/joinblob",
226	"internal/names",	233	"internal/names",
		234	"internal/random",
227	"internal/server/api/types",	235	"internal/server/api/types",
228	"internal/server/hosttoken",	236	"internal/server/hosttoken",
229	"internal/server/hub",	237	"internal/server/hub",
@@ -248,6 +256,12 @@
248	"imports": []	256	"imports": []
249	},	257	},
250	{	258	{
		259	"importPath": "internal/server/config",
		260	"plane": "control",
		261	"synopsis": "Package config defines the eitri-server on-disk JSON configuration schema.",
		262	"imports": []
		263	},
		264	{
251	"importPath": "internal/server/health",	265	"importPath": "internal/server/health",
252	"plane": "control",	266	"plane": "control",
253	"synopsis": "Package health serves the eitri-server liveness and readiness probes.",	267	"synopsis": "Package health serves the eitri-server liveness and readiness probes.",
@@ -288,6 +302,7 @@
288	"plane": "control",	302	"plane": "control",
289	"synopsis": "Package store is the server's durable control-plane state, backed by SQLite: the host registry, enrollment tokens, desired VM specs, and freed CIDRs.",	303	"synopsis": "Package store is the server's durable control-plane state, backed by SQLite: the host registry, enrollment tokens, desired VM specs, and freed CIDRs.",
290	"imports": [	304	"imports": [
		305	"internal/random",
291	"internal/transport"	306	"internal/transport"
292	]	307	]
293	},	308	},

docs/shape.json

Old		New
@@ -33,6 +33,7 @@
33	"imports": [	33	"imports": [
34	"internal/joinblob",	34	"internal/joinblob",
35	"internal/server/api",	35	"internal/server/api",
		36	"internal/server/config",
36	"internal/server/health",	37	"internal/server/health",
37	"internal/server/hub",	38	"internal/server/hub",
38	"internal/server/registry",	39	"internal/server/registry",
@@ -166,6 +167,12 @@
166	"imports": []	167	"imports": []
167	},	168	},
168	{	169	{
		170	"importPath": "internal/random",
		171	"plane": "wire",
		172	"synopsis": "Package random provides small cryptographically-random helpers shared across the control plane, CLIs, and the integration harness — a leaf package so a CLI or test binary can reuse them without importing heavier deps (e.g.",
		173	"imports": []
		174	},
		175	{
169	"importPath": "internal/server/api",	176	"importPath": "internal/server/api",
170	"plane": "control",	177	"plane": "control",
171	"synopsis": "Package api implements the admin REST API and the unauthenticated enrollment endpoint.",	178	"synopsis": "Package api implements the admin REST API and the unauthenticated enrollment endpoint.",
@@ -173,6 +180,7 @@
173	"internal/cloudinit",	180	"internal/cloudinit",
174	"internal/joinblob",	181	"internal/joinblob",
175	"internal/names",	182	"internal/names",
		183	"internal/random",
176	"internal/server/api/types",	184	"internal/server/api/types",
177	"internal/server/hosttoken",	185	"internal/server/hosttoken",
178	"internal/server/hub",	186	"internal/server/hub",
@@ -197,6 +205,12 @@
197	"imports": []	205	"imports": []
198	},	206	},
199	{	207	{
		208	"importPath": "internal/server/config",
		209	"plane": "control",
		210	"synopsis": "Package config defines the eitri-server on-disk JSON configuration schema.",
		211	"imports": []
		212	},
		213	{
200	"importPath": "internal/server/health",	214	"importPath": "internal/server/health",
201	"plane": "control",	215	"plane": "control",
202	"synopsis": "Package health serves the eitri-server liveness and readiness probes.",	216	"synopsis": "Package health serves the eitri-server liveness and readiness probes.",
@@ -237,6 +251,7 @@
237	"plane": "control",	251	"plane": "control",
238	"synopsis": "Package store is the server's durable control-plane state, backed by SQLite: the host registry, enrollment tokens, desired VM specs, and freed CIDRs.",	252	"synopsis": "Package store is the server's durable control-plane state, backed by SQLite: the host registry, enrollment tokens, desired VM specs, and freed CIDRs.",
239	"imports": [	253	"imports": [
		254	"internal/random",
240	"internal/transport"	255	"internal/transport"
241	]	256	]
242	},	257	},

internal/agent/cloudhv/cloudhv.go

Old		New
@@ -45,7 +45,7 @@ type PumpHooks interface {
45	type Provisioner struct {	45	type Provisioner struct {
46	st *state.Store	46	st *state.Store
47	chBin string // path to cloud-hypervisor binary	47	chBin string // path to cloud-hypervisor binary
48	firmware string // path to hypervisor-fw (EFI firmware)	48	firmware string // path to CLOUDHV.fd (UEFI firmware)
49	run agentexec.Runner	49	run agentexec.Runner
50		50
51	// Pumps receives serial-pump lifecycle calls at Boot/Kill. nil = no-op.	51	// Pumps receives serial-pump lifecycle calls at Boot/Kill. nil = no-op.
@@ -125,12 +125,26 @@ func (p *Provisioner) PrepareDisk(ctx context.Context, spec state.VMSpec, basePa
125	if err := os.MkdirAll(p.st.VMDir(spec.VMID), 0o700); err != nil {	125	if err := os.MkdirAll(p.st.VMDir(spec.VMID), 0o700); err != nil {
126	return fmt.Errorf("mkdir %s: %w", p.st.VMDir(spec.VMID), err)	126	return fmt.Errorf("mkdir %s: %w", p.st.VMDir(spec.VMID), err)
127	}	127	}
128	if _, err := p.run(ctx, "cp", "--reflink=auto", basePath, diskPath); err != nil {	128	// Build into a sibling temp file and rename into place, so a create that is
129	return fmt.Errorf("cp --reflink=auto %s %s: %w", basePath, diskPath, err)	129	// killed (ctx cancel on agent stop, ENOSPC) mid-copy/truncate can never leave
		130	// a torn disk.raw at the final path — the reconcile "exists" gate keys off
		131	// rec.BootID, not disk presence, but an atomic artifact keeps the on-disk
		132	// state honest for debugging and any future disk-aware code. The temp is a
		133	// sibling (same dir/filesystem) so cp keeps its reflink and rename is atomic.
		134	tmpPath := diskPath + ".partial"
		135	_ = os.Remove(tmpPath) // clear any leftover from an earlier interrupted create
		136	if _, err := p.run(ctx, "cp", "--reflink=auto", basePath, tmpPath); err != nil {
		137	_ = os.Remove(tmpPath)
		138	return fmt.Errorf("cp --reflink=auto %s %s: %w", basePath, tmpPath, err)
130	}	139	}
131	sizeArg := fmt.Sprintf("%dG", spec.DiskGB)	140	sizeArg := fmt.Sprintf("%dG", spec.DiskGB)
132	if _, err := p.run(ctx, "truncate", "-s", sizeArg, diskPath); err != nil {	141	if _, err := p.run(ctx, "truncate", "-s", sizeArg, tmpPath); err != nil {
133	return fmt.Errorf("truncate -s %s %s: %w", sizeArg, diskPath, err)	142	_ = os.Remove(tmpPath)
		143	return fmt.Errorf("truncate -s %s %s: %w", sizeArg, tmpPath, err)
		144	}
		145	if err := os.Rename(tmpPath, diskPath); err != nil {
		146	_ = os.Remove(tmpPath)
		147	return fmt.Errorf("rename %s -> %s: %w", tmpPath, diskPath, err)
134	}	148	}
135	return nil	149	return nil
136	}	150	}
@@ -182,8 +196,11 @@ func (p *Provisioner) Boot(_ context.Context, vmID string, spec state.VMSpec) er
182	// Write PID file so Running/Shutdown/Kill can find the process later.	196	// Write PID file so Running/Shutdown/Kill can find the process later.
183	pidData := []byte(strconv.Itoa(cmd.Process.Pid))	197	pidData := []byte(strconv.Itoa(cmd.Process.Pid))
184	if err := os.WriteFile(p.pidPath(vmID), pidData, 0o600); err != nil {	198	if err := os.WriteFile(p.pidPath(vmID), pidData, 0o600); err != nil {
185	// Best effort — kill the orphan if we can't track it.	199	// Best effort — kill the orphan if we can't track it, and Wait to reap it
		200	// (the async reaper below is not started on this path, so without Wait the
		201	// killed child would linger as a zombie for the agent's whole lifetime).
186	_ = cmd.Process.Kill()	202	_ = cmd.Process.Kill()
		203	_ = cmd.Wait()
187	return fmt.Errorf("write pidfile %s: %w", vmID, err)	204	return fmt.Errorf("write pidfile %s: %w", vmID, err)
188	}	205	}
189		206

internal/agent/cloudhv/cloudhv_test.go

Old		New
@@ -118,6 +118,9 @@ func TestPrepareDiskUsesReflinkAndResizes(t *testing.T) {
118	var cmds []string	118	var cmds []string
119	run := func(ctx context.Context, name string, args ...string) (string, error) {	119	run := func(ctx context.Context, name string, args ...string) (string, error) {
120	cmds = append(cmds, name+" "+strings.Join(args, " "))	120	cmds = append(cmds, name+" "+strings.Join(args, " "))
		121	if name == "cp" { // simulate the copy so the atomic rename has a file to move
		122	_ = os.WriteFile(args[len(args)-1], []byte("disk"), 0o600)
		123	}
121	return "", nil	124	return "", nil
122	}	125	}
123	st, _ := state.Open(t.TempDir())	126	st, _ := state.Open(t.TempDir())
@@ -126,9 +129,13 @@ func TestPrepareDiskUsesReflinkAndResizes(t *testing.T) {
126	require.NoError(t, p.PrepareDisk(context.Background(),	129	require.NoError(t, p.PrepareDisk(context.Background(),
127	state.VMSpec{VMID: "vm1", DiskGB: 10}, base))	130	state.VMSpec{VMID: "vm1", DiskGB: 10}, base))
128	joined := strings.Join(cmds, "\n")	131	joined := strings.Join(cmds, "\n")
		132	// Artifacts are built at a .partial sibling then renamed into place atomically.
		133	partial := st.DiskPath("vm1") + ".partial"
129	// reflink=auto: instant on XFS/btrfs, silent full-copy fallback on ext4 (spec)	134	// reflink=auto: instant on XFS/btrfs, silent full-copy fallback on ext4 (spec)
130	assert.Contains(t, joined, "cp --reflink=auto "+base+" "+st.DiskPath("vm1"))	135	assert.Contains(t, joined, "cp --reflink=auto "+base+" "+partial)
131	assert.Contains(t, joined, "truncate -s 10G "+st.DiskPath("vm1"))	136	assert.Contains(t, joined, "truncate -s 10G "+partial)
		137	assert.FileExists(t, st.DiskPath("vm1"), "disk.raw must be renamed into place")
		138	assert.NoFileExists(t, partial, "temp must not survive a successful prepare")
132	}	139	}
133		140
134	// TestPrepareDiskRefusesToShrinkBaseImage pins the never-shrink guard:	141	// TestPrepareDiskRefusesToShrinkBaseImage pins the never-shrink guard:
@@ -281,6 +288,9 @@ func TestPrepareDiskShrinkGuardEdgeCases(t *testing.T) {
281	newP := func(t testing.T, cmds []string) *Provisioner {	288	newP := func(t testing.T, cmds []string) *Provisioner {
282	run := func(ctx context.Context, name string, args ...string) (string, error) {	289	run := func(ctx context.Context, name string, args ...string) (string, error) {
283	cmds = append(cmds, name)	290	cmds = append(cmds, name)
		291	if name == "cp" { // simulate the copy so the atomic rename has a file to move
		292	_ = os.WriteFile(args[len(args)-1], []byte("disk"), 0o600)
		293	}
284	return "", nil	294	return "", nil
285	}	295	}
286	st, _ := state.Open(t.TempDir())	296	st, _ := state.Open(t.TempDir())

internal/agent/imagecache/imagecache.go

Old		New
@@ -112,15 +112,24 @@ func (c *Cache) Ensure(ctx context.Context, url, sha string) (string, error) {
112	return "", err	112	return "", err
113	}	113	}
114	defer os.Remove(tmp)	114	defer os.Remove(tmp)
115	// Convert to a temp file first; rename onto final atomically so a crash	115	// Convert to a UNIQUE temp file first, then rename onto final atomically so a
116	// mid-convert cannot leave a corrupt file at the final path.	116	// crash mid-convert cannot leave a corrupt file at the final path. The temp
117	converting := final + ".converting"	117	// name is unique (not a fixed "<sha>.converting") so two concurrent Ensure
		118	// calls for the same sha can never converge on one file and corrupt it. The
		119	// suffix deliberately does NOT end in ".raw", so evict()'s "*.raw" glob never
		120	// sees a half-built temp.
		121	cf, err := os.CreateTemp(c.dir, sha+".raw.converting-*")
		122	if err != nil {
		123	return "", fmt.Errorf("imagecache: create temp: %w", err)
		124	}
		125	converting := cf.Name()
		126	_ = cf.Close()
		127	_ = os.Remove(converting) // reserve the unique name; let qemu-img create it fresh
		128	defer os.Remove(converting)
118	if _, err := c.run(ctx, "qemu-img", "convert", "-O", "raw", tmp, converting); err != nil {	129	if _, err := c.run(ctx, "qemu-img", "convert", "-O", "raw", tmp, converting); err != nil {
119	os.Remove(converting)
120	return "", fmt.Errorf("qemu-img convert: %w", err)	130	return "", fmt.Errorf("qemu-img convert: %w", err)
121	}	131	}
122	if err := os.Rename(converting, final); err != nil {	132	if err := os.Rename(converting, final); err != nil {
123	os.Remove(converting)
124	return "", fmt.Errorf("imagecache: rename to final: %w", err)	133	return "", fmt.Errorf("imagecache: rename to final: %w", err)
125	}	134	}
126	c.evict(final)	135	c.evict(final)

internal/agent/ipalloc/ipalloc.go

Old		New
@@ -1,5 +1,7 @@
1	// Package ipalloc allocates VM IPs within the host's bridge CIDR.	1	// Package ipalloc allocates VM IPs within the host's bridge CIDR.
2	// .0 = network, .1 = bridge gateway, .255 = broadcast (for /24).	2	// The network address (first) is reserved, the gateway takes the next, and the
		3	// subnet broadcast (last) is reserved. For the /24 the server assigns today
		4	// those are .0, .1 and .255 respectively.
3	package ipalloc	5	package ipalloc
4		6
5	import (	7	import (
@@ -20,13 +22,25 @@ func Alloc(cidr string, used []string) (string, error) {
20	inUse[u] = true	22	inUse[u] = true
21	}	23	}
22	network := prefix.Masked().Addr()	24	network := prefix.Masked().Addr()
23	addr := network.Next().Next() // skip network + gateway	25	broadcast := broadcastAddr(prefix) // subnet-relative, not hardcoded .255
		26	addr := network.Next().Next() // skip network + gateway
24	for prefix.Contains(addr) {	27	for prefix.Contains(addr) {
25	a4 := addr.As4()	28	if addr != broadcast && !inUse[addr.String()] {
26	if a4[3] != 255 && !inUse[addr.String()] {
27	return addr.String(), nil	29	return addr.String(), nil
28	}	30	}
29	addr = addr.Next()	31	addr = addr.Next()
30	}	32	}
31	return "", fmt.Errorf("no free IP in %s", cidr)	33	return "", fmt.Errorf("no free IP in %s", cidr)
32	}	34	}
		35
		36	// broadcastAddr returns the all-host-bits-set (broadcast) address of an IPv4
		37	// prefix — e.g. 10.0.5.255 for 10.0.5.0/24, 10.0.0.63 for 10.0.0.0/26. The old
		38	// code hardcoded a last octet of 255, which is only the broadcast for /24 or
		39	// shorter; a /25–/30 bridge CIDR would otherwise hand out its true broadcast.
		40	func broadcastAddr(p netip.Prefix) netip.Addr {
		41	b := p.Masked().Addr().As4()
		42	n := uint32(b[0])<<24 \| uint32(b[1])<<16 \| uint32(b[2])<<8 \| uint32(b[3])
		43	host := uint32(0xffffffff) >> uint(p.Bits()) // low (32-bits) host bits set
		44	n \|= host
		45	return netip.AddrFrom4([4]byte{byte(n >> 24), byte(n >> 16), byte(n >> 8), byte(n)})
		46	}

internal/agent/ipalloc/ipalloc_test.go

Old		New
@@ -3,6 +3,7 @@ package ipalloc
3	import (	3	import (
4	"strconv"	4	"strconv"
5	"testing"	5	"testing"
		6
6	"github.com/stretchr/testify/assert"	7	"github.com/stretchr/testify/assert"
7	"github.com/stretchr/testify/require"	8	"github.com/stretchr/testify/require"
8	)	9	)
@@ -21,6 +22,22 @@ func TestAllocFirstVM(t *testing.T) {
21	assert.Equal(t, "10.77.1.2", ip)	22	assert.Equal(t, "10.77.1.2", ip)
22	}	23	}
23		24
		25	func TestAllocReservesSubnetBroadcastForNonSlash24(t *testing.T) {
		26	// /26 => hosts .0(network) .1(gw) ... .63(broadcast). The old hardcoded-.255
		27	// broadcast would have wrongly handed out .63 and stopped short of .255.
		28	used := make([]string, 0, 60)
		29	for i := 2; i <= 62; i++ { // fill .2..*.62, leaving only .63 (broadcast)
		30	used = append(used, "10.0.0."+itoa(i))
		31	}
		32	_, err := Alloc("10.0.0.0/26", used)
		33	assert.Error(t, err, ".63 is the /26 broadcast and must not be allocated")
		34
		35	// With .62 free it is allocatable (it is a valid host, not the broadcast).
		36	ip, err := Alloc("10.0.0.0/26", used[:len(used)-1])
		37	require.NoError(t, err)
		38	assert.Equal(t, "10.0.0.62", ip)
		39	}
		40
24	func TestAllocExhausted(t *testing.T) {	41	func TestAllocExhausted(t *testing.T) {
25	used := make([]string, 0, 253)	42	used := make([]string, 0, 253)
26	for i := 2; i <= 254; i++ {	43	for i := 2; i <= 254; i++ {

internal/agent/netenv/netenv.go

Old		New
@@ -1,7 +1,7 @@
1	// Package netenv manages the host side of VM networking: bridge eitri0 with	1	// Package netenv manages the host side of VM networking: bridge eitri0 with
2	// the host as .1 gateway, per-VM taps, and NAT for outbound internet. The	2	// the host as .1 gateway, per-VM taps, and NAT for outbound internet. The
3	// bridge is a pure masqueraded underlay; mesh connectivity (per-VM identity	3	// bridge is a pure masqueraded underlay; inbound admin reachability is via the
4	// and routing) is the guest's own concern via the rayfish (ray) binary.	4	// server's SSH-CA jump gate (internal/server/sshgate), not any guest overlay.
5	package netenv	5	package netenv
6		6
7	import (	7	import (

internal/agent/reconcile/reconcile.go

Old		New
@@ -2,18 +2,22 @@
2	//	2	//
3	// Definitions (normative, from the spec):	3	// Definitions (normative, from the spec):
4	//	4	//
5	// exists = state-dir record present AND (disk present OR create completed)	5	// exists = state-dir record present AND create completed (rec.BootID != "")
6	// lost = boot ID changed OR process died without a recorded stop request;	6	// lost = boot ID changed OR process died without a recorded stop request;
7	// a deliberately stopped VM is stopped, NOT lost	7	// a deliberately stopped VM is stopped, NOT lost
8	// destroyed[] ack = level-triggered: every tombstoned vm_id with no local	8	// destroyed[] ack = level-triggered: every tombstoned vm_id with no local
9	// record, repeated until the server hard-deletes it	9	// record, repeated until the server hard-deletes it
10	//	10	//
11	// The "exists" definition deserves a comment:	11	// The "exists" definition deserves a comment:
12	// On real hosts, disk presence is the physical witness that a create completed.	12	// rec.BootID is the sole completion witness. create() sets it to the current
13	// With fake provisioners (tests), no disk file is written, so we fall back to	13	// host boot ID only after every side effect (image, tap, disk, seed, boot) has
14	// the logical witness: a completed create always sets rec.BootID to the current	14	// succeeded, so rec.BootID != "" means — and only means — a create finished.
15	// host boot ID. rec.BootID != "" means create succeeded. Absence of both (no	15	// Disk presence is deliberately NOT consulted: create() writes disk.raw in the
16	// disk AND BootID == "") means an incomplete create that must be retried.	16	// middle of the sequence, so a create that fails after the disk is written but
		17	// before boot leaves a disk on a still-empty BootID; treating that as "exists"
		18	// would divert the retry to converge() (which rebuilds neither disk nor seed)
		19	// and strand the VM. Belt-and-suspenders: PrepareDisk and seed.Build both write
		20	// via a temp file + rename, so a killed create can never leave a torn artifact.
17	package reconcile	21	package reconcile
18		22
19	import (	23	import (
@@ -132,11 +136,7 @@ func (e Engine) Step(ctx context.Context, snap pb.DesiredStateSnapshot) *pb.Ac
132	for _, rec := range recs {	136	for _, rec := range recs {
133	// Fix 4: quarantined VMs belong in Quarantined[], not Vms[].	137	// Fix 4: quarantined VMs belong in Quarantined[], not Vms[].
134	if rec.QuarantinedAt != nil {	138	if rec.QuarantinedAt != nil {
135	grace := e.VanishGrace	139	rep.Quarantined = append(rep.Quarantined, quarantinedEntry(rec, e.graceFor(rec)))
136	if rec.QuarantineTombstoned {
137	grace = e.TombstoneGrace
138	}
139	rep.Quarantined = append(rep.Quarantined, quarantinedEntry(rec, grace))
140	continue	140	continue
141	}	141	}
142	power := "stopped"	142	power := "stopped"
@@ -206,10 +206,7 @@ func (e Engine) Step(ctx context.Context, snap pb.DesiredStateSnapshot) *pb.Ac
206	_ = e.St.SaveVM(rec)	206	_ = e.St.SaveVM(rec)
207	}	207	}
208		208
209	grace := e.VanishGrace	209	grace := e.graceFor(rec)
210	if rec.QuarantineTombstoned {
211	grace = e.TombstoneGrace
212	}
213		210
214	if now.Sub(*rec.QuarantinedAt) >= grace {	211	if now.Sub(*rec.QuarantinedAt) >= grace {
215	// Grace expired: destroy the VM.	212	// Grace expired: destroy the VM.
@@ -219,7 +216,16 @@ func (e Engine) Step(ctx context.Context, snap pb.DesiredStateSnapshot) *pb.Ac
219	// that honors ctx here would skip the destroy until a later tick,	216	// that honors ctx here would skip the destroy until a later tick,
220	// which the level-triggered loop tolerates but delays.	217	// which the level-triggered loop tolerates but delays.
221	_ = e.Prov.Kill(ctx, id)	218	_ = e.Prov.Kill(ctx, id)
222	_ = e.Net.DeleteTap(ctx, state.TapName(id))	219	if err := e.Net.DeleteTap(ctx, state.TapName(id)); err != nil {
		220	// TAP deletion failed — e.g. DeleteTap runs `ip link del` under
		221	// ctx and the ctx expired during a SIGTERM shutdown. KEEP the
		222	// record so a later tick retries; deleting it here would orphan
		223	// the eit-XXXXXXXX interface with nothing left to reap it (agent
		224	// restart's EnsureBridge does not sweep orphan taps). Kill already
		225	// stopped the guest, so re-entering here next tick (grace still
		226	// expired) just retries DeleteTap idempotently until it succeeds.
		227	continue
		228	}
223	_ = e.St.DeleteVM(id)	229	_ = e.St.DeleteVM(id)
224	// Do NOT append to rep.Quarantined — VM is gone.	230	// Do NOT append to rep.Quarantined — VM is gone.
225	} else {	231	} else {
@@ -232,6 +238,14 @@ func (e Engine) Step(ctx context.Context, snap pb.DesiredStateSnapshot) *pb.Ac
232	// Re-load records after reap pass. Every tombstoned ID with NO local record	238	// Re-load records after reap pass. Every tombstoned ID with NO local record
233	// is acked in destroyed[] every report until the server hard-deletes it.	239	// is acked in destroyed[] every report until the server hard-deletes it.
234	recs, _ = e.St.LoadVMs()	240	recs, _ = e.St.LoadVMs()
		241	// LoadVMs returns a nil map on a (transient) ReadDir failure. The converge
		242	// pass below WRITES into recs (recs[d.VmId]=rec after a durable create); a
		243	// write to a nil map panics. Fall back to an empty map so a load blip
		244	// degrades to an empty view (as the old read-only use did) instead of
		245	// crashing the agent.
		246	if recs == nil {
		247	recs = map[string]state.Record{}
		248	}
235	for id := range tombstoned {	249	for id := range tombstoned {
236	if _, hasRecord := recs[id]; !hasRecord {	250	if _, hasRecord := recs[id]; !hasRecord {
237	rep.Destroyed = append(rep.Destroyed, id)	251	rep.Destroyed = append(rep.Destroyed, id)
@@ -245,13 +259,16 @@ func (e Engine) Step(ctx context.Context, snap pb.DesiredStateSnapshot) *pb.Ac
245	continue // tombstoned VMs are handled by reap + destroyed[]	259	continue // tombstoned VMs are handled by reap + destroyed[]
246	}	260	}
247	rec, ok := recs[id]	261	rec, ok := recs[id]
248	// exists = record present AND (disk present OR create completed).	262	// exists = record present AND create completed. rec.BootID is the sole
249	// BootID != "" is the logical witness that create finished: the real	263	// completion witness: create() writes it only after every side effect
250	// provisioner creates the disk; in tests the fake does not, but both	264	// (image, tap, disk, seed, boot) has succeeded. Disk presence is NOT a
251	// set BootID after a successful create.	265	// witness — create() writes disk.raw mid-sequence, so a create that fails
252	exists := ok && (e.St.DiskExists(id) \|\| rec.BootID != "")	266	// after PrepareDisk but before boot leaves a disk on a still-empty BootID.
		267	// Treating that disk as "exists" would divert the retry to converge(),
		268	// which never rebuilds the disk or seed, and the VM would never recover.
		269	exists := ok && rec.BootID != ""
253	if !exists {	270	if !exists {
254	e.create(ctx, d, rec, ok, rep)	271	e.create(ctx, d, rep, recs)
255	} else {	272	} else {
256	e.converge(ctx, d, rec, rep)	273	e.converge(ctx, d, rec, rep)
257	}	274	}
@@ -260,19 +277,31 @@ func (e Engine) Step(ctx context.Context, snap pb.DesiredStateSnapshot) *pb.Ac
260	return rep	277	return rep
261	}	278	}
262		279
		280	// graceFor returns the quarantine grace period for rec: the shorter
		281	// TombstoneGrace for a tombstoned VM, else VanishGrace.
		282	func (e *Engine) graceFor(rec state.Record) time.Duration {
		283	if rec.QuarantineTombstoned {
		284	return e.TombstoneGrace
		285	}
		286	return e.VanishGrace
		287	}
		288
263	// quotaBlock returns a non-empty reason when booting d would exceed a configured	289	// quotaBlock returns a non-empty reason when booting d would exceed a configured
264	// host resource cap, else "". It sums the resources of the currently-committed	290	// host resource cap, else "". It sums the resources of the currently-committed
265	// local VMs — excluding quarantined records (being torn down; their guests are	291	// local VMs — excluding quarantined records (being torn down; their guests are
266	// already stopped) and d's own record (so a retry or spec-edit of an existing	292	// already stopped) and d's own record (so a retry or spec-edit of an existing
267	// VM does not double-count itself) — and adds d's request. The binding	293	// VM does not double-count itself) — and adds d's request. The binding
268	// dimension is named in the message so the operator sees which cap was hit.	294	// dimension is named in the message so the operator sees which cap was hit.
269	func (e Engine) quotaBlock(d pb.VMDesired) string {	295	func (e Engine) quotaBlock(d pb.VMDesired, recs map[string]state.Record) string {
270	if e.MaxVCPUs == 0 && e.MaxMemMB == 0 && e.MaxDiskGB == 0 {	296	if e.MaxVCPUs == 0 && e.MaxMemMB == 0 && e.MaxDiskGB == 0 {
271	return "" // no caps configured — unlimited	297	return "" // no caps configured — unlimited
272	}	298	}
273	spec := specFromDesired(d)	299	spec := specFromDesired(d)
274	vcpus, mem, disk := spec.VCPUs, spec.MemMB, spec.DiskGB	300	vcpus, mem, disk := spec.VCPUs, spec.MemMB, spec.DiskGB
275	recs, _ := e.St.LoadVMs()	301	// recs is the tick's live record map (loaded once post-reap in Step, kept
		302	// intra-tick-consistent as each create commits its record). Reading it —
		303	// rather than re-LoadVMs here — is what lets a second create in the same tick
		304	// see the first's committed resources and trip the cap.
276	for _, r := range recs {	305	for _, r := range recs {
277	if r.QuarantinedAt != nil \|\| r.Spec.VMID == d.VmId {	306	if r.QuarantinedAt != nil \|\| r.Spec.VMID == d.VmId {
278	continue	307	continue
@@ -293,8 +322,20 @@ func (e Engine) quotaBlock(d pb.VMDesired) string {
293	}	322	}
294		323
295	// create attempts to create a new VM from desired state d.	324	// create attempts to create a new VM from desired state d.
296	// rec is the existing (potentially stale) record, ok indicates whether one exists.	325	// recs is the tick's live record map (loaded once in Step); create looks up
297	func (e Engine) create(ctx context.Context, d pb.VMDesired, rec state.Record, ok bool, rep *pb.ActualStateReport) {	326	// d.VmId in it to get the existing (potentially stale) record, if any. create
		327	// also reads recs for the quota sum and the IP used-set INSTEAD of
		328	// re-LoadVMs-ing, and — crucially —
		329	// writes the new record back into it (recs[d.VmId]=rec) the moment that record's
		330	// Spec+IP are durably committed, so a later create in the same Step's converge
		331	// loop observes this VM's committed IP and resources. This reproduces exactly
		332	// what the old per-create LoadVMs did: pre-change it re-read disk and saw the
		333	// earlier create's just-saved record; now it reads the same fact from the shared
		334	// map. Step passes the same map reference to every create, so the write is
		335	// visible to subsequent calls.
		336	func (e Engine) create(ctx context.Context, d pb.VMDesired, rep *pb.ActualStateReport, recs map[string]state.Record) {
		337	rec, ok := recs[d.VmId]
		338
298	// A fired step watchdog is the STEP's failure, not this VM's: don't start	339	// A fired step watchdog is the STEP's failure, not this VM's: don't start
299	// an attempt (which would burn retry budget) with a dead context. Report	340	// an attempt (which would burn retry budget) with a dead context. Report
300	// and let the next tick — with a fresh budget — do the work. Pinned by	341	// and let the next tick — with a fresh budget — do the work. Pinned by
@@ -304,11 +345,13 @@ func (e Engine) create(ctx context.Context, d pb.VMDesired, rec state.Record,
304	return	345	return
305	}	346	}
306		347
		348	spec := specFromDesired(d)
		349
307	// Fix 2: if the desired spec differs from the stored spec, the user edited the	350	// Fix 2: if the desired spec differs from the stored spec, the user edited the
308	// VM definition. Reset CreateAttempts so the new spec gets a fresh retry budget	351	// VM definition. Reset CreateAttempts so the new spec gets a fresh retry budget
309	// instead of being permanently terminal-failed due to the old spec's failures.	352	// instead of being permanently terminal-failed due to the old spec's failures.
310	// VMSpec is fully comparable (all fields are strings/ints/bool), so == is safe.	353	// VMSpec is fully comparable (all fields are strings/ints/bool), so == is safe.
311	if ok && specFromDesired(d) != rec.Spec {	354	if ok && spec != rec.Spec {
312	rec.CreateAttempts = 0	355	rec.CreateAttempts = 0
313	rec.LastError = ""	356	rec.LastError = ""
314	}	357	}
@@ -324,28 +367,35 @@ func (e Engine) create(ctx context.Context, d pb.VMDesired, rec state.Record,
324	// before touching CreateAttempts or any state, so once a running VM is	367	// before touching CreateAttempts or any state, so once a running VM is
325	// removed and room frees, the next level-triggered tick retries and boots.	368	// removed and room frees, the next level-triggered tick retries and boots.
326	// A newly-lowered cap never kills a running guest; only new boots are gated.	369	// A newly-lowered cap never kills a running guest; only new boots are gated.
327	if msg := e.quotaBlock(d); msg != "" {	370	if msg := e.quotaBlock(d, recs); msg != "" {
328	addReport(rep, d.VmId, rec.IP, "stopped", "failed", msg)	371	addReport(rep, d.VmId, rec.IP, "stopped", "failed", msg)
329	return	372	return
330	}	373	}
331		374
332	// Build the spec from desired.	375	// Build the spec from desired.
333	rec.Spec = specFromDesired(d)	376	rec.Spec = spec
334	rec.CreateAttempts++	377	rec.CreateAttempts++
335	rec.CreatedAt = e.Now()	378	rec.CreatedAt = e.Now()
336	rec.LastError = "" // clear for this attempt	379	rec.LastError = "" // clear for this attempt
337		380
338	// Allocate an IP if we don't have one yet.	381	// Allocate an IP if we don't have one yet. The used-set is built from the
		382	// tick's live record map (kept intra-tick-consistent below), so two VMs
		383	// created in one Step never collide on an address.
339	if rec.IP == "" {	384	if rec.IP == "" {
340	allRecs, _ := e.St.LoadVMs()	385	used := make([]string, 0, len(recs))
341	used := make([]string, 0, len(allRecs))	386	for _, r := range recs {
342	for _, r := range allRecs {
343	if r.IP != "" {	387	if r.IP != "" {
344	used = append(used, r.IP)	388	used = append(used, r.IP)
345	}	389	}
346	}	390	}
347	ip, err := e.Net.AllocateIP(ctx, used)	391	ip, err := e.Net.AllocateIP(ctx, used)
348	if err != nil {	392	if err != nil {
		393	// failCreate durably saves this record (Spec set, IP still empty).
		394	// Publish it so a same-tick sibling's quota check counts its Spec —
		395	// matching the old per-create LoadVMs, which re-read the failCreate
		396	// save. Tick-scoped: conservative inclusion only over-counts for one
		397	// tick and self-heals on the next fresh load.
		398	recs[d.VmId] = rec
349	e.failCreate(ctx, rec, err, rep)	399	e.failCreate(ctx, rec, err, rep)
350	return	400	return
351	}	401	}
@@ -354,9 +404,21 @@ func (e Engine) create(ctx context.Context, d pb.VMDesired, rec state.Record,
354		404
355	// Record BEFORE side effects so a crash is recoverable.	405	// Record BEFORE side effects so a crash is recoverable.
356	if err := e.St.SaveVM(rec); err != nil {	406	if err := e.St.SaveVM(rec); err != nil {
		407	// This save failed but failCreate's save may succeed, durably committing
		408	// the allocated IP. Publish so a same-tick sibling excludes that IP from
		409	// its used-set and counts its Spec — preventing a double-allocation the
		410	// old per-create LoadVMs could not produce.
		411	recs[d.VmId] = rec
357	e.failCreate(ctx, rec, err, rep)	412	e.failCreate(ctx, rec, err, rep)
358	return	413	return
359	}	414	}
		415	// Intra-tick consistency: this record's Spec+IP are now durably committed.
		416	// Publish it into the shared tick map so a later create in this same Step
		417	// sees it — its quotaBlock sums this VM's resources and its IP used-set
		418	// excludes this VM's address. This mirrors the old per-create LoadVMs, which
		419	// re-read exactly this just-saved record. Only Spec/IP matter to quota/IP;
		420	// the later BootID/StopRequested save does not change them, so we update here.
		421	recs[d.VmId] = rec
360		422
361	// Resolve base image.	423	// Resolve base image.
362	basePath, err := e.Images(ctx, d.ImageUrl, d.ImageSha256)	424	basePath, err := e.Images(ctx, d.ImageUrl, d.ImageSha256)
@@ -452,6 +514,16 @@ func (e *Engine) failCreate(ctx context.Context, rec state.Record, err error, re
452	addReport(rep, rec.Spec.VMID, rec.IP, "stopped", phase, rec.LastError)	514	addReport(rep, rec.Spec.VMID, rec.IP, "stopped", phase, rec.LastError)
453	}	515	}
454		516
		517	// failConverge is the shared epilogue for the converge restart/boot paths:
		518	// persist err on the record and report it stopped/failed. Unlike failCreate
		519	// there is no create-attempt budget — a converge (already-created VM) failure
		520	// is terminal-for-this-tick and reported failed immediately.
		521	func (e Engine) failConverge(rec state.Record, err error, rep pb.ActualStateReport) {
		522	rec.LastError = err.Error()
		523	_ = e.St.SaveVM(rec)
		524	addReport(rep, rec.Spec.VMID, rec.IP, "stopped", "failed", rec.LastError)
		525	}
		526
455	// converge drives an existing VM toward its desired power state,	527	// converge drives an existing VM toward its desired power state,
456	// handling lost detection and restart logic.	528	// handling lost detection and restart logic.
457	func (e Engine) converge(ctx context.Context, d pb.VMDesired, rec state.Record, rep *pb.ActualStateReport) {	529	func (e Engine) converge(ctx context.Context, d pb.VMDesired, rec state.Record, rep *pb.ActualStateReport) {
@@ -479,15 +551,11 @@ func (e Engine) converge(ctx context.Context, d pb.VMDesired, rec state.Record
479	// ITS message — letting Boot fail instead yields an illegible	551	// ITS message — letting Boot fail instead yields an illegible
480	// cloud-hypervisor error for the same root cause.	552	// cloud-hypervisor error for the same root cause.
481	if err := e.Net.CreateTap(ctx, state.TapName(d.VmId)); err != nil {	553	if err := e.Net.CreateTap(ctx, state.TapName(d.VmId)); err != nil {
482	rec.LastError = err.Error()	554	e.failConverge(rec, err, rep)
483	_ = e.St.SaveVM(rec)
484	addReport(rep, d.VmId, rec.IP, "stopped", "failed", rec.LastError)
485	return	555	return
486	}	556	}
487	if err := e.Prov.Boot(ctx, d.VmId, rec.Spec); err != nil {	557	if err := e.Prov.Boot(ctx, d.VmId, rec.Spec); err != nil {
488	rec.LastError = err.Error()	558	e.failConverge(rec, err, rep)
489	_ = e.St.SaveVM(rec)
490	addReport(rep, d.VmId, rec.IP, "stopped", "failed", rec.LastError)
491	return	559	return
492	}	560	}
493	rec.BootID = bootID	561	rec.BootID = bootID
@@ -509,9 +577,7 @@ func (e Engine) converge(ctx context.Context, d pb.VMDesired, rec state.Record
509	if d.PowerState == "running" && !running {	577	if d.PowerState == "running" && !running {
510	// Start the VM.	578	// Start the VM.
511	if err := e.Prov.Boot(ctx, d.VmId, rec.Spec); err != nil {	579	if err := e.Prov.Boot(ctx, d.VmId, rec.Spec); err != nil {
512	rec.LastError = err.Error()	580	e.failConverge(rec, err, rep)
513	_ = e.St.SaveVM(rec)
514	addReport(rep, d.VmId, rec.IP, "stopped", "failed", rec.LastError)
515	return	581	return
516	}	582	}
517	rec.StopRequested = false	583	rec.StopRequested = false
@@ -540,10 +606,9 @@ func (e Engine) converge(ctx context.Context, d pb.VMDesired, rec state.Record
540	if running {	606	if running {
541	power = "running"	607	power = "running"
542	}	608	}
543	phase := "ready"
544	// Preserve last error in the report field but phase stays ready	609	// Preserve last error in the report field but phase stays ready
545	// (the VM is converged; the error is informational history).	610	// (the VM is converged; the error is informational history).
546	addReport(rep, d.VmId, rec.IP, power, phase, rec.LastError)	611	addReport(rep, d.VmId, rec.IP, power, "ready", rec.LastError)
547	}	612	}
548	}	613	}
549		614

internal/agent/reconcile/reconcile_test.go

@@ @@ -4,6 +4,8 @@ import ( @@
+	"context"
+	"errors"
+	"net/netip"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+func (f *fakeProv) Running(id string) bool { return f.running[id] }
+type fakeNet struct {
+	taps     []string
+	deleted  []string
+	cidr     string
+	allocErr error // one-shot: consumed and cleared on the first AllocateIP call
+func (f *fakeNet) CreateTap(_ context.Context, t string) error {
+// AllocateIP / GuestNetwork mirror netenv's host-local behavior so reconcile
+// tests exercise identical addressing through the seam.
+func (f *fakeNet) AllocateIP(_ context.Context, used []string) (string, error) {
+	if f.allocErr != nil {
+		err := f.allocErr
+		f.allocErr = nil // one-shot
+		return "", err
+	}
+	return ipalloc.Alloc(f.cidr, used)
+func (f *fakeNet) GuestNetwork() (string, int) {
@@ @@ -157,6 +165,81 @@ func TestCreateAllocatesIPPreparesAndBoots(t *testing.T) { @@
+	assert.Equal(t, uint64(1), rep.LastSeenEpoch)
+// TestIncompleteCreateWithDiskPresentIsRetried pins the completion-witness fix:
+// a create that failed after PrepareDisk wrote disk.raw but before boot leaves a
+// record with an empty BootID AND a disk file on disk. The witness is BootID, not
+// disk presence — so the next tick must re-run create() (rebuilding disk + seed),
+// NOT divert to converge() (which rebuilds neither and would strand the VM).
+func TestIncompleteCreateWithDiskPresentIsRetried(t *testing.T) {
+	f := setup(t)
+	const id = "vm1"
+	// Simulate the interrupted create: record saved pre-boot (BootID == ""),
+	// with the disk already materialised so the disk file exists on disk.
+	require.NoError(t, f.st.SaveVM(state.Record{
+		Spec:      state.VMSpec{VMID: id, Name: "vm-" + id, VCPUs: 1, MemMB: 512, DiskGB: 5},
+		IP:        "10.77.1.2",
+		BootID:    "", // create never completed
+		CreatedAt: f.now,
+	}))
+	require.NoError(t, os.MkdirAll(f.st.VMDir(id), 0o700))
+	require.NoError(t, os.WriteFile(f.st.DiskPath(id), []byte("partial"), 0o600))
+	_, statErr := os.Stat(f.st.DiskPath(id))
+	require.NoError(t, statErr, "precondition: disk present")
+	rep := f.eng.Step(context.Background(), snap(1, vm(id)))
+	assert.Equal(t, []string{id}, f.prov.prepared, "must re-run create (PrepareDisk), not converge")
+	assert.Equal(t, []string{id}, f.prov.booted)
+	av := findVM(rep, id)
+	require.NotNil(t, av)
+	assert.Equal(t, "ready", av.Phase)
+}
+// TestCreateSurvivesRecordLoadFailure guards against the nil-map panic: when
+// LoadVMs errors (transient ReadDir failure) it returns a nil map, and the
+// converge pass WRITES into that map (recs[d.VmId]=rec) after a durable create.
+// A write to a nil map panics — crashing the agent where the old read-only use
+// merely degraded. We force the failure by removing the vms/ dir; SaveVM
+// re-creates vms/<id> via MkdirAll, so the create still commits and the write
+// path is exercised.
+func TestCreateSurvivesRecordLoadFailure(t *testing.T) {
+	f := setup(t)
+	vmsDir := filepath.Dir(f.st.VMDir("placeholder"))
+	require.NoError(t, os.RemoveAll(vmsDir))
+	require.NotPanics(t, func() {
+		rep := f.eng.Step(context.Background(), snap(1, vm("vm1")))
+		av := findVM(rep, "vm1")
+		require.NotNil(t, av)
+		assert.Equal(t, "ready", av.Phase)
+	})
+}
+// TestSameTickSiblingCountsFailedCreateAgainstQuota pins that a VM whose create
+// fails early (here: IP allocation) still has its Spec published into the tick's
+// record map, so a SAME-TICK sibling's quota check counts it. Both VMs are
+// identical (2 vCPU) under a 3-vCPU cap, so whichever is processed first fails IP
+// allocation and the other must be quota-blocked — NEITHER boots. Before the fix
+// the failed VM was invisible to the sibling, which wrongly booted.
+func TestSameTickSiblingCountsFailedCreateAgainstQuota(t *testing.T) {
+	f := setup(t)
+	f.eng.MaxVCPUs = 3
+	f.net.allocErr = assert.AnError
+	twoVCPU := func(v *pb.VMDesired) { v.Vcpus = 2 }
+	rep := f.eng.Step(context.Background(), snap(1, vm("vm1", twoVCPU), vm("vm2", twoVCPU)))
+	ready := 0
+	for _, v := range rep.Vms {
+		if v.Phase == "ready" {
+			ready++
+		}
+	}
+	assert.Equal(t, 0, ready, "a failed-create sibling must count against the other's cap")
+	assert.Empty(t, f.prov.booted, "neither VM should boot")
+}
+func TestFenceRefusesLowerEpochWithoutActing(t *testing.T) {
+	f := setup(t)
+	f.eng.Step(context.Background(), snap(5, vm("vm1")))
+	assert.Equal(t, 1, f.prov.prepCalls, "no further provisioner attempts after a permanent failure")
+// TestTwoVMsCreatedInOneStepGetDistinctIPs pins intra-tick IP consistency:
+// when a single Step creates two VMs, the second create must SEE the first's
+// just-allocated IP in its used-set and pick a different one. A naive
+// implementation that threads the pre-create records map (loaded before any
+// create) into the IP used-set would hand both creates the same empty used-set
+// and double-allocate the same address. The reconcile loop's converge order is
+// randomized, so this must hold regardless of which VM is created first.
+func TestTwoVMsCreatedInOneStepGetDistinctIPs(t *testing.T) {
+	f := setup(t)
+	f.eng.Step(context.Background(), snap(1, vm("vm1"), vm("vm2")))
+	recs, err := f.st.LoadVMs()
+	require.NoError(t, err)
+	require.Contains(t, recs, "vm1")
+	require.Contains(t, recs, "vm2")
+	assert.NotEmpty(t, recs["vm1"].IP)
+	assert.NotEmpty(t, recs["vm2"].IP)
+	assert.NotEqual(t, recs["vm1"].IP, recs["vm2"].IP,
+		"two VMs created in one Step must not share an IP (intra-tick used-set)")
+}
+// TestSecondVMInOneStepBustingCapIsBlocked pins intra-tick quota consistency:
+// two 2-vcpu VMs sum to 4 > the 3-vcpu cap, so exactly one may boot per tick.
+// The second create's quotaBlock must SEE the first's just-committed record so
+// the running total (2 + 2 = 4) trips the cap. A naive implementation feeding
+// the pre-create records map to quotaBlock would count 0 committed for both and
+// let both boot (over-commit). Converge order is randomized, so we assert on the
+// count and identify the blocked VM dynamically.
+func TestSecondVMInOneStepBustingCapIsBlocked(t *testing.T) {
+	f := setup(t)
+	f.eng.MaxVCPUs = 3
+	rep := f.eng.Step(context.Background(), snap(1,
+		vm("vm1", withRes(2, 512, 5)),
+		vm("vm2", withRes(2, 512, 5))))
+	require.Len(t, f.prov.booted, 1, "exactly one of two cap-busting VMs may boot in one tick")
+	other := "vm1"
+	if f.prov.booted[0] == "vm1" {
+		other = "vm2"
+	}
+	av := findVM(rep, other)
+	require.NotNil(t, av)
+	assert.Equal(t, "failed", av.Phase, "the second cap-busting VM must be quota-blocked")
+	assert.Contains(t, av.GetLastError(), "capacity limit")
+	assert.Contains(t, av.GetLastError(), "vcpus")
+}
+// TestTransientCreateErrorStillRetries pins the counterpart: unmarked errors
+// keep the existing bounded-retry behavior (phase creating until the budget
+// is spent).

internal/agent/seed/seed.go

Old		New
@@ -55,6 +55,7 @@ func validateParams(p Params) error {
55	value string	55	value string
56	}{	56	}{
57	{"Hostname", p.Hostname},	57	{"Hostname", p.Hostname},
		58	{"InstanceID", p.InstanceID},
58	{"SSHAuthorizedKey", p.SSHAuthorizedKey},	59	{"SSHAuthorizedKey", p.SSHAuthorizedKey},
59	{"IP", p.IP},	60	{"IP", p.IP},
60	{"Gateway", p.Gateway},	61	{"Gateway", p.Gateway},
@@ -237,11 +238,37 @@ func Build(outPath string, p Params) error {
237	}	238	}
238	defer os.RemoveAll(workDir)	239	defer os.RemoveAll(workDir)
239		240
240	d, err := diskfs.Create(outPath, isoSize, isoSectorSize)	241	// Build into a sibling temp file and rename into place, so a Build that fails
		242	// or is killed part-way (MkdirTemp, filesystem create, the write loop, or
		243	// Finalize) can never leave a partial/empty seed.iso at the final path. The
		244	// reconcile "exists" gate keys off rec.BootID rather than seed presence, but
		245	// an atomic artifact keeps on-disk state honest. The deferred guard below
		246	// closes the image (if not already closed) and drops the temp on any error
		247	// path; on success we close explicitly before the rename so the ISO is fully
		248	// flushed, and the guard is disarmed.
		249	tmpPath := outPath + ".partial"
		250	_ = os.Remove(tmpPath)
		251
		252	d, err := diskfs.Create(tmpPath, isoSize, isoSectorSize)
241	if err != nil {	253	if err != nil {
242	return fmt.Errorf("seed: create disk image: %w", err)	254	return fmt.Errorf("seed: create disk image: %w", err)
243	}	255	}
244	defer d.Close()	256	// success and closed track how far Build got. While d is still open, the
		257	// guard closes it before removing the temp file; once Close has been
		258	// attempted (success or fail), it must not be called again — the guard
		259	// then just removes the temp file, matching the two distinct cleanups the
		260	// manual version used at the close-failure and rename-failure sites.
		261	success := false
		262	closed := false
		263	defer func() {
		264	if success {
		265	return
		266	}
		267	if !closed {
		268	d.Close()
		269	}
		270	_ = os.Remove(tmpPath)
		271	}()
245		272
246	fsi, err := d.CreateFilesystem(disk.FilesystemSpec{	273	fsi, err := d.CreateFilesystem(disk.FilesystemSpec{
247	Partition: 0,	274	Partition: 0,
@@ -285,5 +312,16 @@ func Build(outPath string, p Params) error {
285	return fmt.Errorf("seed: finalize iso: %w", err)	312	return fmt.Errorf("seed: finalize iso: %w", err)
286	}	313	}
287		314
		315	closeErr := d.Close()
		316	closed = true
		317	if closeErr != nil {
		318	return fmt.Errorf("seed: close iso: %w", closeErr)
		319	}
		320
		321	if err := os.Rename(tmpPath, outPath); err != nil {
		322	return fmt.Errorf("seed: rename %s -> %s: %w", tmpPath, outPath, err)
		323	}
		324
		325	success = true
288	return nil	326	return nil
289	}	327	}

internal/agent/state/state.go

Old		New
@@ -15,7 +15,7 @@ import (
15	type VMSpec struct {	15	type VMSpec struct {
16	VMID, Name, ImageURL, ImageSHA256, CloudInit, SSHAuthorizedKey string	16	VMID, Name, ImageURL, ImageSHA256, CloudInit, SSHAuthorizedKey string
17	VCPUs, MemMB, DiskGB int64	17	VCPUs, MemMB, DiskGB int64
18	Persistent bool	18	Persistent bool
19	}	19	}
20		20
21	type Record struct {	21	type Record struct {
@@ -32,8 +32,8 @@ type Record struct {
32		32
33	type Identity struct {	33	type Identity struct {
34	HostID, Credential, BridgeCIDR string	34	HostID, Credential, BridgeCIDR string
35	ServerQUICAddr string // host:port for QUIC dial	35	ServerQUICAddr string // host:port for QUIC dial
36	ServerCertSHA256 string // pinned server cert fingerprint	36	ServerCertSHA256 string // pinned server cert fingerprint
37	}	37	}
38		38
39	type Store struct{ dir string }	39	type Store struct{ dir string }
@@ -70,6 +70,12 @@ func (s *Store) SerialSocketPath(vmID string) string {
70	return filepath.Join(s.VMDir(vmID), "serial.sock")	70	return filepath.Join(s.VMDir(vmID), "serial.sock")
71	}	71	}
72		72
		73	// SerialLogPath returns the path of the VM's on-disk serial console log,
		74	// written by serialpump so console history survives an agent restart.
		75	func (s *Store) SerialLogPath(vmID string) string {
		76	return filepath.Join(s.VMDir(vmID), "serial.log")
		77	}
		78
73	// TapName returns the TAP device name for a given vmID. The name is truncated	79	// TapName returns the TAP device name for a given vmID. The name is truncated
74	// to the first 8 characters of the vmID, giving "eit-XXXXXXXX" (12 chars),	80	// to the first 8 characters of the vmID, giving "eit-XXXXXXXX" (12 chars),
75	// which is safely below the 15-char IFNAMSIZ limit.	81	// which is safely below the 15-char IFNAMSIZ limit.
@@ -161,17 +167,9 @@ func (s *Store) DeleteVM(vmID string) error {
161	return os.RemoveAll(s.VMDir(vmID))	167	return os.RemoveAll(s.VMDir(vmID))
162	}	168	}
163		169
164	// DiskExists reports whether the VM's disk image file exists on disk.
165	func (s *Store) DiskExists(vmID string) bool {
166	_, err := os.Stat(s.DiskPath(vmID))
167	return err == nil
168	}
169
170	// epochPath returns the path to the epoch file.	170	// epochPath returns the path to the epoch file.
171	func (s *Store) epochPath() string { return filepath.Join(s.dir, "epoch") }	171	func (s *Store) epochPath() string { return filepath.Join(s.dir, "epoch") }
172		172
173	// Epoch reads the current epoch from disk, returning 0 if the file does not
174	// exist (fresh store).
175	// Epoch returns the highest epoch this agent has seen, or 0 for a fresh	173	// Epoch returns the highest epoch this agent has seen, or 0 for a fresh
176	// store. DELIBERATE FAIL-OPEN: a corrupt epoch file also reads as 0, which	174	// store. DELIBERATE FAIL-OPEN: a corrupt epoch file also reads as 0, which
177	// resets the reaping fence; the quarantine grace period (not the fence) is	175	// resets the reaping fence; the quarantine grace period (not the fence) is

internal/agent/state/state_paths_test.go

@@ @@ -0,0 +1,28 @@ @@
+package state
+import (
+	"path/filepath"
+	"testing"
+)
+// TestStorePathHelpers pins the per-VM path derivations — including
+// SerialLogPath, added for the on-disk serial console log — to their layout
+// under the state root.
+func TestStorePathHelpers(t *testing.T) {
+	s, err := Open(t.TempDir())
+	if err != nil {
+		t.Fatalf("Open: %v", err)
+	}
+	const vm = "vm-42"
+	checks := []struct{ name, got, want string }{
+		{"ImagesDir", s.ImagesDir(), filepath.Join(s.dir, "images")},
+		{"SeedPath", s.SeedPath(vm), filepath.Join(s.VMDir(vm), "seed.iso")},
+		{"SocketPath", s.SocketPath(vm), filepath.Join(s.VMDir(vm), "ch.sock")},
+		{"SerialLogPath", s.SerialLogPath(vm), filepath.Join(s.VMDir(vm), "serial.log")},
+	}
+	for _, c := range checks {
+		if c.got != c.want {
+			t.Errorf("%s = %q, want %q", c.name, c.got, c.want)
+		}
+	}
+}

internal/agent/state/state_test.go

Old		New
@@ -62,10 +62,12 @@ func TestSerialSocketPath(t *testing.T) {
62	assert.Equal(t, filepath.Join(s.VMDir("vm1"), "serial.sock"), p)	62	assert.Equal(t, filepath.Join(s.VMDir("vm1"), "serial.sock"), p)
63	}	63	}
64		64
65	func TestDiskExistsReflectsDiskFile(t *testing.T) {	65	func TestDiskPathLocatesDiskFile(t *testing.T) {
66	s := open(t)	66	s := open(t)
67	require.NoError(t, s.SaveVM(Record{Spec: VMSpec{VMID: "vm1"}}))	67	require.NoError(t, s.SaveVM(Record{Spec: VMSpec{VMID: "vm1"}}))
68	assert.False(t, s.DiskExists("vm1"), "record without disk: VM does not 'exist' (spec)")	68	_, err := os.Stat(s.DiskPath("vm1"))
		69	assert.True(t, os.IsNotExist(err), "record without disk: DiskPath points at a file that does not yet exist")
69	require.NoError(t, os.WriteFile(s.DiskPath("vm1"), []byte("disk"), 0o644))	70	require.NoError(t, os.WriteFile(s.DiskPath("vm1"), []byte("disk"), 0o644))
70	assert.True(t, s.DiskExists("vm1"))	71	_, err = os.Stat(s.DiskPath("vm1"))
		72	assert.NoError(t, err)
71	}	73	}

internal/agent/syncclient/client.go

Old		New
@@ -34,6 +34,10 @@ func HostBootID() string {
34	// capacity returns the host's TOTAL capacity: total disk at stateDir, total mem,	34	// capacity returns the host's TOTAL capacity: total disk at stateDir, total mem,
35	// and CPU count. The server computes allocated/available by subtracting the sum	35	// and CPU count. The server computes allocated/available by subtracting the sum
36	// of live VM specs, so capacity must be totals (not free) for the math to cohere.	36	// of live VM specs, so capacity must be totals (not free) for the math to cohere.
		37	// computeCapacity is the raw-capacity source, indirected through a var so tests
		38	// can count how often the (memoized) computation actually runs.
		39	var computeCapacity = capacity
		40
37	func capacity(stateDir string) *pb.Capacity {	41	func capacity(stateDir string) *pb.Capacity {
38	var fs syscall.Statfs_t	42	var fs syscall.Statfs_t
39	var diskGB int64	43	var diskGB int64
@@ -67,6 +71,16 @@ type Console interface {
67	Attach(ctx context.Context, vmID string, rw io.ReadWriter, onReady func() error) error	71	Attach(ctx context.Context, vmID string, rw io.ReadWriter, onReady func() error) error
68	}	72	}
69		73
		74	// DefaultTickInterval is the fallback report/reconcile cadence when
		75	// Client.TickInterval is zero. It is COUPLED to the server's
		76	// registry.OnlineWindow (30s): the server marks a host offline after
		77	// OnlineWindow of silence, so the agent must report well inside it. The invariant
		78	// OnlineWindow >= 3*DefaultTickInterval gives a ~3-missed-report margin before a
		79	// healthy host flaps Online/Offline; changing either value alone erodes it (see
		80	// registry.OnlineWindow and the invariant test in this package). Independent of
		81	// transport.SyncKeepAlivePeriod, which keeps the QUIC connection itself alive.
		82	const DefaultTickInterval = 10 * time.Second
		83
70	// Client manages the agent's QUIC sync session.	84	// Client manages the agent's QUIC sync session.
71	type Client struct {	85	type Client struct {
72	Engine *reconcile.Engine	86	Engine *reconcile.Engine
@@ -85,7 +99,7 @@ type Client struct {
85		99
86	// TickInterval is the period of the fallback ticker that drives a reconcile	100	// TickInterval is the period of the fallback ticker that drives a reconcile
87	// step even when no new snapshot has arrived (e.g. for periodic health	101	// step even when no new snapshot has arrived (e.g. for periodic health
88	// reports). Zero uses the default of 10 seconds.	102	// reports). Zero uses DefaultTickInterval (coupled to registry.OnlineWindow).
89	TickInterval time.Duration	103	TickInterval time.Duration
90		104
91	// ReconnectBackoff is the sleep between a transient session failure and the	105	// ReconnectBackoff is the sleep between a transient session failure and the
@@ -99,6 +113,16 @@ type Client struct {
99	MaxVCPUs int64	113	MaxVCPUs int64
100	MaxMemMB int64	114	MaxMemMB int64
101	MaxDiskGB int64	115	MaxDiskGB int64
		116
		117	// rawCap memoizes the machine's real capacity (vCPUs/RAM/disk). Host totals
		118	// don't change over a session, so the Statfs+Sysinfo syscalls run once
		119	// instead of on every report; the cheap per-report clamp still applies. Only
		120	// a FULL reading is cached — capacity() silently zeroes a dimension whose
		121	// syscall failed, and caching that would freeze a bad advertisement for the
		122	// whole session, so a zeroed probe is re-tried on the next call. Guarded by
		123	// rawCapMu (hello and report goroutines both call advertisedCapacity).
		124	rawCapMu sync.Mutex
		125	rawCap *pb.Capacity
102	}	126	}
103		127
104	// clampCapacity reduces each advertised dimension to its configured cap	128	// clampCapacity reduces each advertised dimension to its configured cap
@@ -120,9 +144,24 @@ func clampDim(actual, cap int64) int64 {
120	}	144	}
121		145
122	// advertisedCapacity is the machine's real capacity clamped to this agent's	146	// advertisedCapacity is the machine's real capacity clamped to this agent's
123	// configured caps — what the agent reports to the server.	147	// configured caps — what the agent reports to the server. The raw capacity is
		148	// computed once (host totals are fixed for the session) and re-clamped cheaply
		149	// on every call.
124	func (c Client) advertisedCapacity(stateDir string) pb.Capacity {	150	func (c Client) advertisedCapacity(stateDir string) pb.Capacity {
125	return clampCapacity(capacity(stateDir), c.MaxVCPUs, c.MaxMemMB, c.MaxDiskGB)	151	c.rawCapMu.Lock()
		152	raw := c.rawCap
		153	if raw == nil {
		154	raw = computeCapacity(stateDir)
		155	// Cache only a full reading. A zeroed dimension means the Statfs/Sysinfo
		156	// probe failed; caching it would freeze a bad total for the session, so
		157	// leave rawCap nil and re-probe next call (vCPUs is runtime.NumCPU, never
		158	// zero, so only mem/disk gate the memoization).
		159	if raw.GetMemMb() > 0 && raw.GetDiskGb() > 0 {
		160	c.rawCap = raw
		161	}
		162	}
		163	c.rawCapMu.Unlock()
		164	return clampCapacity(raw, c.MaxVCPUs, c.MaxMemMB, c.MaxDiskGB)
126	}	165	}
127		166
128	// errPermanentAuth marks a credential rejection so Run() backs off long instead	167	// errPermanentAuth marks a credential rejection so Run() backs off long instead
@@ -167,8 +206,8 @@ func (c *Client) Run(ctx context.Context) {
167	} else {	206	} else {
168	failures++	207	failures++
169	if failures >= 3 {	208	if failures >= 3 {
170	slog.Warn("control-plane unreachable — QUIC/UDP may be blocked or filtered on this network",	209	slog.Warn("control-plane sync failing repeatedly — server unreachable OR wedged (accepting connections but not serving snapshots); check the server before suspecting the network",
171	"server", c.Identity.ServerQUICAddr, "consecutive_failures", failures)	210	"server", c.Identity.ServerQUICAddr, "consecutive_failures", failures, "last_err", err)
172	}	211	}
173	}	212	}
174	backoff := c.ReconnectBackoff	213	backoff := c.ReconnectBackoff
@@ -200,7 +239,8 @@ var errSessionConnected = errors.New("session was connected")
200	func (c *Client) session(ctx context.Context) error {	239	func (c *Client) session(ctx context.Context) error {
201	tlsConf := transport.ClientTLS(c.Identity.ServerCertSHA256)	240	tlsConf := transport.ClientTLS(c.Identity.ServerCertSHA256)
202	conn, err := quic.DialAddr(ctx, c.Identity.ServerQUICAddr, tlsConf,	241	conn, err := quic.DialAddr(ctx, c.Identity.ServerQUICAddr, tlsConf,
203	&quic.Config{KeepAlivePeriod: 15 * time.Second, MaxIdleTimeout: 30 * time.Second})	242	// Shared with the server listener via transport so the two ends can't drift.
		243	transport.SyncQUICConfig())
204	if err != nil {	244	if err != nil {
205	return classifyErr(err) // transient: Run() backs off	245	return classifyErr(err) // transient: Run() backs off
206	}	246	}
@@ -256,6 +296,15 @@ func (c *Client) session(ctx context.Context) error {
256	stepSignal := make(chan struct{}, 1)	296	stepSignal := make(chan struct{}, 1)
257	errc := make(chan error, 2)	297	errc := make(chan error, 2)
258		298
		299	// sessCtx is cancelled when session returns (for ANY reason), giving every
		300	// worker goroutine a signal to exit. Most sessions end because the recv
		301	// goroutine errored — NOT because the parent ctx was cancelled — so a worker
		302	// that selects only on the long-lived parent ctx (plus a dead stepSignal and
		303	// a stopped ticker) would never observe session end and would leak one
		304	// goroutine per reconnect.
		305	sessCtx, cancel := context.WithCancel(ctx)
		306	defer cancel()
		307
259	step := func() error {	308	step := func() error {
260	mu.Lock()	309	mu.Lock()
261	snap := latest	310	snap := latest
@@ -292,15 +341,15 @@ func (c *Client) session(ctx context.Context) error {
292	// Worker goroutine: owns all up-stream writes (snapshot-driven + ticker).	341	// Worker goroutine: owns all up-stream writes (snapshot-driven + ticker).
293	tick := c.TickInterval	342	tick := c.TickInterval
294	if tick == 0 {	343	if tick == 0 {
295	tick = 10 * time.Second	344	tick = DefaultTickInterval
296	}	345	}
297	ticker := time.NewTicker(tick)	346	ticker := time.NewTicker(tick)
298	defer ticker.Stop()	347	defer ticker.Stop()
299	go func() {	348	go func() {
300	for {	349	for {
301	select {	350	select {
302	case <-ctx.Done():	351	case <-sessCtx.Done():
303	errc <- ctx.Err()	352	errc <- sessCtx.Err()
304	return	353	return
305	case <-stepSignal:	354	case <-stepSignal:
306	if err := step(); err != nil {	355	if err := step(); err != nil {

internal/agent/syncclient/client_test.go

Old		New
@@ -12,6 +12,7 @@ import (
12	"github.com/a73x/eitri/internal/agent/reconcile"	12	"github.com/a73x/eitri/internal/agent/reconcile"
13	"github.com/a73x/eitri/internal/agent/seed"	13	"github.com/a73x/eitri/internal/agent/seed"
14	"github.com/a73x/eitri/internal/agent/state"	14	"github.com/a73x/eitri/internal/agent/state"
		15	"github.com/a73x/eitri/internal/pb"
15	"github.com/a73x/eitri/internal/server/hosttoken"	16	"github.com/a73x/eitri/internal/server/hosttoken"
16	"github.com/a73x/eitri/internal/server/hub"	17	"github.com/a73x/eitri/internal/server/hub"
17	"github.com/a73x/eitri/internal/server/registry"	18	"github.com/a73x/eitri/internal/server/registry"
@@ -42,6 +43,93 @@ func (noopNet) AllocateIP(context.Context, []string) (string, error) {
42	}	43	}
43	func (noopNet) GuestNetwork() (string, int) { return "10.77.1.1", 24 }	44	func (noopNet) GuestNetwork() (string, int) { return "10.77.1.1", 24 }
44		45
		46	// testQUICIdle is the deliberately-short idle timeout the test listeners use so
		47	// reconnect/drop cases don't wait out the production SyncMaxIdleTimeout.
		48	const testQUICIdle = 5 * time.Second
		49
		50	// TestAdvertisedCapacityComputesOnce proves the Statfs+Sysinfo capacity probe
		51	// runs a single time per client (host totals are fixed for the session) while
		52	// the cheap cap clamp still applies on every call.
		53	func TestAdvertisedCapacityComputesOnce(t *testing.T) {
		54	orig := computeCapacity
		55	defer func() { computeCapacity = orig }()
		56	var calls int
		57	computeCapacity = func(string) *pb.Capacity {
		58	calls++
		59	return &pb.Capacity{Vcpus: 8, MemMb: 16384, DiskGb: 500}
		60	}
		61	c := &Client{MaxVCPUs: 2} // cap vCPUs to prove the clamp runs each call
		62	for i := 0; i < 5; i++ {
		63	got := c.advertisedCapacity("/state")
		64	require.Equal(t, int64(2), got.GetVcpus())
		65	require.Equal(t, int64(16384), got.GetMemMb())
		66	}
		67	require.Equal(t, 1, calls, "raw capacity syscalls must run exactly once, not per report")
		68	}
		69
		70	// TestAdvertisedCapacityReprobesUntilSuccess proves a transiently-failed probe
		71	// (capacity() silently zeroes a dimension whose syscall failed) is NOT memoized:
		72	// the agent re-probes until it gets a full reading, then caches that — otherwise
		73	// a boot-time blip would freeze a 0-capacity advertisement for the whole session
		74	// where the old per-report probe self-healed within one tick.
		75	func TestAdvertisedCapacityReprobesUntilSuccess(t *testing.T) {
		76	orig := computeCapacity
		77	defer func() { computeCapacity = orig }()
		78	var calls int
		79	computeCapacity = func(string) *pb.Capacity {
		80	calls++
		81	if calls == 1 {
		82	return &pb.Capacity{Vcpus: 8, MemMb: 0, DiskGb: 500} // transient mem probe failure
		83	}
		84	return &pb.Capacity{Vcpus: 8, MemMb: 16384, DiskGb: 500}
		85	}
		86	c := &Client{}
		87
		88	got1 := c.advertisedCapacity("/state")
		89	require.Equal(t, int64(0), got1.GetMemMb(), "first (failed) probe surfaces the zero, not cached")
		90
		91	got2 := c.advertisedCapacity("/state")
		92	require.Equal(t, int64(16384), got2.GetMemMb(), "must re-probe after a zeroed dimension")
		93	require.Equal(t, 2, calls)
		94
		95	got3 := c.advertisedCapacity("/state")
		96	require.Equal(t, int64(16384), got3.GetMemMb())
		97	require.Equal(t, 2, calls, "a full reading is memoized: no further syscalls")
		98	}
		99
		100	// TestHeartbeatMarginInvariant guards the cross-package coupling documented on
		101	// DefaultTickInterval and registry.OnlineWindow: the server must not declare a
		102	// host offline before it has had room to miss ~3 reports, or healthy hosts flap.
		103	func TestHeartbeatMarginInvariant(t *testing.T) {
		104	require.GreaterOrEqual(t, registry.OnlineWindow, 3*DefaultTickInterval,
		105	"OnlineWindow (%s) must stay >= 3x the agent report tick (%s) to avoid Online/Offline flapping",
		106	registry.OnlineWindow, DefaultTickInterval)
		107	}
		108
		109	// genTestCert returns a fresh self-signed server cert+key and its pinned
		110	// fingerprint — the bootstrap every test QUIC server shares.
		111	func genTestCert(t *testing.T) (certPEM, keyPEM []byte, fp string) {
		112	t.Helper()
		113	var err error
		114	certPEM, keyPEM, err = transport.GenerateServerCert()
		115	require.NoError(t, err)
		116	fp, err = transport.CertFingerprint(certPEM)
		117	require.NoError(t, err)
		118	return
		119	}
		120
		121	// listenTestQUIC opens a loopback QUIC listener with the shared short test idle
		122	// timeout, from an already-generated cert+key (so a harness can re-listen on the
		123	// same port with the same identity).
		124	func listenTestQUIC(t testing.T, addr string, certPEM, keyPEM []byte) quic.Listener {
		125	t.Helper()
		126	tlsConf, err := transport.ServerTLS(certPEM, keyPEM)
		127	require.NoError(t, err)
		128	lis, err := quic.ListenAddr(addr, tlsConf, &quic.Config{MaxIdleTimeout: testQUICIdle})
		129	require.NoError(t, err)
		130	return lis
		131	}
		132
45	// serverHarness owns a real syncsvc.Service on a fixed UDP loopback port so it	133	// serverHarness owns a real syncsvc.Service on a fixed UDP loopback port so it
46	// can be stopped and restarted (TestReconnectAfterDrop) on the same address.	134	// can be stopped and restarted (TestReconnectAfterDrop) on the same address.
47	type serverHarness struct {	135	type serverHarness struct {
@@ -64,10 +152,7 @@ func newServerHarness(t testing.T) serverHarness {
64	st, err := store.Open(t.TempDir()+"/db", "10.77.0.0/16")	152	st, err := store.Open(t.TempDir()+"/db", "10.77.0.0/16")
65	require.NoError(t, err)	153	require.NoError(t, err)
66	t.Cleanup(func() { st.Close() })	154	t.Cleanup(func() { st.Close() })
67	certPEM, keyPEM, err := transport.GenerateServerCert()	155	certPEM, keyPEM, fp := genTestCert(t)
68	require.NoError(t, err)
69	fp, err := transport.CertFingerprint(certPEM)
70	require.NoError(t, err)
71	h := &serverHarness{	156	h := &serverHarness{
72	t: t, st: st, reg: registry.New(time.Now), hub: hub.New(),	157	t: t, st: st, reg: registry.New(time.Now), hub: hub.New(),
73	secret: []byte("s3cret"), certPEM: certPEM, keyPEM: keyPEM, fp: fp,	158	secret: []byte("s3cret"), certPEM: certPEM, keyPEM: keyPEM, fp: fp,
@@ -78,16 +163,12 @@ func newServerHarness(t testing.T) serverHarness {
78	}	163	}
79		164
80	func (h *serverHarness) start(addr string) {	165	func (h *serverHarness) start(addr string) {
81	tlsConf, err := transport.ServerTLS(h.certPEM, h.keyPEM)	166	h.lis = listenTestQUIC(h.t, addr, h.certPEM, h.keyPEM)
82	require.NoError(h.t, err)	167	h.addr = h.lis.Addr().String()
83	lis, err := quic.ListenAddr(addr, tlsConf, &quic.Config{MaxIdleTimeout: 5 * time.Second})
84	require.NoError(h.t, err)
85	h.lis = lis
86	h.addr = lis.Addr().String()
87	ctx, cancel := context.WithCancel(context.Background())	168	ctx, cancel := context.WithCancel(context.Background())
88	h.cancel = cancel	169	h.cancel = cancel
89	h.svc = syncsvc.New(h.st, h.reg, h.hub, h.secret, 0)	170	h.svc = syncsvc.New(h.st, h.reg, h.hub, h.secret, 0)
90	go h.svc.Serve(ctx, lis) //nolint:errcheck	171	go h.svc.Serve(ctx, h.lis) //nolint:errcheck
91	}	172	}
92		173
93	func (h *serverHarness) stop() {	174	func (h *serverHarness) stop() {
@@ -285,25 +366,18 @@ func newCountingServer(t testing.T, onAccept func(int64) int64) serverHarness
285	st, err := store.Open(t.TempDir()+"/db", "10.77.0.0/16")	366	st, err := store.Open(t.TempDir()+"/db", "10.77.0.0/16")
286	require.NoError(t, err)	367	require.NoError(t, err)
287	t.Cleanup(func() { st.Close() })	368	t.Cleanup(func() { st.Close() })
288	certPEM, keyPEM, err := transport.GenerateServerCert()	369	certPEM, keyPEM, fp := genTestCert(t)
289	require.NoError(t, err)
290	fp, err := transport.CertFingerprint(certPEM)
291	require.NoError(t, err)
292	h := &serverHarness{	370	h := &serverHarness{
293	t: t, st: st, reg: registry.New(time.Now), hub: hub.New(),	371	t: t, st: st, reg: registry.New(time.Now), hub: hub.New(),
294	secret: []byte("s3cret"), certPEM: certPEM, keyPEM: keyPEM, fp: fp,	372	secret: []byte("s3cret"), certPEM: certPEM, keyPEM: keyPEM, fp: fp,
295	}	373	}
296	tlsConf, err := transport.ServerTLS(certPEM, keyPEM)	374	h.lis = listenTestQUIC(t, "127.0.0.1:0", certPEM, keyPEM)
297	require.NoError(t, err)	375	h.addr = h.lis.Addr().String()
298	lis, err := quic.ListenAddr("127.0.0.1:0", tlsConf, &quic.Config{MaxIdleTimeout: 5 * time.Second})
299	require.NoError(t, err)
300	h.lis = lis
301	h.addr = lis.Addr().String()
302	ctx, cancel := context.WithCancel(context.Background())	376	ctx, cancel := context.WithCancel(context.Background())
303	h.cancel = cancel	377	h.cancel = cancel
304	go func() {	378	go func() {
305	for {	379	for {
306	conn, err := lis.Accept(ctx)	380	conn, err := h.lis.Accept(ctx)
307	if err != nil {	381	if err != nil {
308	return	382	return
309	}	383	}
@@ -313,6 +387,6 @@ func newCountingServer(t testing.T, onAccept func(int64) int64) serverHarness
313	_ = conn.CloseWithError(transport.CodeAuthRejected, "test reject")	387	_ = conn.CloseWithError(transport.CodeAuthRejected, "test reject")
314	}	388	}
315	}()	389	}()
316	t.Cleanup(func() { cancel(); lis.Close() })	390	t.Cleanup(func() { cancel(); h.lis.Close() })
317	return h	391	return h
318	}	392	}

internal/agent/syncclient/leak_test.go

@@ @@ -0,0 +1,123 @@ @@
+package syncclient
+import (
+	"context"
+	"runtime"
+	"testing"
+	"time"
+	"github.com/a73x/eitri/internal/pb"
+	"github.com/a73x/eitri/internal/transport"
+	"github.com/stretchr/testify/require"
+)
+// settleGoroutines forces GC and polls until the live goroutine count drops to
+// target or the deadline passes, returning the last observed count. It lets any
+// per-session goroutines (recv/worker/console-accept + QUIC internals) unwind
+// before we sample, so the assertion isn't racing normal teardown.
+func settleGoroutines(target int, d time.Duration) int {
+	deadline := time.Now().Add(d)
+	n := runtime.NumGoroutine()
+	for time.Now().Before(deadline) {
+		runtime.GC()
+		n = runtime.NumGoroutine()
+		if n <= target {
+			return n
+		}
+		time.Sleep(20 * time.Millisecond)
+	}
+	return n
+}
+// newDropServer runs a minimal QUIC server that, for every connection, completes
+// the sync handshake (reads Hello, opens the down-stream, pushes one snapshot so
+// the client marks the session connected and exercises a step) and then drops
+// the connection — exactly the production reconnect trigger: the session ends
+// because the down-stream read errors, NOT because the parent ctx was cancelled.
+// It returns the listen addr, the cert fingerprint to pin, and a stop func.
+func newDropServer(t *testing.T) (addr, fp string, stop func()) {
+	t.Helper()
+	var certPEM, keyPEM []byte
+	certPEM, keyPEM, fp = genTestCert(t)
+	lis := listenTestQUIC(t, "127.0.0.1:0", certPEM, keyPEM)
+	ctx, cancel := context.WithCancel(context.Background())
+	go func() {
+		for {
+			conn, err := lis.Accept(ctx)
+			if err != nil {
+				return
+			}
+			go func() {
+				up, err := conn.AcceptStream(ctx)
+				if err != nil {
+					return
+				}
+				var first pb.AgentMessage
+				if err := transport.ReadMsg(up, &first, transport.DefaultMaxFrame); err != nil {
+					return
+				}
+				down, err := conn.OpenStreamSync(ctx)
+				if err != nil {
+					return
+				}
+				// One snapshot makes the client mark the session connected and run a
+				// step; then drop so the client's recv goroutine errors and the
+				// session ends the way a real server drop / idle timeout ends it.
+				_ = transport.WriteMsg(down, &pb.ServerMessage{
+					Msg: &pb.ServerMessage_Snapshot{Snapshot: &pb.DesiredStateSnapshot{}}})
+				time.Sleep(30 * time.Millisecond)
+				_ = conn.CloseWithError(0, "drop")
+			}()
+		}
+	}()
+	return lis.Addr().String(), fp, func() { cancel(); lis.Close() }
+}
+// TestSessionNoWorkerLeakAcrossReconnects drives many session start/stop cycles
+// (each ends by dropping the server, exactly like a production reconnect/deploy)
+// and asserts the live goroutine count returns to its post-warmup baseline.
+//
+// The regression it guards: the up-stream worker goroutine selects only on the
+// long-lived parent ctx, a dead stepSignal, and a ticker that session stops on
+// return — so when a session ends because the RECV goroutine errored (server
+// drop / idle timeout), the worker has no signal to exit and leaks one
+// goroutine per reconnect. Over long uptime these accumulate and the server-side
+// sync eventually wedges.
+func TestSessionNoWorkerLeakAcrossReconnects(t *testing.T) {
+	addr, fp, stop := newDropServer(t)
+	defer stop()
+	c := newClient(t, addr, fp, "host-x", "host-x.deadbeef")
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	// Each session connects, gets one snapshot, then the server drops it, so
+	// session() returns promptly. The parent ctx is never cancelled — precisely
+	// the condition under which the worker goroutine cannot observe session end.
+	runOne := func() {
+		done := make(chan error, 1)
+		go func() { done <- c.session(ctx) }()
+		select {
+		case <-done:
+		case <-time.After(5 * time.Second):
+			t.Fatal("session did not return after server drop")
+		}
+	}
+	// Warm up to a steady goroutine state before sampling the baseline.
+	runOne()
+	runOne()
+	base := settleGoroutines(0, 3*time.Second) // target 0 => returns settled count
+	const cycles = 10
+	for i := 0; i < cycles; i++ {
+		runOne()
+	}
+	// With the leak, each cycle strands one worker goroutine, so the count sits
+	// near base+cycles and never settles back. A small slack absorbs QUIC jitter.
+	got := settleGoroutines(base+2, 5*time.Second)
+	require.LessOrEqualf(t, got, base+2,
+		"goroutine count grew across %d reconnects (base=%d, got=%d): a worker goroutine leaks per session",
+		cycles, base, got)
+}

internal/arch/arch_test.go

@@ @@ -205,6 +205,38 @@ func TestAPITypesIsALeaf(t *testing.T) { @@
+// R7: internal/integration/testutil is test infrastructure only (its package
+// doc says so) — it exists to support the in-process harness and the QEMU
+// sandbox, not to be a dependency of anything shipped. Nothing under
+// internal/server/*, internal/agent/*, or cmd/* may import
+// internal/integration/* — checked transitively, so a wrapper package can't
+// smuggle the dependency in either. The sanctioned exceptions are the
+// test-tooling launcher binaries that ARE the integration infrastructure's
+// entry points (smoketest, devstack, sandbox); the shipped binaries
+// (eitri-server, eitri-agent) and everything else stay clean.
+func TestProductionPlanesDoNotImportIntegrationTestInfra(t *testing.T) {
+	g := internalImports(t)
+	allowed := map[string]bool{
+		module + "/cmd/eitri-smoketest": true, // drives internal/integration/harness
+		module + "/cmd/eitri-devstack":  true, // drives internal/integration/harness
+		module + "/cmd/eitri-sandbox":   true, // drives internal/integration/sandbox + testutil
+	}
+	for pkg := range g {
+		if allowed[pkg] {
+			continue
+		}
+		if !has(pkg, "internal/server/") && !has(pkg, "internal/agent/") && !has(pkg, "cmd/") {
+			continue
+		}
+		deps := transitiveDeps(g, pkg)
+		for d := range deps {
+			if has(d, "internal/integration/") {
+				t.Errorf("production package %s must not import test-infrastructure package %s", short(pkg), short(d))
+			}
+		}
+	}
+}
+// assertNotImported fails if pkg directly imports any path in forbidden.
+func assertNotImported(t *testing.T, g map[string][]string, pkg string, forbidden []string) {
+	t.Helper()

internal/cloudinit/cloudinit.go

internal/cloudinit/cloudinit_test.go

Old		New
@@ -36,7 +36,7 @@ func authKeys(t *testing.T, m map[string]any) []string {
36		36
37	func TestMergeAddsKeyWhenAbsent(t *testing.T) {	37	func TestMergeAddsKeyWhenAbsent(t *testing.T) {
38	in := "#cloud-config\npackages:\n - htop\n"	38	in := "#cloud-config\npackages:\n - htop\n"
39	out, err := MergeSSHKey(in, key)	39	out, err := mergeSSHKey(in, key)
40	require.NoError(t, err)	40	require.NoError(t, err)
41	m := parse(t, out)	41	m := parse(t, out)
42	assert.Equal(t, []string{key}, authKeys(t, m))	42	assert.Equal(t, []string{key}, authKeys(t, m))
@@ -47,14 +47,14 @@ func TestMergeAddsKeyWhenAbsent(t *testing.T) {
47	func TestMergeAppendsToExistingList(t *testing.T) {	47	func TestMergeAppendsToExistingList(t *testing.T) {
48	existing := "ssh-rsa AAAAexisting other@host"	48	existing := "ssh-rsa AAAAexisting other@host"
49	in := "#cloud-config\nssh_authorized_keys:\n - " + existing + "\n"	49	in := "#cloud-config\nssh_authorized_keys:\n - " + existing + "\n"
50	out, err := MergeSSHKey(in, key)	50	out, err := mergeSSHKey(in, key)
51	require.NoError(t, err)	51	require.NoError(t, err)
52	assert.Equal(t, []string{existing, key}, authKeys(t, parse(t, out)))	52	assert.Equal(t, []string{existing, key}, authKeys(t, parse(t, out)))
53	}	53	}
54		54
55	func TestMergeIsIdempotentDedup(t *testing.T) {	55	func TestMergeIsIdempotentDedup(t *testing.T) {
56	in := "#cloud-config\nssh_authorized_keys:\n - " + key + "\n"	56	in := "#cloud-config\nssh_authorized_keys:\n - " + key + "\n"
57	out, err := MergeSSHKey(in, key)	57	out, err := mergeSSHKey(in, key)
58	require.NoError(t, err)	58	require.NoError(t, err)
59	assert.Equal(t, []string{key}, authKeys(t, parse(t, out)), "an already-present key must not be duplicated")	59	assert.Equal(t, []string{key}, authKeys(t, parse(t, out)), "an already-present key must not be duplicated")
60	}	60	}
@@ -64,38 +64,38 @@ func TestMergeNormalizesScalarKey(t *testing.T) {
64	// list and append rather than clobbering the user's existing key.	64	// list and append rather than clobbering the user's existing key.
65	existing := "ssh-rsa AAAAexisting other@host"	65	existing := "ssh-rsa AAAAexisting other@host"
66	in := "#cloud-config\nssh_authorized_keys: " + existing + "\n"	66	in := "#cloud-config\nssh_authorized_keys: " + existing + "\n"
67	out, err := MergeSSHKey(in, key)	67	out, err := mergeSSHKey(in, key)
68	require.NoError(t, err)	68	require.NoError(t, err)
69	assert.Equal(t, []string{existing, key}, authKeys(t, parse(t, out)))	69	assert.Equal(t, []string{existing, key}, authKeys(t, parse(t, out)))
70	}	70	}
71		71
72	func TestMergeHeaderWithLeadingBlankLines(t *testing.T) {	72	func TestMergeHeaderWithLeadingBlankLines(t *testing.T) {
73	in := "\n\n#cloud-config\nruncmd:\n - echo hi\n"	73	in := "\n\n#cloud-config\nruncmd:\n - echo hi\n"
74	out, err := MergeSSHKey(in, key)	74	out, err := mergeSSHKey(in, key)
75	require.NoError(t, err)	75	require.NoError(t, err)
76	assert.Equal(t, []string{key}, authKeys(t, parse(t, out)))	76	assert.Equal(t, []string{key}, authKeys(t, parse(t, out)))
77	}	77	}
78		78
79	func TestMergeRejectsNonCloudConfig(t *testing.T) {	79	func TestMergeRejectsNonCloudConfig(t *testing.T) {
80	for _, in := range []string{	80	for _, in := range []string{
81	"#!/bin/bash\necho hi\n", // shell script user-data	81	"#!/bin/bash\necho hi\n", // shell script user-data
82	"## template: jinja\n#cloud-config\n", // jinja-templated (can't safely edit)	82	"## template: jinja\n#cloud-config\n", // jinja-templated (can't safely edit)
83	"just some text", // not user-data at all	83	"just some text", // not user-data at all
84	} {	84	} {
85	_, err := MergeSSHKey(in, key)	85	_, err := mergeSSHKey(in, key)
86	assert.Error(t, err, "must refuse to merge into non-cloud-config: %q", in)	86	assert.Error(t, err, "must refuse to merge into non-cloud-config: %q", in)
87	}	87	}
88	}	88	}
89		89
90	func TestMergeRejectsMalformedYAML(t *testing.T) {	90	func TestMergeRejectsMalformedYAML(t *testing.T) {
91	_, err := MergeSSHKey("#cloud-config\n bad: : : indent\n\t- x\n", key)	91	_, err := mergeSSHKey("#cloud-config\n bad: : : indent\n\t- x\n", key)
92	assert.Error(t, err)	92	assert.Error(t, err)
93	}	93	}
94		94
95	func TestMergeRejectsSSHAuthorizedKeysWrongType(t *testing.T) {	95	func TestMergeRejectsSSHAuthorizedKeysWrongType(t *testing.T) {
96	// A mapping where we expect a scalar/sequence — don't silently drop it.	96	// A mapping where we expect a scalar/sequence — don't silently drop it.
97	in := "#cloud-config\nssh_authorized_keys:\n nested: value\n"	97	in := "#cloud-config\nssh_authorized_keys:\n nested: value\n"
98	_, err := MergeSSHKey(in, key)	98	_, err := mergeSSHKey(in, key)
99	assert.Error(t, err)	99	assert.Error(t, err)
100	}	100	}
101		101
@@ -109,7 +109,7 @@ func TestMergePreservesScalarFidelity(t *testing.T) {
109	" permissions: 0644\n" +	109	" permissions: 0644\n" +
110	"version: 1.10\n" +	110	"version: 1.10\n" +
111	"stamp: 2020-01-02\n"	111	"stamp: 2020-01-02\n"
112	out, err := MergeSSHKey(in, key)	112	out, err := mergeSSHKey(in, key)
113	require.NoError(t, err)	113	require.NoError(t, err)
114	assert.Contains(t, out, "0644", "octal permissions must survive verbatim, not become 420")	114	assert.Contains(t, out, "0644", "octal permissions must survive verbatim, not become 420")
115	assert.Contains(t, out, "1.10", "trailing zero must survive")	115	assert.Contains(t, out, "1.10", "trailing zero must survive")
@@ -120,7 +120,7 @@ func TestMergePreservesScalarFidelity(t *testing.T) {
120		120
121	func TestMergePreservesComments(t *testing.T) {	121	func TestMergePreservesComments(t *testing.T) {
122	in := "#cloud-config\n# keep me\npackages:\n - htop # inline\n"	122	in := "#cloud-config\n# keep me\npackages:\n - htop # inline\n"
123	out, err := MergeSSHKey(in, key)	123	out, err := mergeSSHKey(in, key)
124	require.NoError(t, err)	124	require.NoError(t, err)
125	assert.Contains(t, out, "# keep me", "top-level comment must survive an in-place edit")	125	assert.Contains(t, out, "# keep me", "top-level comment must survive an in-place edit")
126	}	126	}
@@ -129,13 +129,13 @@ func TestMergeRejectsMultiDocYAML(t *testing.T) {
129	// cloud-init rejects multi-document cloud-config; accepting-and-truncating	129	// cloud-init rejects multi-document cloud-config; accepting-and-truncating
130	// would mask an error the guest would raise. Reject it too.	130	// would mask an error the guest would raise. Reject it too.
131	in := "#cloud-config\npackages: [htop]\n---\nruncmd:\n - echo hi\n"	131	in := "#cloud-config\npackages: [htop]\n---\nruncmd:\n - echo hi\n"
132	_, err := MergeSSHKey(in, key)	132	_, err := mergeSSHKey(in, key)
133	assert.Error(t, err)	133	assert.Error(t, err)
134	}	134	}
135		135
136	func TestMergeEmptyBodyGetsKey(t *testing.T) {	136	func TestMergeEmptyBodyGetsKey(t *testing.T) {
137	// A bare "#cloud-config" with no body is valid; the merge seeds the key.	137	// A bare "#cloud-config" with no body is valid; the merge seeds the key.
138	out, err := MergeSSHKey("#cloud-config\n", key)	138	out, err := mergeSSHKey("#cloud-config\n", key)
139	require.NoError(t, err)	139	require.NoError(t, err)
140	assert.Equal(t, []string{key}, authKeys(t, parse(t, out)))	140	assert.Equal(t, []string{key}, authKeys(t, parse(t, out)))
141	}	141	}

internal/cloudinit/multipart.go

Old		New
@@ -13,40 +13,40 @@ import (
13	"strings"	13	"strings"
14	)	14	)
15		15
16	// Format classifies cloud-init user-data. cloud-init picks a handler by the	16	// format classifies cloud-init user-data. cloud-init picks a handler by the
17	// payload's leading "magic" line (or MIME/gzip framing), so user-data is a	17	// payload's leading "magic" line (or MIME/gzip framing), so user-data is a
18	// tagged union, not just YAML — detection mirrors cloud-init's own sniff.	18	// tagged union, not just YAML — detection mirrors cloud-init's own sniff.
19	type Format int	19	type format int
20		20
21	const (	21	const (
22	FormatUnknown Format = iota	22	formatUnknown format = iota
23	FormatCloudConfig	23	formatCloudConfig
24	FormatShellScript	24	formatShellScript
25	FormatBoothook	25	formatBoothook
26	FormatInclude	26	formatInclude
27	FormatPartHandler	27	formatPartHandler
28	FormatMultipart // MIME multipart/mixed archive	28	formatMultipart // MIME multipart/mixed archive
29	FormatJinja // ## template: jinja — templated; not safe to edit or wrap blind	29	formatJinja // ## template: jinja — templated; not safe to edit or wrap blind
30	FormatGzip // gzip-compressed payload	30	formatGzip // gzip-compressed payload
31	)	31	)
32		32
33	func (f Format) String() string {	33	func (f format) String() string {
34	switch f {	34	switch f {
35	case FormatCloudConfig:	35	case formatCloudConfig:
36	return "cloud-config"	36	return "cloud-config"
37	case FormatShellScript:	37	case formatShellScript:
38	return "shell-script"	38	return "shell-script"
39	case FormatBoothook:	39	case formatBoothook:
40	return "cloud-boothook"	40	return "cloud-boothook"
41	case FormatInclude:	41	case formatInclude:
42	return "include"	42	return "include"
43	case FormatPartHandler:	43	case formatPartHandler:
44	return "part-handler"	44	return "part-handler"
45	case FormatMultipart:	45	case formatMultipart:
46	return "multipart"	46	return "multipart"
47	case FormatJinja:	47	case formatJinja:
48	return "jinja-template"	48	return "jinja-template"
49	case FormatGzip:	49	case formatGzip:
50	return "gzip"	50	return "gzip"
51	default:	51	default:
52	return "unknown"	52	return "unknown"
@@ -57,35 +57,35 @@ func (f Format) String() string {
57	// used to tell a MIME archive (starts with headers) from a #-tagged payload.	57	// used to tell a MIME archive (starts with headers) from a #-tagged payload.
58	var headerRe = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9-]*:`)	58	var headerRe = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9-]*:`)
59		59
60	// DetectFormat classifies user-data the way cloud-init does: gzip magic first,	60	// detectFormat classifies user-data the way cloud-init does: gzip magic first,
61	// then the leading non-blank line's marker (its first whitespace-delimited	61	// then the leading non-blank line's marker (its first whitespace-delimited
62	// token, so a trailing comment is tolerated), then — for a payload that leads	62	// token, so a trailing comment is tolerated), then — for a payload that leads
63	// with RFC 5322 headers rather than a #-marker — a real MIME header-block parse	63	// with RFC 5322 headers rather than a #-marker — a real MIME header-block parse
64	// (an archive may put MIME-Version before Content-Type).	64	// (an archive may put MIME-Version before Content-Type).
65	func DetectFormat(userData string) Format {	65	func detectFormat(userData string) format {
66	if len(userData) >= 2 && userData[0] == 0x1f && userData[1] == 0x8b {	66	if len(userData) >= 2 && userData[0] == 0x1f && userData[1] == 0x8b {
67	return FormatGzip	67	return formatGzip
68	}	68	}
69	first := firstNonBlankLine(userData)	69	first := firstNonBlankLine(userData)
70	if strings.HasPrefix(first, "## template: jinja") {	70	if strings.HasPrefix(first, "## template: jinja") {
71	return FormatJinja	71	return formatJinja
72	}	72	}
73	switch marker := firstMarker(userData); {	73	switch marker := firstMarker(userData); {
74	case marker == "#cloud-config":	74	case marker == "#cloud-config":
75	return FormatCloudConfig	75	return formatCloudConfig
76	case strings.HasPrefix(marker, "#!"):	76	case strings.HasPrefix(marker, "#!"):
77	return FormatShellScript	77	return formatShellScript
78	case marker == "#cloud-boothook":	78	case marker == "#cloud-boothook":
79	return FormatBoothook	79	return formatBoothook
80	case strings.HasPrefix(marker, "#include"): // #include and #include-once	80	case strings.HasPrefix(marker, "#include"): // #include and #include-once
81	return FormatInclude	81	return formatInclude
82	case marker == "#part-handler":	82	case marker == "#part-handler":
83	return FormatPartHandler	83	return formatPartHandler
84	}	84	}
85	if isMIMEMultipart(userData) {	85	if isMIMEMultipart(userData) {
86	return FormatMultipart	86	return formatMultipart
87	}	87	}
88	return FormatUnknown	88	return formatUnknown
89	}	89	}
90		90
91	// isMIMEMultipart reports whether s is a MIME multipart archive by parsing its	91	// isMIMEMultipart reports whether s is a MIME multipart archive by parsing its
@@ -122,18 +122,18 @@ func isMIMEMultipart(s string) bool {
122	// generated #cloud-config, so a newline would produce a garbage key. It is	122	// generated #cloud-config, so a newline would produce a garbage key. It is
123	// added as a YAML scalar node, so it cannot inject structure regardless.	123	// added as a YAML scalar node, so it cannot inject structure regardless.
124	func AddSSHKey(userData, key string) (string, error) {	124	func AddSSHKey(userData, key string) (string, error) {
125	switch f := DetectFormat(userData); f {	125	switch f := detectFormat(userData); f {
126	case FormatCloudConfig:	126	case formatCloudConfig:
127	return MergeSSHKey(userData, key)	127	return mergeSSHKey(userData, key)
128	case FormatShellScript:	128	case formatShellScript:
129	return wrapMultipart([]part{typedPart("text/x-shellscript", userData), keyPart(key)})	129	return wrapMultipart([]part{typedPart("text/x-shellscript", userData), keyPart(key)})
130	case FormatBoothook:	130	case formatBoothook:
131	return wrapMultipart([]part{typedPart("text/cloud-boothook", userData), keyPart(key)})	131	return wrapMultipart([]part{typedPart("text/cloud-boothook", userData), keyPart(key)})
132	case FormatInclude:	132	case formatInclude:
133	return wrapMultipart([]part{typedPart("text/x-include-url", userData), keyPart(key)})	133	return wrapMultipart([]part{typedPart("text/x-include-url", userData), keyPart(key)})
134	case FormatPartHandler:	134	case formatPartHandler:
135	return wrapMultipart([]part{typedPart("text/part-handler", userData), keyPart(key)})	135	return wrapMultipart([]part{typedPart("text/part-handler", userData), keyPart(key)})
136	case FormatMultipart:	136	case formatMultipart:
137	return appendToMultipart(userData, key)	137	return appendToMultipart(userData, key)
138	default:	138	default:
139	return "", fmt.Errorf("cannot add an ssh key to %s user-data; include the key in the user-data itself", f)	139	return "", fmt.Errorf("cannot add an ssh key to %s user-data; include the key in the user-data itself", f)
@@ -156,9 +156,9 @@ func typedPart(contentType, body string) part {
156	}	156	}
157		157
158	// keyPart is the generated #cloud-config carrying just the ssh key, built via	158	// keyPart is the generated #cloud-config carrying just the ssh key, built via
159	// MergeSSHKey so the key lands in a valid ssh_authorized_keys list.	159	// mergeSSHKey so the key lands in a valid ssh_authorized_keys list.
160	func keyPart(key string) part {	160	func keyPart(key string) part {
161	doc, _ := MergeSSHKey("#cloud-config\n", key) // cannot fail on a literal #cloud-config	161	doc, _ := mergeSSHKey("#cloud-config\n", key) // cannot fail on a literal #cloud-config
162	return typedPart("text/cloud-config", doc)	162	return typedPart("text/cloud-config", doc)
163	}	163	}
164		164

internal/cloudinit/multipart_test.go

Old		New
@@ -15,22 +15,22 @@ import (
15	func TestDetectFormat(t *testing.T) {	15	func TestDetectFormat(t *testing.T) {
16	cases := []struct {	16	cases := []struct {
17	in string	17	in string
18	want Format	18	want format
19	}{	19	}{
20	{"#cloud-config\npackages: [htop]\n", FormatCloudConfig},	20	{"#cloud-config\npackages: [htop]\n", formatCloudConfig},
21	{"\n\n#cloud-config\n", FormatCloudConfig}, // leading blank lines tolerated	21	{"\n\n#cloud-config\n", formatCloudConfig}, // leading blank lines tolerated
22	{"#!/bin/bash\necho hi\n", FormatShellScript},	22	{"#!/bin/bash\necho hi\n", formatShellScript},
23	{"#cloud-boothook\n#!/bin/sh\n", FormatBoothook},	23	{"#cloud-boothook\n#!/bin/sh\n", formatBoothook},
24	{"#include\nhttps://example/x\n", FormatInclude},	24	{"#include\nhttps://example/x\n", formatInclude},
25	{"#include-once\nhttps://example/x\n", FormatInclude},	25	{"#include-once\nhttps://example/x\n", formatInclude},
26	{"#part-handler\n", FormatPartHandler},	26	{"#part-handler\n", formatPartHandler},
27	{"## template: jinja\n#cloud-config\n", FormatJinja},	27	{"## template: jinja\n#cloud-config\n", formatJinja},
28	{"Content-Type: multipart/mixed; boundary=\"X\"\n\n", FormatMultipart},	28	{"Content-Type: multipart/mixed; boundary=\"X\"\n\n", formatMultipart},
29	{"\x1f\x8b\x08 gzip bytes", FormatGzip},	29	{"\x1f\x8b\x08 gzip bytes", formatGzip},
30	{"just some text", FormatUnknown},	30	{"just some text", formatUnknown},
31	}	31	}
32	for _, c := range cases {	32	for _, c := range cases {
33	assert.Equal(t, c.want, DetectFormat(c.in), "input %q", c.in)	33	assert.Equal(t, c.want, detectFormat(c.in), "input %q", c.in)
34	}	34	}
35	}	35	}
36		36
@@ -59,7 +59,7 @@ func mimeParts(t *testing.T, s string) map[string]string {
59	}	59	}
60		60
61	func TestAddSSHKeyCloudConfigMerges(t *testing.T) {	61	func TestAddSSHKeyCloudConfigMerges(t *testing.T) {
62	// cloud-config path delegates to MergeSSHKey → a single merged document,	62	// cloud-config path delegates to mergeSSHKey → a single merged document,
63	// NOT a multipart wrapper.	63	// NOT a multipart wrapper.
64	out, err := AddSSHKey("#cloud-config\npackages:\n - htop\n", key)	64	out, err := AddSSHKey("#cloud-config\npackages:\n - htop\n", key)
65	require.NoError(t, err)	65	require.NoError(t, err)
@@ -125,13 +125,13 @@ func TestDetectFormatMultipartHeaderOrder(t *testing.T) {
125	// A valid archive may lead with MIME-Version before Content-Type; detection	125	// A valid archive may lead with MIME-Version before Content-Type; detection
126	// must parse the header block, not just sniff line one.	126	// must parse the header block, not just sniff line one.
127	in := "MIME-Version: 1.0\nContent-Type: multipart/mixed; boundary=\"X\"\n\n--X--\n"	127	in := "MIME-Version: 1.0\nContent-Type: multipart/mixed; boundary=\"X\"\n\n--X--\n"
128	assert.Equal(t, FormatMultipart, DetectFormat(in))	128	assert.Equal(t, formatMultipart, detectFormat(in))
129	}	129	}
130		130
131	func TestDetectFormatCloudConfigTrailingComment(t *testing.T) {	131	func TestDetectFormatCloudConfigTrailingComment(t *testing.T) {
132	// cloud-init detects #cloud-config by prefix, so a trailing comment on the	132	// cloud-init detects #cloud-config by prefix, so a trailing comment on the
133	// marker line is valid — don't reject it.	133	// marker line is valid — don't reject it.
134	assert.Equal(t, FormatCloudConfig, DetectFormat("#cloud-config # my vm\npackages: [htop]\n"))	134	assert.Equal(t, formatCloudConfig, detectFormat("#cloud-config # my vm\npackages: [htop]\n"))
135	}	135	}
136		136
137	func TestAppendPreservesPartHeaders(t *testing.T) {	137	func TestAppendPreservesPartHeaders(t *testing.T) {

internal/random/random.go

@@ @@ -0,0 +1,19 @@ @@
+// Package random provides small cryptographically-random helpers shared across
+// the control plane, CLIs, and the integration harness — a leaf package so a
+// CLI or test binary can reuse them without importing heavier deps (e.g. the
+// sqlite-backed store).
+package random
+import (
+	"crypto/rand"
+	"encoding/hex"
+)
+// Hex returns n cryptographically-random bytes encoded as hex (2n chars).
+// crypto/rand.Read is documented never to fail (Go 1.24+), so its error is
+// intentionally ignored.
+func Hex(n int) string {
+	b := make([]byte, n)
+	rand.Read(b) //nolint:errcheck // crypto/rand.Read never returns an error
+	return hex.EncodeToString(b)
+}

internal/server/api/api.go

Old		New
@@ -18,6 +18,7 @@ import (
18	"github.com/a73x/eitri/internal/cloudinit"	18	"github.com/a73x/eitri/internal/cloudinit"
19	"github.com/a73x/eitri/internal/joinblob"	19	"github.com/a73x/eitri/internal/joinblob"
20	"github.com/a73x/eitri/internal/names"	20	"github.com/a73x/eitri/internal/names"
		21	"github.com/a73x/eitri/internal/random"
21	"github.com/a73x/eitri/internal/server/api/types"	22	"github.com/a73x/eitri/internal/server/api/types"
22	"github.com/a73x/eitri/internal/server/hosttoken"	23	"github.com/a73x/eitri/internal/server/hosttoken"
23	"github.com/a73x/eitri/internal/server/hub"	24	"github.com/a73x/eitri/internal/server/hub"
@@ -48,25 +49,37 @@ type Config struct {
48		49
49	// API is the HTTP handler container.	50	// API is the HTTP handler container.
50	type API struct {	51	type API struct {
51	cfg Config	52	cfg Config
52	st *store.Store	53	st *store.Store
53	reg *registry.Registry	54	reg *registry.Registry
54	hub *hub.Hub	55	hub *hub.Hub
55	notif *notifier	56	notif *notifier
56	enrolls *ipLimiter // per-client-bucket brake on the unauthenticated enroll endpoint (v4: address, v6: /64)	57	enrolls *ipLimiter // per-client-bucket brake on the unauthenticated enroll endpoint (v4: address, v6: /64)
57	tickets *ticketStore // one-time SSE stream tickets	58	tickets *ticketStore // one-time SSE stream tickets
		59	snap *snapshotHub // central SSE snapshot: one marshal fanned to all clients
58	console ConsoleDialer // nil until main wires syncsvc (SetConsoleDialer)	60	console ConsoleDialer // nil until main wires syncsvc (SetConsoleDialer)
59	certs CertMinter // nil until main wires the SSH user CA (SetCertMinter); nil ⇒ gate off	61	certs CertMinter // nil until main wires the SSH user CA (SetCertMinter); nil ⇒ gate off
60	hostCerts HostCertMinter // nil until main wires the SSH host CA (SetHostCertMinter); nil ⇒ gate off	62	hostCerts HostCertMinter // nil until main wires the SSH host CA (SetHostCertMinter); nil ⇒ gate off
61	sshCAKey string // eitri CA public key (authorized_keys form) served by GET /api/v1/ssh-ca; empty ⇒ gate off	63	sshCAKey string // eitri CA public key (authorized_keys form) served by GET /api/v1/ssh-ca; empty ⇒ gate off
62	}	64	}
63		65
64	// New constructs an API.	66	// New constructs an API. It starts the central SSE snapshot hub (one goroutine
		67	// that marshals the fleet snapshot on a 1s tick / desired-state wake and fans
		68	// the identical bytes to all connected clients). Call Close to stop it.
65	func New(cfg Config, st store.Store, reg registry.Registry, h hub.Hub) API {	69	func New(cfg Config, st store.Store, reg registry.Registry, h hub.Hub) API {
66	return &API{cfg: cfg, st: st, reg: reg, hub: h, notif: newNotifier(),	70	a := &API{cfg: cfg, st: st, reg: reg, hub: h, notif: newNotifier(),
67	enrolls: newIPLimiter(time.Now), tickets: newTicketStore(time.Now)}	71	enrolls: newIPLimiter(time.Now), tickets: newTicketStore(time.Now)}
		72	a.snap = newSnapshotHub(a.marshalSnapshot, a.notif)
		73	go a.snap.run()
		74	return a
68	}	75	}
69		76
		77	// Close stops the central snapshot hub goroutine. Idempotent. Tests and any
		78	// future graceful-shutdown path should call it; the server binary itself blocks
		79	// in ListenAndServe for the whole process lifetime and exits via os.Exit (which
		80	// skips deferred cleanup), so the single hub goroutine simply lives until exit.
		81	func (a *API) Close() { a.snap.Close() }
		82
70	// Handler returns the ServeMux with all routes registered from the declared	83	// Handler returns the ServeMux with all routes registered from the declared
71	// route table (routes.go) — the table is the single enumerable surface, shared	84	// route table (routes.go) — the table is the single enumerable surface, shared
72	// with the OpenAPI generator.	85	// with the OpenAPI generator.
@@ -116,10 +129,10 @@ func (a *API) sweepDecommissioned() bool {
116	if h.Status != "decommissioning" {	129	if h.Status != "decommissioning" {
117	continue	130	continue
118	}	131	}
119	n, err := a.st.HostVMCount(h.ID)	132	// RemoveHost re-counts VM rows in-transaction and refuses while any
120	if err != nil \|\| n > 0 {	133	// remain, so a pre-check here would only save a wasted call on hosts
121	continue	134	// that aren't yet drained — every RemoveHost error (drained or not) is
122	}	135	// already handled by simply skipping to the next host.
123	if a.st.RemoveHost(h.ID) == nil {	136	if a.st.RemoveHost(h.ID) == nil {
124	removed = true	137	removed = true
125	}	138	}
@@ -163,18 +176,12 @@ func writeJSON(w http.ResponseWriter, status int, v any) {
163	json.NewEncoder(w).Encode(v) //nolint:errcheck	176	json.NewEncoder(w).Encode(v) //nolint:errcheck
164	}	177	}
165		178
166	// httpError writes a plain-text error with the given status. Thin wrapper over
167	// http.Error kept for symmetry with writeJSON; the message is always explicit.
168	func httpError(w http.ResponseWriter, msg string, status int) {
169	http.Error(w, msg, status)
170	}
171
172	// decodeJSON decodes the request body into v, reporting a 400 with the standard	179	// decodeJSON decodes the request body into v, reporting a 400 with the standard
173	// "bad request" body on failure. Returns false when it has already written the	180	// "bad request" body on failure. Returns false when it has already written the
174	// response (caller must return).	181	// response (caller must return).
175	func decodeJSON(w http.ResponseWriter, r *http.Request, v any) bool {	182	func decodeJSON(w http.ResponseWriter, r *http.Request, v any) bool {
176	if err := json.NewDecoder(r.Body).Decode(v); err != nil {	183	if err := json.NewDecoder(r.Body).Decode(v); err != nil {
177	httpError(w, "bad request", http.StatusBadRequest)	184	http.Error(w, "bad request", http.StatusBadRequest)
178	return false	185	return false
179	}	186	}
180	return true	187	return true
@@ -212,7 +219,7 @@ func tokenHashPrefix(tok string) string {
212		219
213	func (a API) handleEnroll(w http.ResponseWriter, r http.Request) {	220	func (a API) handleEnroll(w http.ResponseWriter, r http.Request) {
214	if !a.enrolls.allow(bucketKey(clientIP(r))) {	221	if !a.enrolls.allow(bucketKey(clientIP(r))) {
215	httpError(w, "rate limited", http.StatusTooManyRequests)	222	http.Error(w, "rate limited", http.StatusTooManyRequests)
216	return	223	return
217	}	224	}
218	// Unauthenticated endpoint: bound the body so junk can't bloat memory or	225	// Unauthenticated endpoint: bound the body so junk can't bloat memory or
@@ -227,7 +234,7 @@ func (a API) handleEnroll(w http.ResponseWriter, r http.Request) {
227	a.audit("host.enroll.denied", map[string]string{	234	a.audit("host.enroll.denied", map[string]string{
228	"remote": clientIP(r), "name": truncate(req.Name, 64),	235	"remote": clientIP(r), "name": truncate(req.Name, 64),
229	"token_hash_prefix": tokenHashPrefix(req.Token)})	236	"token_hash_prefix": tokenHashPrefix(req.Token)})
230	httpError(w, "forbidden", http.StatusForbidden)	237	http.Error(w, "forbidden", http.StatusForbidden)
231	return	238	return
232	}	239	}
233	// host.enroll is audited durably inside the redeem transaction.	240	// host.enroll is audited durably inside the redeem transaction.
@@ -243,14 +250,14 @@ func (a API) handleEnroll(w http.ResponseWriter, r http.Request) {
243	func (a API) handleCreateEnrollToken(w http.ResponseWriter, r http.Request) {	250	func (a API) handleCreateEnrollToken(w http.ResponseWriter, r http.Request) {
244	tok, err := a.st.CreateEnrollmentToken()	251	tok, err := a.st.CreateEnrollmentToken()
245	if err != nil {	252	if err != nil {
246	httpError(w, "internal error", http.StatusInternalServerError)	253	http.Error(w, "internal error", http.StatusInternalServerError)
247	return	254	return
248	}	255	}
249	a.audit("enroll-token.mint", map[string]string{	256	a.audit("enroll-token.mint", map[string]string{
250	"remote": clientIP(r), "token_hash_prefix": tokenHashPrefix(tok)})	257	"remote": clientIP(r), "token_hash_prefix": tokenHashPrefix(tok)})
251	join, err := joinblob.Encode(a.cfg.AdvertiseHTTP, a.cfg.AdvertiseQUIC, tok, a.cfg.ServerCertSHA256)	258	join, err := joinblob.Encode(a.cfg.AdvertiseHTTP, a.cfg.AdvertiseQUIC, tok, a.cfg.ServerCertSHA256)
252	if err != nil {	259	if err != nil {
253	httpError(w, "internal error", http.StatusInternalServerError)	260	http.Error(w, "internal error", http.StatusInternalServerError)
254	return	261	return
255	}	262	}
256	writeJSON(w, http.StatusCreated, types.EnrollTokenResponse{Token: tok, Join: join})	263	writeJSON(w, http.StatusCreated, types.EnrollTokenResponse{Token: tok, Join: join})
@@ -274,6 +281,16 @@ func toHostResponse(h store.Host, st registry.HostState, ok bool, alloc store.Al
274	}	281	}
275	if ok {	282	if ok {
276	hr.Online = st.Online	283	hr.Online = st.Online
		284	hr.Stale = st.Stale
		285	hr.Sessions = st.Sessions
		286	// LastSeen unset ⇒ connected but never reported: leave the age fields
		287	// null rather than emit a bogus "last seen at the zero time".
		288	if !st.LastSeen.IsZero() {
		289	seen := st.LastSeen
		290	secs := int64(st.SinceLastSeen.Seconds())
		291	hr.LastSeen = &seen
		292	hr.SecondsSinceLastSeen = &secs
		293	}
277	hr.Capacity = types.Capacity{	294	hr.Capacity = types.Capacity{
278	VCPUs: st.Capacity.VCPUs,	295	VCPUs: st.Capacity.VCPUs,
279	MemMB: st.Capacity.MemMB,	296	MemMB: st.Capacity.MemMB,
@@ -283,6 +300,32 @@ func toHostResponse(h store.Host, st registry.HostState, ok bool, alloc store.Al
283	return hr	300	return hr
284	}	301	}
285		302
		303	// regState is one host's live registry state, fetched ONCE per snapshot. It
		304	// exists so buildVMResponses can index a host's state instead of calling
		305	// reg.Get(vm.HostID) per VM — registry.Get deep-clones the whole host report on
		306	// every call, so N VMs on a host used to re-clone that report N times.
		307	type regState struct {
		308	st registry.HostState
		309	ok bool
		310	}
		311
		312	// fetchStates fetches each distinct host's registry state exactly once. The
		313	// returned map covers every id passed (a missing registry entry is stored with
		314	// ok=false), so callers can index it without falling back to reg.Get.
		315	func (a *API) fetchStates(ids ...[]string) map[string]regState {
		316	states := map[string]regState{}
		317	for _, group := range ids {
		318	for _, id := range group {
		319	if _, seen := states[id]; seen {
		320	continue
		321	}
		322	st, ok := a.reg.Get(id)
		323	states[id] = regState{st: st, ok: ok}
		324	}
		325	}
		326	return states
		327	}
		328
286	// snapshotHosts builds the wire host list (durable host rows merged with live	329	// snapshotHosts builds the wire host list (durable host rows merged with live
287	// registry state and server-computed allocation) for GET /hosts.	330	// registry state and server-computed allocation) for GET /hosts.
288	func (a *API) snapshotHosts() ([]types.Host, error) {	331	func (a *API) snapshotHosts() ([]types.Host, error) {
@@ -293,15 +336,34 @@ func (a *API) snapshotHosts() ([]types.Host, error) {
293	if err != nil {	336	if err != nil {
294	return nil, err	337	return nil, err
295	}	338	}
296	return a.buildHostResponses(hosts, alloc), nil	339	return a.buildHostResponses(hosts, alloc, a.fetchStates(hostIDs(hosts))), nil
		340	}
		341
		342	// hostIDs projects the host row IDs (registry keys) for a single-fetch states map.
		343	func hostIDs(hosts []store.Host) []string {
		344	ids := make([]string, len(hosts))
		345	for i, h := range hosts {
		346	ids[i] = h.ID
		347	}
		348	return ids
297	}	349	}
298		350
299	// buildHostResponses merges durable host rows with live registry state.	351	// vmHostIDs projects the host IDs referenced by a VM list (registry keys).
300	func (a *API) buildHostResponses(hosts []store.Host, alloc map[string]store.Alloc) []types.Host {	352	func vmHostIDs(vms []store.VM) []string {
		353	ids := make([]string, len(vms))
		354	for i, vm := range vms {
		355	ids[i] = vm.HostID
		356	}
		357	return ids
		358	}
		359
		360	// buildHostResponses merges durable host rows with live registry state, indexing
		361	// the pre-fetched states map (one reg.Get per host).
		362	func (a *API) buildHostResponses(hosts []store.Host, alloc map[string]store.Alloc, states map[string]regState) []types.Host {
301	out := make([]types.Host, len(hosts))	363	out := make([]types.Host, len(hosts))
302	for i, h := range hosts {	364	for i, h := range hosts {
303	st, ok := a.reg.Get(h.ID)	365	rs := states[h.ID]
304	out[i] = toHostResponse(h, st, ok, alloc[h.ID])	366	out[i] = toHostResponse(h, rs.st, rs.ok, alloc[h.ID])
305	}	367	}
306	return out	368	return out
307	}	369	}
@@ -309,7 +371,7 @@ func (a *API) buildHostResponses(hosts []store.Host, alloc map[string]store.Allo
309	func (a API) handleListHosts(w http.ResponseWriter, r http.Request) {	371	func (a API) handleListHosts(w http.ResponseWriter, r http.Request) {
310	out, err := a.snapshotHosts()	372	out, err := a.snapshotHosts()
311	if err != nil {	373	if err != nil {
312	httpError(w, "internal error", http.StatusInternalServerError)	374	http.Error(w, "internal error", http.StatusInternalServerError)
313	return	375	return
314	}	376	}
315	writeJSON(w, http.StatusOK, out)	377	writeJSON(w, http.StatusOK, out)
@@ -343,9 +405,11 @@ func toVMResponse(vm store.VM, actualPower, phase string, destroyAt int64) types
343	}	405	}
344		406
345	// deriveLifecycle folds the orthogonal state axes into one coarse lifecycle	407	// deriveLifecycle folds the orthogonal state axes into one coarse lifecycle
346	// word. This MUST stay in lockstep with vmStatus() in	408	// word. The server owns this derivation exclusively — every types.VM ships
347	// web/src/lib/fleet.svelte.ts — the client fold is retained so the UI degrades	409	// the result as vm.lifecycle, and the client (vmStatus() in
348	// gracefully against an older server, and the two must not disagree.	410	// web/src/lib/fleet.svelte.ts) displays it directly, keeping only a small
		411	// defensive fallback for malformed or missing snapshots rather than
		412	// re-deriving the value itself.
349	func deriveLifecycle(vm store.VM, actualPower, phase string) string {	413	func deriveLifecycle(vm store.VM, actualPower, phase string) string {
350	if vm.DeletedAt != nil {	414	if vm.DeletedAt != nil {
351	return "deleting"	415	return "deleting"
@@ -379,17 +443,18 @@ func (a *API) snapshotVMs() ([]types.VM, error) {
379	if err != nil {	443	if err != nil {
380	return nil, err	444	return nil, err
381	}	445	}
382	return a.buildVMResponses(vms), nil	446	return a.buildVMResponses(vms, a.fetchStates(vmHostIDs(vms))), nil
383	}	447	}
384		448
385	// buildVMResponses merges durable VM rows with live registry actual-state.	449	// buildVMResponses merges durable VM rows with live registry actual-state,
386	func (a *API) buildVMResponses(vms []store.VM) []types.VM {	450	// indexing the pre-fetched states map (one reg.Get per host, not per VM).
		451	func (a *API) buildVMResponses(vms []store.VM, states map[string]regState) []types.VM {
387	out := make([]types.VM, len(vms))	452	out := make([]types.VM, len(vms))
388	for i, vm := range vms {	453	for i, vm := range vms {
389	var actualPower, phase string	454	var actualPower, phase string
390	var destroyAt int64	455	var destroyAt int64
391	if st, ok := a.reg.Get(vm.HostID); ok {	456	if rs := states[vm.HostID]; rs.ok {
392	for _, av := range st.Report.VMs {	457	for _, av := range rs.st.Report.VMs {
393	if av.VMID == vm.ID {	458	if av.VMID == vm.ID {
394	actualPower = av.Power	459	actualPower = av.Power
395	phase = av.Phase	460	phase = av.Phase
@@ -398,7 +463,7 @@ func (a *API) buildVMResponses(vms []store.VM) []types.VM {
398	}	463	}
399	// A tombstoned VM the agent has stopped and quarantined carries a	464	// A tombstoned VM the agent has stopped and quarantined carries a
400	// hard destroy deadline; surface it so clients can render a countdown.	465	// hard destroy deadline; surface it so clients can render a countdown.
401	for _, qv := range st.Report.Quarantined {	466	for _, qv := range rs.st.Report.Quarantined {
402	if qv.VMID == vm.ID {	467	if qv.VMID == vm.ID {
403	destroyAt = qv.DestroyAtUnix	468	destroyAt = qv.DestroyAtUnix
404	break	469	break
@@ -413,7 +478,7 @@ func (a *API) buildVMResponses(vms []store.VM) []types.VM {
413	func (a API) handleListVMs(w http.ResponseWriter, r http.Request) {	478	func (a API) handleListVMs(w http.ResponseWriter, r http.Request) {
414	out, err := a.snapshotVMs()	479	out, err := a.snapshotVMs()
415	if err != nil {	480	if err != nil {
416	httpError(w, "internal error", http.StatusInternalServerError)	481	http.Error(w, "internal error", http.StatusInternalServerError)
417	return	482	return
418	}	483	}
419	writeJSON(w, http.StatusOK, out)	484	writeJSON(w, http.StatusOK, out)
@@ -424,7 +489,7 @@ func (a API) handleListVMs(w http.ResponseWriter, r http.Request) {
424	// validation, not a default.	489	// validation, not a default.
425	func (a API) applyVMDefaults(req types.CreateVMRequest) (string, int) {	490	func (a API) applyVMDefaults(req types.CreateVMRequest) (string, int) {
426	if req.Name == "" {	491	if req.Name == "" {
427	req.Name = "sandbox-" + store.RandHex(3)	492	req.Name = "sandbox-" + random.Hex(3)
428	}	493	}
429	// Image URL+SHA must come as a pair; apply defaults only when BOTH are empty.	494	// Image URL+SHA must come as a pair; apply defaults only when BOTH are empty.
430	if req.ImageURL == "" && req.ImageSHA256 == "" {	495	if req.ImageURL == "" && req.ImageSHA256 == "" {
@@ -482,16 +547,16 @@ func (a API) handleCreateVM(w http.ResponseWriter, r http.Request) {
482	return	547	return
483	}	548	}
484	if req.HostID == "" {	549	if req.HostID == "" {
485	httpError(w, "host_id required", http.StatusBadRequest)	550	http.Error(w, "host_id required", http.StatusBadRequest)
486	return	551	return
487	}	552	}
488		553
489	if msg, code := a.applyVMDefaults(&req); msg != "" {	554	if msg, code := a.applyVMDefaults(&req); msg != "" {
490	httpError(w, msg, code)	555	http.Error(w, msg, code)
491	return	556	return
492	}	557	}
493	if msg, code := validateCreateVM(&req); msg != "" {	558	if msg, code := validateCreateVM(&req); msg != "" {
494	httpError(w, msg, code)	559	http.Error(w, msg, code)
495	return	560	return
496	}	561	}
497		562
@@ -514,7 +579,7 @@ func (a API) handleCreateVM(w http.ResponseWriter, r http.Request) {
514	if req.CloudInit != "" && req.SSHAuthorizedKey != "" {	579	if req.CloudInit != "" && req.SSHAuthorizedKey != "" {
515	merged, err := cloudinit.AddSSHKey(req.CloudInit, req.SSHAuthorizedKey)	580	merged, err := cloudinit.AddSSHKey(req.CloudInit, req.SSHAuthorizedKey)
516	if err != nil {	581	if err != nil {
517	httpError(w, "cannot add ssh_authorized_key to cloud_init: "+err.Error(), http.StatusBadRequest)	582	http.Error(w, "cannot add ssh_authorized_key to cloud_init: "+err.Error(), http.StatusBadRequest)
518	return	583	return
519	}	584	}
520	req.CloudInit = merged	585	req.CloudInit = merged
@@ -522,7 +587,7 @@ func (a API) handleCreateVM(w http.ResponseWriter, r http.Request) {
522	}	587	}
523		588
524	// Generate ID here so we can return it.	589	// Generate ID here so we can return it.
525	id := store.RandHex(16)	590	id := random.Hex(16)
526		591
527	vm := store.VM{	592	vm := store.VM{
528	ID: id,	593	ID: id,
@@ -547,7 +612,7 @@ func (a API) handleCreateVM(w http.ResponseWriter, r http.Request) {
547	if a.hostCerts != nil {	612	if a.hostCerts != nil {
548	keyPEM, cert, err := a.hostCerts.MintHostCert(req.Name)	613	keyPEM, cert, err := a.hostCerts.MintHostCert(req.Name)
549	if err != nil {	614	if err != nil {
550	httpError(w, "internal error", http.StatusInternalServerError)	615	http.Error(w, "internal error", http.StatusInternalServerError)
551	return	616	return
552	}	617	}
553	vm.SSHHostKey = keyPEM	618	vm.SSHHostKey = keyPEM
@@ -557,11 +622,13 @@ func (a API) handleCreateVM(w http.ResponseWriter, r http.Request) {
557	if err := a.st.CreateVM(vm); err != nil {	622	if err := a.st.CreateVM(vm); err != nil {
558	switch {	623	switch {
559	case errors.Is(err, store.ErrNameTaken):	624	case errors.Is(err, store.ErrNameTaken):
560	httpError(w, "name already in use", http.StatusConflict)	625	http.Error(w, "name already in use", http.StatusConflict)
561	case errors.Is(err, store.ErrHostNotFound):	626	case errors.Is(err, store.ErrHostNotFound):
562	httpError(w, "unknown host_id", http.StatusBadRequest)	627	http.Error(w, "unknown host_id", http.StatusBadRequest)
		628	case errors.Is(err, store.ErrHostNotEnrolled):
		629	http.Error(w, "host is not accepting new VMs", http.StatusConflict)
563	default:	630	default:
564	httpError(w, "internal error", http.StatusInternalServerError)	631	http.Error(w, "internal error", http.StatusInternalServerError)
565	}	632	}
566	return	633	return
567	}	634	}
@@ -572,50 +639,57 @@ func (a API) handleCreateVM(w http.ResponseWriter, r http.Request) {
572	writeJSON(w, http.StatusCreated, types.CreateVMResponse{ID: id, Name: req.Name})	639	writeJSON(w, http.StatusCreated, types.CreateVMResponse{ID: id, Name: req.Name})
573	}	640	}
574		641
575	func (a API) handlePatchVM(w http.ResponseWriter, r http.Request) {	642	// mutateVM implements the choreography shared by every VM mutation endpoint
576	id := r.PathValue("id")	643	// (patch/delete/restore): run mutate; a sql.ErrNoRows becomes the caller's
577	var req types.PatchVMRequest	644	// not-found response, any other error a generic 500. On success the row
578	if !decodeJSON(w, r, &req) {	645	// always survives the mutation (power change, tombstone, and restore all
579	return	646	// keep it), so it is scanned for name/host: a best-effort audit row is
580	}	647	// appended and the host's desired-state stream poked (a lookup miss is
581	if req.PowerState != "running" && req.PowerState != "stopped" {	648	// non-fatal — the mutation already succeeded). Finally SSE watchers are
582	httpError(w, "power_state must be running or stopped", http.StatusBadRequest)	649	// notified and the response is written as 204, unconditionally.
583	return	650	func (a *API) mutateVM(w http.ResponseWriter, id string, mutate func(id string) error,
584	}	651	notFoundMsg string, notFoundStatus int,
585	if err := a.st.SetVMPower(id, req.PowerState); err != nil {	652	auditAction string, auditDetail func(vm store.VM) map[string]string) {
586	if err == sql.ErrNoRows {	653	if err := mutate(id); err != nil {
587	httpError(w, "not found", http.StatusNotFound)	654	if errors.Is(err, sql.ErrNoRows) {
		655	http.Error(w, notFoundMsg, notFoundStatus)
588	return	656	return
589	}	657	}
590	httpError(w, "internal error", http.StatusInternalServerError)	658	http.Error(w, "internal error", http.StatusInternalServerError)
591	return	659	return
592	}	660	}
593	// The row survives a power change, so we can still scan it for name/host.
594	if vm, ok := a.vmByID(id); ok {	661	if vm, ok := a.vmByID(id); ok {
595	a.audit("vm.power", map[string]string{"vm_id": id, "name": vm.Name, "power": req.PowerState})	662	a.audit(auditAction, auditDetail(vm))
596	a.hub.Poke(vm.HostID)	663	a.hub.Poke(vm.HostID)
597	}	664	}
598	a.notif.notify()	665	a.notif.notify()
599	w.WriteHeader(http.StatusNoContent)	666	w.WriteHeader(http.StatusNoContent)
600	}	667	}
601		668
602	func (a API) handleDeleteVM(w http.ResponseWriter, r http.Request) {	669	func (a API) handlePatchVM(w http.ResponseWriter, r http.Request) {
603	id := r.PathValue("id")	670	id := r.PathValue("id")
604	if err := a.st.TombstoneVM(id); err != nil {	671	var req types.PatchVMRequest
605	if err == sql.ErrNoRows {	672	if !decodeJSON(w, r, &req) {
606	httpError(w, "not found", http.StatusNotFound)
607	return
608	}
609	httpError(w, "internal error", http.StatusInternalServerError)
610	return	673	return
611	}	674	}
612	// TombstoneVM keeps the row, so we can still scan for name/host.	675	if req.PowerState != "running" && req.PowerState != "stopped" {
613	if vm, ok := a.vmByID(id); ok {	676	http.Error(w, "power_state must be running or stopped", http.StatusBadRequest)
614	a.audit("vm.delete", map[string]string{"vm_id": id, "name": vm.Name})	677	return
615	a.hub.Poke(vm.HostID)
616	}	678	}
617	a.notif.notify()	679	a.mutateVM(w, id, func(id string) error { return a.st.SetVMPower(id, req.PowerState) },
618	w.WriteHeader(http.StatusNoContent)	680	"not found", http.StatusNotFound,
		681	"vm.power", func(vm store.VM) map[string]string {
		682	return map[string]string{"vm_id": id, "name": vm.Name, "power": req.PowerState}
		683	})
		684	}
		685
		686	func (a API) handleDeleteVM(w http.ResponseWriter, r http.Request) {
		687	id := r.PathValue("id")
		688	a.mutateVM(w, id, a.st.TombstoneVM,
		689	"not found", http.StatusNotFound,
		690	"vm.delete", func(vm store.VM) map[string]string {
		691	return map[string]string{"vm_id": id, "name": vm.Name}
		692	})
619	}	693	}
620		694
621	// handleRestoreVM un-tombstones a VM that is still within the teardown grace	695	// handleRestoreVM un-tombstones a VM that is still within the teardown grace
@@ -624,35 +698,20 @@ func (a API) handleDeleteVM(w http.ResponseWriter, r http.Request) {
624	// converges it back toward its power_state — so the server only pokes.	698	// converges it back toward its power_state — so the server only pokes.
625	func (a API) handleRestoreVM(w http.ResponseWriter, r http.Request) {	699	func (a API) handleRestoreVM(w http.ResponseWriter, r http.Request) {
626	id := r.PathValue("id")	700	id := r.PathValue("id")
627	if err := a.st.RestoreVM(id); err != nil {	701	a.mutateVM(w, id, a.st.RestoreVM,
628	if errors.Is(err, sql.ErrNoRows) {	702	"vm not restorable (already destroyed or not deleted)", http.StatusConflict,
629	httpError(w, "vm not restorable (already destroyed or not deleted)", http.StatusConflict)	703	"vm.restore", func(vm store.VM) map[string]string {
630	return	704	return map[string]string{"vm_id": id, "name": vm.Name}
631	}	705	})
632	httpError(w, "internal error", http.StatusInternalServerError)
633	return
634	}
635	// RestoreVM clears deleted_at, so the row is back in desired state; scan it
636	// for name/host. A lookup miss is non-fatal — the restore already succeeded.
637	if vm, ok := a.vmByID(id); ok {
638	a.audit("vm.restore", map[string]string{"vm_id": id, "name": vm.Name})
639	a.hub.Poke(vm.HostID)
640	}
641	a.notif.notify()
642	w.WriteHeader(http.StatusNoContent)
643	}	706	}
644		707
645	// vmByID scans ListVMs for the desired-state VM row with the given id.	708	// vmByID fetches the VM row with the given id via an indexed primary-key
646	// Phase 1 scale: linear scan is acceptable (callers need name + host_id).	709	// lookup (callers need name + host_id). Any error — including a missing row —
		710	// reports ok=false, matching the callers' non-fatal not-found handling.
647	func (a *API) vmByID(vmID string) (store.VM, bool) {	711	func (a *API) vmByID(vmID string) (store.VM, bool) {
648	vms, err := a.st.ListVMs()	712	vm, err := a.st.GetVM(vmID)
649	if err != nil {	713	if err != nil {
650	return store.VM{}, false	714	return store.VM{}, false
651	}	715	}
652	for _, vm := range vms {	716	return vm, true
653	if vm.ID == vmID {
654	return vm, true
655	}
656	}
657	return store.VM{}, false
658	}	717	}

internal/server/api/api_test.go

@@ @@ -131,6 +131,52 @@ func TestCreateVMUnknownHostReturns400(t *testing.T) { @@
+	assert.Equal(t, 400, resp.StatusCode)
+// TestHostResponseSurfacesSyncHealth pins the sync-health mapping in
+// toHostResponse: a live host surfaces last_seen, seconds_since_last_seen,
+// stale and sessions; a host with no registry entry (never connected) and a
+// host that connected but never reported both leave the age fields null.
+func TestHostResponseSurfacesSyncHealth(t *testing.T) {
+	h := store.Host{ID: "h1", Name: "host-1"}
+	t.Run("online and stale", func(t *testing.T) {
+		st := registry.HostState{
+			LastSeen:      time.Unix(1700000000, 0),
+			SinceLastSeen: 20 * time.Second,
+			Online:        true,
+			Stale:         true,
+			Sessions:      3,
+		}
+		hr := toHostResponse(h, st, true, store.Alloc{})
+		require.NotNil(t, hr.LastSeen)
+		assert.True(t, hr.LastSeen.Equal(time.Unix(1700000000, 0)))
+		require.NotNil(t, hr.SecondsSinceLastSeen)
+		assert.Equal(t, int64(20), *hr.SecondsSinceLastSeen)
+		assert.True(t, hr.Online)
+		assert.True(t, hr.Stale, "past half the online window ⇒ stale")
+		assert.Equal(t, 3, hr.Sessions)
+	})
+	t.Run("connected but never reported", func(t *testing.T) {
+		// RecordConnect created an entry (ok=true, Sessions>0) but LastSeen is
+		// unset, so the age fields must stay null rather than emit the zero time.
+		st := registry.HostState{Sessions: 1}
+		hr := toHostResponse(h, st, true, store.Alloc{})
+		assert.Nil(t, hr.LastSeen)
+		assert.Nil(t, hr.SecondsSinceLastSeen)
+		assert.False(t, hr.Online)
+		assert.Equal(t, 1, hr.Sessions)
+	})
+	t.Run("no registry entry", func(t *testing.T) {
+		hr := toHostResponse(h, registry.HostState{}, false, store.Alloc{})
+		assert.Nil(t, hr.LastSeen)
+		assert.Nil(t, hr.SecondsSinceLastSeen)
+		assert.False(t, hr.Online)
+		assert.False(t, hr.Stale)
+		assert.Equal(t, 0, hr.Sessions)
+	})
+}
+// newServer is the shared builder. It also returns the *API itself for tests
+// that need post-construction wiring (SetConsoleDialer, SetCertMinter).
+func newServer(t *testing.T) (*httptest.Server, *store.Store, *hub.Hub, *registry.Registry, *API) {
+	}, st, reg, h)
+	ts := httptest.NewServer(a.Handler())
+	t.Cleanup(ts.Close)
+	t.Cleanup(a.Close) // stop the snapshot hub goroutine
+	return ts, st, h, reg, a

internal/server/api/console.go

Old		New
@@ -32,17 +32,19 @@ const consoleOpenTimeout = 10 * time.Second
32	// the admin token never appears in a URL.	32	// the admin token never appears in a URL.
33	func (a API) handleConsoleWS(w http.ResponseWriter, r http.Request) {	33	func (a API) handleConsoleWS(w http.ResponseWriter, r http.Request) {
34	if !a.tickets.consume(r.URL.Query().Get("ticket")) {	34	if !a.tickets.consume(r.URL.Query().Get("ticket")) {
35	httpError(w, "unauthorized", http.StatusUnauthorized)	35	http.Error(w, "unauthorized", http.StatusUnauthorized)
36	return	36	return
37	}	37	}
38	if a.console == nil {	38	if a.console == nil {
39	httpError(w, "console unavailable", http.StatusServiceUnavailable)	39	http.Error(w, "console unavailable", http.StatusServiceUnavailable)
40	return	40	return
41	}	41	}
42	id := r.PathValue("id")	42	id := r.PathValue("id")
43	vm, ok := a.vmByID(id)	43	vm, ok := a.vmByID(id)
44	if !ok {	44	if !ok \|\| vm.DeletedAt != nil {
45	httpError(w, "not found", http.StatusNotFound)	45	// GetVM does not filter tombstones; a console to a VM being decommissioned
		46	// or deleted must not open (the agent leg would refuse it anyway).
		47	http.Error(w, "not found", http.StatusNotFound)
46	return	48	return
47	}	49	}
48		50

internal/server/api/console_test.go

Old		New
@@ -93,6 +93,21 @@ func TestConsoleWSRequiresTicket(t *testing.T) {
93	assert.Equal(t, 401, resp.StatusCode)	93	assert.Equal(t, 401, resp.StatusCode)
94	}	94	}
95		95
		96	func TestConsoleWSRejectsDeletedVM(t *testing.T) {
		97	a, fix := newConsoleAPI(t)
		98	a.SetConsoleDialer(&fakeConsole{})
		99
		100	// Tombstone the VM (GetVM does not filter tombstones).
		101	resp := do(t, "DELETE", fix.ts.URL+"/api/v1/vms/"+fix.vmID, "admintok", nil)
		102	require.Equal(t, 204, resp.StatusCode)
		103
		104	ticket := mintTicket(t, fix.ts.URL, "admintok")
		105	r, err := fix.ts.Client().Get(fix.ts.URL + "/api/v1/vms/" + fix.vmID + "/console/ws?ticket=" + ticket)
		106	require.NoError(t, err)
		107	defer r.Body.Close()
		108	assert.Equal(t, 404, r.StatusCode, "console to a tombstoned VM must be rejected")
		109	}
		110
96	func TestConsoleWSBridgesBytes(t *testing.T) {	111	func TestConsoleWSBridgesBytes(t *testing.T) {
97	a, fix := newConsoleAPI(t)	112	a, fix := newConsoleAPI(t)
98	fc := &fakeConsole{opened: make(chan struct{}, 1)}	113	fc := &fakeConsole{opened: make(chan struct{}, 1)}

internal/server/api/decommission_poke_test.go

@@ @@ -0,0 +1,66 @@ @@
+package api
+import (
+	"net/http"
+	"testing"
+	"time"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+// TestDecommissionPokesAgent pins the fix for the decommission stall: the
+// handler must wake the host's desired-state stream (like every VM mutation
+// handler does), otherwise an online host never learns its VMs were tombstoned
+// and stalls in `decommissioning` forever.
+func TestDecommissionPokesAgent(t *testing.T) {
+	ts, a, _ := apiServer(t)
+	out := enroll(t, ts)
+	hostID := out["host_id"]
+	ch, cancel := a.hub.Subscribe(hostID)
+	defer cancel()
+	resp := do(t, "DELETE", ts.URL+"/api/v1/hosts/"+hostID, "admintok", nil)
+	require.Equal(t, http.StatusAccepted, resp.StatusCode)
+	select {
+	case <-ch:
+	case <-time.After(2 * time.Second):
+		t.Fatal("decommission must poke the host's desired-state stream")
+	}
+}
+// TestForceDecommissionRemovesHostWithVMs pins the dead-hardware escape hatch:
+// ?force=true purges VM rows and removes the host immediately, without waiting
+// for an agent drain that (for dead hardware) can never happen.
+func TestForceDecommissionRemovesHostWithVMs(t *testing.T) {
+	ts, _, _ := apiServer(t)
+	out := enroll(t, ts)
+	hostID := out["host_id"]
+	resp := do(t, "POST", ts.URL+"/api/v1/vms", "admintok", map[string]any{"host_id": hostID, "name": "vm-a"})
+	require.Equal(t, http.StatusCreated, resp.StatusCode)
+	// Graceful delete would only tombstone and wait; force removes now.
+	resp = do(t, "DELETE", ts.URL+"/api/v1/hosts/"+hostID+"?force=true", "admintok", nil)
+	require.Equal(t, http.StatusOK, resp.StatusCode)
+	hosts := decodeJSONKeys(t, do(t, "GET", ts.URL+"/api/v1/hosts", "admintok", nil))
+	assert.Empty(t, hosts, "force must remove the host immediately")
+}
+// TestCreateVMRejectedOnDecommissioningHost pins that a host mid-decommission
+// no longer accepts new VMs (previously only the FK was enforced, so a create
+// could land on a host being torn down).
+func TestCreateVMRejectedOnDecommissioningHost(t *testing.T) {
+	ts, _, _ := apiServer(t)
+	out := enroll(t, ts)
+	hostID := out["host_id"]
+	resp := do(t, "DELETE", ts.URL+"/api/v1/hosts/"+hostID, "admintok", nil)
+	require.Equal(t, http.StatusAccepted, resp.StatusCode)
+	resp = do(t, "POST", ts.URL+"/api/v1/vms", "admintok", map[string]any{"host_id": hostID, "name": "vm-late"})
+	assert.Equal(t, http.StatusConflict, resp.StatusCode, "create on a decommissioning host must be rejected")
+}

internal/server/api/events.go

Old		New
@@ -1,7 +1,6 @@
1	package api	1	package api
2		2
3	import (	3	import (
4	"bytes"
5	"database/sql"	4	"database/sql"
6	"encoding/json"	5	"encoding/json"
7	"errors"	6	"errors"
@@ -11,25 +10,68 @@ import (
11	"time"	10	"time"
12		11
13	"github.com/a73x/eitri/internal/server/api/types"	12	"github.com/a73x/eitri/internal/server/api/types"
		13	"github.com/a73x/eitri/internal/server/store"
14	)	14	)
15		15
16	// handleDecommissionHost begins graceful host decommission: its VMs are	16	// handleDecommissionHost begins graceful host decommission: its VMs are
17	// tombstoned and reaped, then the sweeper removes the host and frees its CIDR.	17	// tombstoned and reaped by the agent, then the sweeper removes the host and
		18	// frees its CIDR. The agent only learns of the tombstones when its desired-state
		19	// stream is poked, so we poke here exactly as the VM mutation handlers do —
		20	// without it a healthy host never drains and stalls in `decommissioning`.
		21	//
		22	// ?force=true is the escape hatch for dead hardware whose agent will never
		23	// report: it purges the VM rows and removes the host immediately (their compute
		24	// is gone with the box), reclaiming the CIDR without waiting for a drain that
		25	// can never happen.
18	func (a API) handleDecommissionHost(w http.ResponseWriter, r http.Request) {	26	func (a API) handleDecommissionHost(w http.ResponseWriter, r http.Request) {
19	id := r.PathValue("id")	27	id := r.PathValue("id")
		28
		29	if forceParam(r) {
		30	purged, err := a.st.ForceRemoveHost(id)
		31	switch {
		32	case errors.Is(err, sql.ErrNoRows):
		33	http.Error(w, "host not found", http.StatusNotFound)
		34	return
		35	case err != nil:
		36	http.Error(w, "internal error", http.StatusInternalServerError)
		37	return
		38	}
		39	a.audit("host.decommission", map[string]string{
		40	"host_id": id, "remote": clientIP(r),
		41	"force": "true", "vms_purged": strconv.Itoa(purged),
		42	})
		43	a.hub.Poke(id)
		44	a.notif.notify()
		45	w.WriteHeader(http.StatusOK)
		46	return
		47	}
		48
20	switch err := a.st.DecommissionHost(id); {	49	switch err := a.st.DecommissionHost(id); {
21	case errors.Is(err, sql.ErrNoRows):	50	case errors.Is(err, sql.ErrNoRows):
22	httpError(w, "host not found", http.StatusNotFound)	51	http.Error(w, "host not found", http.StatusNotFound)
23	return	52	return
24	case err != nil:	53	case err != nil:
25	httpError(w, "internal error", http.StatusInternalServerError)	54	http.Error(w, "internal error", http.StatusInternalServerError)
26	return	55	return
27	}	56	}
28	a.audit("host.decommission", map[string]string{"host_id": id, "remote": clientIP(r)})	57	a.audit("host.decommission", map[string]string{"host_id": id, "remote": clientIP(r)})
		58	a.hub.Poke(id)
29	a.notif.notify()	59	a.notif.notify()
30	w.WriteHeader(http.StatusAccepted)	60	w.WriteHeader(http.StatusAccepted)
31	}	61	}
32		62
		63	// forceParam reports whether the request opts into forced removal via ?force,
		64	// accepting a bare ?force or ?force=true. An absent key, or any other value, is
		65	// false (the default graceful path).
		66	func forceParam(r *http.Request) bool {
		67	q := r.URL.Query()
		68	if !q.Has("force") {
		69	return false
		70	}
		71	v := q.Get("force")
		72	return v == "" \|\| v == "true"
		73	}
		74
33	// handleRevokeCredential bumps the host's credential generation, revoking its	75	// handleRevokeCredential bumps the host's credential generation, revoking its
34	// outstanding credential WITHOUT rotating the fleet secret. The agent's live	76	// outstanding credential WITHOUT rotating the fleet secret. The agent's live
35	// session is closed by syncsvc within one report tick; the host stays dark	77	// session is closed by syncsvc within one report tick; the host stays dark
@@ -41,33 +83,35 @@ func (a API) handleRevokeCredential(w http.ResponseWriter, r http.Request) {
41	_, err := a.st.BumpCredGeneration(id, clientIP(r))	83	_, err := a.st.BumpCredGeneration(id, clientIP(r))
42	switch {	84	switch {
43	case errors.Is(err, sql.ErrNoRows):	85	case errors.Is(err, sql.ErrNoRows):
44	httpError(w, "host not found", http.StatusNotFound)	86	http.Error(w, "host not found", http.StatusNotFound)
45	return	87	return
46	case err != nil:	88	case err != nil:
47	httpError(w, "internal error", http.StatusInternalServerError)	89	http.Error(w, "internal error", http.StatusInternalServerError)
48	return	90	return
49	}	91	}
50	w.WriteHeader(http.StatusNoContent)	92	w.WriteHeader(http.StatusNoContent)
51	}	93	}
52		94
53	// handleListAudit returns the newest audit rows (default 100, ?limit=N caps	95	// parseLimit reads the shared ?limit=N query param (default 100, capped to
54	// at 1000). Completes the forensic story: rows were previously reachable only	96	// 1..1000). On a bad value it writes the 400 and returns ok=false so the caller
55	// by opening the SQLite file.	97	// just returns.
56	func (a API) handleListAudit(w http.ResponseWriter, r http.Request) {	98	func parseLimit(w http.ResponseWriter, r *http.Request) (int, bool) {
57	limit := 100	99	limit := 100
58	if v := r.URL.Query().Get("limit"); v != "" {	100	if v := r.URL.Query().Get("limit"); v != "" {
59	n, err := strconv.Atoi(v)	101	n, err := strconv.Atoi(v)
60	if err != nil \|\| n < 1 \|\| n > 1000 {	102	if err != nil \|\| n < 1 \|\| n > 1000 {
61	httpError(w, "limit must be 1..1000", http.StatusBadRequest)	103	http.Error(w, "limit must be 1..1000", http.StatusBadRequest)
62	return	104	return 0, false
63	}	105	}
64	limit = n	106	limit = n
65	}	107	}
66	rows, err := a.st.ListAudit(limit)	108	return limit, true
67	if err != nil {	109	}
68	httpError(w, "internal error", http.StatusInternalServerError)	110
69	return	111	// auditRowsToResponse maps store rows to the wire shape, embedding each detail
70	}	112	// as raw JSON and defensively re-marshalling anything that isn't valid JSON so
		113	// the endpoint never emits a malformed body.
		114	func auditRowsToResponse(rows []store.AuditEntry) []types.AuditEvent {
71	out := make([]types.AuditEvent, len(rows))	115	out := make([]types.AuditEvent, len(rows))
72	for i, e := range rows {	116	for i, e := range rows {
73	detail := json.RawMessage(e.Detail)	117	detail := json.RawMessage(e.Detail)
@@ -76,7 +120,23 @@ func (a API) handleListAudit(w http.ResponseWriter, r http.Request) {
76	}	120	}
77	out[i] = types.AuditEvent{At: e.At, Action: e.Action, Detail: detail}	121	out[i] = types.AuditEvent{At: e.At, Action: e.Action, Detail: detail}
78	}	122	}
79	writeJSON(w, http.StatusOK, out)	123	return out
		124	}
		125
		126	// handleListAudit returns the newest audit rows (default 100, ?limit=N caps
		127	// at 1000). Completes the forensic story: rows were previously reachable only
		128	// by opening the SQLite file.
		129	func (a API) handleListAudit(w http.ResponseWriter, r http.Request) {
		130	limit, ok := parseLimit(w, r)
		131	if !ok {
		132	return
		133	}
		134	rows, err := a.st.ListAudit(limit)
		135	if err != nil {
		136	http.Error(w, "internal error", http.StatusInternalServerError)
		137	return
		138	}
		139	writeJSON(w, http.StatusOK, auditRowsToResponse(rows))
80	}	140	}
81		141
82	// handleListVMEvents returns one VM's lifecycle timeline: the audit rows whose	142	// handleListVMEvents returns one VM's lifecycle timeline: the audit rows whose
@@ -86,29 +146,16 @@ func (a API) handleListAudit(w http.ResponseWriter, r http.Request) {
86	// handleListAudit.	146	// handleListAudit.
87	func (a API) handleListVMEvents(w http.ResponseWriter, r http.Request) {	147	func (a API) handleListVMEvents(w http.ResponseWriter, r http.Request) {
88	id := r.PathValue("id")	148	id := r.PathValue("id")
89	limit := 100	149	limit, ok := parseLimit(w, r)
90	if v := r.URL.Query().Get("limit"); v != "" {	150	if !ok {
91	n, err := strconv.Atoi(v)	151	return
92	if err != nil \|\| n < 1 \|\| n > 1000 {
93	httpError(w, "limit must be 1..1000", http.StatusBadRequest)
94	return
95	}
96	limit = n
97	}	152	}
98	rows, err := a.st.ListVMEvents(id, limit)	153	rows, err := a.st.ListVMEvents(id, limit)
99	if err != nil {	154	if err != nil {
100	httpError(w, "internal error", http.StatusInternalServerError)	155	http.Error(w, "internal error", http.StatusInternalServerError)
101	return	156	return
102	}	157	}
103	out := make([]types.AuditEvent, len(rows))	158	writeJSON(w, http.StatusOK, auditRowsToResponse(rows))
104	for i, e := range rows {
105	detail := json.RawMessage(e.Detail)
106	if !json.Valid(detail) { // defensive: never emit invalid JSON
107	detail, _ = json.Marshal(e.Detail)
108	}
109	out[i] = types.AuditEvent{At: e.At, Action: e.Action, Detail: detail}
110	}
111	writeJSON(w, http.StatusOK, out)
112	}	159	}
113		160
114	// handleMintStreamTicket issues a one-time short-TTL ticket for the SSE	161	// handleMintStreamTicket issues a one-time short-TTL ticket for the SSE
@@ -118,40 +165,37 @@ func (a API) handleMintStreamTicket(w http.ResponseWriter, r http.Request) {
118	writeJSON(w, http.StatusCreated, types.StreamTicketResponse{Ticket: a.tickets.mint()})	165	writeJSON(w, http.StatusCreated, types.StreamTicketResponse{Ticket: a.tickets.mint()})
119	}	166	}
120		167
121	// handleEvents streams the fleet snapshot as Server-Sent Events. It pushes on	168	// handleEvents streams the fleet snapshot as Server-Sent Events. It subscribes
122	// every desired-state change (via the notifier) and re-checks on a 1s tick to	169	// to the central snapshot hub — which marshals ONE shared snapshot on a 1s tick
123	// catch agent-reported actual-state changes, sending only when the snapshot	170	// / desired-state wake and fans the identical bytes to every client — and writes
124	// actually changed. A periodic comment keeps the connection alive.	171	// each delivered snapshot as an `event: state` frame. The hub delivers the
		172	// current snapshot immediately on subscribe, so a new client gets initial state
		173	// without doing its own marshal, and suppresses unchanged snapshots so no frame
		174	// is pushed when nothing changed. A periodic comment keeps the connection alive.
125	func (a API) handleEvents(w http.ResponseWriter, r http.Request) {	175	func (a API) handleEvents(w http.ResponseWriter, r http.Request) {
126	if !a.tickets.consume(r.URL.Query().Get("ticket")) {	176	// Assert streaming support BEFORE consuming the one-time ticket, so a 500 on
127	http.Error(w, "unauthorized", http.StatusUnauthorized)	177	// a non-flushing ResponseWriter doesn't burn the client's ticket.
128	return
129	}
130	flusher, ok := w.(http.Flusher)	178	flusher, ok := w.(http.Flusher)
131	if !ok {	179	if !ok {
132	http.Error(w, "streaming unsupported", http.StatusInternalServerError)	180	http.Error(w, "streaming unsupported", http.StatusInternalServerError)
133	return	181	return
134	}	182	}
		183	if !a.tickets.consume(r.URL.Query().Get("ticket")) {
		184	http.Error(w, "unauthorized", http.StatusUnauthorized)
		185	return
		186	}
135		187
136	w.Header().Set("Content-Type", "text/event-stream")	188	w.Header().Set("Content-Type", "text/event-stream")
137	w.Header().Set("Cache-Control", "no-cache")	189	w.Header().Set("Cache-Control", "no-cache")
138	w.Header().Set("Connection", "keep-alive")	190	w.Header().Set("Connection", "keep-alive")
139		191
140	wake, unsub := a.notif.subscribe()	192	snaps, unsub := a.snap.subscribe()
141	defer unsub()	193	defer unsub()
142		194
143	tick := time.NewTicker(time.Second)
144	defer tick.Stop()
145	heartbeat := time.NewTicker(15 * time.Second)	195	heartbeat := time.NewTicker(15 * time.Second)
146	defer heartbeat.Stop()	196	defer heartbeat.Stop()
147		197
148	var last []byte	198	sendState := func(payload []byte) bool {
149	sendIfChanged := func() bool {
150	payload, err := a.marshalSnapshot()
151	if err != nil \|\| bytes.Equal(payload, last) {
152	return err == nil
153	}
154	last = payload
155	if _, err := fmt.Fprintf(w, "event: state\ndata: %s\n\n", payload); err != nil {	199	if _, err := fmt.Fprintf(w, "event: state\ndata: %s\n\n", payload); err != nil {
156	return false	200	return false
157	}	201	}
@@ -159,18 +203,13 @@ func (a API) handleEvents(w http.ResponseWriter, r http.Request) {
159	return true	203	return true
160	}	204	}
161		205
162	sendIfChanged() // initial snapshot
163	ctx := r.Context()	206	ctx := r.Context()
164	for {	207	for {
165	select {	208	select {
166	case <-ctx.Done():	209	case <-ctx.Done():
167	return	210	return
168	case <-wake:	211	case payload := <-snaps:
169	if !sendIfChanged() {	212	if !sendState(payload) {
170	return
171	}
172	case <-tick.C:
173	if !sendIfChanged() {
174	return	213	return
175	}	214	}
176	case <-heartbeat.C:	215	case <-heartbeat.C:
@@ -184,14 +223,17 @@ func (a API) handleEvents(w http.ResponseWriter, r http.Request) {
184		223
185	// marshalSnapshot builds and JSON-encodes the current fleet snapshot. The	224	// marshalSnapshot builds and JSON-encodes the current fleet snapshot. The
186	// store reads happen in one transaction (store.Snapshot) so hosts, allocation	225	// store reads happen in one transaction (store.Snapshot) so hosts, allocation
187	// and VMs can never mix state from two different epochs.	226	// and VMs can never mix state from two different epochs. Each referenced host's
		227	// registry state is fetched ONCE (not once per VM), then shared by both the host
		228	// and VM builders — registry.Get deep-clones the whole host report per call.
188	func (a *API) marshalSnapshot() ([]byte, error) {	229	func (a *API) marshalSnapshot() ([]byte, error) {
189	hosts, alloc, vms, err := a.st.Snapshot()	230	hosts, alloc, vms, err := a.st.Snapshot()
190	if err != nil {	231	if err != nil {
191	return nil, err	232	return nil, err
192	}	233	}
		234	states := a.fetchStates(hostIDs(hosts), vmHostIDs(vms))
193	return json.Marshal(types.StateSnapshot{	235	return json.Marshal(types.StateSnapshot{
194	Hosts: a.buildHostResponses(hosts, alloc),	236	Hosts: a.buildHostResponses(hosts, alloc, states),
195	VMs: a.buildVMResponses(vms),	237	VMs: a.buildVMResponses(vms, states),
196	})	238	})
197	}	239	}

internal/server/api/snapshot_hub.go

@@ @@ -0,0 +1,130 @@ @@
+package api
+import (
+	"bytes"
+	"sync"
+	"time"
+)
+// snapshotHub computes the fleet SSE snapshot ONCE, centrally, and fans the
+// identical bytes out to every connected client. Before this, each SSE client
+// goroutine ran its own 1s ticker and marshalled its own snapshot (a DB tx +
+// full build + json.Marshal), so M clients meant M full snapshots every second.
+// The hub collapses that to one build per tick/wake regardless of client count.
+//
+// Fan-out is latest-wins / non-blocking: each subscriber holds a buffer-1
+// channel carrying only the newest snapshot. A lagging client drops intermediate
+// snapshots (correct — SSE state is a full snapshot, so the newest supersedes any
+// it missed), and the central loop NEVER blocks on a slow reader, so a stuck
+// client can neither wedge the hub nor grow memory unbounded.
+type snapshotHub struct {
+	build      func() ([]byte, error) // marshals the current snapshot (a.marshalSnapshot)
+	wake       <-chan struct{}        // desired-state wake source (single subscription)
+	notifUnsub func()                 // releases the notifier subscription on Close
+	mu      sync.Mutex
+	current []byte // latest marshalled snapshot; delivered to new subscribers immediately
+	subs    map[chan []byte]struct{}
+	stop     chan struct{}
+	done     chan struct{}
+	stopOnce sync.Once
+}
+// newSnapshotHub builds the hub, subscribes to the notifier SYNCHRONOUSLY (so no
+// wake can be lost in the window before run's goroutine is scheduled), and
+// computes the INITIAL snapshot, so a client that subscribes before the first
+// tick still gets current state. The caller must launch run() (once).
+func newSnapshotHub(build func() ([]byte, error), notif *notifier) *snapshotHub {
+	wake, unsub := notif.subscribe()
+	h := &snapshotHub{
+		build:      build,
+		wake:       wake,
+		notifUnsub: unsub,
+		subs:       make(map[chan []byte]struct{}),
+		stop:       make(chan struct{}),
+		done:       make(chan struct{}),
+	}
+	if b, err := build(); err == nil {
+		h.current = b
+	}
+	return h
+}
+// run is the hub's single goroutine: it recomputes the snapshot on a 1s tick (to
+// catch agent-reported actual-state changes) or on a desired-state wake, at most
+// once per event. It returns when Close is called.
+func (h *snapshotHub) run() {
+	defer close(h.done)
+	tick := time.NewTicker(time.Second)
+	defer tick.Stop()
+	for {
+		select {
+		case <-h.stop:
+			return
+		case <-h.wake:
+			h.recompute()
+		case <-tick.C:
+			h.recompute()
+		}
+	}
+}
+// recompute builds the snapshot once and, if the bytes changed, stores them and
+// fans them to all subscribers. Unchanged bytes are suppressed (no push), matching
+// the prior per-client diff behaviour.
+func (h *snapshotHub) recompute() {
+	b, err := h.build()
+	if err != nil {
+		return // transient store error: keep serving the last good snapshot
+	}
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	if bytes.Equal(b, h.current) {
+		return
+	}
+	h.current = b
+	for ch := range h.subs {
+		// Latest-wins: drain any stale pending snapshot, then send the newest.
+		// Both sends are non-blocking; because subscribers only ever receive
+		// (never send) and we hold h.mu, the post-drain buffer has room and the
+		// hub can never block on a slow client.
+		select {
+		case <-ch:
+		default:
+		}
+		select {
+		case ch <- b:
+		default:
+		}
+	}
+}
+// subscribe registers a client and immediately delivers the CURRENT snapshot so
+// a new connection gets initial state without doing its own marshal. It returns
+// the client's buffer-1 channel and an unsubscribe func.
+func (h *snapshotHub) subscribe() (<-chan []byte, func()) {
+	ch := make(chan []byte, 1)
+	h.mu.Lock()
+	if h.current != nil {
+		ch <- h.current // buffer-1 and empty ⇒ never blocks under the lock
+	}
+	h.subs[ch] = struct{}{}
+	h.mu.Unlock()
+	return ch, func() {
+		h.mu.Lock()
+		delete(h.subs, ch)
+		h.mu.Unlock()
+	}
+}
+// Close stops the hub goroutine, waits for it to exit, and releases the notifier
+// subscription. Idempotent — safe to call more than once (e.g. a shutdown path
+// plus a test cleanup).
+func (h *snapshotHub) Close() {
+	h.stopOnce.Do(func() {
+		close(h.stop)
+		<-h.done
+		h.notifUnsub()
+	})
+}

internal/server/api/snapshot_hub_test.go

@@ @@ -0,0 +1,153 @@ @@
+package api
+import (
+	"fmt"
+	"sync/atomic"
+	"testing"
+	"time"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+// waitFor polls cond until it is true or the deadline elapses.
+func waitFor(t *testing.T, d time.Duration, cond func() bool) {
+	t.Helper()
+	deadline := time.Now().Add(d)
+	for time.Now().Before(deadline) {
+		if cond() {
+			return
+		}
+		time.Sleep(time.Millisecond)
+	}
+	require.True(t, cond(), "condition not met within %s", d)
+}
+// TestSnapshotHubCloseIdempotent proves Close can be called more than once
+// without panicking (close-of-closed-channel) — a shutdown path plus a test
+// cleanup must both be safe.
+func TestSnapshotHubCloseIdempotent(t *testing.T) {
+	h := newSnapshotHub(func() ([]byte, error) { return []byte("x"), nil }, newNotifier())
+	go h.run()
+	h.Close()
+	h.Close() // must not panic
+}
+// TestSnapshotHubSingleBuildFanout proves the whole point of the hub: one
+// underlying snapshot build is fanned to every subscriber as identical bytes,
+// and a wake drives exactly one recompute regardless of how many clients are
+// attached (M clients must NOT cause M builds).
+func TestSnapshotHubSingleBuildFanout(t *testing.T) {
+	var builds int64
+	build := func() ([]byte, error) {
+		n := atomic.AddInt64(&builds, 1)
+		return []byte(fmt.Sprintf("snap-%d", n)), nil
+	}
+	notif := newNotifier()
+	h := newSnapshotHub(build, notif) // computes the initial snapshot (build #1)
+	require.Equal(t, int64(1), atomic.LoadInt64(&builds))
+	go h.run()
+	defer h.Close()
+	// Three subscribers each get the CURRENT snapshot immediately, without
+	// triggering their own build.
+	const n = 3
+	chans := make([]<-chan []byte, n)
+	for i := 0; i < n; i++ {
+		ch, unsub := h.subscribe()
+		defer unsub()
+		chans[i] = ch
+	}
+	for i, ch := range chans {
+		select {
+		case b := <-ch:
+			assert.Equal(t, "snap-1", string(b), "subscriber %d initial snapshot", i)
+		case <-time.After(time.Second):
+			t.Fatalf("subscriber %d got no initial snapshot", i)
+		}
+	}
+	// Still exactly one build despite three subscribers.
+	assert.Equal(t, int64(1), atomic.LoadInt64(&builds), "subscribing must not build")
+	// One wake ⇒ exactly one recompute, fanned identically to all three.
+	// (The 1s ticker cannot fire within this sub-second window, so builds is
+	// driven solely by the wake here.)
+	notif.notify()
+	waitFor(t, time.Second, func() bool { return atomic.LoadInt64(&builds) == 2 })
+	for i, ch := range chans {
+		select {
+		case b := <-ch:
+			assert.Equal(t, "snap-2", string(b), "subscriber %d wake snapshot", i)
+		case <-time.After(time.Second):
+			t.Fatalf("subscriber %d got no wake snapshot", i)
+		}
+	}
+	assert.Equal(t, int64(2), atomic.LoadInt64(&builds),
+		"one wake must drive exactly one build regardless of client count")
+}
+// TestSnapshotHubSuppressesUnchanged pins change-suppression: identical bytes
+// on recompute produce no push to subscribers.
+func TestSnapshotHubSuppressesUnchanged(t *testing.T) {
+	var builds int64
+	build := func() ([]byte, error) {
+		atomic.AddInt64(&builds, 1)
+		return []byte("constant"), nil // never changes
+	}
+	notif := newNotifier()
+	h := newSnapshotHub(build, notif)
+	go h.run()
+	defer h.Close()
+	ch, unsub := h.subscribe()
+	defer unsub()
+	// Drain the immediate initial delivery.
+	select {
+	case b := <-ch:
+		assert.Equal(t, "constant", string(b))
+	case <-time.After(time.Second):
+		t.Fatal("no initial snapshot")
+	}
+	notif.notify()
+	waitFor(t, time.Second, func() bool { return atomic.LoadInt64(&builds) >= 2 })
+	// Bytes are unchanged, so nothing new must be delivered.
+	select {
+	case b := <-ch:
+		t.Fatalf("unexpected push of unchanged snapshot: %q", b)
+	case <-time.After(100 * time.Millisecond):
+	}
+}
+// TestSnapshotHubLatestWins proves the fan-out is non-blocking / latest-wins: a
+// slow (never-draining) subscriber's channel holds only the newest snapshot and
+// the central loop never blocks on it.
+func TestSnapshotHubLatestWins(t *testing.T) {
+	var n int64
+	build := func() ([]byte, error) {
+		return []byte(fmt.Sprintf("v%d", atomic.AddInt64(&n, 1))), nil
+	}
+	notif := newNotifier()
+	h := newSnapshotHub(build, notif)
+	go h.run()
+	defer h.Close()
+	// Subscribe but never read: the buffer-1 channel already holds the initial
+	// snapshot. Further recomputes must overwrite it (drain-stale-then-send),
+	// never block the hub.
+	ch, unsub := h.subscribe()
+	defer unsub()
+	for i := 0; i < 5; i++ {
+		notif.notify()
+		waitFor(t, time.Second, func() bool { return atomic.LoadInt64(&n) >= int64(i+2) })
+	}
+	// The lone buffered value must be the LATEST, not a stale early one.
+	last := fmt.Sprintf("v%d", atomic.LoadInt64(&n))
+	select {
+	case b := <-ch:
+		assert.Equal(t, last, string(b), "slow subscriber must hold the newest snapshot")
+	case <-time.After(time.Second):
+		t.Fatal("slow subscriber holds nothing")
+	}
+}

internal/server/api/sshcert.go

Old		New
@@ -105,7 +105,7 @@ func (a *API) SetSSHCAAuthorizedKey(line string) { a.sshCAKey = line }
105	// 404s when the jump gate is off.	105	// 404s when the jump gate is off.
106	func (a API) handleSSHCA(w http.ResponseWriter, r http.Request) {	106	func (a API) handleSSHCA(w http.ResponseWriter, r http.Request) {
107	if a.sshCAKey == "" {	107	if a.sshCAKey == "" {
108	httpError(w, "ssh jump gate not enabled", http.StatusNotFound)	108	http.Error(w, "ssh jump gate not enabled", http.StatusNotFound)
109	return	109	return
110	}	110	}
111	writeJSON(w, http.StatusOK, types.SSHCAResponse{CA: a.sshCAKey})	111	writeJSON(w, http.StatusOK, types.SSHCAResponse{CA: a.sshCAKey})
@@ -142,7 +142,7 @@ func (m *HostMinter) MintHostCert(principal string) (keyPEM, cert string, err er
142	// gate is off (no CA wired).	142	// gate is off (no CA wired).
143	func (a API) handleMintSSHCert(w http.ResponseWriter, r http.Request) {	143	func (a API) handleMintSSHCert(w http.ResponseWriter, r http.Request) {
144	if a.certs == nil {	144	if a.certs == nil {
145	httpError(w, "ssh jump gate not enabled", http.StatusNotFound)	145	http.Error(w, "ssh jump gate not enabled", http.StatusNotFound)
146	return	146	return
147	}	147	}
148	var req types.SSHCertRequest	148	var req types.SSHCertRequest
@@ -151,12 +151,12 @@ func (a API) handleMintSSHCert(w http.ResponseWriter, r http.Request) {
151	}	151	}
152	pub, _, _, _, err := ssh.ParseAuthorizedKey([]byte(req.PublicKey))	152	pub, _, _, _, err := ssh.ParseAuthorizedKey([]byte(req.PublicKey))
153	if err != nil {	153	if err != nil {
154	httpError(w, "invalid public_key", http.StatusBadRequest)	154	http.Error(w, "invalid public_key", http.StatusBadRequest)
155	return	155	return
156	}	156	}
157	cert, err := a.certs.Mint(pub)	157	cert, err := a.certs.Mint(pub)
158	if err != nil {	158	if err != nil {
159	httpError(w, "internal error", http.StatusInternalServerError)	159	http.Error(w, "internal error", http.StatusInternalServerError)
160	return	160	return
161	}	161	}
162	a.audit("ssh-cert.mint", map[string]string{	162	a.audit("ssh-cert.mint", map[string]string{
@@ -189,24 +189,24 @@ func (a API) handleRevokeSSHCert(w http.ResponseWriter, r http.Request) {
189	// signature is checked. A non-cert key line is a clear 400.	189	// signature is checked. A non-cert key line is a clear 400.
190	pk, _, _, _, err := ssh.ParseAuthorizedKey([]byte(req.Certificate))	190	pk, _, _, _, err := ssh.ParseAuthorizedKey([]byte(req.Certificate))
191	if err != nil {	191	if err != nil {
192	httpError(w, "invalid certificate", http.StatusBadRequest)	192	http.Error(w, "invalid certificate", http.StatusBadRequest)
193	return	193	return
194	}	194	}
195	cert, ok := pk.(*ssh.Certificate)	195	cert, ok := pk.(*ssh.Certificate)
196	if !ok {	196	if !ok {
197	httpError(w, "not a certificate", http.StatusBadRequest)	197	http.Error(w, "not a certificate", http.StatusBadRequest)
198	return	198	return
199	}	199	}
200	serial = cert.Serial	200	serial = cert.Serial
201	case req.Serial != nil:	201	case req.Serial != nil:
202	serial = *req.Serial	202	serial = *req.Serial
203	default:	203	default:
204	httpError(w, "serial or certificate required", http.StatusBadRequest)	204	http.Error(w, "serial or certificate required", http.StatusBadRequest)
205	return	205	return
206	}	206	}
207		207
208	if err := a.st.RevokeSSHCert(serial, req.Reason); err != nil {	208	if err := a.st.RevokeSSHCert(serial, req.Reason); err != nil {
209	httpError(w, "internal error", http.StatusInternalServerError)	209	http.Error(w, "internal error", http.StatusInternalServerError)
210	return	210	return
211	}	211	}
212	a.audit("ssh-cert.revoke", map[string]string{	212	a.audit("ssh-cert.revoke", map[string]string{
@@ -222,7 +222,7 @@ func (a API) handleRevokeSSHCert(w http.ResponseWriter, r http.Request) {
222	func (a API) handleListRevokedSSHCerts(w http.ResponseWriter, r http.Request) {	222	func (a API) handleListRevokedSSHCerts(w http.ResponseWriter, r http.Request) {
223	revoked, err := a.st.ListRevokedSSHCerts()	223	revoked, err := a.st.ListRevokedSSHCerts()
224	if err != nil {	224	if err != nil {
225	httpError(w, "internal error", http.StatusInternalServerError)	225	http.Error(w, "internal error", http.StatusInternalServerError)
226	return	226	return
227	}	227	}
228	out := make([]types.RevokedCert, len(revoked))	228	out := make([]types.RevokedCert, len(revoked))

internal/server/api/testdata/host.golden.json

Old		New
@@ -8,6 +8,10 @@
8	"status": "active",	8	"status": "active",
9	"enrolled_at": "2026-07-27T12:00:00Z",	9	"enrolled_at": "2026-07-27T12:00:00Z",
10	"online": true,	10	"online": true,
		11	"last_seen": null,
		12	"seconds_since_last_seen": null,
		13	"stale": false,
		14	"sessions": 0,
11	"capacity": {	15	"capacity": {
12	"vcpus": 16,	16	"vcpus": 16,
13	"mem_mb": 32768,	17	"mem_mb": 32768,

internal/server/api/testdata/snapshot.golden.json

Old		New
@@ -10,6 +10,10 @@
10	"status": "active",	10	"status": "active",
11	"enrolled_at": "2026-07-27T12:00:00Z",	11	"enrolled_at": "2026-07-27T12:00:00Z",
12	"online": true,	12	"online": true,
		13	"last_seen": null,
		14	"seconds_since_last_seen": null,
		15	"stale": false,
		16	"sessions": 0,
13	"capacity": {	17	"capacity": {
14	"vcpus": 16,	18	"vcpus": 16,
15	"mem_mb": 32768,	19	"mem_mb": 32768,

internal/server/api/ticket.go

Old		New
@@ -4,7 +4,7 @@ import (
4	"sync"	4	"sync"
5	"time"	5	"time"
6		6
7	"github.com/a73x/eitri/internal/server/store"	7	"github.com/a73x/eitri/internal/random"
8	)	8	)
9		9
10	// streamTicketTTL bounds how long a minted stream ticket stays redeemable.	10	// streamTicketTTL bounds how long a minted stream ticket stays redeemable.
@@ -40,7 +40,7 @@ func (t *ticketStore) mint() string {
40	delete(t.tickets, k)	40	delete(t.tickets, k)
41	}	41	}
42	}	42	}
43	tick := store.RandHex(16)	43	tick := random.Hex(16)
44	t.tickets[tick] = now.Add(streamTicketTTL)	44	t.tickets[tick] = now.Add(streamTicketTTL)
45	return tick	45	return tick
46	}	46	}

internal/server/api/types/types.go

Old		New
@@ -32,8 +32,17 @@ type Host struct {
32	Status string `json:"status"`	32	Status string `json:"status"`
33	EnrolledAt time.Time `json:"enrolled_at"`	33	EnrolledAt time.Time `json:"enrolled_at"`
34	Online bool `json:"online"`	34	Online bool `json:"online"`
35	Capacity Capacity `json:"capacity"` // host TOTALS (when online)	35	// Sync-health signals so degradation is visible at a glance. LastSeen
36	Allocated Capacity `json:"allocated"` // committed to live VMs (server-computed)	36	// and SecondsSinceLastSeen are nil until the host has reported at least
		37	// once (why clients can see last_seen: null). Stale trips before Online
		38	// clears — the early "sync is degrading" warning. Sessions is the agent
		39	// (re)connect count: rising while Online means the link is flapping.
		40	LastSeen *time.Time `json:"last_seen"`
		41	SecondsSinceLastSeen *int64 `json:"seconds_since_last_seen"`
		42	Stale bool `json:"stale"`
		43	Sessions int `json:"sessions"`
		44	Capacity Capacity `json:"capacity"` // host TOTALS (when online)
		45	Allocated Capacity `json:"allocated"` // committed to live VMs (server-computed)
37	}	46	}
38		47
39	// VM is the explicit snake_case wire representation of a VM, served by	48	// VM is the explicit snake_case wire representation of a VM, served by

internal/server/config/config.go

@@ @@ -0,0 +1,48 @@ @@
+// Package config defines the eitri-server on-disk JSON configuration schema.
+// It is shared between the server binary (cmd/eitri-server) and the
+// integration harness (internal/integration/harness), which renders a config
+// file for the server subprocess it launches — sharing the type makes the
+// compiler, not a comment, keep the two in agreement.
+package config
+// Config is the eitri-server config file schema (decoded from JSON).
+type Config struct {
+	HTTPListen      string `json:"http_listen"`
+	QUICListen      string `json:"quic_listen"`
+	DBPath          string `json:"db_path"`
+	AdminToken      string `json:"admin_token"`
+	HostSecret      string `json:"host_secret"`
+	CIDRPool        string `json:"cidr_pool"`
+	DefaultImageURL string `json:"default_image_url"`
+	DefaultImageSHA string `json:"default_image_sha256"`
+	AdvertiseHTTP   string `json:"advertise_http"`
+	AdvertiseQUIC   string `json:"advertise_quic"`
+	// CredentialMaxAge optionally bounds host credential age (Go duration,
+	// e.g. "2160h" for 90 days). Empty/zero disables — revocation via
+	// POST /api/v1/hosts/{id}/revoke-credential is the primary mechanism;
+	// max-age forces periodic re-enrollment and is opt-in defense-in-depth.
+	CredentialMaxAge string `json:"credential_max_age"`
+	// AuditRetention bounds the audit_log age (Go duration; default "2160h" =
+	// 90 days; "0" disables pruning). Pruned at startup and daily.
+	AuditRetention string `json:"audit_retention"`
+	// SSHCAKey is the path to the persistent SSH user CA private key
+	// (auto-created 0600 if absent, handled like AdminToken — never logged).
+	// The CA signs the short-lived certs the jump gate accepts.
+	SSHCAKey string `json:"ssh_ca_key"`
+	// SSHHostKey is the path to the gate's persistent SSH host key
+	// (auto-created 0600 if absent, never regenerated on restart so users
+	// don't see host-key-changed warnings).
+	SSHHostKey string `json:"ssh_host_key"`
+	// SSHListen is the jump-gate listen address. Empty ⇒ gate is OFF (no
+	// key material is loaded and no listener is started).
+	SSHListen string `json:"ssh_listen"`
+	// SSHGateDomain is the hostname clients dial the gate as (the principal put
+	// on the gate's signed HOST certificate). Empty ⇒ derived from SSHListen's
+	// host part; if that is also empty (e.g. ":2222") it falls back to
+	// "localhost". It must match the host in EITRI_GATE so `@cert-authority`
+	// verification accepts the presented host cert.
+	SSHGateDomain string `json:"ssh_gate_domain"`
+	// SSHCertTTL bounds minted user-cert validity (Go duration; default 10m).
+	// Set server-side; client-requested validity is never honored.
+	SSHCertTTL string `json:"ssh_cert_ttl"`
+}

internal/server/registry/registry.go

Old		New
@@ -9,8 +9,21 @@ import (
9	"time"	9	"time"
10	)	10	)
11		11
		12	// OnlineWindow is how long a host may go without a report before it is marked
		13	// offline. It is COUPLED to the agent's report cadence
		14	// (syncclient.DefaultTickInterval, 10s): the invariant
		15	// OnlineWindow >= 3*DefaultTickInterval keeps a healthy host from flapping
		16	// Online/Offline on a couple of jittered/missed reports. Lowering this (or
		17	// raising the agent tick) without preserving that margin re-introduces flapping;
		18	// the two live in different packages, so an invariant test in syncclient guards
		19	// the relationship.
12	const OnlineWindow = 30 * time.Second	20	const OnlineWindow = 30 * time.Second
13		21
		22	// StaleWindow is the "degrading" threshold: a host whose last report is older
		23	// than this — but still within OnlineWindow — is Online yet Stale, the early
		24	// warning that sync is flapping or falling behind before it drops fully offline.
		25	const StaleWindow = OnlineWindow / 2
		26
14	type Capacity struct{ VCPUs, MemMB, DiskGB int64 }	27	type Capacity struct{ VCPUs, MemMB, DiskGB int64 }
15		28
16	type ActualVM struct {	29	type ActualVM struct {
@@ -34,7 +47,19 @@ type Report struct {
34	type HostState struct {	47	type HostState struct {
35	Report	48	Report
36	LastSeen time.Time	49	LastSeen time.Time
37	Online bool	50	// Sessions counts agent (re)connects since server start. A steadily rising
		51	// value with a live LastSeen means the agent is churning/flapping its QUIC
		52	// session even though it looks online; a flat value means a stable link.
		53	Sessions int
		54	// The following are derived on each Get from LastSeen and the clock; they
		55	// are not stored.
		56	Online bool
		57	// Stale trips before Online clears (last report older than StaleWindow): the
		58	// at-a-glance "sync is degrading" signal.
		59	Stale bool
		60	// SinceLastSeen is the age of the last report at read time. Zero when the
		61	// host has never reported (LastSeen unset).
		62	SinceLastSeen time.Duration
38	}	63	}
39		64
40	type Registry struct {	65	type Registry struct {
@@ -50,7 +75,21 @@ func New(now func() time.Time) *Registry {
50	func (r *Registry) UpdateReport(hostID string, rep Report) {	75	func (r *Registry) UpdateReport(hostID string, rep Report) {
51	r.mu.Lock()	76	r.mu.Lock()
52	defer r.mu.Unlock()	77	defer r.mu.Unlock()
53	r.m[hostID] = HostState{Report: rep, LastSeen: r.now()}	78	// Preserve the session counter across reports: UpdateReport replaces the
		79	// whole HostState, and Sessions is owned by RecordConnect, not the report.
		80	sessions := r.m[hostID].Sessions
		81	r.m[hostID] = HostState{Report: rep, LastSeen: r.now(), Sessions: sessions}
		82	}
		83
		84	// RecordConnect increments the host's session counter, marking one agent
		85	// (re)connect. It preserves any existing report/LastSeen so a reconnect that
		86	// arrives before the first fresh report does not blank live state.
		87	func (r *Registry) RecordConnect(hostID string) {
		88	r.mu.Lock()
		89	defer r.mu.Unlock()
		90	st := r.m[hostID]
		91	st.Sessions++
		92	r.m[hostID] = st
54	}	93	}
55		94
56	func (r *Registry) Get(hostID string) (HostState, bool) {	95	func (r *Registry) Get(hostID string) (HostState, bool) {
@@ -60,7 +99,15 @@ func (r *Registry) Get(hostID string) (HostState, bool) {
60	if !ok {	99	if !ok {
61	return st, false	100	return st, false
62	}	101	}
63	st.Online = r.now().Sub(st.LastSeen) < OnlineWindow	102	// Derive liveness from the last report's age. A host that has connected but
		103	// never reported (LastSeen unset) is neither online nor meaningfully "stale
		104	// for N seconds", so leave SinceLastSeen zero and report it offline.
		105	if !st.LastSeen.IsZero() {
		106	elapsed := r.now().Sub(st.LastSeen)
		107	st.SinceLastSeen = elapsed
		108	st.Online = elapsed < OnlineWindow
		109	st.Stale = elapsed >= StaleWindow
		110	}
64	// Deep-copy slices so callers cannot corrupt registry state.	111	// Deep-copy slices so callers cannot corrupt registry state.
65	st.Report.VMs = slices.Clone(st.Report.VMs)	112	st.Report.VMs = slices.Clone(st.Report.VMs)
66	quarantined := slices.Clone(st.Report.Quarantined)	113	quarantined := slices.Clone(st.Report.Quarantined)

internal/server/registry/registry_test.go

@@ @@ -3,6 +3,7 @@ package registry @@
+import (
+	"testing"
+	"time"
+	"github.com/stretchr/testify/assert"
@@ @@ -43,3 +44,65 @@ func TestUnknownHostNotFound(t *testing.T) { @@
+	_, ok := r.Get("nope")
+	assert.False(t, ok)
+// TestStalenessDerivation pins the online/stale/age signals derived from the age
+// of the last report: fresh is online and not stale; past half the window it is
+// still online but stale (the early "degrading" warning); past the full window
+// it is offline (and remains stale).
+func TestStalenessDerivation(t *testing.T) {
+	cases := []struct {
+		name       string
+		age        time.Duration
+		wantOnline bool
+		wantStale  bool
+	}{
+		{"fresh", 0, true, false},
+		{"just under stale", StaleWindow - time.Second, true, false},
+		{"at stale window", StaleWindow, true, true},
+		{"online but stale", OnlineWindow - time.Second, true, true},
+		{"just offline", OnlineWindow, false, true},
+		{"long offline", 5 * OnlineWindow, false, true},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			now := time.Now()
+			r := New(func() time.Time { return now })
+			r.UpdateReport("h1", Report{})
+			now = now.Add(tc.age)
+			st, ok := r.Get("h1")
+			assert.True(t, ok)
+			assert.Equal(t, tc.wantOnline, st.Online, "online")
+			assert.Equal(t, tc.wantStale, st.Stale, "stale")
+			assert.Equal(t, tc.age, st.SinceLastSeen, "age since last seen")
+		})
+	}
+}
+// TestSessionsCountAndReportPreservation pins that RecordConnect counts agent
+// (re)connects, that a report preserves the counter (flapping stays visible),
+// and that a connect before any report leaves the host offline with no age.
+func TestSessionsCountAndReportPreservation(t *testing.T) {
+	now := time.Now()
+	r := New(func() time.Time { return now })
+	// Connect before any report: counted, but offline with no last-seen age.
+	r.RecordConnect("h1")
+	st, ok := r.Get("h1")
+	assert.True(t, ok)
+	assert.Equal(t, 1, st.Sessions)
+	assert.False(t, st.Online)
+	assert.True(t, st.LastSeen.IsZero())
+	assert.Zero(t, st.SinceLastSeen)
+	// A report does not reset the session counter.
+	r.UpdateReport("h1", Report{Capacity: Capacity{VCPUs: 4}})
+	st, _ = r.Get("h1")
+	assert.Equal(t, 1, st.Sessions, "report must preserve session count")
+	assert.True(t, st.Online)
+	// A reconnect increments and keeps the live report.
+	r.RecordConnect("h1")
+	st, _ = r.Get("h1")
+	assert.Equal(t, 2, st.Sessions)
+	assert.Equal(t, int64(4), st.Capacity.VCPUs, "reconnect must not blank live state")
+}

internal/server/sshgate/gate.go

Old		New
@@ -17,6 +17,7 @@ import (
17	"log/slog"	17	"log/slog"
18	"net"	18	"net"
19	"strings"	19	"strings"
		20	"time"
20		21
21	"golang.org/x/crypto/ssh"	22	"golang.org/x/crypto/ssh"
22	)	23	)
@@ -135,13 +136,24 @@ func (g *Gate) Serve(l net.Listener) error {
135	}	136	}
136	}	137	}
137		138
		139	// handshakeGrace bounds the unauthenticated SSH handshake, mirroring sshd's
		140	// LoginGraceTime. Without it a client that connects and then stalls (no or
		141	// partial banner) parks a goroutine + fd indefinitely, and enough such
		142	// connections starve the bastion of legitimate admin logins. A var (not const)
		143	// so tests can shrink it; not part of the public API.
		144	var handshakeGrace = 30 * time.Second
		145
138	// handleConn runs the SSH handshake and dispatches channels for one connection.	146	// handleConn runs the SSH handshake and dispatches channels for one connection.
139	func (g *Gate) handleConn(nConn net.Conn) {	147	func (g *Gate) handleConn(nConn net.Conn) {
140	defer nConn.Close()	148	defer nConn.Close()
		149	// Deadline covers only the pre-auth handshake; cleared once it completes so
		150	// it never applies to the long-lived tunnel that follows.
		151	_ = nConn.SetDeadline(time.Now().Add(handshakeGrace))
141	sConn, chans, reqs, err := ssh.NewServerConn(nConn, g.cfg)	152	sConn, chans, reqs, err := ssh.NewServerConn(nConn, g.cfg)
142	if err != nil {	153	if err != nil {
143	return // handshake or auth failure — nothing to serve	154	return // handshake, auth failure, or grace timeout — nothing to serve
144	}	155	}
		156	_ = nConn.SetDeadline(time.Time{})
145	defer sConn.Close()	157	defer sConn.Close()
146		158
147	// Refuse EVERY out-of-band global request. tcpip-forward (`ssh -R`) would turn	159	// Refuse EVERY out-of-band global request. tcpip-forward (`ssh -R`) would turn

internal/server/sshgate/gate_test.go

@@ @@ -296,6 +296,33 @@ func TestGateDirectTCPIPToPort22RoundTrips(t *testing.T) { @@
+	assert.Equal(t, want, got)
+// TestGateHandshakeGraceDropsStalledConn pins the pre-auth DoS guard: a client
+// that connects and then never completes the SSH handshake must be dropped by
+// the handshake grace, not parked forever holding a goroutine + fd.
+func TestGateHandshakeGraceDropsStalledConn(t *testing.T) {
+	orig := handshakeGrace
+	handshakeGrace = 150 * time.Millisecond
+	t.Cleanup(func() { handshakeGrace = orig })
+	ca := newSigner(t)
+	tg := startGate(t, ca.PublicKey(), true)
+	conn, err := net.Dial("tcp", tg.addr)
+	require.NoError(t, err)
+	defer conn.Close()
+	// Never send a client identification string. The gate emits its banner then
+	// blocks reading ours; the grace deadline must fire and close the connection,
+	// so our read drains the banner and hits a clean EOF well before this generous
+	// client deadline. Without the grace the server would block forever and we'd
+	// instead trip our own read timeout.
+	start := time.Now()
+	require.NoError(t, conn.SetReadDeadline(time.Now().Add(5*time.Second)))
+	_, err = io.ReadAll(conn)
+	require.NoError(t, err, "server must close the stalled conn (EOF), not leave us to time out")
+	assert.Less(t, time.Since(start), 3*time.Second, "stalled conn must drop near the handshake grace")
+}
+func TestGateDirectTCPIPToNonSSHPortRejected(t *testing.T) {
+	ca := newSigner(t)
+	tg := startGate(t, ca.PublicKey(), true)

internal/server/store/allocation_test.go

Old		New
@@ -3,6 +3,7 @@ package store
3	import (	3	import (
4	"testing"	4	"testing"
5		5
		6	"github.com/a73x/eitri/internal/random"
6	"github.com/stretchr/testify/assert"	7	"github.com/stretchr/testify/assert"
7	"github.com/stretchr/testify/require"	8	"github.com/stretchr/testify/require"
8	)	9	)
@@ -10,7 +11,7 @@ import (
10	func vmWithResources(t testing.T, s Store, h Host, name string, vcpus, mem, disk int64) VM {	11	func vmWithResources(t testing.T, s Store, h Host, name string, vcpus, mem, disk int64) VM {
11	t.Helper()	12	t.Helper()
12	vm := VM{	13	vm := VM{
13	ID: RandHex(8), HostID: h.ID, Name: name,	14	ID: random.Hex(8), HostID: h.ID, Name: name,
14	ImageURL: "http://img", ImageSHA256: "abc",	15	ImageURL: "http://img", ImageSHA256: "abc",
15	VCPUs: vcpus, MemMB: mem, DiskGB: disk, PowerState: "running",	16	VCPUs: vcpus, MemMB: mem, DiskGB: disk, PowerState: "running",
16	}	17	}
@@ -59,7 +60,7 @@ func TestAllocatedByHostSeparatesHosts(t *testing.T) {
59		60
60	func (s Store) mustAllocated(t testing.T) map[string]Alloc {	61	func (s Store) mustAllocated(t testing.T) map[string]Alloc {
61	t.Helper()	62	t.Helper()
62	a, err := s.AllocatedByHost()	63	_, alloc, _, err := s.Snapshot()
63	require.NoError(t, err)	64	require.NoError(t, err)
64	return a	65	return alloc
65	}	66	}

internal/server/store/decommission_test.go

Old		New
@@ -3,6 +3,7 @@ package store
3	import (	3	import (
4	"testing"	4	"testing"
5		5
		6	"github.com/a73x/eitri/internal/random"
6	"github.com/stretchr/testify/assert"	7	"github.com/stretchr/testify/assert"
7	"github.com/stretchr/testify/require"	8	"github.com/stretchr/testify/require"
8	)	9	)
@@ -10,7 +11,7 @@ import (
10	func makeVM(t testing.T, s Store, host Host, name string) VM {	11	func makeVM(t testing.T, s Store, host Host, name string) VM {
11	t.Helper()	12	t.Helper()
12	vm := VM{	13	vm := VM{
13	ID: RandHex(8), HostID: host.ID, Name: name,	14	ID: random.Hex(8), HostID: host.ID, Name: name,
14	ImageURL: "http://img", ImageSHA256: "abc",	15	ImageURL: "http://img", ImageSHA256: "abc",
15	VCPUs: 1, MemMB: 512, DiskGB: 5, PowerState: "running",	16	VCPUs: 1, MemMB: 512, DiskGB: 5, PowerState: "running",
16	}	17	}
@@ -79,21 +80,27 @@ func TestRemoveHostRefusesWhileVMsRemain(t *testing.T) {
79	assert.Error(t, err, "RemoveHost must refuse while VM rows remain")	80	assert.Error(t, err, "RemoveHost must refuse while VM rows remain")
80	}	81	}
81		82
82	func TestHostVMCount(t *testing.T) {	83	// TestRemoveHostDrainSequence pins the same VM-row-counting behavior the
		84	// deleted HostVMCount used to expose directly, but through RemoveHost (its
		85	// only production caller): decommission refuses to finalize while a VM row
		86	// remains — live or merely tombstoned-but-not-yet-reaped — and succeeds the
		87	// instant the row is hard-deleted.
		88	func TestRemoveHostDrainSequence(t *testing.T) {
83	s := newStore(t)	89	s := newStore(t)
84	h := enrollHost(t, s)	90	h := enrollHost(t, s)
85	assert.Equal(t, 0, s.mustHostVMCount(t, h.ID))	91	require.NoError(t, s.DecommissionHost(h.ID))
86	vm := makeVM(t, s, h, "vm-a")	92	require.NoError(t, s.RemoveHost(h.ID), "no VMs: RemoveHost should succeed immediately")
87	assert.Equal(t, 1, s.mustHostVMCount(t, h.ID))
88	require.NoError(t, s.TombstoneVM(vm.ID))
89	assert.Equal(t, 1, s.mustHostVMCount(t, h.ID), "tombstoned but not reaped still counts")
90	require.NoError(t, s.HardDeleteVM(vm.ID))
91	assert.Equal(t, 0, s.mustHostVMCount(t, h.ID))
92	}
93		93
94	func (s Store) mustHostVMCount(t testing.T, id string) int {	94	s2 := newStore(t)
95	t.Helper()	95	h2 := enrollHost(t, s2)
96	n, err := s.HostVMCount(id)	96	vm := makeVM(t, s2, h2, "vm-a")
97	require.NoError(t, err)	97	assert.Error(t, s2.RemoveHost(h2.ID), "live VM row: RemoveHost must refuse")
98	return n	98
		99	// DecommissionHost tombstones the host's VMs but leaves the rows in place
		100	// until the agent acks their destroy — still not drained.
		101	require.NoError(t, s2.DecommissionHost(h2.ID))
		102	assert.Error(t, s2.RemoveHost(h2.ID), "tombstoned but not reaped: RemoveHost must still refuse")
		103
		104	require.NoError(t, s2.HardDeleteVM(vm.ID))
		105	require.NoError(t, s2.RemoveHost(h2.ID), "reaped: RemoveHost should now succeed")
99	}	106	}

internal/server/store/store.go

Old		New
@@ -6,7 +6,6 @@ package store
6		6
7	import (	7	import (
8	"context"	8	"context"
9	"crypto/rand"
10	"crypto/sha256"	9	"crypto/sha256"
11	"database/sql"	10	"database/sql"
12	"encoding/hex"	11	"encoding/hex"
@@ -20,6 +19,7 @@ import (
20	"strings"	19	"strings"
21	"time"	20	"time"
22		21
		22	"github.com/a73x/eitri/internal/random"
23	"github.com/a73x/eitri/internal/transport"	23	"github.com/a73x/eitri/internal/transport"
24	_ "modernc.org/sqlite"	24	_ "modernc.org/sqlite"
25	)	25	)
@@ -30,13 +30,9 @@ var ErrNameTaken = errors.New("vm name already in use")
30	// ErrHostNotFound is returned by CreateVM when the host_id does not exist.	30	// ErrHostNotFound is returned by CreateVM when the host_id does not exist.
31	var ErrHostNotFound = errors.New("host not found")	31	var ErrHostNotFound = errors.New("host not found")
32		32
33	// RandHex returns n cryptographically-random bytes encoded as hex. rand.Read is	33	// ErrHostNotEnrolled is returned by CreateVM when the target host exists but is
34	// documented never to fail (Go 1.24+), so its error is intentionally ignored.	34	// not accepting new VMs (e.g. it is decommissioning).
35	func RandHex(n int) string {	35	var ErrHostNotEnrolled = errors.New("host not accepting new VMs")
36	b := make([]byte, n)
37	rand.Read(b) //nolint:errcheck // crypto/rand.Read never returns an error
38	return hex.EncodeToString(b)
39	}
40		36
41	type Store struct {	37	type Store struct {
42	db *sql.DB	38	db *sql.DB
@@ -64,8 +60,9 @@ type VM struct {
64	// never returned in vmResponse and never logged. SSHHostCert is the matching	60	// never returned in vmResponse and never logged. SSHHostCert is the matching
65	// CA-signed host cert (authorized_keys form); public, but grouped here.	61	// CA-signed host cert (authorized_keys form); public, but grouped here.
66	SSHHostKey, SSHHostCert string	62	SSHHostKey, SSHHostCert string
67	CreatedAt time.Time	63	// The VM's reachable address is AssignedIP (the agent-reported bridge IP).
68	DeletedAt *time.Time	64	CreatedAt time.Time
		65	DeletedAt *time.Time
69	}	66	}
70		67
71	const schema = `	68	const schema = `
@@ -179,6 +176,10 @@ func (s *Store) Close() error { return s.db.Close() }
179	// the readiness probe; the context bounds a wedged driver.	176	// the readiness probe; the context bounds a wedged driver.
180	func (s *Store) Ping(ctx context.Context) error { return s.db.PingContext(ctx) }	177	func (s *Store) Ping(ctx context.Context) error { return s.db.PingContext(ctx) }
181		178
		179	// Epoch reads the current epoch value directly. Production callers get the
		180	// epoch via DesiredForHost (paired with a matching desired-VM read in the
		181	// same tx); this exists as a test/observability hook for asserting exactly
		182	// which mutations bump the epoch.
182	func (s *Store) Epoch() (uint64, error) {	183	func (s *Store) Epoch() (uint64, error) {
183	var v uint64	184	var v uint64
184	err := s.db.QueryRow(`SELECT CAST(value AS INTEGER) FROM meta WHERE key='epoch'`).Scan(&v)	185	err := s.db.QueryRow(`SELECT CAST(value AS INTEGER) FROM meta WHERE key='epoch'`).Scan(&v)
@@ -210,7 +211,7 @@ func subnetForIndex(pool netip.Prefix, idx int64) (string, error) {
210	}	211	}
211		212
212	func (s *Store) CreateEnrollmentToken() (string, error) {	213	func (s *Store) CreateEnrollmentToken() (string, error) {
213	tok := RandHex(32)	214	tok := random.Hex(32)
214	h := sha256.Sum256([]byte(tok))	215	h := sha256.Sum256([]byte(tok))
215	hash := hex.EncodeToString(h[:])	216	hash := hex.EncodeToString(h[:])
216	expiresAt := time.Now().UTC().Add(15 * time.Minute)	217	expiresAt := time.Now().UTC().Add(15 * time.Minute)
@@ -284,7 +285,7 @@ func (s *Store) RedeemEnrollmentToken(tok, name, osName, arch, provisioner, remo
284	return Host{}, fmt.Errorf("read freed_cidrs: %w", err)	285	return Host{}, fmt.Errorf("read freed_cidrs: %w", err)
285	}	286	}
286		287
287	id := RandHex(16)	288	id := random.Hex(16)
288		289
289	if _, err := tx.Exec(	290	if _, err := tx.Exec(
290	`INSERT INTO hosts(id, name, os, arch, provisioner, bridge_cidr, enrolled_at) VALUES (?,?,?,?,?,?,?)`,	291	`INSERT INTO hosts(id, name, os, arch, provisioner, bridge_cidr, enrolled_at) VALUES (?,?,?,?,?,?,?)`,
@@ -296,10 +297,9 @@ func (s *Store) RedeemEnrollmentToken(tok, name, osName, arch, provisioner, remo
296	// Audit in the SAME tx: the durable record of who enrolled cannot be lost	297	// Audit in the SAME tx: the durable record of who enrolled cannot be lost
297	// once the enrollment itself commits. The token appears only as a hash	298	// once the enrollment itself commits. The token appears only as a hash
298	// prefix (matches the enrollment_tokens.token_hash the mint row logs).	299	// prefix (matches the enrollment_tokens.token_hash the mint row logs).
299	tokSum := sha256.Sum256([]byte(tok))
300	detail, _ := json.Marshal(map[string]string{	300	detail, _ := json.Marshal(map[string]string{
301	"host_id": id, "name": name, "os": osName, "arch": arch,	301	"host_id": id, "name": name, "os": osName, "arch": arch,
302	"remote": remote, "token_hash_prefix": hex.EncodeToString(tokSum[:])[:8],	302	"remote": remote, "token_hash_prefix": hash[:8],
303	})	303	})
304	if _, err := tx.Exec(`INSERT INTO audit_log(at, action, detail) VALUES (?, ?, ?)`,	304	if _, err := tx.Exec(`INSERT INTO audit_log(at, action, detail) VALUES (?, ?, ?)`,
305	now.Format(time.RFC3339), "host.enroll", string(detail)); err != nil {	305	now.Format(time.RFC3339), "host.enroll", string(detail)); err != nil {
@@ -399,7 +399,19 @@ func (s *Store) CreateVM(vm VM) error {
399		399
400	now := time.Now().UTC()	400	now := time.Now().UTC()
401	if vm.ID == "" {	401	if vm.ID == "" {
402	vm.ID = RandHex(16)	402	vm.ID = random.Hex(16)
		403	}
		404
		405	// Refuse to place a VM on a host that is not enrolled (e.g. mid-decommission);
		406	// the FK below only proves the host row exists, not that it accepts new VMs.
		407	var hostStatus string
		408	switch err := tx.QueryRow(`SELECT status FROM hosts WHERE id=?`, vm.HostID).Scan(&hostStatus); {
		409	case errors.Is(err, sql.ErrNoRows):
		410	return ErrHostNotFound
		411	case err != nil:
		412	return fmt.Errorf("lookup host status: %w", err)
		413	case hostStatus != "enrolled":
		414	return ErrHostNotEnrolled
403	}	415	}
404		416
405	_, err = tx.Exec(	417	_, err = tx.Exec(
@@ -476,24 +488,7 @@ func (s *Store) RestoreVM(id string) error {
476	// sql.ErrNoRows otherwise) and bumps the epoch. Called after the agent acks the	488	// sql.ErrNoRows otherwise) and bumps the epoch. Called after the agent acks the
477	// destroy.	489	// destroy.
478	func (s *Store) HardDeleteVM(id string) error {	490	func (s *Store) HardDeleteVM(id string) error {
479	tx, err := s.db.Begin()	491	return s.mutate(`DELETE FROM vms WHERE id=? AND deleted_at IS NOT NULL`, id)
480	if err != nil {
481	return err
482	}
483	defer tx.Rollback()
484
485	res, err := tx.Exec(`DELETE FROM vms WHERE id=? AND deleted_at IS NOT NULL`, id)
486	if err != nil {
487	return fmt.Errorf("delete vm: %w", err)
488	}
489	if n, _ := res.RowsAffected(); n == 0 {
490	return sql.ErrNoRows
491	}
492
493	if err := bumpEpoch(tx); err != nil {
494	return err
495	}
496	return tx.Commit()
497	}	492	}
498		493
499	// DecommissionHost marks a host as decommissioning and tombstones all its live	494	// DecommissionHost marks a host as decommissioning and tombstones all its live
@@ -528,10 +523,11 @@ func (s *Store) DecommissionHost(id string) error {
528	// Alloc is the sum of resources committed to live VMs on a host.	523	// Alloc is the sum of resources committed to live VMs on a host.
529	type Alloc struct{ VCPUs, MemMB, DiskGB int64 }	524	type Alloc struct{ VCPUs, MemMB, DiskGB int64 }
530		525
531	// AllocatedByHost returns, per host, the resources allocated to its live	526	// allocatedByHost returns, per host, the resources allocated to its live
532	// (non-tombstoned) VMs. Hosts with no live VMs are absent from the map.	527	// (non-tombstoned) VMs. Hosts with no live VMs are absent from the map. Used
533	func (s *Store) AllocatedByHost() (map[string]Alloc, error) { return allocatedByHost(s.db) }	528	// by Snapshot; there is no standalone exported accessor (nothing outside the
534		529	// package needs allocation without hosts/VMs, and Snapshot is the consistent
		530	// way to get all three together).
535	func allocatedByHost(q querier) (map[string]Alloc, error) {	531	func allocatedByHost(q querier) (map[string]Alloc, error) {
536	rows, err := q.Query(`	532	rows, err := q.Query(`
537	SELECT host_id, COALESCE(SUM(vcpus),0), COALESCE(SUM(mem_mb),0), COALESCE(SUM(disk_gb),0)	533	SELECT host_id, COALESCE(SUM(vcpus),0), COALESCE(SUM(mem_mb),0), COALESCE(SUM(disk_gb),0)
@@ -567,12 +563,10 @@ func (s *Store) AppendAudit(action, detail string) error {
567	return err	563	return err
568	}	564	}
569		565
570	// ListAudit returns up to limit audit entries, newest first.	566	// scanAuditRows drains an `at, action, detail`-shaped result set into
571	func (s *Store) ListAudit(limit int) ([]AuditEntry, error) {	567	// AuditEntry values, newest-first per the caller's ORDER BY. Both audit queries
572	rows, err := s.db.Query(`SELECT at, action, detail FROM audit_log ORDER BY id DESC LIMIT ?`, limit)	568	// share the same three columns and RFC3339 at-parse, so they share this.
573	if err != nil {	569	func scanAuditRows(rows *sql.Rows) ([]AuditEntry, error) {
574	return nil, err
575	}
576	defer rows.Close()	570	defer rows.Close()
577	var out []AuditEntry	571	var out []AuditEntry
578	for rows.Next() {	572	for rows.Next() {
@@ -587,6 +581,15 @@ func (s *Store) ListAudit(limit int) ([]AuditEntry, error) {
587	return out, rows.Err()	581	return out, rows.Err()
588	}	582	}
589		583
		584	// ListAudit returns up to limit audit entries, newest first.
		585	func (s *Store) ListAudit(limit int) ([]AuditEntry, error) {
		586	rows, err := s.db.Query(`SELECT at, action, detail FROM audit_log ORDER BY id DESC LIMIT ?`, limit)
		587	if err != nil {
		588	return nil, err
		589	}
		590	return scanAuditRows(rows)
		591	}
		592
590	// ListVMEvents returns up to limit audit rows whose detail JSON carries the	593	// ListVMEvents returns up to limit audit rows whose detail JSON carries the
591	// given vm_id (the lifecycle timeline for one VM), newest first. It filters on	594	// given vm_id (the lifecycle timeline for one VM), newest first. It filters on
592	// json_extract(detail,'$.vm_id'), so every lifecycle emitter must key the VM id	595	// json_extract(detail,'$.vm_id'), so every lifecycle emitter must key the VM id
@@ -600,18 +603,7 @@ func (s *Store) ListVMEvents(vmID string, limit int) ([]AuditEntry, error) {
600	if err != nil {	603	if err != nil {
601	return nil, err	604	return nil, err
602	}	605	}
603	defer rows.Close()	606	return scanAuditRows(rows)
604	var out []AuditEntry
605	for rows.Next() {
606	var e AuditEntry
607	var at string
608	if err := rows.Scan(&at, &e.Action, &e.Detail); err != nil {
609	return nil, err
610	}
611	e.At, _ = time.Parse(time.RFC3339, at)
612	out = append(out, e)
613	}
614	return out, rows.Err()
615	}	607	}
616		608
617	// PruneAudit deletes audit rows older than olderThan and reports how many	609	// PruneAudit deletes audit rows older than olderThan and reports how many
@@ -691,18 +683,9 @@ func (s *Store) ListRevokedSSHCerts() ([]RevokedCert, error) {
691	return out, rows.Err()	683	return out, rows.Err()
692	}	684	}
693		685
694	// HostVMCount returns the number of VM rows for a host (live + tombstoned).
695	// Rows are hard-deleted only after the agent acks destroy, so a count of 0 means
696	// the host is fully drained.
697	func (s *Store) HostVMCount(id string) (int, error) {
698	var n int
699	err := s.db.QueryRow(`SELECT COUNT(*) FROM vms WHERE host_id=?`, id).Scan(&n)
700	return n, err
701	}
702
703	// RemoveHost finalizes decommission: it returns the host's bridge CIDR to the	686	// RemoveHost finalizes decommission: it returns the host's bridge CIDR to the
704	// pool and deletes the host row. It refuses while any VM rows remain (not yet	687	// pool and deletes the host row. It refuses (in-transaction) while any VM rows
705	// reaped), so it must be called only after HostVMCount reaches 0.	688	// remain for the host (not yet reaped).
706	func (s *Store) RemoveHost(id string) error {	689	func (s *Store) RemoveHost(id string) error {
707	tx, err := s.db.Begin()	690	tx, err := s.db.Begin()
708	if err != nil {	691	if err != nil {
@@ -738,26 +721,89 @@ func (s *Store) RemoveHost(id string) error {
738	return tx.Commit()	721	return tx.Commit()
739	}	722	}
740		723
		724	// ForceRemoveHost finalizes a host whose agent will never drain it (dead
		725	// hardware): it purges every VM row for the host, returns the bridge CIDR to
		726	// the pool, and deletes the host row — all in one transaction. Unlike the
		727	// graceful path it does NOT wait for the agent to ack destroys, so it must only
		728	// be used when the host is known gone; any VMs still physically running are
		729	// orphaned with the hardware. Returns the number of VM rows purged.
		730	func (s *Store) ForceRemoveHost(id string) (int, error) {
		731	tx, err := s.db.Begin()
		732	if err != nil {
		733	return 0, err
		734	}
		735	defer tx.Rollback()
		736
		737	var bridgeCIDR string
		738	switch err := tx.QueryRow(`SELECT bridge_cidr FROM hosts WHERE id=?`, id).Scan(&bridgeCIDR); {
		739	case errors.Is(err, sql.ErrNoRows):
		740	return 0, sql.ErrNoRows
		741	case err != nil:
		742	return 0, fmt.Errorf("lookup host cidr: %w", err)
		743	}
		744
		745	res, err := tx.Exec(`DELETE FROM vms WHERE host_id=?`, id)
		746	if err != nil {
		747	return 0, fmt.Errorf("purge host vms: %w", err)
		748	}
		749	purged, _ := res.RowsAffected()
		750
		751	if _, err := tx.Exec(`INSERT INTO freed_cidrs(bridge_cidr) VALUES(?) ON CONFLICT DO NOTHING`, bridgeCIDR); err != nil {
		752	return 0, fmt.Errorf("free cidr: %w", err)
		753	}
		754	if _, err := tx.Exec(`DELETE FROM hosts WHERE id=?`, id); err != nil {
		755	return 0, fmt.Errorf("delete host: %w", err)
		756	}
		757	if err := bumpEpoch(tx); err != nil {
		758	return 0, fmt.Errorf("bump epoch: %w", err)
		759	}
		760	if err := tx.Commit(); err != nil {
		761	return 0, err
		762	}
		763	return int(purged), nil
		764	}
		765
		766	// ipWithinHostCIDR reports whether ip falls inside the owning host's bridge_cidr.
		767	// A missing row, an unparseable stored CIDR, or an unparseable ip all yield
		768	// (false, nil) — the IP simply cannot be validated, so callers drop it. Only a
		769	// genuine DB error on the lookup is surfaced.
		770	func (s *Store) ipWithinHostCIDR(vmID, ip string) (bool, error) {
		771	var cidrStr string
		772	switch err := s.db.QueryRow(
		773	`SELECT h.bridge_cidr FROM vms v JOIN hosts h ON h.id = v.host_id WHERE v.id=?`, vmID,
		774	).Scan(&cidrStr); {
		775	case errors.Is(err, sql.ErrNoRows):
		776	return false, nil
		777	case err != nil:
		778	return false, fmt.Errorf("lookup host cidr: %w", err)
		779	}
		780	prefix, err := netip.ParsePrefix(cidrStr)
		781	if err != nil {
		782	return false, nil
		783	}
		784	addr, err := netip.ParseAddr(ip)
		785	if err != nil {
		786	return false, nil
		787	}
		788	return prefix.Contains(addr), nil
		789	}
		790
741	func (s *Store) RecordVMStatus(id, status, lastErr, ip string) error {	791	func (s *Store) RecordVMStatus(id, status, lastErr, ip string) error {
		792	// A reported IP outside the owning host's bridge_cidr (a DHCP hiccup, a
		793	// link-local/APIPA address, or an agent bug) must NOT block the (status,
		794	// last_error) transition: a genuinely-failed VM still needs status=failed
		795	// persisted durably. So an out-of-CIDR (or unparseable) IP is DROPPED —
		796	// assigned_ip keeps its prior value via the CASE below — while the status
		797	// still writes. This preserves the integrity guarantee (a bogus IP never
		798	// lands in the row) without stalling the transition or spamming per-tick
		799	// rejection warnings from syncsvc.applyReport.
742	if ip != "" {	800	if ip != "" {
743	// Validate IP is within the owning host's bridge_cidr.	801	within, err := s.ipWithinHostCIDR(id, ip)
744	var cidrStr string
745	err := s.db.QueryRow(
746	`SELECT h.bridge_cidr FROM vms v JOIN hosts h ON h.id = v.host_id WHERE v.id=?`, id,
747	).Scan(&cidrStr)
748	if err != nil {
749	return fmt.Errorf("lookup host cidr: %w", err)
750	}
751	prefix, err := netip.ParsePrefix(cidrStr)
752	if err != nil {	802	if err != nil {
753	return fmt.Errorf("parse bridge_cidr: %w", err)	803	return err
754	}	804	}
755	addr, err := netip.ParseAddr(ip)	805	if !within {
756	if err != nil {	806	ip = ""
757	return fmt.Errorf("parse ip: %w", err)
758	}
759	if !prefix.Contains(addr) {
760	return fmt.Errorf("ip %s is outside host cidr %s", ip, cidrStr)
761	}	807	}
762	}	808	}
763		809
@@ -775,8 +821,18 @@ func (s *Store) RecordVMStatus(id, status, lastErr, ip string) error {
775	return nil	821	return nil
776	}	822	}
777		823
778	// scanVM's column order must match the SELECT lists in listVMs and	824	// vmColumns is the positional column list every VM SELECT must use, so the
779	// DesiredForHost exactly — it is positional, not name-based.	825	// order stays locked to scanVM's Scan below (which is positional, not
		826	// name-based). queryVMs is the sole caller, so adding a column is a single
		827	// edit here plus scanVM — every VM query goes through it and can't drift out
		828	// of lockstep.
		829	const vmColumns = `id, host_id, name, image_url, image_sha256, cloud_init, ssh_authorized_key,
		830	ssh_host_key, ssh_host_cert,
		831	vcpus, mem_mb, disk_gb, persistent, power_state, status, last_error, assigned_ip,
		832	created_at, deleted_at`
		833
		834	// scanVM's column order must match vmColumns exactly — it is positional, not
		835	// name-based.
780	func scanVM(rows *sql.Rows) (VM, error) {	836	func scanVM(rows *sql.Rows) (VM, error) {
781	var vm VM	837	var vm VM
782	var createdAt string	838	var createdAt string
@@ -799,15 +855,13 @@ func scanVM(rows *sql.Rows) (VM, error) {
799	return vm, nil	855	return vm, nil
800	}	856	}
801		857
802	func (s *Store) ListVMs() ([]VM, error) { return listVMs(s.db) }	858	// queryVMs runs a vmColumns-projected SELECT against vms, with where appended
803		859	// verbatim after `FROM vms` (e.g. " WHERE id=?", or "" for none) and args
804	func listVMs(q querier) ([]VM, error) {	860	// bound in order, scanning every matching row. listVMs, VMByName, GetVM, and
805	rows, err := q.Query(	861	// DesiredForHost all share this — they differ only in WHERE clause, row-count
806	`SELECT id, host_id, name, image_url, image_sha256, cloud_init, ssh_authorized_key,	862	// expectations, and whether q is sql.DB or an in-flight sql.Tx.
807	ssh_host_key, ssh_host_cert,	863	func queryVMs(q querier, where string, args ...any) ([]VM, error) {
808	vcpus, mem_mb, disk_gb, persistent, power_state, status, last_error, assigned_ip,	864	rows, err := q.Query(`SELECT `+vmColumns+` FROM vms`+where, args...)
809	created_at, deleted_at FROM vms`,
810	)
811	if err != nil {	865	if err != nil {
812	return nil, err	866	return nil, err
813	}	867	}
@@ -823,28 +877,38 @@ func listVMs(q querier) ([]VM, error) {
823	return vms, rows.Err()	877	return vms, rows.Err()
824	}	878	}
825		879
		880	func (s *Store) ListVMs() ([]VM, error) { return listVMs(s.db) }
		881
		882	func listVMs(q querier) ([]VM, error) { return queryVMs(q, "") }
		883
826	// VMByName returns the live (non-tombstoned) VM with the given name. The	884	// VMByName returns the live (non-tombstoned) VM with the given name. The
827	// vms_name unique index guarantees at most one match. sql.ErrNoRows ⇒ no such	885	// vms_name unique index guarantees at most one match. sql.ErrNoRows ⇒ no such
828	// VM. Read-only; used by the SSH jump gate to resolve `ssh -J gate user@<name>`	886	// VM. Read-only; used by the SSH jump gate to resolve `ssh -J gate user@<name>`
829	// to a host/VM ID pair.	887	// to a host/VM ID pair.
830	func (s *Store) VMByName(name string) (VM, error) {	888	func (s *Store) VMByName(name string) (VM, error) {
831	rows, err := s.db.Query(	889	vms, err := queryVMs(s.db, ` WHERE name=? AND deleted_at IS NULL`, name)
832	`SELECT id, host_id, name, image_url, image_sha256, cloud_init, ssh_authorized_key,
833	ssh_host_key, ssh_host_cert,
834	vcpus, mem_mb, disk_gb, persistent, power_state, status, last_error, assigned_ip,
835	created_at, deleted_at FROM vms WHERE name=? AND deleted_at IS NULL`, name,
836	)
837	if err != nil {	890	if err != nil {
838	return VM{}, err	891	return VM{}, err
839	}	892	}
840	defer rows.Close()	893	if len(vms) == 0 {
841	if !rows.Next() {	894	return VM{}, sql.ErrNoRows
842	if err := rows.Err(); err != nil {	895	}
843	return VM{}, err	896	return vms[0], nil
844	}	897	}
		898
		899	// GetVM returns the VM with the given id via a single indexed lookup on the
		900	// primary key. Unlike VMByName it does NOT filter on deleted_at: the
		901	// patch/delete/restore callers operate on VMs that may be tombstoned, so the
		902	// row must be found regardless of tombstone state. sql.ErrNoRows ⇒ no such VM.
		903	func (s *Store) GetVM(id string) (VM, error) {
		904	vms, err := queryVMs(s.db, ` WHERE id=?`, id)
		905	if err != nil {
		906	return VM{}, err
		907	}
		908	if len(vms) == 0 {
845	return VM{}, sql.ErrNoRows	909	return VM{}, sql.ErrNoRows
846	}	910	}
847	return scanVM(rows)	911	return vms[0], nil
848	}	912	}
849		913
850	// Snapshot reads hosts, per-host allocation, and VMs in a single read	914	// Snapshot reads hosts, per-host allocation, and VMs in a single read
@@ -884,28 +948,10 @@ func (s *Store) DesiredForHost(hostID string) (uint64, []VM, error) {
884	return 0, nil, fmt.Errorf("read epoch: %w", err)	948	return 0, nil, fmt.Errorf("read epoch: %w", err)
885	}	949	}
886		950
887	rows, err := tx.Query(	951	vms, err := queryVMs(tx, ` WHERE host_id=?`, hostID)
888	`SELECT id, host_id, name, image_url, image_sha256, cloud_init, ssh_authorized_key,
889	ssh_host_key, ssh_host_cert,
890	vcpus, mem_mb, disk_gb, persistent, power_state, status, last_error, assigned_ip,
891	created_at, deleted_at FROM vms WHERE host_id=?`, hostID,
892	)
893	if err != nil {	952	if err != nil {
894	return 0, nil, err	953	return 0, nil, err
895	}	954	}
896	defer rows.Close()
897
898	var vms []VM
899	for rows.Next() {
900	vm, err := scanVM(rows)
901	if err != nil {
902	return 0, nil, err
903	}
904	vms = append(vms, vm)
905	}
906	if err := rows.Err(); err != nil {
907	return 0, nil, err
908	}
909		955
910	if err := tx.Commit(); err != nil {	956	if err := tx.Commit(); err != nil {
911	return 0, nil, err	957	return 0, nil, err

internal/server/store/store_test.go

Old		New
@@ -7,6 +7,7 @@ import (
7	"testing"	7	"testing"
8	"time"	8	"time"
9		9
		10	"github.com/a73x/eitri/internal/random"
10	"github.com/stretchr/testify/assert"	11	"github.com/stretchr/testify/assert"
11	"github.com/stretchr/testify/require"	12	"github.com/stretchr/testify/require"
12	)	13	)
@@ -56,6 +57,30 @@ func TestVMByName(t *testing.T) {
56	assert.ErrorIs(t, err, sql.ErrNoRows)	57	assert.ErrorIs(t, err, sql.ErrNoRows)
57	}	58	}
58		59
		60	func TestGetVM(t *testing.T) {
		61	s := newStore(t)
		62	h := enrollHost(t, s)
		63	vm := makeVM(t, s, h, "web-1")
		64
		65	got, err := s.GetVM(vm.ID)
		66	require.NoError(t, err)
		67	assert.Equal(t, vm.ID, got.ID)
		68	assert.Equal(t, vm.Name, got.Name)
		69	assert.Equal(t, h.ID, got.HostID)
		70
		71	// Unknown id ⇒ ErrNoRows (callers read this as not-found).
		72	_, err = s.GetVM("no-such-id")
		73	assert.ErrorIs(t, err, sql.ErrNoRows)
		74
		75	// Unlike VMByName, GetVM must still find a tombstoned row — the
		76	// patch/delete/restore callers operate on VMs that may be tombstoned.
		77	require.NoError(t, s.TombstoneVM(vm.ID))
		78	got, err = s.GetVM(vm.ID)
		79	require.NoError(t, err)
		80	assert.Equal(t, vm.ID, got.ID)
		81	assert.NotNil(t, got.DeletedAt)
		82	}
		83
59	func TestVMHostKeyAndCertPersist(t *testing.T) {	84	func TestVMHostKeyAndCertPersist(t *testing.T) {
60	s := newStore(t)	85	s := newStore(t)
61	h := enrollHost(t, s)	86	h := enrollHost(t, s)
@@ -206,14 +231,27 @@ func TestDesiredForHostIncludesTombstonedAndEpochConsistently(t *testing.T) {
206	assert.NotNil(t, vms[0].DeletedAt, "tombstoned rows stay in the snapshot until acked")	231	assert.NotNil(t, vms[0].DeletedAt, "tombstoned rows stay in the snapshot until acked")
207	}	232	}
208		233
209	func TestRecordVMStatusValidatesIPWithinHostCIDR(t *testing.T) {	234	func TestRecordVMStatusDropsOutOfCIDRIPButKeepsStatus(t *testing.T) {
210	s := newStore(t)	235	s := newStore(t)
211	h := enrollHost(t, s) // 10.77.1.0/24	236	h := enrollHost(t, s) // 10.77.1.0/24
212	require.NoError(t, s.CreateVM(VM{ID: "vm1", HostID: h.ID, Name: "a", ImageURL: "u",	237	require.NoError(t, s.CreateVM(VM{ID: "vm1", HostID: h.ID, Name: "a", ImageURL: "u",
213	ImageSHA256: "s", VCPUs: 1, MemMB: 512, DiskGB: 5, PowerState: "running"}))	238	ImageSHA256: "s", VCPUs: 1, MemMB: 512, DiskGB: 5, PowerState: "running"}))
214	assert.Error(t, s.RecordVMStatus("vm1", "ready", "", "10.77.2.9"),	239
215	"IP outside the host's CIDR must be rejected")	240	// A good IP lands.
216	assert.NoError(t, s.RecordVMStatus("vm1", "ready", "", "10.77.1.9"))	241	require.NoError(t, s.RecordVMStatus("vm1", "ready", "", "10.77.1.9"))
		242	vm, err := s.GetVM("vm1")
		243	require.NoError(t, err)
		244	assert.Equal(t, "10.77.1.9", vm.AssignedIP)
		245
		246	// An out-of-CIDR IP alongside a failed transition must NOT block the status
		247	// write, and must NOT overwrite the prior good IP with the bogus value.
		248	require.NoError(t, s.RecordVMStatus("vm1", "failed", "boom", "10.77.2.9"),
		249	"out-of-CIDR IP must not fail the status write")
		250	vm, err = s.GetVM("vm1")
		251	require.NoError(t, err)
		252	assert.Equal(t, "failed", vm.Status, "status transition must persist despite the bad IP")
		253	assert.Equal(t, "boom", vm.LastError)
		254	assert.Equal(t, "10.77.1.9", vm.AssignedIP, "bogus IP must be dropped, prior kept")
217	}	255	}
218		256
219	func TestEnrollmentCIDRWorksForNonSlash16Pools(t *testing.T) {	257	func TestEnrollmentCIDRWorksForNonSlash16Pools(t *testing.T) {
@@ -245,6 +283,55 @@ func TestRecordVMStatusUnknownVMErrors(t *testing.T) {
245	assert.Error(t, err)	283	assert.Error(t, err)
246	}	284	}
247		285
		286	func TestForceRemoveHostPurgesVMsAndFreesCIDR(t *testing.T) {
		287	s := newStore(t)
		288	h := enrollHost(t, s)
		289	makeVM(t, s, h, "vm-a")
		290	vmB := makeVM(t, s, h, "vm-b")
		291	// Tombstone one of the two so the purge is exercised against both live and
		292	// tombstoned rows (ForceRemoveHost deletes by host_id with no deleted_at
		293	// filter, so a tombstoned row is purged — and counted — like a live one).
		294	require.NoError(t, s.TombstoneVM(vmB.ID))
		295
		296	purged, err := s.ForceRemoveHost(h.ID)
		297	require.NoError(t, err)
		298	assert.Equal(t, 2, purged, "both VM rows — live and tombstoned — must be purged")
		299
		300	_, err = s.GetHost(h.ID)
		301	assert.Error(t, err, "host row must be gone")
		302
		303	// Independently verify against the DB, not just the return value: no VM
		304	// row (including tombstoned ones) may remain for the purged host.
		305	vms, err := s.ListVMs()
		306	require.NoError(t, err)
		307	for _, vm := range vms {
		308	assert.NotEqual(t, h.ID, vm.HostID, "no VM rows should remain for the force-removed host")
		309	}
		310
		311	// The freed CIDR is consulted before the monotonic allocator, so a fresh
		312	// enrollment reuses it.
		313	tok, err := s.CreateEnrollmentToken()
		314	require.NoError(t, err)
		315	h2, err := s.RedeemEnrollmentToken(tok, "host-b", "linux", "amd64", "cloudhv", "")
		316	require.NoError(t, err)
		317	assert.Equal(t, h.BridgeCIDR, h2.BridgeCIDR, "force-removed host's CIDR must return to the pool")
		318	}
		319
		320	func TestForceRemoveHostUnknownIsNoRows(t *testing.T) {
		321	s := newStore(t)
		322	_, err := s.ForceRemoveHost("nope")
		323	assert.ErrorIs(t, err, sql.ErrNoRows)
		324	}
		325
		326	func TestCreateVMRejectsNonEnrolledHost(t *testing.T) {
		327	s := newStore(t)
		328	h := enrollHost(t, s)
		329	require.NoError(t, s.DecommissionHost(h.ID))
		330	err := s.CreateVM(VM{ID: "late", HostID: h.ID, Name: "late", ImageURL: "u",
		331	ImageSHA256: "s", VCPUs: 1, MemMB: 512, DiskGB: 5, PowerState: "running"})
		332	assert.ErrorIs(t, err, ErrHostNotEnrolled)
		333	}
		334
248	func TestServerCertLoadOrCreatePersists(t *testing.T) {	335	func TestServerCertLoadOrCreatePersists(t *testing.T) {
249	dir := t.TempDir()	336	dir := t.TempDir()
250	st, err := Open(filepath.Join(dir, "x.db"), "10.77.0.0/16")	337	st, err := Open(filepath.Join(dir, "x.db"), "10.77.0.0/16")
@@ -282,10 +369,22 @@ func TestSnapshotReadsAllThreeConsistently(t *testing.T) {
282		369
283	wantHosts, err := s.ListHosts()	370	wantHosts, err := s.ListHosts()
284	require.NoError(t, err)	371	require.NoError(t, err)
285	wantAlloc, err := s.AllocatedByHost()
286	require.NoError(t, err)
287	wantVMs, err := s.ListVMs()	372	wantVMs, err := s.ListVMs()
288	require.NoError(t, err)	373	require.NoError(t, err)
		374	// Derive the expected allocation from the independently-read VM list
		375	// (rather than a second Snapshot() call, which would just check Snapshot
		376	// against itself) — the same pattern TestSnapshotIsAtomicUnderConcurrentWrites
		377	// uses to pin that alloc is exactly the sum of live VM resources per host.
		378	wantAlloc := map[string]Alloc{}
		379	for _, vm := range wantVMs {
		380	if vm.DeletedAt == nil {
		381	a := wantAlloc[vm.HostID]
		382	a.VCPUs += vm.VCPUs
		383	a.MemMB += vm.MemMB
		384	a.DiskGB += vm.DiskGB
		385	wantAlloc[vm.HostID] = a
		386	}
		387	}
289		388
290	assert.Equal(t, wantHosts, hosts)	389	assert.Equal(t, wantHosts, hosts)
291	assert.Equal(t, wantAlloc, alloc)	390	assert.Equal(t, wantAlloc, alloc)
@@ -306,7 +405,7 @@ func TestSnapshotIsAtomicUnderConcurrentWrites(t *testing.T) {
306	go func() {	405	go func() {
307	defer close(done)	406	defer close(done)
308	for i := 0; i < 300; i++ {	407	for i := 0; i < 300; i++ {
309	id := RandHex(8)	408	id := random.Hex(8)
310	_ = s.CreateVM(VM{ID: id, HostID: h.ID, Name: "vm-" + id,	409	_ = s.CreateVM(VM{ID: id, HostID: h.ID, Name: "vm-" + id,
311	ImageURL: "http://x/i", ImageSHA256: "abc",	410	ImageURL: "http://x/i", ImageSHA256: "abc",
312	VCPUs: 1, MemMB: 256, DiskGB: 1, PowerState: "running"})	411	VCPUs: 1, MemMB: 256, DiskGB: 1, PowerState: "running"})

internal/server/syncsvc/syncsvc.go

Old		New
@@ -26,12 +26,26 @@ import (
26	// fills.	26	// fills.
27	const defaultWriteTimeout = 30 * time.Second	27	const defaultWriteTimeout = 30 * time.Second
28		28
		29	// vmStatusRecorder is the durable-write seam applyReport uses to record VM
		30	// lifecycle status. *store.Store satisfies it. Keeping it a field (rather than a
		31	// direct s.st call) lets tests substitute a counting fake to prove that
		32	// unchanged reports perform no write.
		33	type vmStatusRecorder interface {
		34	RecordVMStatus(id, status, lastErr, ip string) error
		35	}
		36
29	// Service is the QUIC server end of the agent reconcile stream.	37	// Service is the QUIC server end of the agent reconcile stream.
30	type Service struct {	38	type Service struct {
31	st *store.Store	39	st *store.Store
32	reg *registry.Registry	40	reg *registry.Registry
33	hub *hub.Hub	41	hub *hub.Hub
34	secret []byte	42	secret []byte
		43	// recorder is the durable VM-status write seam (defaults to st). tracker
		44	// remembers the last durably-written status per VM so applyReport can skip
		45	// RecordVMStatus (a SELECT+UPDATE on the single SQLite conn) when nothing
		46	// changed.
		47	recorder vmStatusRecorder
		48	tracker *statusTracker
35	// maxCredAge, when non-zero, rejects credentials whose issued-at is older.	49	// maxCredAge, when non-zero, rejects credentials whose issued-at is older.
36	// Zero disables the age check (default: expiry without an auto-renewal	50	// Zero disables the age check (default: expiry without an auto-renewal
37	// channel would force periodic re-enrolls; per-host generation revocation	51	// channel would force periodic re-enrolls; per-host generation revocation
@@ -72,7 +86,7 @@ func newWithWriteTimeout(st store.Store, reg registry.Registry, h *hub.Hub, se
72	writeTimeout = defaultWriteTimeout	86	writeTimeout = defaultWriteTimeout
73	}	87	}
74	return &Service{st: st, reg: reg, hub: h, secret: secret, maxCredAge: maxCredAge, writeTimeout: writeTimeout,	88	return &Service{st: st, reg: reg, hub: h, secret: secret, maxCredAge: maxCredAge, writeTimeout: writeTimeout,
75	conns: map[string]quic.Connection{}}	89	conns: map[string]quic.Connection{}, recorder: st, tracker: newStatusTracker()}
76	}	90	}
77		91
78	// Serve accepts QUIC connections until ctx is cancelled.	92	// Serve accepts QUIC connections until ctx is cancelled.
@@ -86,6 +100,19 @@ func (s Service) Serve(ctx context.Context, lis quic.Listener) error {
86	}	100	}
87	}	101	}
88		102
		103	// credStale evaluates a credential's claims against the host row's current
		104	// generation: stale is true when the credential has been revoked (its
		105	// generation no longer matches the host's current one) or has exceeded
		106	// maxCredAge (expired, reported separately so callers that log an "expired"
		107	// field can distinguish the reason). It is pure — no I/O — so callers decide
		108	// when to fetch hostRow and what to do with a stale verdict (initial auth
		109	// rejects; the per-report re-check closes the session).
		110	func (s *Service) credStale(claims hosttoken.Claims, hostRow store.Host) (stale, expired bool) {
		111	expired = s.maxCredAge > 0 && time.Since(claims.IssuedAt) > s.maxCredAge
		112	stale = expired \|\| claims.Generation != hostRow.CredGeneration
		113	return stale, expired
		114	}
		115
89	// NOTE (verified Task 1): quic-go v0.48.2 uses INTERFACES quic.Connection and	116	// NOTE (verified Task 1): quic-go v0.48.2 uses INTERFACES quic.Connection and
90	// quic.Stream (not quic.Conn/quic.Stream, which only exist in v0.49+).	117	// quic.Stream (not quic.Conn/quic.Stream, which only exist in v0.49+).
91	func (s *Service) handleConn(ctx context.Context, conn quic.Connection) {	118	func (s *Service) handleConn(ctx context.Context, conn quic.Connection) {
@@ -112,20 +139,27 @@ func (s *Service) handleConn(ctx context.Context, conn quic.Connection) {
112	return	139	return
113	}	140	}
114	hostID := claims.HostID	141	hostID := claims.HostID
115	if s.maxCredAge > 0 && time.Since(claims.IssuedAt) > s.maxCredAge {
116	_ = conn.CloseWithError(transport.CodeAuthRejected, "credential expired — re-enroll this host")
117	return
118	}
119	hostRow, err := s.st.GetHost(hostID)	142	hostRow, err := s.st.GetHost(hostID)
120	if err != nil {	143	if err != nil {
121	_ = conn.CloseWithError(transport.CodeAuthRejected, "host not found")	144	_ = conn.CloseWithError(transport.CodeAuthRejected, "host not found")
122	return	145	return
123	}	146	}
124	// Per-host revocation: a credential minted at an older generation is dead.	147	// credStale covers both per-host revocation (a credential minted at an
125	if claims.Generation != hostRow.CredGeneration {	148	// older generation is dead) and max-age expiry in a single predicate; the
126	_ = conn.CloseWithError(transport.CodeAuthRejected, "credential revoked — re-enroll this host")	149	// close message distinguishes the two so an operator reading the close
		150	// reason can tell a routine re-enroll (expiry) from something more
		151	// suspicious (revocation). The ordering is deliberate: host-not-found is
		152	// checked BEFORE expiry, so it wins for a purged host — earlier code
		153	// rejected an expired credential before ever touching the DB.
		154	if stale, expired := s.credStale(claims, hostRow); stale {
		155	msg := "credential revoked — re-enroll this host"
		156	if expired {
		157	msg = "credential expired — re-enroll this host"
		158	}
		159	_ = conn.CloseWithError(transport.CodeAuthRejected, msg)
127	return	160	return
128	}	161	}
		162	s.reg.RecordConnect(hostID)
129	slog.Info("agent connected", "host", hostID, "provisioner", h.GetProvisioner(), "last_seen_epoch", h.GetLastSeenEpoch())	163	slog.Info("agent connected", "host", hostID, "provisioner", h.GetProvisioner(), "last_seen_epoch", h.GetLastSeenEpoch())
130		164
131	// Down-stream: server opens it; first write makes it visible to the agent.	165	// Down-stream: server opens it; first write makes it visible to the agent.
@@ -192,8 +226,7 @@ func (s *Service) handleConn(ctx context.Context, conn quic.Connection) {
192	_ = conn.CloseWithError(0, "credential re-check unavailable")	226	_ = conn.CloseWithError(0, "credential re-check unavailable")
193	return	227	return
194	}	228	}
195	expired := s.maxCredAge > 0 && time.Since(claims.IssuedAt) > s.maxCredAge	229	if stale, expired := s.credStale(claims, row); stale {
196	if row.CredGeneration != claims.Generation \|\| expired {
197	slog.Info("credential no longer valid mid-session; closing", "host", hostID, "expired", expired)	230	slog.Info("credential no longer valid mid-session; closing", "host", hostID, "expired", expired)
198	_ = conn.CloseWithError(transport.CodeAuthRejected, "credential revoked or expired — re-enroll this host")	231	_ = conn.CloseWithError(transport.CodeAuthRejected, "credential revoked or expired — re-enroll this host")
199	return	232	return
@@ -267,13 +300,27 @@ func (s Service) applyReport(hostID string, rep pb.ActualStateReport) {
267	s.reg.UpdateReport(hostID, r)	300	s.reg.UpdateReport(hostID, r)
268		301
269	// Write-through durable status for lifecycle phases ready/failed only.	302	// Write-through durable status for lifecycle phases ready/failed only.
		303	// Skip the SELECT+UPDATE when the durable triple (status, last_error,
		304	// effective assigned_ip) is unchanged from the last write we recorded.
		305	// writeThrough runs decide→write→commit atomically per VM so overlapping
		306	// reports for the same host can never reorder the write and the cache commit;
		307	// the cache is updated only after a successful write, so a rejected write
		308	// never suppresses the next retry.
270	for _, v := range rep.GetVms() {	309	for _, v := range rep.GetVms() {
271	phase := v.GetPhase()	310	phase := v.GetPhase()
272	if phase != "ready" && phase != "failed" {	311	if phase != "ready" && phase != "failed" {
273	continue	312	continue
274	}	313	}
275	if err := s.st.RecordVMStatus(v.GetVmId(), phase, v.GetLastError(), v.GetIp()); err != nil {	314	vmID := v.GetVmId()
276	slog.Warn("RecordVMStatus rejected", "vm", v.GetVmId(), "host", hostID, "err", err)	315	// Pass the RAW reported ip to RecordVMStatus to preserve its
		316	// empty-ip-keeps-prior UPDATE semantics.
		317	err := s.tracker.writeThrough(vmID, phase, v.GetLastError(), v.GetIp(), func() error {
		318	return s.recorder.RecordVMStatus(vmID, phase, v.GetLastError(), v.GetIp())
		319	})
		320	if err != nil {
		321	// ErrNoRows (row gone) or a validation error: the write did not land
		322	// and the cache is unchanged, so the next report retries.
		323	slog.Warn("RecordVMStatus rejected", "vm", vmID, "host", hostID, "err", err)
277	}	324	}
278	}	325	}
279		326
@@ -289,6 +336,11 @@ func (s Service) applyReport(hostID string, rep pb.ActualStateReport) {
289	// and re-acks every tick forever (log spam) until some other edit pokes it.	336	// and re-acks every tick forever (log spam) until some other edit pokes it.
290	anyDeleted := false	337	anyDeleted := false
291	for _, id := range rep.GetDestroyed() {	338	for _, id := range rep.GetDestroyed() {
		339	// Prune the dedup cache: this VM's row is being hard-deleted, so its
		340	// tracked status is dead weight (and a future id reuse must not inherit
		341	// a stale cached triple — ids are unique, but forgetting is the correct,
		342	// memory-bounding thing regardless of whether the delete succeeds).
		343	s.tracker.forget(id)
292	if err := s.st.HardDeleteVM(id); err != nil {	344	if err := s.st.HardDeleteVM(id); err != nil {
293	slog.Warn("HardDeleteVM failed", "vm", id, "host", hostID, "err", err)	345	slog.Warn("HardDeleteVM failed", "vm", id, "host", hostID, "err", err)
294	} else {	346	} else {
@@ -362,14 +414,19 @@ var ErrAgentOffline = errors.New("agent not connected")
362	// deadline — consoles are long-lived.	414	// deadline — consoles are long-lived.
363	const consoleHandshakeTimeout = 10 * time.Second	415	const consoleHandshakeTimeout = 10 * time.Second
364		416
365	// OpenConsole opens a console stream to vmID's agent on the live sync	417	// openStream is the shared server-initiated stream procedure behind OpenConsole
366	// connection: sends ConsoleOpen, awaits ConsoleOpened, and returns the stream	418	// and OpenTCP: find the live agent conn, open a QUIC stream, send `open` under a
367	// as a raw byte pipe. The returned Close tears down both directions.	419	// handshake deadline, await the reply, and return the stream as a raw byte pipe
		420	// once the agent acks. checkReply extracts the typed ack — present=false when
		421	// the reply wasn't the expected ack message, ok=false with a reason when the
		422	// agent refused. label ("console"/"tcp") prefixes the error messages. The
		423	// returned Close tears down both directions.
368	//	424	//
369	// ctx bounds stream OPENING only; the handshake that follows is bounded by	425	// ctx bounds stream OPENING only; the handshake that follows is bounded by
370	// consoleHandshakeTimeout instead, so a call can outlive ctx cancellation by	426	// consoleHandshakeTimeout instead, so a call can outlive ctx cancellation by
371	// up to that long (10s) before returning.	427	// up to that long (10s) before returning.
372	func (s *Service) OpenConsole(ctx context.Context, hostID, vmID string) (io.ReadWriteCloser, error) {	428	func (s Service) openStream(ctx context.Context, hostID, label string, open pb.ServerMessage,
		429	checkReply func(*pb.AgentMessage) (ok bool, reason string, present bool)) (io.ReadWriteCloser, error) {
373	s.consoleMu.Lock()	430	s.consoleMu.Lock()
374	conn, ok := s.conns[hostID]	431	conn, ok := s.conns[hostID]
375	s.consoleMu.Unlock()	432	s.consoleMu.Unlock()
@@ -378,31 +435,30 @@ func (s *Service) OpenConsole(ctx context.Context, hostID, vmID string) (io.Read
378	}	435	}
379	st, err := conn.OpenStreamSync(ctx)	436	st, err := conn.OpenStreamSync(ctx)
380	if err != nil {	437	if err != nil {
381	return nil, fmt.Errorf("open console stream: %w", err)	438	return nil, fmt.Errorf("open %s stream: %w", label, err)
382	}	439	}
383	cs := consoleStream{st}	440	cs := consoleStream{st}
384	if err := st.SetDeadline(time.Now().Add(consoleHandshakeTimeout)); err != nil {	441	if err := st.SetDeadline(time.Now().Add(consoleHandshakeTimeout)); err != nil {
385	cs.Close()	442	cs.Close()
386	return nil, err	443	return nil, err
387	}	444	}
388	open := &pb.ServerMessage{Msg: &pb.ServerMessage_ConsoleOpen{ConsoleOpen: &pb.ConsoleOpen{VmId: vmID}}}
389	if err := transport.WriteMsg(st, open); err != nil {	445	if err := transport.WriteMsg(st, open); err != nil {
390	cs.Close()	446	cs.Close()
391	return nil, fmt.Errorf("console open: %w", err)	447	return nil, fmt.Errorf("%s open: %w", label, err)
392	}	448	}
393	var reply pb.AgentMessage	449	var reply pb.AgentMessage
394	if err := transport.ReadMsg(st, &reply, transport.DefaultMaxFrame); err != nil {	450	if err := transport.ReadMsg(st, &reply, transport.DefaultMaxFrame); err != nil {
395	cs.Close()	451	cs.Close()
396	return nil, fmt.Errorf("console reply: %w", err)	452	return nil, fmt.Errorf("%s reply: %w", label, err)
397	}	453	}
398	co := reply.GetConsoleOpened()	454	replyOK, reason, present := checkReply(&reply)
399	if co == nil {	455	if !present {
400	cs.Close()	456	cs.Close()
401	return nil, errors.New("console refused: unexpected reply")	457	return nil, fmt.Errorf("%s refused: unexpected reply", label)
402	}	458	}
403	if !co.GetOk() {	459	if !replyOK {
404	cs.Close()	460	cs.Close()
405	return nil, fmt.Errorf("console refused: %s", co.GetError())	461	return nil, fmt.Errorf("%s refused: %s", label, reason)
406	}	462	}
407	// Handshake done — clear the deadline; the session is long-lived.	463	// Handshake done — clear the deadline; the session is long-lived.
408	if err := st.SetDeadline(time.Time{}); err != nil {	464	if err := st.SetDeadline(time.Time{}); err != nil {
@@ -412,56 +468,32 @@ func (s *Service) OpenConsole(ctx context.Context, hostID, vmID string) (io.Read
412	return cs, nil	468	return cs, nil
413	}	469	}
414		470
		471	// OpenConsole opens a console stream to vmID's agent on the live sync
		472	// connection: sends ConsoleOpen, awaits ConsoleOpened, and returns the stream
		473	// as a raw byte pipe.
		474	func (s *Service) OpenConsole(ctx context.Context, hostID, vmID string) (io.ReadWriteCloser, error) {
		475	open := &pb.ServerMessage{Msg: &pb.ServerMessage_ConsoleOpen{ConsoleOpen: &pb.ConsoleOpen{VmId: vmID}}}
		476	return s.openStream(ctx, hostID, "console", open, func(m *pb.AgentMessage) (bool, string, bool) {
		477	co := m.GetConsoleOpened()
		478	if co == nil {
		479	return false, "", false
		480	}
		481	return co.GetOk(), co.GetError(), true
		482	})
		483	}
		484
415	// OpenTCP opens a tunnel stream to vmID's agent on the live sync connection:	485	// OpenTCP opens a tunnel stream to vmID's agent on the live sync connection:
416	// sends TCPOpen (naming the VM and guest TCP port), awaits TCPOpened, and	486	// sends TCPOpen (naming the VM and guest TCP port), awaits TCPOpened, and
417	// returns the stream as a raw byte pipe. The returned Close tears down both	487	// returns the stream as a raw byte pipe.
418	// directions. It mirrors OpenConsole exactly, differing only in the handshake
419	// message pair.
420	//
421	// ctx bounds stream OPENING only; the handshake that follows is bounded by
422	// consoleHandshakeTimeout instead, so a call can outlive ctx cancellation by
423	// up to that long (10s) before returning.
424	func (s *Service) OpenTCP(ctx context.Context, hostID, vmID string, port uint32) (io.ReadWriteCloser, error) {	488	func (s *Service) OpenTCP(ctx context.Context, hostID, vmID string, port uint32) (io.ReadWriteCloser, error) {
425	s.consoleMu.Lock()
426	conn, ok := s.conns[hostID]
427	s.consoleMu.Unlock()
428	if !ok {
429	return nil, ErrAgentOffline
430	}
431	st, err := conn.OpenStreamSync(ctx)
432	if err != nil {
433	return nil, fmt.Errorf("open tcp stream: %w", err)
434	}
435	cs := consoleStream{st}
436	if err := st.SetDeadline(time.Now().Add(consoleHandshakeTimeout)); err != nil {
437	cs.Close()
438	return nil, err
439	}
440	open := &pb.ServerMessage{Msg: &pb.ServerMessage_TcpOpen{TcpOpen: &pb.TCPOpen{VmId: vmID, Port: port}}}	489	open := &pb.ServerMessage{Msg: &pb.ServerMessage_TcpOpen{TcpOpen: &pb.TCPOpen{VmId: vmID, Port: port}}}
441	if err := transport.WriteMsg(st, open); err != nil {	490	return s.openStream(ctx, hostID, "tcp", open, func(m *pb.AgentMessage) (bool, string, bool) {
442	cs.Close()	491	to := m.GetTcpOpened()
443	return nil, fmt.Errorf("tcp open: %w", err)	492	if to == nil {
444	}	493	return false, "", false
445	var reply pb.AgentMessage	494	}
446	if err := transport.ReadMsg(st, &reply, transport.DefaultMaxFrame); err != nil {	495	return to.GetOk(), to.GetError(), true
447	cs.Close()	496	})
448	return nil, fmt.Errorf("tcp reply: %w", err)
449	}
450	to := reply.GetTcpOpened()
451	if to == nil {
452	cs.Close()
453	return nil, errors.New("tcp refused: unexpected reply")
454	}
455	if !to.GetOk() {
456	cs.Close()
457	return nil, fmt.Errorf("tcp refused: %s", to.GetError())
458	}
459	// Handshake done — clear the deadline; the session is long-lived.
460	if err := st.SetDeadline(time.Time{}); err != nil {
461	cs.Close()
462	return nil, err
463	}
464	return cs, nil
465	}	497	}
466		498
467	// consoleStream adapts a quic.Stream to io.ReadWriteCloser with a Close that	499	// consoleStream adapts a quic.Stream to io.ReadWriteCloser with a Close that

internal/server/syncsvc/syncsvc_test.go

@@ @@ -6,6 +6,7 @@ import ( @@
+	"io"
+	"runtime"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+	assert.Equal(t, int64(8), st.Capacity.VCPUs)
+// countingRecorder wraps a real recorder and counts (with a mutex, since host
+// read-loop goroutines call concurrently) how many RecordVMStatus writes reach
+// the store, so a test can prove that a repeated unchanged report performs none.
+type countingRecorder struct {
+	mu    sync.Mutex
+	inner vmStatusRecorder
+	calls int
+}
+func (c *countingRecorder) RecordVMStatus(id, status, lastErr, ip string) error {
+	c.mu.Lock()
+	c.calls++
+	c.mu.Unlock()
+	return c.inner.RecordVMStatus(id, status, lastErr, ip)
+}
+func (c *countingRecorder) count() int {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.calls
+}
+// TestApplyReportSkipsUnchangedWrites proves the dedup: the SAME ready report
+// sent twice performs exactly ONE durable write; a subsequent report that
+// changes the ip performs another; and the DB's final (status, assigned_ip) is
+// correct across the unchanged→changed sequence.
+func TestApplyReportSkipsUnchangedWrites(t *testing.T) {
+	f := setup(t)
+	require.NoError(t, f.st.CreateVM(store.VM{ID: "vm1", HostID: f.host.ID, Name: "a",
+		ImageURL: "u", ImageSHA256: "s", VCPUs: 1, MemMB: 512, DiskGB: 5, PowerState: "running"}))
+	// Swap in a counting recorder before any report is applied.
+	rec := &countingRecorder{inner: f.svc.recorder}
+	f.svc.recorder = rec
+	c := mustDial(t, f)
+	c.recv(t) // initial snapshot
+	ready := func(ip string) *pb.AgentMessage {
+		return &pb.AgentMessage{Msg: &pb.AgentMessage_Report{
+			Report: &pb.ActualStateReport{Vms: []*pb.ActualVM{
+				{VmId: "vm1", Power: "running", Phase: "ready", Ip: ip},
+			}}}}
+	}
+	// First report writes and lands in the DB.
+	c.send(t, ready("10.77.1.2"))
+	require.Eventually(t, func() bool {
+		vms, _ := f.st.ListVMs()
+		return len(vms) == 1 && vms[0].Status == "ready" && vms[0].AssignedIP == "10.77.1.2"
+	}, 2*time.Second, 20*time.Millisecond, "first report must write through")
+	require.Equal(t, 1, rec.count(), "first report writes exactly once")
+	// Identical report: must perform NO additional write. Assert the count stays
+	// at 1 for a sustained window (a redundant write would bump it).
+	c.send(t, ready("10.77.1.2"))
+	require.Never(t, func() bool {
+		return rec.count() != 1
+	}, 500*time.Millisecond, 25*time.Millisecond, "unchanged report must not write again")
+	// A changed ip must write again and update the DB.
+	c.send(t, ready("10.77.1.3"))
+	require.Eventually(t, func() bool {
+		return rec.count() == 2
+	}, 2*time.Second, 20*time.Millisecond, "changed ip must write again")
+	require.Eventually(t, func() bool {
+		vms, _ := f.st.ListVMs()
+		return len(vms) == 1 && vms[0].Status == "ready" && vms[0].AssignedIP == "10.77.1.3"
+	}, 2*time.Second, 20*time.Millisecond, "changed ip must land in the DB")
+}
+func TestReportWithValidIPUpdatesRegistryAndStore(t *testing.T) {
+	f := setup(t)
+	require.NoError(t, f.st.CreateVM(store.VM{ID: "vm1", HostID: f.host.ID, Name: "a",

internal/server/syncsvc/tracker.go

@@ @@ -0,0 +1,68 @@ @@
+package syncsvc
+import "sync"
+// vmStatus is the durable status triple a successful RecordVMStatus persists:
+// the (status, last_error, assigned_ip) that ends up stored in the vms row.
+type vmStatus struct {
+	status  string
+	lastErr string
+	ip      string
+}
+// statusTracker remembers the last DURABLY-WRITTEN status triple per VM so
+// applyReport can skip the SELECT+UPDATE that RecordVMStatus does when nothing
+// changed. Multiple host read-loop goroutines call into it concurrently, so all
+// access is guarded by mu. Keyed by vmID (globally unique).
+type statusTracker struct {
+	mu   sync.Mutex
+	last map[string]vmStatus
+}
+func newStatusTracker() *statusTracker {
+	return &statusTracker{last: map[string]vmStatus{}}
+}
+// writeThrough runs the decide→durable-write→commit sequence for one VM under a
+// single lock, so it is ATOMIC per VM: concurrent reports for the same host
+// (e.g. an agent reconnect where the old and new sessions both deliver a report)
+// cannot interleave the write and the cache update and leave the cache
+// disagreeing with the row — a divergence that would then suppress every later
+// identical report indefinitely.
+//
+// write performs the durable RecordVMStatus call; it runs ONLY when the durable
+// triple (status, last_error, effective assigned_ip) changed, and the cache is
+// updated ONLY if write returns nil, so a rejected write never suppresses the
+// next retry. write receives no arguments: the caller passes the RAW reported ip
+// to RecordVMStatus (whose UPDATE keeps the prior ip on empty), while the cache
+// stores the effective ip computed here.
+//
+// The effective ip mirrors RecordVMStatus's UPDATE SET clause
+// `assigned_ip = CASE WHEN ?=” THEN assigned_ip ELSE ? END`: an EMPTY reported
+// ip keeps the prior stored ip (prior.ip=="" when there is no prior entry); a
+// non-empty ip replaces it.
+func (t *statusTracker) writeThrough(vmID, status, lastErr, ip string, write func() error) error {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+	prior, ok := t.last[vmID]
+	effIP := ip
+	if effIP == "" {
+		effIP = prior.ip
+	}
+	if ok && prior.status == status && prior.lastErr == lastErr && prior.ip == effIP {
+		return nil // unchanged — skip the write
+	}
+	if err := write(); err != nil {
+		return err // rejected — do not cache; let the next report retry
+	}
+	t.last[vmID] = vmStatus{status: status, lastErr: lastErr, ip: effIP}
+	return nil
+}
+// forget drops a VM's cached state to bound memory. Called when the agent acks
+// a VM as destroyed (the row is being hard-deleted).
+func (t *statusTracker) forget(vmID string) {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+	delete(t.last, vmID)
+}

internal/server/syncsvc/tracker_test.go

@@ @@ -0,0 +1,153 @@ @@
+package syncsvc
+import (
+	"errors"
+	"sync"
+	"testing"
+	"github.com/stretchr/testify/require"
+)
+// write is a tiny helper that runs writeThrough with a SUCCESSFUL durable write,
+// returning whether a write actually happened (the triple changed).
+func (t *statusTracker) write(vmID, status, lastErr, ip string) bool {
+	wrote := false
+	_ = t.writeThrough(vmID, status, lastErr, ip, func() error { wrote = true; return nil })
+	return wrote
+}
+func TestStatusTrackerDecide(t *testing.T) {
+	tr := newStatusTracker()
+	// First call for a VM must write.
+	if !tr.write("vm1", "ready", "", "10.0.0.1") {
+		t.Fatal("first call must write")
+	}
+	// Identical repeat must not write.
+	if tr.write("vm1", "ready", "", "10.0.0.1") {
+		t.Fatal("identical repeat must not write")
+	}
+	// Changed phase must write.
+	if !tr.write("vm1", "failed", "", "10.0.0.1") {
+		t.Fatal("changed phase must write")
+	}
+	// Changed lastError must write.
+	if !tr.write("vm1", "failed", "boom", "10.0.0.1") {
+		t.Fatal("changed lastError must write")
+	}
+	// Changed non-empty ip must write.
+	if !tr.write("vm1", "failed", "boom", "10.0.0.2") {
+		t.Fatal("changed non-empty ip must write")
+	}
+	// ip="" with same phase/err keeps the prior ip → unchanged → no write.
+	if tr.write("vm1", "failed", "boom", "") {
+		t.Fatal("empty ip keeps prior; unchanged must not write")
+	}
+}
+func TestStatusTrackerEmptyThenNonEmptyIP(t *testing.T) {
+	tr := newStatusTracker()
+	// First report has NO ip: effective ip is empty.
+	if !tr.write("vm1", "ready", "", "") {
+		t.Fatal("first call must write")
+	}
+	// Same phase/err, still empty ip: unchanged (empty keeps prior empty).
+	if tr.write("vm1", "ready", "", "") {
+		t.Fatal("repeat empty-ip must not write")
+	}
+	// Later a non-empty ip arrives with same phase/err: this is a change.
+	if !tr.write("vm1", "ready", "", "10.0.0.5") {
+		t.Fatal("empty→non-empty ip must write")
+	}
+	// Now an empty ip keeps that prior non-empty ip → unchanged.
+	if tr.write("vm1", "ready", "", "") {
+		t.Fatal("empty ip after non-empty keeps prior; must not write")
+	}
+}
+func TestStatusTrackerFailedWriteNotCached(t *testing.T) {
+	tr := newStatusTracker()
+	// The durable write FAILS: writeThrough must surface the error and NOT cache.
+	sentinel := errors.New("record rejected")
+	if err := tr.writeThrough("vm1", "ready", "", "10.0.0.1", func() error { return sentinel }); err != sentinel {
+		t.Fatalf("failed write must surface its error, got %v", err)
+	}
+	// Because the write failed (uncached), the next report must still attempt it —
+	// a failed write must never suppress the retry.
+	attempted := false
+	if err := tr.writeThrough("vm1", "ready", "", "10.0.0.1", func() error { attempted = true; return nil }); err != nil {
+		t.Fatalf("retry write errored: %v", err)
+	}
+	if !attempted {
+		t.Fatal("failed (uncached) write must not suppress the retry")
+	}
+	// After that successful write, an identical report must not write again.
+	if tr.write("vm1", "ready", "", "10.0.0.1") {
+		t.Fatal("after a successful write an identical report must not write")
+	}
+}
+// TestStatusTrackerWriteThroughAtomic pins the invariant Fix D restores: under
+// concurrent conflicting reports for one VM, the cached triple always equals the
+// LAST durable write. writeThrough holds the lock across write+commit, so the
+// two can never be reordered across goroutines and leave the cache disagreeing
+// with the row (the divergence that would suppress every later report). Runs
+// under -race to exercise the interleavings.
+func TestStatusTrackerWriteThroughAtomic(t *testing.T) {
+	tr := newStatusTracker()
+	var mu sync.Mutex
+	lastWritten := "" // status of the most recent successful durable write
+	writer := func(status string) {
+		_ = tr.writeThrough("vm1", status, "", "10.0.0.1", func() error {
+			mu.Lock()
+			lastWritten = status // recorded inside writeThrough's critical section
+			mu.Unlock()
+			return nil
+		})
+	}
+	var wg sync.WaitGroup
+	for i := 0; i < 300; i++ {
+		wg.Add(2)
+		go func() { defer wg.Done(); writer("ready") }()
+		go func() { defer wg.Done(); writer("failed") }()
+	}
+	wg.Wait()
+	tr.mu.Lock()
+	cached := tr.last["vm1"].status
+	tr.mu.Unlock()
+	mu.Lock()
+	dbLast := lastWritten
+	mu.Unlock()
+	require.Equal(t, dbLast, cached, "cache must equal the last durable write")
+}
+func TestStatusTrackerForget(t *testing.T) {
+	tr := newStatusTracker()
+	if !tr.write("vm1", "ready", "", "10.0.0.1") {
+		t.Fatal("first call must write")
+	}
+	tr.forget("vm1")
+	// After forget the VM is unknown again, so the same report writes afresh.
+	if !tr.write("vm1", "ready", "", "10.0.0.1") {
+		t.Fatal("after forget the same report must write again")
+	}
+}
+func TestStatusTrackerPerVMKeys(t *testing.T) {
+	tr := newStatusTracker()
+	if !tr.write("vm1", "ready", "", "10.0.0.1") {
+		t.Fatal("vm1 first write")
+	}
+	// A different VM with the same triple is independent → must write.
+	if !tr.write("vm2", "ready", "", "10.0.0.1") {
+		t.Fatal("vm2 is a distinct key and must write")
+	}
+	if tr.write("vm1", "ready", "", "10.0.0.1") {
+		t.Fatal("vm1 unchanged must not write")
+	}
+}

internal/server/web/spa_test.go

Old		New
@@ -31,8 +31,8 @@ func TestSPAServesIndexAtRoot(t *testing.T) {
31		31
32	func TestSPAServesRealAsset(t *testing.T) {	32	func TestSPAServesRealAsset(t *testing.T) {
33	h := spaHandler(fstest.MapFS{	33	h := spaHandler(fstest.MapFS{
34	"index.html": {Data: []byte("index")},	34	"index.html": {Data: []byte("index")},
35	"_app/app.js": {Data: []byte("console.log(1)")},	35	"_app/app.js": {Data: []byte("console.log(1)")},
36	})	36	})
37	code, body := get(t, h, "/_app/app.js")	37	code, body := get(t, h, "/_app/app.js")
38	assert.Equal(t, http.StatusOK, code)	38	assert.Equal(t, http.StatusOK, code)

internal/shape/classify.go

Old		New
@@ -33,7 +33,8 @@ func classify(rel string) Plane {
33	strings.HasPrefix(rel, "internal/transport"),	33	strings.HasPrefix(rel, "internal/transport"),
34	strings.HasPrefix(rel, "internal/joinblob"),	34	strings.HasPrefix(rel, "internal/joinblob"),
35	strings.HasPrefix(rel, "internal/cloudinit"),	35	strings.HasPrefix(rel, "internal/cloudinit"),
36	strings.HasPrefix(rel, "internal/names"):	36	strings.HasPrefix(rel, "internal/names"),
		37	strings.HasPrefix(rel, "internal/random"):
37	return PlaneWire	38	return PlaneWire
38	case strings.HasPrefix(rel, "cmd/"):	39	case strings.HasPrefix(rel, "cmd/"):
39	return PlaneBinaries	40	return PlaneBinaries

internal/shape/classify_test.go

Old		New
@@ -29,6 +29,7 @@ func TestClassifyAssignsPlaneByPrefix(t *testing.T) {
29	"internal/pb": PlaneWire,	29	"internal/pb": PlaneWire,
30	"internal/transport": PlaneWire,	30	"internal/transport": PlaneWire,
31	"internal/names": PlaneWire,	31	"internal/names": PlaneWire,
		32	"internal/random": PlaneWire,
32	"cmd/eitri-server": PlaneBinaries,	33	"cmd/eitri-server": PlaneBinaries,
33	"cmd/eitri-shape": PlaneBinaries,	34	"cmd/eitri-shape": PlaneBinaries,
34	"internal/arch": PlaneTooling,	35	"internal/arch": PlaneTooling,

internal/shape/generate.go

Old		New
@@ -3,7 +3,7 @@ package shape
3	// Generate runs the real `go list`, builds the model, and returns its JSON	3	// Generate runs the real `go list`, builds the model, and returns its JSON
4	// rendering. It is the one entry point cmd/eitri-shape calls.	4	// rendering. It is the one entry point cmd/eitri-shape calls.
5	func Generate() ([]byte, error) {	5	func Generate() ([]byte, error) {
6	raw, err := goListLister()()	6	raw, err := goList()
7	if err != nil {	7	if err != nil {
8	return nil, err	8	return nil, err
9	}	9	}

internal/shape/golist.go

Old		New
@@ -8,29 +8,27 @@ import (
8	"os/exec"	8	"os/exec"
9	)	9	)
10		10
11	// goListLister returns a lister backed by `go list -json` over this module's	11	// goList returns the raw package list for the module via `go list -json` over
12	// internal/... and cmd/... packages. GOOS is pinned to linux: the stack is	12	// its internal/... and cmd/... packages. GOOS is pinned to linux: the stack is
13	// Linux-only (netlink, cloud-hypervisor), so pinning keeps output identical	13	// Linux-only (netlink, cloud-hypervisor), so pinning keeps output identical
14	// regardless of the contributor's OS. `go list` only resolves build	14	// regardless of the contributor's OS. `go list` only resolves build
15	// constraints here — it does not compile — so this is safe cross-platform.	15	// constraints here — it does not compile — so this is safe cross-platform.
16	func goListLister() lister {	16	func goList() ([]rawPackage, error) {
17	return func() ([]rawPackage, error) {	17	cmd := exec.Command("go", "list", "-json",
18	cmd := exec.Command("go", "list", "-json",	18	module+"/internal/...", module+"/cmd/...")
19	module+"/internal/...", module+"/cmd/...")	19	cmd.Env = append(os.Environ(), "GOOS=linux")
20	cmd.Env = append(os.Environ(), "GOOS=linux")	20	out, err := cmd.Output()
21	out, err := cmd.Output()	21	if err != nil {
22	if err != nil {	22	return nil, fmt.Errorf("go list: %w", err)
23	return nil, fmt.Errorf("go list: %w", err)	23	}
24	}	24	var pkgs []rawPackage
25	var pkgs []rawPackage	25	dec := json.NewDecoder(bytes.NewReader(out))
26	dec := json.NewDecoder(bytes.NewReader(out))	26	for dec.More() {
27	for dec.More() {	27	var p rawPackage
28	var p rawPackage	28	if err := dec.Decode(&p); err != nil {
29	if err := dec.Decode(&p); err != nil {	29	return nil, fmt.Errorf("decode go list output: %w", err)
30	return nil, fmt.Errorf("decode go list output: %w", err)
31	}
32	pkgs = append(pkgs, p)
33	}	30	}
34	return pkgs, nil	31	pkgs = append(pkgs, p)
35	}	32	}
		33	return pkgs, nil
36	}	34	}

internal/shape/golist_test.go

Old		New
@@ -6,7 +6,7 @@ import "testing"
6	// If a new top-level package appears that matches no rule in classify(), it	6	// If a new top-level package appears that matches no rule in classify(), it
7	// lands in Unclassified and this fails — forcing a deliberate classification.	7	// lands in Unclassified and this fails — forcing a deliberate classification.
8	func TestNoUnclassifiedPackagesInModule(t *testing.T) {	8	func TestNoUnclassifiedPackagesInModule(t *testing.T) {
9	raw, err := goListLister()()	9	raw, err := goList()
10	if err != nil {	10	if err != nil {
11	t.Fatalf("go list: %v", err)	11	t.Fatalf("go list: %v", err)
12	}	12	}

internal/shape/shape.go

Old		New
@@ -25,11 +25,6 @@ type rawPackage struct {
25	Imports []string	25	Imports []string
26	}	26	}
27		27
28	// lister returns the raw package list for the module. It is a concrete function
29	// type (not an interface) so injecting a fake in tests does not trip the
30	// block-tier ireturn linter, and so production code can shell out to `go list`.
31	type lister func() ([]rawPackage, error)
32
33	// Package is one node in the shape graph. All string fields are module-relative.	28	// Package is one node in the shape graph. All string fields are module-relative.
34	type Package struct {	29	type Package struct {
35	ImportPath string `json:"importPath"`	30	ImportPath string `json:"importPath"`

internal/transport/tlsconf.go

Old		New
@@ -13,6 +13,8 @@ import (
13	"fmt"	13	"fmt"
14	"math/big"	14	"math/big"
15	"time"	15	"time"
		16
		17	"github.com/quic-go/quic-go"
16	)	18	)
17		19
18	// ALPN is the QUIC application-layer protocol token. Both ends MUST set it; the	20	// ALPN is the QUIC application-layer protocol token. Both ends MUST set it; the
@@ -25,6 +27,28 @@ const (
25	CodeAuthRejected = 1 // permanent: bad/expired host credential — do not tight-loop	27	CodeAuthRejected = 1 // permanent: bad/expired host credential — do not tight-loop
26	)	28	)
27		29
		30	// Sync QUIC keepalive/idle timings. The agent dialer and the server listener
		31	// MUST build their quic.Config from SyncQUICConfig: the effective idle timeout
		32	// is the min of the two peers, so tuning one side alone silently shortens it.
		33	// 10s keepalive under a 60s idle gives ~5 keepalive attempts before teardown,
		34	// so a single missed PING or a brief GC/stall on either side does not kill a
		35	// healthy control-plane session (the old 15s/30s allowed only one attempt and
		36	// caused reconnect churn).
		37	const (
		38	SyncKeepAlivePeriod = 10 * time.Second
		39	SyncMaxIdleTimeout = 60 * time.Second
		40	)
		41
		42	// SyncQUICConfig returns the QUIC transport config shared by the eitri-agent
		43	// dialer and the eitri-server listener. Colocating it with ALPN and the TLS
		44	// builders is what keeps the two ends from drifting.
		45	func SyncQUICConfig() *quic.Config {
		46	return &quic.Config{
		47	KeepAlivePeriod: SyncKeepAlivePeriod,
		48	MaxIdleTimeout: SyncMaxIdleTimeout,
		49	}
		50	}
		51
28	// GenerateServerCert returns a fresh self-signed ECDSA cert+key as PEM.	52	// GenerateServerCert returns a fresh self-signed ECDSA cert+key as PEM.
29	func GenerateServerCert() (certPEM, keyPEM []byte, err error) {	53	func GenerateServerCert() (certPEM, keyPEM []byte, err error) {
30	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)	54	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)

scripts/deploy.env.example

Old		New
@@ -21,9 +21,13 @@ AGENT_LOG="/var/lib/eitri-agent/agent.log"
21		21
22	# Agent launch flags shared by all hosts.	22	# Agent launch flags shared by all hosts.
23	CH_BIN="/usr/local/bin/cloud-hypervisor"	23	CH_BIN="/usr/local/bin/cloud-hypervisor"
24	FIRMWARE="/usr/share/eitri/hypervisor-fw"	24	# FIRMWARE is the guest firmware path ON each host. UEFI (CLOUDHV.fd) boots the
25	MESH_BIN_URL="http://192.168.0.190:8090/ray.bin"	25	# guest's own GRUB -> guest kernel+initrd from disk (modern images boot).
26	MESH_BIN_SHA256="e806d523cf50bec454ef299be3bb5b8aad123ded93911fa86a66d70341915d05"	26	FIRMWARE="/usr/share/eitri/CLOUDHV.fd"
		27	# FIRMWARE_SRC (optional): a LOCAL firmware file to scp+install to $FIRMWARE on
		28	# every host during deploy. Set it to roll a firmware change with the deploy;
		29	# leave unset to keep whatever firmware already exists on each host.
		30	# FIRMWARE_SRC="$HOME/.cache/eitri/CLOUDHV.fd"
27		31
28	# Optional overrides (leave unset to use the agent's built-in defaults).	32	# Optional overrides (leave unset to use the agent's built-in defaults).
29	# TOMBSTONE_GRACE — how long a deleted VM lingers (stopped) before destroy.	33	# TOMBSTONE_GRACE — how long a deleted VM lingers (stopped) before destroy.

scripts/deploy.sh

Old		New
@@ -57,8 +57,8 @@ install -m 0755 bin/eitri-server "$SERVER_BIN"
57	setsid "$SERVER_BIN" --config "$SERVER_CONFIG" </dev/null >>"$SERVER_LOG" 2>&1 &	57	setsid "$SERVER_BIN" --config "$SERVER_CONFIG" </dev/null >>"$SERVER_LOG" 2>&1 &
58	# Liveness gate: /livez is 200 as soon as the HTTP mux serves — an honest	58	# Liveness gate: /livez is 200 as soon as the HTTP mux serves — an honest
59	# "process is up" signal (unlike GET /, which returns the SPA even before the	59	# "process is up" signal (unlike GET /, which returns the SPA even before the
60	# app is wired). Readiness (DB + warden) is gated separately below, after the	60	# app is wired). Readiness (DB) is gated separately below, after the agents
61	# agents have had a chance to redial.	61	# have had a chance to redial.
62	for _ in $(seq 1 30); do	62	for _ in $(seq 1 30); do
63	curl -fsS -o /dev/null "$SERVER_URL/livez" 2>/dev/null && break	63	curl -fsS -o /dev/null "$SERVER_URL/livez" 2>/dev/null && break
64	sleep 0.5	64	sleep 0.5
@@ -82,6 +82,14 @@ for entry in $AGENT_HOSTS; do
82	port="${entry##*:}"; [[ "$port" == "$entry" ]] && port=22	82	port="${entry##*:}"; [[ "$port" == "$entry" ]] && port=22
83	bold "Rolling agent -> $userhost (port $port)"	83	bold "Rolling agent -> $userhost (port $port)"
84	scp -q -P "$port" -o ConnectTimeout=10 bin/eitri-agent "$userhost:/tmp/eitri-agent-new"	84	scp -q -P "$port" -o ConnectTimeout=10 bin/eitri-agent "$userhost:/tmp/eitri-agent-new"
		85	# Optionally ship the guest firmware (e.g. CLOUDHV.fd) so a firmware change
		86	# rolls WITH the deploy instead of needing a manual per-host install. When
		87	# FIRMWARE_SRC is unset, hosts keep whatever firmware is already at $FIRMWARE.
		88	fw_install=""
		89	if [[ -n "${FIRMWARE_SRC:-}" ]]; then
		90	scp -q -P "$port" -o ConnectTimeout=10 "$FIRMWARE_SRC" "$userhost:/tmp/eitri-firmware-new"
		91	fw_install="sudo install -Dm0644 /tmp/eitri-firmware-new \"$FIRMWARE\" && rm -f /tmp/eitri-firmware-new"
		92	fi
85	# Unquoted heredoc: local vars expand here; \$(...) runs on the remote.	93	# Unquoted heredoc: local vars expand here; \$(...) runs on the remote.
86	ssh -p "$port" -o BatchMode=yes "$userhost" bash -s <<REMOTE	94	ssh -p "$port" -o BatchMode=yes "$userhost" bash -s <<REMOTE
87	set -e	95	set -e
@@ -90,6 +98,7 @@ for _ in \$(seq 1 20); do pgrep -x eitri-agent >/dev/null \|\| break; sleep 0.5; d
90	if pgrep -x eitri-agent >/dev/null; then echo "agent did not stop on \$(hostname)" >&2; exit 1; fi	98	if pgrep -x eitri-agent >/dev/null; then echo "agent did not stop on \$(hostname)" >&2; exit 1; fi
91	sudo install -m 0755 /tmp/eitri-agent-new "$AGENT_BIN"	99	sudo install -m 0755 /tmp/eitri-agent-new "$AGENT_BIN"
92	rm -f /tmp/eitri-agent-new	100	rm -f /tmp/eitri-agent-new
		101	$fw_install
93	sudo bash -c "setsid $AGENT_BIN $agent_flags </dev/null >>$AGENT_LOG 2>&1 &"	102	sudo bash -c "setsid $AGENT_BIN $agent_flags </dev/null >>$AGENT_LOG 2>&1 &"
94	sleep 2	103	sleep 2
95	pgrep -x eitri-agent >/dev/null \|\| { echo "agent FAILED to start on \$(hostname) (see $AGENT_LOG)" >&2; exit 1; }	104	pgrep -x eitri-agent >/dev/null \|\| { echo "agent FAILED to start on \$(hostname) (see $AGENT_LOG)" >&2; exit 1; }
@@ -99,10 +108,9 @@ done
99		108
100	# ── 3. Post-deploy verification ───────────────────────────────────────────────	109	# ── 3. Post-deploy verification ───────────────────────────────────────────────
101	bold "Verifying fleet on $SHA"	110	bold "Verifying fleet on $SHA"
102	# Readiness gate: /readyz 200s only when the server's dependencies (DB open +	111	# Readiness gate: /readyz 200s only when the server's dependencies (DB open) are
103	# warden daemon reachable) are healthy. Poll briefly — the warden socket is	112	# healthy. Poll briefly — the DB is local and up whenever the box is, so this
104	# local and up whenever the box is, so this settles fast; a persistent 503	113	# settles fast; a persistent 503 names the failed dependency in its JSON body.
105	# names the failed dependency in its JSON body.
106	ready=""	114	ready=""
107	for _ in $(seq 1 15); do	115	for _ in $(seq 1 15); do
108	ready="$(curl -fsS "$SERVER_URL/readyz" 2>/dev/null)" && break	116	ready="$(curl -fsS "$SERVER_URL/readyz" 2>/dev/null)" && break
@@ -136,4 +144,27 @@ if [[ -n "${ADMIN_TOKEN_FILE:-}" && -f "$ADMIN_TOKEN_FILE" ]]; then
136	else	144	else
137	echo "(no ADMIN_TOKEN_FILE set — skipping API verification)"	145	echo "(no ADMIN_TOKEN_FILE set — skipping API verification)"
138	fi	146	fi
		147
		148	# ── 4. Boot gate ──────────────────────────────────────────────────────────────
		149	# readyz + hosts-online prove the CONTROL PLANE is up, but a deploy can still
		150	# ship a change that breaks VM BOOT (e.g. an unknown cloud-hypervisor --disk
		151	# option that makes CH refuse to start — exactly how the image_type=raw
		152	# regression reached the fleet). The only way to catch that is to boot a real
		153	# VM. This gate does so and FAILS the deploy if it can't. Set SKIP_BOOT_SMOKE=1
		154	# to bypass for a server-only / docs-only change.
		155	if [[ "${SKIP_BOOT_SMOKE:-0}" == "1" ]]; then
		156	echo "(boot gate skipped: SKIP_BOOT_SMOKE=1)"
		157	elif [[ -n "${ADMIN_TOKEN_FILE:-}" ]]; then
		158	bold "Boot gate: create a throwaway VM and confirm it actually boots"
		159	if EITRI_DEPLOY_ENV="$ENV_FILE" "$REPO_ROOT/scripts/remote-smoke.sh"; then
		160	echo "boot gate: PASS"
		161	else
		162	echo "DEPLOY FAILED boot gate: a VM did not boot after this deploy." >&2
		163	echo " The fleet is likely serving a boot-breaking change — investigate before relying on it." >&2
		164	exit 1
		165	fi
		166	else
		167	echo "(boot gate skipped — needs ADMIN_TOKEN_FILE)"
		168	fi
		169
139	bold "Deploy complete: $SHA"	170	bold "Deploy complete: $SHA"

scripts/remote-smoke.sh

@@ @@ -0,0 +1,101 @@ @@
+#!/usr/bin/env bash
+# Remote smoke: validate the LIVE fleet end-to-end. Creates a VM through the real
+# API, confirms the GUEST actually booted (reads its serial console on the host),
+# optionally checks reachability, then reaps it and verifies the hard-delete.
+#
+# Unlike scripts/smoke.sh this needs NO local privilege — the agent host does the
+# privileged tap/nft work. Config is sourced from $EITRI_DEPLOY_ENV (the same
+# file deploy.sh uses); the admin token is read from $ADMIN_TOKEN_FILE and never
+# printed.
+set -euo pipefail
+ENV_FILE="${EITRI_DEPLOY_ENV:-$HOME/eitri-deploy/deploy.env}"
+[[ -f "$ENV_FILE" ]] || { echo "remote-smoke: config not found: $ENV_FILE" >&2; exit 1; }
+# shellcheck disable=SC1090
+source "$ENV_FILE"
+: "${SERVER_URL:?}" "${ADMIN_TOKEN_FILE:?}" "${AGENT_HOSTS:?}" "${AGENT_STATE_DIR:?}"
+tok="$(cat "$ADMIN_TOKEN_FILE")"
+api() { curl -fsS -H "Authorization: Bearer $tok" "$@"; }
+say() { printf '\n\033[1m== %s ==\033[0m\n' "$*"; }
+# First agent host = where we read the serial console.
+entry="${AGENT_HOSTS%% *}"
+userhost="${entry%%:*}"; port="${entry##*:}"; [[ "$port" == "$entry" ]] && port=22
+ssh_host() { ssh -p "$port" -o BatchMode=yes -o ConnectTimeout=10 "$userhost" "$@"; }
+# Echo the requested fields (space-separated) for THIS VM from the /vms list in a
+# single query; each field is '' when the VM is absent or the value is null. Used
+# by the ready-poll (phase + ip together, so no second fetch on the ready
+# iteration) and the reap-poll (presence check via the id field).
+vm_fields() {
+  api "$SERVER_URL/api/v1/vms" | python3 -c "
+import sys,json
+want='$*'.split()
+vm=next((v for v in json.load(sys.stdin) if v['id']=='$VM_ID'), {})
+print(' '.join('' if vm.get(f) is None else str(vm.get(f)) for f in want))
+"
+}
+say "Fleet"
+api "$SERVER_URL/api/v1/hosts" | python3 -m json.tool
+HOST_ID=$(api "$SERVER_URL/api/v1/hosts" \
+  | python3 -c 'import sys,json; print(json.load(sys.stdin)[0]["id"])')
+say "Create VM"
+SSH_KEY=""
+for f in ~/.ssh/id_ed25519.pub ~/.ssh/id_rsa.pub; do
+  [[ -f "$f" ]] && { SSH_KEY=$(cat "$f"); break; }
+done
+START=$(date +%s)
+VM_ID=$(api -X POST "$SERVER_URL/api/v1/vms" -H 'Content-Type: application/json' \
+  -d "{\"host_id\":\"$HOST_ID\",\"ssh_authorized_key\":\"$SSH_KEY\"}" \
+  | python3 -c 'import sys,json; print(json.load(sys.stdin)["id"])')
+echo "VM id: $VM_ID"
+say "Wait for ready (first run downloads the image — allow a few minutes)"
+DEADLINE=$(( $(date +%s) + 600 )); IP=""; PHASE=""
+while [[ "$(date +%s)" -lt "$DEADLINE" ]]; do
+  read -r PHASE IP <<<"$(vm_fields phase assigned_ip)"
+  [[ "$PHASE" == "ready" ]] && break
+  echo "  phase=$PHASE — waiting..."
+  sleep 5
+done
+COLD=$(( $(date +%s) - START ))
+[[ "$PHASE" == "ready" && -n "$IP" ]] || { echo "FAIL: VM not ready within 600s (phase=$PHASE)"; exit 1; }
+echo "ready: ip=$IP  cold_start=${COLD}s"
+# Boot GATE — 'ready' only means CH is up + IP assigned; it does NOT mean the
+# guest booted (a bad firmware/image leaves CH running but the guest dead). So
+# poll the guest serial console until we see real userspace, and FAIL on a
+# kernel panic or a 180s timeout. This is the assertion that the UEFI ->
+# guest-GRUB -> guest-kernel chain actually worked.
+say "Boot proof — poll guest serial on $userhost until userspace"
+SERIAL="$AGENT_STATE_DIR/vms/$VM_ID/serial.log"
+BOOT_DEADLINE=$(( $(date +%s) + 180 )); BOOTED=""
+while [[ "$(date +%s)" -lt "$BOOT_DEADLINE" ]]; do
+  serial="$(ssh_host "sudo cat '$SERIAL' 2>/dev/null | tr -cd '\\11\\12\\15\\40-\\176'" 2>/dev/null || true)"
+  if grep -qaiE 'Kernel panic|Cannot open root' <<<"$serial"; then
+    echo "FAIL: guest panic / root-mount failure:"; grep -aiE 'panic|Cannot open root' <<<"$serial" | tail -5
+    exit 1
+  fi
+  if grep -qaiE 'Welcome to.*Ubuntu|login:|Reached target.*Multi-User' <<<"$serial"; then
+    BOOTED=1
+    grep -aiE 'BdsDxe|Linux version|Welcome to.*Ubuntu|login:' <<<"$serial" | head -6
+    break
+  fi
+  echo "  guest still booting..."; sleep 6
+done
+[[ -n "$BOOTED" ]] || { echo "FAIL: no userspace boot evidence in serial within 180s"; exit 1; }
+say "Reap — verify quarantine -> destroy -> hard-delete"
+api -X DELETE "$SERVER_URL/api/v1/vms/$VM_ID" >/dev/null
+REAP_DEADLINE=$(( $(date +%s) + 120 )); REAPED=""
+while [[ "$(date +%s)" -lt "$REAP_DEADLINE" ]]; do
+  [[ -z "$(vm_fields id)" ]] && { REAPED=1; break; }
+  echo "  vm still present — waiting for destroy+ack..."
+  sleep 5
+done
+[[ -n "$REAPED" ]] || { echo "FAIL: VM not hard-deleted within 120s of tombstone"; exit 1; }
+say "REMOTE SMOKE COMPLETE — booted under UEFI, cold_start=${COLD}s, reaped OK"

web/package-lock.json

Old		New
@@ -15,6 +15,7 @@
15	"@sveltejs/adapter-static": "^3.0.10",	15	"@sveltejs/adapter-static": "^3.0.10",
16	"@sveltejs/kit": "^2.63.0",	16	"@sveltejs/kit": "^2.63.0",
17	"@sveltejs/vite-plugin-svelte": "^7.1.2",	17	"@sveltejs/vite-plugin-svelte": "^7.1.2",
		18	"@types/node": "^26.1.1",
18	"openapi-typescript": "^7.13.0",	19	"openapi-typescript": "^7.13.0",
19	"svelte": "^5.56.1",	20	"svelte": "^5.56.1",
20	"svelte-check": "^4.6.0",	21	"svelte-check": "^4.6.0",
@@ -629,6 +630,16 @@
629	"dev": true,	630	"dev": true,
630	"license": "MIT"	631	"license": "MIT"
631	},	632	},
		633	"node_modules/@types/node": {
		634	"version": "26.1.1",
		635	"resolved": "https://registry.npmjs.org/@types/node/-/node-26.1.1.tgz",
		636	"integrity": "sha512-nxAkRSVkN1Y0JC1W8ky/fTfkGsMmcrRsbx+3XoZE+rMOX71kLYTV7fLXpqud1GpbpP5TuffXFqfX7fH2GgZREw==",
		637	"dev": true,
		638	"license": "MIT",
		639	"dependencies": {
		640	"undici-types": "~8.3.0"
		641	}
		642	},
632	"node_modules/@types/trusted-types": {	643	"node_modules/@types/trusted-types": {
633	"version": "2.0.7",	644	"version": "2.0.7",
634	"resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.7.tgz",	645	"resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.7.tgz",
@@ -1668,6 +1679,13 @@
1668	"node": ">=14.17"	1679	"node": ">=14.17"
1669	}	1680	}
1670	},	1681	},
		1682	"node_modules/undici-types": {
		1683	"version": "8.3.0",
		1684	"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.3.0.tgz",
		1685	"integrity": "sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ==",
		1686	"dev": true,
		1687	"license": "MIT"
		1688	},
1671	"node_modules/uri-js-replace": {	1689	"node_modules/uri-js-replace": {
1672	"version": "1.0.1",	1690	"version": "1.0.1",
1673	"resolved": "https://registry.npmjs.org/uri-js-replace/-/uri-js-replace-1.0.1.tgz",	1691	"resolved": "https://registry.npmjs.org/uri-js-replace/-/uri-js-replace-1.0.1.tgz",

web/package.json

Old		New
@@ -22,6 +22,7 @@
22	"@sveltejs/adapter-static": "^3.0.10",	22	"@sveltejs/adapter-static": "^3.0.10",
23	"@sveltejs/kit": "^2.63.0",	23	"@sveltejs/kit": "^2.63.0",
24	"@sveltejs/vite-plugin-svelte": "^7.1.2",	24	"@sveltejs/vite-plugin-svelte": "^7.1.2",
		25	"@types/node": "^26.1.1",
25	"openapi-typescript": "^7.13.0",	26	"openapi-typescript": "^7.13.0",
26	"svelte": "^5.56.1",	27	"svelte": "^5.56.1",
27	"svelte-check": "^4.6.0",	28	"svelte-check": "^4.6.0",

web/src/lib/api-types.ts

Old		New
@@ -916,10 +916,15 @@ export interface components {
916	/** Format: date-time */	916	/** Format: date-time */
917	enrolled_at: string;	917	enrolled_at: string;
918	id: string;	918	id: string;
		919	/** Format: date-time */
		920	last_seen?: string \| null;
919	name: string;	921	name: string;
920	online: boolean;	922	online: boolean;
921	os: string;	923	os: string;
922	provisioner: string;	924	provisioner: string;
		925	seconds_since_last_seen?: number \| null;
		926	sessions: number;
		927	stale: boolean;
923	status: string;	928	status: string;
924	};	929	};
925	PatchVMRequest: {	930	PatchVMRequest: {

Old		New
@@ -138,16 +138,14 @@ shape-check:
138	# entrypoint — every main() in cmd/. Rooting at the binaries (NOT -test) is what	138	# entrypoint — every main() in cmd/. Rooting at the binaries (NOT -test) is what
139	# catches production code kept alive only by its own tests; the fix is to remove	139	# catches production code kept alive only by its own tests; the fix is to remove
140	# it, wire it into a real path, or move it into a _test.go. The smoke/sandbox tags	140	# it, wire it into a real path, or move it into a _test.go. The smoke/sandbox tags
141	# compile the tag-gated code so it is analysed too. Three sanctioned exceptions,	141	# compile the tag-gated code so it is analysed too. Two sanctioned exceptions,
142	# all production code that only a CROSS-package test can reach (so none can be	142	# both production code that only a CROSS-package test can reach (so neither can be
143	# a _test.go): internal/integration (the e2e/harness tree), reconcile.Engine.Stop	143	# a _test.go): internal/integration (the e2e/harness tree) and reconcile.Engine.Stop
144	# (terminal teardown that must not run in production — it would report every VM as	144	# (terminal teardown that must not run in production — it would report every VM as
145	# vanished — used only by an integration test's cleanup), and	145	# vanished — used only by an integration test's cleanup).
146	# store.Store.AllocatedByHost (the single-tx Snapshot path now serves GET /hosts;
147	# the standalone accessor is exercised only by the store's own allocation tests).
148	deadcode:	146	deadcode:
149	@out=$$(go run golang.org/x/tools/cmd/deadcode@$(DEADCODE_VERSION) -tags=smoke,sandbox ./... \	147	@out=$$(go run golang.org/x/tools/cmd/deadcode@$(DEADCODE_VERSION) -tags=smoke,sandbox ./... \
150	\| { grep -vE '^internal/integration/\|unreachable func: Engine\.Stop$$\|unreachable func: Store\.AllocatedByHost$$' \|\| true; }); \	148	\| { grep -vE '^internal/integration/\|unreachable func: Engine\.Stop$$' \|\| true; }); \
151	if [ -n "$$out" ]; then \	149	if [ -n "$$out" ]; then \
152	echo "deadcode: unreachable from any cmd/ entrypoint (remove it, wire it in, or move it to a _test.go):"; \	150	echo "deadcode: unreachable from any cmd/ entrypoint (remove it, wire it in, or move it to a _test.go):"; \
153	echo "$$out"; exit 1; \	151	echo "$$out"; exit 1; \

Old		New
@@ -39,6 +39,7 @@ bridge IP (`assigned_ip`) via the agent.
39	\| R4 \| Pure domain packages (`agent/state`, `agent/seed`, `agent/ipalloc`, `server/registry`) don't depend on the transport stack (HTTP/QUIC/`transport`). `server/store` may use `transport` (cert helpers) but not HTTP/QUIC. \| `internal/arch` `TestDomainDoesNotImportTransportStack` + `depguard` `domain-no-transport`. \|	39	\| R4 \| Pure domain packages (`agent/state`, `agent/seed`, `agent/ipalloc`, `server/registry`) don't depend on the transport stack (HTTP/QUIC/`transport`). `server/store` may use `transport` (cert helpers) but not HTTP/QUIC. \| `internal/arch` `TestDomainDoesNotImportTransportStack` + `depguard` `domain-no-transport`. \|
40	\| R5 \| The reconcile boundary interfaces (`Provisioner`, `NetEnv`) stay consumer-owned and small; the IPAM seam (`NetEnv.AllocateIP`/`GuestNetwork`) is where a future central allocator plugs in. \| Convention (below) + `ireturn` allow-list keeps the seams' interface returns honest. \|	40	\| R5 \| The reconcile boundary interfaces (`Provisioner`, `NetEnv`) stay consumer-owned and small; the IPAM seam (`NetEnv.AllocateIP`/`GuestNetwork`) is where a future central allocator plugs in. \| Convention (below) + `ireturn` allow-list keeps the seams' interface returns honest. \|
41	\| R6 \| All external process execution in the data plane funnels through `agent/exec.Runner`. The sole exception is `agent/cloudhv`, which launches the long-lived cloud-hypervisor process directly. Checked transitively (reaching `os/exec` via the sanctioned `cloudhv` is fine). \| `internal/arch` `TestOnlyCloudhvImportsOsExecInDataPlane` (transitive). \|	41	\| R6 \| All external process execution in the data plane funnels through `agent/exec.Runner`. The sole exception is `agent/cloudhv`, which launches the long-lived cloud-hypervisor process directly. Checked transitively (reaching `os/exec` via the sanctioned `cloudhv` is fine). \| `internal/arch` `TestOnlyCloudhvImportsOsExecInDataPlane` (transitive). \|
		42	\| R7 \| `internal/integration/` is test infrastructure only — no package under `internal/server/`, `internal/agent/`, or `cmd/` may import it, even transitively. The sanctioned exceptions are the test-tooling launcher binaries that are the infrastructure's entry points: `cmd/eitri-smoketest`, `cmd/eitri-devstack`, `cmd/eitri-sandbox`. \| `internal/arch` `TestProductionPlanesDoNotImportIntegrationTestInfra` (transitive). \|
42		43
43	> The `internal/arch` tests shell out to `go list`, so Go's test cache can't see	44	> The `internal/arch` tests shell out to `go list`, so Go's test cache can't see
44	> edges changing elsewhere in the module. Always run them with `-count=1`	45	> edges changing elsewhere in the module. Always run them with `-count=1`
@@ -80,7 +81,7 @@ why the invariants hold:
80	\|------\|--------\|---------------\|	81	\|------\|--------\|---------------\|
81	\| Compile all packages \| `make build-go` \| yes \|	82	\| Compile all packages \| `make build-go` \| yes \|
82	\| `go vet` \| `make vet` \| yes \|	83	\| `go vet` \| `make vet` \| yes \|
83	\| Architecture fitness tests (R1–R6) \| `make arch` \| yes \|	84	\| Architecture fitness tests (R1–R7) \| `make arch` \| yes \|
84	\| Block-tier lint (boundaries + correctness) \| `make lint` \| yes \|	85	\| Block-tier lint (boundaries + correctness) \| `make lint` \| yes \|
85	\| Race-detector tests \| `make test` \| yes \|	86	\| Race-detector tests \| `make test` \| yes \|
86	\| Per-package coverage ratchet \| `make cover` \| yes \|	87	\| Per-package coverage ratchet \| `make cover` \| yes \|

Old		New
@@ -18,13 +18,13 @@ import (
18	"gopkg.in/yaml.v3"	18	"gopkg.in/yaml.v3"
19	)	19	)
20		20
21	// ErrNotCloudConfig reports user-data that is not a `#cloud-config` document —	21	// errNotCloudConfig reports user-data that is not a `#cloud-config` document —
22	// a shell script, a jinja-templated config, MIME multipart, etc. There is no	22	// a shell script, a jinja-templated config, MIME multipart, etc. There is no
23	// ssh_authorized_keys to merge into, so the caller must reject rather than	23	// ssh_authorized_keys to merge into, so the caller must reject rather than
24	// silently drop the key (the exact bug this package removes).	24	// silently drop the key (the exact bug this package removes).
25	var ErrNotCloudConfig = errors.New("user-data is not #cloud-config; cannot merge an ssh key into it")	25	var errNotCloudConfig = errors.New("user-data is not #cloud-config; cannot merge an ssh key into it")
26		26
27	// MergeSSHKey returns userData with key added to the top-level	27	// mergeSSHKey returns userData with key added to the top-level
28	// ssh_authorized_keys list — cloud-init's canonical way to add a key to the	28	// ssh_authorized_keys list — cloud-init's canonical way to add a key to the
29	// image's default user, robust whether or not the doc defines a `users:` block.	29	// image's default user, robust whether or not the doc defines a `users:` block.
30	// (An operator who REPLACES the default user with their own named user should	30	// (An operator who REPLACES the default user with their own named user should
@@ -44,9 +44,9 @@ var ErrNotCloudConfig = errors.New("user-data is not #cloud-config; cannot merge
44	// added as a YAML scalar node (not string-interpolated), so a value with	44	// added as a YAML scalar node (not string-interpolated), so a value with
45	// YAML-significant characters — or a bypassed multi-line value — is emitted as	45	// YAML-significant characters — or a bypassed multi-line value — is emitted as
46	// a quoted/block scalar and cannot inject structure.	46	// a quoted/block scalar and cannot inject structure.
47	func MergeSSHKey(userData, key string) (string, error) {	47	func mergeSSHKey(userData, key string) (string, error) {
48	if !isCloudConfig(userData) {	48	if !isCloudConfig(userData) {
49	return "", ErrNotCloudConfig	49	return "", errNotCloudConfig
50	}	50	}
51		51
52	dec := yaml.NewDecoder(strings.NewReader(userData))	52	dec := yaml.NewDecoder(strings.NewReader(userData))