15c5a499 — eitri

cmd/eitri-agent/main.go

Old		New
@@ -1,293 +1,23 @@
1	// eitri-agent: BYO-hardware agent. Enrolls the host via "join <blob>" and	1	// eitri-agent: BYO-hardware agent. Enrolls the host via "join <blob>" and then
2	// then runs the reconcile + sync loop indefinitely.	2	// runs the reconcile + sync loop indefinitely. All behavior lives in
		3	// internal/agent/run (RunCLI); this package is wiring only (arch R14).
3	package main	4	package main
4		5
5	import (	6	import (
6	"context"
7	"errors"
8	"flag"
9	"fmt"	7	"fmt"
10	"log/slog"
11	"os"	8	"os"
12	"os/exec"
13	"os/signal"
14	"runtime"
15	"syscall"
16	"time"
17		9
18	"github.com/a73x/eitri/internal/agent/bootstrap"	10	"github.com/a73x/eitri/internal/agent/run"
19	"github.com/a73x/eitri/internal/agent/cloudhv"
20	"github.com/a73x/eitri/internal/agent/enrollclient"
21	"github.com/a73x/eitri/internal/agent/imagecache"
22	"github.com/a73x/eitri/internal/agent/netenv"
23	"github.com/a73x/eitri/internal/agent/reconcile"
24	"github.com/a73x/eitri/internal/agent/seed"
25	"github.com/a73x/eitri/internal/agent/serialpump"
26	"github.com/a73x/eitri/internal/agent/state"
27	"github.com/a73x/eitri/internal/agent/syncclient"
28	"github.com/a73x/eitri/internal/covsnap"
29	"github.com/a73x/eitri/internal/joinblob"
30	"github.com/a73x/eitri/internal/version"	11	"github.com/a73x/eitri/internal/version"
31	)	12	)
32		13
33	// agentConfig carries runAgent's wiring, replacing a long positional list.
34	type agentConfig struct {
35	StateDir, CHBin, Firmware string
36	BootstrapURL string
37	TombstoneGrace, VanishGrace time.Duration
38	VMTimeout time.Duration
39	ImageCacheMaxGB int64
40	// MaxConcurrentCreates bounds how much of the per-VM workers' create
41	// concurrency reaches the disk at once (0 = unlimited).
42	MaxConcurrentCreates int
43	// MaxVCPUs, MaxMemMB, MaxDiskGB cap the host resources this agent offers the
44	// fleet (0 = unlimited): advertised to the server AND enforced at VM boot.
45	MaxVCPUs, MaxMemMB, MaxDiskGB int64
46	}
47
48	func main() {	14	func main() {
49	if len(os.Args) > 1 && os.Args[1] == "--version" {	15	if len(os.Args) > 1 && os.Args[1] == "--version" {
50	fmt.Println(version.Version)	16	fmt.Println(version.Version)
51	return	17	return
52	}	18	}
53	stateDir := flag.String("state-dir", "/var/lib/eitri-agent", "agent state directory")	19	if err := run.RunCLI(os.Args[1:]); err != nil {
54	chBin := flag.String("ch-bin", "cloud-hypervisor", "path to cloud-hypervisor binary")	20	fmt.Fprintln(os.Stderr, "eitri-agent:", err)
55	firmware := flag.String("firmware", "/usr/share/eitri/CLOUDHV.fd", "path to CH UEFI firmware (CLOUDHV.fd)")
56	bootstrapURL := flag.String("bootstrap-url", "https://eitri.sh/dl/latest/manifest.json", "eitri.sh release manifest to fetch cloud-hypervisor/firmware from if missing at startup (empty disables bootstrap)")
57	tombstoneGrace := flag.Duration("tombstone-grace", 5*time.Minute, "quarantine period for tombstoned VMs before destroy")
58	vanishGrace := flag.Duration("vanish-grace", time.Hour, "quarantine period for VMs that vanished without a tombstone")
59	vmTimeout := flag.Duration("vm-timeout", 15*time.Minute, "watchdog bound on ONE VM's reconcile pass (0 disables); keep above the 10m image-download timeout")
60	imageCacheMaxGB := flag.Int64("image-cache-max-gb", 20, "evict least-recently-used cached base images beyond this size (0 = never evict)")
61	maxConcurrentCreates := flag.Int("max-concurrent-creates", 4, "cap how many VMs may be inside the I/O-heavy part of create at once (image fetch + disk copy); 0 = unlimited")
62	maxVCPUs := flag.Int64("max-vcpus", 0, "cap the total vCPUs this host offers the fleet (0 = unlimited; reserves headroom, advertised + enforced at boot)")
63	maxMemMB := flag.Int64("max-mem-mb", 0, "cap the total memory (MB) this host offers the fleet (0 = unlimited)")
64	maxDiskGB := flag.Int64("max-disk-gb", 0, "cap the total disk (GB) this host offers the fleet (0 = unlimited)")
65	flag.Parse()
66
67	for name, v := range map[string]int64{"max-vcpus": maxVCPUs, "max-mem-mb": maxMemMB, "max-disk-gb": *maxDiskGB} {
68	if v < 0 {
69	slog.Error("resource cap must be >= 0 (0 = unlimited)", "flag", "--"+name, "value", v)
70	os.Exit(1)
71	}
72	}
73
74	st, err := state.Open(*stateDir)
75	if err != nil {
76	slog.Error("open state dir", "err", err)
77	os.Exit(1)
78	}
79
80	if flag.Arg(0) == "join" {
81	runJoin(st, flag.Arg(1))
82	return
83	}
84
85	runAgent(st, agentConfig{
86	StateDir: *stateDir,
87	CHBin: *chBin,
88	Firmware: *firmware,
89	BootstrapURL: *bootstrapURL,
90	TombstoneGrace: *tombstoneGrace,
91	VanishGrace: *vanishGrace,
92	VMTimeout: *vmTimeout,
93	ImageCacheMaxGB: *imageCacheMaxGB,
94	MaxConcurrentCreates: *maxConcurrentCreates,
95	MaxVCPUs: *maxVCPUs,
96	MaxMemMB: *maxMemMB,
97	MaxDiskGB: *maxDiskGB,
98	})
99	}
100
101	// runJoin handles the "join <blob>" subcommand: decode the join blob, enroll,
102	// and persist identity — pinning the server cert from the blob (the enroll
103	// response's fingerprint is ignored, so the blob is the sole trust root).
104	func runJoin(st *state.Store, blob string) {
105	if blob == "" {
106	fmt.Fprintln(os.Stderr, "usage: eitri-agent join <join-blob>")
107	os.Exit(1)
108	}
109	f, err := joinblob.Decode(blob)
110	if err != nil {
111	// Never echo the blob itself — it carries a bearer token.
112	fmt.Fprintf(os.Stderr, "invalid join blob: %v\n", err)
113	os.Exit(1)
114	}
115
116	hostname, err := os.Hostname()
117	if err != nil {
118	hostname = "unknown"
119	}
120
121	result, err := enrollclient.New(f.HTTPURL).Enroll(context.Background(), enrollclient.Request{
122	Token: f.Token,
123	Name: hostname,
124	OS: runtime.GOOS,
125	Arch: runtime.GOARCH,
126	Provisioner: "cloudhv",
127	})
128	if errors.Is(err, enrollclient.ErrTokenRejected) {
129	fmt.Fprintln(os.Stderr, "enroll rejected: token already used or expired — mint a new join token")
130	os.Exit(1)
131	}
132	if err != nil {
133	fmt.Fprintf(os.Stderr, "enroll failed: %v\n", err)
134	os.Exit(1)
135	}
136
137	id := state.Identity{
138	HostID: result.HostID,
139	Credential: result.Credential,
140	BridgeCIDR: result.BridgeCIDR,
141	ServerQUICAddr: f.QUICAddr,
142	ServerCertSHA256: f.CertFP, // authoritative; response fingerprint ignored
143	}
144	if err := st.SaveIdentity(id); err != nil {
145	slog.Error("save identity", "err", err)
146	os.Exit(1)
147	}
148	fmt.Printf("Enrolled: host_id=%s bridge_cidr=%s\n", result.HostID, result.BridgeCIDR)
149	}
150
151	// realRunner creates a subprocess and returns its combined output.
152	func realRunner(ctx context.Context, name string, args ...string) (string, error) {
153	cmd := exec.CommandContext(ctx, name, args...)
154	out, err := cmd.CombinedOutput()
155	return string(out), err
156	}
157
158	// runAgent handles the normal (no subcommand) run mode.
159	func runAgent(st *state.Store, cfg agentConfig) {
160	id, ok := st.Identity()
161	if !ok {
162	fmt.Fprintln(os.Stderr, "not enrolled — run with 'join <blob>' subcommand first")
163	os.Exit(1)
164	}
165
166	ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
167	defer cancel()
168
169	// Flush integration-coverage counters on SIGUSR1 (no-op unless built with
170	// -cover and GOCOVERDIR is set). Lets the deploy boot-gate snapshot coverage
171	// from the live agent without bouncing the process.
172	covsnap.Install(ctx)
173
174	net, err := netenv.New(realRunner, id.BridgeCIDR)
175	if err != nil {
176	slog.Error("netenv init", "err", err)
177	os.Exit(1)
178	}
179
180	if err := net.EnsureBridge(ctx); err != nil {
181	slog.Error("ensure bridge", "err", err)
182	os.Exit(1)
183	}
184
185	// Bootstrap cloud-hypervisor and its UEFI firmware before anything tries
186	// to launch a VM: a bare host that just joined has neither, and the agent
187	// is useless without at least the hypervisor binary. BootstrapDest maps
188	// the --ch-bin value (usually a bare $PATH name) to a real install path.
189	bs := &bootstrap.Bootstrapper{CHPath: cloudhv.BootstrapDest(cfg.CHBin), FirmwarePath: cfg.Firmware, ManifestURL: cfg.BootstrapURL}
190	if err := bs.Ensure(ctx); err != nil {
191	slog.Error("bootstrap runtime", "err", err)
192	os.Exit(1)
193	}
194
195	prov := cloudhv.New(st, cfg.CHBin, cfg.Firmware, realRunner)
196
197	// Serial console pumps: one per running VM, started at Boot (cloudhv hook)
198	// and reattached here for VMs that survived an agent restart (CH runs in
199	// its own process group; the pump reconnects to the still-listening
200	// serial socket).
201	pumps := serialpump.NewManager(st.SerialSocketPath, st.SerialLogPath)
202	prov.Pumps = pumps
203	if recs, err := st.LoadVMs(); err == nil {
204	for _, rec := range recs {
205	if rec.IP != "" {
206	net.AddReservation(rec.Spec.VMID, rec.IP)
207	}
208	if prov.Running(rec.Spec.VMID) {
209	pumps.Ensure(rec.Spec.VMID)
210	}
211	}
212	} else {
213	slog.Warn("state load failed; surviving VMs' consoles will be silent AND their DHCP reservations are not rebuilt (they may fail to renew until reconcile recreates them)", "err", err)
214	}
215
216	// Start the embedded DHCP responder AFTER reservations are rebuilt from
217	// state, so a surviving guest's renewal is never answered from an empty
218	// table (fail-closed + reservation preload close the gap).
219	if err := net.StartDHCP(ctx); err != nil {
220	slog.Error("start dhcp", "err", err)
221	os.Exit(1)	21	os.Exit(1)
222	}	22	}
223
224	cache := imagecache.New(st.ImagesDir(), realRunner)
225	// Clamp before shifting: GB<<30 overflows int64 for absurd flag values —
226	// same trap class cloudhv's maxDiskGB comment documents. 1 PiB is beyond
227	// any real cache; anything above disables eviction just like 0 would.
228	imageCacheMaxGB := cfg.ImageCacheMaxGB
229	if imageCacheMaxGB < 0 \|\| imageCacheMaxGB > 1<<20 {
230	slog.Warn("image-cache-max-gb out of range [0, 2^20]; disabling eviction", "value", imageCacheMaxGB)
231	imageCacheMaxGB = 0
232	}
233	cache.MaxBytes = imageCacheMaxGB << 30
234	// Reclaim temps left by a previous agent killed mid-fetch or mid-convert.
235	// Must run here, before the reconcile loop starts fetching: a sweep cannot
236	// tell an abandoned temp from one an in-flight fetch is still writing.
237	cache.SweepTemps()
238
239	engine := &reconcile.Engine{
240	St: st,
241	Prov: prov,
242	Net: net,
243	Images: cache.Ensure,
244	Seed: seed.Build,
245	BootID: syncclient.HostBootID,
246	Now: time.Now,
247	TombstoneGrace: cfg.TombstoneGrace,
248	VanishGrace: cfg.VanishGrace,
249	MaxCreateAttempts: 3,
250	MaxConcurrentCreates: cfg.MaxConcurrentCreates,
251	VMTimeout: cfg.VMTimeout,
252	MaxVCPUs: cfg.MaxVCPUs,
253	MaxMemMB: cfg.MaxMemMB,
254	MaxDiskGB: cfg.MaxDiskGB,
255	}
256
257	// Seed the admission ledger from persisted records so the first reconcile
258	// accounts for VMs that survived the agent restart (their compute must
259	// count against the caps before any new create is admitted).
260	if recs, err := st.LoadVMs(); err == nil {
261	engine.SeedLedger(recs)
262	}
263
264	// The per-VM reconcile workers are deliberately NOT torn down here. Stopping
265	// the engine is terminal — it would make the next report an empty actual state,
266	// which the control plane reads as every VM on this host having vanished — and
267	// it would race an unjoined session worker that can still be mid-Step when ctx
268	// is cancelled (see the note below where pumps are torn down). The process is
269	// exiting; the OS reclaims the goroutines.
270
271	client := &syncclient.Client{
272	Engine: engine,
273	St: st,
274	Identity: id,
275	StateDir: cfg.StateDir,
276	Runner: realRunner,
277	Console: pumps,
278	MaxVCPUs: cfg.MaxVCPUs,
279	MaxMemMB: cfg.MaxMemMB,
280	MaxDiskGB: cfg.MaxDiskGB,
281	}
282
283	slog.Info("agent started", "host_id", id.HostID, "bridge_cidr", id.BridgeCIDR)
284	client.Run(ctx)
285
286	// client.Run blocks until ctx is cancelled (SIGINT/SIGTERM) and only then
287	// returns, so the reconcile/sync loop is done driving VMs and no further
288	// Ensure/Stop calls are expected to reach pumps (an unjoined session
289	// worker could in principle still be mid-Engine.Step, but it has nothing
290	// left to drive once client.Run has returned). Tear down every serial
291	// console pump here, at the very end of agent shutdown.
292	pumps.StopAll()
293	}	23	}

cmd/eitri-mcp/main.go

Old		New
@@ -3,27 +3,16 @@
3	// eitri API plus SSH; it embeds no control-plane code. It holds its OWN	3	// eitri API plus SSH; it embeds no control-plane code. It holds its OWN
4	// persistent user CA, uploads that CA's public key to its tenant once at	4	// persistent user CA, uploads that CA's public key to its tenant once at
5	// startup, and self-signs short-lived user certs locally (BYO model — the	5	// startup, and self-signs short-lived user certs locally (BYO model — the
6	// server no longer mints user certs). Wiring only — logic lives in	6	// server no longer mints user certs). All behavior lives in internal/mcpserver
7	// internal/mcpserver.	7	// (RunCLI); this package is wiring only (arch R14).
8	package main	8	package main
9		9
10	import (	10	import (
11	"context"
12	"crypto/ed25519"
13	"crypto/rand"
14	"encoding/pem"
15	"flag"
16	"fmt"	11	"fmt"
17	"os"	12	"os"
18	"os/signal"
19	"syscall"
20		13
21	"github.com/a73x/eitri/internal/gateclient"
22	"github.com/a73x/eitri/internal/mcpserver"	14	"github.com/a73x/eitri/internal/mcpserver"
23	"github.com/a73x/eitri/internal/server/api/client"
24	"github.com/a73x/eitri/internal/version"	15	"github.com/a73x/eitri/internal/version"
25	"github.com/modelcontextprotocol/go-sdk/mcp"
26	"golang.org/x/crypto/ssh"
27	)	16	)
28		17
29	func main() {	18	func main() {
@@ -31,125 +20,8 @@ func main() {
31	fmt.Println(version.Version)	20	fmt.Println(version.Version)
32	return	21	return
33	}	22	}
34	defaultCfg := "~/.config/eitri-mcp/config.json"	23	if err := mcpserver.RunCLI(os.Args[1:]); err != nil {
35	if env := os.Getenv("EITRI_MCP_CONFIG"); env != "" {
36	defaultCfg = env
37	}
38	cfgPath := flag.String("config", defaultCfg, "path to eitri-mcp config.json")
39	flag.Parse()
40
41	if err := run(*cfgPath); err != nil {
42	fmt.Fprintln(os.Stderr, "eitri-mcp:", err)	24	fmt.Fprintln(os.Stderr, "eitri-mcp:", err)
43	os.Exit(1)	25	os.Exit(1)
44	}	26	}
45	}	27	}
46
47	func run(cfgPath string) error {
48	cfg, err := mcpserver.LoadConfig(cfgPath)
49	if err != nil {
50	return err
51	}
52	// This client owns its own persistent user CA (BYO model): it self-signs
53	// short-lived user certs locally rather than asking the server to mint them.
54	userCA, err := loadOrCreateCA(cfg.CAKeyPath)
55	if err != nil {
56	return fmt.Errorf("load mcp user CA: %w", err)
57	}
58	// The Runner reaches VMs by name through the eitri SSH-CA jump gate: it
59	// authenticates with short-lived user certs it self-signs on demand with its
60	// own user CA, and verifies both hops' host certs against the eitri host CA.
61	// The user CA's public key is uploaded to the tenant once (Register, below)
62	// so VMs trust those certs. GateAuth is backed by the same API client.
63	api := &client.Client{BaseURL: cfg.ServerURL, Token: cfg.Token}
64	gateAuth := gateclient.NewGateAuth(api, userCA, cfg.Tenant, nil)
65	tools := &mcpserver.Tools{
66	API: mcpserver.API{Client: api},
67	Runner: mcpserver.NewRunner(mcpserver.RunnerConfig{
68	Gate: cfg.Gate,
69	Auth: gateAuth,
70	VMUser: cfg.VMUser,
71	}),
72	Gate: cfg.Gate,
73	VMUser: cfg.VMUser,
74	}
75
76	server := mcp.NewServer(&mcp.Implementation{Name: "eitri", Version: "0.1.0"}, nil)
77	register(server, "vm_create", "Create an eitri VM (persistent). Waits for ready+cloud-init by default.", tools.VMCreate)
78	register(server, "vm_list", "List all VMs on the eitri fleet.", tools.VMList)
79	register(server, "vm_info", "Show one VM's state and how to reach it.", tools.VMInfo)
80	register(server, "vm_exec", "Run a shell command in a VM over SSH; returns stdout/stderr/exit code.", tools.VMExec)
81	register(server, "vm_write_file", "Write content to a file in a VM (parents created).", tools.VMWriteFile)
82	register(server, "vm_read_file", "Read a file from a VM (capped at 1 MiB).", tools.VMReadFile)
83	register(server, "vm_destroy", "Destroy a VM by id or EXACT name. Explicit-only; never called automatically.", tools.VMDestroy)
84
85	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
86	defer stop()
87
88	// Upload our user CA to the tenant before serving, so vm_create's precondition
89	// (a registered user CA) is satisfied and VMs trust the certs we sign.
90	if err := gateAuth.Register(ctx); err != nil {
91	return fmt.Errorf("register mcp user CA: %w", err)
92	}
93	return server.Run(ctx, &mcp.StdioTransport{})
94	}
95
96	// loadOrCreateCA returns a stable ssh.Signer for the key at path. If the file
97	// is absent it generates a fresh ed25519 key, writes it 0600 with O_EXCL (so a
98	// concurrent creator can't clobber it and a symlink can't be followed), and
99	// returns its signer; if present it parses and returns the existing key. This
100	// is deliberately local to the MCP (a pure API client) and does NOT import the
101	// server's sshca package. Never logs or returns key material in errors.
102	func loadOrCreateCA(path string) (ssh.Signer, error) {
103	pemBytes, err := os.ReadFile(path)
104	if err == nil {
105	signer, perr := ssh.ParsePrivateKey(pemBytes)
106	if perr != nil {
107	return nil, fmt.Errorf("parse ssh key %q: %w", path, perr)
108	}
109	return signer, nil
110	}
111	if !os.IsNotExist(err) {
112	return nil, fmt.Errorf("read ssh key %q: %w", path, err)
113	}
114
115	_, priv, err := ed25519.GenerateKey(rand.Reader)
116	if err != nil {
117	return nil, fmt.Errorf("generate ssh key: %w", err)
118	}
119	block, err := ssh.MarshalPrivateKey(priv, "")
120	if err != nil {
121	return nil, fmt.Errorf("marshal ssh key: %w", err)
122	}
123	signer, err := ssh.NewSignerFromSigner(priv)
124	if err != nil {
125	return nil, fmt.Errorf("new signer: %w", err)
126	}
127	f, err := os.OpenFile(path, os.O_WRONLY\|os.O_CREATE\|os.O_EXCL, 0o600)
128	if err != nil {
129	return nil, fmt.Errorf("create ssh key %q: %w", path, err)
130	}
131	if _, werr := f.Write(pem.EncodeToMemory(block)); werr != nil {
132	f.Close()
133	return nil, fmt.Errorf("write ssh key %q: %w", path, werr)
134	}
135	if cerr := f.Close(); cerr != nil {
136	return nil, fmt.Errorf("close ssh key %q: %w", path, cerr)
137	}
138	return signer, nil
139	}
140
141	// register adapts a Tools method to the SDK. This is the ONLY place that
142	// touches SDK generics; if the SDK's handler signature changes, change it here.
143	//
144	// Note on the hand-off contract: the SDK drops the Out value when the handler
145	// returns a non-nil error — StructuredContent is left unset and only err.Error()
146	// reaches the model (as IsError text content). VMCreate's degraded-path errors
147	// are self-sufficient (they name the VM id+name), so the model can still find
148	// and destroy the VM from the error text.
149	func register[In, Out any](s *mcp.Server, name, desc string, fn func(context.Context, In) (Out, error)) {
150	mcp.AddTool(s, &mcp.Tool{Name: name, Description: desc},
151	func(ctx context.Context, req mcp.CallToolRequest, in In) (mcp.CallToolResult, Out, error) {
152	out, err := fn(ctx, in)
153	return nil, out, err
154	})
155	}

cmd/eitri-server/main.go

Old		New
@@ -1,324 +1,23 @@
1	// eitri-server: single-node control plane (Phase 1: static admin token, no TLS	1	// eitri-server: single-node control plane (no TLS termination here — front
2	// termination here — front with a reverse proxy for TLS).	2	// with a reverse proxy for TLS). All behavior lives in internal/server/boot
		3	// (RunCLI); this package is version-check-and-dispatch wiring only (arch R14).
3	package main	4	package main
4		5
5	import (	6	import (
6	"context"
7	"encoding/json"
8	"errors"
9	"flag"
10	"fmt"	7	"fmt"
11	"log/slog"
12	"net/http"
13	"net/url"
14	"os"	8	"os"
15	"os/signal"
16	"regexp"
17	"strings"
18	"syscall"
19	"time"
20		9
21	"github.com/a73x/eitri/internal/covsnap"	10	"github.com/a73x/eitri/internal/server/boot"
22	"github.com/a73x/eitri/internal/joinblob"
23	"github.com/a73x/eitri/internal/server/api"
24	serverconfig "github.com/a73x/eitri/internal/server/config"
25	"github.com/a73x/eitri/internal/server/health"
26	"github.com/a73x/eitri/internal/server/hub"
27	"github.com/a73x/eitri/internal/server/registry"
28	"github.com/a73x/eitri/internal/server/release"
29	"github.com/a73x/eitri/internal/server/store"
30	"github.com/a73x/eitri/internal/server/syncsvc"
31	"github.com/a73x/eitri/internal/server/web"
32	"github.com/a73x/eitri/internal/transport"
33	"github.com/a73x/eitri/internal/version"	11	"github.com/a73x/eitri/internal/version"
34	"github.com/quic-go/quic-go"
35	)	12	)
36		13
37	// defaultImageSHARe matches a valid lowercase hex SHA-256 digest. Kept local
38	// to the control plane rather than shared with the agent's imagecache: R1
39	// forbids the control plane importing the data plane, even transitively.
40	var defaultImageSHARe = regexp.MustCompile(`^[a-f0-9]{64}$`)
41
42	// parseDurationCfg parses raw (a config duration string) for the knob named
43	// name (the JSON field label echoed in error logs). Empty raw keeps def. A
44	// parse failure is fatal. valid, when non-nil, is the knob's range rule; a
45	// value failing it is fatal, with rule (e.g. ">= 0", "> 0") spelling the rule
46	// in the log line. A nil valid skips the range check.
47	func parseDurationCfg(name, raw string, def time.Duration, valid func(time.Duration) bool, rule string) time.Duration {
48	if raw == "" {
49	return def
50	}
51	d, err := time.ParseDuration(raw)
52	if err != nil {
53	slog.Error(name+" invalid", "err", err)
54	os.Exit(1)
55	}
56	if valid != nil && !valid(d) {
57	slog.Error(name+" must be "+rule, "value", raw)
58	os.Exit(1)
59	}
60	return d
61	}
62
63	// config is the on-disk JSON schema, shared with the integration harness
64	// (which renders this file for the server subprocess it launches) via
65	// internal/server/config so the compiler keeps the two in agreement.
66	type config = serverconfig.Config
67
68	func main() {	14	func main() {
69	if len(os.Args) > 1 && os.Args[1] == "--version" {	15	if len(os.Args) > 1 && os.Args[1] == "--version" {
70	fmt.Println(version.Version)	16	fmt.Println(version.Version)
71	return	17	return
72	}	18	}
73	cfgPath := flag.String("config", "/etc/eitri/server.json", "config file")	19	if err := boot.RunCLI(os.Args[1:]); err != nil {
74	flag.Parse()	20	fmt.Fprintln(os.Stderr, "eitri-server:", err)
75	raw, err := os.ReadFile(*cfgPath)
76	if err != nil {
77	slog.Error("read config", "err", err)
78	os.Exit(1)
79	}
80	var cfg config
81	if err := json.Unmarshal(raw, &cfg); err != nil {
82	slog.Error("parse config", "err", err)
83	os.Exit(1)
84	}
85	if cfg.HostSecret == "" {
86	slog.Error("host_secret is required")
87	os.Exit(1)
88	}
89	// The server is a pure OIDC relying party (spec §2): issuer, client_id and
90	// public_url are required. Collect every missing key so the operator fixes
91	// server.json in one pass rather than one restart per key.
92	var missingOIDC []string
93	if cfg.OIDC.Issuer == "" {
94	missingOIDC = append(missingOIDC, "oidc.issuer")
95	}
96	if cfg.OIDC.ClientID == "" {
97	missingOIDC = append(missingOIDC, "oidc.client_id")
98	}
99	if cfg.OIDC.PublicURL == "" {
100	missingOIDC = append(missingOIDC, "oidc.public_url")
101	}
102	if len(missingOIDC) > 0 {
103	slog.Error("server.json: missing required keys (point them at eitri-oidc or your IdP)", "keys", strings.Join(missingOIDC, ", "))
104	os.Exit(1)
105	}
106	// public_url builds the OIDC callback URL, so it must be an absolute
107	// http(s) URL with a host — catch a bare host, missing scheme, or
108	// scheme-only URL at boot, not at the first redirect.
109	if u, err := url.Parse(cfg.OIDC.PublicURL); err != nil \|\| !u.IsAbs() \|\| u.Host == "" \|\| (u.Scheme != "http" && u.Scheme != "https") {
110	slog.Error("server.json: oidc.public_url must be an absolute http(s) URL", "value", cfg.OIDC.PublicURL)
111	os.Exit(1)
112	}
113	// admin_token is retired (spec §7): ignored if present, still passed to
114	// api.Config until the PAT/session middleware replaces it. Warn once so the
115	// operator prunes the stale key.
116	if cfg.AdminToken != "" {
117	slog.Warn("server.json: admin_token is no longer used and is ignored; remove it")
118	}
119	if cfg.AdvertiseHTTP == "" \|\| cfg.AdvertiseQUIC == "" {
120	slog.Error("advertise_http and advertise_quic are required (the addresses agents use to reach this server, e.g. http://192.168.0.190:8080 and 192.168.0.190:8443)")
121	os.Exit(1)
122	}
123	// [carry-forward] Fail fast on a malformed default image digest rather
124	// than letting every mint silently propagate a bad hash. A local regex
125	// (not the agent's imagecache) because R1 forbids the control plane
126	// importing the data plane, even transitively.
127	if cfg.DefaultImageSHA != "" && !defaultImageSHARe.MatchString(cfg.DefaultImageSHA) {
128	slog.Error("default_image_sha256 malformed (want 64 lowercase hex chars)", "value", cfg.DefaultImageSHA)
129	os.Exit(1)
130	}
131
132	st, err := store.Open(cfg.DBPath, cfg.CIDRPool)
133	if err != nil {
134	slog.Error("open store", "err", err)
135	os.Exit(1)	21	os.Exit(1)
136	}	22	}
137
138	certPEM, certFP, err := st.ServerCert()
139	if err != nil {
140	slog.Error("server cert", "err", err)
141	os.Exit(1)
142	}
143	keyPEM, err := st.ServerKeyPEM()
144	if err != nil {
145	slog.Error("server key", "err", err)
146	os.Exit(1)
147	}
148
149	// Log the identity agents pin — the operator verifies this out-of-band
150	// during the rotation ceremony (docs/cert-rotation.md step 3).
151	slog.Info("server cert", "fingerprint", certFP)
152
153	// Rotation nudge: the agent pin ignores expiry so nothing breaks at
154	// NotAfter, but a long-lived key is a widening forgery window. Warn while
155	// inside the renewal window — at startup AND daily, because servers here
156	// are long-lived daemons that can cross into the window (or past expiry)
157	// without ever restarting. See docs/cert-rotation.md for the ceremony.
158	warnIfRenewalDue := func() {
159	notAfter, due := transport.CertRenewalDue(certPEM, time.Now())
160	if !due {
161	return
162	}
163	if notAfter.IsZero() {
164	slog.Warn("server cert unparseable — inspect server.crt (docs/cert-rotation.md)")
165	return
166	}
167	slog.Warn("server cert renewal due — rotate and re-enroll agents (docs/cert-rotation.md)",
168	"not_after", notAfter.Format(time.RFC3339))
169	}
170	warnIfRenewalDue()
171
172	// Audit retention: bound the append-only log (default 90 days, 0 disables).
173	// A negative value is almost certainly a typo — refuse rather than silently
174	// keeping the audit log forever ("0" is the explicit disable spelling).
175	auditRetention := parseDurationCfg("audit_retention", cfg.AuditRetention, 9024time.Hour,
176	func(d time.Duration) bool { return d >= 0 }, ">= 0")
177	pruneAudit := func() {
178	if auditRetention <= 0 {
179	return
180	}
181	if n, err := st.PruneAudit(auditRetention); err != nil {
182	slog.Warn("audit prune failed", "err", err)
183	} else if n > 0 {
184	slog.Info("audit pruned", "rows", n, "retention", auditRetention)
185	}
186	}
187	pruneAudit()
188
189	// Daily housekeeping: cert-renewal nudge + audit retention.
190	go func() {
191	for range time.Tick(24 * time.Hour) {
192	warnIfRenewalDue()
193	pruneAudit()
194	}
195	}()
196
197	// Fail fast if the advertised addresses are non-empty but malformed (e.g. a
198	// URL with no scheme): otherwise every enroll-token mint would 500 at runtime.
199	if _, err := joinblob.Encode(cfg.AdvertiseHTTP, cfg.AdvertiseQUIC, "startup-probe", certFP); err != nil {
200	slog.Error("advertise_http/advertise_quic invalid", "err", err)
201	os.Exit(1)
202	}
203
204	// SSH jump gate (§B): nil when ssh_listen is unset (gate OFF); see
205	// setupSSHGate. The listener itself is started below, once syncsvc.Service
206	// (the tunnel dialer) exists.
207	sshGate := setupSSHGate(cfg)
208
209	reg := registry.New(time.Now)
210	h := hub.New()
211
212	a := api.New(api.Config{HostSecret: []byte(cfg.HostSecret),
213	DefaultImage: api.DefaultImage{URL: cfg.DefaultImageURL, SHA256: cfg.DefaultImageSHA},
214	ServerCertSHA256: certFP,
215	AdvertiseHTTP: cfg.AdvertiseHTTP,
216	AdvertiseQUIC: cfg.AdvertiseQUIC,
217	OIDC: api.OIDCConfig{
218	Issuer: cfg.OIDC.Issuer,
219	ClientID: cfg.OIDC.ClientID,
220	ClientSecret: cfg.OIDC.ClientSecret,
221	PublicURL: cfg.OIDC.PublicURL,
222	AllowedDomains: cfg.OIDC.AllowedDomains,
223	AllowedIdentities: cfg.OIDC.AllowedIdentities,
224	}},
225	st, reg, h)
226
227	tlsConf, err := transport.ServerTLS(certPEM, keyPEM)
228	if err != nil {
229	slog.Error("server tls", "err", err)
230	os.Exit(1)
231	}
232	lis, err := quic.ListenAddr(cfg.QUICListen, tlsConf,
233	// Shared with the agent dialer via transport so the two ends can't drift.
234	transport.SyncQUICConfig())
235	if err != nil {
236	slog.Error("quic listen", "err", err)
237	os.Exit(1)
238	}
239	maxCredAge := parseDurationCfg("credential_max_age", cfg.CredentialMaxAge, 0, nil, "")
240	// SSH cert minters: no-op when the gate is off, so the endpoint 404s.
241	sshGate.wireAPI(a)
242
243	svc := syncsvc.New(st, reg, h, []byte(cfg.HostSecret), maxCredAge)
244	// Console broker: the API bridges browser WebSockets to agent console
245	// streams over the live sync connections the service tracks.
246	a.SetConsoleDialer(svc)
247
248	// SSH jump gate listener (no-op when off); fatal on a failed bind, like
249	// QUIC/HTTP below — a dead gate must not run silently.
250	sshGate.startListener(st, svc)
251
252	go func() {
253	slog.Info("quic listening", "addr", cfg.QUICListen)
254	if err := svc.Serve(context.Background(), lis); err != nil {
255	slog.Error("quic serve", "err", err)
256	os.Exit(1)
257	}
258	}()
259
260	// Background: finalize drained decommissioning hosts.
261	go a.StartBackground(context.Background())
262
263	// Serve the REST API + SSE under /api/ and the embedded SPA everywhere else.
264	root := http.NewServeMux()
265	root.Handle("/api/", a.Handler())
266	// Sign-in endpoints live OUTSIDE /api/ and its auth middleware: they are how
267	// a browser establishes a session in the first place (spec §2).
268	root.Handle("/auth/", a.AuthHandler())
269	// Unauthenticated probes (outside /api/, so a load balancer or the deploy
270	// script needs no token). /livez is process-up; /readyz gates on the
271	// dependencies the server needs to actually serve — the DB. The QUIC
272	// listener bind is a startup invariant: quic.ListenAddr above exits the
273	// process on failure and runs before this HTTP server, so a response here
274	// already implies QUIC bound.
275	root.HandleFunc("/livez", health.Live)
276	root.Handle("/readyz", health.Ready(3*time.Second,
277	health.Check{Name: "db", Probe: st.Ping},
278	))
279	root.Handle("/", web.Handler())
280
281	// Graceful shutdown: SIGINT/SIGTERM stops accepting, drains in-flight
282	// requests, then closes the API — stopping the SSE snapshot-hub goroutine and
283	// releasing its notifier subscription. The QUIC listener and background worker
284	// die with the process.
285	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
286	defer stop()
287
288	// Release discovery: absent field ⇒ eitri.sh default; explicit "" disables.
289	manifestURL := "https://eitri.sh/dl/latest/manifest.json"
290	if cfg.ReleaseManifestURL != nil {
291	manifestURL = *cfg.ReleaseManifestURL
292	}
293	if manifestURL != "" {
294	rel := release.New(manifestURL)
295	go rel.Poll(ctx, 24*time.Hour, func(err error) {
296	slog.Warn("release manifest refresh failed", "err", err)
297	})
298	a.SetReleaseSource(rel)
299	}
300	a.SetAgentUpgrader(svc)
301
302	// Flush integration-coverage counters on SIGUSR1 (no-op unless built with
303	// -cover and GOCOVERDIR is set). Lets the deploy boot-gate snapshot coverage
304	// from the live server without bouncing the process.
305	covsnap.Install(ctx)
306
307	srv := &http.Server{Addr: cfg.HTTPListen, Handler: root}
308	go func() {
309	slog.Info("http listening", "addr", cfg.HTTPListen)
310	if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
311	slog.Error("http serve", "err", err)
312	os.Exit(1)
313	}
314	}()
315
316	<-ctx.Done()
317	slog.Info("shutting down")
318	shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
319	defer cancel()
320	if err := srv.Shutdown(shutdownCtx); err != nil {
321	slog.Warn("http graceful shutdown", "err", err)
322	}
323	a.Close()
324	}	23	}

cmd/eitri-server/sshgate.go

Old		New
@@ -1,159 +0,0 @@
1	package main
2
3	import (
4	"log/slog"
5	"net"
6	"os"
7	"time"
8
9	"github.com/a73x/eitri/internal/server/api"
10	"github.com/a73x/eitri/internal/server/sshca"
11	"github.com/a73x/eitri/internal/server/sshgate"
12	"github.com/a73x/eitri/internal/server/store"
13	"github.com/a73x/eitri/internal/server/syncsvc"
14	"golang.org/x/crypto/ssh"
15	)
16
17	// sshGateSetup carries the jump-gate state from config-time setup to the later
18	// wiring points in startup (API cert minters, sync snapshot, gate listener).
19	// A nil *sshGateSetup means the gate is OFF: every method is a no-op on a nil
20	// receiver, so main holds one value instead of repeating `!= nil` guards.
21	type sshGateSetup struct {
22	ca *sshca.CA
23	listen string // cfg.SSHListen
24	domain string // cfg.SSHGateDomain
25	}
26
27	// setupSSHGate wires the SSH jump gate (§B): OFF unless ssh_listen is set
28	// (returns nil). When enabled, load or create the persistent user CA + gate
29	// host key (0600, never logged).
30	func setupSSHGate(cfg config) *sshGateSetup {
31	if cfg.SSHListen == "" {
32	return nil
33	}
34	if cfg.SSHCAKey == "" \|\| cfg.SSHHostKey == "" {
35	slog.Error("ssh_ca_key and ssh_host_key are required when ssh_listen is set")
36	os.Exit(1)
37	}
38	sshGate, err := sshca.New(cfg.SSHCAKey, cfg.SSHHostKey)
39	if err != nil {
40	slog.Error("ssh ca", "err", err)
41	os.Exit(1)
42	}
43	// Log the HOST CA identity operators pin via @cert-authority for gate + VM
44	// host verification. Only the public key is ever logged (private material
45	// never is). eitri holds no user CA — those are BYO per-tenant.
46	slog.Info("ssh jump gate configured", "listen", cfg.SSHListen,
47	"host_ca", string(sshGate.HostCAAuthorizedKey()))
48	// The gate listener itself is started later (startListener), once
49	// syncsvc.Service (the tunnel dialer) exists.
50	return &sshGateSetup{ca: sshGate, listen: cfg.SSHListen, domain: cfg.SSHGateDomain}
51	}
52
53	// wireAPI installs the per-VM host-cert minter and publishes the HOST CA: when
54	// the jump gate is enabled, eitri signs a persistent host key + cert at each VM
55	// create and serves the host CA pubkey via GET /api/v1/ssh-ca. eitri never
56	// mints user certs — user CAs are BYO per-tenant (uploaded, never held here).
57	// Left unwired when the gate is off, so the ssh-ca endpoint 404s.
58	func (g sshGateSetup) wireAPI(a api.API) {
59	if g == nil {
60	return
61	}
62	// Per-VM host certs: sign a persistent host key + cert at each VM create,
63	// so VMs present verifiable host keys (clients accept via @cert-authority).
64	a.SetHostCertMinter(api.NewHostMinter(g.ca.HostCA()))
65	// Publish the HOST CA public key so clients can pin `@cert-authority` for
66	// host verification of both the gate and every VM.
67	a.SetSSHCAAuthorizedKey(string(g.ca.HostCAAuthorizedKey()))
68	}
69
70	// startListener starts the SSH jump gate listener: when enabled, front
71	// `ssh -J gate ubuntu@<vm>` with the hardened bastion. It resolves VM names
72	// against the store, tunnels port 22 through the sync connection (svc.OpenTCP),
73	// and trusts only certs signed by the user CA. A failed bind is fatal (like
74	// QUIC/HTTP): a dead gate must not run silently.
75	func (g sshGateSetup) startListener(st store.Store, svc *syncsvc.Service) {
76	if g == nil {
77	return
78	}
79	// The gate host cert's principal is the name clients dial. Prefer the
80	// configured domain; else the host part of ssh_listen; else "localhost".
81	gateDomain := g.domain
82	if gateDomain == "" {
83	if h, _, err := net.SplitHostPort(g.listen); err == nil {
84	gateDomain = h
85	}
86	}
87	if gateDomain == "" {
88	gateDomain = "localhost"
89	}
90	slog.Info("ssh gate host cert", "principal", gateDomain)
91	// resolve is tenant-scoped: the bare name is looked up WITHIN the
92	// connection's tenant only.
93	resolve := func(tenant, name string) (hostID, vmID string, ok bool) {
94	vm, err := st.VMByTenantName(tenant, name)
95	if err != nil {
96	return "", "", false
97	}
98	return vm.HostID, vm.ID, true
99	}
100	// authorize re-reads the VM row and requires tenant equality: the tenant
101	// that the connection's user cert resolved to (via its per-tenant CA) must
102	// own the VM, or the connection is refused.
103	authorize := func(tenant, vmID string) bool {
104	vm, err := st.GetVM(vmID)
105	return err == nil && vm.DeletedAt == nil && vm.Tenant == tenant
106	}
107	// Sign a long-lived HOST cert for the gate's own host key and present THAT
108	// (via a cert signer) instead of the bare key, so a client verifying with
109	// `@cert-authority` accepts the gate on first connect — no TOFU window.
110	gateCert, err := sshca.SignHostCert(g.ca.HostCA(), g.ca.HostKey().PublicKey(),
111	[]string{gateDomain}, "eitri-gate", time.Now(), sshca.HostCertTTL)
112	if err != nil {
113	slog.Error("sign gate host cert", "err", err)
114	os.Exit(1)
115	}
116	gateHostSigner, err := ssh.NewCertSigner(gateCert, g.ca.HostKey())
117	if err != nil {
118	slog.Error("gate host cert signer", "err", err)
119	os.Exit(1)
120	}
121	// isRevoked gates every cert auth against the revocation list. Fail-CLOSED
122	// for the single connection on a DB error: a store hiccup rejects THAT
123	// login (returns revoked=true) rather than fail-open (which would let a
124	// possibly-revoked cert through) or fail-the-whole-gate (which a global
125	// close would amount to, DoSing every login on any transient error).
126	isRevoked := func(serial uint64) bool {
127	revoked, err := st.IsSSHCertRevoked(serial)
128	if err != nil {
129	slog.Error("ssh cert revocation lookup failed; rejecting connection", "err", err)
130	return true
131	}
132	return revoked
133	}
134	// The gate trusts the DB-registered set of tenant user CAs and stamps each
135	// connection with the tenant that registered the signing CA. Look up by the
136	// SAME canonical authorized_keys line the store persists (ca_pubkey), so the
137	// bytes agree. A lookup error fails closed (rejects the cert).
138	userCALookup := func(pub ssh.PublicKey) (string, bool) {
139	tenant, ok, err := st.TenantForUserCA(sshca.AuthorizedKeyLine(pub))
140	if err != nil {
141	slog.Error("tenant user-ca lookup failed; rejecting", "err", err)
142	return "", false
143	}
144	return tenant, ok
145	}
146	gate := sshgate.New(gateHostSigner, userCALookup, resolve, authorize, svc.OpenTCP, isRevoked)
147	ln, err := net.Listen("tcp", g.listen)
148	if err != nil {
149	slog.Error("ssh gate listen", "err", err)
150	os.Exit(1)
151	}
152	go func() {
153	slog.Info("ssh jump gate listening", "addr", g.listen)
154	if err := gate.Serve(ln); err != nil {
155	slog.Error("ssh gate serve", "err", err)
156	os.Exit(1)
157	}
158	}()
159	}

cmd/eitri-site/main.go

Old		New
@@ -1,10 +1,9 @@
1	// Command eitri-site generates the eitri.sh static site (default) or the	1	// Command eitri-site generates the eitri.sh static site (default) or the
2	// release manifest (the "manifest" subcommand). See internal/site.	2	// release manifest (the "manifest" subcommand). All behavior lives in
		3	// internal/site (RunCLI); this package is wiring only (arch R14).
3	package main	4	package main
4		5
5	import (	6	import (
6	"encoding/json"
7	"flag"
8	"fmt"	7	"fmt"
9	"os"	8	"os"
10		9
@@ -17,54 +16,8 @@ func main() {
17	fmt.Println(version.Version)	16	fmt.Println(version.Version)
18	return	17	return
19	}	18	}
20	if len(os.Args) > 1 && os.Args[1] == "manifest" {	19	if err := site.RunCLI(os.Args[1:]); err != nil {
21	if err := runManifest(os.Args[2:]); err != nil {
22	fmt.Fprintln(os.Stderr, "eitri-site manifest:", err)
23	os.Exit(1)
24	}
25	return
26	}
27	if err := runBuild(os.Args[1:]); err != nil {
28	fmt.Fprintln(os.Stderr, "eitri-site:", err)	20	fmt.Fprintln(os.Stderr, "eitri-site:", err)
29	os.Exit(1)	21	os.Exit(1)
30	}	22	}
31	}	23	}
32
33	func runBuild(args []string) error {
34	fs := flag.NewFlagSet("eitri-site", flag.ExitOnError)
35	docs := fs.String("docs", "docs", "docs directory (markdown sources)")
36	siteDir := fs.String("site", "site", "site directory (index.md, template.html, style.css)")
37	dist := fs.String("dist", "", "optional dist/<version> dir with release artifacts")
38	out := fs.String("out", "site/dist", "output webroot")
39	if err := fs.Parse(args); err != nil {
40	return err
41	}
42	return site.Build(site.Config{DocsDir: docs, SiteDir: siteDir, DistDir: dist, OutDir: out})
43	}
44
45	func runManifest(args []string) error {
46	fs := flag.NewFlagSet("eitri-site manifest", flag.ExitOnError)
47	ver := fs.String("version", "", "release version (vX.Y.Z)")
48	dist := fs.String("dist", "", "dist/<version> dir holding bare agent binaries")
49	base := fs.String("base", "", "base URL artifacts are served from")
50	out := fs.String("out", "", "output path (default <dist>/manifest.json)")
51	if err := fs.Parse(args); err != nil {
52	return err
53	}
54	if ver == "" \|\| dist == "" \|\| *base == "" {
55	return fmt.Errorf("-version, -dist, and -base are required")
56	}
57	m, err := site.BuildManifest(ver, dist, *base)
58	if err != nil {
59	return err
60	}
61	raw, err := json.MarshalIndent(m, "", " ")
62	if err != nil {
63	return err
64	}
65	path := *out
66	if path == "" {
67	path = *dist + "/manifest.json"
68	}
69	return os.WriteFile(path, append(raw, '\n'), 0o644)
70	}

cmd/eitri-smoke/config.go

Old		New
@@ -1,106 +0,0 @@
1	// Command eitri-smoke drives the live eitri fleet through create -> boot-proof
2	// -> reap of one throwaway VM, exiting non-zero on any failure. It is a client
3	// of the eitri API plus SSH; it embeds no control-plane or agent code.
4	package main
5
6	import (
7	"fmt"
8	"strconv"
9	"strings"
10	)
11
12	// Config holds the environment-sourced settings for one smoke run. It mirrors
13	// the variables the deploy boot-gate reads from $EITRI_DEPLOY_ENV.
14	type Config struct {
15	ServerURL string
16	CIUser string
17	CIPasswordFile string
18	CIPATFile string
19	AgentUserHost string
20	AgentPort int
21	AgentStateDir string
22
23	// Carried for a later coverage-collection task; optional here.
24	ServerGocoverdir string
25	AgentGocoverdir string
26	CoverOut string
27
28	// SSH-CA gate check. When SmokeGate and SmokeUserCAFile are both set, the
29	// scenario proves guest access through the gate (a hard gate). Optional.
30	SmokeGate string // SMOKE_GATE, "<gate-domain>:<port>"
31	SmokeVMUser string // SMOKE_VM_USER, default "ubuntu"
32	SmokeUserCAFile string // SMOKE_USER_CA_FILE, load-or-create user CA key
33	}
34
35	// loadConfig reads the required smoke settings via getenv (never the real
36	// process environment directly, so callers can inject a fake for tests). It
37	// returns an error naming the first missing required variable.
38	func loadConfig(getenv func(string) string) (Config, error) {
39	serverURL := getenv("SERVER_URL")
40	ciUser := getenv("CI_USER")
41	ciPasswordFile := getenv("CI_PASSWORD_FILE")
42	ciPATFile := getenv("CI_PAT_FILE")
43	agentHosts := getenv("AGENT_HOSTS")
44	agentStateDir := getenv("AGENT_STATE_DIR")
45
46	var missing []string
47	if serverURL == "" {
48	missing = append(missing, "SERVER_URL")
49	}
50	if ciUser == "" {
51	missing = append(missing, "CI_USER")
52	}
53	if ciPasswordFile == "" {
54	missing = append(missing, "CI_PASSWORD_FILE")
55	}
56	if ciPATFile == "" {
57	missing = append(missing, "CI_PAT_FILE")
58	}
59	if agentHosts == "" {
60	missing = append(missing, "AGENT_HOSTS")
61	}
62	if agentStateDir == "" {
63	missing = append(missing, "AGENT_STATE_DIR")
64	}
65	if len(missing) > 0 {
66	return Config{}, fmt.Errorf("missing required env var(s): %s", strings.Join(missing, ", "))
67	}
68
69	userHost, port := parseAgentHost(agentHosts)
70
71	smokeVMUser := getenv("SMOKE_VM_USER")
72	if smokeVMUser == "" {
73	smokeVMUser = "ubuntu"
74	}
75
76	return Config{
77	ServerURL: serverURL,
78	CIUser: ciUser,
79	CIPasswordFile: ciPasswordFile,
80	CIPATFile: ciPATFile,
81	AgentUserHost: userHost,
82	AgentPort: port,
83	AgentStateDir: agentStateDir,
84	ServerGocoverdir: getenv("SERVER_GOCOVERDIR"),
85	AgentGocoverdir: getenv("AGENT_GOCOVERDIR"),
86	CoverOut: getenv("COVER_OUT"),
87	SmokeGate: getenv("SMOKE_GATE"),
88	SmokeVMUser: smokeVMUser,
89	SmokeUserCAFile: getenv("SMOKE_USER_CA_FILE"),
90	}, nil
91	}
92
93	// parseAgentHost takes the AGENT_HOSTS value (space-separated "user@host[:port]"
94	// entries) and returns the first entry's user@host plus its port. The port is
95	// the substring after the LAST colon when that substring is entirely digits;
96	// otherwise there is no port and it defaults to 22.
97	func parseAgentHost(agentHosts string) (userHost string, port int) {
98	entry := strings.Fields(agentHosts)[0]
99
100	if idx := strings.LastIndex(entry, ":"); idx != -1 {
101	if p, err := strconv.Atoi(entry[idx+1:]); err == nil {
102	return entry[:idx], p
103	}
104	}
105	return entry, 22
106	}

cmd/eitri-smoke/config_test.go

Old		New
@@ -1,153 +0,0 @@
1	package main
2
3	import (
4	"strings"
5	"testing"
6	)
7
8	// fakeGetenv returns a getenv func backed by a map, so tests never touch the
9	// real process environment.
10	func fakeGetenv(vals map[string]string) func(string) string {
11	return func(key string) string { return vals[key] }
12	}
13
14	func requiredVals() map[string]string {
15	return map[string]string{
16	"SERVER_URL": "https://server.example:8443",
17	"CI_USER": "ci@eitri.local",
18	"CI_PASSWORD_FILE": "/etc/eitri/ci-password",
19	"CI_PAT_FILE": "/etc/eitri/ci-pat",
20	"AGENT_HOSTS": "ubuntu@10.0.0.5:2222",
21	"AGENT_STATE_DIR": "/var/lib/eitri-agent",
22	}
23	}
24
25	func TestLoadConfigParsesAgentHostWithPort(t *testing.T) {
26	cfg, err := loadConfig(fakeGetenv(requiredVals()))
27	if err != nil {
28	t.Fatalf("loadConfig: %v", err)
29	}
30	if cfg.AgentUserHost != "ubuntu@10.0.0.5" {
31	t.Errorf("AgentUserHost = %q, want ubuntu@10.0.0.5", cfg.AgentUserHost)
32	}
33	if cfg.AgentPort != 2222 {
34	t.Errorf("AgentPort = %d, want 2222", cfg.AgentPort)
35	}
36	}
37
38	func TestLoadConfigDefaultsPortWithoutColon(t *testing.T) {
39	vals := requiredVals()
40	vals["AGENT_HOSTS"] = "ubuntu@10.0.0.5"
41	cfg, err := loadConfig(fakeGetenv(vals))
42	if err != nil {
43	t.Fatalf("loadConfig: %v", err)
44	}
45	if cfg.AgentUserHost != "ubuntu@10.0.0.5" {
46	t.Errorf("AgentUserHost = %q, want ubuntu@10.0.0.5", cfg.AgentUserHost)
47	}
48	if cfg.AgentPort != 22 {
49	t.Errorf("AgentPort = %d, want 22", cfg.AgentPort)
50	}
51	}
52
53	func TestLoadConfigMultiEntryTakesFirst(t *testing.T) {
54	vals := requiredVals()
55	vals["AGENT_HOSTS"] = "ubuntu@10.0.0.5:2200 ubuntu@10.0.0.6:2201"
56	cfg, err := loadConfig(fakeGetenv(vals))
57	if err != nil {
58	t.Fatalf("loadConfig: %v", err)
59	}
60	if cfg.AgentUserHost != "ubuntu@10.0.0.5" \|\| cfg.AgentPort != 2200 {
61	t.Errorf("got %q:%d, want ubuntu@10.0.0.5:2200", cfg.AgentUserHost, cfg.AgentPort)
62	}
63	}
64
65	func TestLoadConfigMissingRequiredVars(t *testing.T) {
66	cases := []struct {
67	name string
68	unset string
69	wantErr string
70	}{
71	{"missing server url", "SERVER_URL", "SERVER_URL"},
72	{"missing ci user", "CI_USER", "CI_USER"},
73	{"missing ci password file", "CI_PASSWORD_FILE", "CI_PASSWORD_FILE"},
74	{"missing ci pat file", "CI_PAT_FILE", "CI_PAT_FILE"},
75	{"missing agent hosts", "AGENT_HOSTS", "AGENT_HOSTS"},
76	{"missing agent state dir", "AGENT_STATE_DIR", "AGENT_STATE_DIR"},
77	}
78	for _, tc := range cases {
79	t.Run(tc.name, func(t *testing.T) {
80	vals := requiredVals()
81	delete(vals, tc.unset)
82	_, err := loadConfig(fakeGetenv(vals))
83	if err == nil {
84	t.Fatal("loadConfig: want error, got nil")
85	}
86	if !strings.Contains(err.Error(), tc.wantErr) {
87	t.Errorf("error = %q, want to mention %q", err.Error(), tc.wantErr)
88	}
89	})
90	}
91	}
92
93	func TestLoadConfigMissingAllRequiredVars(t *testing.T) {
94	_, err := loadConfig(fakeGetenv(nil))
95	if err == nil {
96	t.Fatal("loadConfig: want error, got nil")
97	}
98	for _, want := range []string{"SERVER_URL", "CI_USER", "CI_PASSWORD_FILE", "CI_PAT_FILE", "AGENT_HOSTS", "AGENT_STATE_DIR"} {
99	if !strings.Contains(err.Error(), want) {
100	t.Errorf("error = %q, missing %q", err.Error(), want)
101	}
102	}
103	}
104
105	func TestLoadConfigSmokeGateDefaults(t *testing.T) {
106	cfg, err := loadConfig(fakeGetenv(requiredVals()))
107	if err != nil {
108	t.Fatalf("loadConfig: %v", err)
109	}
110	if cfg.SmokeGate != "" {
111	t.Errorf("SmokeGate = %q, want empty", cfg.SmokeGate)
112	}
113	if cfg.SmokeVMUser != "ubuntu" {
114	t.Errorf("SmokeVMUser = %q, want ubuntu", cfg.SmokeVMUser)
115	}
116	if cfg.SmokeUserCAFile != "" {
117	t.Errorf("SmokeUserCAFile = %q, want empty", cfg.SmokeUserCAFile)
118	}
119	}
120
121	func TestLoadConfigSmokeGatePassThrough(t *testing.T) {
122	vals := requiredVals()
123	vals["SMOKE_GATE"] = "gate.example:2222"
124	vals["SMOKE_VM_USER"] = "debian"
125	vals["SMOKE_USER_CA_FILE"] = "/etc/eitri-smoke/user_ca"
126	cfg, err := loadConfig(fakeGetenv(vals))
127	if err != nil {
128	t.Fatalf("loadConfig: %v", err)
129	}
130	if cfg.SmokeGate != "gate.example:2222" {
131	t.Errorf("SmokeGate = %q, want gate.example:2222", cfg.SmokeGate)
132	}
133	if cfg.SmokeVMUser != "debian" {
134	t.Errorf("SmokeVMUser = %q, want debian", cfg.SmokeVMUser)
135	}
136	if cfg.SmokeUserCAFile != "/etc/eitri-smoke/user_ca" {
137	t.Errorf("SmokeUserCAFile = %q, want /etc/eitri-smoke/user_ca", cfg.SmokeUserCAFile)
138	}
139	}
140
141	func TestLoadConfigOptionalCoverageVarsPassThrough(t *testing.T) {
142	vals := requiredVals()
143	vals["SERVER_GOCOVERDIR"] = "/tmp/server-cover"
144	vals["AGENT_GOCOVERDIR"] = "/tmp/agent-cover"
145	vals["COVER_OUT"] = "/tmp/out"
146	cfg, err := loadConfig(fakeGetenv(vals))
147	if err != nil {
148	t.Fatalf("loadConfig: %v", err)
149	}
150	if cfg.ServerGocoverdir != "/tmp/server-cover" \|\| cfg.AgentGocoverdir != "/tmp/agent-cover" \|\| cfg.CoverOut != "/tmp/out" {
151	t.Errorf("optional coverage vars not passed through: %+v", cfg)
152	}
153	}

cmd/eitri-smoke/coverage.go

Old		New
@@ -1,139 +0,0 @@
1	package main
2
3	import (
4	"bytes"
5	"context"
6	"fmt"
7	"os"
8	"os/exec"
9	"path/filepath"
10	"strconv"
11	"strings"
12	"time"
13	)
14
15	// covdataMergeArgs builds the `go tool covdata merge` argument list that unions
16	// the per-binary coverage directories in inputDirs into outDir. Pure so the
17	// command assembly is unit-testable without running the toolchain.
18	func covdataMergeArgs(inputDirs []string, outDir string) []string {
19	return []string{"tool", "covdata", "merge",
20	"-i=" + strings.Join(inputDirs, ","),
21	"-o=" + outDir}
22	}
23
24	// covdataTextfmtArgs builds the `go tool covdata textfmt` argument list that
25	// renders the merged coverage in inDir as a textual profile at outFile.
26	func covdataTextfmtArgs(inDir, outFile string) []string {
27	return []string{"tool", "covdata", "textfmt",
28	"-i=" + inDir,
29	"-o=" + outFile}
30	}
31
32	// collectCoverage snapshots live coverage from the running server and agent,
33	// pulls both GOCOVERDIRs together, and merges them into cfg.CoverOut, printing
34	// the total. It needs both cfg.ServerGocoverdir and cfg.AgentGocoverdir; when
35	// either is empty it is a no-op with an explanatory note. Any failure here is
36	// the caller's to treat as non-fatal — the boot gate is the authoritative check.
37	func collectCoverage(ctx context.Context, cfg Config) error {
38	if cfg.ServerGocoverdir == "" \|\| cfg.AgentGocoverdir == "" {
39	fmt.Println("coverage: SERVER_GOCOVERDIR/AGENT_GOCOVERDIR not both set — skipping collection")
40	return nil
41	}
42
43	// 1. Snapshot: SIGUSR1 makes each covsnap handler flush counters into its
44	// GOCOVERDIR. The server is local; the agent runs as root on the remote, so
45	// its pkill needs sudo over SSH.
46	if err := runCmd(exec.CommandContext(ctx, "pkill", "-USR1", "-x", "eitri-server")); err != nil {
47	fmt.Fprintln(os.Stderr, "coverage: signalling local eitri-server:", err)
48	}
49	if err := runCmd(sshCommand(ctx, cfg, "sudo pkill -USR1 -x eitri-agent")); err != nil {
50	fmt.Fprintln(os.Stderr, "coverage: signalling remote eitri-agent:", err)
51	}
52	// Let both handlers finish writing before we read their dirs.
53	time.Sleep(2 * time.Second)
54
55	// 2. Collect: the server dir is local; stream the agent's over SSH via a
56	// sudo tar pipe (its files are root-owned, so a plain scp can't read them).
57	agentDir, err := os.MkdirTemp("", "eitri-smoke-agentcov-")
58	if err != nil {
59	return fmt.Errorf("temp dir for agent coverage: %w", err)
60	}
61	defer os.RemoveAll(agentDir)
62	if err := pullAgentCoverage(ctx, cfg, agentDir); err != nil {
63	return err
64	}
65
66	// 3. Merge both binaries' data, render a textual profile, and read the total.
67	// Start from a clean CoverOut: covdata refuses to read a directory that
68	// mixes covermodes, so a stale profile from an earlier deploy would clash
69	// with this run's data.
70	if err := os.RemoveAll(cfg.CoverOut); err != nil {
71	return fmt.Errorf("clean cover out %q: %w", cfg.CoverOut, err)
72	}
73	if err := os.MkdirAll(cfg.CoverOut, 0o755); err != nil {
74	return fmt.Errorf("mkdir cover out %q: %w", cfg.CoverOut, err)
75	}
76	if err := runCmd(exec.CommandContext(ctx, "go", covdataMergeArgs([]string{cfg.ServerGocoverdir, agentDir}, cfg.CoverOut)...)); err != nil {
77	return fmt.Errorf("covdata merge: %w", err)
78	}
79	textOut := filepath.Join(cfg.CoverOut, "coverage.txt")
80	if err := runCmd(exec.CommandContext(ctx, "go", covdataTextfmtArgs(cfg.CoverOut, textOut)...)); err != nil {
81	return fmt.Errorf("covdata textfmt: %w", err)
82	}
83
84	total, err := coverageTotal(ctx, textOut)
85	if err != nil {
86	return err
87	}
88	fmt.Printf("coverage: %s (profile: %s)\n", total, textOut)
89	return nil
90	}
91
92	// pullAgentCoverage streams the agent's GOCOVERDIR contents into destDir over a
93	// sudo tar pipe (the coverage files are root-owned on the remote).
94	func pullAgentCoverage(ctx context.Context, cfg Config, destDir string) error {
95	tarball, err := sshCommand(ctx, cfg, "sudo tar -C '"+cfg.AgentGocoverdir+"' -cf - .").Output()
96	if err != nil {
97	return fmt.Errorf("stream agent coverage over ssh: %w", err)
98	}
99	untar := exec.CommandContext(ctx, "tar", "-C", destDir, "-xf", "-")
100	untar.Stdin = bytes.NewReader(tarball)
101	if err := runCmd(untar); err != nil {
102	return fmt.Errorf("extract agent coverage: %w", err)
103	}
104	return nil
105	}
106
107	// coverageTotal returns the final "total:" line of `go tool cover -func`.
108	func coverageTotal(ctx context.Context, profile string) (string, error) {
109	out, err := exec.CommandContext(ctx, "go", "tool", "cover", "-func="+profile).Output()
110	if err != nil {
111	return "", fmt.Errorf("go tool cover -func: %w", err)
112	}
113	lines := strings.Split(strings.TrimSpace(string(out)), "\n")
114	return lines[len(lines)-1], nil
115	}
116
117	// sshCommand builds the ssh invocation used to reach the agent host, matching
118	// the flags the deploy boot-gate uses to reach the agent host.
119	func sshCommand(ctx context.Context, cfg Config, remoteCmd string) *exec.Cmd {
120	return exec.CommandContext(ctx, "ssh",
121	"-p", strconv.Itoa(cfg.AgentPort),
122	"-o", "BatchMode=yes",
123	"-o", "ConnectTimeout=10",
124	cfg.AgentUserHost, remoteCmd)
125	}
126
127	// runCmd runs cmd, capturing stderr so a failure carries the command's own
128	// diagnostic rather than a bare exit code.
129	func runCmd(cmd *exec.Cmd) error {
130	var errb bytes.Buffer
131	cmd.Stderr = &errb
132	if err := cmd.Run(); err != nil {
133	if msg := strings.TrimSpace(errb.String()); msg != "" {
134	return fmt.Errorf("%s: %w: %s", cmd.Args[0], err, msg)
135	}
136	return fmt.Errorf("%s: %w", cmd.Args[0], err)
137	}
138	return nil
139	}

cmd/eitri-smoke/coverage_test.go

Old		New
@@ -1,30 +0,0 @@
1	package main
2
3	import (
4	"slices"
5	"testing"
6	)
7
8	func TestCovdataMergeArgs(t *testing.T) {
9	got := covdataMergeArgs([]string{"/cov/server", "/cov/agent"}, "/cov/out")
10	want := []string{"tool", "covdata", "merge", "-i=/cov/server,/cov/agent", "-o=/cov/out"}
11	if !slices.Equal(got, want) {
12	t.Fatalf("covdataMergeArgs = %v, want %v", got, want)
13	}
14	}
15
16	func TestCovdataMergeArgsSingleInput(t *testing.T) {
17	got := covdataMergeArgs([]string{"/only"}, "/out")
18	want := []string{"tool", "covdata", "merge", "-i=/only", "-o=/out"}
19	if !slices.Equal(got, want) {
20	t.Fatalf("covdataMergeArgs = %v, want %v", got, want)
21	}
22	}
23
24	func TestCovdataTextfmtArgs(t *testing.T) {
25	got := covdataTextfmtArgs("/cov/out", "/cov/out/coverage.txt")
26	want := []string{"tool", "covdata", "textfmt", "-i=/cov/out", "-o=/cov/out/coverage.txt"}
27	if !slices.Equal(got, want) {
28	t.Fatalf("covdataTextfmtArgs = %v, want %v", got, want)
29	}
30	}

cmd/eitri-smoke/gatecheck.go

Old		New
@@ -1,78 +0,0 @@
1	package main
2
3	import (
4	"context"
5	"errors"
6	"fmt"
7	"strings"
8	"time"
9
10	"github.com/a73x/eitri/internal/gateclient"
11	"golang.org/x/crypto/ssh"
12	)
13
14	// realGateHooks builds the live SSH-CA gate steps: register the smoke user CA
15	// with the tenant, and reach the guest through the gate to prove access. The
16	// tenant is the operator PAT's own tenant (derived via Me() by the caller), so
17	// the CA registration and connect names match the fleet's real partition.
18	func realGateHooks(cfg Config, tenant string, ca gateclient.CertAuthority, userCA ssh.Signer, now func() time.Time, sleep func(time.Duration)) *gateHooks {
19	auth := gateclient.NewGateAuth(ca, userCA, tenant, now)
20	return &gateHooks{
21	register: func(ctx context.Context) error { return auth.Register(ctx) },
22	exec: func(ctx context.Context, vmName string) error { return gateExec(ctx, cfg, auth, vmName, now, sleep) },
23	}
24	}
25
26	// gateExec proves the guest is reachable through the SSH-CA gate: it retries
27	// dial+login over the guest's pre-sshd boot window (a dial or session error
28	// mid-boot is expected, not fatal) until it logs in as cfg.SmokeVMUser and
29	// confirms `id -un` echoes that same user, or the 120s deadline expires.
30	func gateExec(ctx context.Context, cfg Config, auth gateclient.Credentials, vmName string, now func() time.Time, sleep func(time.Duration)) error {
31	var lastErr error
32	err := pollLoop(ctx, now, sleep, 120time.Second, 5time.Second, func() (bool, error) {
33	client, dialErr := gateclient.Dial(ctx, gateclient.DialConfig{
34	Gate: cfg.SmokeGate,
35	VMUser: cfg.SmokeVMUser,
36	Auth: auth,
37	}, vmName)
38	if dialErr != nil {
39	lastErr = dialErr
40	return false, nil
41	}
42	defer client.Close()
43
44	out, runErr := runGuestCommand(client, "id -un")
45	if runErr != nil {
46	lastErr = runErr
47	return false, nil
48	}
49
50	got := strings.TrimSpace(out)
51	if got != cfg.SmokeVMUser {
52	return false, fmt.Errorf("gate SSH logged into %q as %q, want %q", vmName, got, cfg.SmokeVMUser)
53	}
54	return true, nil
55	})
56	if err != nil {
57	if errors.Is(err, errPollTimeout) {
58	return fmt.Errorf("FAIL: could not reach guest %q through the gate within 120s: %w", vmName, lastErr)
59	}
60	return err
61	}
62	return nil
63	}
64
65	// runGuestCommand runs cmd in a new session on client and returns its stdout.
66	func runGuestCommand(client *ssh.Client, cmd string) (string, error) {
67	session, err := client.NewSession()
68	if err != nil {
69	return "", fmt.Errorf("open ssh session: %w", err)
70	}
71	defer session.Close()
72
73	out, err := session.Output(cmd)
74	if err != nil {
75	return "", fmt.Errorf("run %q: %w", cmd, err)
76	}
77	return string(out), nil
78	}

cmd/eitri-smoke/login.go

Old		New
@@ -1,108 +0,0 @@
1	package main
2
3	import (
4	"fmt"
5	"net/http"
6	"net/http/cookiejar"
7	"net/url"
8	"time"
9
10	"github.com/a73x/eitri/internal/server/api/client"
11	)
12
13	// sessionCookie is the name of the console session cookie the callback sets.
14	// Its presence in the jar after the credential POST is the one honest signal
15	// that sign-in succeeded — a failed password re-renders the login form as a
16	// plain 200 without ever setting it.
17	const sessionCookie = "eitri_session"
18
19	// loginPAT signs in through the real OIDC code flow — the same doors a browser
20	// walks — and mints a short-lived PAT for this run. There is no special grant
21	// for machines (spec §3): CI registers a flat-file user and then drives the
22	// standard authorization-code flow headlessly against the plain-HTML login
23	// form, lands an ordinary session, and mints a PAT through the normal route.
24	func loginPAT(serverURL, email, password string) (string, error) {
25	base, err := url.Parse(serverURL)
26	if err != nil {
27	return "", fmt.Errorf("parse server url %q: %w", serverURL, err)
28	}
29	jar, err := cookiejar.New(nil)
30	if err != nil {
31	return "", fmt.Errorf("cookie jar: %w", err)
32	}
33	// A default (redirect-following) client: GET /auth/login bounces through the
34	// issuer's authorize endpoint to the login form, and the credential POST
35	// runs issuer -> /auth/callback -> / — we want every hop followed so the
36	// session cookie lands in the jar and resp.Request.URL is the form's URL.
37	hc := &http.Client{Jar: jar, Timeout: 30 * time.Second}
38
39	// GET /auth/login follows the redirect chain to the issuer's login form.
40	// We POST credentials straight back to resp.Request.URL — the authorize
41	// URL we landed on, query intact — so no HTML parsing is needed and the
42	// form's action attribute is never consulted. (eitri-oidc accepts the
43	// credential POST on that same URL by contract, spec §2.1.)
44	resp, err := hc.Get(serverURL + "/auth/login")
45	if err != nil {
46	return "", fmt.Errorf("GET /auth/login: %w", err)
47	}
48	resp.Body.Close()
49	if resp.StatusCode != http.StatusOK {
50	return "", fmt.Errorf("sign-in: login form GET returned %d (want 200) at %s", resp.StatusCode, resp.Request.URL)
51	}
52	formURL := resp.Request.URL.String()
53
54	// POST credentials to the form URL; the redirects run issuer -> /auth/callback
55	// -> / and the callback plants the session cookie in the jar on success.
56	resp2, err := hc.PostForm(formURL, url.Values{
57	"email": {email},
58	"password": {password},
59	})
60	if err != nil {
61	return "", fmt.Errorf("POST credentials: %w", err)
62	}
63	resp2.Body.Close()
64
65	// A failed sign-in re-renders the login form as a 200 without a session
66	// cookie. Detect success by the cookie's presence in the jar, never by the
67	// status. The password is never included in the error.
68	if cookieByName(jar.Cookies(base), sessionCookie) == nil {
69	return "", fmt.Errorf("sign-in failed for %q: no %s cookie after credential POST "+
70	"(final status %d at %s) — check the CI user exists in eitri-oidc and the password matches",
71	email, sessionCookie, resp2.StatusCode, resp2.Request.URL)
72	}
73
74	// Mint the PAT through the shared client, riding the session cookie in the
75	// jar (Token left empty so no Bearer header is sent).
76	c := &client.Client{BaseURL: serverURL, HTTP: hc}
77	tok, err := c.CreateAPIToken("boot-gate", time.Hour)
78	if err != nil {
79	return "", fmt.Errorf("mint boot-gate PAT: %w", err)
80	}
81	return tok.Token, nil
82	}
83
84	// proveCredentialChain is the boot-gate's phase-1 proof: it confirms the minted
85	// PAT resolves to a non-empty tenant via GET /api/v1/me, exercising the whole
86	// issuer -> login -> session -> PAT-mint chain without touching VMs. It returns
87	// the tenant handle so the caller can log which tenant the ci user landed in.
88	func proveCredentialChain(serverURL, token string) (string, error) {
89	c := &client.Client{BaseURL: serverURL, Token: token, HTTP: &http.Client{Timeout: 30 * time.Second}}
90	me, err := c.Me()
91	if err != nil {
92	return "", fmt.Errorf("credential-chain proof: Me() with minted PAT: %w", err)
93	}
94	if me.Tenant == "" {
95	return "", fmt.Errorf("credential-chain proof: minted PAT resolved to an empty tenant")
96	}
97	return me.Tenant, nil
98	}
99
100	// cookieByName returns the named cookie from cookies, or nil if absent.
101	func cookieByName(cookies []http.Cookie, name string) http.Cookie {
102	for _, c := range cookies {
103	if c.Name == name {
104	return c
105	}
106	}
107	return nil
108	}

cmd/eitri-smoke/login_test.go

Old		New
@@ -1,171 +0,0 @@
1	package main
2
3	import (
4	"context"
5	"net/http"
6	"net/http/httptest"
7	"path/filepath"
8	"strings"
9	"testing"
10	"time"
11
12	"github.com/a73x/eitri/internal/oidcprovider"
13	"github.com/a73x/eitri/internal/server/api"
14	"github.com/a73x/eitri/internal/server/api/client"
15	"github.com/a73x/eitri/internal/server/hub"
16	"github.com/a73x/eitri/internal/server/registry"
17	"github.com/a73x/eitri/internal/server/store"
18	)
19
20	// smokeLoginEnv is a whole eitri-server /auth + /api stack wired to a real
21	// internal/oidcprovider issuer — the exact credential chain a deploy walks. It
22	// exists to prove loginPAT end-to-end: sign in through the login form, land a
23	// session, mint a PAT, and use it against the API.
24	type smokeLoginEnv struct {
25	serverURL string
26	st *store.Store
27	}
28
29	// newSmokeLoginEnv stands up the issuer and server with one seeded user and
30	// returns the server's base URL. It mirrors internal/server/api/auth_test.go's
31	// fixture: the httptest listener binds on construction so we know the server's
32	// address (hence the OIDC redirect URL) before wiring the two ends.
33	func newSmokeLoginEnv(t *testing.T, email, password string) smokeLoginEnv {
34	t.Helper()
35
36	apiSrv := httptest.NewUnstartedServer(nil)
37	publicURL := "http://" + apiSrv.Listener.Addr().String()
38
39	st, err := store.Open(t.TempDir()+"/db", "10.77.0.0/16")
40	if err != nil {
41	t.Fatalf("store.Open: %v", err)
42	}
43	t.Cleanup(func() { st.Close() })
44
45	usersPath := filepath.Join(t.TempDir(), "users.json")
46	if err := oidcprovider.AddUser(usersPath, email, password); err != nil {
47	t.Fatalf("AddUser: %v", err)
48	}
49	prov, err := oidcprovider.New(oidcprovider.Config{
50	UsersFile: usersPath,
51	SigningKey: filepath.Join(t.TempDir(), "signing.key"),
52	Clients: []oidcprovider.Client{{ID: "eitri-console", RedirectURL: publicURL + "/auth/callback"}},
53	})
54	if err != nil {
55	t.Fatalf("oidcprovider.New: %v", err)
56	}
57	oidcSrv := httptest.NewServer(prov.Handler())
58	t.Cleanup(oidcSrv.Close)
59	prov.SetIssuer(oidcSrv.URL)
60
61	a := api.New(api.Config{
62	HostSecret: []byte("hostsecret"),
63	OIDC: api.OIDCConfig{Issuer: oidcSrv.URL, ClientID: "eitri-console", PublicURL: publicURL},
64	}, st, registry.New(time.Now), hub.New())
65	t.Cleanup(a.Close)
66
67	root := http.NewServeMux()
68	root.Handle("/api/", a.Handler())
69	root.Handle("/auth/", a.AuthHandler())
70	apiSrv.Config.Handler = root
71	apiSrv.Start()
72	t.Cleanup(apiSrv.Close)
73
74	return smokeLoginEnv{serverURL: publicURL, st: st}
75	}
76
77	func TestLoginPATMintsUsableToken(t *testing.T) {
78	const (
79	email = "ci@eitri.local"
80	password = "hunter2hunter2"
81	)
82	env := newSmokeLoginEnv(t, email, password)
83
84	token, err := loginPAT(env.serverURL, email, password)
85	if err != nil {
86	t.Fatalf("loginPAT: %v", err)
87	}
88	if token == "" {
89	t.Fatal("loginPAT returned an empty token")
90	}
91
92	// The PAT must actually authenticate an ordinary API call.
93	c := &client.Client{BaseURL: env.serverURL, Token: token, HTTP: &http.Client{Timeout: 10 * time.Second}}
94	me, err := c.Me()
95	if err != nil {
96	t.Fatalf("Me() with minted PAT: %v", err)
97	}
98	if me.Email != email {
99	t.Errorf("Me().Email = %q, want %q", me.Email, email)
100	}
101	if _, err := c.ListHosts(context.Background()); err != nil {
102	t.Errorf("ListHosts() with minted PAT: %v", err)
103	}
104	}
105
106	// TestProveCredentialChain is the phase-1 proof: signing in and minting a PAT
107	// must resolve to a non-empty tenant via Me() (the ci user's JIT tenant),
108	// without any VM involvement.
109	func TestProveCredentialChain(t *testing.T) {
110	const (
111	email = "ci@eitri.local"
112	password = "hunter2hunter2"
113	)
114	env := newSmokeLoginEnv(t, email, password)
115
116	token, err := loginPAT(env.serverURL, email, password)
117	if err != nil {
118	t.Fatalf("loginPAT: %v", err)
119	}
120	tenant, err := proveCredentialChain(env.serverURL, token)
121	if err != nil {
122	t.Fatalf("proveCredentialChain: %v", err)
123	}
124	if tenant == "" {
125	t.Fatal("proveCredentialChain returned an empty tenant")
126	}
127	}
128
129	// TestScenarioPATDerivesTenant is the phase-2 contract: a scenario client built
130	// from an operator-minted PAT derives its tenant via Me() — not from any env or
131	// hardcoded default. The store mints the PAT directly, standing in for the
132	// console-minted "deploy" token a real operator saves to CI_PAT_FILE.
133	func TestScenarioPATDerivesTenant(t *testing.T) {
134	env := newSmokeLoginEnv(t, "ci@eitri.local", "hunter2hunter2")
135
136	// A distinct operator tenant with a row (handleMe 401s on a rowless tenant).
137	tn, err := env.st.CreateTenantForIdentity("https://op.example", "op-subject", "op@example.com")
138	if err != nil {
139	t.Fatalf("CreateTenantForIdentity: %v", err)
140	}
141	secret, _, err := env.st.CreateAPIToken(tn.ID, "deploy", 0)
142	if err != nil {
143	t.Fatalf("CreateAPIToken: %v", err)
144	}
145
146	c := &client.Client{BaseURL: env.serverURL, Token: secret, HTTP: &http.Client{Timeout: 10 * time.Second}}
147	me, err := c.Me()
148	if err != nil {
149	t.Fatalf("Me() with store-minted PAT: %v", err)
150	}
151	if me.Tenant != tn.ID {
152	t.Errorf("Me().Tenant = %q, want %q (derived, not assumed)", me.Tenant, tn.ID)
153	}
154	}
155
156	func TestLoginPATWrongPasswordFails(t *testing.T) {
157	const email = "ci@eitri.local"
158	env := newSmokeLoginEnv(t, email, "the-right-password")
159
160	_, err := loginPAT(env.serverURL, email, "the-wrong-password")
161	if err == nil {
162	t.Fatal("loginPAT: want error on wrong password, got nil")
163	}
164	// The failure must name the sign-in problem and must not echo the password.
165	if !strings.Contains(err.Error(), "sign-in failed") \|\| !strings.Contains(err.Error(), email) {
166	t.Errorf("error = %q, want it to mention the sign-in failure and the user", err)
167	}
168	if strings.Contains(err.Error(), "the-wrong-password") {
169	t.Errorf("error must not echo the password: %q", err)
170	}
171	}

cmd/eitri-smoke/main.go

Old		New
@@ -1,19 +1,14 @@
		1	// eitri-smoke: the deploy boot-gate harness. It signs in through the real OIDC
		2	// flow, then creates a throwaway VM, proves it boots and is reachable through
		3	// the SSH-CA gate, and reaps it — failing the deploy on any error. All behavior
		4	// lives in internal/smoke (Run); this package is wiring only (arch R14).
1	package main	5	package main
2		6
3	import (	7	import (
4	"context"
5	"crypto/rand"
6	"encoding/hex"
7	"fmt"	8	"fmt"
8	"net/http"
9	"os"	9	"os"
10	"os/exec"
11	"path/filepath"
12	"strconv"
13	"strings"
14	"time"
15		10
16	"github.com/a73x/eitri/internal/server/api/client"	11	"github.com/a73x/eitri/internal/smoke"
17	"github.com/a73x/eitri/internal/version"	12	"github.com/a73x/eitri/internal/version"
18	)	13	)
19		14
@@ -22,140 +17,8 @@ func main() {
22	fmt.Println(version.Version)	17	fmt.Println(version.Version)
23	return	18	return
24	}	19	}
25	if err := run(); err != nil {	20	if err := smoke.Run(); err != nil {
26	fmt.Fprintln(os.Stderr, "eitri-smoke:", err)	21	fmt.Fprintln(os.Stderr, "eitri-smoke:", err)
27	os.Exit(1)	22	os.Exit(1)
28	}	23	}
29	}	24	}
30
31	func run() error {
32	cfg, err := loadConfig(os.Getenv)
33	if err != nil {
34	return err
35	}
36
37	// Phase 1 — credential-chain proof. The admin token is gone: the boot-gate
38	// authenticates like a human. Read the machine identity's password (CI_USER,
39	// the deploy identity), sign in through the real OIDC code flow, mint a
40	// short-lived PAT, and confirm it resolves to a tenant (spec §3). This proves
41	// issuer, login form, session, and PAT mint end to end; it deliberately never
42	// touches VMs — the machine identity's JIT tenant owns no hosts (the fleet's
43	// `default` tenant is human-owned), so the lifecycle half runs as the operator
44	// below.
45	pwBytes, err := os.ReadFile(cfg.CIPasswordFile)
46	if err != nil {
47	return fmt.Errorf("read CI password file %q: %w", cfg.CIPasswordFile, err)
48	}
49	password := strings.TrimRight(string(pwBytes), "\r\n")
50
51	token, err := loginPAT(cfg.ServerURL, cfg.CIUser, password)
52	if err != nil {
53	return err
54	}
55	ciTenant, err := proveCredentialChain(cfg.ServerURL, token)
56	if err != nil {
57	return err
58	}
59	fmt.Printf("credential chain OK (tenant %s)\n", ciTenant)
60
61	// Phase 2 — VM lifecycle. Authenticate with an operator-minted PAT read from
62	// CI_PAT_FILE (a normal, tenant-scoped console PAT — spec §3's automation
63	// story). The scenario's tenant is DERIVED via Me(), never assumed, and
64	// threaded into the user-CA registration and gate connect name below.
65	patBytes, err := os.ReadFile(cfg.CIPATFile)
66	if err != nil {
67	return fmt.Errorf("read CI PAT file %q: %w", cfg.CIPATFile, err)
68	}
69	pat := strings.TrimRight(string(patBytes), "\r\n")
70
71	api := &client.Client{
72	BaseURL: cfg.ServerURL,
73	Token: pat,
74	UserCALabel: "eitri-smoke",
75	HTTP: &http.Client{Timeout: 30 * time.Second},
76	}
77	me, err := api.Me()
78	if err != nil {
79	return fmt.Errorf("resolve operator PAT tenant: %w", err)
80	}
81	if me.Tenant == "" {
82	return fmt.Errorf("operator PAT (%s) resolved to an empty tenant", cfg.CIPATFile)
83	}
84	tenant := me.Tenant
85	fmt.Printf("operator PAT tenant: %s\n", tenant)
86
87	vmName, err := randVMName()
88	if err != nil {
89	return fmt.Errorf("generate vm name: %w", err)
90	}
91
92	var gate *gateHooks
93	if cfg.SmokeGate != "" && cfg.SmokeUserCAFile != "" {
94	userCA, err := loadOrCreateUserCA(cfg.SmokeUserCAFile)
95	if err != nil {
96	return fmt.Errorf("load smoke user CA: %w", err)
97	}
98	gate = realGateHooks(cfg, tenant, api, userCA, time.Now, time.Sleep)
99	} else {
100	fmt.Fprintln(os.Stderr, "eitri-smoke: gate SSH check skipped (SMOKE_GATE/SMOKE_USER_CA_FILE not set)")
101	}
102
103	ctx := context.Background()
104	msg, err := runScenario(ctx, cfg, vmName, api, realRunSSH(cfg.AgentUserHost, cfg.AgentPort), gate, time.Now, time.Sleep, realReadPubKey)
105	if err != nil {
106	return err
107	}
108	fmt.Println(msg)
109
110	// The boot gate has passed. Collect integration coverage as a by-product
111	// when COVER_OUT is set; a failure here must NOT fail the deploy, so warn
112	// and carry on — the gate above is the authoritative check.
113	if cfg.CoverOut != "" {
114	if err := collectCoverage(ctx, cfg); err != nil {
115	fmt.Fprintln(os.Stderr, "eitri-smoke: coverage collection failed (boot gate still passed):", err)
116	}
117	}
118	return nil
119	}
120
121	// randVMName generates a throwaway VM name of the form "smoke-<8 hex digits>",
122	// unique enough that concurrent smoke runs don't collide.
123	func randVMName() (string, error) {
124	var b [4]byte
125	if _, err := rand.Read(b[:]); err != nil {
126	return "", err
127	}
128	return "smoke-" + hex.EncodeToString(b[:]), nil
129	}
130
131	// realRunSSH returns the real sshFunc: it shells out to the ssh binary
132	// against userHost:port with the deploy boot-gate's ssh flags, running
133	// remoteCmd non-interactively and returning its stdout.
134	func realRunSSH(userHost string, port int) sshFunc {
135	return func(ctx context.Context, remoteCmd string) (string, error) {
136	cmd := exec.CommandContext(ctx, "ssh",
137	"-p", strconv.Itoa(port),
138	"-o", "BatchMode=yes",
139	"-o", "ConnectTimeout=10",
140	userHost, remoteCmd)
141	out, err := cmd.Output()
142	return string(out), err
143	}
144	}
145
146	// realReadPubKey reads the local operator's SSH public key, trying
147	// ~/.ssh/id_ed25519.pub then ~/.ssh/id_rsa.pub. It returns "" (not an error)
148	// if neither is present — the scenario tolerates a keyless VM.
149	func realReadPubKey() string {
150	home, err := os.UserHomeDir()
151	if err != nil {
152	return ""
153	}
154	for _, name := range []string{"id_ed25519.pub", "id_rsa.pub"} {
155	data, err := os.ReadFile(filepath.Join(home, ".ssh", name))
156	if err == nil {
157	return strings.TrimSpace(string(data))
158	}
159	}
160	return ""
161	}

cmd/eitri-smoke/scenario.go

Old		New
@@ -1,196 +0,0 @@
1	package main
2
3	import (
4	"context"
5	"errors"
6	"fmt"
7	"regexp"
8	"time"
9
10	"github.com/a73x/eitri/internal/server/api/client"
11	)
12
13	// bootedPattern / panickedPattern classify a guest's serial console text. They
14	// mirror the boot-gate's grep -aiE serial-console patterns exactly.
15	var (
16	bootedPattern = regexp.MustCompile(`(?i)Welcome to.Ubuntu\|login:\|Reached target.Multi-User`)
17	panickedPattern = regexp.MustCompile(`(?i)Kernel panic\|Cannot open root`)
18	errPollTimeout = errors.New("poll timeout")
19	)
20
21	// classifySerial inspects a guest's serial console text and reports whether it
22	// shows evidence of a successful userspace boot and/or a kernel panic /
23	// root-mount failure. It is pure — no I/O — so the boot-proof classification
24	// logic is fully unit-testable.
25	func classifySerial(text string) (booted, panicked bool) {
26	return bootedPattern.MatchString(text), panickedPattern.MatchString(text)
27	}
28
29	// bootProofCommand builds the remote shell command run over SSH on the agent
30	// host to read and sanitize a VM's serial console log.
31	func bootProofCommand(agentStateDir, vmID string) string {
32	return fmt.Sprintf(`sudo cat '%s/vms/%s/serial.log' 2>/dev/null \| tr -cd '\11\12\15\40-\176'`, agentStateDir, vmID)
33	}
34
35	// vmAPI is the subset of the shared API client the scenario needs. Declaring
36	// it lets tests supply a fake instead of a real HTTP-backed *client.Client.
37	type vmAPI interface {
38	ListHosts(ctx context.Context) ([]client.Host, error)
39	CreateVM(ctx context.Context, req client.CreateVMRequest) (client.CreateVMResponse, error)
40	ListVMs(ctx context.Context) ([]client.VM, error)
41	DeleteVM(ctx context.Context, id string) error
42	}
43
44	// getVM finds the VM with the given id in the current listing. The bool
45	// return reports whether it was present. There is deliberately no GET-one
46	// endpoint, so this list-filter is the smoke's lookup.
47	func getVM(ctx context.Context, c vmAPI, id string) (client.VM, bool, error) {
48	vms, err := c.ListVMs(ctx)
49	if err != nil {
50	return client.VM{}, false, err
51	}
52	for _, vm := range vms {
53	if vm.ID == id {
54	return vm, true, nil
55	}
56	}
57	return client.VM{}, false, nil
58	}
59
60	// sshFunc runs remoteCmd on the agent host over SSH and returns its stdout.
61	type sshFunc func(ctx context.Context, remoteCmd string) (string, error)
62
63	// gateHooks bundles the optional SSH-CA gate steps. nil means "skip the gate".
64	type gateHooks struct {
65	register func(ctx context.Context) error // upload the smoke user CA to the tenant (before create)
66	exec func(ctx context.Context, vmName string) error // reach the guest through the gate (after boot)
67	}
68
69	// pollLoop calls attempt repeatedly (with sleep between calls) until attempt
70	// reports done, returns a non-nil error, or the deadline (now()+timeout) is
71	// reached, in which case it returns errPollTimeout. now and sleep are
72	// injected so callers can drive the deadline logic with a virtual clock.
73	func pollLoop(ctx context.Context, now func() time.Time, sleep func(time.Duration), timeout, interval time.Duration, attempt func() (done bool, err error)) error {
74	deadline := now().Add(timeout)
75	for {
76	if err := ctx.Err(); err != nil {
77	return err
78	}
79	done, err := attempt()
80	if err != nil {
81	return err
82	}
83	if done {
84	return nil
85	}
86	if !now().Before(deadline) {
87	return errPollTimeout
88	}
89	sleep(interval)
90	}
91	}
92
93	// runScenario drives the full register -> create -> ready -> boot-proof ->
94	// gate-exec -> reap sequence against c (the API), runSSH (the boot-proof
95	// transport), and readPubKey (the local SSH key source). vmName is the
96	// pre-generated name for the throwaway VM. gate, when non-nil, registers the
97	// smoke's user CA with the tenant before create (the guest bakes its trusted
98	// CAs at boot, so registration MUST happen first) and proves gate SSH access
99	// after the boot-proof — a hard gate, so a failure there fails the scenario.
100	// now/sleep are the injected clock so the poll deadlines are unit-testable
101	// without real waiting. On success it returns the human-readable COMPLETE
102	// line; on any failure it returns a descriptive error.
103	func runScenario(ctx context.Context, cfg Config, vmName string, c vmAPI, runSSH sshFunc, gate *gateHooks, now func() time.Time, sleep func(time.Duration), readPubKey func() string) (string, error) {
104	if gate != nil {
105	if err := gate.register(ctx); err != nil {
106	return "", fmt.Errorf("register smoke user CA: %w", err)
107	}
108	}
109
110	hostList, err := c.ListHosts(ctx)
111	if err != nil {
112	return "", fmt.Errorf("list hosts: %w", err)
113	}
114	if len(hostList) == 0 {
115	return "", errors.New("no hosts available")
116	}
117	hostID := hostList[0].ID
118
119	sshKey := readPubKey()
120
121	start := now()
122	created, err := c.CreateVM(ctx, client.CreateVMRequest{HostID: hostID, Name: vmName, SSHAuthorizedKey: sshKey})
123	if err != nil {
124	return "", fmt.Errorf("create vm: %w", err)
125	}
126	vmID := created.ID
127
128	var lastPhase string
129	err = pollLoop(ctx, now, sleep, 600time.Second, 5time.Second, func() (bool, error) {
130	vm, _, err := getVM(ctx, c, vmID)
131	if err != nil {
132	return false, fmt.Errorf("poll vm ready: %w", err)
133	}
134	lastPhase = vm.Phase
135	return vm.Phase == "ready" && vm.AssignedIP != "", nil
136	})
137	if err != nil {
138	if errors.Is(err, errPollTimeout) {
139	return "", fmt.Errorf("FAIL: VM not ready within 600s (phase=%s)", lastPhase)
140	}
141	return "", err
142	}
143	coldStart := now().Sub(start)
144
145	remoteCmd := bootProofCommand(cfg.AgentStateDir, vmID)
146	err = pollLoop(ctx, now, sleep, 180time.Second, 6time.Second, func() (bool, error) {
147	serial, sshErr := runSSH(ctx, remoteCmd)
148	if sshErr != nil {
149	// Match the bash scenario's "\|\| true": an SSH hiccup mid-boot is
150	// not fatal on its own, just an empty-serial iteration.
151	serial = ""
152	}
153	booted, panicked := classifySerial(serial)
154	if panicked {
155	return false, errors.New("FAIL: guest panic / root-mount failure in serial log")
156	}
157	return booted, nil
158	})
159	if err != nil {
160	if errors.Is(err, errPollTimeout) {
161	return "", errors.New("FAIL: no userspace boot evidence in serial within 180s")
162	}
163	return "", err
164	}
165
166	gateOK := false
167	if gate != nil {
168	if err := gate.exec(ctx, vmName); err != nil {
169	return "", err
170	}
171	gateOK = true
172	}
173
174	if err := c.DeleteVM(ctx, vmID); err != nil {
175	return "", fmt.Errorf("delete vm: %w", err)
176	}
177	err = pollLoop(ctx, now, sleep, 120time.Second, 5time.Second, func() (bool, error) {
178	_, present, err := getVM(ctx, c, vmID)
179	if err != nil {
180	return false, fmt.Errorf("poll vm reaped: %w", err)
181	}
182	return !present, nil
183	})
184	if err != nil {
185	if errors.Is(err, errPollTimeout) {
186	return "", errors.New("FAIL: VM not hard-deleted within 120s of tombstone")
187	}
188	return "", err
189	}
190
191	msg := fmt.Sprintf("SMOKE COMPLETE — booted under UEFI, cold_start=%ds, reaped OK", int64(coldStart.Seconds()))
192	if gateOK {
193	msg += ", gate SSH: ok"
194	}
195	return msg, nil
196	}

cmd/eitri-smoke/scenario_test.go

Old		New
@@ -1,455 +0,0 @@
1	package main
2
3	import (
4	"context"
5	"errors"
6	"strings"
7	"testing"
8	"time"
9
10	"github.com/a73x/eitri/internal/server/api/client"
11	)
12
13	// --- classifySerial -------------------------------------------------------
14
15	func TestClassifySerialBooted(t *testing.T) {
16	text := "[ 5.123456] Ubuntu 22.04.3 LTS ubuntu-vm ttyS0\n\nubuntu-vm login: "
17	booted, panicked := classifySerial(text)
18	if !booted {
19	t.Error("booted = false, want true")
20	}
21	if panicked {
22	t.Error("panicked = true, want false")
23	}
24	}
25
26	func TestClassifySerialPanic(t *testing.T) {
27	text := "[ 2.345678] Kernel panic - not syncing: VFS: Unable to mount root fs on unknown-block(0,0)"
28	booted, panicked := classifySerial(text)
29	if booted {
30	t.Error("booted = true, want false")
31	}
32	if !panicked {
33	t.Error("panicked = false, want true")
34	}
35	}
36
37	func TestClassifySerialStillBooting(t *testing.T) {
38	text := "[ 0.123456] Booting Linux on physical CPU 0x0\n[ 0.234567] Linux version 6.5.0"
39	booted, panicked := classifySerial(text)
40	if booted {
41	t.Error("booted = true, want false")
42	}
43	if panicked {
44	t.Error("panicked = true, want false")
45	}
46	}
47
48	// --- fakes for runScenario -------------------------------------------------
49
50	// fakeClock is a controllable now()/sleep() pair: sleep advances the virtual
51	// clock instead of waiting, so deadline logic runs at test speed.
52	type fakeClock struct{ t time.Time }
53
54	func (c *fakeClock) now() time.Time { return c.t }
55	func (c *fakeClock) sleep(d time.Duration) { c.t = c.t.Add(d) }
56
57	// testAPI implements vmAPI by delegating to per-test closures.
58	type testAPI struct {
59	listHostsFunc func(ctx context.Context) ([]client.Host, error)
60	createVMFunc func(ctx context.Context, req client.CreateVMRequest) (client.CreateVMResponse, error)
61	listVMsFunc func(ctx context.Context) ([]client.VM, error)
62	deleteVMFunc func(ctx context.Context, id string) error
63	}
64
65	func (a *testAPI) ListHosts(ctx context.Context) ([]client.Host, error) {
66	return a.listHostsFunc(ctx)
67	}
68	func (a *testAPI) CreateVM(ctx context.Context, req client.CreateVMRequest) (client.CreateVMResponse, error) {
69	return a.createVMFunc(ctx, req)
70	}
71	func (a *testAPI) ListVMs(ctx context.Context) ([]client.VM, error) {
72	return a.listVMsFunc(ctx)
73	}
74	func (a *testAPI) DeleteVM(ctx context.Context, id string) error { return a.deleteVMFunc(ctx, id) }
75
76	func baseCfg() Config {
77	return Config{AgentStateDir: "/var/lib/eitri-agent", AgentUserHost: "ubuntu@10.0.0.5", AgentPort: 22}
78	}
79
80	func noopReadPubKey() string { return "ssh-ed25519 AAAAfake test@smoke" }
81
82	// --- runScenario: success path ---------------------------------------------
83
84	func TestRunScenarioSuccess(t *testing.T) {
85	listVMsCalls := 0
86	api := &testAPI{
87	listHostsFunc: func(ctx context.Context) ([]client.Host, error) {
88	return []client.Host{{ID: "host-1"}}, nil
89	},
90	createVMFunc: func(ctx context.Context, req client.CreateVMRequest) (client.CreateVMResponse, error) {
91	if req.HostID != "host-1" {
92	t.Errorf("CreateVM HostID = %q, want host-1", req.HostID)
93	}
94	return client.CreateVMResponse{ID: "vm-1"}, nil
95	},
96	listVMsFunc: func(ctx context.Context) ([]client.VM, error) {
97	listVMsCalls++
98	switch {
99	case listVMsCalls < 3:
100	return []client.VM{{ID: "vm-1", Phase: "booting"}}, nil
101	case listVMsCalls == 3:
102	return []client.VM{{ID: "vm-1", Phase: "ready", AssignedIP: "10.0.0.9"}}, nil
103	default:
104	// Reap poll: absent from the first check.
105	return nil, nil
106	}
107	},
108	deleteVMFunc: func(ctx context.Context, id string) error {
109	if id != "vm-1" {
110	t.Errorf("deleteVM id = %q, want vm-1", id)
111	}
112	return nil
113	},
114	}
115
116	sshCalls := 0
117	runSSH := func(ctx context.Context, remoteCmd string) (string, error) {
118	sshCalls++
119	if !strings.Contains(remoteCmd, "vm-1") {
120	t.Errorf("remoteCmd = %q, want it to reference vm-1", remoteCmd)
121	}
122	if sshCalls < 2 {
123	return "[ 0.1] Booting Linux...", nil
124	}
125	return "Ubuntu 24.04 LTS ubuntu-vm login: ", nil
126	}
127
128	clock := &fakeClock{t: time.Unix(0, 0)}
129	msg, err := runScenario(context.Background(), baseCfg(), "smoke-test", api, runSSH, nil, clock.now, clock.sleep, noopReadPubKey)
130	if err != nil {
131	t.Fatalf("runScenario: %v", err)
132	}
133	if !strings.Contains(msg, "SMOKE COMPLETE") \|\| !strings.Contains(msg, "reaped OK") {
134	t.Errorf("message = %q, want SMOKE COMPLETE ... reaped OK", msg)
135	}
136	if !strings.Contains(msg, "cold_start=") {
137	t.Errorf("message = %q, want cold_start=", msg)
138	}
139	}
140
141	// --- runScenario: serial panic -----------------------------------------
142
143	func TestRunScenarioSerialPanicFails(t *testing.T) {
144	api := &testAPI{
145	listHostsFunc: func(ctx context.Context) ([]client.Host, error) {
146	return []client.Host{{ID: "host-1"}}, nil
147	},
148	createVMFunc: func(ctx context.Context, req client.CreateVMRequest) (client.CreateVMResponse, error) {
149	return client.CreateVMResponse{ID: "vm-1"}, nil
150	},
151	listVMsFunc: func(ctx context.Context) ([]client.VM, error) {
152	return []client.VM{{ID: "vm-1", Phase: "ready", AssignedIP: "10.0.0.9"}}, nil
153	},
154	deleteVMFunc: func(ctx context.Context, id string) error {
155	t.Fatal("DeleteVM should not be called after a panic")
156	return nil
157	},
158	}
159	runSSH := func(ctx context.Context, remoteCmd string) (string, error) {
160	return "Kernel panic - not syncing: VFS: Unable to mount root fs", nil
161	}
162
163	clock := &fakeClock{t: time.Unix(0, 0)}
164	_, err := runScenario(context.Background(), baseCfg(), "smoke-test", api, runSSH, nil, clock.now, clock.sleep, noopReadPubKey)
165	if err == nil {
166	t.Fatal("runScenario: want error, got nil")
167	}
168	if !strings.Contains(err.Error(), "panic") {
169	t.Errorf("error = %q, want it to mention panic", err.Error())
170	}
171	}
172
173	// --- runScenario: never-ready timeout ---------------------------------
174
175	func TestRunScenarioNeverReadyTimesOut(t *testing.T) {
176	api := &testAPI{
177	listHostsFunc: func(ctx context.Context) ([]client.Host, error) {
178	return []client.Host{{ID: "host-1"}}, nil
179	},
180	createVMFunc: func(ctx context.Context, req client.CreateVMRequest) (client.CreateVMResponse, error) {
181	return client.CreateVMResponse{ID: "vm-1"}, nil
182	},
183	listVMsFunc: func(ctx context.Context) ([]client.VM, error) {
184	return []client.VM{{ID: "vm-1", Phase: "booting"}}, nil
185	},
186	deleteVMFunc: func(ctx context.Context, id string) error {
187	t.Fatal("DeleteVM should not be called when the VM never becomes ready")
188	return nil
189	},
190	}
191	runSSH := func(ctx context.Context, remoteCmd string) (string, error) {
192	t.Fatal("runSSH should not be called when the VM never becomes ready")
193	return "", nil
194	}
195
196	clock := &fakeClock{t: time.Unix(0, 0)}
197	_, err := runScenario(context.Background(), baseCfg(), "smoke-test", api, runSSH, nil, clock.now, clock.sleep, noopReadPubKey)
198	if err == nil {
199	t.Fatal("runScenario: want error, got nil")
200	}
201	if !strings.Contains(err.Error(), "FAIL: VM not ready within 600s") {
202	t.Errorf("error = %q, want FAIL: VM not ready within 600s ...", err.Error())
203	}
204	if !strings.Contains(err.Error(), "phase=booting") {
205	t.Errorf("error = %q, want it to include phase=booting", err.Error())
206	}
207	}
208
209	// --- runScenario: never-reaped timeout ---------------------------------
210
211	func TestRunScenarioNeverReapedTimesOut(t *testing.T) {
212	api := &testAPI{
213	listHostsFunc: func(ctx context.Context) ([]client.Host, error) {
214	return []client.Host{{ID: "host-1"}}, nil
215	},
216	createVMFunc: func(ctx context.Context, req client.CreateVMRequest) (client.CreateVMResponse, error) {
217	return client.CreateVMResponse{ID: "vm-1"}, nil
218	},
219	listVMsFunc: func(ctx context.Context) ([]client.VM, error) {
220	// Always present, even after delete — models a stuck reap.
221	return []client.VM{{ID: "vm-1", Phase: "ready", AssignedIP: "10.0.0.9"}}, nil
222	},
223	deleteVMFunc: func(ctx context.Context, id string) error { return nil },
224	}
225	runSSH := func(ctx context.Context, remoteCmd string) (string, error) {
226	return "Ubuntu 24.04 LTS ubuntu-vm login: ", nil
227	}
228
229	clock := &fakeClock{t: time.Unix(0, 0)}
230	_, err := runScenario(context.Background(), baseCfg(), "smoke-test", api, runSSH, nil, clock.now, clock.sleep, noopReadPubKey)
231	if err == nil {
232	t.Fatal("runScenario: want error, got nil")
233	}
234	if !strings.Contains(err.Error(), "FAIL: VM not hard-deleted within 120s") {
235	t.Errorf("error = %q, want FAIL: VM not hard-deleted within 120s ...", err.Error())
236	}
237	}
238
239	// --- runScenario: no hosts ------------------------------------------------
240
241	func TestRunScenarioNoHosts(t *testing.T) {
242	api := &testAPI{
243	listHostsFunc: func(ctx context.Context) ([]client.Host, error) { return nil, nil },
244	createVMFunc: func(ctx context.Context, req client.CreateVMRequest) (client.CreateVMResponse, error) {
245	t.Fatal("CreateVM should not be called with no hosts")
246	return client.CreateVMResponse{}, nil
247	},
248	listVMsFunc: func(ctx context.Context) ([]client.VM, error) {
249	t.Fatal("ListVMs should not be called with no hosts")
250	return nil, nil
251	},
252	deleteVMFunc: func(ctx context.Context, id string) error {
253	t.Fatal("DeleteVM should not be called with no hosts")
254	return nil
255	},
256	}
257	clock := &fakeClock{t: time.Unix(0, 0)}
258	_, err := runScenario(context.Background(), baseCfg(), "smoke-test", api, nil, nil, clock.now, clock.sleep, noopReadPubKey)
259	if err == nil {
260	t.Fatal("runScenario: want error, got nil")
261	}
262	}
263
264	// --- runScenario: gate hooks -----------------------------------------------
265
266	// happyPathAPI returns a testAPI that succeeds all the way through reap,
267	// tracking call order in calls (a shared slice each hook also appends to).
268	func happyPathAPI(t testing.T, calls []string) *testAPI {
269	t.Helper()
270	listVMsCalls := 0
271	return &testAPI{
272	listHostsFunc: func(ctx context.Context) ([]client.Host, error) {
273	return []client.Host{{ID: "host-1"}}, nil
274	},
275	createVMFunc: func(ctx context.Context, req client.CreateVMRequest) (client.CreateVMResponse, error) {
276	calls = append(calls, "createVM")
277	return client.CreateVMResponse{ID: "vm-1"}, nil
278	},
279	listVMsFunc: func(ctx context.Context) ([]client.VM, error) {
280	listVMsCalls++
281	switch {
282	case listVMsCalls < 3:
283	return []client.VM{{ID: "vm-1", Phase: "booting"}}, nil
284	case listVMsCalls == 3:
285	return []client.VM{{ID: "vm-1", Phase: "ready", AssignedIP: "10.0.0.9"}}, nil
286	default:
287	return nil, nil
288	}
289	},
290	deleteVMFunc: func(ctx context.Context, id string) error { return nil },
291	}
292	}
293
294	func happyPathRunSSH() sshFunc {
295	sshCalls := 0
296	return func(ctx context.Context, remoteCmd string) (string, error) {
297	sshCalls++
298	if sshCalls < 2 {
299	return "[ 0.1] Booting Linux...", nil
300	}
301	return "Ubuntu 24.04 LTS ubuntu-vm login: ", nil
302	}
303	}
304
305	func TestRunScenarioGateRegistersBeforeCreateAndExecsAfterBoot(t *testing.T) {
306	var calls []string
307	api := happyPathAPI(t, &calls)
308	gate := &gateHooks{
309	register: func(ctx context.Context) error {
310	calls = append(calls, "register")
311	return nil
312	},
313	exec: func(ctx context.Context, vmName string) error {
314	if vmName != "smoke-test" {
315	t.Errorf("gate.exec vmName = %q, want smoke-test", vmName)
316	}
317	calls = append(calls, "exec")
318	return nil
319	},
320	}
321
322	clock := &fakeClock{t: time.Unix(0, 0)}
323	msg, err := runScenario(context.Background(), baseCfg(), "smoke-test", api, happyPathRunSSH(), gate, clock.now, clock.sleep, noopReadPubKey)
324	if err != nil {
325	t.Fatalf("runScenario: %v", err)
326	}
327	if !strings.Contains(msg, "gate SSH: ok") {
328	t.Errorf("message = %q, want it to mention gate SSH: ok", msg)
329	}
330
331	want := []string{"register", "createVM", "exec"}
332	if len(calls) != len(want) {
333	t.Fatalf("call order = %v, want %v", calls, want)
334	}
335	for i, c := range want {
336	if calls[i] != c {
337	t.Errorf("call order = %v, want %v", calls, want)
338	break
339	}
340	}
341	}
342
343	func TestRunScenarioGateRegisterErrorAbortsBeforeCreate(t *testing.T) {
344	api := &testAPI{
345	listHostsFunc: func(ctx context.Context) ([]client.Host, error) {
346	t.Fatal("ListHosts should not be called when register fails")
347	return nil, nil
348	},
349	createVMFunc: func(ctx context.Context, req client.CreateVMRequest) (client.CreateVMResponse, error) {
350	t.Fatal("CreateVM should not be called when register fails")
351	return client.CreateVMResponse{}, nil
352	},
353	listVMsFunc: func(ctx context.Context) ([]client.VM, error) {
354	t.Fatal("ListVMs should not be called when register fails")
355	return nil, nil
356	},
357	deleteVMFunc: func(ctx context.Context, id string) error {
358	t.Fatal("DeleteVM should not be called when register fails")
359	return nil
360	},
361	}
362	gate := &gateHooks{
363	register: func(ctx context.Context) error { return errors.New("upload boom") },
364	exec: func(ctx context.Context, vmName string) error {
365	t.Fatal("exec should not be called when register fails")
366	return nil
367	},
368	}
369
370	clock := &fakeClock{t: time.Unix(0, 0)}
371	_, err := runScenario(context.Background(), baseCfg(), "smoke-test", api, nil, gate, clock.now, clock.sleep, noopReadPubKey)
372	if err == nil {
373	t.Fatal("runScenario: want error, got nil")
374	}
375	if !strings.Contains(err.Error(), "register smoke user CA") {
376	t.Errorf("error = %q, want it to mention register smoke user CA", err.Error())
377	}
378	}
379
380	func TestRunScenarioGateExecErrorFails(t *testing.T) {
381	var calls []string
382	api := happyPathAPI(t, &calls)
383	api.deleteVMFunc = func(ctx context.Context, id string) error {
384	t.Fatal("DeleteVM should not be called when gate exec fails")
385	return nil
386	}
387	gate := &gateHooks{
388	register: func(ctx context.Context) error { return nil },
389	exec: func(ctx context.Context, vmName string) error {
390	return errors.New("FAIL: gate SSH boom")
391	},
392	}
393
394	clock := &fakeClock{t: time.Unix(0, 0)}
395	_, err := runScenario(context.Background(), baseCfg(), "smoke-test", api, happyPathRunSSH(), gate, clock.now, clock.sleep, noopReadPubKey)
396	if err == nil {
397	t.Fatal("runScenario: want error, got nil")
398	}
399	if !strings.Contains(err.Error(), "gate SSH boom") {
400	t.Errorf("error = %q, want it to mention gate SSH boom", err.Error())
401	}
402	}
403
404	// --- pollLoop ---------------------------------------------------------
405
406	func TestPollLoopReturnsErrPollTimeoutAtDeadline(t *testing.T) {
407	clock := &fakeClock{t: time.Unix(0, 0)}
408	calls := 0
409	err := pollLoop(context.Background(), clock.now, clock.sleep, 10time.Second, 5time.Second, func() (bool, error) {
410	calls++
411	return false, nil
412	})
413	if !errors.Is(err, errPollTimeout) {
414	t.Errorf("err = %v, want errPollTimeout", err)
415	}
416	if calls == 0 {
417	t.Error("attempt was never called")
418	}
419	}
420
421	func TestPollLoopPropagatesAttemptError(t *testing.T) {
422	wantErr := errors.New("boom")
423	clock := &fakeClock{t: time.Unix(0, 0)}
424	err := pollLoop(context.Background(), clock.now, clock.sleep, 10time.Second, 5time.Second, func() (bool, error) {
425	return false, wantErr
426	})
427	if !errors.Is(err, wantErr) {
428	t.Errorf("err = %v, want %v", err, wantErr)
429	}
430	}
431
432	// TestGetVMFiltersById pins that getVM discriminates by ID within a listing
433	// that contains other VMs — on the live fleet the list always does (eitri-dev
434	// at minimum), so a match-first regression would poll the wrong VM's
435	// phase/IP during the ready wait.
436	func TestGetVMFiltersById(t *testing.T) {
437	api := &testAPI{listVMsFunc: func(ctx context.Context) ([]client.VM, error) {
438	return []client.VM{
439	{ID: "vm-1", Phase: "creating"},
440	{ID: "vm-2", Phase: "ready", AssignedIP: "10.77.1.9"},
441	}, nil
442	}}
443
444	vm, present, err := getVM(context.Background(), api, "vm-2")
445	if err != nil \|\| !present {
446	t.Fatalf("getVM(vm-2) = present %v, err %v; want present", present, err)
447	}
448	if vm.ID != "vm-2" \|\| vm.Phase != "ready" \|\| vm.AssignedIP != "10.77.1.9" {
449	t.Errorf("getVM(vm-2) returned the wrong row: %+v", vm)
450	}
451
452	if _, present, err := getVM(context.Background(), api, "vm-3"); err != nil \|\| present {
453	t.Errorf("getVM(vm-3) against a non-empty list = present %v, err %v; want absent", present, err)
454	}
455	}

cmd/eitri-smoke/userca.go

Old		New
@@ -1,56 +0,0 @@
1	package main
2
3	import (
4	"crypto/ed25519"
5	"crypto/rand"
6	"encoding/pem"
7	"errors"
8	"fmt"
9	"io/fs"
10	"os"
11	"path/filepath"
12
13	"golang.org/x/crypto/ssh"
14	)
15
16	// loadOrCreateUserCA loads the smoke's persistent user CA key from path,
17	// generating and persisting a fresh ed25519 key on first run so subsequent
18	// smoke runs reuse the same CA identity (the gate check registers this key's
19	// public half with the tenant once; a fresh key every run would mean an
20	// ever-growing set of trusted, never-reused CAs).
21	func loadOrCreateUserCA(path string) (ssh.Signer, error) {
22	data, err := os.ReadFile(path)
23	if err == nil {
24	signer, err := ssh.ParsePrivateKey(data)
25	if err != nil {
26	return nil, fmt.Errorf("parse user CA key %q: %w", path, err)
27	}
28	return signer, nil
29	}
30	if !errors.Is(err, fs.ErrNotExist) {
31	return nil, fmt.Errorf("read user CA key %q: %w", path, err)
32	}
33
34	_, priv, err := ed25519.GenerateKey(rand.Reader)
35	if err != nil {
36	return nil, fmt.Errorf("generate user CA key: %w", err)
37	}
38	block, err := ssh.MarshalPrivateKey(priv, "")
39	if err != nil {
40	return nil, fmt.Errorf("marshal user CA key: %w", err)
41	}
42	pemBytes := pem.EncodeToMemory(block)
43
44	if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
45	return nil, fmt.Errorf("mkdir user CA key dir: %w", err)
46	}
47	if err := os.WriteFile(path, pemBytes, 0o600); err != nil {
48	return nil, fmt.Errorf("write user CA key %q: %w", path, err)
49	}
50
51	signer, err := ssh.NewSignerFromSigner(priv)
52	if err != nil {
53	return nil, fmt.Errorf("build user CA signer: %w", err)
54	}
55	return signer, nil
56	}

cmd/eitri-smoke/userca_test.go

Old		New
@@ -1,37 +0,0 @@
1	package main
2
3	import (
4	"os"
5	"path/filepath"
6	"testing"
7
8	"golang.org/x/crypto/ssh"
9	)
10
11	func TestLoadOrCreateUserCACreatesThenReuses(t *testing.T) {
12	dir := t.TempDir()
13	path := filepath.Join(dir, "nested", "user_ca")
14
15	s1, err := loadOrCreateUserCA(path)
16	if err != nil {
17	t.Fatalf("loadOrCreateUserCA (create): %v", err)
18	}
19	s2, err := loadOrCreateUserCA(path)
20	if err != nil {
21	t.Fatalf("loadOrCreateUserCA (reuse): %v", err)
22	}
23
24	line1 := string(ssh.MarshalAuthorizedKey(s1.PublicKey()))
25	line2 := string(ssh.MarshalAuthorizedKey(s2.PublicKey()))
26	if line1 != line2 {
27	t.Errorf("public keys differ across calls: %q vs %q", line1, line2)
28	}
29
30	info, err := os.Stat(path)
31	if err != nil {
32	t.Fatalf("stat key file: %v", err)
33	}
34	if mode := info.Mode().Perm(); mode != 0o600 {
35	t.Errorf("file mode = %o, want 0600", mode)
36	}
37	}

cmd/eitri/main.go

Old		New
@@ -1,10 +1,11 @@
1	// Command eitri is the end-user client: SSH into fleet VMs through the	1	// Command eitri is the end-user client: SSH into fleet VMs through the
2	// jump gate with self-signed short-lived certs (eitri ssh) and register	2	// jump gate with self-signed short-lived certs (eitri ssh) and register
3	// tenant user CAs (eitri ca upload). See internal/cli and docs/ssh-access.md.	3	// tenant user CAs (eitri ca upload). All behavior lives in internal/cli
		4	// (Main); this package is wiring only (arch R14). See docs/ssh-access.md.
4	package main	5	package main
5		6
6	import (	7	import (
7	"context"	8	"errors"
8	"fmt"	9	"fmt"
9	"os"	10	"os"
10		11
@@ -12,31 +13,14 @@ import (
12	"github.com/a73x/eitri/internal/version"	13	"github.com/a73x/eitri/internal/version"
13	)	14	)
14		15
15	const usage = `usage:
16	eitri ssh <vm> [ssh args / remote command...]
17	eitri ca upload [<tenant>] <ca-public-key-file>
18	eitri --version
19
20	env: EITRI_URL, EITRI_GATE (required for ssh); EITRI_TOKEN (required for ca);
21	EITRI_CA, EITRI_TENANT, EITRI_KEY, EITRI_KNOWN_HOSTS (optional)`
22
23	func main() {	16	func main() {
24	if len(os.Args) > 1 && os.Args[1] == "--version" {	17	if len(os.Args) > 1 && os.Args[1] == "--version" {
25	fmt.Println(version.Version)	18	fmt.Println(version.Version)
26	return	19	return
27	}	20	}
28	if len(os.Args) < 2 {	21	err := cli.Main(os.Args[1:], os.Stdout)
29	fmt.Fprintln(os.Stderr, usage)	22	if errors.Is(err, cli.ErrUsage) {
30	os.Exit(2)	23	fmt.Fprintln(os.Stderr, cli.Usage)
31	}
32	var err error
33	switch os.Args[1] {
34	case "ssh":
35	err = runSSH(os.Args[2:])
36	case "ca":
37	err = runCA(os.Args[2:])
38	default:
39	fmt.Fprintln(os.Stderr, usage)
40	os.Exit(2)	24	os.Exit(2)
41	}	25	}
42	if err != nil {	26	if err != nil {
@@ -44,46 +28,3 @@ func main() {
44	os.Exit(1)	28	os.Exit(1)
45	}	29	}
46	}	30	}
47
48	func runSSH(args []string) error {
49	if len(args) >= 1 && (args[0] == "-h" \|\| args[0] == "--help") {
50	fmt.Println("usage: eitri ssh <vm> [ssh args / remote command...]")
51	return nil
52	}
53	if len(args) < 1 {
54	return fmt.Errorf("usage: eitri ssh <vm> [ssh args / remote command...]")
55	}
56	env, err := cli.FromEnv()
57	if err != nil {
58	return err
59	}
60	return cli.RunSSH(context.Background(), env, args[0], args[1:])
61	}
62
63	func runCA(args []string) error {
64	if len(args) < 1 \|\| args[0] != "upload" {
65	return fmt.Errorf("usage: eitri ca upload [<tenant>] <ca-public-key-file>")
66	}
67	rest := args[1:]
68	if len(rest) < 1 \|\| len(rest) > 2 {
69	return fmt.Errorf("usage: eitri ca upload [<tenant>] <ca-public-key-file>")
70	}
71	tenant, pub := os.Getenv("EITRI_TENANT"), rest[0]
72	if len(rest) == 2 {
73	tenant, pub = rest[0], rest[1]
74	}
75	if tenant == "" {
76	return fmt.Errorf("no tenant: pass one (eitri ca upload <tenant> <file>) or set EITRI_TENANT")
77	}
78	url := os.Getenv("EITRI_URL")
79	token := os.Getenv("EITRI_TOKEN")
80	if url == "" \|\| token == "" {
81	return fmt.Errorf("set EITRI_URL and EITRI_TOKEN (a personal access token)")
82	}
83	out, err := cli.UploadUserCA(context.Background(), url, token, tenant, pub)
84	if err != nil {
85	return err
86	}
87	fmt.Println(out)
88	return nil
89	}

docs/shape.html

Old		New
@@ -66,18 +66,7 @@
66	"plane": "binaries",	66	"plane": "binaries",
67	"synopsis": "eitri-agent: BYO-hardware agent.",	67	"synopsis": "eitri-agent: BYO-hardware agent.",
68	"imports": [	68	"imports": [
69	"internal/agent/bootstrap",	69	"internal/agent/run",
70	"internal/agent/cloudhv",
71	"internal/agent/enrollclient",
72	"internal/agent/imagecache",
73	"internal/agent/netenv",
74	"internal/agent/reconcile",
75	"internal/agent/seed",
76	"internal/agent/serialpump",
77	"internal/agent/state",
78	"internal/agent/syncclient",
79	"internal/covsnap",
80	"internal/joinblob",
81	"internal/version"	70	"internal/version"
82	]	71	]
83	},	72	},
@@ -94,9 +83,7 @@
94	"plane": "binaries",	83	"plane": "binaries",
95	"synopsis": "Command eitri-mcp is an MCP server exposing eitri VM tools to Claude: create/list/info/exec/write_file/read_file/destroy.",	84	"synopsis": "Command eitri-mcp is an MCP server exposing eitri VM tools to Claude: create/list/info/exec/write_file/read_file/destroy.",
96	"imports": [	85	"imports": [
97	"internal/gateclient",
98	"internal/mcpserver",	86	"internal/mcpserver",
99	"internal/server/api/client",
100	"internal/version"	87	"internal/version"
101	]	88	]
102	},	89	},
@@ -112,22 +99,9 @@
112	{	99	{
113	"importPath": "cmd/eitri-server",	100	"importPath": "cmd/eitri-server",
114	"plane": "binaries",	101	"plane": "binaries",
115	"synopsis": "eitri-server: single-node control plane (Phase 1: static admin token, no TLS termination here — front with a reverse proxy for TLS).",	102	"synopsis": "eitri-server: single-node control plane (no TLS termination here — front with a reverse proxy for TLS).",
116	"imports": [	103	"imports": [
117	"internal/covsnap",	104	"internal/server/boot",
118	"internal/joinblob",
119	"internal/server/api",
120	"internal/server/config",
121	"internal/server/health",
122	"internal/server/hub",
123	"internal/server/registry",
124	"internal/server/release",
125	"internal/server/sshca",
126	"internal/server/sshgate",
127	"internal/server/store",
128	"internal/server/syncsvc",
129	"internal/server/web",
130	"internal/transport",
131	"internal/version"	105	"internal/version"
132	]	106	]
133	},	107	},
@@ -151,10 +125,9 @@
151	{	125	{
152	"importPath": "cmd/eitri-smoke",	126	"importPath": "cmd/eitri-smoke",
153	"plane": "binaries",	127	"plane": "binaries",
154	"synopsis": "Command eitri-smoke drives the live eitri fleet through create -\u003e boot-proof -\u003e reap of one throwaway VM, exiting non-zero on any failure.",	128	"synopsis": "eitri-smoke: the deploy boot-gate harness.",
155	"imports": [	129	"imports": [
156	"internal/gateclient",	130	"internal/smoke",
157	"internal/server/api/client",
158	"internal/version"	131	"internal/version"
159	]	132	]
160	},	133	},
@@ -239,6 +212,25 @@
239	]	212	]
240	},	213	},
241	{	214	{
		215	"importPath": "internal/agent/run",
		216	"plane": "data",
		217	"synopsis": "Package run implements the eitri-agent command line behind a tested RunCLI so cmd/eitri-agent stays thin wiring (arch R14).",
		218	"imports": [
		219	"internal/agent/bootstrap",
		220	"internal/agent/cloudhv",
		221	"internal/agent/enrollclient",
		222	"internal/agent/imagecache",
		223	"internal/agent/netenv",
		224	"internal/agent/reconcile",
		225	"internal/agent/seed",
		226	"internal/agent/serialpump",
		227	"internal/agent/state",
		228	"internal/agent/syncclient",
		229	"internal/covsnap",
		230	"internal/joinblob"
		231	]
		232	},
		233	{
242	"importPath": "internal/agent/seed",	234	"importPath": "internal/agent/seed",
243	"plane": "data",	235	"plane": "data",
244	"synopsis": "Package seed builds the cloud-init NoCloud config-drive ISO (label CIDATA).",	236	"synopsis": "Package seed builds the cloud-init NoCloud config-drive ISO (label CIDATA).",
@@ -398,6 +390,27 @@
398	"imports": []	390	"imports": []
399	},	391	},
400	{	392	{
		393	"importPath": "internal/server/boot",
		394	"plane": "control",
		395	"synopsis": "Package boot implements the eitri-server command line behind a tested RunCLI so cmd/eitri-server stays thin wiring (arch R14).",
		396	"imports": [
		397	"internal/covsnap",
		398	"internal/joinblob",
		399	"internal/server/api",
		400	"internal/server/config",
		401	"internal/server/health",
		402	"internal/server/hub",
		403	"internal/server/registry",
		404	"internal/server/release",
		405	"internal/server/sshca",
		406	"internal/server/sshgate",
		407	"internal/server/store",
		408	"internal/server/syncsvc",
		409	"internal/server/web",
		410	"internal/transport"
		411	]
		412	},
		413	{
401	"importPath": "internal/server/config",	414	"importPath": "internal/server/config",
402	"plane": "control",	415	"plane": "control",
403	"synopsis": "Package config defines the eitri-server on-disk JSON configuration schema, loaded by the server binary (cmd/eitri-server) at startup.",	416	"synopsis": "Package config defines the eitri-server on-disk JSON configuration schema, loaded by the server binary (cmd/eitri-server) at startup.",
@@ -490,6 +503,15 @@
490	]	503	]
491	},	504	},
492	{	505	{
		506	"importPath": "internal/smoke",
		507	"plane": "tooling",
		508	"synopsis": "Package smoke is the deploy boot-gate harness.",
		509	"imports": [
		510	"internal/gateclient",
		511	"internal/server/api/client"
		512	]
		513	},
		514	{
493	"importPath": "internal/transport",	515	"importPath": "internal/transport",
494	"plane": "wire",	516	"plane": "wire",
495	"synopsis": "Package transport carries the agent↔server sync protocol over QUIC.",	517	"synopsis": "Package transport carries the agent↔server sync protocol over QUIC.",

docs/shape.json

Old		New
@@ -15,18 +15,7 @@
15	"plane": "binaries",	15	"plane": "binaries",
16	"synopsis": "eitri-agent: BYO-hardware agent.",	16	"synopsis": "eitri-agent: BYO-hardware agent.",
17	"imports": [	17	"imports": [
18	"internal/agent/bootstrap",	18	"internal/agent/run",
19	"internal/agent/cloudhv",
20	"internal/agent/enrollclient",
21	"internal/agent/imagecache",
22	"internal/agent/netenv",
23	"internal/agent/reconcile",
24	"internal/agent/seed",
25	"internal/agent/serialpump",
26	"internal/agent/state",
27	"internal/agent/syncclient",
28	"internal/covsnap",
29	"internal/joinblob",
30	"internal/version"	19	"internal/version"
31	]	20	]
32	},	21	},
@@ -43,9 +32,7 @@
43	"plane": "binaries",	32	"plane": "binaries",
44	"synopsis": "Command eitri-mcp is an MCP server exposing eitri VM tools to Claude: create/list/info/exec/write_file/read_file/destroy.",	33	"synopsis": "Command eitri-mcp is an MCP server exposing eitri VM tools to Claude: create/list/info/exec/write_file/read_file/destroy.",
45	"imports": [	34	"imports": [
46	"internal/gateclient",
47	"internal/mcpserver",	35	"internal/mcpserver",
48	"internal/server/api/client",
49	"internal/version"	36	"internal/version"
50	]	37	]
51	},	38	},
@@ -61,22 +48,9 @@
61	{	48	{
62	"importPath": "cmd/eitri-server",	49	"importPath": "cmd/eitri-server",
63	"plane": "binaries",	50	"plane": "binaries",
64	"synopsis": "eitri-server: single-node control plane (Phase 1: static admin token, no TLS termination here — front with a reverse proxy for TLS).",	51	"synopsis": "eitri-server: single-node control plane (no TLS termination here — front with a reverse proxy for TLS).",
65	"imports": [	52	"imports": [
66	"internal/covsnap",	53	"internal/server/boot",
67	"internal/joinblob",
68	"internal/server/api",
69	"internal/server/config",
70	"internal/server/health",
71	"internal/server/hub",
72	"internal/server/registry",
73	"internal/server/release",
74	"internal/server/sshca",
75	"internal/server/sshgate",
76	"internal/server/store",
77	"internal/server/syncsvc",
78	"internal/server/web",
79	"internal/transport",
80	"internal/version"	54	"internal/version"
81	]	55	]
82	},	56	},
@@ -100,10 +74,9 @@
100	{	74	{
101	"importPath": "cmd/eitri-smoke",	75	"importPath": "cmd/eitri-smoke",
102	"plane": "binaries",	76	"plane": "binaries",
103	"synopsis": "Command eitri-smoke drives the live eitri fleet through create -\u003e boot-proof -\u003e reap of one throwaway VM, exiting non-zero on any failure.",	77	"synopsis": "eitri-smoke: the deploy boot-gate harness.",
104	"imports": [	78	"imports": [
105	"internal/gateclient",	79	"internal/smoke",
106	"internal/server/api/client",
107	"internal/version"	80	"internal/version"
108	]	81	]
109	},	82	},
@@ -188,6 +161,25 @@
188	]	161	]
189	},	162	},
190	{	163	{
		164	"importPath": "internal/agent/run",
		165	"plane": "data",
		166	"synopsis": "Package run implements the eitri-agent command line behind a tested RunCLI so cmd/eitri-agent stays thin wiring (arch R14).",
		167	"imports": [
		168	"internal/agent/bootstrap",
		169	"internal/agent/cloudhv",
		170	"internal/agent/enrollclient",
		171	"internal/agent/imagecache",
		172	"internal/agent/netenv",
		173	"internal/agent/reconcile",
		174	"internal/agent/seed",
		175	"internal/agent/serialpump",
		176	"internal/agent/state",
		177	"internal/agent/syncclient",
		178	"internal/covsnap",
		179	"internal/joinblob"
		180	]
		181	},
		182	{
191	"importPath": "internal/agent/seed",	183	"importPath": "internal/agent/seed",
192	"plane": "data",	184	"plane": "data",
193	"synopsis": "Package seed builds the cloud-init NoCloud config-drive ISO (label CIDATA).",	185	"synopsis": "Package seed builds the cloud-init NoCloud config-drive ISO (label CIDATA).",
@@ -347,6 +339,27 @@
347	"imports": []	339	"imports": []
348	},	340	},
349	{	341	{
		342	"importPath": "internal/server/boot",
		343	"plane": "control",
		344	"synopsis": "Package boot implements the eitri-server command line behind a tested RunCLI so cmd/eitri-server stays thin wiring (arch R14).",
		345	"imports": [
		346	"internal/covsnap",
		347	"internal/joinblob",
		348	"internal/server/api",
		349	"internal/server/config",
		350	"internal/server/health",
		351	"internal/server/hub",
		352	"internal/server/registry",
		353	"internal/server/release",
		354	"internal/server/sshca",
		355	"internal/server/sshgate",
		356	"internal/server/store",
		357	"internal/server/syncsvc",
		358	"internal/server/web",
		359	"internal/transport"
		360	]
		361	},
		362	{
350	"importPath": "internal/server/config",	363	"importPath": "internal/server/config",
351	"plane": "control",	364	"plane": "control",
352	"synopsis": "Package config defines the eitri-server on-disk JSON configuration schema, loaded by the server binary (cmd/eitri-server) at startup.",	365	"synopsis": "Package config defines the eitri-server on-disk JSON configuration schema, loaded by the server binary (cmd/eitri-server) at startup.",
@@ -439,6 +452,15 @@
439	]	452	]
440	},	453	},
441	{	454	{
		455	"importPath": "internal/smoke",
		456	"plane": "tooling",
		457	"synopsis": "Package smoke is the deploy boot-gate harness.",
		458	"imports": [
		459	"internal/gateclient",
		460	"internal/server/api/client"
		461	]
		462	},
		463	{
442	"importPath": "internal/transport",	464	"importPath": "internal/transport",
443	"plane": "wire",	465	"plane": "wire",
444	"synopsis": "Package transport carries the agent↔server sync protocol over QUIC.",	466	"synopsis": "Package transport carries the agent↔server sync protocol over QUIC.",

internal/agent/cloudhv/cloudhv.go

Old		New
@@ -49,6 +49,17 @@ func New(st state.Store, chBin, firmware string, run agentexec.Runner) Provisi
49	return &Provisioner{st: st, chBin: chBin, firmware: firmware, run: run}	49	return &Provisioner{st: st, chBin: chBin, firmware: firmware, run: run}
50	}	50	}
51		51
		52	// RealRunner is the production one-shot command runner the host-touching agent
		53	// packages (netenv, imagecache, cloudhv, syncclient) run with. It spawns
		54	// name+args, waits, and returns their combined stdout/stderr. It lives here
		55	// because cloudhv is the data plane's one sanctioned os/exec site (arch R6/R7);
		56	// every other agent package receives it as an injected agentexec.Runner and so
		57	// never imports os/exec itself.
		58	func RealRunner(ctx context.Context, name string, args ...string) (string, error) {
		59	out, err := exec.CommandContext(ctx, name, args...).CombinedOutput()
		60	return string(out), err
		61	}
		62
52	// BootstrapDest maps a --ch-bin value to the filesystem path bootstrap may	63	// BootstrapDest maps a --ch-bin value to the filesystem path bootstrap may
53	// install the binary at. The two are different vocabularies: --ch-bin is	64	// install the binary at. The two are different vocabularies: --ch-bin is
54	// usually a bare command name resolved on $PATH at launch, which names no	65	// usually a bare command name resolved on $PATH at launch, which names no

internal/agent/run/cli.go

@@ @@ -0,0 +1,309 @@ @@
+// cli.go is the eitri-agent command line: `eitri-agent join <blob>` enrolls the
+// host once, and `eitri-agent` (no subcommand) runs the reconcile + sync loop
+// indefinitely. It lives here rather than in cmd/eitri-agent so the flag/config
+// assembly, validation, and dispatch are testable and coverage-gated (arch R14:
+// main packages are wiring only).
+// Package run implements the eitri-agent command line behind a tested RunCLI so
+// cmd/eitri-agent stays thin wiring (arch R14).
+package run
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"log/slog"
+	"os"
+	"os/signal"
+	"runtime"
+	"syscall"
+	"time"
+	"github.com/a73x/eitri/internal/agent/bootstrap"
+	"github.com/a73x/eitri/internal/agent/cloudhv"
+	"github.com/a73x/eitri/internal/agent/enrollclient"
+	"github.com/a73x/eitri/internal/agent/imagecache"
+	"github.com/a73x/eitri/internal/agent/netenv"
+	"github.com/a73x/eitri/internal/agent/reconcile"
+	"github.com/a73x/eitri/internal/agent/seed"
+	"github.com/a73x/eitri/internal/agent/serialpump"
+	"github.com/a73x/eitri/internal/agent/state"
+	"github.com/a73x/eitri/internal/agent/syncclient"
+	"github.com/a73x/eitri/internal/covsnap"
+	"github.com/a73x/eitri/internal/joinblob"
+)
+// Config carries serve's wiring, replacing a long positional list.
+type Config struct {
+	StateDir, CHBin, Firmware   string
+	BootstrapURL                string
+	TombstoneGrace, VanishGrace time.Duration
+	VMTimeout                   time.Duration
+	ImageCacheMaxGB             int64
+	// MaxConcurrentCreates bounds how much of the per-VM workers' create
+	// concurrency reaches the disk at once (0 = unlimited).
+	MaxConcurrentCreates int
+	// MaxVCPUs, MaxMemMB, MaxDiskGB cap the host resources this agent offers the
+	// fleet (0 = unlimited): advertised to the server AND enforced at VM boot.
+	MaxVCPUs, MaxMemMB, MaxDiskGB int64
+}
+// RunCLI dispatches the eitri-agent command line (everything after the binary
+// name, --version excluded — that stays in cmd/eitri-agent). It parses flags,
+// opens the state directory, then either enrolls the host ("join <blob>") or
+// runs the agent.
+func RunCLI(args []string) error {
+	cfg, rest, err := parseConfig(args)
+	if err != nil {
+		return err
+	}
+	st, err := state.Open(cfg.StateDir)
+	if err != nil {
+		return fmt.Errorf("open state dir: %w", err)
+	}
+	if len(rest) > 0 && rest[0] == "join" {
+		var blob string
+		if len(rest) > 1 {
+			blob = rest[1]
+		}
+		return join(st, blob)
+	}
+	return serve(st, cfg)
+}
+// parseConfig defines and parses the agent flags, returning the assembled
+// Config and any positional arguments (the "join <blob>" subcommand). Resource
+// caps are validated here so a negative cap is rejected before any state is
+// touched.
+func parseConfig(args []string) (Config, []string, error) {
+	fs := flag.NewFlagSet("eitri-agent", flag.ContinueOnError)
+	stateDir := fs.String("state-dir", "/var/lib/eitri-agent", "agent state directory")
+	chBin := fs.String("ch-bin", "cloud-hypervisor", "path to cloud-hypervisor binary")
+	firmware := fs.String("firmware", "/usr/share/eitri/CLOUDHV.fd", "path to CH UEFI firmware (CLOUDHV.fd)")
+	bootstrapURL := fs.String("bootstrap-url", "https://eitri.sh/dl/latest/manifest.json", "eitri.sh release manifest to fetch cloud-hypervisor/firmware from if missing at startup (empty disables bootstrap)")
+	tombstoneGrace := fs.Duration("tombstone-grace", 5*time.Minute, "quarantine period for tombstoned VMs before destroy")
+	vanishGrace := fs.Duration("vanish-grace", time.Hour, "quarantine period for VMs that vanished without a tombstone")
+	vmTimeout := fs.Duration("vm-timeout", 15*time.Minute, "watchdog bound on ONE VM's reconcile pass (0 disables); keep above the 10m image-download timeout")
+	imageCacheMaxGB := fs.Int64("image-cache-max-gb", 20, "evict least-recently-used cached base images beyond this size (0 = never evict)")
+	maxConcurrentCreates := fs.Int("max-concurrent-creates", 4, "cap how many VMs may be inside the I/O-heavy part of create at once (image fetch + disk copy); 0 = unlimited")
+	maxVCPUs := fs.Int64("max-vcpus", 0, "cap the total vCPUs this host offers the fleet (0 = unlimited; reserves headroom, advertised + enforced at boot)")
+	maxMemMB := fs.Int64("max-mem-mb", 0, "cap the total memory (MB) this host offers the fleet (0 = unlimited)")
+	maxDiskGB := fs.Int64("max-disk-gb", 0, "cap the total disk (GB) this host offers the fleet (0 = unlimited)")
+	if err := fs.Parse(args); err != nil {
+		return Config{}, nil, err
+	}
+	// Fixed order so the reported cap is deterministic when more than one is bad.
+	caps := []struct {
+		flag string
+		v    int64
+	}{
+		{"max-vcpus", *maxVCPUs},
+		{"max-mem-mb", *maxMemMB},
+		{"max-disk-gb", *maxDiskGB},
+	}
+	for _, c := range caps {
+		if c.v < 0 {
+			return Config{}, nil, fmt.Errorf("resource cap --%s must be >= 0 (0 = unlimited), got %d", c.flag, c.v)
+		}
+	}
+	return Config{
+		StateDir:             *stateDir,
+		CHBin:                *chBin,
+		Firmware:             *firmware,
+		BootstrapURL:         *bootstrapURL,
+		TombstoneGrace:       *tombstoneGrace,
+		VanishGrace:          *vanishGrace,
+		VMTimeout:            *vmTimeout,
+		ImageCacheMaxGB:      *imageCacheMaxGB,
+		MaxConcurrentCreates: *maxConcurrentCreates,
+		MaxVCPUs:             *maxVCPUs,
+		MaxMemMB:             *maxMemMB,
+		MaxDiskGB:            *maxDiskGB,
+	}, fs.Args(), nil
+}
+// join handles the "join <blob>" subcommand: decode the join blob, enroll,
+// and persist identity — pinning the server cert from the blob (the enroll
+// response's fingerprint is ignored, so the blob is the sole trust root).
+func join(st *state.Store, blob string) error {
+	if blob == "" {
+		return errors.New("usage: eitri-agent join <join-blob>")
+	}
+	f, err := joinblob.Decode(blob)
+	if err != nil {
+		// Never echo the blob itself — it carries a bearer token.
+		return fmt.Errorf("invalid join blob: %w", err)
+	}
+	hostname, err := os.Hostname()
+	if err != nil {
+		hostname = "unknown"
+	}
+	result, err := enrollclient.New(f.HTTPURL).Enroll(context.Background(), enrollclient.Request{
+		Token:       f.Token,
+		Name:        hostname,
+		OS:          runtime.GOOS,
+		Arch:        runtime.GOARCH,
+		Provisioner: "cloudhv",
+	})
+	if errors.Is(err, enrollclient.ErrTokenRejected) {
+		return errors.New("enroll rejected: token already used or expired — mint a new join token")
+	}
+	if err != nil {
+		return fmt.Errorf("enroll failed: %w", err)
+	}
+	id := state.Identity{
+		HostID:           result.HostID,
+		Credential:       result.Credential,
+		BridgeCIDR:       result.BridgeCIDR,
+		ServerQUICAddr:   f.QUICAddr,
+		ServerCertSHA256: f.CertFP, // authoritative; response fingerprint ignored
+	}
+	if err := st.SaveIdentity(id); err != nil {
+		return fmt.Errorf("save identity: %w", err)
+	}
+	fmt.Printf("Enrolled: host_id=%s bridge_cidr=%s\n", result.HostID, result.BridgeCIDR)
+	return nil
+}
+// serve handles the normal (no subcommand) run mode: it wires the reconcile
+// engine and sync client and blocks until SIGINT/SIGTERM.
+func serve(st *state.Store, cfg Config) error {
+	id, ok := st.Identity()
+	if !ok {
+		return errors.New("not enrolled — run with 'join <blob>' subcommand first")
+	}
+	ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
+	defer cancel()
+	// Flush integration-coverage counters on SIGUSR1 (no-op unless built with
+	// -cover and GOCOVERDIR is set). Lets the deploy boot-gate snapshot coverage
+	// from the live agent without bouncing the process.
+	covsnap.Install(ctx)
+	net, err := netenv.New(cloudhv.RealRunner, id.BridgeCIDR)
+	if err != nil {
+		return fmt.Errorf("netenv init: %w", err)
+	}
+	if err := net.EnsureBridge(ctx); err != nil {
+		return fmt.Errorf("ensure bridge: %w", err)
+	}
+	// Bootstrap cloud-hypervisor and its UEFI firmware before anything tries
+	// to launch a VM: a bare host that just joined has neither, and the agent
+	// is useless without at least the hypervisor binary. BootstrapDest maps
+	// the --ch-bin value (usually a bare $PATH name) to a real install path.
+	bs := &bootstrap.Bootstrapper{CHPath: cloudhv.BootstrapDest(cfg.CHBin), FirmwarePath: cfg.Firmware, ManifestURL: cfg.BootstrapURL}
+	if err := bs.Ensure(ctx); err != nil {
+		return fmt.Errorf("bootstrap runtime: %w", err)
+	}
+	prov := cloudhv.New(st, cfg.CHBin, cfg.Firmware, cloudhv.RealRunner)
+	// Serial console pumps: one per running VM, started at Boot (cloudhv hook)
+	// and reattached here for VMs that survived an agent restart (CH runs in
+	// its own process group; the pump reconnects to the still-listening
+	// serial socket).
+	pumps := serialpump.NewManager(st.SerialSocketPath, st.SerialLogPath)
+	prov.Pumps = pumps
+	if recs, err := st.LoadVMs(); err == nil {
+		for _, rec := range recs {
+			if rec.IP != "" {
+				net.AddReservation(rec.Spec.VMID, rec.IP)
+			}
+			if prov.Running(rec.Spec.VMID) {
+				pumps.Ensure(rec.Spec.VMID)
+			}
+		}
+	} else {
+		slog.Warn("state load failed; surviving VMs' consoles will be silent AND their DHCP reservations are not rebuilt (they may fail to renew until reconcile recreates them)", "err", err)
+	}
+	// Start the embedded DHCP responder AFTER reservations are rebuilt from
+	// state, so a surviving guest's renewal is never answered from an empty
+	// table (fail-closed + reservation preload close the gap).
+	if err := net.StartDHCP(ctx); err != nil {
+		return fmt.Errorf("start dhcp: %w", err)
+	}
+	cache := imagecache.New(st.ImagesDir(), cloudhv.RealRunner)
+	// Clamp before shifting: GB<<30 overflows int64 for absurd flag values —
+	// same trap class cloudhv's maxDiskGB comment documents. 1 PiB is beyond
+	// any real cache; anything above disables eviction just like 0 would.
+	imageCacheMaxGB := cfg.ImageCacheMaxGB
+	if imageCacheMaxGB < 0 || imageCacheMaxGB > 1<<20 {
+		slog.Warn("image-cache-max-gb out of range [0, 2^20]; disabling eviction", "value", imageCacheMaxGB)
+		imageCacheMaxGB = 0
+	}
+	cache.MaxBytes = imageCacheMaxGB << 30
+	// Reclaim temps left by a previous agent killed mid-fetch or mid-convert.
+	// Must run here, before the reconcile loop starts fetching: a sweep cannot
+	// tell an abandoned temp from one an in-flight fetch is still writing.
+	cache.SweepTemps()
+	engine := &reconcile.Engine{
+		St:                   st,
+		Prov:                 prov,
+		Net:                  net,
+		Images:               cache.Ensure,
+		Seed:                 seed.Build,
+		BootID:               syncclient.HostBootID,
+		Now:                  time.Now,
+		TombstoneGrace:       cfg.TombstoneGrace,
+		VanishGrace:          cfg.VanishGrace,
+		MaxCreateAttempts:    3,
+		MaxConcurrentCreates: cfg.MaxConcurrentCreates,
+		VMTimeout:            cfg.VMTimeout,
+		MaxVCPUs:             cfg.MaxVCPUs,
+		MaxMemMB:             cfg.MaxMemMB,
+		MaxDiskGB:            cfg.MaxDiskGB,
+	}
+	// Seed the admission ledger from persisted records so the first reconcile
+	// accounts for VMs that survived the agent restart (their compute must
+	// count against the caps before any new create is admitted).
+	if recs, err := st.LoadVMs(); err == nil {
+		engine.SeedLedger(recs)
+	}
+	// The per-VM reconcile workers are deliberately NOT torn down here. Stopping
+	// the engine is terminal — it would make the next report an empty actual state,
+	// which the control plane reads as every VM on this host having vanished — and
+	// it would race an unjoined session worker that can still be mid-Step when ctx
+	// is cancelled (see the note below where pumps are torn down). The process is
+	// exiting; the OS reclaims the goroutines.
+	client := &syncclient.Client{
+		Engine:    engine,
+		St:        st,
+		Identity:  id,
+		StateDir:  cfg.StateDir,
+		Runner:    cloudhv.RealRunner,
+		Console:   pumps,
+		MaxVCPUs:  cfg.MaxVCPUs,
+		MaxMemMB:  cfg.MaxMemMB,
+		MaxDiskGB: cfg.MaxDiskGB,
+	}
+	slog.Info("agent started", "host_id", id.HostID, "bridge_cidr", id.BridgeCIDR)
+	client.Run(ctx)
+	// client.Run blocks until ctx is cancelled (SIGINT/SIGTERM) and only then
+	// returns, so the reconcile/sync loop is done driving VMs and no further
+	// Ensure/Stop calls are expected to reach pumps (an unjoined session
+	// worker could in principle still be mid-Engine.Step, but it has nothing
+	// left to drive once client.Run has returned). Tear down every serial
+	// console pump here, at the very end of agent shutdown.
+	pumps.StopAll()
+	return nil
+}

internal/agent/run/cli_test.go

@@ @@ -0,0 +1,116 @@ @@
+package run
+import (
+	"strings"
+	"testing"
+	"time"
+	"github.com/a73x/eitri/internal/agent/state"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+// TestParseConfigDefaults pins every flag default: this binary runs the fleet,
+// so a silent change to any default is a production behavior change.
+func TestParseConfigDefaults(t *testing.T) {
+	cfg, rest, err := parseConfig(nil)
+	require.NoError(t, err)
+	assert.Empty(t, rest)
+	assert.Equal(t, "/var/lib/eitri-agent", cfg.StateDir)
+	assert.Equal(t, "cloud-hypervisor", cfg.CHBin)
+	assert.Equal(t, "/usr/share/eitri/CLOUDHV.fd", cfg.Firmware)
+	assert.Equal(t, "https://eitri.sh/dl/latest/manifest.json", cfg.BootstrapURL)
+	assert.Equal(t, 5*time.Minute, cfg.TombstoneGrace)
+	assert.Equal(t, time.Hour, cfg.VanishGrace)
+	assert.Equal(t, 15*time.Minute, cfg.VMTimeout)
+	assert.Equal(t, int64(20), cfg.ImageCacheMaxGB)
+	assert.Equal(t, 4, cfg.MaxConcurrentCreates)
+	assert.Equal(t, int64(0), cfg.MaxVCPUs)
+	assert.Equal(t, int64(0), cfg.MaxMemMB)
+	assert.Equal(t, int64(0), cfg.MaxDiskGB)
+}
+// TestParseConfigOverrides confirms every flag threads through to the Config.
+func TestParseConfigOverrides(t *testing.T) {
+	cfg, rest, err := parseConfig([]string{
+		"--state-dir=/srv/state",
+		"--ch-bin=/opt/ch",
+		"--firmware=/opt/fw.fd",
+		"--bootstrap-url=",
+		"--tombstone-grace=90s",
+		"--vanish-grace=2h",
+		"--vm-timeout=0",
+		"--image-cache-max-gb=50",
+		"--max-concurrent-creates=1",
+		"--max-vcpus=8",
+		"--max-mem-mb=4096",
+		"--max-disk-gb=100",
+	})
+	require.NoError(t, err)
+	assert.Empty(t, rest)
+	assert.Equal(t, "/srv/state", cfg.StateDir)
+	assert.Equal(t, "/opt/ch", cfg.CHBin)
+	assert.Equal(t, "/opt/fw.fd", cfg.Firmware)
+	assert.Equal(t, "", cfg.BootstrapURL)
+	assert.Equal(t, 90*time.Second, cfg.TombstoneGrace)
+	assert.Equal(t, 2*time.Hour, cfg.VanishGrace)
+	assert.Equal(t, time.Duration(0), cfg.VMTimeout)
+	assert.Equal(t, int64(50), cfg.ImageCacheMaxGB)
+	assert.Equal(t, 1, cfg.MaxConcurrentCreates)
+	assert.Equal(t, int64(8), cfg.MaxVCPUs)
+	assert.Equal(t, int64(4096), cfg.MaxMemMB)
+	assert.Equal(t, int64(100), cfg.MaxDiskGB)
+}
+// TestParseConfigJoinSubcommand returns the positional args so RunCLI can
+// dispatch the join flow.
+func TestParseConfigJoinSubcommand(t *testing.T) {
+	_, rest, err := parseConfig([]string{"--state-dir=/srv/state", "join", "the-blob"})
+	require.NoError(t, err)
+	require.Equal(t, []string{"join", "the-blob"}, rest)
+}
+// TestParseConfigRejectsNegativeCaps rejects a negative resource cap (0 means
+// unlimited) and names the offending flag deterministically.
+func TestParseConfigRejectsNegativeCaps(t *testing.T) {
+	for _, flag := range []string{"max-vcpus", "max-mem-mb", "max-disk-gb"} {
+		_, _, err := parseConfig([]string{"--" + flag + "=-1"})
+		require.Error(t, err)
+		assert.Contains(t, err.Error(), "--"+flag)
+	}
+}
+// TestParseConfigBadFlag surfaces an unknown flag as an error rather than
+// exiting the process (ContinueOnError).
+func TestParseConfigBadFlag(t *testing.T) {
+	_, _, err := parseConfig([]string{"--nonesuch"})
+	require.Error(t, err)
+}
+// TestJoinEmptyBlob rejects a missing blob before any decode or network call;
+// st is never touched, so nil is safe.
+func TestJoinEmptyBlob(t *testing.T) {
+	err := join(nil, "")
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "usage: eitri-agent join")
+}
+// TestJoinInvalidBlob rejects an undecodable blob and never echoes the blob
+// itself (it carries a bearer token). Decode fails before st is used.
+func TestJoinInvalidBlob(t *testing.T) {
+	const secret = "not-a-valid-join-blob-with-secret"
+	err := join(nil, secret)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "invalid join blob")
+	assert.NotContains(t, err.Error(), secret, "the blob carries a bearer token and must never appear in errors")
+}
+// TestServeNotEnrolled refuses to run before the host has enrolled, before any
+// network or signal handler is set up.
+func TestServeNotEnrolled(t *testing.T) {
+	st, err := state.Open(t.TempDir())
+	require.NoError(t, err)
+	err = serve(st, Config{StateDir: t.TempDir()})
+	require.Error(t, err)
+	assert.True(t, strings.Contains(err.Error(), "not enrolled"), "got %q", err.Error())
+}

internal/arch/arch_test.go

Old		New
@@ -229,17 +229,19 @@ func TestOnlyCloudhvImportsOsExecInDataPlane(t *testing.T) {
229		229
230	// R7: external process execution anywhere in internal/ is confined to a	230	// R7: external process execution anywhere in internal/ is confined to a
231	// sanctioned allowlist — cloudhv (the data plane's one exec funnel, R6's	231	// sanctioned allowlist — cloudhv (the data plane's one exec funnel, R6's
232	// narrower story), cli (interactive ssh must be the real OpenSSH client), and	232	// narrower story), cli (interactive ssh must be the real OpenSSH client), shape
233	// shape (architecture tooling that shells out to `go list`, the same	233	// (architecture tooling that shells out to `go list`, the same introspection
234	// introspection internal/arch's own tests do). R2/R6 tell the per-plane	234	// internal/arch's own tests do), and smoke (the deploy boot-gate harness, an
235	// stories; R7 is the whole-tree backstop that makes a new exec site anywhere	235	// external test rig that shells out to ssh, pkill, and `go tool covdata`).
236	// an explicit, reviewed decision.	236	// R2/R6 tell the per-plane stories; R7 is the whole-tree backstop that makes a
		237	// new exec site anywhere an explicit, reviewed decision.
237	func TestExecIsConfinedToSanctionedPackages(t *testing.T) {	238	func TestExecIsConfinedToSanctionedPackages(t *testing.T) {
238	g := directImports(t)	239	g := directImports(t)
239	allowed := map[string]bool{	240	allowed := map[string]bool{
240	module + "/internal/agent/cloudhv": true, // the data plane's one sanctioned exec funnel (R6's story)	241	module + "/internal/agent/cloudhv": true, // the data plane's one sanctioned exec funnel (R6's story)
241	module + "/internal/cli": true, // interactive sessions must be the real OpenSSH client	242	module + "/internal/cli": true, // interactive sessions must be the real OpenSSH client
242	module + "/internal/shape": true, // architecture tooling: shells `go list -json` to build the module graph — the same shell-out internal/arch's own tests make	243	module + "/internal/shape": true, // architecture tooling: shells `go list -json` to build the module graph — the same shell-out internal/arch's own tests make
		244	module + "/internal/smoke": true, // deploy boot-gate harness: shells out to ssh, pkill, and `go tool covdata` against the live fleet
243	}	245	}
244	for pkg, offenders := range execViolations(g, module, "internal/", allowed) {	246	for pkg, offenders := range execViolations(g, module, "internal/", allowed) {
245	for _, o := range offenders {	247	for _, o := range offenders {
@@ -328,7 +330,7 @@ func TestAPITypesIsALeaf(t *testing.T) {
328	// (or any other) import would relink the issuer into the relying party and	330	// (or any other) import would relink the issuer into the relying party and
329	// silently rebuild the embedded-IdP coupling this split exists to prevent.	331	// silently rebuild the embedded-IdP coupling this split exists to prevent.
330	// Test files anywhere may still import it as an in-process IdP	332	// Test files anywhere may still import it as an in-process IdP
331	// (server/api's auth_test, eitri-smoke's login_test); those edges are	333	// (server/api's auth_test, internal/smoke's login_test); those edges are
332	// test-only and never enter the production graph go list reports here.	334	// test-only and never enter the production graph go list reports here.
333	// (b) the go-oidc verifier module belongs to internal/server/api, the relying	335	// (b) the go-oidc verifier module belongs to internal/server/api, the relying
334	// party, alone. It is the client half of the protocol — anywhere else it	336	// party, alone. It is the client half of the protocol — anywhere else it
@@ -410,3 +412,53 @@ func assertNotImported(t *testing.T, g map[string][]string, pkg string, forbidde
410	t.Errorf("domain package %s must not import: %s", short(pkg), strings.Join(hits, ", "))	412	t.Errorf("domain package %s must not import: %s", short(pkg), strings.Join(hits, ", "))
411	}	413	}
412	}	414	}
		415
		416	// R14: main packages are wiring, not logic. A cmd/* package may import
		417	// packages from this module plus a tiny stdlib allowlist — enough to print a
		418	// version, dispatch to an internal Run, and exit non-zero. Every other import
		419	// (net/http, encoding/json, flag, crypto, third-party modules) is evidence of
		420	// logic living in a package that the coverage gate cannot see (scripts/
		421	// coverage.sh floors internal/, not cmd/) — move it behind a tested
		422	// `Run(...) error` in an internal package instead.
		423	//
		424	// r14Grandfathered names the mains that predate the rule and still carry
		425	// logic. The list is a ratchet: entries may only be REMOVED (as their logic
		426	// is extracted); never add one, and never grow an entry's import surface.
		427	func TestMainPackagesAreWiringOnly(t *testing.T) {
		428	allowedStd := map[string]bool{
		429	"errors": true, // errors.Is on sentinel errors from the internal Run
		430	"fmt": true,
		431	"log": true,
		432	"log/slog": true,
		433	"os": true,
		434	}
		435	r14Grandfathered := map[string]bool{
		436	// Empty: every main is now wiring over a tested internal Run. The map
		437	// stays as the ratchet mechanism — a new main has nowhere to hide.
		438	}
		439	g := directImports(t)
		440	seen := 0
		441	for pkg, deps := range g {
		442	if !strings.HasPrefix(pkg, module+"/cmd/") {
		443	continue
		444	}
		445	seen++
		446	if r14Grandfathered[pkg] {
		447	continue
		448	}
		449	var bad []string
		450	for _, d := range deps {
		451	if strings.HasPrefix(d, module+"/") \|\| allowedStd[d] {
		452	continue
		453	}
		454	bad = append(bad, d)
		455	}
		456	if len(bad) > 0 {
		457	sort.Strings(bad)
		458	t.Errorf("main package %s imports %s — mains are wiring only (R14): move the logic behind a tested Run() in an internal package", short(pkg), strings.Join(bad, ", "))
		459	}
		460	}
		461	if seen == 0 {
		462	t.Error("sweep found no cmd/ packages at all — did the go list pattern break?")
		463	}
		464	}

internal/cli/main.go

@@ @@ -0,0 +1,89 @@ @@
+// main.go is the eitri client's command dispatch. It lives here rather than in
+// cmd/eitri so it is testable and coverage-gated (arch R14: main packages are
+// wiring only).
+package cli
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+)
+// Usage is the top-level help text, printed by cmd/eitri on ErrUsage.
+const Usage = `usage:
+  eitri ssh <vm> [ssh args / remote command...]
+  eitri ca upload [<tenant>] <ca-public-key-file>
+  eitri --version
+env: EITRI_URL, EITRI_GATE, EITRI_TENANT (required for ssh);
+     EITRI_URL, EITRI_TOKEN and a tenant (argument or EITRI_TENANT) for ca;
+     EITRI_CA, EITRI_KEY, EITRI_KNOWN_HOSTS (optional, default under ~/.ssh)`
+// ErrUsage marks a bad invocation: the caller prints Usage and exits 2 rather
+// than treating it as a runtime failure.
+var ErrUsage = errors.New("bad usage")
+// Main dispatches the eitri command line (everything after the binary name,
+// --version excluded — that stays in cmd/eitri). stdout carries command
+// output; prompts and confirmations go to stderr inside the subcommands.
+func Main(args []string, stdout io.Writer) error {
+	if len(args) < 1 {
+		return ErrUsage
+	}
+	switch args[0] {
+	case "ssh":
+		return runSSH(args[1:], stdout)
+	case "ca":
+		return runCA(args[1:], stdout)
+	default:
+		return ErrUsage
+	}
+}
+func runSSH(args []string, stdout io.Writer) error {
+	if len(args) >= 1 && (args[0] == "-h" || args[0] == "--help") {
+		fmt.Fprintln(stdout, "usage: eitri ssh <vm> [ssh args / remote command...]")
+		return nil
+	}
+	if len(args) < 1 {
+		return fmt.Errorf("usage: eitri ssh <vm> [ssh args / remote command...]")
+	}
+	env, err := FromEnv()
+	if err != nil {
+		return err
+	}
+	return RunSSH(context.Background(), env, args[0], args[1:])
+}
+func runCA(args []string, stdout io.Writer) error {
+	if len(args) < 1 || args[0] != "upload" {
+		return fmt.Errorf("usage: eitri ca upload [<tenant>] <ca-public-key-file>")
+	}
+	rest := args[1:]
+	if len(rest) < 1 || len(rest) > 2 {
+		return fmt.Errorf("usage: eitri ca upload [<tenant>] <ca-public-key-file>")
+	}
+	// The tenant comes from the positional when given, else EITRI_TENANT.
+	// There is no implicit tenant (a guess would surface as an opaque 404).
+	tenant, pub := os.Getenv("EITRI_TENANT"), rest[0]
+	if len(rest) == 2 {
+		tenant, pub = rest[0], rest[1]
+	}
+	if tenant == "" {
+		return fmt.Errorf("no tenant: pass one (eitri ca upload <tenant> <file>) or set EITRI_TENANT")
+	}
+	url := os.Getenv("EITRI_URL")
+	token := os.Getenv("EITRI_TOKEN")
+	if url == "" || token == "" {
+		return fmt.Errorf("set EITRI_URL and EITRI_TOKEN (a personal access token)")
+	}
+	out, err := UploadUserCA(context.Background(), url, token, tenant, pub)
+	if err != nil {
+		return err
+	}
+	fmt.Fprintln(stdout, out)
+	return nil
+}

internal/cli/main_test.go

@@ @@ -0,0 +1,63 @@ @@
+package cli
+import (
+	"bytes"
+	"errors"
+	"strings"
+	"testing"
+)
+func TestMainDispatch(t *testing.T) {
+	var out bytes.Buffer
+	if err := Main(nil, &out); !errors.Is(err, ErrUsage) {
+		t.Errorf("no args: got %v, want ErrUsage", err)
+	}
+	if err := Main([]string{"frobnicate"}, &out); !errors.Is(err, ErrUsage) {
+		t.Errorf("unknown subcommand: got %v, want ErrUsage", err)
+	}
+	// `ssh -h` prints usage and succeeds (exit 0), it is not an error.
+	if err := Main([]string{"ssh", "-h"}, &out); err != nil {
+		t.Errorf("ssh -h: %v", err)
+	}
+	if !strings.Contains(out.String(), "eitri ssh <vm>") {
+		t.Errorf("ssh -h must print usage, got %q", out.String())
+	}
+	if err := Main([]string{"ssh"}, &out); err == nil {
+		t.Error("bare ssh must error with usage")
+	}
+}
+// TestRunCATenantResolution pins how `eitri ca upload` finds its tenant:
+// positional wins, EITRI_TENANT is the fallback, and no tenant at all is an
+// actionable error before anything touches the network.
+func TestRunCATenantResolution(t *testing.T) {
+	var out bytes.Buffer
+	t.Setenv("EITRI_URL", "")
+	t.Setenv("EITRI_TOKEN", "")
+	t.Setenv("EITRI_TENANT", "")
+	if err := Main([]string{"ca"}, &out); err == nil || !strings.Contains(err.Error(), "usage:") {
+		t.Errorf("bare ca: got %v", err)
+	}
+	if err := Main([]string{"ca", "upload"}, &out); err == nil || !strings.Contains(err.Error(), "usage:") {
+		t.Errorf("upload with no args: got %v", err)
+	}
+	// No positional tenant, no EITRI_TENANT: the error names both remedies.
+	err := Main([]string{"ca", "upload", "ca.pub"}, &out)
+	if err == nil || !strings.Contains(err.Error(), "EITRI_TENANT") {
+		t.Errorf("tenantless: got %v", err)
+	}
+	// A tenant (positional or env) but no URL/token: fails on the env contract
+	// — proving tenant resolution passed.
+	err = Main([]string{"ca", "upload", "acme", "ca.pub"}, &out)
+	if err == nil || !strings.Contains(err.Error(), "EITRI_URL and EITRI_TOKEN") {
+		t.Errorf("positional tenant: got %v", err)
+	}
+	t.Setenv("EITRI_TENANT", "acme")
+	err = Main([]string{"ca", "upload", "ca.pub"}, &out)
+	if err == nil || !strings.Contains(err.Error(), "EITRI_URL and EITRI_TOKEN") {
+		t.Errorf("env tenant: got %v", err)
+	}
+}

internal/mcpserver/cli.go

@@ @@ -0,0 +1,149 @@ @@
+// cli.go is the eitri-mcp command line: it loads the config, self-signs with a
+// persistent per-client user CA, uploads that CA to the tenant, and serves the
+// MCP tools over stdio. It lives here rather than in cmd/eitri-mcp so it is
+// testable and coverage-gated (arch R14: main packages are wiring only).
+package mcpserver
+import (
+	"context"
+	"crypto/ed25519"
+	"crypto/rand"
+	"encoding/pem"
+	"flag"
+	"fmt"
+	"os"
+	"os/signal"
+	"syscall"
+	"github.com/a73x/eitri/internal/gateclient"
+	"github.com/a73x/eitri/internal/server/api/client"
+	"github.com/modelcontextprotocol/go-sdk/mcp"
+	"golang.org/x/crypto/ssh"
+)
+// RunCLI dispatches the eitri-mcp command line (everything after the binary
+// name, --version excluded — that stays in cmd/eitri-mcp). The config path
+// defaults to EITRI_MCP_CONFIG, else ~/.config/eitri-mcp/config.json.
+func RunCLI(args []string) error {
+	defaultCfg := "~/.config/eitri-mcp/config.json"
+	if env := os.Getenv("EITRI_MCP_CONFIG"); env != "" {
+		defaultCfg = env
+	}
+	fs := flag.NewFlagSet("eitri-mcp", flag.ExitOnError)
+	cfgPath := fs.String("config", defaultCfg, "path to eitri-mcp config.json")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+	return run(*cfgPath)
+}
+func run(cfgPath string) error {
+	cfg, err := LoadConfig(cfgPath)
+	if err != nil {
+		return err
+	}
+	// This client owns its own persistent user CA (BYO model): it self-signs
+	// short-lived user certs locally rather than asking the server to mint them.
+	userCA, err := loadOrCreateCA(cfg.CAKeyPath)
+	if err != nil {
+		return fmt.Errorf("load mcp user CA: %w", err)
+	}
+	// The Runner reaches VMs by name through the eitri SSH-CA jump gate: it
+	// authenticates with short-lived user certs it self-signs on demand with its
+	// own user CA, and verifies both hops' host certs against the eitri host CA.
+	// The user CA's public key is uploaded to the tenant once (Register, below)
+	// so VMs trust those certs. GateAuth is backed by the same API client.
+	api := &client.Client{BaseURL: cfg.ServerURL, Token: cfg.Token}
+	gateAuth := gateclient.NewGateAuth(api, userCA, cfg.Tenant, nil)
+	tools := &Tools{
+		API: API{Client: api},
+		Runner: NewRunner(RunnerConfig{
+			Gate:   cfg.Gate,
+			Auth:   gateAuth,
+			VMUser: cfg.VMUser,
+		}),
+		Gate:   cfg.Gate,
+		VMUser: cfg.VMUser,
+	}
+	server := mcp.NewServer(&mcp.Implementation{Name: "eitri", Version: "0.1.0"}, nil)
+	register(server, "vm_create", "Create an eitri VM (persistent). Waits for ready+cloud-init by default.", tools.VMCreate)
+	register(server, "vm_list", "List all VMs on the eitri fleet.", tools.VMList)
+	register(server, "vm_info", "Show one VM's state and how to reach it.", tools.VMInfo)
+	register(server, "vm_exec", "Run a shell command in a VM over SSH; returns stdout/stderr/exit code.", tools.VMExec)
+	register(server, "vm_write_file", "Write content to a file in a VM (parents created).", tools.VMWriteFile)
+	register(server, "vm_read_file", "Read a file from a VM (capped at 1 MiB).", tools.VMReadFile)
+	register(server, "vm_destroy", "Destroy a VM by id or EXACT name. Explicit-only; never called automatically.", tools.VMDestroy)
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+	// Upload our user CA to the tenant before serving, so vm_create's precondition
+	// (a registered user CA) is satisfied and VMs trust the certs we sign.
+	if err := gateAuth.Register(ctx); err != nil {
+		return fmt.Errorf("register mcp user CA: %w", err)
+	}
+	return server.Run(ctx, &mcp.StdioTransport{})
+}
+// loadOrCreateCA returns a stable ssh.Signer for the key at path. If the file
+// is absent it generates a fresh ed25519 key, writes it 0600 with O_EXCL (so a
+// concurrent creator can't clobber it and a symlink can't be followed), and
+// returns its signer; if present it parses and returns the existing key. This
+// is deliberately local to the MCP (a pure API client) and does NOT import the
+// server's sshca package. Never logs or returns key material in errors.
+func loadOrCreateCA(path string) (ssh.Signer, error) {
+	pemBytes, err := os.ReadFile(path)
+	if err == nil {
+		signer, perr := ssh.ParsePrivateKey(pemBytes)
+		if perr != nil {
+			return nil, fmt.Errorf("parse ssh key %q: %w", path, perr)
+		}
+		return signer, nil
+	}
+	if !os.IsNotExist(err) {
+		return nil, fmt.Errorf("read ssh key %q: %w", path, err)
+	}
+	_, priv, err := ed25519.GenerateKey(rand.Reader)
+	if err != nil {
+		return nil, fmt.Errorf("generate ssh key: %w", err)
+	}
+	block, err := ssh.MarshalPrivateKey(priv, "")
+	if err != nil {
+		return nil, fmt.Errorf("marshal ssh key: %w", err)
+	}
+	signer, err := ssh.NewSignerFromSigner(priv)
+	if err != nil {
+		return nil, fmt.Errorf("new signer: %w", err)
+	}
+	f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
+	if err != nil {
+		return nil, fmt.Errorf("create ssh key %q: %w", path, err)
+	}
+	if _, werr := f.Write(pem.EncodeToMemory(block)); werr != nil {
+		f.Close()
+		return nil, fmt.Errorf("write ssh key %q: %w", path, werr)
+	}
+	if cerr := f.Close(); cerr != nil {
+		return nil, fmt.Errorf("close ssh key %q: %w", path, cerr)
+	}
+	return signer, nil
+}
+// register adapts a Tools method to the SDK. This is the ONLY place that
+// touches SDK generics; if the SDK's handler signature changes, change it here.
+//
+// Note on the hand-off contract: the SDK drops the Out value when the handler
+// returns a non-nil error — StructuredContent is left unset and only err.Error()
+// reaches the model (as IsError text content). VMCreate's degraded-path errors
+// are self-sufficient (they name the VM id+name), so the model can still find
+// and destroy the VM from the error text.
+func register[In, Out any](s *mcp.Server, name, desc string, fn func(context.Context, In) (Out, error)) {
+	mcp.AddTool(s, &mcp.Tool{Name: name, Description: desc},
+		func(ctx context.Context, req *mcp.CallToolRequest, in In) (*mcp.CallToolResult, Out, error) {
+			out, err := fn(ctx, in)
+			return nil, out, err
+		})
+}

internal/mcpserver/cli_test.go

@@ @@ -0,0 +1,60 @@ @@
+package mcpserver
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"testing"
+	"github.com/modelcontextprotocol/go-sdk/mcp"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+// TestLoadOrCreateCACreatesThenLoads covers the two steady-state paths: a fresh
+// key is generated 0600 when absent, and the identical key loads back when
+// present.
+func TestLoadOrCreateCACreatesThenLoads(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "user_ca")
+	signer, err := loadOrCreateCA(path)
+	require.NoError(t, err)
+	require.NotNil(t, signer)
+	info, err := os.Stat(path)
+	require.NoError(t, err)
+	assert.Equal(t, os.FileMode(0o600), info.Mode().Perm(), "key material must be written 0600")
+	again, err := loadOrCreateCA(path)
+	require.NoError(t, err)
+	assert.Equal(t, signer.PublicKey().Marshal(), again.PublicKey().Marshal(),
+		"an existing key must load back unchanged, not be regenerated")
+}
+// TestLoadOrCreateCARejectsGarbage rejects an unparseable key file without
+// leaking its bytes in the error.
+func TestLoadOrCreateCARejectsGarbage(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "user_ca")
+	require.NoError(t, os.WriteFile(path, []byte("this is not a pem key"), 0o600))
+	_, err := loadOrCreateCA(path)
+	require.Error(t, err)
+	assert.NotContains(t, err.Error(), "not a pem key", "key bytes must never appear in errors")
+}
+// TestLoadOrCreateCAReadError treats a non-not-exist read failure (a directory
+// at the path) as a fatal error, not a signal to generate a key.
+func TestLoadOrCreateCAReadError(t *testing.T) {
+	dir := t.TempDir()
+	_, err := loadOrCreateCA(dir)
+	require.Error(t, err)
+}
+// TestRegisterAddsTool pins the single SDK-generics adapter: it must wire a
+// typed handler onto the server without panicking.
+func TestRegisterAddsTool(t *testing.T) {
+	s := mcp.NewServer(&mcp.Implementation{Name: "test", Version: "0"}, nil)
+	register(s, "noop", "does nothing", func(context.Context, struct{}) (struct{}, error) {
+		return struct{}{}, nil
+	})
+}

internal/server/boot/boot.go

@@ @@ -0,0 +1,268 @@ @@
+// Package boot implements the eitri-server command line behind a tested RunCLI
+// so cmd/eitri-server stays thin wiring (arch R14). It opens the store, handles
+// the server certificate, starts the housekeeping goroutines, binds the QUIC
+// and HTTP listeners, constructs the API, and wires the SSH jump gate — the
+// single-node control plane's whole boot sequence.
+//
+// No TLS termination happens here — front eitri-server with a reverse proxy for
+// TLS.
+package boot
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"log/slog"
+	"net/http"
+	"os/signal"
+	"syscall"
+	"time"
+	"github.com/a73x/eitri/internal/covsnap"
+	"github.com/a73x/eitri/internal/joinblob"
+	"github.com/a73x/eitri/internal/server/api"
+	serverconfig "github.com/a73x/eitri/internal/server/config"
+	"github.com/a73x/eitri/internal/server/health"
+	"github.com/a73x/eitri/internal/server/hub"
+	"github.com/a73x/eitri/internal/server/registry"
+	"github.com/a73x/eitri/internal/server/release"
+	"github.com/a73x/eitri/internal/server/store"
+	"github.com/a73x/eitri/internal/server/syncsvc"
+	"github.com/a73x/eitri/internal/server/web"
+	"github.com/a73x/eitri/internal/transport"
+	"github.com/quic-go/quic-go"
+)
+// RunCLI dispatches the eitri-server command line (everything after the binary
+// name, --version excluded — that stays in cmd/eitri-server). It parses the
+// -config flag and runs the control plane until SIGINT/SIGTERM.
+func RunCLI(args []string) error {
+	fs := flag.NewFlagSet("eitri-server", flag.ContinueOnError)
+	cfgPath := fs.String("config", "/etc/eitri/server.json", "config file")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+	return run(*cfgPath)
+}
+// run wires and serves the control plane, blocking until SIGINT/SIGTERM (a nil
+// return) or an always-on server fails (a non-nil return). Every synchronous
+// startup invariant surfaces as a returned error so cmd/eitri-server can exit
+// non-zero — the process-fatal outcome each site previously reached directly.
+func run(cfgPath string) error {
+	// Load enforces every startup invariant (required keys, OIDC block, URL
+	// shapes) — see internal/server/config, where the rules are tested.
+	cfg, err := serverconfig.Load(cfgPath)
+	if err != nil {
+		return fmt.Errorf("config %s: %w", cfgPath, err)
+	}
+	st, err := store.Open(cfg.DBPath, cfg.CIDRPool)
+	if err != nil {
+		return fmt.Errorf("open store: %w", err)
+	}
+	certPEM, certFP, err := st.ServerCert()
+	if err != nil {
+		return fmt.Errorf("server cert: %w", err)
+	}
+	keyPEM, err := st.ServerKeyPEM()
+	if err != nil {
+		return fmt.Errorf("server key: %w", err)
+	}
+	// Log the identity agents pin — the operator verifies this out-of-band
+	// during the rotation ceremony (docs/cert-rotation.md step 3).
+	slog.Info("server cert", "fingerprint", certFP)
+	// Rotation nudge: the agent pin ignores expiry so nothing breaks at
+	// NotAfter, but a long-lived key is a widening forgery window. Warn while
+	// inside the renewal window — at startup AND daily, because servers here
+	// are long-lived daemons that can cross into the window (or past expiry)
+	// without ever restarting. See docs/cert-rotation.md for the ceremony.
+	warnIfRenewalDue := func() {
+		notAfter, due := transport.CertRenewalDue(certPEM, time.Now())
+		if !due {
+			return
+		}
+		if notAfter.IsZero() {
+			slog.Warn("server cert unparseable — inspect server.crt (docs/cert-rotation.md)")
+			return
+		}
+		slog.Warn("server cert renewal due — rotate and re-enroll agents (docs/cert-rotation.md)",
+			"not_after", notAfter.Format(time.RFC3339))
+	}
+	warnIfRenewalDue()
+	// Audit retention: bound the append-only log (default 90 days, 0 disables).
+	// A negative value is almost certainly a typo — refuse rather than silently
+	// keeping the audit log forever ("0" is the explicit disable spelling).
+	auditRetention, err := serverconfig.ParseDuration("audit_retention", cfg.AuditRetention, 90*24*time.Hour,
+		func(d time.Duration) bool { return d >= 0 }, ">= 0")
+	if err != nil {
+		return fmt.Errorf("config: %w", err)
+	}
+	pruneAudit := func() {
+		if auditRetention <= 0 {
+			return
+		}
+		if n, err := st.PruneAudit(auditRetention); err != nil {
+			slog.Warn("audit prune failed", "err", err)
+		} else if n > 0 {
+			slog.Info("audit pruned", "rows", n, "retention", auditRetention)
+		}
+	}
+	pruneAudit()
+	// Daily housekeeping: cert-renewal nudge + audit retention.
+	go func() {
+		for range time.Tick(24 * time.Hour) {
+			warnIfRenewalDue()
+			pruneAudit()
+		}
+	}()
+	// Fail fast if the advertised addresses are non-empty but malformed (e.g. a
+	// URL with no scheme): otherwise every enroll-token mint would 500 at runtime.
+	if _, err := joinblob.Encode(cfg.AdvertiseHTTP, cfg.AdvertiseQUIC, "startup-probe", certFP); err != nil {
+		return fmt.Errorf("advertise_http/advertise_quic invalid: %w", err)
+	}
+	// SSH jump gate (§B): nil when ssh_listen is unset (gate OFF); see
+	// setupSSHGate. The listener itself is started below, once syncsvc.Service
+	// (the tunnel dialer) exists.
+	sshGate, err := setupSSHGate(cfg)
+	if err != nil {
+		return err
+	}
+	reg := registry.New(time.Now)
+	h := hub.New()
+	a := api.New(api.Config{HostSecret: []byte(cfg.HostSecret),
+		DefaultImage:     api.DefaultImage{URL: cfg.DefaultImageURL, SHA256: cfg.DefaultImageSHA},
+		ServerCertSHA256: certFP,
+		AdvertiseHTTP:    cfg.AdvertiseHTTP,
+		AdvertiseQUIC:    cfg.AdvertiseQUIC,
+		OIDC: api.OIDCConfig{
+			Issuer:            cfg.OIDC.Issuer,
+			ClientID:          cfg.OIDC.ClientID,
+			ClientSecret:      cfg.OIDC.ClientSecret,
+			PublicURL:         cfg.OIDC.PublicURL,
+			AllowedDomains:    cfg.OIDC.AllowedDomains,
+			AllowedIdentities: cfg.OIDC.AllowedIdentities,
+		}},
+		st, reg, h)
+	tlsConf, err := transport.ServerTLS(certPEM, keyPEM)
+	if err != nil {
+		return fmt.Errorf("server tls: %w", err)
+	}
+	lis, err := quic.ListenAddr(cfg.QUICListen, tlsConf,
+		// Shared with the agent dialer via transport so the two ends can't drift.
+		transport.SyncQUICConfig())
+	if err != nil {
+		return fmt.Errorf("quic listen: %w", err)
+	}
+	maxCredAge, err := serverconfig.ParseDuration("credential_max_age", cfg.CredentialMaxAge, 0, nil, "")
+	if err != nil {
+		return fmt.Errorf("config: %w", err)
+	}
+	// SSH cert minters: no-op when the gate is off, so the endpoint 404s.
+	sshGate.wireAPI(a)
+	svc := syncsvc.New(st, reg, h, []byte(cfg.HostSecret), maxCredAge)
+	// Console broker: the API bridges browser WebSockets to agent console
+	// streams over the live sync connections the service tracks.
+	a.SetConsoleDialer(svc)
+	// fatal collects the first failure from any always-on server goroutine (the
+	// SSH gate, QUIC, HTTP). Buffered so a failing server never blocks on send;
+	// run returns the first error and cmd/eitri-server exits non-zero — the same
+	// process-fatal outcome each site previously reached via os.Exit(1). A dead
+	// server must not run silently.
+	fatal := make(chan error, 3)
+	// SSH jump gate listener (no-op when off); a failed bind is fatal, like
+	// QUIC/HTTP below.
+	if err := sshGate.startListener(st, svc, fatal); err != nil {
+		return err
+	}
+	go func() {
+		slog.Info("quic listening", "addr", cfg.QUICListen)
+		if err := svc.Serve(context.Background(), lis); err != nil {
+			fatal <- fmt.Errorf("quic serve: %w", err)
+		}
+	}()
+	// Background: finalize drained decommissioning hosts.
+	go a.StartBackground(context.Background())
+	// Serve the REST API + SSE under /api/ and the embedded SPA everywhere else.
+	root := http.NewServeMux()
+	root.Handle("/api/", a.Handler())
+	// Sign-in endpoints live OUTSIDE /api/ and its auth middleware: they are how
+	// a browser establishes a session in the first place (spec §2).
+	root.Handle("/auth/", a.AuthHandler())
+	// Unauthenticated probes (outside /api/, so a load balancer or the deploy
+	// script needs no token). /livez is process-up; /readyz gates on the
+	// dependencies the server needs to actually serve — the DB. The QUIC
+	// listener bind is a startup invariant: quic.ListenAddr above returns an
+	// error that stops the process before this HTTP server serves, so a response
+	// here already implies QUIC bound.
+	root.HandleFunc("/livez", health.Live)
+	root.Handle("/readyz", health.Ready(3*time.Second,
+		health.Check{Name: "db", Probe: st.Ping},
+	))
+	root.Handle("/", web.Handler())
+	// Graceful shutdown: SIGINT/SIGTERM stops accepting, drains in-flight
+	// requests, then closes the API — stopping the SSE snapshot-hub goroutine and
+	// releasing its notifier subscription. The QUIC listener and background worker
+	// die with the process.
+	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
+	defer stop()
+	// Release discovery: absent field ⇒ eitri.sh default; explicit "" disables.
+	manifestURL := "https://eitri.sh/dl/latest/manifest.json"
+	if cfg.ReleaseManifestURL != nil {
+		manifestURL = *cfg.ReleaseManifestURL
+	}
+	if manifestURL != "" {
+		rel := release.New(manifestURL)
+		go rel.Poll(ctx, 24*time.Hour, func(err error) {
+			slog.Warn("release manifest refresh failed", "err", err)
+		})
+		a.SetReleaseSource(rel)
+	}
+	a.SetAgentUpgrader(svc)
+	// Flush integration-coverage counters on SIGUSR1 (no-op unless built with
+	// -cover and GOCOVERDIR is set). Lets the deploy boot-gate snapshot coverage
+	// from the live server without bouncing the process.
+	covsnap.Install(ctx)
+	srv := &http.Server{Addr: cfg.HTTPListen, Handler: root}
+	go func() {
+		slog.Info("http listening", "addr", cfg.HTTPListen)
+		if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
+			fatal <- fmt.Errorf("http serve: %w", err)
+		}
+	}()
+	select {
+	case err := <-fatal:
+		return err
+	case <-ctx.Done():
+	}
+	slog.Info("shutting down")
+	shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	if err := srv.Shutdown(shutdownCtx); err != nil {
+		slog.Warn("http graceful shutdown", "err", err)
+	}
+	a.Close()
+	return nil
+}

internal/server/boot/cli_test.go

@@ @@ -0,0 +1,25 @@ @@
+package boot
+import (
+	"testing"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+// TestRunCLIBadFlag surfaces an unknown flag as an error rather than exiting the
+// process (ContinueOnError).
+func TestRunCLIBadFlag(t *testing.T) {
+	err := RunCLI([]string{"--nonesuch"})
+	require.Error(t, err)
+}
+// TestRunCLIMissingConfig threads the -config flag through to the loader and
+// surfaces a missing config as an error naming the path (the process would then
+// exit non-zero), without starting any listener.
+func TestRunCLIMissingConfig(t *testing.T) {
+	missing := t.TempDir() + "/no-such-server.json"
+	err := RunCLI([]string{"-config", missing})
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), missing)
+}

internal/server/boot/sshgate.go

@@ @@ -0,0 +1,172 @@ @@
+package boot
+import (
+	"errors"
+	"fmt"
+	"log/slog"
+	"net"
+	"time"
+	"github.com/a73x/eitri/internal/server/api"
+	serverconfig "github.com/a73x/eitri/internal/server/config"
+	"github.com/a73x/eitri/internal/server/sshca"
+	"github.com/a73x/eitri/internal/server/sshgate"
+	"github.com/a73x/eitri/internal/server/store"
+	"github.com/a73x/eitri/internal/server/syncsvc"
+	"golang.org/x/crypto/ssh"
+)
+// sshGateSetup carries the jump-gate state from config-time setup to the later
+// wiring points in startup (API cert minters, sync snapshot, gate listener).
+// A nil *sshGateSetup means the gate is OFF: every method is a no-op on a nil
+// receiver, so run holds one value instead of repeating `!= nil` guards.
+type sshGateSetup struct {
+	ca     *sshca.CA
+	listen string // cfg.SSHListen
+	domain string // cfg.SSHGateDomain
+}
+// setupSSHGate wires the SSH jump gate (§B): OFF unless ssh_listen is set
+// (returns nil). When enabled, load or create the persistent user CA + gate
+// host key (0600, never logged).
+func setupSSHGate(cfg serverconfig.Config) (*sshGateSetup, error) {
+	if cfg.SSHListen == "" {
+		return nil, nil
+	}
+	if cfg.SSHCAKey == "" || cfg.SSHHostKey == "" {
+		return nil, errors.New("ssh_ca_key and ssh_host_key are required when ssh_listen is set")
+	}
+	sshGate, err := sshca.New(cfg.SSHCAKey, cfg.SSHHostKey)
+	if err != nil {
+		return nil, fmt.Errorf("ssh ca: %w", err)
+	}
+	// Log the HOST CA identity operators pin via @cert-authority for gate + VM
+	// host verification. Only the *public* key is ever logged (private material
+	// never is). eitri holds no user CA — those are BYO per-tenant.
+	slog.Info("ssh jump gate configured", "listen", cfg.SSHListen,
+		"host_ca", string(sshGate.HostCAAuthorizedKey()))
+	// The gate listener itself is started later (startListener), once
+	// syncsvc.Service (the tunnel dialer) exists.
+	return &sshGateSetup{ca: sshGate, listen: cfg.SSHListen, domain: cfg.SSHGateDomain}, nil
+}
+// wireAPI installs the per-VM host-cert minter and publishes the HOST CA: when
+// the jump gate is enabled, eitri signs a persistent host key + cert at each VM
+// create and serves the host CA pubkey via GET /api/v1/ssh-ca. eitri never
+// mints user certs — user CAs are BYO per-tenant (uploaded, never held here).
+// Left unwired when the gate is off, so the ssh-ca endpoint 404s.
+func (g *sshGateSetup) wireAPI(a *api.API) {
+	if g == nil {
+		return
+	}
+	// Per-VM host certs: sign a persistent host key + cert at each VM create,
+	// so VMs present verifiable host keys (clients accept via @cert-authority).
+	a.SetHostCertMinter(api.NewHostMinter(g.ca.HostCA()))
+	// Publish the HOST CA public key so clients can pin `@cert-authority` for
+	// host verification of both the gate and every VM.
+	a.SetSSHCAAuthorizedKey(string(g.ca.HostCAAuthorizedKey()))
+}
+// startListener starts the SSH jump gate listener: when enabled, front
+// `ssh -J gate ubuntu@<vm>` with the hardened bastion. It resolves VM names
+// against the store, tunnels port 22 through the sync connection (svc.OpenTCP),
+// and trusts only certs signed by a registered tenant user CA. A failed bind or
+// cert-sign is fatal (returned to run, like QUIC/HTTP); a serve failure after
+// bind lands on fatal so a dead gate does not run silently.
+func (g *sshGateSetup) startListener(st *store.Store, svc *syncsvc.Service, fatal chan<- error) error {
+	if g == nil {
+		return nil
+	}
+	// The gate host cert's principal is the name clients dial. Prefer the
+	// configured domain; else the host part of ssh_listen; else "localhost".
+	gateDomain := g.domain
+	if gateDomain == "" {
+		if h, _, err := net.SplitHostPort(g.listen); err == nil {
+			gateDomain = h
+		}
+	}
+	if gateDomain == "" {
+		gateDomain = "localhost"
+	}
+	slog.Info("ssh gate host cert", "principal", gateDomain)
+	// Sign a long-lived HOST cert for the gate's own host key and present THAT
+	// (via a cert signer) instead of the bare key, so a client verifying with
+	// `@cert-authority` accepts the gate on first connect — no TOFU window.
+	gateCert, err := sshca.SignHostCert(g.ca.HostCA(), g.ca.HostKey().PublicKey(),
+		[]string{gateDomain}, "eitri-gate", time.Now(), sshca.HostCertTTL)
+	if err != nil {
+		return fmt.Errorf("sign gate host cert: %w", err)
+	}
+	gateHostSigner, err := ssh.NewCertSigner(gateCert, g.ca.HostKey())
+	if err != nil {
+		return fmt.Errorf("gate host cert signer: %w", err)
+	}
+	gate := sshgate.New(gateHostSigner, userCALookup(st), resolveVM(st), authorizeVM(st), svc.OpenTCP, revokedCert(st))
+	ln, err := net.Listen("tcp", g.listen)
+	if err != nil {
+		return fmt.Errorf("ssh gate listen: %w", err)
+	}
+	go func() {
+		slog.Info("ssh jump gate listening", "addr", g.listen)
+		if err := gate.Serve(ln); err != nil {
+			fatal <- fmt.Errorf("ssh gate serve: %w", err)
+		}
+	}()
+	return nil
+}
+// resolveVM is tenant-scoped: the bare name is looked up WITHIN the connection's
+// tenant only (VMByTenantName), so a name never resolves across tenants. A
+// lookup error (unknown or tombstoned VM) reports ok=false rather than tunneling
+// to a dead or foreign VM.
+func resolveVM(st *store.Store) sshgate.Resolver {
+	return func(tenant, name string) (hostID, vmID string, ok bool) {
+		vm, err := st.VMByTenantName(tenant, name)
+		if err != nil {
+			return "", "", false
+		}
+		return vm.HostID, vm.ID, true
+	}
+}
+// authorizeVM re-reads the VM row and requires tenant equality: the tenant that
+// the connection's user cert resolved to (via its per-tenant CA) must own the
+// VM, or the connection is refused. A tombstoned VM (DeletedAt set) or a store
+// error is refused too.
+func authorizeVM(st *store.Store) sshgate.Authorizer {
+	return func(tenant, vmID string) bool {
+		vm, err := st.GetVM(vmID)
+		return err == nil && vm.DeletedAt == nil && vm.Tenant == tenant
+	}
+}
+// revokedCert gates every cert auth against the revocation list. Fail-CLOSED
+// for the single connection on a DB error: a store hiccup rejects THAT login
+// (returns revoked=true) rather than fail-open (which would let a possibly-
+// revoked cert through) or fail-the-whole-gate (which a global close would
+// amount to, DoSing every login on any transient error).
+func revokedCert(st *store.Store) sshgate.Revoker {
+	return func(serial uint64) bool {
+		revoked, err := st.IsSSHCertRevoked(serial)
+		if err != nil {
+			slog.Error("ssh cert revocation lookup failed; rejecting connection", "err", err)
+			return true
+		}
+		return revoked
+	}
+}
+// userCALookup trusts the DB-registered set of tenant user CAs and stamps each
+// connection with the tenant that registered the signing CA. It looks up by the
+// SAME canonical authorized_keys line the store persists (ca_pubkey), so the
+// bytes agree. A lookup error fails closed (rejects the cert).
+func userCALookup(st *store.Store) sshgate.UserCALookup {
+	return func(pub ssh.PublicKey) (string, bool) {
+		tenant, ok, err := st.TenantForUserCA(sshca.AuthorizedKeyLine(pub))
+		if err != nil {
+			slog.Error("tenant user-ca lookup failed; rejecting", "err", err)
+			return "", false
+		}
+		return tenant, ok
+	}
+}

internal/server/boot/sshgate_test.go

@@ @@ -0,0 +1,162 @@ @@
+package boot
+import (
+	"crypto/ed25519"
+	"testing"
+	"github.com/a73x/eitri/internal/server/sshca"
+	"github.com/a73x/eitri/internal/server/store"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/crypto/ssh"
+)
+// newStore opens a fresh store on a temp DB. The gate closures resolve VMs and
+// tenants against a REAL store — the same one the live gate uses — so these
+// tests exercise the actual SQL, not a mock.
+func newStore(t *testing.T) *store.Store {
+	t.Helper()
+	s, err := store.Open(t.TempDir()+"/eitri.db", "10.77.0.0/16")
+	require.NoError(t, err)
+	t.Cleanup(func() { s.Close() })
+	return s
+}
+// makeTenantHost provisions a tenant through the real JIT path and enrolls one
+// host into it, returning the tenant ID and host. Enrollment tokens require an
+// existing tenant (FK enforced), so the tenant must be created first.
+func makeTenantHost(t *testing.T, s *store.Store, subject, email string) (string, store.Host) {
+	t.Helper()
+	tn, err := s.CreateTenantForIdentity("https://idp", subject, email)
+	require.NoError(t, err)
+	tok, err := s.CreateEnrollmentToken(tn.ID)
+	require.NoError(t, err)
+	h, err := s.RedeemEnrollmentToken(tok, "host-"+tn.ID, "linux", "amd64", "cloudhv", "")
+	require.NoError(t, err)
+	return tn.ID, h
+}
+func makeVM(t *testing.T, s *store.Store, host store.Host, name string) store.VM {
+	t.Helper()
+	vm := store.VM{
+		ID: "vm-" + name, HostID: host.ID, Name: name,
+		ImageURL: "http://img", ImageSHA256: "abc",
+		VCPUs: 1, MemMB: 512, DiskGB: 5, PowerState: "running",
+	}
+	require.NoError(t, s.CreateVM(vm))
+	got, err := s.GetVM(vm.ID)
+	require.NoError(t, err)
+	return got
+}
+// TestResolveVMIsTenantScoped pins that the gate resolver looks a bare VM name
+// up WITHIN the connection's tenant only: the same name in another tenant does
+// not resolve, so one tenant can never tunnel to another's VM by guessing names.
+func TestResolveVMIsTenantScoped(t *testing.T) {
+	s := newStore(t)
+	tenantA, hostA := makeTenantHost(t, s, "sub-a", "alpha@x.com")
+	tenantB, _ := makeTenantHost(t, s, "sub-b", "beta@x.com")
+	vm := makeVM(t, s, hostA, "web")
+	resolve := resolveVM(s)
+	hostID, vmID, ok := resolve(tenantA, "web")
+	require.True(t, ok, "the owning tenant must resolve its own VM")
+	assert.Equal(t, hostA.ID, hostID)
+	assert.Equal(t, vm.ID, vmID)
+	_, _, ok = resolve(tenantB, "web")
+	assert.False(t, ok, "resolution must not cross tenants")
+	_, _, ok = resolve(tenantA, "nope")
+	assert.False(t, ok, "an unknown name does not resolve")
+}
+// TestResolveVMTombstonedNotResolvable pins that a tombstoned VM stops
+// resolving: the gate must not tunnel to a dead VM.
+func TestResolveVMTombstonedNotResolvable(t *testing.T) {
+	s := newStore(t)
+	tenant, host := makeTenantHost(t, s, "sub-a", "alpha@x.com")
+	vm := makeVM(t, s, host, "web")
+	resolve := resolveVM(s)
+	_, _, ok := resolve(tenant, "web")
+	require.True(t, ok)
+	require.NoError(t, s.TombstoneVM(vm.ID))
+	_, _, ok = resolve(tenant, "web")
+	assert.False(t, ok, "a tombstoned VM must not resolve")
+}
+// TestAuthorizeVM pins the second gate check: the connection's tenant must OWN
+// the VM (tenant equality), the VM must be live, and any store error refuses.
+func TestAuthorizeVM(t *testing.T) {
+	s := newStore(t)
+	tenantA, hostA := makeTenantHost(t, s, "sub-a", "alpha@x.com")
+	tenantB, _ := makeTenantHost(t, s, "sub-b", "beta@x.com")
+	vm := makeVM(t, s, hostA, "web")
+	authorize := authorizeVM(s)
+	assert.True(t, authorize(tenantA, vm.ID), "the owning tenant is authorized")
+	assert.False(t, authorize(tenantB, vm.ID), "a different tenant must be rejected")
+	assert.False(t, authorize(tenantA, "no-such-vm"), "an unknown VM must be rejected")
+	// A tombstoned VM (DeletedAt set) is refused even for its owner.
+	require.NoError(t, s.TombstoneVM(vm.ID))
+	assert.False(t, authorize(tenantA, vm.ID), "a tombstoned VM must be rejected")
+}
+// TestRevokedCertFailsClosed pins the revocation gate: a known-revoked serial
+// reads revoked, an unknown serial does not, and a store error fails CLOSED
+// (reports revoked=true) so a DB hiccup rejects THAT login rather than letting a
+// possibly-revoked cert through.
+func TestRevokedCertFailsClosed(t *testing.T) {
+	s := newStore(t)
+	tenant, _ := makeTenantHost(t, s, "sub-a", "alpha@x.com")
+	revoked := revokedCert(s)
+	assert.False(t, revoked(42), "an unknown serial is not revoked")
+	require.NoError(t, s.RevokeSSHCert(tenant, 42, "leaked laptop"))
+	assert.True(t, revoked(42), "a revoked serial reads revoked")
+	// Force a store error: after Close the DB handle is dead and the lookup
+	// errors. The gate must fail closed — reject the connection.
+	require.NoError(t, s.Close())
+	assert.True(t, revoked(43), "a store error must fail closed (reject)")
+}
+// TestUserCALookupFailsClosed pins tenant attribution: a cert signed by a
+// registered tenant CA resolves to that tenant, an unknown CA does not, and a
+// store error fails CLOSED (rejects the cert).
+func TestUserCALookupFailsClosed(t *testing.T) {
+	s := newStore(t)
+	tenant, _ := makeTenantHost(t, s, "sub-a", "alpha@x.com")
+	_, priv, err := ed25519.GenerateKey(nil)
+	require.NoError(t, err)
+	pub, err := ssh.NewPublicKey(priv.Public())
+	require.NoError(t, err)
+	caLine := sshca.AuthorizedKeyLine(pub)
+	require.NoError(t, s.AddTenantUserCA(tenant, caLine, "tenant", "laptop", "admin"))
+	lookup := userCALookup(s)
+	gotTenant, ok := lookup(pub)
+	require.True(t, ok, "a registered CA must resolve")
+	assert.Equal(t, tenant, gotTenant, "the CA maps to the tenant that registered it")
+	// An unregistered CA does not resolve (no error, just not found).
+	_, otherPriv, err := ed25519.GenerateKey(nil)
+	require.NoError(t, err)
+	otherPub, err := ssh.NewPublicKey(otherPriv.Public())
+	require.NoError(t, err)
+	_, ok = lookup(otherPub)
+	assert.False(t, ok, "an unregistered CA must not resolve")
+	// Force a store error: after Close the lookup errors and must fail closed.
+	require.NoError(t, s.Close())
+	_, ok = lookup(pub)
+	assert.False(t, ok, "a store error must fail closed (reject)")
+}

internal/server/config/load.go

@@ @@ -0,0 +1,99 @@ @@
+// load.go reads and validates the server config. It lives here rather than in
+// cmd/eitri-server so the rules are testable and coverage-gated (arch R14:
+// main packages are wiring only).
+package config
+import (
+	"encoding/json"
+	"fmt"
+	"log/slog"
+	"net/url"
+	"os"
+	"regexp"
+	"strings"
+	"time"
+)
+// defaultImageSHARe matches a valid lowercase hex SHA-256 digest. Kept local
+// to the control plane rather than shared with the agent's imagecache: R1
+// forbids the control plane importing the data plane, even transitively.
+var defaultImageSHARe = regexp.MustCompile(`^[a-f0-9]{64}$`)
+// Load reads path, decodes the JSON, and enforces every startup invariant the
+// server refuses to boot without. It warns (but does not fail) on the retired
+// admin_token key so an old config gets pruned rather than silently carried.
+func Load(path string) (Config, error) {
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return Config{}, fmt.Errorf("read config: %w", err)
+	}
+	var cfg Config
+	if err := json.Unmarshal(raw, &cfg); err != nil {
+		return Config{}, fmt.Errorf("parse config: %w", err)
+	}
+	if err := validate(cfg); err != nil {
+		return Config{}, err
+	}
+	// admin_token is retired: sign-in is OIDC and console credentials are
+	// sessions/PATs. Warn once so the operator prunes the stale key.
+	if cfg.AdminToken != "" {
+		slog.Warn("server.json: admin_token is no longer used and is ignored; remove it")
+	}
+	return cfg, nil
+}
+func validate(cfg Config) error {
+	if cfg.HostSecret == "" {
+		return fmt.Errorf("host_secret is required")
+	}
+	// The server is a pure OIDC relying party (spec §2): issuer, client_id and
+	// public_url are required. Collect every missing key so the operator fixes
+	// server.json in one pass rather than one restart per key.
+	var missingOIDC []string
+	if cfg.OIDC.Issuer == "" {
+		missingOIDC = append(missingOIDC, "oidc.issuer")
+	}
+	if cfg.OIDC.ClientID == "" {
+		missingOIDC = append(missingOIDC, "oidc.client_id")
+	}
+	if cfg.OIDC.PublicURL == "" {
+		missingOIDC = append(missingOIDC, "oidc.public_url")
+	}
+	if len(missingOIDC) > 0 {
+		return fmt.Errorf("missing required keys (point them at eitri-oidc or your IdP): %s", strings.Join(missingOIDC, ", "))
+	}
+	// public_url builds the OIDC callback URL, so it must be an absolute
+	// http(s) URL with a host — catch a bare host, missing scheme, or
+	// scheme-only URL at boot, not at the first redirect.
+	if u, err := url.Parse(cfg.OIDC.PublicURL); err != nil || !u.IsAbs() || u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") {
+		return fmt.Errorf("oidc.public_url must be an absolute http(s) URL, got %q", cfg.OIDC.PublicURL)
+	}
+	if cfg.AdvertiseHTTP == "" || cfg.AdvertiseQUIC == "" {
+		return fmt.Errorf("advertise_http and advertise_quic are required (the addresses agents use to reach this server, e.g. http://192.168.0.190:8080 and 192.168.0.190:8443)")
+	}
+	// Fail fast on a malformed default image digest rather than letting every
+	// mint silently propagate a bad hash.
+	if cfg.DefaultImageSHA != "" && !defaultImageSHARe.MatchString(cfg.DefaultImageSHA) {
+		return fmt.Errorf("default_image_sha256 malformed (want 64 lowercase hex chars), got %q", cfg.DefaultImageSHA)
+	}
+	return nil
+}
+// ParseDuration parses raw (a config duration string) for the knob named name
+// (the JSON field label echoed in errors). Empty raw keeps def. valid, when
+// non-nil, is the knob's range rule; rule (e.g. ">= 0") spells it in the
+// error. A nil valid skips the range check.
+func ParseDuration(name, raw string, def time.Duration, valid func(time.Duration) bool, rule string) (time.Duration, error) {
+	if raw == "" {
+		return def, nil
+	}
+	d, err := time.ParseDuration(raw)
+	if err != nil {
+		return 0, fmt.Errorf("%s invalid: %w", name, err)
+	}
+	if valid != nil && !valid(d) {
+		return 0, fmt.Errorf("%s must be %s, got %q", name, rule, raw)
+	}
+	return d, nil
+}

internal/server/config/load_test.go

@@ @@ -0,0 +1,136 @@ @@
+package config
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+)
+// minimal is a config that passes every validation rule.
+const minimal = `{
+	"host_secret": "s3cret",
+	"advertise_http": "http://192.0.2.1:8080",
+	"advertise_quic": "192.0.2.1:8443",
+	"oidc": {
+		"issuer": "http://127.0.0.1:9111",
+		"client_id": "eitri-console",
+		"public_url": "http://192.0.2.1:8080"
+	}
+}`
+func write(t *testing.T, content string) string {
+	t.Helper()
+	p := filepath.Join(t.TempDir(), "server.json")
+	if err := os.WriteFile(p, []byte(content), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	return p
+}
+func TestLoadValid(t *testing.T) {
+	cfg, err := Load(write(t, minimal))
+	if err != nil {
+		t.Fatalf("Load: %v", err)
+	}
+	if cfg.HostSecret != "s3cret" || cfg.OIDC.ClientID != "eitri-console" {
+		t.Errorf("fields not decoded: %+v", cfg)
+	}
+}
+func TestLoadFileErrors(t *testing.T) {
+	if _, err := Load(filepath.Join(t.TempDir(), "absent.json")); err == nil {
+		t.Error("missing file must error")
+	}
+	if _, err := Load(write(t, "{nope")); err == nil || !strings.Contains(err.Error(), "parse config") {
+		t.Error("malformed json must error with parse context")
+	}
+}
+func TestLoadRequiresHostSecret(t *testing.T) {
+	_, err := Load(write(t, strings.Replace(minimal, `"host_secret": "s3cret",`, "", 1)))
+	if err == nil || !strings.Contains(err.Error(), "host_secret") {
+		t.Errorf("got %v", err)
+	}
+}
+// TestLoadNamesEveryMissingOIDCKey pins that the operator sees all missing
+// oidc keys in ONE error, not one restart per key.
+func TestLoadNamesEveryMissingOIDCKey(t *testing.T) {
+	_, err := Load(write(t, `{"host_secret": "s", "advertise_http": "h", "advertise_quic": "q"}`))
+	if err == nil {
+		t.Fatal("empty oidc block accepted")
+	}
+	for _, want := range []string{"oidc.issuer", "oidc.client_id", "oidc.public_url"} {
+		if !strings.Contains(err.Error(), want) {
+			t.Errorf("error %q must name %q", err, want)
+		}
+	}
+}
+// TestLoadRejectsMalformedPublicURL pins the boot-time URL check: public_url
+// builds the OIDC callback, so a bare host, missing scheme, or non-http(s)
+// scheme fails at startup rather than at the first redirect.
+func TestLoadRejectsMalformedPublicURL(t *testing.T) {
+	for _, bad := range []string{"192.0.2.1:8080", "example.com", "http://", "ftp://x.example", "/just/a/path"} {
+		_, err := Load(write(t, strings.Replace(minimal, `"public_url": "http://192.0.2.1:8080"`, `"public_url": "`+bad+`"`, 1)))
+		if err == nil || !strings.Contains(err.Error(), "public_url") {
+			t.Errorf("public_url %q: got %v, want rejection", bad, err)
+		}
+	}
+	for _, good := range []string{"http://192.0.2.1:8080", "https://eitri.example.com"} {
+		if _, err := Load(write(t, strings.Replace(minimal, `"public_url": "http://192.0.2.1:8080"`, `"public_url": "`+good+`"`, 1))); err != nil {
+			t.Errorf("public_url %q: unexpected %v", good, err)
+		}
+	}
+}
+func TestLoadRequiresAdvertiseAddrs(t *testing.T) {
+	_, err := Load(write(t, strings.Replace(minimal, `"advertise_quic": "192.0.2.1:8443",`, "", 1)))
+	if err == nil || !strings.Contains(err.Error(), "advertise_http and advertise_quic") {
+		t.Errorf("got %v", err)
+	}
+}
+func TestLoadRejectsMalformedImageSHA(t *testing.T) {
+	withSHA := strings.Replace(minimal, `"host_secret": "s3cret",`,
+		`"host_secret": "s3cret", "default_image_sha256": "NOTHEX",`, 1)
+	if _, err := Load(write(t, withSHA)); err == nil || !strings.Contains(err.Error(), "default_image_sha256") {
+		t.Errorf("got %v", err)
+	}
+	okSHA := strings.Replace(minimal, `"host_secret": "s3cret",`,
+		`"host_secret": "s3cret", "default_image_sha256": "`+strings.Repeat("a", 64)+`",`, 1)
+	if _, err := Load(write(t, okSHA)); err != nil {
+		t.Errorf("valid sha rejected: %v", err)
+	}
+}
+// TestLoadToleratesRetiredAdminToken pins that a stale admin_token key warns
+// but does not fail — old configs keep booting.
+func TestLoadToleratesRetiredAdminToken(t *testing.T) {
+	withTok := strings.Replace(minimal, `"host_secret": "s3cret",`,
+		`"host_secret": "s3cret", "admin_token": "stale",`, 1)
+	if _, err := Load(write(t, withTok)); err != nil {
+		t.Errorf("admin_token must warn, not fail: %v", err)
+	}
+}
+func TestParseDuration(t *testing.T) {
+	// Empty keeps the default.
+	d, err := ParseDuration("knob", "", 90*time.Hour, nil, "")
+	if err != nil || d != 90*time.Hour {
+		t.Errorf("empty: %v %v", d, err)
+	}
+	d, err = ParseDuration("knob", "2h", 0, nil, "")
+	if err != nil || d != 2*time.Hour {
+		t.Errorf("2h: %v %v", d, err)
+	}
+	if _, err := ParseDuration("knob", "banana", 0, nil, ""); err == nil || !strings.Contains(err.Error(), "knob invalid") {
+		t.Errorf("malformed: %v", err)
+	}
+	// The range rule is enforced and named in the error.
+	if _, err := ParseDuration("knob", "-1h", 0, func(d time.Duration) bool { return d >= 0 }, ">= 0"); err == nil || !strings.Contains(err.Error(), ">= 0") {
+		t.Errorf("range: %v", err)
+	}
+}

internal/shape/classify.go

Old		New
@@ -47,6 +47,7 @@ func classify(rel string) Plane {
47	strings.HasPrefix(rel, "internal/oidcprovider"),	47	strings.HasPrefix(rel, "internal/oidcprovider"),
48	strings.HasPrefix(rel, "internal/site"),	48	strings.HasPrefix(rel, "internal/site"),
49	strings.HasPrefix(rel, "internal/shape"),	49	strings.HasPrefix(rel, "internal/shape"),
		50	strings.HasPrefix(rel, "internal/smoke"),
50	strings.HasPrefix(rel, "internal/cli"):	51	strings.HasPrefix(rel, "internal/cli"):
51	return PlaneTooling	52	return PlaneTooling
52	default:	53	default:

internal/shape/classify_test.go

Old		New
@@ -35,6 +35,7 @@ func TestClassifyAssignsPlaneByPrefix(t *testing.T) {
35	"internal/covsnap": PlaneTooling,	35	"internal/covsnap": PlaneTooling,
36	"internal/gateclient": PlaneTooling,	36	"internal/gateclient": PlaneTooling,
37	"internal/shape": PlaneTooling,	37	"internal/shape": PlaneTooling,
		38	"internal/smoke": PlaneTooling,
38	"internal/somethingnew": PlaneUnclassified,	39	"internal/somethingnew": PlaneUnclassified,
39	"pkg/whatever": PlaneUnclassified,	40	"pkg/whatever": PlaneUnclassified,
40	}	41	}

internal/site/cli.go

@@ @@ -0,0 +1,65 @@ @@
+// cli.go is the eitri-site command line: `eitri-site` (no subcommand) renders
+// the static site, `eitri-site manifest` writes the release manifest. It lives
+// here rather than in cmd/eitri-site so it is testable and coverage-gated
+// (arch R14: main packages are wiring only).
+package site
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"os"
+)
+// RunCLI dispatches the eitri-site command line (everything after the binary
+// name, --version excluded — that stays in cmd/eitri-site). The default path
+// renders the site; the "manifest" subcommand writes the release manifest.
+func RunCLI(args []string) error {
+	if len(args) > 0 && args[0] == "manifest" {
+		if err := runManifest(args[1:]); err != nil {
+			return fmt.Errorf("manifest: %w", err)
+		}
+		return nil
+	}
+	return runBuild(args)
+}
+func runBuild(args []string) error {
+	fs := flag.NewFlagSet("eitri-site", flag.ExitOnError)
+	docs := fs.String("docs", "docs", "docs directory (markdown sources)")
+	siteDir := fs.String("site", "site", "site directory (index.md, template.html, style.css)")
+	dist := fs.String("dist", "", "optional dist/<version> dir with release artifacts")
+	out := fs.String("out", "site/dist", "output webroot")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+	return Build(Config{DocsDir: *docs, SiteDir: *siteDir, DistDir: *dist, OutDir: *out})
+}
+func runManifest(args []string) error {
+	fs := flag.NewFlagSet("eitri-site manifest", flag.ExitOnError)
+	ver := fs.String("version", "", "release version (vX.Y.Z)")
+	dist := fs.String("dist", "", "dist/<version> dir holding bare agent binaries")
+	base := fs.String("base", "", "base URL artifacts are served from")
+	out := fs.String("out", "", "output path (default <dist>/manifest.json)")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+	if *ver == "" || *dist == "" || *base == "" {
+		return fmt.Errorf("-version, -dist, and -base are required")
+	}
+	m, err := BuildManifest(*ver, *dist, *base)
+	if err != nil {
+		return err
+	}
+	raw, err := json.MarshalIndent(m, "", "  ")
+	if err != nil {
+		return err
+	}
+	path := *out
+	if path == "" {
+		path = *dist + "/manifest.json"
+	}
+	return os.WriteFile(path, append(raw, '\n'), 0o644)
+}

internal/site/cli_test.go

@@ @@ -0,0 +1,78 @@ @@
+package site
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"github.com/a73x/eitri/internal/server/release"
+)
+// TestRunCLIBuild renders the site through the command-line entry point,
+// exercising the default (no-subcommand) flag path.
+func TestRunCLIBuild(t *testing.T) {
+	root, docs, siteDir := writeFixture(t)
+	out := filepath.Join(root, "out")
+	if err := RunCLI([]string{"-docs", docs, "-site", siteDir, "-out", out}); err != nil {
+		t.Fatalf("RunCLI build: %v", err)
+	}
+	if _, err := os.Stat(filepath.Join(out, "index.html")); err != nil {
+		t.Errorf("index.html not emitted: %v", err)
+	}
+}
+// TestRunCLIBuildError surfaces a build failure (an absent tree) as an error,
+// not a panic.
+func TestRunCLIBuildError(t *testing.T) {
+	absent := filepath.Join(t.TempDir(), "absent")
+	if err := RunCLI([]string{"-docs", absent, "-site", absent, "-out", absent}); err == nil {
+		t.Error("build over an absent tree must error")
+	}
+}
+// TestRunCLIManifest writes a release manifest through the "manifest"
+// subcommand, to both an explicit -out and the default <dist>/manifest.json.
+func TestRunCLIManifest(t *testing.T) {
+	dist := t.TempDir()
+	if err := os.WriteFile(filepath.Join(dist, "eitri-agent_linux_amd64"), []byte("x"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	outPath := filepath.Join(dist, "m.json")
+	if err := RunCLI([]string{"manifest", "-version", "v0.0.1", "-dist", dist, "-base", "https://eitri.sh/dl/v0.0.1", "-out", outPath}); err != nil {
+		t.Fatalf("RunCLI manifest: %v", err)
+	}
+	raw, err := os.ReadFile(outPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var m release.Manifest
+	if err := json.Unmarshal(raw, &m); err != nil {
+		t.Fatalf("manifest not valid json: %v", err)
+	}
+	if m.Version != "v0.0.1" {
+		t.Errorf("manifest version = %q", m.Version)
+	}
+	// With -out omitted the manifest lands at <dist>/manifest.json.
+	if err := RunCLI([]string{"manifest", "-version", "v0.0.1", "-dist", dist, "-base", "https://eitri.sh/dl/v0.0.1"}); err != nil {
+		t.Fatalf("RunCLI manifest default out: %v", err)
+	}
+	if _, err := os.Stat(filepath.Join(dist, "manifest.json")); err != nil {
+		t.Errorf("default manifest.json not written: %v", err)
+	}
+}
+// TestRunCLIManifestRequiresFlags names the missing required flags, prefixed so
+// the caller can tell a manifest failure from a build failure.
+func TestRunCLIManifestRequiresFlags(t *testing.T) {
+	err := RunCLI([]string{"manifest", "-version", "v0.0.1"})
+	if err == nil || !strings.Contains(err.Error(), "required") {
+		t.Errorf("manifest without -dist/-base: got %v", err)
+	}
+	if err != nil && !strings.Contains(err.Error(), "manifest:") {
+		t.Errorf("manifest error must be prefixed: got %v", err)
+	}
+}

internal/smoke/config.go

@@ @@ -0,0 +1,103 @@ @@
+package smoke
+import (
+	"fmt"
+	"strconv"
+	"strings"
+)
+// Config holds the environment-sourced settings for one smoke run. It mirrors
+// the variables the deploy boot-gate reads from $EITRI_DEPLOY_ENV.
+type Config struct {
+	ServerURL      string
+	CIUser         string
+	CIPasswordFile string
+	CIPATFile      string
+	AgentUserHost  string
+	AgentPort      int
+	AgentStateDir  string
+	// Carried for a later coverage-collection task; optional here.
+	ServerGocoverdir string
+	AgentGocoverdir  string
+	CoverOut         string
+	// SSH-CA gate check. When SmokeGate and SmokeUserCAFile are both set, the
+	// scenario proves guest access through the gate (a hard gate). Optional.
+	SmokeGate       string // SMOKE_GATE, "<gate-domain>:<port>"
+	SmokeVMUser     string // SMOKE_VM_USER, default "ubuntu"
+	SmokeUserCAFile string // SMOKE_USER_CA_FILE, load-or-create user CA key
+}
+// loadConfig reads the required smoke settings via getenv (never the real
+// process environment directly, so callers can inject a fake for tests). It
+// returns an error naming the first missing required variable.
+func loadConfig(getenv func(string) string) (Config, error) {
+	serverURL := getenv("SERVER_URL")
+	ciUser := getenv("CI_USER")
+	ciPasswordFile := getenv("CI_PASSWORD_FILE")
+	ciPATFile := getenv("CI_PAT_FILE")
+	agentHosts := getenv("AGENT_HOSTS")
+	agentStateDir := getenv("AGENT_STATE_DIR")
+	var missing []string
+	if serverURL == "" {
+		missing = append(missing, "SERVER_URL")
+	}
+	if ciUser == "" {
+		missing = append(missing, "CI_USER")
+	}
+	if ciPasswordFile == "" {
+		missing = append(missing, "CI_PASSWORD_FILE")
+	}
+	if ciPATFile == "" {
+		missing = append(missing, "CI_PAT_FILE")
+	}
+	if agentHosts == "" {
+		missing = append(missing, "AGENT_HOSTS")
+	}
+	if agentStateDir == "" {
+		missing = append(missing, "AGENT_STATE_DIR")
+	}
+	if len(missing) > 0 {
+		return Config{}, fmt.Errorf("missing required env var(s): %s", strings.Join(missing, ", "))
+	}
+	userHost, port := parseAgentHost(agentHosts)
+	smokeVMUser := getenv("SMOKE_VM_USER")
+	if smokeVMUser == "" {
+		smokeVMUser = "ubuntu"
+	}
+	return Config{
+		ServerURL:        serverURL,
+		CIUser:           ciUser,
+		CIPasswordFile:   ciPasswordFile,
+		CIPATFile:        ciPATFile,
+		AgentUserHost:    userHost,
+		AgentPort:        port,
+		AgentStateDir:    agentStateDir,
+		ServerGocoverdir: getenv("SERVER_GOCOVERDIR"),
+		AgentGocoverdir:  getenv("AGENT_GOCOVERDIR"),
+		CoverOut:         getenv("COVER_OUT"),
+		SmokeGate:        getenv("SMOKE_GATE"),
+		SmokeVMUser:      smokeVMUser,
+		SmokeUserCAFile:  getenv("SMOKE_USER_CA_FILE"),
+	}, nil
+}
+// parseAgentHost takes the AGENT_HOSTS value (space-separated "user@host[:port]"
+// entries) and returns the first entry's user@host plus its port. The port is
+// the substring after the LAST colon when that substring is entirely digits;
+// otherwise there is no port and it defaults to 22.
+func parseAgentHost(agentHosts string) (userHost string, port int) {
+	entry := strings.Fields(agentHosts)[0]
+	if idx := strings.LastIndex(entry, ":"); idx != -1 {
+		if p, err := strconv.Atoi(entry[idx+1:]); err == nil {
+			return entry[:idx], p
+		}
+	}
+	return entry, 22
+}

internal/smoke/config_test.go

@@ @@ -0,0 +1,153 @@ @@
+package smoke
+import (
+	"strings"
+	"testing"
+)
+// fakeGetenv returns a getenv func backed by a map, so tests never touch the
+// real process environment.
+func fakeGetenv(vals map[string]string) func(string) string {
+	return func(key string) string { return vals[key] }
+}
+func requiredVals() map[string]string {
+	return map[string]string{
+		"SERVER_URL":       "https://server.example:8443",
+		"CI_USER":          "ci@eitri.local",
+		"CI_PASSWORD_FILE": "/etc/eitri/ci-password",
+		"CI_PAT_FILE":      "/etc/eitri/ci-pat",
+		"AGENT_HOSTS":      "ubuntu@10.0.0.5:2222",
+		"AGENT_STATE_DIR":  "/var/lib/eitri-agent",
+	}
+}
+func TestLoadConfigParsesAgentHostWithPort(t *testing.T) {
+	cfg, err := loadConfig(fakeGetenv(requiredVals()))
+	if err != nil {
+		t.Fatalf("loadConfig: %v", err)
+	}
+	if cfg.AgentUserHost != "ubuntu@10.0.0.5" {
+		t.Errorf("AgentUserHost = %q, want ubuntu@10.0.0.5", cfg.AgentUserHost)
+	}
+	if cfg.AgentPort != 2222 {
+		t.Errorf("AgentPort = %d, want 2222", cfg.AgentPort)
+	}
+}
+func TestLoadConfigDefaultsPortWithoutColon(t *testing.T) {
+	vals := requiredVals()
+	vals["AGENT_HOSTS"] = "ubuntu@10.0.0.5"
+	cfg, err := loadConfig(fakeGetenv(vals))
+	if err != nil {
+		t.Fatalf("loadConfig: %v", err)
+	}
+	if cfg.AgentUserHost != "ubuntu@10.0.0.5" {
+		t.Errorf("AgentUserHost = %q, want ubuntu@10.0.0.5", cfg.AgentUserHost)
+	}
+	if cfg.AgentPort != 22 {
+		t.Errorf("AgentPort = %d, want 22", cfg.AgentPort)
+	}
+}
+func TestLoadConfigMultiEntryTakesFirst(t *testing.T) {
+	vals := requiredVals()
+	vals["AGENT_HOSTS"] = "ubuntu@10.0.0.5:2200 ubuntu@10.0.0.6:2201"
+	cfg, err := loadConfig(fakeGetenv(vals))
+	if err != nil {
+		t.Fatalf("loadConfig: %v", err)
+	}
+	if cfg.AgentUserHost != "ubuntu@10.0.0.5" || cfg.AgentPort != 2200 {
+		t.Errorf("got %q:%d, want ubuntu@10.0.0.5:2200", cfg.AgentUserHost, cfg.AgentPort)
+	}
+}
+func TestLoadConfigMissingRequiredVars(t *testing.T) {
+	cases := []struct {
+		name    string
+		unset   string
+		wantErr string
+	}{
+		{"missing server url", "SERVER_URL", "SERVER_URL"},
+		{"missing ci user", "CI_USER", "CI_USER"},
+		{"missing ci password file", "CI_PASSWORD_FILE", "CI_PASSWORD_FILE"},
+		{"missing ci pat file", "CI_PAT_FILE", "CI_PAT_FILE"},
+		{"missing agent hosts", "AGENT_HOSTS", "AGENT_HOSTS"},
+		{"missing agent state dir", "AGENT_STATE_DIR", "AGENT_STATE_DIR"},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			vals := requiredVals()
+			delete(vals, tc.unset)
+			_, err := loadConfig(fakeGetenv(vals))
+			if err == nil {
+				t.Fatal("loadConfig: want error, got nil")
+			}
+			if !strings.Contains(err.Error(), tc.wantErr) {
+				t.Errorf("error = %q, want to mention %q", err.Error(), tc.wantErr)
+			}
+		})
+	}
+}
+func TestLoadConfigMissingAllRequiredVars(t *testing.T) {
+	_, err := loadConfig(fakeGetenv(nil))
+	if err == nil {
+		t.Fatal("loadConfig: want error, got nil")
+	}
+	for _, want := range []string{"SERVER_URL", "CI_USER", "CI_PASSWORD_FILE", "CI_PAT_FILE", "AGENT_HOSTS", "AGENT_STATE_DIR"} {
+		if !strings.Contains(err.Error(), want) {
+			t.Errorf("error = %q, missing %q", err.Error(), want)
+		}
+	}
+}
+func TestLoadConfigSmokeGateDefaults(t *testing.T) {
+	cfg, err := loadConfig(fakeGetenv(requiredVals()))
+	if err != nil {
+		t.Fatalf("loadConfig: %v", err)
+	}
+	if cfg.SmokeGate != "" {
+		t.Errorf("SmokeGate = %q, want empty", cfg.SmokeGate)
+	}
+	if cfg.SmokeVMUser != "ubuntu" {
+		t.Errorf("SmokeVMUser = %q, want ubuntu", cfg.SmokeVMUser)
+	}
+	if cfg.SmokeUserCAFile != "" {
+		t.Errorf("SmokeUserCAFile = %q, want empty", cfg.SmokeUserCAFile)
+	}
+}
+func TestLoadConfigSmokeGatePassThrough(t *testing.T) {
+	vals := requiredVals()
+	vals["SMOKE_GATE"] = "gate.example:2222"
+	vals["SMOKE_VM_USER"] = "debian"
+	vals["SMOKE_USER_CA_FILE"] = "/etc/eitri-smoke/user_ca"
+	cfg, err := loadConfig(fakeGetenv(vals))
+	if err != nil {
+		t.Fatalf("loadConfig: %v", err)
+	}
+	if cfg.SmokeGate != "gate.example:2222" {
+		t.Errorf("SmokeGate = %q, want gate.example:2222", cfg.SmokeGate)
+	}
+	if cfg.SmokeVMUser != "debian" {
+		t.Errorf("SmokeVMUser = %q, want debian", cfg.SmokeVMUser)
+	}
+	if cfg.SmokeUserCAFile != "/etc/eitri-smoke/user_ca" {
+		t.Errorf("SmokeUserCAFile = %q, want /etc/eitri-smoke/user_ca", cfg.SmokeUserCAFile)
+	}
+}
+func TestLoadConfigOptionalCoverageVarsPassThrough(t *testing.T) {
+	vals := requiredVals()
+	vals["SERVER_GOCOVERDIR"] = "/tmp/server-cover"
+	vals["AGENT_GOCOVERDIR"] = "/tmp/agent-cover"
+	vals["COVER_OUT"] = "/tmp/out"
+	cfg, err := loadConfig(fakeGetenv(vals))
+	if err != nil {
+		t.Fatalf("loadConfig: %v", err)
+	}
+	if cfg.ServerGocoverdir != "/tmp/server-cover" || cfg.AgentGocoverdir != "/tmp/agent-cover" || cfg.CoverOut != "/tmp/out" {
+		t.Errorf("optional coverage vars not passed through: %+v", cfg)
+	}
+}

internal/smoke/coverage.go

@@ @@ -0,0 +1,139 @@ @@
+package smoke
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+)
+// covdataMergeArgs builds the `go tool covdata merge` argument list that unions
+// the per-binary coverage directories in inputDirs into outDir. Pure so the
+// command assembly is unit-testable without running the toolchain.
+func covdataMergeArgs(inputDirs []string, outDir string) []string {
+	return []string{"tool", "covdata", "merge",
+		"-i=" + strings.Join(inputDirs, ","),
+		"-o=" + outDir}
+}
+// covdataTextfmtArgs builds the `go tool covdata textfmt` argument list that
+// renders the merged coverage in inDir as a textual profile at outFile.
+func covdataTextfmtArgs(inDir, outFile string) []string {
+	return []string{"tool", "covdata", "textfmt",
+		"-i=" + inDir,
+		"-o=" + outFile}
+}
+// collectCoverage snapshots live coverage from the running server and agent,
+// pulls both GOCOVERDIRs together, and merges them into cfg.CoverOut, printing
+// the total. It needs both cfg.ServerGocoverdir and cfg.AgentGocoverdir; when
+// either is empty it is a no-op with an explanatory note. Any failure here is
+// the caller's to treat as non-fatal — the boot gate is the authoritative check.
+func collectCoverage(ctx context.Context, cfg Config) error {
+	if cfg.ServerGocoverdir == "" || cfg.AgentGocoverdir == "" {
+		fmt.Println("coverage: SERVER_GOCOVERDIR/AGENT_GOCOVERDIR not both set — skipping collection")
+		return nil
+	}
+	// 1. Snapshot: SIGUSR1 makes each covsnap handler flush counters into its
+	// GOCOVERDIR. The server is local; the agent runs as root on the remote, so
+	// its pkill needs sudo over SSH.
+	if err := runCmd(exec.CommandContext(ctx, "pkill", "-USR1", "-x", "eitri-server")); err != nil {
+		fmt.Fprintln(os.Stderr, "coverage: signalling local eitri-server:", err)
+	}
+	if err := runCmd(sshCommand(ctx, cfg, "sudo pkill -USR1 -x eitri-agent")); err != nil {
+		fmt.Fprintln(os.Stderr, "coverage: signalling remote eitri-agent:", err)
+	}
+	// Let both handlers finish writing before we read their dirs.
+	time.Sleep(2 * time.Second)
+	// 2. Collect: the server dir is local; stream the agent's over SSH via a
+	// sudo tar pipe (its files are root-owned, so a plain scp can't read them).
+	agentDir, err := os.MkdirTemp("", "eitri-smoke-agentcov-")
+	if err != nil {
+		return fmt.Errorf("temp dir for agent coverage: %w", err)
+	}
+	defer os.RemoveAll(agentDir)
+	if err := pullAgentCoverage(ctx, cfg, agentDir); err != nil {
+		return err
+	}
+	// 3. Merge both binaries' data, render a textual profile, and read the total.
+	// Start from a clean CoverOut: covdata refuses to read a directory that
+	// mixes covermodes, so a stale profile from an earlier deploy would clash
+	// with this run's data.
+	if err := os.RemoveAll(cfg.CoverOut); err != nil {
+		return fmt.Errorf("clean cover out %q: %w", cfg.CoverOut, err)
+	}
+	if err := os.MkdirAll(cfg.CoverOut, 0o755); err != nil {
+		return fmt.Errorf("mkdir cover out %q: %w", cfg.CoverOut, err)
+	}
+	if err := runCmd(exec.CommandContext(ctx, "go", covdataMergeArgs([]string{cfg.ServerGocoverdir, agentDir}, cfg.CoverOut)...)); err != nil {
+		return fmt.Errorf("covdata merge: %w", err)
+	}
+	textOut := filepath.Join(cfg.CoverOut, "coverage.txt")
+	if err := runCmd(exec.CommandContext(ctx, "go", covdataTextfmtArgs(cfg.CoverOut, textOut)...)); err != nil {
+		return fmt.Errorf("covdata textfmt: %w", err)
+	}
+	total, err := coverageTotal(ctx, textOut)
+	if err != nil {
+		return err
+	}
+	fmt.Printf("coverage: %s (profile: %s)\n", total, textOut)
+	return nil
+}
+// pullAgentCoverage streams the agent's GOCOVERDIR contents into destDir over a
+// sudo tar pipe (the coverage files are root-owned on the remote).
+func pullAgentCoverage(ctx context.Context, cfg Config, destDir string) error {
+	tarball, err := sshCommand(ctx, cfg, "sudo tar -C '"+cfg.AgentGocoverdir+"' -cf - .").Output()
+	if err != nil {
+		return fmt.Errorf("stream agent coverage over ssh: %w", err)
+	}
+	untar := exec.CommandContext(ctx, "tar", "-C", destDir, "-xf", "-")
+	untar.Stdin = bytes.NewReader(tarball)
+	if err := runCmd(untar); err != nil {
+		return fmt.Errorf("extract agent coverage: %w", err)
+	}
+	return nil
+}
+// coverageTotal returns the final "total:" line of `go tool cover -func`.
+func coverageTotal(ctx context.Context, profile string) (string, error) {
+	out, err := exec.CommandContext(ctx, "go", "tool", "cover", "-func="+profile).Output()
+	if err != nil {
+		return "", fmt.Errorf("go tool cover -func: %w", err)
+	}
+	lines := strings.Split(strings.TrimSpace(string(out)), "\n")
+	return lines[len(lines)-1], nil
+}
+// sshCommand builds the ssh invocation used to reach the agent host, matching
+// the flags the deploy boot-gate uses to reach the agent host.
+func sshCommand(ctx context.Context, cfg Config, remoteCmd string) *exec.Cmd {
+	return exec.CommandContext(ctx, "ssh",
+		"-p", strconv.Itoa(cfg.AgentPort),
+		"-o", "BatchMode=yes",
+		"-o", "ConnectTimeout=10",
+		cfg.AgentUserHost, remoteCmd)
+}
+// runCmd runs cmd, capturing stderr so a failure carries the command's own
+// diagnostic rather than a bare exit code.
+func runCmd(cmd *exec.Cmd) error {
+	var errb bytes.Buffer
+	cmd.Stderr = &errb
+	if err := cmd.Run(); err != nil {
+		if msg := strings.TrimSpace(errb.String()); msg != "" {
+			return fmt.Errorf("%s: %w: %s", cmd.Args[0], err, msg)
+		}
+		return fmt.Errorf("%s: %w", cmd.Args[0], err)
+	}
+	return nil
+}

internal/smoke/coverage_test.go

@@ @@ -0,0 +1,30 @@ @@
+package smoke
+import (
+	"slices"
+	"testing"
+)
+func TestCovdataMergeArgs(t *testing.T) {
+	got := covdataMergeArgs([]string{"/cov/server", "/cov/agent"}, "/cov/out")
+	want := []string{"tool", "covdata", "merge", "-i=/cov/server,/cov/agent", "-o=/cov/out"}
+	if !slices.Equal(got, want) {
+		t.Fatalf("covdataMergeArgs = %v, want %v", got, want)
+	}
+}
+func TestCovdataMergeArgsSingleInput(t *testing.T) {
+	got := covdataMergeArgs([]string{"/only"}, "/out")
+	want := []string{"tool", "covdata", "merge", "-i=/only", "-o=/out"}
+	if !slices.Equal(got, want) {
+		t.Fatalf("covdataMergeArgs = %v, want %v", got, want)
+	}
+}
+func TestCovdataTextfmtArgs(t *testing.T) {
+	got := covdataTextfmtArgs("/cov/out", "/cov/out/coverage.txt")
+	want := []string{"tool", "covdata", "textfmt", "-i=/cov/out", "-o=/cov/out/coverage.txt"}
+	if !slices.Equal(got, want) {
+		t.Fatalf("covdataTextfmtArgs = %v, want %v", got, want)
+	}
+}

internal/smoke/gatecheck.go

@@ @@ -0,0 +1,78 @@ @@
+package smoke
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+	"github.com/a73x/eitri/internal/gateclient"
+	"golang.org/x/crypto/ssh"
+)
+// realGateHooks builds the live SSH-CA gate steps: register the smoke user CA
+// with the tenant, and reach the guest through the gate to prove access. The
+// tenant is the operator PAT's own tenant (derived via Me() by the caller), so
+// the CA registration and connect names match the fleet's real partition.
+func realGateHooks(cfg Config, tenant string, ca gateclient.CertAuthority, userCA ssh.Signer, now func() time.Time, sleep func(time.Duration)) *gateHooks {
+	auth := gateclient.NewGateAuth(ca, userCA, tenant, now)
+	return &gateHooks{
+		register: func(ctx context.Context) error { return auth.Register(ctx) },
+		exec:     func(ctx context.Context, vmName string) error { return gateExec(ctx, cfg, auth, vmName, now, sleep) },
+	}
+}
+// gateExec proves the guest is reachable through the SSH-CA gate: it retries
+// dial+login over the guest's pre-sshd boot window (a dial or session error
+// mid-boot is expected, not fatal) until it logs in as cfg.SmokeVMUser and
+// confirms `id -un` echoes that same user, or the 120s deadline expires.
+func gateExec(ctx context.Context, cfg Config, auth gateclient.Credentials, vmName string, now func() time.Time, sleep func(time.Duration)) error {
+	var lastErr error
+	err := pollLoop(ctx, now, sleep, 120*time.Second, 5*time.Second, func() (bool, error) {
+		client, dialErr := gateclient.Dial(ctx, gateclient.DialConfig{
+			Gate:   cfg.SmokeGate,
+			VMUser: cfg.SmokeVMUser,
+			Auth:   auth,
+		}, vmName)
+		if dialErr != nil {
+			lastErr = dialErr
+			return false, nil
+		}
+		defer client.Close()
+		out, runErr := runGuestCommand(client, "id -un")
+		if runErr != nil {
+			lastErr = runErr
+			return false, nil
+		}
+		got := strings.TrimSpace(out)
+		if got != cfg.SmokeVMUser {
+			return false, fmt.Errorf("gate SSH logged into %q as %q, want %q", vmName, got, cfg.SmokeVMUser)
+		}
+		return true, nil
+	})
+	if err != nil {
+		if errors.Is(err, errPollTimeout) {
+			return fmt.Errorf("FAIL: could not reach guest %q through the gate within 120s: %w", vmName, lastErr)
+		}
+		return err
+	}
+	return nil
+}
+// runGuestCommand runs cmd in a new session on client and returns its stdout.
+func runGuestCommand(client *ssh.Client, cmd string) (string, error) {
+	session, err := client.NewSession()
+	if err != nil {
+		return "", fmt.Errorf("open ssh session: %w", err)
+	}
+	defer session.Close()
+	out, err := session.Output(cmd)
+	if err != nil {
+		return "", fmt.Errorf("run %q: %w", cmd, err)
+	}
+	return string(out), nil
+}

internal/smoke/login.go

@@ @@ -0,0 +1,108 @@ @@
+package smoke
+import (
+	"fmt"
+	"net/http"
+	"net/http/cookiejar"
+	"net/url"
+	"time"
+	"github.com/a73x/eitri/internal/server/api/client"
+)
+// sessionCookie is the name of the console session cookie the callback sets.
+// Its presence in the jar after the credential POST is the one honest signal
+// that sign-in succeeded — a failed password re-renders the login form as a
+// plain 200 without ever setting it.
+const sessionCookie = "eitri_session"
+// loginPAT signs in through the real OIDC code flow — the same doors a browser
+// walks — and mints a short-lived PAT for this run. There is no special grant
+// for machines (spec §3): CI registers a flat-file user and then drives the
+// standard authorization-code flow headlessly against the plain-HTML login
+// form, lands an ordinary session, and mints a PAT through the normal route.
+func loginPAT(serverURL, email, password string) (string, error) {
+	base, err := url.Parse(serverURL)
+	if err != nil {
+		return "", fmt.Errorf("parse server url %q: %w", serverURL, err)
+	}
+	jar, err := cookiejar.New(nil)
+	if err != nil {
+		return "", fmt.Errorf("cookie jar: %w", err)
+	}
+	// A default (redirect-following) client: GET /auth/login bounces through the
+	// issuer's authorize endpoint to the login form, and the credential POST
+	// runs issuer -> /auth/callback -> / — we want every hop followed so the
+	// session cookie lands in the jar and resp.Request.URL is the form's URL.
+	hc := &http.Client{Jar: jar, Timeout: 30 * time.Second}
+	// GET /auth/login follows the redirect chain to the issuer's login form.
+	// We POST credentials straight back to resp.Request.URL — the authorize
+	// URL we landed on, query intact — so no HTML parsing is needed and the
+	// form's action attribute is never consulted. (eitri-oidc accepts the
+	// credential POST on that same URL by contract, spec §2.1.)
+	resp, err := hc.Get(serverURL + "/auth/login")
+	if err != nil {
+		return "", fmt.Errorf("GET /auth/login: %w", err)
+	}
+	resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("sign-in: login form GET returned %d (want 200) at %s", resp.StatusCode, resp.Request.URL)
+	}
+	formURL := resp.Request.URL.String()
+	// POST credentials to the form URL; the redirects run issuer -> /auth/callback
+	// -> / and the callback plants the session cookie in the jar on success.
+	resp2, err := hc.PostForm(formURL, url.Values{
+		"email":    {email},
+		"password": {password},
+	})
+	if err != nil {
+		return "", fmt.Errorf("POST credentials: %w", err)
+	}
+	resp2.Body.Close()
+	// A failed sign-in re-renders the login form as a 200 without a session
+	// cookie. Detect success by the cookie's presence in the jar, never by the
+	// status. The password is never included in the error.
+	if cookieByName(jar.Cookies(base), sessionCookie) == nil {
+		return "", fmt.Errorf("sign-in failed for %q: no %s cookie after credential POST "+
+			"(final status %d at %s) — check the CI user exists in eitri-oidc and the password matches",
+			email, sessionCookie, resp2.StatusCode, resp2.Request.URL)
+	}
+	// Mint the PAT through the shared client, riding the session cookie in the
+	// jar (Token left empty so no Bearer header is sent).
+	c := &client.Client{BaseURL: serverURL, HTTP: hc}
+	tok, err := c.CreateAPIToken("boot-gate", time.Hour)
+	if err != nil {
+		return "", fmt.Errorf("mint boot-gate PAT: %w", err)
+	}
+	return tok.Token, nil
+}
+// proveCredentialChain is the boot-gate's phase-1 proof: it confirms the minted
+// PAT resolves to a non-empty tenant via GET /api/v1/me, exercising the whole
+// issuer -> login -> session -> PAT-mint chain without touching VMs. It returns
+// the tenant handle so the caller can log which tenant the ci user landed in.
+func proveCredentialChain(serverURL, token string) (string, error) {
+	c := &client.Client{BaseURL: serverURL, Token: token, HTTP: &http.Client{Timeout: 30 * time.Second}}
+	me, err := c.Me()
+	if err != nil {
+		return "", fmt.Errorf("credential-chain proof: Me() with minted PAT: %w", err)
+	}
+	if me.Tenant == "" {
+		return "", fmt.Errorf("credential-chain proof: minted PAT resolved to an empty tenant")
+	}
+	return me.Tenant, nil
+}
+// cookieByName returns the named cookie from cookies, or nil if absent.
+func cookieByName(cookies []*http.Cookie, name string) *http.Cookie {
+	for _, c := range cookies {
+		if c.Name == name {
+			return c
+		}
+	}
+	return nil
+}

internal/smoke/login_test.go

@@ @@ -0,0 +1,171 @@ @@
+package smoke
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+	"github.com/a73x/eitri/internal/oidcprovider"
+	"github.com/a73x/eitri/internal/server/api"
+	"github.com/a73x/eitri/internal/server/api/client"
+	"github.com/a73x/eitri/internal/server/hub"
+	"github.com/a73x/eitri/internal/server/registry"
+	"github.com/a73x/eitri/internal/server/store"
+)
+// smokeLoginEnv is a whole eitri-server /auth + /api stack wired to a real
+// internal/oidcprovider issuer — the exact credential chain a deploy walks. It
+// exists to prove loginPAT end-to-end: sign in through the login form, land a
+// session, mint a PAT, and use it against the API.
+type smokeLoginEnv struct {
+	serverURL string
+	st        *store.Store
+}
+// newSmokeLoginEnv stands up the issuer and server with one seeded user and
+// returns the server's base URL. It mirrors internal/server/api/auth_test.go's
+// fixture: the httptest listener binds on construction so we know the server's
+// address (hence the OIDC redirect URL) before wiring the two ends.
+func newSmokeLoginEnv(t *testing.T, email, password string) smokeLoginEnv {
+	t.Helper()
+	apiSrv := httptest.NewUnstartedServer(nil)
+	publicURL := "http://" + apiSrv.Listener.Addr().String()
+	st, err := store.Open(t.TempDir()+"/db", "10.77.0.0/16")
+	if err != nil {
+		t.Fatalf("store.Open: %v", err)
+	}
+	t.Cleanup(func() { st.Close() })
+	usersPath := filepath.Join(t.TempDir(), "users.json")
+	if err := oidcprovider.AddUser(usersPath, email, password); err != nil {
+		t.Fatalf("AddUser: %v", err)
+	}
+	prov, err := oidcprovider.New(oidcprovider.Config{
+		UsersFile:  usersPath,
+		SigningKey: filepath.Join(t.TempDir(), "signing.key"),
+		Clients:    []oidcprovider.Client{{ID: "eitri-console", RedirectURL: publicURL + "/auth/callback"}},
+	})
+	if err != nil {
+		t.Fatalf("oidcprovider.New: %v", err)
+	}
+	oidcSrv := httptest.NewServer(prov.Handler())
+	t.Cleanup(oidcSrv.Close)
+	prov.SetIssuer(oidcSrv.URL)
+	a := api.New(api.Config{
+		HostSecret: []byte("hostsecret"),
+		OIDC:       api.OIDCConfig{Issuer: oidcSrv.URL, ClientID: "eitri-console", PublicURL: publicURL},
+	}, st, registry.New(time.Now), hub.New())
+	t.Cleanup(a.Close)
+	root := http.NewServeMux()
+	root.Handle("/api/", a.Handler())
+	root.Handle("/auth/", a.AuthHandler())
+	apiSrv.Config.Handler = root
+	apiSrv.Start()
+	t.Cleanup(apiSrv.Close)
+	return smokeLoginEnv{serverURL: publicURL, st: st}
+}
+func TestLoginPATMintsUsableToken(t *testing.T) {
+	const (
+		email    = "ci@eitri.local"
+		password = "hunter2hunter2"
+	)
+	env := newSmokeLoginEnv(t, email, password)
+	token, err := loginPAT(env.serverURL, email, password)
+	if err != nil {
+		t.Fatalf("loginPAT: %v", err)
+	}
+	if token == "" {
+		t.Fatal("loginPAT returned an empty token")
+	}
+	// The PAT must actually authenticate an ordinary API call.
+	c := &client.Client{BaseURL: env.serverURL, Token: token, HTTP: &http.Client{Timeout: 10 * time.Second}}
+	me, err := c.Me()
+	if err != nil {
+		t.Fatalf("Me() with minted PAT: %v", err)
+	}
+	if me.Email != email {
+		t.Errorf("Me().Email = %q, want %q", me.Email, email)
+	}
+	if _, err := c.ListHosts(context.Background()); err != nil {
+		t.Errorf("ListHosts() with minted PAT: %v", err)
+	}
+}
+// TestProveCredentialChain is the phase-1 proof: signing in and minting a PAT
+// must resolve to a non-empty tenant via Me() (the ci user's JIT tenant),
+// without any VM involvement.
+func TestProveCredentialChain(t *testing.T) {
+	const (
+		email    = "ci@eitri.local"
+		password = "hunter2hunter2"
+	)
+	env := newSmokeLoginEnv(t, email, password)
+	token, err := loginPAT(env.serverURL, email, password)
+	if err != nil {
+		t.Fatalf("loginPAT: %v", err)
+	}
+	tenant, err := proveCredentialChain(env.serverURL, token)
+	if err != nil {
+		t.Fatalf("proveCredentialChain: %v", err)
+	}
+	if tenant == "" {
+		t.Fatal("proveCredentialChain returned an empty tenant")
+	}
+}
+// TestScenarioPATDerivesTenant is the phase-2 contract: a scenario client built
+// from an operator-minted PAT derives its tenant via Me() — not from any env or
+// hardcoded default. The store mints the PAT directly, standing in for the
+// console-minted "deploy" token a real operator saves to CI_PAT_FILE.
+func TestScenarioPATDerivesTenant(t *testing.T) {
+	env := newSmokeLoginEnv(t, "ci@eitri.local", "hunter2hunter2")
+	// A distinct operator tenant with a row (handleMe 401s on a rowless tenant).
+	tn, err := env.st.CreateTenantForIdentity("https://op.example", "op-subject", "op@example.com")
+	if err != nil {
+		t.Fatalf("CreateTenantForIdentity: %v", err)
+	}
+	secret, _, err := env.st.CreateAPIToken(tn.ID, "deploy", 0)
+	if err != nil {
+		t.Fatalf("CreateAPIToken: %v", err)
+	}
+	c := &client.Client{BaseURL: env.serverURL, Token: secret, HTTP: &http.Client{Timeout: 10 * time.Second}}
+	me, err := c.Me()
+	if err != nil {
+		t.Fatalf("Me() with store-minted PAT: %v", err)
+	}
+	if me.Tenant != tn.ID {
+		t.Errorf("Me().Tenant = %q, want %q (derived, not assumed)", me.Tenant, tn.ID)
+	}
+}
+func TestLoginPATWrongPasswordFails(t *testing.T) {
+	const email = "ci@eitri.local"
+	env := newSmokeLoginEnv(t, email, "the-right-password")
+	_, err := loginPAT(env.serverURL, email, "the-wrong-password")
+	if err == nil {
+		t.Fatal("loginPAT: want error on wrong password, got nil")
+	}
+	// The failure must name the sign-in problem and must not echo the password.
+	if !strings.Contains(err.Error(), "sign-in failed") || !strings.Contains(err.Error(), email) {
+		t.Errorf("error = %q, want it to mention the sign-in failure and the user", err)
+	}
+	if strings.Contains(err.Error(), "the-wrong-password") {
+		t.Errorf("error must not echo the password: %q", err)
+	}
+}

internal/smoke/run.go

@@ @@ -0,0 +1,157 @@ @@
+// Package smoke is the deploy boot-gate harness. It drives the live eitri
+// fleet through a credential-chain proof and a create -> boot-proof -> gate-SSH
+// -> reap of one throwaway VM, returning an error on any failure. It is a
+// client of the eitri API plus SSH; it embeds no control-plane or agent code.
+// The eitri-smoke command is thin wiring over Run (arch R14).
+package smoke
+import (
+	"context"
+	"crypto/rand"
+	"encoding/hex"
+	"fmt"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+	"github.com/a73x/eitri/internal/server/api/client"
+)
+// Run executes one boot-gate pass, reading its settings from the process
+// environment. It returns an error describing the first failed step; a nil
+// return means the credential chain, VM boot-proof, and reap all passed.
+func Run() error {
+	cfg, err := loadConfig(os.Getenv)
+	if err != nil {
+		return err
+	}
+	// Phase 1 — credential-chain proof. The admin token is gone: the boot-gate
+	// authenticates like a human. Read the machine identity's password (CI_USER,
+	// the deploy identity), sign in through the real OIDC code flow, mint a
+	// short-lived PAT, and confirm it resolves to a tenant (spec §3). This proves
+	// issuer, login form, session, and PAT mint end to end; it deliberately never
+	// touches VMs — the machine identity's JIT tenant owns no hosts (the fleet's
+	// `default` tenant is human-owned), so the lifecycle half runs as the operator
+	// below.
+	pwBytes, err := os.ReadFile(cfg.CIPasswordFile)
+	if err != nil {
+		return fmt.Errorf("read CI password file %q: %w", cfg.CIPasswordFile, err)
+	}
+	password := strings.TrimRight(string(pwBytes), "\r\n")
+	token, err := loginPAT(cfg.ServerURL, cfg.CIUser, password)
+	if err != nil {
+		return err
+	}
+	ciTenant, err := proveCredentialChain(cfg.ServerURL, token)
+	if err != nil {
+		return err
+	}
+	fmt.Printf("credential chain OK (tenant %s)\n", ciTenant)
+	// Phase 2 — VM lifecycle. Authenticate with an operator-minted PAT read from
+	// CI_PAT_FILE (a normal, tenant-scoped console PAT — spec §3's automation
+	// story). The scenario's tenant is DERIVED via Me(), never assumed, and
+	// threaded into the user-CA registration and gate connect name below.
+	patBytes, err := os.ReadFile(cfg.CIPATFile)
+	if err != nil {
+		return fmt.Errorf("read CI PAT file %q: %w", cfg.CIPATFile, err)
+	}
+	pat := strings.TrimRight(string(patBytes), "\r\n")
+	api := &client.Client{
+		BaseURL:     cfg.ServerURL,
+		Token:       pat,
+		UserCALabel: "eitri-smoke",
+		HTTP:        &http.Client{Timeout: 30 * time.Second},
+	}
+	me, err := api.Me()
+	if err != nil {
+		return fmt.Errorf("resolve operator PAT tenant: %w", err)
+	}
+	if me.Tenant == "" {
+		return fmt.Errorf("operator PAT (%s) resolved to an empty tenant", cfg.CIPATFile)
+	}
+	tenant := me.Tenant
+	fmt.Printf("operator PAT tenant: %s\n", tenant)
+	vmName, err := randVMName()
+	if err != nil {
+		return fmt.Errorf("generate vm name: %w", err)
+	}
+	var gate *gateHooks
+	if cfg.SmokeGate != "" && cfg.SmokeUserCAFile != "" {
+		userCA, err := loadOrCreateUserCA(cfg.SmokeUserCAFile)
+		if err != nil {
+			return fmt.Errorf("load smoke user CA: %w", err)
+		}
+		gate = realGateHooks(cfg, tenant, api, userCA, time.Now, time.Sleep)
+	} else {
+		fmt.Fprintln(os.Stderr, "eitri-smoke: gate SSH check skipped (SMOKE_GATE/SMOKE_USER_CA_FILE not set)")
+	}
+	ctx := context.Background()
+	msg, err := runScenario(ctx, cfg, vmName, api, realRunSSH(cfg.AgentUserHost, cfg.AgentPort), gate, time.Now, time.Sleep, realReadPubKey)
+	if err != nil {
+		return err
+	}
+	fmt.Println(msg)
+	// The boot gate has passed. Collect integration coverage as a by-product
+	// when COVER_OUT is set; a failure here must NOT fail the deploy, so warn
+	// and carry on — the gate above is the authoritative check.
+	if cfg.CoverOut != "" {
+		if err := collectCoverage(ctx, cfg); err != nil {
+			fmt.Fprintln(os.Stderr, "eitri-smoke: coverage collection failed (boot gate still passed):", err)
+		}
+	}
+	return nil
+}
+// randVMName generates a throwaway VM name of the form "smoke-<8 hex digits>",
+// unique enough that concurrent smoke runs don't collide.
+func randVMName() (string, error) {
+	var b [4]byte
+	if _, err := rand.Read(b[:]); err != nil {
+		return "", err
+	}
+	return "smoke-" + hex.EncodeToString(b[:]), nil
+}
+// realRunSSH returns the real sshFunc: it shells out to the ssh binary
+// against userHost:port with the deploy boot-gate's ssh flags, running
+// remoteCmd non-interactively and returning its stdout.
+func realRunSSH(userHost string, port int) sshFunc {
+	return func(ctx context.Context, remoteCmd string) (string, error) {
+		cmd := exec.CommandContext(ctx, "ssh",
+			"-p", strconv.Itoa(port),
+			"-o", "BatchMode=yes",
+			"-o", "ConnectTimeout=10",
+			userHost, remoteCmd)
+		out, err := cmd.Output()
+		return string(out), err
+	}
+}
+// realReadPubKey reads the local operator's SSH public key, trying
+// ~/.ssh/id_ed25519.pub then ~/.ssh/id_rsa.pub. It returns "" (not an error)
+// if neither is present — the scenario tolerates a keyless VM.
+func realReadPubKey() string {
+	home, err := os.UserHomeDir()
+	if err != nil {
+		return ""
+	}
+	for _, name := range []string{"id_ed25519.pub", "id_rsa.pub"} {
+		data, err := os.ReadFile(filepath.Join(home, ".ssh", name))
+		if err == nil {
+			return strings.TrimSpace(string(data))
+		}
+	}
+	return ""
+}

internal/smoke/scenario.go

@@ @@ -0,0 +1,196 @@ @@
+package smoke
+import (
+	"context"
+	"errors"
+	"fmt"
+	"regexp"
+	"time"
+	"github.com/a73x/eitri/internal/server/api/client"
+)
+// bootedPattern / panickedPattern classify a guest's serial console text. They
+// mirror the boot-gate's grep -aiE serial-console patterns exactly.
+var (
+	bootedPattern   = regexp.MustCompile(`(?i)Welcome to.*Ubuntu|login:|Reached target.*Multi-User`)
+	panickedPattern = regexp.MustCompile(`(?i)Kernel panic|Cannot open root`)
+	errPollTimeout  = errors.New("poll timeout")
+)
+// classifySerial inspects a guest's serial console text and reports whether it
+// shows evidence of a successful userspace boot and/or a kernel panic /
+// root-mount failure. It is pure — no I/O — so the boot-proof classification
+// logic is fully unit-testable.
+func classifySerial(text string) (booted, panicked bool) {
+	return bootedPattern.MatchString(text), panickedPattern.MatchString(text)
+}
+// bootProofCommand builds the remote shell command run over SSH on the agent
+// host to read and sanitize a VM's serial console log.
+func bootProofCommand(agentStateDir, vmID string) string {
+	return fmt.Sprintf(`sudo cat '%s/vms/%s/serial.log' 2>/dev/null | tr -cd '\11\12\15\40-\176'`, agentStateDir, vmID)
+}
+// vmAPI is the subset of the shared API client the scenario needs. Declaring
+// it lets tests supply a fake instead of a real HTTP-backed *client.Client.
+type vmAPI interface {
+	ListHosts(ctx context.Context) ([]client.Host, error)
+	CreateVM(ctx context.Context, req client.CreateVMRequest) (client.CreateVMResponse, error)
+	ListVMs(ctx context.Context) ([]client.VM, error)
+	DeleteVM(ctx context.Context, id string) error
+}
+// getVM finds the VM with the given id in the current listing. The bool
+// return reports whether it was present. There is deliberately no GET-one
+// endpoint, so this list-filter is the smoke's lookup.
+func getVM(ctx context.Context, c vmAPI, id string) (client.VM, bool, error) {
+	vms, err := c.ListVMs(ctx)
+	if err != nil {
+		return client.VM{}, false, err
+	}
+	for _, vm := range vms {
+		if vm.ID == id {
+			return vm, true, nil
+		}
+	}
+	return client.VM{}, false, nil
+}
+// sshFunc runs remoteCmd on the agent host over SSH and returns its stdout.
+type sshFunc func(ctx context.Context, remoteCmd string) (string, error)
+// gateHooks bundles the optional SSH-CA gate steps. nil means "skip the gate".
+type gateHooks struct {
+	register func(ctx context.Context) error                // upload the smoke user CA to the tenant (before create)
+	exec     func(ctx context.Context, vmName string) error // reach the guest through the gate (after boot)
+}
+// pollLoop calls attempt repeatedly (with sleep between calls) until attempt
+// reports done, returns a non-nil error, or the deadline (now()+timeout) is
+// reached, in which case it returns errPollTimeout. now and sleep are
+// injected so callers can drive the deadline logic with a virtual clock.
+func pollLoop(ctx context.Context, now func() time.Time, sleep func(time.Duration), timeout, interval time.Duration, attempt func() (done bool, err error)) error {
+	deadline := now().Add(timeout)
+	for {
+		if err := ctx.Err(); err != nil {
+			return err
+		}
+		done, err := attempt()
+		if err != nil {
+			return err
+		}
+		if done {
+			return nil
+		}
+		if !now().Before(deadline) {
+			return errPollTimeout
+		}
+		sleep(interval)
+	}
+}
+// runScenario drives the full register -> create -> ready -> boot-proof ->
+// gate-exec -> reap sequence against c (the API), runSSH (the boot-proof
+// transport), and readPubKey (the local SSH key source). vmName is the
+// pre-generated name for the throwaway VM. gate, when non-nil, registers the
+// smoke's user CA with the tenant before create (the guest bakes its trusted
+// CAs at boot, so registration MUST happen first) and proves gate SSH access
+// after the boot-proof — a hard gate, so a failure there fails the scenario.
+// now/sleep are the injected clock so the poll deadlines are unit-testable
+// without real waiting. On success it returns the human-readable COMPLETE
+// line; on any failure it returns a descriptive error.
+func runScenario(ctx context.Context, cfg Config, vmName string, c vmAPI, runSSH sshFunc, gate *gateHooks, now func() time.Time, sleep func(time.Duration), readPubKey func() string) (string, error) {
+	if gate != nil {
+		if err := gate.register(ctx); err != nil {
+			return "", fmt.Errorf("register smoke user CA: %w", err)
+		}
+	}
+	hostList, err := c.ListHosts(ctx)
+	if err != nil {
+		return "", fmt.Errorf("list hosts: %w", err)
+	}
+	if len(hostList) == 0 {
+		return "", errors.New("no hosts available")
+	}
+	hostID := hostList[0].ID
+	sshKey := readPubKey()
+	start := now()
+	created, err := c.CreateVM(ctx, client.CreateVMRequest{HostID: hostID, Name: vmName, SSHAuthorizedKey: sshKey})
+	if err != nil {
+		return "", fmt.Errorf("create vm: %w", err)
+	}
+	vmID := created.ID
+	var lastPhase string
+	err = pollLoop(ctx, now, sleep, 600*time.Second, 5*time.Second, func() (bool, error) {
+		vm, _, err := getVM(ctx, c, vmID)
+		if err != nil {
+			return false, fmt.Errorf("poll vm ready: %w", err)
+		}
+		lastPhase = vm.Phase
+		return vm.Phase == "ready" && vm.AssignedIP != "", nil
+	})
+	if err != nil {
+		if errors.Is(err, errPollTimeout) {
+			return "", fmt.Errorf("FAIL: VM not ready within 600s (phase=%s)", lastPhase)
+		}
+		return "", err
+	}
+	coldStart := now().Sub(start)
+	remoteCmd := bootProofCommand(cfg.AgentStateDir, vmID)
+	err = pollLoop(ctx, now, sleep, 180*time.Second, 6*time.Second, func() (bool, error) {
+		serial, sshErr := runSSH(ctx, remoteCmd)
+		if sshErr != nil {
+			// Match the bash scenario's "|| true": an SSH hiccup mid-boot is
+			// not fatal on its own, just an empty-serial iteration.
+			serial = ""
+		}
+		booted, panicked := classifySerial(serial)
+		if panicked {
+			return false, errors.New("FAIL: guest panic / root-mount failure in serial log")
+		}
+		return booted, nil
+	})
+	if err != nil {
+		if errors.Is(err, errPollTimeout) {
+			return "", errors.New("FAIL: no userspace boot evidence in serial within 180s")
+		}
+		return "", err
+	}
+	gateOK := false
+	if gate != nil {
+		if err := gate.exec(ctx, vmName); err != nil {
+			return "", err
+		}
+		gateOK = true
+	}
+	if err := c.DeleteVM(ctx, vmID); err != nil {
+		return "", fmt.Errorf("delete vm: %w", err)
+	}
+	err = pollLoop(ctx, now, sleep, 120*time.Second, 5*time.Second, func() (bool, error) {
+		_, present, err := getVM(ctx, c, vmID)
+		if err != nil {
+			return false, fmt.Errorf("poll vm reaped: %w", err)
+		}
+		return !present, nil
+	})
+	if err != nil {
+		if errors.Is(err, errPollTimeout) {
+			return "", errors.New("FAIL: VM not hard-deleted within 120s of tombstone")
+		}
+		return "", err
+	}
+	msg := fmt.Sprintf("SMOKE COMPLETE — booted under UEFI, cold_start=%ds, reaped OK", int64(coldStart.Seconds()))
+	if gateOK {
+		msg += ", gate SSH: ok"
+	}
+	return msg, nil
+}

internal/smoke/scenario_test.go

@@ @@ -0,0 +1,455 @@ @@
+package smoke
+import (
+	"context"
+	"errors"
+	"strings"
+	"testing"
+	"time"
+	"github.com/a73x/eitri/internal/server/api/client"
+)
+// --- classifySerial -------------------------------------------------------
+func TestClassifySerialBooted(t *testing.T) {
+	text := "[    5.123456] Ubuntu 22.04.3 LTS ubuntu-vm ttyS0\n\nubuntu-vm login: "
+	booted, panicked := classifySerial(text)
+	if !booted {
+		t.Error("booted = false, want true")
+	}
+	if panicked {
+		t.Error("panicked = true, want false")
+	}
+}
+func TestClassifySerialPanic(t *testing.T) {
+	text := "[    2.345678] Kernel panic - not syncing: VFS: Unable to mount root fs on unknown-block(0,0)"
+	booted, panicked := classifySerial(text)
+	if booted {
+		t.Error("booted = true, want false")
+	}
+	if !panicked {
+		t.Error("panicked = false, want true")
+	}
+}
+func TestClassifySerialStillBooting(t *testing.T) {
+	text := "[    0.123456] Booting Linux on physical CPU 0x0\n[    0.234567] Linux version 6.5.0"
+	booted, panicked := classifySerial(text)
+	if booted {
+		t.Error("booted = true, want false")
+	}
+	if panicked {
+		t.Error("panicked = true, want false")
+	}
+}
+// --- fakes for runScenario -------------------------------------------------
+// fakeClock is a controllable now()/sleep() pair: sleep advances the virtual
+// clock instead of waiting, so deadline logic runs at test speed.
+type fakeClock struct{ t time.Time }
+func (c *fakeClock) now() time.Time        { return c.t }
+func (c *fakeClock) sleep(d time.Duration) { c.t = c.t.Add(d) }
+// testAPI implements vmAPI by delegating to per-test closures.
+type testAPI struct {
+	listHostsFunc func(ctx context.Context) ([]client.Host, error)
+	createVMFunc  func(ctx context.Context, req client.CreateVMRequest) (client.CreateVMResponse, error)
+	listVMsFunc   func(ctx context.Context) ([]client.VM, error)
+	deleteVMFunc  func(ctx context.Context, id string) error
+}
+func (a *testAPI) ListHosts(ctx context.Context) ([]client.Host, error) {
+	return a.listHostsFunc(ctx)
+}
+func (a *testAPI) CreateVM(ctx context.Context, req client.CreateVMRequest) (client.CreateVMResponse, error) {
+	return a.createVMFunc(ctx, req)
+}
+func (a *testAPI) ListVMs(ctx context.Context) ([]client.VM, error) {
+	return a.listVMsFunc(ctx)
+}
+func (a *testAPI) DeleteVM(ctx context.Context, id string) error { return a.deleteVMFunc(ctx, id) }
+func baseCfg() Config {
+	return Config{AgentStateDir: "/var/lib/eitri-agent", AgentUserHost: "ubuntu@10.0.0.5", AgentPort: 22}
+}
+func noopReadPubKey() string { return "ssh-ed25519 AAAAfake test@smoke" }
+// --- runScenario: success path ---------------------------------------------
+func TestRunScenarioSuccess(t *testing.T) {
+	listVMsCalls := 0
+	api := &testAPI{
+		listHostsFunc: func(ctx context.Context) ([]client.Host, error) {
+			return []client.Host{{ID: "host-1"}}, nil
+		},
+		createVMFunc: func(ctx context.Context, req client.CreateVMRequest) (client.CreateVMResponse, error) {
+			if req.HostID != "host-1" {
+				t.Errorf("CreateVM HostID = %q, want host-1", req.HostID)
+			}
+			return client.CreateVMResponse{ID: "vm-1"}, nil
+		},
+		listVMsFunc: func(ctx context.Context) ([]client.VM, error) {
+			listVMsCalls++
+			switch {
+			case listVMsCalls < 3:
+				return []client.VM{{ID: "vm-1", Phase: "booting"}}, nil
+			case listVMsCalls == 3:
+				return []client.VM{{ID: "vm-1", Phase: "ready", AssignedIP: "10.0.0.9"}}, nil
+			default:
+				// Reap poll: absent from the first check.
+				return nil, nil
+			}
+		},
+		deleteVMFunc: func(ctx context.Context, id string) error {
+			if id != "vm-1" {
+				t.Errorf("deleteVM id = %q, want vm-1", id)
+			}
+			return nil
+		},
+	}
+	sshCalls := 0
+	runSSH := func(ctx context.Context, remoteCmd string) (string, error) {
+		sshCalls++
+		if !strings.Contains(remoteCmd, "vm-1") {
+			t.Errorf("remoteCmd = %q, want it to reference vm-1", remoteCmd)
+		}
+		if sshCalls < 2 {
+			return "[    0.1] Booting Linux...", nil
+		}
+		return "Ubuntu 24.04 LTS ubuntu-vm login: ", nil
+	}
+	clock := &fakeClock{t: time.Unix(0, 0)}
+	msg, err := runScenario(context.Background(), baseCfg(), "smoke-test", api, runSSH, nil, clock.now, clock.sleep, noopReadPubKey)
+	if err != nil {
+		t.Fatalf("runScenario: %v", err)
+	}
+	if !strings.Contains(msg, "SMOKE COMPLETE") || !strings.Contains(msg, "reaped OK") {
+		t.Errorf("message = %q, want SMOKE COMPLETE ... reaped OK", msg)
+	}
+	if !strings.Contains(msg, "cold_start=") {
+		t.Errorf("message = %q, want cold_start=", msg)
+	}
+}
+// --- runScenario: serial panic -----------------------------------------
+func TestRunScenarioSerialPanicFails(t *testing.T) {
+	api := &testAPI{
+		listHostsFunc: func(ctx context.Context) ([]client.Host, error) {
+			return []client.Host{{ID: "host-1"}}, nil
+		},
+		createVMFunc: func(ctx context.Context, req client.CreateVMRequest) (client.CreateVMResponse, error) {
+			return client.CreateVMResponse{ID: "vm-1"}, nil
+		},
+		listVMsFunc: func(ctx context.Context) ([]client.VM, error) {
+			return []client.VM{{ID: "vm-1", Phase: "ready", AssignedIP: "10.0.0.9"}}, nil
+		},
+		deleteVMFunc: func(ctx context.Context, id string) error {
+			t.Fatal("DeleteVM should not be called after a panic")
+			return nil
+		},
+	}
+	runSSH := func(ctx context.Context, remoteCmd string) (string, error) {
+		return "Kernel panic - not syncing: VFS: Unable to mount root fs", nil
+	}
+	clock := &fakeClock{t: time.Unix(0, 0)}
+	_, err := runScenario(context.Background(), baseCfg(), "smoke-test", api, runSSH, nil, clock.now, clock.sleep, noopReadPubKey)
+	if err == nil {
+		t.Fatal("runScenario: want error, got nil")
+	}
+	if !strings.Contains(err.Error(), "panic") {
+		t.Errorf("error = %q, want it to mention panic", err.Error())
+	}
+}
+// --- runScenario: never-ready timeout ---------------------------------
+func TestRunScenarioNeverReadyTimesOut(t *testing.T) {
+	api := &testAPI{
+		listHostsFunc: func(ctx context.Context) ([]client.Host, error) {
+			return []client.Host{{ID: "host-1"}}, nil
+		},
+		createVMFunc: func(ctx context.Context, req client.CreateVMRequest) (client.CreateVMResponse, error) {
+			return client.CreateVMResponse{ID: "vm-1"}, nil
+		},
+		listVMsFunc: func(ctx context.Context) ([]client.VM, error) {
+			return []client.VM{{ID: "vm-1", Phase: "booting"}}, nil
+		},
+		deleteVMFunc: func(ctx context.Context, id string) error {
+			t.Fatal("DeleteVM should not be called when the VM never becomes ready")
+			return nil
+		},
+	}
+	runSSH := func(ctx context.Context, remoteCmd string) (string, error) {
+		t.Fatal("runSSH should not be called when the VM never becomes ready")
+		return "", nil
+	}
+	clock := &fakeClock{t: time.Unix(0, 0)}
+	_, err := runScenario(context.Background(), baseCfg(), "smoke-test", api, runSSH, nil, clock.now, clock.sleep, noopReadPubKey)
+	if err == nil {
+		t.Fatal("runScenario: want error, got nil")
+	}
+	if !strings.Contains(err.Error(), "FAIL: VM not ready within 600s") {
+		t.Errorf("error = %q, want FAIL: VM not ready within 600s ...", err.Error())
+	}
+	if !strings.Contains(err.Error(), "phase=booting") {
+		t.Errorf("error = %q, want it to include phase=booting", err.Error())
+	}
+}
+// --- runScenario: never-reaped timeout ---------------------------------
+func TestRunScenarioNeverReapedTimesOut(t *testing.T) {
+	api := &testAPI{
+		listHostsFunc: func(ctx context.Context) ([]client.Host, error) {
+			return []client.Host{{ID: "host-1"}}, nil
+		},
+		createVMFunc: func(ctx context.Context, req client.CreateVMRequest) (client.CreateVMResponse, error) {
+			return client.CreateVMResponse{ID: "vm-1"}, nil
+		},
+		listVMsFunc: func(ctx context.Context) ([]client.VM, error) {
+			// Always present, even after delete — models a stuck reap.
+			return []client.VM{{ID: "vm-1", Phase: "ready", AssignedIP: "10.0.0.9"}}, nil
+		},
+		deleteVMFunc: func(ctx context.Context, id string) error { return nil },
+	}
+	runSSH := func(ctx context.Context, remoteCmd string) (string, error) {
+		return "Ubuntu 24.04 LTS ubuntu-vm login: ", nil
+	}
+	clock := &fakeClock{t: time.Unix(0, 0)}
+	_, err := runScenario(context.Background(), baseCfg(), "smoke-test", api, runSSH, nil, clock.now, clock.sleep, noopReadPubKey)
+	if err == nil {
+		t.Fatal("runScenario: want error, got nil")
+	}
+	if !strings.Contains(err.Error(), "FAIL: VM not hard-deleted within 120s") {
+		t.Errorf("error = %q, want FAIL: VM not hard-deleted within 120s ...", err.Error())
+	}
+}
+// --- runScenario: no hosts ------------------------------------------------
+func TestRunScenarioNoHosts(t *testing.T) {
+	api := &testAPI{
+		listHostsFunc: func(ctx context.Context) ([]client.Host, error) { return nil, nil },
+		createVMFunc: func(ctx context.Context, req client.CreateVMRequest) (client.CreateVMResponse, error) {
+			t.Fatal("CreateVM should not be called with no hosts")
+			return client.CreateVMResponse{}, nil
+		},
+		listVMsFunc: func(ctx context.Context) ([]client.VM, error) {
+			t.Fatal("ListVMs should not be called with no hosts")
+			return nil, nil
+		},
+		deleteVMFunc: func(ctx context.Context, id string) error {
+			t.Fatal("DeleteVM should not be called with no hosts")
+			return nil
+		},
+	}
+	clock := &fakeClock{t: time.Unix(0, 0)}
+	_, err := runScenario(context.Background(), baseCfg(), "smoke-test", api, nil, nil, clock.now, clock.sleep, noopReadPubKey)
+	if err == nil {
+		t.Fatal("runScenario: want error, got nil")
+	}
+}
+// --- runScenario: gate hooks -----------------------------------------------
+// happyPathAPI returns a testAPI that succeeds all the way through reap,
+// tracking call order in calls (a shared slice each hook also appends to).
+func happyPathAPI(t *testing.T, calls *[]string) *testAPI {
+	t.Helper()
+	listVMsCalls := 0
+	return &testAPI{
+		listHostsFunc: func(ctx context.Context) ([]client.Host, error) {
+			return []client.Host{{ID: "host-1"}}, nil
+		},
+		createVMFunc: func(ctx context.Context, req client.CreateVMRequest) (client.CreateVMResponse, error) {
+			*calls = append(*calls, "createVM")
+			return client.CreateVMResponse{ID: "vm-1"}, nil
+		},
+		listVMsFunc: func(ctx context.Context) ([]client.VM, error) {
+			listVMsCalls++
+			switch {
+			case listVMsCalls < 3:
+				return []client.VM{{ID: "vm-1", Phase: "booting"}}, nil
+			case listVMsCalls == 3:
+				return []client.VM{{ID: "vm-1", Phase: "ready", AssignedIP: "10.0.0.9"}}, nil
+			default:
+				return nil, nil
+			}
+		},
+		deleteVMFunc: func(ctx context.Context, id string) error { return nil },
+	}
+}
+func happyPathRunSSH() sshFunc {
+	sshCalls := 0
+	return func(ctx context.Context, remoteCmd string) (string, error) {
+		sshCalls++
+		if sshCalls < 2 {
+			return "[    0.1] Booting Linux...", nil
+		}
+		return "Ubuntu 24.04 LTS ubuntu-vm login: ", nil
+	}
+}
+func TestRunScenarioGateRegistersBeforeCreateAndExecsAfterBoot(t *testing.T) {
+	var calls []string
+	api := happyPathAPI(t, &calls)
+	gate := &gateHooks{
+		register: func(ctx context.Context) error {
+			calls = append(calls, "register")
+			return nil
+		},
+		exec: func(ctx context.Context, vmName string) error {
+			if vmName != "smoke-test" {
+				t.Errorf("gate.exec vmName = %q, want smoke-test", vmName)
+			}
+			calls = append(calls, "exec")
+			return nil
+		},
+	}
+	clock := &fakeClock{t: time.Unix(0, 0)}
+	msg, err := runScenario(context.Background(), baseCfg(), "smoke-test", api, happyPathRunSSH(), gate, clock.now, clock.sleep, noopReadPubKey)
+	if err != nil {
+		t.Fatalf("runScenario: %v", err)
+	}
+	if !strings.Contains(msg, "gate SSH: ok") {
+		t.Errorf("message = %q, want it to mention gate SSH: ok", msg)
+	}
+	want := []string{"register", "createVM", "exec"}
+	if len(calls) != len(want) {
+		t.Fatalf("call order = %v, want %v", calls, want)
+	}
+	for i, c := range want {
+		if calls[i] != c {
+			t.Errorf("call order = %v, want %v", calls, want)
+			break
+		}
+	}
+}
+func TestRunScenarioGateRegisterErrorAbortsBeforeCreate(t *testing.T) {
+	api := &testAPI{
+		listHostsFunc: func(ctx context.Context) ([]client.Host, error) {
+			t.Fatal("ListHosts should not be called when register fails")
+			return nil, nil
+		},
+		createVMFunc: func(ctx context.Context, req client.CreateVMRequest) (client.CreateVMResponse, error) {
+			t.Fatal("CreateVM should not be called when register fails")
+			return client.CreateVMResponse{}, nil
+		},
+		listVMsFunc: func(ctx context.Context) ([]client.VM, error) {
+			t.Fatal("ListVMs should not be called when register fails")
+			return nil, nil
+		},
+		deleteVMFunc: func(ctx context.Context, id string) error {
+			t.Fatal("DeleteVM should not be called when register fails")
+			return nil
+		},
+	}
+	gate := &gateHooks{
+		register: func(ctx context.Context) error { return errors.New("upload boom") },
+		exec: func(ctx context.Context, vmName string) error {
+			t.Fatal("exec should not be called when register fails")
+			return nil
+		},
+	}
+	clock := &fakeClock{t: time.Unix(0, 0)}
+	_, err := runScenario(context.Background(), baseCfg(), "smoke-test", api, nil, gate, clock.now, clock.sleep, noopReadPubKey)
+	if err == nil {
+		t.Fatal("runScenario: want error, got nil")
+	}
+	if !strings.Contains(err.Error(), "register smoke user CA") {
+		t.Errorf("error = %q, want it to mention register smoke user CA", err.Error())
+	}
+}
+func TestRunScenarioGateExecErrorFails(t *testing.T) {
+	var calls []string
+	api := happyPathAPI(t, &calls)
+	api.deleteVMFunc = func(ctx context.Context, id string) error {
+		t.Fatal("DeleteVM should not be called when gate exec fails")
+		return nil
+	}
+	gate := &gateHooks{
+		register: func(ctx context.Context) error { return nil },
+		exec: func(ctx context.Context, vmName string) error {
+			return errors.New("FAIL: gate SSH boom")
+		},
+	}
+	clock := &fakeClock{t: time.Unix(0, 0)}
+	_, err := runScenario(context.Background(), baseCfg(), "smoke-test", api, happyPathRunSSH(), gate, clock.now, clock.sleep, noopReadPubKey)
+	if err == nil {
+		t.Fatal("runScenario: want error, got nil")
+	}
+	if !strings.Contains(err.Error(), "gate SSH boom") {
+		t.Errorf("error = %q, want it to mention gate SSH boom", err.Error())
+	}
+}
+// --- pollLoop ---------------------------------------------------------
+func TestPollLoopReturnsErrPollTimeoutAtDeadline(t *testing.T) {
+	clock := &fakeClock{t: time.Unix(0, 0)}
+	calls := 0
+	err := pollLoop(context.Background(), clock.now, clock.sleep, 10*time.Second, 5*time.Second, func() (bool, error) {
+		calls++
+		return false, nil
+	})
+	if !errors.Is(err, errPollTimeout) {
+		t.Errorf("err = %v, want errPollTimeout", err)
+	}
+	if calls == 0 {
+		t.Error("attempt was never called")
+	}
+}
+func TestPollLoopPropagatesAttemptError(t *testing.T) {
+	wantErr := errors.New("boom")
+	clock := &fakeClock{t: time.Unix(0, 0)}
+	err := pollLoop(context.Background(), clock.now, clock.sleep, 10*time.Second, 5*time.Second, func() (bool, error) {
+		return false, wantErr
+	})
+	if !errors.Is(err, wantErr) {
+		t.Errorf("err = %v, want %v", err, wantErr)
+	}
+}
+// TestGetVMFiltersById pins that getVM discriminates by ID within a listing
+// that contains other VMs — on the live fleet the list always does (eitri-dev
+// at minimum), so a match-first regression would poll the wrong VM's
+// phase/IP during the ready wait.
+func TestGetVMFiltersById(t *testing.T) {
+	api := &testAPI{listVMsFunc: func(ctx context.Context) ([]client.VM, error) {
+		return []client.VM{
+			{ID: "vm-1", Phase: "creating"},
+			{ID: "vm-2", Phase: "ready", AssignedIP: "10.77.1.9"},
+		}, nil
+	}}
+	vm, present, err := getVM(context.Background(), api, "vm-2")
+	if err != nil || !present {
+		t.Fatalf("getVM(vm-2) = present %v, err %v; want present", present, err)
+	}
+	if vm.ID != "vm-2" || vm.Phase != "ready" || vm.AssignedIP != "10.77.1.9" {
+		t.Errorf("getVM(vm-2) returned the wrong row: %+v", vm)
+	}
+	if _, present, err := getVM(context.Background(), api, "vm-3"); err != nil || present {
+		t.Errorf("getVM(vm-3) against a non-empty list = present %v, err %v; want absent", present, err)
+	}
+}

internal/smoke/userca.go

@@ @@ -0,0 +1,56 @@ @@
+package smoke
+import (
+	"crypto/ed25519"
+	"crypto/rand"
+	"encoding/pem"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"golang.org/x/crypto/ssh"
+)
+// loadOrCreateUserCA loads the smoke's persistent user CA key from path,
+// generating and persisting a fresh ed25519 key on first run so subsequent
+// smoke runs reuse the same CA identity (the gate check registers this key's
+// public half with the tenant once; a fresh key every run would mean an
+// ever-growing set of trusted, never-reused CAs).
+func loadOrCreateUserCA(path string) (ssh.Signer, error) {
+	data, err := os.ReadFile(path)
+	if err == nil {
+		signer, err := ssh.ParsePrivateKey(data)
+		if err != nil {
+			return nil, fmt.Errorf("parse user CA key %q: %w", path, err)
+		}
+		return signer, nil
+	}
+	if !errors.Is(err, fs.ErrNotExist) {
+		return nil, fmt.Errorf("read user CA key %q: %w", path, err)
+	}
+	_, priv, err := ed25519.GenerateKey(rand.Reader)
+	if err != nil {
+		return nil, fmt.Errorf("generate user CA key: %w", err)
+	}
+	block, err := ssh.MarshalPrivateKey(priv, "")
+	if err != nil {
+		return nil, fmt.Errorf("marshal user CA key: %w", err)
+	}
+	pemBytes := pem.EncodeToMemory(block)
+	if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
+		return nil, fmt.Errorf("mkdir user CA key dir: %w", err)
+	}
+	if err := os.WriteFile(path, pemBytes, 0o600); err != nil {
+		return nil, fmt.Errorf("write user CA key %q: %w", path, err)
+	}
+	signer, err := ssh.NewSignerFromSigner(priv)
+	if err != nil {
+		return nil, fmt.Errorf("build user CA signer: %w", err)
+	}
+	return signer, nil
+}

internal/smoke/userca_test.go

@@ @@ -0,0 +1,37 @@ @@
+package smoke
+import (
+	"os"
+	"path/filepath"
+	"testing"
+	"golang.org/x/crypto/ssh"
+)
+func TestLoadOrCreateUserCACreatesThenReuses(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "nested", "user_ca")
+	s1, err := loadOrCreateUserCA(path)
+	if err != nil {
+		t.Fatalf("loadOrCreateUserCA (create): %v", err)
+	}
+	s2, err := loadOrCreateUserCA(path)
+	if err != nil {
+		t.Fatalf("loadOrCreateUserCA (reuse): %v", err)
+	}
+	line1 := string(ssh.MarshalAuthorizedKey(s1.PublicKey()))
+	line2 := string(ssh.MarshalAuthorizedKey(s2.PublicKey()))
+	if line1 != line2 {
+		t.Errorf("public keys differ across calls: %q vs %q", line1, line2)
+	}
+	info, err := os.Stat(path)
+	if err != nil {
+		t.Fatalf("stat key file: %v", err)
+	}
+	if mode := info.Mode().Perm(); mode != 0o600 {
+		t.Errorf("file mode = %o, want 0600", mode)
+	}
+}

scripts/coverage.sh

Old		New
@@ -25,6 +25,7 @@ declare -A FLOOR=(
25	[internal/agent/cloudhv]=40	25	[internal/agent/cloudhv]=40
26	[internal/agent/syncclient]=74	26	[internal/agent/syncclient]=74
27	[internal/server/api]=76	27	[internal/server/api]=76
		28	[internal/server/boot]=17
28	[internal/server/api/client]=89	29	[internal/server/api/client]=89
29	[internal/server/api/spec]=91	30	[internal/server/api/spec]=91
30	[internal/server/store]=76	31	[internal/server/store]=76
@@ -32,15 +33,19 @@ declare -A FLOOR=(
32	[internal/server/release]=90	33	[internal/server/release]=90
33	[internal/agent/selfupdate]=65	34	[internal/agent/selfupdate]=65
34	[internal/agent/bootstrap]=70	35	[internal/agent/bootstrap]=70
		36	[internal/agent/run]=30
35	[internal/server/hosttoken]=95	37	[internal/server/hosttoken]=95
36	[internal/server/hub]=90	38	[internal/server/hub]=90
37	[internal/server/syncsvc]=75	39	[internal/server/syncsvc]=75
38	[internal/server/web]=90	40	[internal/server/web]=90
39	[internal/transport]=77	41	[internal/transport]=77
40	[internal/shape]=88	42	[internal/shape]=88
41	[internal/site]=80	43	[internal/site]=84
42	[internal/cli]=60	44	[internal/smoke]=46
43	[internal/oidcprovider]=78	45	[internal/cli]=64
		46	[internal/mcpserver]=57
		47	[internal/oidcprovider]=79
		48	[internal/server/config]=95
44	)	49	)
45		50
46	profile="$(mktemp)"	51	profile="$(mktemp)"