2ff23e41 — eitri

deploy/server/README.md

Old		New
@@ -343,7 +343,7 @@ The guest is a macvtap child of the dev machine's LAN uplink (`DEVHOST_UPLINK`,
343	default `enp4s0`), so the LAN's router leases it an address and the workstation	343	default `enp4s0`), so the LAN's router leases it an address and the workstation
344	dials it directly. macvtap isolates a guest from its own host by design: **the	344	dials it directly. macvtap isolates a guest from its own host by design: **the
345	dev machine cannot reach the guest it runs.** That costs nothing here, because	345	dev machine cannot reach the guest it runs.** That costs nothing here, because
346	every gate connection — `deploy.sh` over ssh, the smoke's serial-log reads —	346	every gate connection — `deploy.sh` over ssh, the smoke's coverage pull —
347	comes from the workstation anyway. It does mean the address is discovered	347	comes from the workstation anyway. It does mean the address is discovered
348	rather than assigned: `create` waits for the guest agent to report it, then	348	rather than assigned: `create` waits for the guest agent to report it, then
349	prints the `AGENT_HOSTS` and `AGENT_EXTRA_FLAGS` lines for	349	prints the `AGENT_HOSTS` and `AGENT_EXTRA_FLAGS` lines for

docs/assumptions.md

+python-urllib drew Cloudflare error 1010 (browser integrity check) while Go's
+client and any self-naming User-Agent passed. A proxied `/mcp` must exempt
+API user agents — or clients must send their own.
+### The console shows a boot as well as the host's own log did
+The serial console the control plane serves carries the same evidence the
+agent's `serial.log` does — the host replays its backlog to whoever attaches,
+so a boot is readable however late the watcher arrives. Underpins the boot gate
+proving a guest booted through the API rather than by logging into its host,
+which is what makes the gate host-agnostic: nothing it proves needs an ssh
+target, a sudo rule, or a path on any machine in the fleet.
+**Proven** on the branch gate: the same login prompt satisfies the first-boot
+and post-power-cycle proofs, read from the console instead of the file. Console
+history survives a restart, so the reboot proof marks the stream at the start
+command and counts only what arrives after it — the freshness the old proof got
+by truncating the file.

internal/server/api/client/console.go

@@ @@ -0,0 +1,64 @@ @@
+package client
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+	"github.com/coder/websocket"
+	"github.com/a73x/eitri/internal/server/api/types"
+)
+// MintStreamTicket mints a one-time, short-TTL ticket for the SSE stream or a
+// VM's console WebSocket. The ticket carries the minting credential's tenant
+// and is the only thing eitri ever puts in a URL: neither an EventSource nor a
+// WebSocket dial can carry a header, so the PAT authenticates THIS request and
+// the ticket stands in for it on the stream.
+func (c *Client) MintStreamTicket(ctx context.Context) (string, error) {
+	var out types.StreamTicketResponse
+	if err := c.do(ctx, http.MethodPost, "/api/v1/stream-tickets", nil, &out); err != nil {
+		return "", err
+	}
+	if out.Ticket == "" {
+		return "", fmt.Errorf("client: POST /api/v1/stream-tickets: no ticket in the response")
+	}
+	return out.Ticket, nil
+}
+// consoleWSURL renders a VM's console endpoint as the URL to dial: the API's
+// own origin with a WebSocket scheme, and the ticket as its only credential.
+func consoleWSURL(baseURL, vmID, ticket string) string {
+	base := strings.TrimRight(baseURL, "/")
+	if rest, ok := strings.CutPrefix(base, "https://"); ok {
+		base = "wss://" + rest
+	} else if rest, ok := strings.CutPrefix(base, "http://"); ok {
+		base = "ws://" + rest
+	}
+	return base + "/api/v1/vms/" + url.PathEscape(vmID) + "/console/ws?ticket=" + url.QueryEscape(ticket)
+}
+// DialConsole attaches to a VM's serial console and hands back the raw byte
+// pipe: reads are what the guest printed (the host replays its recent backlog
+// on attach, then live output follows), writes are keystrokes to it. It is the
+// browser's path exactly — mint a ticket, dial the WebSocket with it — so a
+// caller watching a guest boot watches it through the same authenticated
+// endpoint an operator does, with nothing on the host to log into.
+//
+// ctx bounds the whole session, not just the dial: cancelling it ends the
+// stream. The caller closes the returned pipe.
+func (c *Client) DialConsole(ctx context.Context, vmID string) (io.ReadWriteCloser, error) {
+	ticket, err := c.MintStreamTicket(ctx)
+	if err != nil {
+		return nil, err
+	}
+	//nolint:bodyclose // the handshake response body is the library's to close
+	conn, _, err := websocket.Dial(ctx, consoleWSURL(c.BaseURL, vmID, ticket), &websocket.DialOptions{HTTPClient: c.HTTP})
+	if err != nil {
+		return nil, fmt.Errorf("client: dial console for vm %s: %w", vmID, err)
+	}
+	return websocket.NetConn(ctx, conn, websocket.MessageBinary), nil
+}

internal/server/api/client/console_test.go

@@ @@ -0,0 +1,160 @@ @@
+package client_test
+import (
+	"context"
+	"errors"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+	"github.com/coder/websocket"
+	"github.com/a73x/eitri/internal/server/api/client"
+)
+func TestMintStreamTicket(t *testing.T) {
+	var cap capture
+	srv := serve(t, &cap, http.StatusCreated, `{"ticket":"tkt-123"}`)
+	c := &client.Client{BaseURL: srv.URL, Token: "tok"}
+	ticket, err := c.MintStreamTicket(context.Background())
+	if err != nil {
+		t.Fatalf("MintStreamTicket: %v", err)
+	}
+	if ticket != "tkt-123" {
+		t.Errorf("ticket = %q, want tkt-123", ticket)
+	}
+	if cap.method != http.MethodPost || cap.path != "/api/v1/stream-tickets" {
+		t.Errorf("request = %s %s, want POST /api/v1/stream-tickets", cap.method, cap.path)
+	}
+	if cap.auth != "Bearer tok" {
+		t.Errorf("auth = %q, want the PAT in the header", cap.auth)
+	}
+}
+// TestMintStreamTicketRejectsAnEmptyTicket: an empty ticket dials a console
+// that answers 401, so the failure belongs at the mint, naming the mint.
+func TestMintStreamTicketRejectsAnEmptyTicket(t *testing.T) {
+	var cap capture
+	srv := serve(t, &cap, http.StatusCreated, `{"ticket":""}`)
+	c := &client.Client{BaseURL: srv.URL, Token: "tok"}
+	if _, err := c.MintStreamTicket(context.Background()); err == nil {
+		t.Fatal("MintStreamTicket: want error for an empty ticket, got nil")
+	} else if !strings.Contains(err.Error(), "stream-tickets") {
+		t.Errorf("error = %q, want it to name the endpoint", err.Error())
+	}
+}
+func TestMintStreamTicketPropagatesTheAPIError(t *testing.T) {
+	var cap capture
+	srv := serve(t, &cap, http.StatusUnauthorized, `unauthorized`)
+	c := &client.Client{BaseURL: srv.URL, Token: "stale"}
+	_, err := c.MintStreamTicket(context.Background())
+	var apiErr *client.Error
+	if !errors.As(err, &apiErr) {
+		t.Fatalf("error = %v, want a *client.Error", err)
+	}
+	if apiErr.Status != http.StatusUnauthorized {
+		t.Errorf("status = %d, want 401", apiErr.Status)
+	}
+}
+// consoleServer stands in for the control plane's console endpoint: it mints a
+// ticket on POST /api/v1/stream-tickets and accepts the WebSocket only when
+// that same ticket comes back in the query, writing text to whoever attaches.
+func consoleServer(t *testing.T, text string) (*httptest.Server, *string) {
+	t.Helper()
+	dialedPath := new(string)
+	mux := http.NewServeMux()
+	mux.HandleFunc("POST /api/v1/stream-tickets", func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusCreated)
+		io.WriteString(w, `{"ticket":"tkt-abc"}`)
+	})
+	mux.HandleFunc("GET /api/v1/vms/{id}/console/ws", func(w http.ResponseWriter, r *http.Request) {
+		*dialedPath = r.URL.EscapedPath() + "?" + r.URL.RawQuery
+		if r.URL.Query().Get("ticket") != "tkt-abc" {
+			http.Error(w, "unauthorized", http.StatusUnauthorized)
+			return
+		}
+		c, err := websocket.Accept(w, r, &websocket.AcceptOptions{InsecureSkipVerify: true})
+		if err != nil {
+			return
+		}
+		defer c.CloseNow()
+		nc := websocket.NetConn(r.Context(), c, websocket.MessageBinary)
+		io.WriteString(nc, text)
+		time.Sleep(50 * time.Millisecond)
+		_ = c.Close(websocket.StatusNormalClosure, "")
+	})
+	srv := httptest.NewServer(mux)
+	t.Cleanup(srv.Close)
+	return srv, dialedPath
+}
+// TestDialConsoleMintsThenDials pins the two-step: the console is reached with
+// a freshly minted ticket in the URL, and the bytes come back raw.
+func TestDialConsoleMintsThenDials(t *testing.T) {
+	srv, dialedPath := consoleServer(t, "ubuntu-vm login: ")
+	c := &client.Client{BaseURL: srv.URL, Token: "tok"}
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	stream, err := c.DialConsole(ctx, "vm-1")
+	if err != nil {
+		t.Fatalf("DialConsole: %v", err)
+	}
+	defer stream.Close()
+	got, err := io.ReadAll(stream)
+	if err != nil && err != io.EOF {
+		t.Fatalf("read console: %v", err)
+	}
+	if !strings.Contains(string(got), "login:") {
+		t.Errorf("console bytes = %q, want the guest's output", string(got))
+	}
+	if *dialedPath != "/api/v1/vms/vm-1/console/ws?ticket=tkt-abc" {
+		t.Errorf("dialed %q, want the VM's console path carrying the minted ticket", *dialedPath)
+	}
+}
+// TestDialConsoleFailsWhenTheTicketIsRefused: a console that will not open is
+// an error at dial, not a silent stream that never says anything.
+func TestDialConsoleFailsWhenTheTicketIsRefused(t *testing.T) {
+	mux := http.NewServeMux()
+	mux.HandleFunc("POST /api/v1/stream-tickets", func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusCreated)
+		io.WriteString(w, `{"ticket":"expired"}`)
+	})
+	mux.HandleFunc("GET /api/v1/vms/{id}/console/ws", func(w http.ResponseWriter, r *http.Request) {
+		http.Error(w, "unauthorized", http.StatusUnauthorized)
+	})
+	srv := httptest.NewServer(mux)
+	defer srv.Close()
+	c := &client.Client{BaseURL: srv.URL, Token: "tok"}
+	if _, err := c.DialConsole(context.Background(), "vm-1"); err == nil {
+		t.Fatal("DialConsole: want error when the console refuses the ticket, got nil")
+	} else if !strings.Contains(err.Error(), "vm-1") {
+		t.Errorf("error = %q, want it to name the VM", err.Error())
+	}
+}
+// TestDialConsoleEscapesTheVMID pins that an id with URL-significant characters
+// addresses one VM rather than reshaping the path.
+func TestDialConsoleEscapesTheVMID(t *testing.T) {
+	srv, dialedPath := consoleServer(t, "x")
+	c := &client.Client{BaseURL: srv.URL, Token: "tok"}
+	stream, err := c.DialConsole(context.Background(), "vm/1")
+	if err == nil {
+		stream.Close()
+	}
+	if !strings.Contains(*dialedPath, "vm%2F1") {
+		t.Errorf("dialed %q, want the id escaped into one path segment", *dialedPath)
+	}
+}

internal/smoke/config.go

Old		New
@@ -20,10 +20,15 @@ type Config struct {
20	// CIPATFile is an operator-minted token for the VM lifecycle. Empty ⇒ the	20	// CIPATFile is an operator-minted token for the VM lifecycle. Empty ⇒ the
21	// PAT minted by the sign-in above is used instead, which is right whenever	21	// PAT minted by the sign-in above is used instead, which is right whenever
22	// the identity that signs in is itself the operator.	22	// the identity that signs in is itself the operator.
23	CIPATFile string	23	CIPATFile string
		24
		25	// AgentUserHost/AgentPort are the ssh target coverage collection pulls the
		26	// agent's raw profile from (AGENT_HOSTS' first entry). Nothing the gate
		27	// PROVES goes near them: the run itself only ever speaks to the control
		28	// plane, so a plane whose hosts the runner cannot log into is still fully
		29	// gated — it just collects no agent-side coverage.
24	AgentUserHost string	30	AgentUserHost string
25	AgentPort int	31	AgentPort int
26	AgentStateDir string
27		32
28	// MCPURLs are the origins the remote-MCP leg exercises (SMOKE_MCP_URL, a	33	// MCPURLs are the origins the remote-MCP leg exercises (SMOKE_MCP_URL, a
29	// space-separated list; a single ServerURL when unset). The FIRST entry gets	34	// space-separated list; a single ServerURL when unset). The FIRST entry gets
@@ -55,7 +60,7 @@ func loadConfig(getenv func(string) string) (Config, error) {
55	ciPasswordFile := getenv("CI_PASSWORD_FILE")	60	ciPasswordFile := getenv("CI_PASSWORD_FILE")
56	ciPATFile := getenv("CI_PAT_FILE")	61	ciPATFile := getenv("CI_PAT_FILE")
57	agentHosts := getenv("AGENT_HOSTS")	62	agentHosts := getenv("AGENT_HOSTS")
58	agentStateDir := getenv("AGENT_STATE_DIR")	63	agentGocoverdir := getenv("AGENT_GOCOVERDIR")
59		64
60	var missing []string	65	var missing []string
61	if serverURL == "" {	66	if serverURL == "" {
@@ -76,11 +81,12 @@ func loadConfig(getenv func(string) string) (Config, error) {
76	if ciPATFile == "" && ciUser == "" {	81	if ciPATFile == "" && ciUser == "" {
77	missing = append(missing, "CI_PAT_FILE (or CI_USER, to mint one by signing in)")	82	missing = append(missing, "CI_PAT_FILE (or CI_USER, to mint one by signing in)")
78	}	83	}
79	if agentHosts == "" {	84	// AGENT_HOSTS is required only by coverage collection, which is the one
80	missing = append(missing, "AGENT_HOSTS")	85	// thing here that reaches past the API onto a host. A run that collects no
81	}	86	// agent coverage needs no ssh target, and asking for one would be asking a
82	if agentStateDir == "" {	87	// hosted plane for a login it has no reason to hand out.
83	missing = append(missing, "AGENT_STATE_DIR")	88	if agentHosts == "" && agentGocoverdir != "" {
		89	missing = append(missing, "AGENT_HOSTS (to pull AGENT_GOCOVERDIR from the host)")
84	}	90	}
85	if len(missing) > 0 {	91	if len(missing) > 0 {
86	return Config{}, fmt.Errorf("missing required env var(s): %s", strings.Join(missing, ", "))	92	return Config{}, fmt.Errorf("missing required env var(s): %s", strings.Join(missing, ", "))
@@ -106,9 +112,8 @@ func loadConfig(getenv func(string) string) (Config, error) {
106	CIPATFile: ciPATFile,	112	CIPATFile: ciPATFile,
107	AgentUserHost: userHost,	113	AgentUserHost: userHost,
108	AgentPort: port,	114	AgentPort: port,
109	AgentStateDir: agentStateDir,
110	ServerGocoverdir: getenv("SERVER_GOCOVERDIR"),	115	ServerGocoverdir: getenv("SERVER_GOCOVERDIR"),
111	AgentGocoverdir: getenv("AGENT_GOCOVERDIR"),	116	AgentGocoverdir: agentGocoverdir,
112	CoverOut: getenv("COVER_OUT"),	117	CoverOut: getenv("COVER_OUT"),
113	SmokeGate: getenv("SMOKE_GATE"),	118	SmokeGate: getenv("SMOKE_GATE"),
114	SmokeVMUser: smokeVMUser,	119	SmokeVMUser: smokeVMUser,
@@ -119,9 +124,14 @@ func loadConfig(getenv func(string) string) (Config, error) {
119	// parseAgentHost takes the AGENT_HOSTS value (space-separated "user@host[:port]"	124	// parseAgentHost takes the AGENT_HOSTS value (space-separated "user@host[:port]"
120	// entries) and returns the first entry's user@host plus its port. The port is	125	// entries) and returns the first entry's user@host plus its port. The port is
121	// the substring after the LAST colon when that substring is entirely digits;	126	// the substring after the LAST colon when that substring is entirely digits;
122	// otherwise there is no port and it defaults to 22.	127	// otherwise there is no port and it defaults to 22. An empty value yields an
		128	// empty target: no host was named, and none is needed.
123	func parseAgentHost(agentHosts string) (userHost string, port int) {	129	func parseAgentHost(agentHosts string) (userHost string, port int) {
124	entry := strings.Fields(agentHosts)[0]	130	entries := strings.Fields(agentHosts)
		131	if len(entries) == 0 {
		132	return "", 0
		133	}
		134	entry := entries[0]
125		135
126	if idx := strings.LastIndex(entry, ":"); idx != -1 {	136	if idx := strings.LastIndex(entry, ":"); idx != -1 {
127	if p, err := strconv.Atoi(entry[idx+1:]); err == nil {	137	if p, err := strconv.Atoi(entry[idx+1:]); err == nil {

internal/smoke/config_test.go

Old		New
@@ -18,7 +18,6 @@ func requiredVals() map[string]string {
18	"CI_PASSWORD_FILE": "/etc/eitri/ci-password",	18	"CI_PASSWORD_FILE": "/etc/eitri/ci-password",
19	"CI_PAT_FILE": "/etc/eitri/ci-pat",	19	"CI_PAT_FILE": "/etc/eitri/ci-pat",
20	"AGENT_HOSTS": "ubuntu@10.0.0.5:2222",	20	"AGENT_HOSTS": "ubuntu@10.0.0.5:2222",
21	"AGENT_STATE_DIR": "/var/lib/eitri-agent",
22	}	21	}
23	}	22	}
24		23
@@ -69,8 +68,6 @@ func TestLoadConfigMissingRequiredVars(t *testing.T) {
69	wantErr string	68	wantErr string
70	}{	69	}{
71	{"missing server url", "SERVER_URL", "SERVER_URL"},	70	{"missing server url", "SERVER_URL", "SERVER_URL"},
72	{"missing agent hosts", "AGENT_HOSTS", "AGENT_HOSTS"},
73	{"missing agent state dir", "AGENT_STATE_DIR", "AGENT_STATE_DIR"},
74	}	71	}
75	for _, tc := range cases {	72	for _, tc := range cases {
76	t.Run(tc.name, func(t *testing.T) {	73	t.Run(tc.name, func(t *testing.T) {
@@ -92,7 +89,7 @@ func TestLoadConfigMissingAllRequiredVars(t *testing.T) {
92	if err == nil {	89	if err == nil {
93	t.Fatal("loadConfig: want error, got nil")	90	t.Fatal("loadConfig: want error, got nil")
94	}	91	}
95	for _, want := range []string{"SERVER_URL", "CI_PAT_FILE", "AGENT_HOSTS", "AGENT_STATE_DIR"} {	92	for _, want := range []string{"SERVER_URL", "CI_PAT_FILE"} {
96	if !strings.Contains(err.Error(), want) {	93	if !strings.Contains(err.Error(), want) {
97	t.Errorf("error = %q, missing %q", err.Error(), want)	94	t.Errorf("error = %q, missing %q", err.Error(), want)
98	}	95	}
@@ -237,6 +234,37 @@ func TestLoadConfigSmokeGatePassThrough(t *testing.T) {
237	}	234	}
238	}	235	}
239		236
		237	// TestLoadConfigNeedsNoHostWithoutCoverage pins what the gate itself needs: a
		238	// server URL and a credential. Nothing it proves goes near a host, so a run
		239	// that collects no agent coverage names no ssh target at all.
		240	func TestLoadConfigNeedsNoHostWithoutCoverage(t *testing.T) {
		241	vals := requiredVals()
		242	delete(vals, "AGENT_HOSTS")
		243	cfg, err := loadConfig(fakeGetenv(vals))
		244	if err != nil {
		245	t.Fatalf("loadConfig: %v", err)
		246	}
		247	if cfg.AgentUserHost != "" {
		248	t.Errorf("AgentUserHost = %q, want empty", cfg.AgentUserHost)
		249	}
		250	}
		251
		252	// TestLoadConfigNeedsAHostToCollectAgentCoverage: pulling the agent's raw
		253	// profile is the one thing here that reaches onto a host, so that is the one
		254	// setting that makes an ssh target required.
		255	func TestLoadConfigNeedsAHostToCollectAgentCoverage(t *testing.T) {
		256	vals := requiredVals()
		257	delete(vals, "AGENT_HOSTS")
		258	vals["AGENT_GOCOVERDIR"] = "/var/lib/eitri-agent/coverage"
		259	_, err := loadConfig(fakeGetenv(vals))
		260	if err == nil {
		261	t.Fatal("loadConfig: want error when coverage has no host to pull from, got nil")
		262	}
		263	if !strings.Contains(err.Error(), "AGENT_HOSTS") {
		264	t.Errorf("error = %q, want it to name AGENT_HOSTS", err.Error())
		265	}
		266	}
		267
240	func TestLoadConfigOptionalCoverageVarsPassThrough(t *testing.T) {	268	func TestLoadConfigOptionalCoverageVarsPassThrough(t *testing.T) {
241	vals := requiredVals()	269	vals := requiredVals()
242	vals["SERVER_GOCOVERDIR"] = "/tmp/server-cover"	270	vals["SERVER_GOCOVERDIR"] = "/tmp/server-cover"

internal/smoke/console.go

@@ @@ -0,0 +1,163 @@ @@
+package smoke
+import (
+	"context"
+	"io"
+	"sync"
+	"time"
+)
+const (
+	// serialTailMax bounds what one tail holds in memory. It matches the
+	// agent's own console ring, so the smoke keeps exactly as much history as
+	// the host is willing to replay — enough for a boot log.
+	serialTailMax = 256 << 10
+	// consoleRetryDelay paces re-attaching after the stream ends. The console
+	// goes away for real mid-run: a host tears a VM's console down when the VM
+	// powers off, so the whole power-cycle window is one long re-attach.
+	consoleRetryDelay = time.Second
+)
+// consoleDialer attaches to a VM's serial console and returns the byte pipe.
+// The live implementation is the API client's ticket-then-dial WebSocket.
+type consoleDialer func(ctx context.Context, vmID string) (io.ReadWriteCloser, error)
+// consoleTail watches one VM's serial console. It keeps the console attached —
+// re-dialing whenever the stream ends, because a guest that is mid-power-cycle
+// has no console to attach to — and accumulates the printable text the guest
+// has produced, which the boot proofs read with text().
+//
+// mark() draws a line under everything collected so far. That is what makes a
+// second boot provable: the console replays a host's recent backlog on attach,
+// so without a line the first boot's login prompt would answer for the second.
+type consoleTail struct {
+	mu   sync.Mutex
+	buf  []byte
+	last error // most recent dial/read failure, for a proof that times out
+	cancel context.CancelFunc
+	done   chan struct{}
+}
+// attachConsole starts watching vmID's console. The tail runs until close().
+func attachConsole(ctx context.Context, dial consoleDialer, vmID string) *consoleTail {
+	return newConsoleTail(ctx, dial, vmID, consoleRetryDelay)
+}
+// newConsoleTail is attachConsole with the re-attach pace injected, so tests
+// drive the drop-and-redial path without waiting out a real backoff.
+func newConsoleTail(ctx context.Context, dial consoleDialer, vmID string, retry time.Duration) *consoleTail {
+	ctx, cancel := context.WithCancel(ctx)
+	t := &consoleTail{cancel: cancel, done: make(chan struct{})}
+	go t.run(ctx, dial, vmID, retry)
+	return t
+}
+// run keeps a stream attached for as long as the tail lives. A dial that fails
+// and a stream that ends are the same thing here — the console is not there
+// right now — and neither is fatal: the proofs' own deadlines decide when a
+// console that never arrives becomes a failure.
+func (t *consoleTail) run(ctx context.Context, dial consoleDialer, vmID string, retry time.Duration) {
+	defer close(t.done)
+	for ctx.Err() == nil {
+		stream, err := dial(ctx, vmID)
+		if err == nil {
+			// Attached: whatever went wrong last time is history, and reporting
+			// it against a console that is answering would be a lie.
+			t.setErr(nil)
+			t.consume(stream)
+			stream.Close()
+		} else {
+			t.setErr(err)
+		}
+		select {
+		case <-ctx.Done():
+			return
+		case <-time.After(retry):
+		}
+	}
+}
+// consume reads the stream to its end, collecting as it goes.
+func (t *consoleTail) consume(r io.Reader) {
+	buf := make([]byte, 4096)
+	for {
+		n, err := r.Read(buf)
+		if n > 0 {
+			t.collect(buf[:n])
+		}
+		if err != nil {
+			t.setErr(err)
+			return
+		}
+	}
+}
+// collect appends b's printable text to the tail, trimming the front past the
+// cap.
+func (t *consoleTail) collect(b []byte) {
+	clean := sanitizeSerial(b)
+	if len(clean) == 0 {
+		return
+	}
+	t.mu.Lock()
+	defer t.mu.Unlock()
+	t.buf = append(t.buf, clean...)
+	if over := len(t.buf) - serialTailMax; over > 0 {
+		t.buf = t.buf[over:]
+	}
+}
+func (t *consoleTail) setErr(err error) {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+	t.last = err
+}
+// text is what the guest has printed since the last mark.
+func (t *consoleTail) text() string {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+	return string(t.buf)
+}
+// mark discards everything collected so far, so only what the guest prints
+// after it can satisfy a later proof.
+func (t *consoleTail) mark() {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+	t.buf = nil
+	t.last = nil
+}
+// why renders the last transport failure as a trailing clause, so a proof that
+// times out on an empty console says whether it was even connected.
+func (t *consoleTail) why() string {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+	if t.last == nil {
+		return ""
+	}
+	return "; last console error: " + t.last.Error()
+}
+// close detaches and waits for the watcher to finish.
+func (t *consoleTail) close() {
+	t.cancel()
+	<-t.done
+}
+// sanitizeSerial keeps only what a serial console's text evidence lives in —
+// tab, newline, carriage return and printable ASCII — dropping the escape
+// sequences and control bytes a guest terminal emits, which would otherwise
+// split a login prompt across bytes no pattern matches.
+func sanitizeSerial(b []byte) []byte {
+	out := make([]byte, 0, len(b))
+	for _, c := range b {
+		if c == '\t' || c == '\n' || c == '\r' || (c >= 0x20 && c <= 0x7e) {
+			out = append(out, c)
+		}
+	}
+	return out
+}

internal/smoke/console_test.go

@@ @@ -0,0 +1,254 @@ @@
+package smoke
+import (
+	"context"
+	"errors"
+	"io"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+)
+// fakeConsole is a scripted serial console. Every attach opens a new stream and
+// replays history, the way a host replays its console backlog to a viewer that
+// attaches late; write puts fresh output on the stream currently attached and
+// returns once the tail has taken it, so a test can order what it sends against
+// what the scenario does next.
+type fakeConsole struct {
+	mu      sync.Mutex
+	history string
+	dials   int
+	err     error // when non-nil, attaching fails instead
+	w       *io.PipeWriter
+	replayed chan int // dial number, once its history has been consumed
+}
+func newFakeConsole(history string) *fakeConsole {
+	return &fakeConsole{history: history, replayed: make(chan int, 16)}
+}
+// dial is the consoleDialer under test.
+func (f *fakeConsole) dial(ctx context.Context, vmID string) (io.ReadWriteCloser, error) {
+	f.mu.Lock()
+	f.dials++
+	n := f.dials
+	history, err := f.history, f.err
+	if err != nil {
+		f.mu.Unlock()
+		return nil, err
+	}
+	pr, pw := io.Pipe()
+	f.w = pw
+	f.mu.Unlock()
+	go func() {
+		if history != "" {
+			if _, err := pw.Write([]byte(history)); err != nil {
+				return
+			}
+			// A pipe write returns once the reader has taken the bytes, which is
+			// a moment before it has collected them. This second write returns
+			// only after the reader comes back for more — which it does only
+			// after collecting the first — so the signal below means "collected",
+			// and a test that waits on it can safely mark the tail.
+			if _, err := pw.Write([]byte("\n")); err != nil {
+				return
+			}
+		}
+		f.replayed <- n
+		<-ctx.Done()
+		pw.CloseWithError(io.EOF)
+	}()
+	return pipeStream{r: pr, w: pw}, nil
+}
+// write puts text on the attached stream, blocking until it is consumed.
+func (f *fakeConsole) write(t *testing.T, text string) {
+	t.Helper()
+	f.mu.Lock()
+	w := f.w
+	f.mu.Unlock()
+	if w == nil {
+		t.Fatal("fakeConsole: nothing is attached")
+	}
+	if _, err := w.Write([]byte(text)); err != nil {
+		t.Fatalf("fakeConsole write: %v", err)
+	}
+}
+// waitReplayed blocks until attach number n has had its history consumed.
+func (f *fakeConsole) waitReplayed(t *testing.T, n int) {
+	t.Helper()
+	for {
+		select {
+		case got := <-f.replayed:
+			if got >= n {
+				return
+			}
+		case <-time.After(5 * time.Second):
+			t.Fatalf("fakeConsole: attach %d never replayed its history", n)
+		}
+	}
+}
+func (f *fakeConsole) attachCount() int {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return f.dials
+}
+func (f *fakeConsole) failWith(err error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.err = err
+}
+// pipeStream is one attached console: reads deliver the guest's output, writes
+// (keystrokes) go nowhere.
+type pipeStream struct {
+	r *io.PipeReader
+	w *io.PipeWriter
+}
+func (p pipeStream) Read(b []byte) (int, error)  { return p.r.Read(b) }
+func (p pipeStream) Write(b []byte) (int, error) { return len(b), nil }
+func (p pipeStream) Close() error {
+	p.r.Close()
+	p.w.Close()
+	return nil
+}
+// waitForText blocks until the tail holds want, or fails the test.
+func waitForText(t *testing.T, tail *consoleTail, want string) {
+	t.Helper()
+	deadline := time.Now().Add(5 * time.Second)
+	for time.Now().Before(deadline) {
+		if strings.Contains(tail.text(), want) {
+			return
+		}
+		time.Sleep(time.Millisecond)
+	}
+	t.Fatalf("console tail = %q, want it to contain %q", tail.text(), want)
+}
+func TestConsoleTailCollectsWhatTheGuestPrints(t *testing.T) {
+	console := newFakeConsole("Ubuntu 24.04 LTS ubuntu-vm login: ")
+	tail := newConsoleTail(context.Background(), console.dial, "vm-1", time.Millisecond)
+	defer tail.close()
+	waitForText(t, tail, "login:")
+	if booted, _ := classifySerial(tail.text()); !booted {
+		t.Errorf("console text %q does not read as a booted guest", tail.text())
+	}
+}
+// TestConsoleTailKeepsOnlyPrintableText pins the sanitizing: a raw console
+// carries escape sequences and control bytes, and a login prompt split by them
+// must still read as one.
+func TestConsoleTailKeepsOnlyPrintableText(t *testing.T) {
+	console := newFakeConsole("\x1b[0;32m\x00ubuntu-vm\x07 login: \x1b[0m")
+	tail := newConsoleTail(context.Background(), console.dial, "vm-1", time.Millisecond)
+	defer tail.close()
+	waitForText(t, tail, "ubuntu-vm login: ")
+	if strings.ContainsAny(tail.text(), "\x00\x07\x1b") {
+		t.Errorf("console text %q still carries control bytes", tail.text())
+	}
+}
+// TestConsoleTailMarkDropsWhatCameBefore is the freshness rule the reboot proof
+// rests on: a console replays history, so everything collected before the mark
+// must be unable to answer for what happens after it.
+func TestConsoleTailMarkDropsWhatCameBefore(t *testing.T) {
+	console := newFakeConsole("first boot: ubuntu-vm login: ")
+	tail := newConsoleTail(context.Background(), console.dial, "vm-1", time.Millisecond)
+	defer tail.close()
+	waitForText(t, tail, "first boot")
+	tail.mark()
+	if booted, _ := classifySerial(tail.text()); booted {
+		t.Errorf("after the mark the tail still reads as booted: %q", tail.text())
+	}
+	console.write(t, "[    0.9] Booting Linux\nubuntu-vm login: ")
+	waitForText(t, tail, "Booting Linux")
+	if booted, _ := classifySerial(tail.text()); !booted {
+		t.Errorf("post-mark output %q should read as a booted guest", tail.text())
+	}
+}
+// TestConsoleTailReattachesAfterTheStreamDrops: the console goes away for real
+// mid-run — a host tears it down while the VM is off — so the tail must come
+// back on its own rather than leaving the proof watching nothing.
+func TestConsoleTailReattachesAfterTheStreamDrops(t *testing.T) {
+	console := newFakeConsole("ubuntu-vm login: ")
+	ctx, cancel := context.WithCancel(context.Background())
+	tail := newConsoleTail(ctx, console.dial, "vm-1", time.Millisecond)
+	defer tail.close()
+	console.waitReplayed(t, 1)
+	// Drop the stream under the tail: the writer's close ends the read.
+	console.mu.Lock()
+	w := console.w
+	console.mu.Unlock()
+	w.CloseWithError(io.ErrUnexpectedEOF)
+	console.waitReplayed(t, 2)
+	if console.attachCount() < 2 {
+		t.Errorf("attaches = %d, want the tail to re-attach after the drop", console.attachCount())
+	}
+	waitForText(t, tail, "login:")
+	cancel()
+}
+// TestConsoleTailKeepsTryingWhenTheConsoleRefuses: a console that is not there
+// yet is not a failure — the proof's deadline decides that — but the reason is
+// kept, so a proof that times out can say it was never connected.
+func TestConsoleTailKeepsTryingWhenTheConsoleRefuses(t *testing.T) {
+	console := newFakeConsole("ubuntu-vm login: ")
+	console.failWith(errors.New("console unavailable: host offline"))
+	tail := newConsoleTail(context.Background(), console.dial, "vm-1", time.Millisecond)
+	defer tail.close()
+	deadline := time.Now().Add(5 * time.Second)
+	for console.attachCount() < 3 && time.Now().Before(deadline) {
+		time.Sleep(time.Millisecond)
+	}
+	if console.attachCount() < 3 {
+		t.Fatalf("attaches = %d, want the tail to keep retrying a refused console", console.attachCount())
+	}
+	if why := tail.why(); !strings.Contains(why, "host offline") {
+		t.Errorf("why = %q, want the refusal to be reportable", why)
+	}
+	console.failWith(nil)
+	waitForText(t, tail, "login:")
+	if tail.why() != "" {
+		t.Errorf("why = %q, want no error once the console answers", tail.why())
+	}
+}
+func TestConsoleTailCloseStopsWatching(t *testing.T) {
+	console := newFakeConsole("ubuntu-vm login: ")
+	tail := newConsoleTail(context.Background(), console.dial, "vm-1", time.Millisecond)
+	console.waitReplayed(t, 1)
+	tail.close()
+	attaches := console.attachCount()
+	time.Sleep(20 * time.Millisecond)
+	if console.attachCount() != attaches {
+		t.Errorf("attaches went %d -> %d after close; the watcher is still running", attaches, console.attachCount())
+	}
+}
+// TestSanitizeSerialKeepsTheTextAndDropsTheRest pins the same character set the
+// boot gate has always classified on: tab, newline, carriage return and
+// printable ASCII survive; NULs, escapes and high bytes do not.
+func TestSanitizeSerialKeepsTheTextAndDropsTheRest(t *testing.T) {
+	const want = "a\tb\r\nc[1md"
+	if got := string(sanitizeSerial([]byte("a\tb\r\nc\x00\x1b[1md\x80"))); got != want {
+		t.Errorf("sanitizeSerial = %q, want %q", got, want)
+	}
+}

internal/smoke/exposure.go

Old		New
@@ -21,14 +21,10 @@ const sshBannerPrefix = "SSH-2.0"
21	// bannerFunc reads the first bytes a TCP peer sends after accepting.	21	// bannerFunc reads the first bytes a TCP peer sends after accepting.
22	type bannerFunc func(ctx context.Context, addr string) (string, error)	22	type bannerFunc func(ctx context.Context, addr string) (string, error)
23		23
24	// proveExposure publishes the smoke VM's ssh port on its host, dials the host	24	// proveExposure publishes the smoke VM's ssh port on its host, dials the
25	// address raw, and expects an SSH banner. hostAddr is the address the host	25	// address the grant names, and expects an SSH banner. The exposure is revoked
26	// itself reported; the exposure is revoked before the leg returns, whatever	26	// before the leg returns, whatever the outcome.
27	// the outcome.	27	func proveExposure(ctx context.Context, c vmAPI, vmID string, now func() time.Time, sleep func(time.Duration), dial bannerFunc) error {
28	func proveExposure(ctx context.Context, c vmAPI, hostAddr, vmID string, now func() time.Time, sleep func(time.Duration), dial bannerFunc) error {
29	if hostAddr == "" {
30	return errors.New("FAIL: the VM's host reported no uplink address; nothing to dial a published port on")
31	}
32	exp, err := c.CreateExposure(ctx, vmID, 22, 0)	28	exp, err := c.CreateExposure(ctx, vmID, 22, 0)
33	if err != nil {	29	if err != nil {
34	return fmt.Errorf("create exposure: %w", err)	30	return fmt.Errorf("create exposure: %w", err)
@@ -43,7 +39,10 @@ func proveExposure(ctx context.Context, c vmAPI, hostAddr, vmID string, now func
43	}	39	}
44	}()	40	}()
45		41
46	target := net.JoinHostPort(hostAddr, strconv.FormatInt(exp.HostPort, 10))	42	if exp.HostAddr == "" {
		43	return errors.New("FAIL: the exposure names no host address; nothing to dial a published port on")
		44	}
		45	target := net.JoinHostPort(exp.HostAddr, strconv.FormatInt(exp.HostPort, 10))
47	var lastErr error	46	var lastErr error
48	err = pollLoop(ctx, now, sleep, 60time.Second, 3time.Second, func() (bool, error) {	47	err = pollLoop(ctx, now, sleep, 60time.Second, 3time.Second, func() (bool, error) {
49	banner, derr := dial(ctx, target)	48	banner, derr := dial(ctx, target)

internal/smoke/exposure_test.go

Old		New
@@ -11,6 +11,10 @@ import (
11	"github.com/a73x/eitri/internal/server/api/client"	11	"github.com/a73x/eitri/internal/server/api/client"
12	)	12	)
13		13
		14	// TestProveExposureReadsTheBannerBack also pins where the dial target comes
		15	// from: the grant names its own address and port, so the leg dials what the
		16	// fleet published rather than an address assembled from anything else (the
		17	// fake's host listing is deliberately absent — asking it would panic).
14	func TestProveExposureReadsTheBannerBack(t *testing.T) {	18	func TestProveExposureReadsTheBannerBack(t *testing.T) {
15	clock := &fakeClock{}	19	clock := &fakeClock{}
16	deleted := ""	20	deleted := ""
@@ -25,7 +29,7 @@ func TestProveExposureReadsTheBannerBack(t *testing.T) {
25	if host != 0 {	29	if host != 0 {
26	t.Errorf("CreateExposure hostPort = %d, want 0 — the fleet allocates the host port", host)	30	t.Errorf("CreateExposure hostPort = %d, want 0 — the fleet allocates the host port", host)
27	}	31	}
28	return client.Exposure{ID: "x-1", HostPort: 30080}, nil	32	return client.Exposure{ID: "x-1", HostPort: 30080, HostAddr: "192.168.0.190"}, nil
29	},	33	},
30	deleteExposureFunc: func(ctx context.Context, id string) error { deleted = id; return nil },	34	deleteExposureFunc: func(ctx context.Context, id string) error { deleted = id; return nil },
31	}	35	}
@@ -35,7 +39,7 @@ func TestProveExposureReadsTheBannerBack(t *testing.T) {
35	return "SSH-2.0-OpenSSH_9.6\r\n", nil	39	return "SSH-2.0-OpenSSH_9.6\r\n", nil
36	}	40	}
37		41
38	if err := proveExposure(context.Background(), api, "192.168.0.190", "vm-1", clock.now, clock.sleep, dial); err != nil {	42	if err := proveExposure(context.Background(), api, "vm-1", clock.now, clock.sleep, dial); err != nil {
39	t.Fatalf("proveExposure: %v", err)	43	t.Fatalf("proveExposure: %v", err)
40	}	44	}
41	if dialed != "192.168.0.190:30080" {	45	if dialed != "192.168.0.190:30080" {
@@ -53,7 +57,7 @@ func TestProveExposureDialsAnIPv6Uplink(t *testing.T) {
53	clock := &fakeClock{}	57	clock := &fakeClock{}
54	api := &testAPI{	58	api := &testAPI{
55	createExposureFunc: func(context.Context, string, int64, int64) (client.Exposure, error) {	59	createExposureFunc: func(context.Context, string, int64, int64) (client.Exposure, error) {
56	return client.Exposure{ID: "x-1", HostPort: 30080}, nil	60	return client.Exposure{ID: "x-1", HostPort: 30080, HostAddr: "2001:db8::1"}, nil
57	},	61	},
58	deleteExposureFunc: func(context.Context, string) error { return nil },	62	deleteExposureFunc: func(context.Context, string) error { return nil },
59	}	63	}
@@ -63,7 +67,7 @@ func TestProveExposureDialsAnIPv6Uplink(t *testing.T) {
63	return "SSH-2.0-OpenSSH_9.6\r\n", nil	67	return "SSH-2.0-OpenSSH_9.6\r\n", nil
64	}	68	}
65		69
66	if err := proveExposure(context.Background(), api, "2001:db8::1", "vm-1", clock.now, clock.sleep, dial); err != nil {	70	if err := proveExposure(context.Background(), api, "vm-1", clock.now, clock.sleep, dial); err != nil {
67	t.Fatalf("proveExposure: %v", err)	71	t.Fatalf("proveExposure: %v", err)
68	}	72	}
69	if dialed != "[2001:db8::1]:30080" {	73	if dialed != "[2001:db8::1]:30080" {
@@ -75,7 +79,7 @@ func TestProveExposureRetriesUntilTheListenerConverges(t *testing.T) {
75	clock := &fakeClock{}	79	clock := &fakeClock{}
76	api := &testAPI{	80	api := &testAPI{
77	createExposureFunc: func(context.Context, string, int64, int64) (client.Exposure, error) {	81	createExposureFunc: func(context.Context, string, int64, int64) (client.Exposure, error) {
78	return client.Exposure{ID: "x-1", HostPort: 30080}, nil	82	return client.Exposure{ID: "x-1", HostPort: 30080, HostAddr: "192.168.0.190"}, nil
79	},	83	},
80	deleteExposureFunc: func(context.Context, string) error { return nil },	84	deleteExposureFunc: func(context.Context, string) error { return nil },
81	}	85	}
@@ -88,7 +92,7 @@ func TestProveExposureRetriesUntilTheListenerConverges(t *testing.T) {
88	return "SSH-2.0-OpenSSH_9.6\r\n", nil	92	return "SSH-2.0-OpenSSH_9.6\r\n", nil
89	}	93	}
90		94
91	if err := proveExposure(context.Background(), api, "192.168.0.190", "vm-1", clock.now, clock.sleep, dial); err != nil {	95	if err := proveExposure(context.Background(), api, "vm-1", clock.now, clock.sleep, dial); err != nil {
92	t.Fatalf("proveExposure: %v", err)	96	t.Fatalf("proveExposure: %v", err)
93	}	97	}
94	if calls != 3 {	98	if calls != 3 {
@@ -100,13 +104,13 @@ func TestProveExposureFailsOnTheWrongBanner(t *testing.T) {
100	clock := &fakeClock{}	104	clock := &fakeClock{}
101	api := &testAPI{	105	api := &testAPI{
102	createExposureFunc: func(context.Context, string, int64, int64) (client.Exposure, error) {	106	createExposureFunc: func(context.Context, string, int64, int64) (client.Exposure, error) {
103	return client.Exposure{ID: "x-1", HostPort: 30080}, nil	107	return client.Exposure{ID: "x-1", HostPort: 30080, HostAddr: "192.168.0.190"}, nil
104	},	108	},
105	deleteExposureFunc: func(context.Context, string) error { return nil },	109	deleteExposureFunc: func(context.Context, string) error { return nil },
106	}	110	}
107	dial := func(ctx context.Context, addr string) (string, error) { return "HTTP/1.1 400\r\n", nil }	111	dial := func(ctx context.Context, addr string) (string, error) { return "HTTP/1.1 400\r\n", nil }
108		112
109	err := proveExposure(context.Background(), api, "192.168.0.190", "vm-1", clock.now, clock.sleep, dial)	113	err := proveExposure(context.Background(), api, "vm-1", clock.now, clock.sleep, dial)
110	if err == nil {	114	if err == nil {
111	t.Fatal("proveExposure: want error for a non-sshd answer, got nil")	115	t.Fatal("proveExposure: want error for a non-sshd answer, got nil")
112	}	116	}
@@ -115,15 +119,27 @@ func TestProveExposureFailsOnTheWrongBanner(t *testing.T) {
115	}	119	}
116	}	120	}
117		121
118	func TestProveExposureFailsWhenTheHostNamesNoAddress(t *testing.T) {	122	// TestProveExposureFailsWhenTheGrantNamesNoAddress: an exposure whose host has
		123	// not said where it answers is published nowhere, and the leg says so instead
		124	// of dialing a port on an empty host.
		125	func TestProveExposureFailsWhenTheGrantNamesNoAddress(t *testing.T) {
119	clock := &fakeClock{}	126	clock := &fakeClock{}
120	api := &testAPI{}	127	revoked := ""
121	err := proveExposure(context.Background(), api, "", "vm-1", clock.now, clock.sleep, nil)	128	api := &testAPI{
		129	createExposureFunc: func(context.Context, string, int64, int64) (client.Exposure, error) {
		130	return client.Exposure{ID: "x-1", HostPort: 30080}, nil
		131	},
		132	deleteExposureFunc: func(_ context.Context, id string) error { revoked = id; return nil },
		133	}
		134	err := proveExposure(context.Background(), api, "vm-1", clock.now, clock.sleep, nil)
122	if err == nil {	135	if err == nil {
123	t.Fatal("proveExposure: want error when the host reported no address, got nil")	136	t.Fatal("proveExposure: want error when the exposure named no address, got nil")
		137	}
		138	if !strings.Contains(err.Error(), "host address") {
		139	t.Errorf("error = %q, want it to mention the missing host address", err.Error())
124	}	140	}
125	if !strings.Contains(err.Error(), "uplink") {	141	if revoked != "x-1" {
126	t.Errorf("error = %q, want it to mention the uplink address", err.Error())	142	t.Errorf("revoked exposure = %q, want the grant revoked even when it named nowhere to dial", revoked)
127	}	143	}
128	}	144	}
129		145

internal/smoke/run.go

Old		New
@@ -1,8 +1,10 @@
1	// Package smoke is the deploy boot-gate harness. It drives the live eitri	1	// Package smoke is the deploy boot-gate harness. It drives the live eitri
2	// fleet through a credential-chain proof and a create -> boot-proof -> gate-SSH	2	// fleet through a credential-chain proof and a create -> boot-proof -> gate-SSH
3	// -> reap of one throwaway VM, returning an error on any failure. It is a	3	// -> reap of one throwaway VM, returning an error on any failure. Everything it
4	// client of the eitri API plus SSH; it embeds no control-plane or agent code.	4	// proves it proves through eitri's own front door — the API, the serial console,
5	// The eitri-smoke command is thin wiring over Run (arch R14).	5	// the SSH-CA gate, a published port — so it needs no login on any host in the
		6	// fleet and embeds no control-plane or agent code. The eitri-smoke command is
		7	// thin wiring over Run (arch R14).
6	package smoke	8	package smoke
7		9
8	import (	10	import (
@@ -12,9 +14,7 @@ import (
12	"fmt"	14	"fmt"
13	"net/http"	15	"net/http"
14	"os"	16	"os"
15	"os/exec"
16	"path/filepath"	17	"path/filepath"
17	"strconv"
18	"strings"	18	"strings"
19	"time"	19	"time"
20		20
@@ -171,7 +171,7 @@ func Run() error {
171	}, time.Now, time.Sleep, readBanner)	171	}, time.Now, time.Sleep, readBanner)
172	}	172	}
173		173
174	msg, err := runScenario(ctx, cfg, vmName, api, realRunSSH(cfg.AgentUserHost, cfg.AgentPort), gate, mcp, time.Now, time.Sleep, realReadPubKey, readBanner)	174	msg, err := runScenario(ctx, vmName, api, api.DialConsole, gate, mcp, time.Now, time.Sleep, realReadPubKey, readBanner)
175	if err != nil {	175	if err != nil {
176	return err	176	return err
177	}	177	}
@@ -198,21 +198,6 @@ func randVMName() (string, error) {
198	return "smoke-" + hex.EncodeToString(b[:]), nil	198	return "smoke-" + hex.EncodeToString(b[:]), nil
199	}	199	}
200		200
201	// realRunSSH returns the real sshFunc: it shells out to the ssh binary
202	// against userHost:port with the deploy boot-gate's ssh flags, running
203	// remoteCmd non-interactively and returning its stdout.
204	func realRunSSH(userHost string, port int) sshFunc {
205	return func(ctx context.Context, remoteCmd string) (string, error) {
206	cmd := exec.CommandContext(ctx, "ssh",
207	"-p", strconv.Itoa(port),
208	"-o", "BatchMode=yes",
209	"-o", "ConnectTimeout=10",
210	userHost, remoteCmd)
211	out, err := cmd.Output()
212	return string(out), err
213	}
214	}
215
216	// realReadPubKey reads the local operator's SSH public key, trying	201	// realReadPubKey reads the local operator's SSH public key, trying
217	// ~/.ssh/id_ed25519.pub then ~/.ssh/id_rsa.pub. It returns "" (not an error)	202	// ~/.ssh/id_ed25519.pub then ~/.ssh/id_rsa.pub. It returns "" (not an error)
218	// if neither is present — the scenario tolerates a keyless VM.	203	// if neither is present — the scenario tolerates a keyless VM.

internal/smoke/scenario.go

Old		New
@@ -26,40 +26,21 @@ func classifySerial(text string) (booted, panicked bool) {
26	return bootedPattern.MatchString(text), panickedPattern.MatchString(text)	26	return bootedPattern.MatchString(text), panickedPattern.MatchString(text)
27	}	27	}
28		28
29	// bootProofCommand builds the remote shell command run over SSH on the agent	29	// proveBoot watches a VM's serial console until it shows a userspace boot
30	// host to read and sanitize a VM's serial console log.	30	// (login prompt), a panic/root-mount failure (immediate FAIL), or the deadline
31	func bootProofCommand(agentStateDir, vmID string) string {	31	// passes. phase names which boot is being proven in the failure messages
32	return fmt.Sprintf(`sudo cat '%s/vms/%s/serial.log' 2>/dev/null \| tr -cd '\11\12\15\40-\176'`, agentStateDir, vmID)	32	// ("first boot", "after power cycle"). A console that is not there yet is just
33	}	33	// an empty iteration — the deadline is what decides.
34		34	func proveBoot(ctx context.Context, tail *consoleTail, now func() time.Time, sleep func(time.Duration), phase string) error {
35	// truncateSerialCommand empties a VM's serial log on the agent host. The log
36	// deliberately survives VM restarts, so a post-restart boot proof would pass
37	// on the FIRST boot's bytes unless it starts from an empty file.
38	func truncateSerialCommand(agentStateDir, vmID string) string {
39	return fmt.Sprintf(`sudo truncate -s 0 '%s/vms/%s/serial.log'`, agentStateDir, vmID)
40	}
41
42	// proveBoot polls the VM's serial log over SSH until it shows a userspace
43	// boot (login prompt), a panic/root-mount failure (immediate FAIL), or the
44	// deadline passes. phase names which boot is being proven in the failure
45	// messages ("first boot", "after power cycle").
46	func proveBoot(ctx context.Context, cfg Config, vmID string, runSSH sshFunc, now func() time.Time, sleep func(time.Duration), phase string) error {
47	remoteCmd := bootProofCommand(cfg.AgentStateDir, vmID)
48	err := pollLoop(ctx, now, sleep, 180time.Second, 6time.Second, func() (bool, error) {	35	err := pollLoop(ctx, now, sleep, 180time.Second, 6time.Second, func() (bool, error) {
49	serial, sshErr := runSSH(ctx, remoteCmd)	36	booted, panicked := classifySerial(tail.text())
50	if sshErr != nil {
51	// Match the bash scenario's "\|\| true": an SSH hiccup mid-boot is
52	// not fatal on its own, just an empty-serial iteration.
53	serial = ""
54	}
55	booted, panicked := classifySerial(serial)
56	if panicked {	37	if panicked {
57	return false, fmt.Errorf("FAIL: guest panic / root-mount failure in serial log (%s)", phase)	38	return false, fmt.Errorf("FAIL: guest panic / root-mount failure on the console (%s)", phase)
58	}	39	}
59	return booted, nil	40	return booted, nil
60	})	41	})
61	if errors.Is(err, errPollTimeout) {	42	if errors.Is(err, errPollTimeout) {
62	return fmt.Errorf("FAIL: no userspace boot evidence in serial within 180s (%s)", phase)	43	return fmt.Errorf("FAIL: no userspace boot evidence on the console within 180s (%s)%s", phase, tail.why())
63	}	44	}
64	return err	45	return err
65	}	46	}
@@ -92,24 +73,6 @@ func getVM(ctx context.Context, c vmAPI, id string) (client.VM, bool, error) {
92	return client.VM{}, false, nil	73	return client.VM{}, false, nil
93	}	74	}
94		75
95	// uplinkFor re-reads the host's own reported address — the one an operator
96	// pastes to reach a port published on it.
97	func uplinkFor(ctx context.Context, c vmAPI, hostID string) (string, error) {
98	hosts, err := c.ListHosts(ctx)
99	if err != nil {
100	return "", fmt.Errorf("list hosts for uplink address: %w", err)
101	}
102	for _, h := range hosts {
103	if h.ID == hostID {
104	return h.UplinkAddr, nil
105	}
106	}
107	return "", fmt.Errorf("host %s vanished from the fleet mid-run", hostID)
108	}
109
110	// sshFunc runs remoteCmd on the agent host over SSH and returns its stdout.
111	type sshFunc func(ctx context.Context, remoteCmd string) (string, error)
112
113	// gateHooks bundles the optional SSH-CA gate steps. nil means "skip the gate".	76	// gateHooks bundles the optional SSH-CA gate steps. nil means "skip the gate".
114	type gateHooks struct {	77	type gateHooks struct {
115	register func(ctx context.Context) error // upload the smoke user CA to the tenant (before create)	78	register func(ctx context.Context) error // upload the smoke user CA to the tenant (before create)
@@ -145,9 +108,9 @@ func pollLoop(ctx context.Context, now func() time.Time, sleep func(time.Duratio
145	}	108	}
146		109
147	// runScenario drives the full register -> create -> ready -> boot-proof ->	110	// runScenario drives the full register -> create -> ready -> boot-proof ->
148	// gate-exec -> published-port -> reap sequence against c (the API), runSSH	111	// gate-exec -> published-port -> reap sequence against c (the API), dialConsole
149	// (the boot-proof transport), readPubKey (the local SSH key source), and	112	// (the serial console the boot proofs watch), readPubKey (the local SSH key
150	// dialBanner (the published-port transport). vmName is the	113	// source), and dialBanner (the published-port transport). vmName is the
151	// pre-generated name for the throwaway VM. gate, when non-nil, registers the	114	// pre-generated name for the throwaway VM. gate, when non-nil, registers the
152	// smoke's user CA with the tenant before create (the guest bakes its trusted	115	// smoke's user CA with the tenant before create (the guest bakes its trusted
153	// CAs at boot, so registration MUST happen first) and proves gate SSH access	116	// CAs at boot, so registration MUST happen first) and proves gate SSH access
@@ -158,7 +121,7 @@ func pollLoop(ctx context.Context, now func() time.Time, sleep func(time.Duratio
158	// now/sleep are the injected clock so the poll deadlines are unit-testable	121	// now/sleep are the injected clock so the poll deadlines are unit-testable
159	// without real waiting. On success it returns the human-readable COMPLETE	122	// without real waiting. On success it returns the human-readable COMPLETE
160	// line; on any failure it returns a descriptive error.	123	// line; on any failure it returns a descriptive error.
161	func runScenario(ctx context.Context, cfg Config, vmName string, c vmAPI, runSSH sshFunc, gate *gateHooks, mcp mcpLeg, now func() time.Time, sleep func(time.Duration), readPubKey func() string, dialBanner bannerFunc) (string, error) {	124	func runScenario(ctx context.Context, vmName string, c vmAPI, dialConsole consoleDialer, gate *gateHooks, mcp mcpLeg, now func() time.Time, sleep func(time.Duration), readPubKey func() string, dialBanner bannerFunc) (string, error) {
162	if gate != nil {	125	if gate != nil {
163	if err := gate.register(ctx); err != nil {	126	if err := gate.register(ctx); err != nil {
164	return "", fmt.Errorf("register smoke user CA: %w", err)	127	return "", fmt.Errorf("register smoke user CA: %w", err)
@@ -200,8 +163,15 @@ func runScenario(ctx context.Context, cfg Config, vmName string, c vmAPI, runSSH
200	}	163	}
201	coldStart := now().Sub(start)	164	coldStart := now().Sub(start)
202		165
203	if err := proveBoot(ctx, cfg, vmID, runSSH, now, sleep, "first boot"); err != nil {	166	// The boot proof watches the guest's own serial console through the control
204	return "", err	167	// plane, exactly as an operator watching the VM come up in a browser does:
		168	// the host replays its console backlog on attach, so the whole boot is
		169	// there to read however late the attach lands.
		170	boot := attachConsole(ctx, dialConsole, vmID)
		171	bootErr := proveBoot(ctx, boot, now, sleep, "first boot")
		172	boot.close()
		173	if bootErr != nil {
		174	return "", bootErr
205	}	175	}
206		176
207	gateOK := false	177	gateOK := false
@@ -214,14 +184,10 @@ func runScenario(ctx context.Context, cfg Config, vmName string, c vmAPI, runSSH
214	}	184	}
215		185
216	// Publish the guest's own sshd on its host and read the banner back	186	// Publish the guest's own sshd on its host and read the banner back
217	// through the listener. The host's address comes from the host itself,	187	// through the listener. The address comes from the exposure record itself —
218	// re-read here rather than reused from the placement lookup, so the leg	188	// the grant names where it is published, so the leg dials what the fleet
219	// dials whatever the host is answering on right now.	189	// told it to dial rather than an address assembled on the side.
220	hostAddr, err := uplinkFor(ctx, c, hostID)	190	if err := proveExposure(ctx, c, vmID, now, sleep, dialBanner); err != nil {
221	if err != nil {
222	return "", err
223	}
224	if err := proveExposure(ctx, c, hostAddr, vmID, now, sleep, dialBanner); err != nil {
225	return "", err	191	return "", err
226	}	192	}
227	exposureOK = true	193	exposureOK = true
@@ -241,6 +207,15 @@ func runScenario(ctx context.Context, cfg Config, vmName string, c vmAPI, runSSH
241	// on-disk GPT survived growpart. The sector-0 regression hid exactly	207	// on-disk GPT survived growpart. The sector-0 regression hid exactly
242	// here — every first boot green, every persistent VM lost at its first	208	// here — every first boot green, every persistent VM lost at its first
243	// reboot.	209	// reboot.
		210	//
		211	// The console is attached BEFORE the cycle starts and held across it, so
		212	// the second boot is watched from a stream that was already open when the
		213	// guest went down. The mark below is what keeps the proof honest: console
		214	// history survives a restart, so only bytes that arrive after the start
		215	// command can answer for the boot that command triggers.
		216	reboot := attachConsole(ctx, dialConsole, vmID)
		217	defer reboot.close()
		218
244	if err := c.PatchVM(ctx, vmID, "stopped"); err != nil {	219	if err := c.PatchVM(ctx, vmID, "stopped"); err != nil {
245	return "", fmt.Errorf("patch vm stopped: %w", err)	220	return "", fmt.Errorf("patch vm stopped: %w", err)
246	}	221	}
@@ -257,9 +232,7 @@ func runScenario(ctx context.Context, cfg Config, vmName string, c vmAPI, runSSH
257	}	232	}
258	return "", err	233	return "", err
259	}	234	}
260	if _, err := runSSH(ctx, truncateSerialCommand(cfg.AgentStateDir, vmID)); err != nil {	235	reboot.mark()
261	return "", fmt.Errorf("truncate serial log before reboot proof: %w", err)
262	}
263	if err := c.PatchVM(ctx, vmID, "running"); err != nil {	236	if err := c.PatchVM(ctx, vmID, "running"); err != nil {
264	return "", fmt.Errorf("patch vm running: %w", err)	237	return "", fmt.Errorf("patch vm running: %w", err)
265	}	238	}
@@ -276,7 +249,7 @@ func runScenario(ctx context.Context, cfg Config, vmName string, c vmAPI, runSSH
276	}	249	}
277	return "", err	250	return "", err
278	}	251	}
279	if err := proveBoot(ctx, cfg, vmID, runSSH, now, sleep, "after power cycle"); err != nil {	252	if err := proveBoot(ctx, reboot, now, sleep, "after power cycle"); err != nil {
280	return "", err	253	return "", err
281	}	254	}
282	if gate != nil {	255	if gate != nil {
@@ -288,7 +261,12 @@ func runScenario(ctx context.Context, cfg Config, vmName string, c vmAPI, runSSH
288	if err := c.DeleteVM(ctx, vmID); err != nil {	261	if err := c.DeleteVM(ctx, vmID); err != nil {
289	return "", fmt.Errorf("delete vm: %w", err)	262	return "", fmt.Errorf("delete vm: %w", err)
290	}	263	}
291	err = pollLoop(ctx, now, sleep, 120time.Second, 5time.Second, func() (bool, error) {	264	// The reap window must outlast the agent's tombstone grace — the quarantine
		265	// a deleted VM sits in before the agent destroys it. That grace defaults to
		266	// five minutes, and a fielded plane legitimately runs the default, so the
		267	// window is that plus margin for teardown and the sync ack. A healthy reap
		268	// still ends the poll the tick it lands; only a broken one waits this out.
		269	err = pollLoop(ctx, now, sleep, 7time.Minute, 5time.Second, func() (bool, error) {
292	_, present, err := getVM(ctx, c, vmID)	270	_, present, err := getVM(ctx, c, vmID)
293	if err != nil {	271	if err != nil {
294	return false, fmt.Errorf("poll vm reaped: %w", err)	272	return false, fmt.Errorf("poll vm reaped: %w", err)
@@ -297,7 +275,7 @@ func runScenario(ctx context.Context, cfg Config, vmName string, c vmAPI, runSSH
297	})	275	})
298	if err != nil {	276	if err != nil {
299	if errors.Is(err, errPollTimeout) {	277	if errors.Is(err, errPollTimeout) {
300	return "", errors.New("FAIL: VM not hard-deleted within 120s of tombstone")	278	return "", errors.New("FAIL: VM not hard-deleted within 7m of tombstone")
301	}	279	}
302	return "", err	280	return "", err
303	}	281	}

internal/smoke/scenario_test.go

Old		New
@@ -3,6 +3,7 @@ package smoke
3	import (	3	import (
4	"context"	4	"context"
5	"errors"	5	"errors"
		6	"io"
6	"strings"	7	"strings"
7	"testing"	8	"testing"
8	"time"	9	"time"
@@ -48,11 +49,17 @@ func TestClassifySerialStillBooting(t *testing.T) {
48	// --- fakes for runScenario -------------------------------------------------	49	// --- fakes for runScenario -------------------------------------------------
49		50
50	// fakeClock is a controllable now()/sleep() pair: sleep advances the virtual	51	// fakeClock is a controllable now()/sleep() pair: sleep advances the virtual
51	// clock instead of waiting, so deadline logic runs at test speed.	52	// clock instead of waiting, so deadline logic runs at test speed. It yields for
		53	// a moment as it does so, because what the boot proofs poll is filled by a real
		54	// goroutine reading a real stream — a poll loop that never gave up the
		55	// processor would spend its whole virtual deadline before that goroutine ran.
52	type fakeClock struct{ t time.Time }	56	type fakeClock struct{ t time.Time }
53		57
54	func (c *fakeClock) now() time.Time { return c.t }	58	func (c *fakeClock) now() time.Time { return c.t }
55	func (c *fakeClock) sleep(d time.Duration) { c.t = c.t.Add(d) }	59	func (c *fakeClock) sleep(d time.Duration) {
		60	c.t = c.t.Add(d)
		61	time.Sleep(time.Millisecond)
		62	}
56		63
57	// testAPI implements vmAPI by delegating to per-test closures. patchVMFunc,	64	// testAPI implements vmAPI by delegating to per-test closures. patchVMFunc,
58	// createExposureFunc, and deleteExposureFunc default to a benign answer when	65	// createExposureFunc, and deleteExposureFunc default to a benign answer when
@@ -85,7 +92,7 @@ func (a *testAPI) PatchVM(ctx context.Context, id, powerState string) error {
85	}	92	}
86	func (a *testAPI) CreateExposure(ctx context.Context, vmID string, guestPort, hostPort int64) (client.Exposure, error) {	93	func (a *testAPI) CreateExposure(ctx context.Context, vmID string, guestPort, hostPort int64) (client.Exposure, error) {
87	if a.createExposureFunc == nil {	94	if a.createExposureFunc == nil {
88	return client.Exposure{ID: "x-fake", HostPort: 30080}, nil	95	return client.Exposure{ID: "x-fake", HostPort: 30080, HostAddr: "192.168.0.190"}, nil
89	}	96	}
90	return a.createExposureFunc(ctx, vmID, guestPort, hostPort)	97	return a.createExposureFunc(ctx, vmID, guestPort, hostPort)
91	}	98	}
@@ -97,12 +104,38 @@ func (a *testAPI) DeleteExposure(ctx context.Context, id string) error {
97	return a.deleteExposureFunc(ctx, id)	104	return a.deleteExposureFunc(ctx, id)
98	}	105	}
99		106
100	func baseCfg() Config {
101	return Config{AgentStateDir: "/var/lib/eitri-agent", AgentUserHost: "ubuntu@10.0.0.5", AgentPort: 22}
102	}
103
104	func noopReadPubKey() string { return "ssh-ed25519 AAAAfake test@smoke" }	107	func noopReadPubKey() string { return "ssh-ed25519 AAAAfake test@smoke" }
105		108
		109	// bootedGuest is what a guest that reached userspace has on its console.
		110	const bootedGuest = "Ubuntu 24.04 LTS ubuntu-vm login: "
		111
		112	// powerCycleConsole gives api a console and the power hooks that make one
		113	// scripted power cycle observable: attaching replays the console's history (as
		114	// a host does), the stop waits until the tail attached for the reboot proof has
		115	// taken that history — so the proof's mark has something to discard — and the
		116	// start puts afterStart on the stream. It wraps whatever power hook api already
		117	// has, so the fake keeps actuating its own power state.
		118	func powerCycleConsole(t testing.T, api testAPI, history, afterStart string) *fakeConsole {
		119	t.Helper()
		120	console := newFakeConsole(history)
		121	actuate := api.patchVMFunc
		122	api.patchVMFunc = func(ctx context.Context, id, powerState string) error {
		123	if actuate != nil {
		124	if err := actuate(ctx, id, powerState); err != nil {
		125	return err
		126	}
		127	}
		128	switch powerState {
		129	case "stopped":
		130	console.waitReplayed(t, 2) // the reboot proof's own attach
		131	case "running":
		132	console.write(t, afterStart)
		133	}
		134	return nil
		135	}
		136	return console
		137	}
		138
106	// okBanner is the banner a converged listener answers with, for tests whose	139	// okBanner is the banner a converged listener answers with, for tests whose
107	// subject is not the exposure leg.	140	// subject is not the exposure leg.
108	func okBanner(ctx context.Context, addr string) (string, error) { return "SSH-2.0-Test\r\n", nil }	141	func okBanner(ctx context.Context, addr string) (string, error) { return "SSH-2.0-Test\r\n", nil }
@@ -151,25 +184,13 @@ func TestRunScenarioSuccess(t *testing.T) {
151	},	184	},
152	}	185	}
153		186
154	sshCalls := 0	187	// The console replays the first boot on every attach — including the one
155	sawTruncate := false	188	// the reboot proof makes — and the guest prints its second boot when the
156	runSSH := func(ctx context.Context, remoteCmd string) (string, error) {	189	// start command lands.
157	sshCalls++	190	console := powerCycleConsole(t, api, bootedGuest, "[ 0.9] Booting Linux\nubuntu-vm login: ")
158	if !strings.Contains(remoteCmd, "vm-1") {
159	t.Errorf("remoteCmd = %q, want it to reference vm-1", remoteCmd)
160	}
161	if strings.Contains(remoteCmd, "truncate") {
162	sawTruncate = true
163	return "", nil
164	}
165	if !sawTruncate && sshCalls < 2 {
166	return "[ 0.1] Booting Linux...", nil
167	}
168	return "Ubuntu 24.04 LTS ubuntu-vm login: ", nil
169	}
170		191
171	clock := &fakeClock{t: time.Unix(0, 0)}	192	clock := &fakeClock{t: time.Unix(0, 0)}
172	msg, err := runScenario(context.Background(), baseCfg(), "smoke-test", api, runSSH, nil, nil, clock.now, clock.sleep, noopReadPubKey, okBanner)	193	msg, err := runScenario(context.Background(), "smoke-test", api, console.dial, nil, nil, clock.now, clock.sleep, noopReadPubKey, okBanner)
173	if err != nil {	194	if err != nil {
174	t.Fatalf("runScenario: %v", err)	195	t.Fatalf("runScenario: %v", err)
175	}	196	}
@@ -185,15 +206,19 @@ func TestRunScenarioSuccess(t *testing.T) {
185	if !strings.Contains(msg, "exposed port: ok") {	206	if !strings.Contains(msg, "exposed port: ok") {
186	t.Errorf("message = %q, want exposed port: ok", msg)	207	t.Errorf("message = %q, want exposed port: ok", msg)
187	}	208	}
188	if !sawTruncate {	209	// One attach for the first boot, a second held across the power cycle.
189	t.Error("serial log was never truncated before the reboot proof — proof could pass on first-boot bytes")	210	if console.attachCount() < 2 {
		211	t.Errorf("console attaches = %d, want the reboot proof to attach its own stream", console.attachCount())
190	}	212	}
191	}	213	}
192		214
193	// TestRunScenarioRebootDeathFails pins the regression that motivated the	215	// TestRunScenarioRebootDeathFails pins the regression that motivated the
194	// power-cycle leg: a VM whose second boot never reaches userspace (stale	216	// power-cycle leg, and with it the freshness rule the console proof rests on: a
195	// on-disk GPT → initramfs emergency mode) must FAIL the smoke, not pass on	217	// VM whose second boot never reaches userspace (stale on-disk GPT → initramfs
196	// its green first boot.	218	// emergency mode) must FAIL the smoke. The console here replays the FIRST
		219	// boot's login prompt to the stream watching the restart — exactly what a host
		220	// does — so a proof that counted replayed history would call this dead VM
		221	// green.
197	func TestRunScenarioRebootDeathFails(t *testing.T) {	222	func TestRunScenarioRebootDeathFails(t *testing.T) {
198	power := "running"	223	power := "running"
199	api := &testAPI{	224	api := &testAPI{
@@ -215,22 +240,11 @@ func TestRunScenarioRebootDeathFails(t *testing.T) {
215	return nil	240	return nil
216	},	241	},
217	}	242	}
218		243	// After the restart: an emergency shell, never a login prompt.
219	sawTruncate := false	244	console := powerCycleConsole(t, api, bootedGuest, "Press Enter for system maintenance")
220	runSSH := func(ctx context.Context, remoteCmd string) (string, error) {
221	if strings.Contains(remoteCmd, "truncate") {
222	sawTruncate = true
223	return "", nil
224	}
225	if sawTruncate {
226	// Post-reboot serial: emergency shell, never a login prompt.
227	return "Press Enter for system maintenance", nil
228	}
229	return "Ubuntu 24.04 LTS ubuntu-vm login: ", nil
230	}
231		245
232	clock := &fakeClock{t: time.Unix(0, 0)}	246	clock := &fakeClock{t: time.Unix(0, 0)}
233	_, err := runScenario(context.Background(), baseCfg(), "smoke-test", api, runSSH, nil, nil, clock.now, clock.sleep, noopReadPubKey, okBanner)	247	_, err := runScenario(context.Background(), "smoke-test", api, console.dial, nil, nil, clock.now, clock.sleep, noopReadPubKey, okBanner)
234	if err == nil {	248	if err == nil {
235	t.Fatal("runScenario: want error for a VM that never came back, got nil")	249	t.Fatal("runScenario: want error for a VM that never came back, got nil")
236	}	250	}
@@ -256,7 +270,7 @@ func TestRunScenarioExposureFailureFails(t *testing.T) {
256	badBanner := func(ctx context.Context, addr string) (string, error) { return "HTTP/1.1 400\r\n", nil }	270	badBanner := func(ctx context.Context, addr string) (string, error) { return "HTTP/1.1 400\r\n", nil }
257		271
258	clock := &fakeClock{t: time.Unix(0, 0)}	272	clock := &fakeClock{t: time.Unix(0, 0)}
259	_, err := runScenario(context.Background(), baseCfg(), "smoke-test", api, happyPathRunSSH(), nil, nil, clock.now, clock.sleep, noopReadPubKey, badBanner)	273	_, err := runScenario(context.Background(), "smoke-test", api, newFakeConsole(bootedGuest).dial, nil, nil, clock.now, clock.sleep, noopReadPubKey, badBanner)
260	if err == nil {	274	if err == nil {
261	t.Fatal("runScenario: want error, got nil")	275	t.Fatal("runScenario: want error, got nil")
262	}	276	}
@@ -286,12 +300,10 @@ func TestRunScenarioSerialPanicFails(t *testing.T) {
286	return nil	300	return nil
287	},	301	},
288	}	302	}
289	runSSH := func(ctx context.Context, remoteCmd string) (string, error) {	303	console := newFakeConsole("Kernel panic - not syncing: VFS: Unable to mount root fs")
290	return "Kernel panic - not syncing: VFS: Unable to mount root fs", nil
291	}
292		304
293	clock := &fakeClock{t: time.Unix(0, 0)}	305	clock := &fakeClock{t: time.Unix(0, 0)}
294	_, err := runScenario(context.Background(), baseCfg(), "smoke-test", api, runSSH, nil, nil, clock.now, clock.sleep, noopReadPubKey, okBanner)	306	_, err := runScenario(context.Background(), "smoke-test", api, console.dial, nil, nil, clock.now, clock.sleep, noopReadPubKey, okBanner)
295	if err == nil {	307	if err == nil {
296	t.Fatal("runScenario: want error, got nil")	308	t.Fatal("runScenario: want error, got nil")
297	}	309	}
@@ -318,16 +330,20 @@ func TestRunScenarioNeverReadyTimesOut(t *testing.T) {
318	return nil	330	return nil
319	},	331	},
320	}	332	}
321	runSSH := func(ctx context.Context, remoteCmd string) (string, error) {	333	attached := 0
322	t.Fatal("runSSH should not be called when the VM never becomes ready")	334	console := func(ctx context.Context, vmID string) (io.ReadWriteCloser, error) {
323	return "", nil	335	attached++
		336	return nil, errors.New("console should not be attached before the VM is ready")
324	}	337	}
325		338
326	clock := &fakeClock{t: time.Unix(0, 0)}	339	clock := &fakeClock{t: time.Unix(0, 0)}
327	_, err := runScenario(context.Background(), baseCfg(), "smoke-test", api, runSSH, nil, nil, clock.now, clock.sleep, noopReadPubKey, okBanner)	340	_, err := runScenario(context.Background(), "smoke-test", api, console, nil, nil, clock.now, clock.sleep, noopReadPubKey, okBanner)
328	if err == nil {	341	if err == nil {
329	t.Fatal("runScenario: want error, got nil")	342	t.Fatal("runScenario: want error, got nil")
330	}	343	}
		344	if attached != 0 {
		345	t.Errorf("console attaches = %d, want none before the VM is ready", attached)
		346	}
331	if !strings.Contains(err.Error(), "FAIL: VM not ready within 600s") {	347	if !strings.Contains(err.Error(), "FAIL: VM not ready within 600s") {
332	t.Errorf("error = %q, want FAIL: VM not ready within 600s ...", err.Error())	348	t.Errorf("error = %q, want FAIL: VM not ready within 600s ...", err.Error())
333	}	349	}
@@ -357,17 +373,15 @@ func TestRunScenarioNeverReapedTimesOut(t *testing.T) {
357	},	373	},
358	deleteVMFunc: func(ctx context.Context, id string) error { return nil },	374	deleteVMFunc: func(ctx context.Context, id string) error { return nil },
359	}	375	}
360	runSSH := func(ctx context.Context, remoteCmd string) (string, error) {	376	console := powerCycleConsole(t, api, bootedGuest, "ubuntu-vm login: ")
361	return "Ubuntu 24.04 LTS ubuntu-vm login: ", nil
362	}
363		377
364	clock := &fakeClock{t: time.Unix(0, 0)}	378	clock := &fakeClock{t: time.Unix(0, 0)}
365	_, err := runScenario(context.Background(), baseCfg(), "smoke-test", api, runSSH, nil, nil, clock.now, clock.sleep, noopReadPubKey, okBanner)	379	_, err := runScenario(context.Background(), "smoke-test", api, console.dial, nil, nil, clock.now, clock.sleep, noopReadPubKey, okBanner)
366	if err == nil {	380	if err == nil {
367	t.Fatal("runScenario: want error, got nil")	381	t.Fatal("runScenario: want error, got nil")
368	}	382	}
369	if !strings.Contains(err.Error(), "FAIL: VM not hard-deleted within 120s") {	383	if !strings.Contains(err.Error(), "FAIL: VM not hard-deleted within 7m") {
370	t.Errorf("error = %q, want FAIL: VM not hard-deleted within 120s ...", err.Error())	384	t.Errorf("error = %q, want FAIL: VM not hard-deleted within 7m ...", err.Error())
371	}	385	}
372	}	386	}
373		387
@@ -390,7 +404,7 @@ func TestRunScenarioNoHosts(t *testing.T) {
390	},	404	},
391	}	405	}
392	clock := &fakeClock{t: time.Unix(0, 0)}	406	clock := &fakeClock{t: time.Unix(0, 0)}
393	_, err := runScenario(context.Background(), baseCfg(), "smoke-test", api, nil, nil, nil, clock.now, clock.sleep, noopReadPubKey, okBanner)	407	_, err := runScenario(context.Background(), "smoke-test", api, nil, nil, nil, clock.now, clock.sleep, noopReadPubKey, okBanner)
394	if err == nil {	408	if err == nil {
395	t.Fatal("runScenario: want error, got nil")	409	t.Fatal("runScenario: want error, got nil")
396	}	410	}
@@ -435,17 +449,6 @@ func happyPathAPI(t testing.T, calls []string) *testAPI {
435	}	449	}
436	}	450	}
437		451
438	func happyPathRunSSH() sshFunc {
439	sshCalls := 0
440	return func(ctx context.Context, remoteCmd string) (string, error) {
441	sshCalls++
442	if sshCalls < 2 {
443	return "[ 0.1] Booting Linux...", nil
444	}
445	return "Ubuntu 24.04 LTS ubuntu-vm login: ", nil
446	}
447	}
448
449	func TestRunScenarioGateRegistersBeforeCreateAndExecsAfterBoot(t *testing.T) {	452	func TestRunScenarioGateRegistersBeforeCreateAndExecsAfterBoot(t *testing.T) {
450	var calls []string	453	var calls []string
451	api := happyPathAPI(t, &calls)	454	api := happyPathAPI(t, &calls)
@@ -464,7 +467,7 @@ func TestRunScenarioGateRegistersBeforeCreateAndExecsAfterBoot(t *testing.T) {
464	}	467	}
465		468
466	clock := &fakeClock{t: time.Unix(0, 0)}	469	clock := &fakeClock{t: time.Unix(0, 0)}
467	msg, err := runScenario(context.Background(), baseCfg(), "smoke-test", api, happyPathRunSSH(), gate, nil, clock.now, clock.sleep, noopReadPubKey, okBanner)	470	msg, err := runScenario(context.Background(), "smoke-test", api, powerCycleConsole(t, api, bootedGuest, "ubuntu-vm login: ").dial, gate, nil, clock.now, clock.sleep, noopReadPubKey, okBanner)
468	if err != nil {	471	if err != nil {
469	t.Fatalf("runScenario: %v", err)	472	t.Fatalf("runScenario: %v", err)
470	}	473	}
@@ -514,7 +517,7 @@ func TestRunScenarioGateRegisterErrorAbortsBeforeCreate(t *testing.T) {
514	}	517	}
515		518
516	clock := &fakeClock{t: time.Unix(0, 0)}	519	clock := &fakeClock{t: time.Unix(0, 0)}
517	_, err := runScenario(context.Background(), baseCfg(), "smoke-test", api, nil, gate, nil, clock.now, clock.sleep, noopReadPubKey, okBanner)	520	_, err := runScenario(context.Background(), "smoke-test", api, nil, gate, nil, clock.now, clock.sleep, noopReadPubKey, okBanner)
518	if err == nil {	521	if err == nil {
519	t.Fatal("runScenario: want error, got nil")	522	t.Fatal("runScenario: want error, got nil")
520	}	523	}
@@ -538,7 +541,7 @@ func TestRunScenarioGateExecErrorFails(t *testing.T) {
538	}	541	}
539		542
540	clock := &fakeClock{t: time.Unix(0, 0)}	543	clock := &fakeClock{t: time.Unix(0, 0)}
541	_, err := runScenario(context.Background(), baseCfg(), "smoke-test", api, happyPathRunSSH(), gate, nil, clock.now, clock.sleep, noopReadPubKey, okBanner)	544	_, err := runScenario(context.Background(), "smoke-test", api, powerCycleConsole(t, api, bootedGuest, "ubuntu-vm login: ").dial, gate, nil, clock.now, clock.sleep, noopReadPubKey, okBanner)
542	if err == nil {	545	if err == nil {
543	t.Fatal("runScenario: want error, got nil")	546	t.Fatal("runScenario: want error, got nil")
544	}	547	}
@@ -615,7 +618,7 @@ func TestRunScenarioRunsTheMCPLegOnItsOwnVM(t *testing.T) {
615	}	618	}
616		619
617	clock := &fakeClock{t: time.Unix(0, 0)}	620	clock := &fakeClock{t: time.Unix(0, 0)}
618	msg, err := runScenario(context.Background(), baseCfg(), "smoke-test", api, happyPathRunSSH(), nil, mcp, clock.now, clock.sleep, noopReadPubKey, okBanner)	621	msg, err := runScenario(context.Background(), "smoke-test", api, powerCycleConsole(t, api, bootedGuest, "ubuntu-vm login: ").dial, nil, mcp, clock.now, clock.sleep, noopReadPubKey, okBanner)
619	if err != nil {	622	if err != nil {
620	t.Fatalf("runScenario: %v", err)	623	t.Fatalf("runScenario: %v", err)
621	}	624	}
@@ -635,7 +638,7 @@ func TestRunScenarioFailsWhenTheMCPLegFails(t *testing.T) {
635	mcp := func(context.Context, string) error { return errors.New("FAIL: register refused") }	638	mcp := func(context.Context, string) error { return errors.New("FAIL: register refused") }
636		639
637	clock := &fakeClock{t: time.Unix(0, 0)}	640	clock := &fakeClock{t: time.Unix(0, 0)}
638	_, err := runScenario(context.Background(), baseCfg(), "smoke-test", api, happyPathRunSSH(), nil, mcp, clock.now, clock.sleep, noopReadPubKey, okBanner)	641	_, err := runScenario(context.Background(), "smoke-test", api, powerCycleConsole(t, api, bootedGuest, "ubuntu-vm login: ").dial, nil, mcp, clock.now, clock.sleep, noopReadPubKey, okBanner)
639	if err == nil \|\| !strings.Contains(err.Error(), "register refused") {	642	if err == nil \|\| !strings.Contains(err.Error(), "register refused") {
640	t.Fatalf("runScenario err = %v, want the MCP leg's failure to fail the gate", err)	643	t.Fatalf("runScenario err = %v, want the MCP leg's failure to fail the gate", err)
641	}	644	}

scripts/deploy.sh

Old		New
@@ -352,9 +352,11 @@ else
352	# to a tenant. Phase 2 authenticates the VM lifecycle with the operator PAT in	352	# to a tenant. Phase 2 authenticates the VM lifecycle with the operator PAT in
353	# CI_PAT_FILE and derives its tenant via Me() — no admin token, no hardcoded	353	# CI_PAT_FILE and derives its tenant via Me() — no admin token, no hardcoded
354	# tenant.	354	# tenant.
355	# COVER_OUT triggers the post-gate coverage merge (server+agent).	355	# COVER_OUT triggers the post-gate coverage merge (server+agent); AGENT_HOSTS
		356	# is here for that merge alone — the gate itself proves everything through
		357	# the API, the console and the gate, and logs into no host in the fleet.
356	export SERVER_URL="$smoke_server_url"	358	export SERVER_URL="$smoke_server_url"
357	export CI_USER CI_PASSWORD_FILE CI_PAT_FILE AGENT_HOSTS AGENT_STATE_DIR SERVER_GOCOVERDIR AGENT_GOCOVERDIR	359	export CI_USER CI_PASSWORD_FILE CI_PAT_FILE AGENT_HOSTS SERVER_GOCOVERDIR AGENT_GOCOVERDIR
358	export SMOKE_GATE SMOKE_VM_USER SMOKE_USER_CA_FILE	360	export SMOKE_GATE SMOKE_VM_USER SMOKE_USER_CA_FILE
359	if COVER_OUT="$REPO_ROOT/coverage/integration" "$REPO_ROOT/bin/eitri-smoke"; then	361	if COVER_OUT="$REPO_ROOT/coverage/integration" "$REPO_ROOT/bin/eitri-smoke"; then
360	echo "boot gate: PASS"	362	echo "boot gate: PASS"

scripts/ship.env.example

Old		New
@@ -16,13 +16,6 @@ SITE_IMAGE="registry.example.com/eitri-site"
16	# Image platform for the site image; match the node arch in the plane file.	16	# Image platform for the site image; match the node arch in the plane file.
17	SITE_PLATFORM="linux/arm64"	17	SITE_PLATFORM="linux/arm64"
18		18
19	# ── The plane's fleet, for the hosted smoke ───────────────────────────────────
20	# The smoke reads the guest's serial log over ssh to its host and dials that
21	# host's own uplink for the exposure leg, so these are LAN addresses of the
22	# hosts joined to THIS plane — unrelated to where the control plane runs.
23	AGENT_HOSTS="ubuntu@192.168.0.193:2222"
24	AGENT_STATE_DIR="/var/lib/eitri-agent"
25
26	# ── Credentials ───────────────────────────────────────────────────────────────	19	# ── Credentials ───────────────────────────────────────────────────────────────
27	# Name ONE of the two arrangements. ship.sh forwards whatever it finds here and	20	# Name ONE of the two arrangements. ship.sh forwards whatever it finds here and
28	# clears the rest, so the plane decides and the pipeline stays the same script.	21	# clears the rest, so the plane decides and the pipeline stays the same script.

scripts/ship.sh

Old		New
@@ -499,10 +499,10 @@ fi
499		499
500	# ── 8. Hosted smoke ───────────────────────────────────────────────────────────	500	# ── 8. Hosted smoke ───────────────────────────────────────────────────────────
501	# The same binary the branch gate runs, pointed at this plane's public names.	501	# The same binary the branch gate runs, pointed at this plane's public names.
502	# The host-facing legs are unchanged by where the control plane lives: the	502	# It needs nothing but those names and a credential: every leg goes through the
503	# boot proof reads a serial log over ssh to the agent host and the exposure leg	503	# plane's own front door, so a hosted run logs into no host in the fleet. What
504	# dials that host's own uplink. What IS new is that the gate leg now crosses the	504	# IS new is that the gate leg crosses the internet, so GATE_HOST:GATE_PORT must
505	# internet, so GATE_HOST:GATE_PORT must be reachable from here.	505	# be reachable from here.
506	#	506	#
507	# Which credentials the run uses is the plane's business, not this script's: it	507	# Which credentials the run uses is the plane's business, not this script's: it
508	# forwards whatever ship.env names and nothing else. A plane with a password	508	# forwards whatever ship.env names and nothing else. A plane with a password
@@ -512,7 +512,6 @@ fi
512	# hosted binaries are not coverage-instrumented.	512	# hosted binaries are not coverage-instrumented.
513	if [[ "$FROM" -le 8 && "$SKIP_SMOKE" != "1" ]]; then	513	if [[ "$FROM" -le 8 && "$SKIP_SMOKE" != "1" ]]; then
514	bold "8. Hosted smoke against $TARGET"	514	bold "8. Hosted smoke against $TARGET"
515	: "${AGENT_HOSTS:?set in $SHIP_ENV}" "${AGENT_STATE_DIR:?}"
516	if [[ -z "${CI_USER:-}" && -z "${CI_PAT_FILE:-}" ]]; then	515	if [[ -z "${CI_USER:-}" && -z "${CI_PAT_FILE:-}" ]]; then
517	fail "$SHIP_ENV names no credential for the smoke.	516	fail "$SHIP_ENV names no credential for the smoke.
518	Set CI_USER + CI_PASSWORD_FILE for a plane with a password issuer, or	517	Set CI_USER + CI_PASSWORD_FILE for a plane with a password issuer, or
@@ -530,8 +529,6 @@ if [[ "$FROM" -le 8 && "$SKIP_SMOKE" != "1" ]]; then
530	SMOKE_GATE="$GATE_HOST:$GATE_PORT"	529	SMOKE_GATE="$GATE_HOST:$GATE_PORT"
531	SMOKE_MCP_URL="$SMOKE_MCP_URL"	530	SMOKE_MCP_URL="$SMOKE_MCP_URL"
532	SMOKE_USER_CA_FILE="$SMOKE_USER_CA_FILE"	531	SMOKE_USER_CA_FILE="$SMOKE_USER_CA_FILE"
533	AGENT_HOSTS="$AGENT_HOSTS"
534	AGENT_STATE_DIR="$AGENT_STATE_DIR"
535	)	532	)
536	if [[ -n "${CI_USER:-}" ]]; then	533	if [[ -n "${CI_USER:-}" ]]; then
537	: "${CI_PASSWORD_FILE:?set it alongside CI_USER in $SHIP_ENV}"	534	: "${CI_PASSWORD_FILE:?set it alongside CI_USER in $SHIP_ENV}"