e679b201

ship: a release is proven on the fleet it actually converged

a73x 2026-08-18 14:46

Commit message

ship: a release is proven on the fleet it actually converged

Stage 7 rolls the control-plane server, and that ends every agent's sync
session at once. Stage 8 read the fleet seconds later, found it entirely
dark, wrote every host off as disconnected and broke out of its converge
loop reporting nothing to converge — then stage 9 placed its VM on whichever
agent had reconnected by then, proved a release-old agent green, and the
ship exited 0 with the plane at the tag and the hosts a release behind it.
That shipped twice, v0.0.6 and v0.0.7 on prod, and both were finished by
hand with --from 8 once the agents were back.

A dark host is now waited on for 90s before it is written off. Inside that
window it joins the waiting list like a host with an offer outstanding, so
the loop keeps polling and offers it the tag the moment it reconnects; past
it, the warn-once-and-skip stands, because a genuinely parked machine must
never hold a release.

The smoke stops taking the host it lands on for granted. A ship names the
release it is proving (SMOKE_EXPECT_AGENT_VERSION) and the scenario refuses
a host whose agent reports anything else, before it creates a VM there — the
failure says the fleet did not converge and how to finish it, because every
leg the smoke proves is satisfied just as well by the release before this
one. Empty is no expectation, which is every run but a ship's: the boot gate
proves binaries built from a working tree.

Stage 8's sign-in is retried six times, ten seconds apart. The issuer rolled
seconds earlier too, and one still coming up walks the redirect chain and
sets no session at the end of it — indistinguishable, from here, from a
refused password, which is why the retry is blind and the original message
is what the last attempt prints.

internal/smoke/config.go

Old		New
@@ -46,6 +46,15 @@ type Config struct {
46	AgentGocoverdir string	46	AgentGocoverdir string
47	CoverOut string	47	CoverOut string
48		48
		49	// ExpectAgentVersion (SMOKE_EXPECT_AGENT_VERSION) is the release this run is
		50	// proving. Set it and the scenario refuses to place its VM on a host whose
		51	// agent reports some other version — the run proves the release, not merely
		52	// the plane serving it, so a fleet that never converged fails the smoke
		53	// instead of passing it on a release-old agent. Empty ⇒ no such expectation,
		54	// which is what every run but a ship's has: the boot gate proves binaries
		55	// built from a working tree, whose stamped version is "dev".
		56	ExpectAgentVersion string
		57
49	// SSH-CA gate check. When SmokeGate and SmokeUserCAFile are both set, the	58	// SSH-CA gate check. When SmokeGate and SmokeUserCAFile are both set, the
50	// scenario proves guest access through the gate (a hard gate). Optional.	59	// scenario proves guest access through the gate (a hard gate). Optional.
51	SmokeGate string // SMOKE_GATE, "<gate-domain>:<port>"	60	SmokeGate string // SMOKE_GATE, "<gate-domain>:<port>"
@@ -107,19 +116,20 @@ func loadConfig(getenv func(string) string) (Config, error) {
107	}	116	}
108		117
109	return Config{	118	return Config{
110	ServerURL: serverURL,	119	ServerURL: serverURL,
111	MCPURLs: mcpURLs,	120	MCPURLs: mcpURLs,
112	CIUser: ciUser,	121	ExpectAgentVersion: getenv("SMOKE_EXPECT_AGENT_VERSION"),
113	CIPasswordFile: ciPasswordFile,	122	CIUser: ciUser,
114	CIPATFile: ciPATFile,	123	CIPasswordFile: ciPasswordFile,
115	AgentUserHost: userHost,	124	CIPATFile: ciPATFile,
116	AgentPort: port,	125	AgentUserHost: userHost,
117	ServerGocoverdir: getenv("SERVER_GOCOVERDIR"),	126	AgentPort: port,
118	AgentGocoverdir: agentGocoverdir,	127	ServerGocoverdir: getenv("SERVER_GOCOVERDIR"),
119	CoverOut: getenv("COVER_OUT"),	128	AgentGocoverdir: agentGocoverdir,
120	SmokeGate: getenv("SMOKE_GATE"),	129	CoverOut: getenv("COVER_OUT"),
121	SmokeVMUser: smokeVMUser,	130	SmokeGate: getenv("SMOKE_GATE"),
122	SmokeUserCAFile: getenv("SMOKE_USER_CA_FILE"),	131	SmokeVMUser: smokeVMUser,
		132	SmokeUserCAFile: getenv("SMOKE_USER_CA_FILE"),
123	}, nil	133	}, nil
124	}	134	}
125		135

internal/smoke/config_test.go

+		t.Errorf("optional coverage vars not passed through: %+v", cfg)
+// TestLoadConfigCarriesTheExpectedAgentVersion: a ship names the release it is
+// proving, and everything else leaves the expectation empty — a boot gate
+// proves binaries built from a working tree, whose stamped version is "dev".
+func TestLoadConfigCarriesTheExpectedAgentVersion(t *testing.T) {
+	cfg, err := loadConfig(fakeGetenv(requiredVals()))
+	if err != nil {
+		t.Fatalf("loadConfig: %v", err)
+	}
+	if cfg.ExpectAgentVersion != "" {
+		t.Errorf("ExpectAgentVersion = %q, want empty when nothing names a release", cfg.ExpectAgentVersion)
+	}
+	vals := requiredVals()
+	vals["SMOKE_EXPECT_AGENT_VERSION"] = "v0.0.7"
+	cfg, err = loadConfig(fakeGetenv(vals))
+	if err != nil {
+		t.Fatalf("loadConfig: %v", err)
+	}
+	if cfg.ExpectAgentVersion != "v0.0.7" {
+		t.Errorf("ExpectAgentVersion = %q, want v0.0.7", cfg.ExpectAgentVersion)
+	}
+}

internal/smoke/run.go

Old		New
@@ -171,7 +171,7 @@ func Run() error {
171	}, time.Now, time.Sleep, readBanner, readEcho)	171	}, time.Now, time.Sleep, readBanner, readEcho)
172	}	172	}
173		173
174	msg, err := runScenario(ctx, vmName, api, api.DialConsole, gate, mcp, time.Now, time.Sleep, realReadPubKey, readBanner)	174	msg, err := runScenario(ctx, vmName, api, api.DialConsole, gate, mcp, cfg.ExpectAgentVersion, time.Now, time.Sleep, realReadPubKey, readBanner)
175	if err != nil {	175	if err != nil {
176	return err	176	return err
177	}	177	}

internal/smoke/scenario.go

Old		New
@@ -73,6 +73,37 @@ func getVM(ctx context.Context, c vmAPI, id string) (client.VM, bool, error) {
73	return client.VM{}, false, nil	73	return client.VM{}, false, nil
74	}	74	}
75		75
		76	// proveHostRelease refuses a host whose agent is not running the release this
		77	// run is proving. want == "" is no expectation at all, which is every run but a
		78	// ship's: a boot gate proves binaries built from a working tree and their
		79	// stamped version is "dev".
		80	//
		81	// It exists because a VM proves whatever agent it lands on, and every leg of
		82	// this scenario — create, boot, gate SSH, published port, reap — is satisfied
		83	// just as well by the release before this one. A ship that rolls the plane and
		84	// fails to converge the fleet therefore looked, from the smoke, exactly like a
		85	// ship that worked; twice it exited green with the hosts a release behind. What
		86	// a mismatch here means is never that the VM is broken: it is that the fleet
		87	// did not converge, so the message says so and says how to finish it.
		88	//
		89	// It guards this scenario's VM only. The MCP leg's VM places itself through the
		90	// plane's own default placement and is deliberately left unguarded — out of
		91	// scope here, and pinning its host would cost that leg the one thing it proves
		92	// about placement, which is that the plane's default choice works at all.
		93	func proveHostRelease(h client.Host, want string) error {
		94	if want == "" \|\| h.AgentVersion == want {
		95	return nil
		96	}
		97	reported := h.AgentVersion
		98	if reported == "" {
		99	reported = "no version at all"
		100	}
		101	return fmt.Errorf("FAIL: host %s is running agent %s, not %s — the plane is serving %s, "+
		102	"so the fleet did not converge. A VM placed here would prove the release before "+
		103	"this one. Converge the fleet and resume: scripts/ship.sh --from 8",
		104	h.Name, reported, want, want)
		105	}
		106
76	// gateHooks bundles the optional SSH-CA gate steps. nil means "skip the gate".	107	// gateHooks bundles the optional SSH-CA gate steps. nil means "skip the gate".
77	type gateHooks struct {	108	type gateHooks struct {
78	register func(ctx context.Context) error // upload the smoke user CA to the tenant (before create)	109	register func(ctx context.Context) error // upload the smoke user CA to the tenant (before create)
@@ -118,10 +149,12 @@ func pollLoop(ctx context.Context, now func() time.Time, sleep func(time.Duratio
118	// mcp, when non-nil, drives a second VM's whole life through the remote MCP	149	// mcp, when non-nil, drives a second VM's whole life through the remote MCP
119	// endpoint — that leg owns its own VM, created after it registers its own CA	150	// endpoint — that leg owns its own VM, created after it registers its own CA
120	// with the tenant, so the guest trusts the certificates eitri presents.	151	// with the tenant, so the guest trusts the certificates eitri presents.
		152	// expectAgentVersion, when non-empty, is the release the host the VM lands on
		153	// must be running (see proveHostRelease); empty skips that check.
121	// now/sleep are the injected clock so the poll deadlines are unit-testable	154	// now/sleep are the injected clock so the poll deadlines are unit-testable
122	// without real waiting. On success it returns the human-readable COMPLETE	155	// without real waiting. On success it returns the human-readable COMPLETE
123	// line; on any failure it returns a descriptive error.	156	// line; on any failure it returns a descriptive error.
124	func runScenario(ctx context.Context, vmName string, c vmAPI, dialConsole consoleDialer, gate *gateHooks, mcp mcpLeg, now func() time.Time, sleep func(time.Duration), readPubKey func() string, dialBanner bannerFunc) (string, error) {	157	func runScenario(ctx context.Context, vmName string, c vmAPI, dialConsole consoleDialer, gate *gateHooks, mcp mcpLeg, expectAgentVersion string, now func() time.Time, sleep func(time.Duration), readPubKey func() string, dialBanner bannerFunc) (string, error) {
125	if gate != nil {	158	if gate != nil {
126	if err := gate.register(ctx); err != nil {	159	if err := gate.register(ctx); err != nil {
127	return "", fmt.Errorf("register smoke user CA: %w", err)	160	return "", fmt.Errorf("register smoke user CA: %w", err)
@@ -135,7 +168,16 @@ func runScenario(ctx context.Context, vmName string, c vmAPI, dialConsole consol
135	if len(hostList) == 0 {	168	if len(hostList) == 0 {
136	return "", errors.New("no hosts available")	169	return "", errors.New("no hosts available")
137	}	170	}
138	hostID := hostList[0].ID	171	host := hostList[0]
		172	// Before anything is created: there is no reason to boot-prove a VM on an
		173	// agent this run was never meant to prove. The list is not searched for a
		174	// host that does match, either — a fleet where only some hosts converged is
		175	// itself the defect, and shopping for a working host would hide exactly the
		176	// bug this check exists to expose.
		177	if err := proveHostRelease(host, expectAgentVersion); err != nil {
		178	return "", err
		179	}
		180	hostID := host.ID
139		181
140	sshKey := readPubKey()	182	sshKey := readPubKey()
141		183

internal/smoke/scenario_test.go

Old		New
@@ -190,7 +190,7 @@ func TestRunScenarioSuccess(t *testing.T) {
190	console := powerCycleConsole(t, api, bootedGuest, "[ 0.9] Booting Linux\nubuntu-vm login: ")	190	console := powerCycleConsole(t, api, bootedGuest, "[ 0.9] Booting Linux\nubuntu-vm login: ")
191		191
192	clock := &fakeClock{t: time.Unix(0, 0)}	192	clock := &fakeClock{t: time.Unix(0, 0)}
193	msg, err := runScenario(context.Background(), "smoke-test", api, console.dial, nil, nil, clock.now, clock.sleep, noopReadPubKey, okBanner)	193	msg, err := runScenario(context.Background(), "smoke-test", api, console.dial, nil, nil, "", clock.now, clock.sleep, noopReadPubKey, okBanner)
194	if err != nil {	194	if err != nil {
195	t.Fatalf("runScenario: %v", err)	195	t.Fatalf("runScenario: %v", err)
196	}	196	}
@@ -244,7 +244,7 @@ func TestRunScenarioRebootDeathFails(t *testing.T) {
244	console := powerCycleConsole(t, api, bootedGuest, "Press Enter for system maintenance")	244	console := powerCycleConsole(t, api, bootedGuest, "Press Enter for system maintenance")
245		245
246	clock := &fakeClock{t: time.Unix(0, 0)}	246	clock := &fakeClock{t: time.Unix(0, 0)}
247	_, err := runScenario(context.Background(), "smoke-test", api, console.dial, nil, nil, clock.now, clock.sleep, noopReadPubKey, okBanner)	247	_, err := runScenario(context.Background(), "smoke-test", api, console.dial, nil, nil, "", clock.now, clock.sleep, noopReadPubKey, okBanner)
248	if err == nil {	248	if err == nil {
249	t.Fatal("runScenario: want error for a VM that never came back, got nil")	249	t.Fatal("runScenario: want error for a VM that never came back, got nil")
250	}	250	}
@@ -270,7 +270,7 @@ func TestRunScenarioExposureFailureFails(t *testing.T) {
270	badBanner := func(ctx context.Context, addr string) (string, error) { return "HTTP/1.1 400\r\n", nil }	270	badBanner := func(ctx context.Context, addr string) (string, error) { return "HTTP/1.1 400\r\n", nil }
271		271
272	clock := &fakeClock{t: time.Unix(0, 0)}	272	clock := &fakeClock{t: time.Unix(0, 0)}
273	_, err := runScenario(context.Background(), "smoke-test", api, newFakeConsole(bootedGuest).dial, nil, nil, clock.now, clock.sleep, noopReadPubKey, badBanner)	273	_, err := runScenario(context.Background(), "smoke-test", api, newFakeConsole(bootedGuest).dial, nil, nil, "", clock.now, clock.sleep, noopReadPubKey, badBanner)
274	if err == nil {	274	if err == nil {
275	t.Fatal("runScenario: want error, got nil")	275	t.Fatal("runScenario: want error, got nil")
276	}	276	}
@@ -303,7 +303,7 @@ func TestRunScenarioSerialPanicFails(t *testing.T) {
303	console := newFakeConsole("Kernel panic - not syncing: VFS: Unable to mount root fs")	303	console := newFakeConsole("Kernel panic - not syncing: VFS: Unable to mount root fs")
304		304
305	clock := &fakeClock{t: time.Unix(0, 0)}	305	clock := &fakeClock{t: time.Unix(0, 0)}
306	_, err := runScenario(context.Background(), "smoke-test", api, console.dial, nil, nil, clock.now, clock.sleep, noopReadPubKey, okBanner)	306	_, err := runScenario(context.Background(), "smoke-test", api, console.dial, nil, nil, "", clock.now, clock.sleep, noopReadPubKey, okBanner)
307	if err == nil {	307	if err == nil {
308	t.Fatal("runScenario: want error, got nil")	308	t.Fatal("runScenario: want error, got nil")
309	}	309	}
@@ -337,7 +337,7 @@ func TestRunScenarioNeverReadyTimesOut(t *testing.T) {
337	}	337	}
338		338
339	clock := &fakeClock{t: time.Unix(0, 0)}	339	clock := &fakeClock{t: time.Unix(0, 0)}
340	_, err := runScenario(context.Background(), "smoke-test", api, console, nil, nil, clock.now, clock.sleep, noopReadPubKey, okBanner)	340	_, err := runScenario(context.Background(), "smoke-test", api, console, nil, nil, "", clock.now, clock.sleep, noopReadPubKey, okBanner)
341	if err == nil {	341	if err == nil {
342	t.Fatal("runScenario: want error, got nil")	342	t.Fatal("runScenario: want error, got nil")
343	}	343	}
@@ -376,7 +376,7 @@ func TestRunScenarioNeverReapedTimesOut(t *testing.T) {
376	console := powerCycleConsole(t, api, bootedGuest, "ubuntu-vm login: ")	376	console := powerCycleConsole(t, api, bootedGuest, "ubuntu-vm login: ")
377		377
378	clock := &fakeClock{t: time.Unix(0, 0)}	378	clock := &fakeClock{t: time.Unix(0, 0)}
379	_, err := runScenario(context.Background(), "smoke-test", api, console.dial, nil, nil, clock.now, clock.sleep, noopReadPubKey, okBanner)	379	_, err := runScenario(context.Background(), "smoke-test", api, console.dial, nil, nil, "", clock.now, clock.sleep, noopReadPubKey, okBanner)
380	if err == nil {	380	if err == nil {
381	t.Fatal("runScenario: want error, got nil")	381	t.Fatal("runScenario: want error, got nil")
382	}	382	}
@@ -404,7 +404,7 @@ func TestRunScenarioNoHosts(t *testing.T) {
404	},	404	},
405	}	405	}
406	clock := &fakeClock{t: time.Unix(0, 0)}	406	clock := &fakeClock{t: time.Unix(0, 0)}
407	_, err := runScenario(context.Background(), "smoke-test", api, nil, nil, nil, clock.now, clock.sleep, noopReadPubKey, okBanner)	407	_, err := runScenario(context.Background(), "smoke-test", api, nil, nil, nil, "", clock.now, clock.sleep, noopReadPubKey, okBanner)
408	if err == nil {	408	if err == nil {
409	t.Fatal("runScenario: want error, got nil")	409	t.Fatal("runScenario: want error, got nil")
410	}	410	}
@@ -467,7 +467,7 @@ func TestRunScenarioGateRegistersBeforeCreateAndExecsAfterBoot(t *testing.T) {
467	}	467	}
468		468
469	clock := &fakeClock{t: time.Unix(0, 0)}	469	clock := &fakeClock{t: time.Unix(0, 0)}
470	msg, err := runScenario(context.Background(), "smoke-test", api, powerCycleConsole(t, api, bootedGuest, "ubuntu-vm login: ").dial, gate, nil, clock.now, clock.sleep, noopReadPubKey, okBanner)	470	msg, err := runScenario(context.Background(), "smoke-test", api, powerCycleConsole(t, api, bootedGuest, "ubuntu-vm login: ").dial, gate, nil, "", clock.now, clock.sleep, noopReadPubKey, okBanner)
471	if err != nil {	471	if err != nil {
472	t.Fatalf("runScenario: %v", err)	472	t.Fatalf("runScenario: %v", err)
473	}	473	}
@@ -517,7 +517,7 @@ func TestRunScenarioGateRegisterErrorAbortsBeforeCreate(t *testing.T) {
517	}	517	}
518		518
519	clock := &fakeClock{t: time.Unix(0, 0)}	519	clock := &fakeClock{t: time.Unix(0, 0)}
520	_, err := runScenario(context.Background(), "smoke-test", api, nil, gate, nil, clock.now, clock.sleep, noopReadPubKey, okBanner)	520	_, err := runScenario(context.Background(), "smoke-test", api, nil, gate, nil, "", clock.now, clock.sleep, noopReadPubKey, okBanner)
521	if err == nil {	521	if err == nil {
522	t.Fatal("runScenario: want error, got nil")	522	t.Fatal("runScenario: want error, got nil")
523	}	523	}
@@ -541,7 +541,7 @@ func TestRunScenarioGateExecErrorFails(t *testing.T) {
541	}	541	}
542		542
543	clock := &fakeClock{t: time.Unix(0, 0)}	543	clock := &fakeClock{t: time.Unix(0, 0)}
544	_, err := runScenario(context.Background(), "smoke-test", api, powerCycleConsole(t, api, bootedGuest, "ubuntu-vm login: ").dial, gate, nil, clock.now, clock.sleep, noopReadPubKey, okBanner)	544	_, err := runScenario(context.Background(), "smoke-test", api, powerCycleConsole(t, api, bootedGuest, "ubuntu-vm login: ").dial, gate, nil, "", clock.now, clock.sleep, noopReadPubKey, okBanner)
545	if err == nil {	545	if err == nil {
546	t.Fatal("runScenario: want error, got nil")	546	t.Fatal("runScenario: want error, got nil")
547	}	547	}
@@ -618,7 +618,7 @@ func TestRunScenarioRunsTheMCPLegOnItsOwnVM(t *testing.T) {
618	}	618	}
619		619
620	clock := &fakeClock{t: time.Unix(0, 0)}	620	clock := &fakeClock{t: time.Unix(0, 0)}
621	msg, err := runScenario(context.Background(), "smoke-test", api, powerCycleConsole(t, api, bootedGuest, "ubuntu-vm login: ").dial, nil, mcp, clock.now, clock.sleep, noopReadPubKey, okBanner)	621	msg, err := runScenario(context.Background(), "smoke-test", api, powerCycleConsole(t, api, bootedGuest, "ubuntu-vm login: ").dial, nil, mcp, "", clock.now, clock.sleep, noopReadPubKey, okBanner)
622	if err != nil {	622	if err != nil {
623	t.Fatalf("runScenario: %v", err)	623	t.Fatalf("runScenario: %v", err)
624	}	624	}
@@ -638,8 +638,113 @@ func TestRunScenarioFailsWhenTheMCPLegFails(t *testing.T) {
638	mcp := func(context.Context, string) error { return errors.New("FAIL: register refused") }	638	mcp := func(context.Context, string) error { return errors.New("FAIL: register refused") }
639		639
640	clock := &fakeClock{t: time.Unix(0, 0)}	640	clock := &fakeClock{t: time.Unix(0, 0)}
641	_, err := runScenario(context.Background(), "smoke-test", api, powerCycleConsole(t, api, bootedGuest, "ubuntu-vm login: ").dial, nil, mcp, clock.now, clock.sleep, noopReadPubKey, okBanner)	641	_, err := runScenario(context.Background(), "smoke-test", api, powerCycleConsole(t, api, bootedGuest, "ubuntu-vm login: ").dial, nil, mcp, "", clock.now, clock.sleep, noopReadPubKey, okBanner)
642	if err == nil \|\| !strings.Contains(err.Error(), "register refused") {	642	if err == nil \|\| !strings.Contains(err.Error(), "register refused") {
643	t.Fatalf("runScenario err = %v, want the MCP leg's failure to fail the gate", err)	643	t.Fatalf("runScenario err = %v, want the MCP leg's failure to fail the gate", err)
644	}	644	}
645	}	645	}
		646
		647	// --- runScenario: the host must be running the release ----------------------
		648
		649	// TestRunScenarioRefusesAHostBehindTheRelease pins the half of the #33 story
		650	// that lives in the smoke. A ship rolls the plane, converges the fleet, and
		651	// then proves the release here — but the proof is only ever as good as the
		652	// agent the VM lands on, and every leg of this scenario is satisfied just as
		653	// well by the release before this one. Twice a ship reached this point with an
		654	// unconverged fleet and exited green. With an expectation set, it cannot: the
		655	// mismatch fails the run before a VM is created, and says the fleet is what
		656	// went wrong.
		657	func TestRunScenarioRefusesAHostBehindTheRelease(t *testing.T) {
		658	api := &testAPI{
		659	listHostsFunc: func(ctx context.Context) ([]client.Host, error) {
		660	return []client.Host{{ID: "host-1", Name: "onyx", AgentVersion: "v0.0.6"}}, nil
		661	},
		662	createVMFunc: func(ctx context.Context, req client.CreateVMRequest) (client.CreateVMResponse, error) {
		663	t.Fatal("CreateVM must not be called on a host that is not running the release")
		664	return client.CreateVMResponse{}, nil
		665	},
		666	listVMsFunc: func(ctx context.Context) ([]client.VM, error) {
		667	t.Fatal("ListVMs must not be called on a host that is not running the release")
		668	return nil, nil
		669	},
		670	deleteVMFunc: func(ctx context.Context, id string) error {
		671	t.Fatal("DeleteVM must not be called on a host that is not running the release")
		672	return nil
		673	},
		674	}
		675
		676	clock := &fakeClock{t: time.Unix(0, 0)}
		677	_, err := runScenario(context.Background(), "smoke-test", api, nil, nil, nil, "v0.0.7", clock.now, clock.sleep, noopReadPubKey, okBanner)
		678	if err == nil {
		679	t.Fatal("runScenario: want a failure on a host a release behind, got nil")
		680	}
		681	// Both versions, or the reader cannot tell which side is stale.
		682	for _, want := range []string{"onyx", "v0.0.6", "v0.0.7", "did not converge", "--from 8"} {
		683	if !strings.Contains(err.Error(), want) {
		684	t.Errorf("error = %q, want it to mention %q", err.Error(), want)
		685	}
		686	}
		687	}
		688
		689	// TestRunScenarioAcceptsTheHostRunningTheRelease: the expectation met is the
		690	// ordinary case, and it must be invisible — the same one ListHosts call, the
		691	// same COMPLETE line.
		692	func TestRunScenarioAcceptsTheHostRunningTheRelease(t *testing.T) {
		693	var calls []string
		694	api := happyPathAPI(t, &calls)
		695	listHostsCalls := 0
		696	api.listHostsFunc = func(ctx context.Context) ([]client.Host, error) {
		697	listHostsCalls++
		698	return []client.Host{{ID: "host-1", Name: "onyx", UplinkAddr: "192.168.0.190", AgentVersion: "v0.0.7"}}, nil
		699	}
		700
		701	clock := &fakeClock{t: time.Unix(0, 0)}
		702	msg, err := runScenario(context.Background(), "smoke-test", api, powerCycleConsole(t, api, bootedGuest, "ubuntu-vm login: ").dial, nil, nil, "v0.0.7", clock.now, clock.sleep, noopReadPubKey, okBanner)
		703	if err != nil {
		704	t.Fatalf("runScenario: %v", err)
		705	}
		706	if !strings.Contains(msg, "SMOKE COMPLETE") {
		707	t.Errorf("message = %q, want SMOKE COMPLETE", msg)
		708	}
		709	if listHostsCalls != 1 {
		710	t.Errorf("ListHosts calls = %d, want 1 — the check reads the listing the scenario already made", listHostsCalls)
		711	}
		712	}
		713
		714	// TestRunScenarioWithoutAnExpectationJudgesNoVersion: no expectation is no
		715	// check and no extra traffic. This is every run but a ship's — a boot gate
		716	// proves binaries built from a working tree, whose stamped version is "dev".
		717	func TestRunScenarioWithoutAnExpectationJudgesNoVersion(t *testing.T) {
		718	var calls []string
		719	api := happyPathAPI(t, &calls)
		720	listHostsCalls := 0
		721	api.listHostsFunc = func(ctx context.Context) ([]client.Host, error) {
		722	listHostsCalls++
		723	return []client.Host{{ID: "host-1", Name: "onyx", UplinkAddr: "192.168.0.190", AgentVersion: "dev"}}, nil
		724	}
		725
		726	clock := &fakeClock{t: time.Unix(0, 0)}
		727	msg, err := runScenario(context.Background(), "smoke-test", api, powerCycleConsole(t, api, bootedGuest, "ubuntu-vm login: ").dial, nil, nil, "", clock.now, clock.sleep, noopReadPubKey, okBanner)
		728	if err != nil {
		729	t.Fatalf("runScenario: %v", err)
		730	}
		731	if !strings.Contains(msg, "SMOKE COMPLETE") {
		732	t.Errorf("message = %q, want SMOKE COMPLETE", msg)
		733	}
		734	if listHostsCalls != 1 {
		735	t.Errorf("ListHosts calls = %d, want 1 — an expectation-free run asks nothing extra", listHostsCalls)
		736	}
		737	}
		738
		739	// TestProveHostReleaseNamesAnAgentTooOldToNameItself: an agent from before
		740	// versions were reported says nothing at all, and "" in the middle of a
		741	// sentence reads as a bug in the message rather than a fact about the host.
		742	func TestProveHostReleaseNamesAnAgentTooOldToNameItself(t *testing.T) {
		743	err := proveHostRelease(client.Host{ID: "host-1", Name: "onyx"}, "v0.0.7")
		744	if err == nil {
		745	t.Fatal("proveHostRelease: want a failure for a host reporting no version, got nil")
		746	}
		747	if !strings.Contains(err.Error(), "no version at all") {
		748	t.Errorf("error = %q, want it to say the host reports no version at all", err.Error())
		749	}
		750	}

scripts/ship.sh

Old		New
@@ -628,6 +628,18 @@ if [[ "$FROM" -le 8 && "$HAVE_CREDENTIAL" == "1" ]]; then
628	CONVERGE_TIMEOUT_S=300	628	CONVERGE_TIMEOUT_S=300
629	CONVERGE_POLL_S=5	629	CONVERGE_POLL_S=5
630		630
		631	# How long a DARK host is waited on before it is written off as parked. This
		632	# stage runs seconds after stage 7 rolled the server, and that roll ended
		633	# every agent's sync session at once — so for as long as it takes the new
		634	# pod to serve and the agents to come back (a 5s reconnect backoff, several
		635	# attempts while the plane is still starting), the whole fleet reads as
		636	# disconnected here. A host that is dark inside this window is waited on, not
		637	# skipped; past it, the warn-and-skip below stands, because a genuinely
		638	# parked machine must never hold a release. 90s is the roll plus a handful of
		639	# backoffs, and well inside CONVERGE_TIMEOUT_S so the grace can never be what
		640	# ends the loop.
		641	DARK_GRACE_S=90
		642
631	API="https://$CONSOLE_HOST"	643	API="https://$CONSOLE_HOST"
632		644
633	# The credential is the one the plane already names for the smoke, chosen the	645	# The credential is the one the plane already names for the smoke, chosen the
@@ -662,15 +674,47 @@ if [[ "$FROM" -le 8 && "$HAVE_CREDENTIAL" == "1" ]]; then
662	# to the URL we landed on. Success is the session cookie in the jar and	674	# to the URL we landed on. Success is the session cookie in the jar and
663	# nothing else — a wrong password re-renders the form as a plain 200.	675	# nothing else — a wrong password re-renders the form as a plain 200.
664	(umask 077; printf '%s' "$(cat "$CI_PASSWORD_FILE")" >"$cdir/pw")	676	(umask 077; printf '%s' "$(cat "$CI_PASSWORD_FILE")" >"$cdir/pw")
665	form_url="$(curl -sS -L --max-time 30 -c "$cdir/jar" -b "$cdir/jar" \	677
666	-o /dev/null -w '%{url_effective}' "$API/auth/login")" \|\|	678	# Retried, because stage 7 rolled BOTH the server and the issuer seconds
667	fail "GET $API/auth/login did not answer, so nothing can sign in to the plane just rolled"	679	# ago and a sign-in is only as settled as the pair of them: an issuer still
668	curl -sS -L --max-time 30 -c "$cdir/jar" -b "$cdir/jar" -o /dev/null \	680	# coming up walks the redirect chain and sets no session at the end of it —
669	--data-urlencode "email=$CI_USER" --data-urlencode "password@$cdir/pw" "$form_url" \|\|	681	# which is, by the line above, indistinguishable from a refused password.
670	fail "posting $CI_USER's credentials to the issuer's login form at $form_url failed"	682	# So the retry is blind on purpose. There is nothing in the answer to tell
671	grep -q eitri_session "$cdir/jar" 2>/dev/null \|\|	683	# a plane that is not ready from a password that is wrong, and the wrong
672	fail "signing in as $CI_USER left no session — check the identity exists in this	684	# password costs a minute before printing the same failure this stage
		685	# always printed. A stg ship lost a run to that race and then signed in,
		686	# unchanged, a minute later.
		687	SIGNIN_ATTEMPTS=6
		688	SIGNIN_RETRY_S=10
		689	signin_attempt=1
		690	while :; do
		691	# A jar from a failed attempt must never be what the next one reads:
		692	# the session cookie in it is the only thing that says "signed in".
		693	rm -f "$cdir/jar"
		694	signin_err=""
		695	form_url="$(curl -sS -L --max-time 30 -c "$cdir/jar" -b "$cdir/jar" \
		696	-o /dev/null -w '%{url_effective}' "$API/auth/login")" \|\|
		697	signin_err="GET $API/auth/login did not answer, so nothing can sign in to the plane just rolled"
		698	if [[ -z "$signin_err" ]]; then
		699	curl -sS -L --max-time 30 -c "$cdir/jar" -b "$cdir/jar" -o /dev/null \
		700	--data-urlencode "email=$CI_USER" --data-urlencode "password@$cdir/pw" "$form_url" \|\|
		701	signin_err="posting $CI_USER's credentials to the issuer's login form at $form_url failed"
		702	fi
		703	if [[ -z "$signin_err" ]]; then
		704	grep -q eitri_session "$cdir/jar" 2>/dev/null \|\|
		705	signin_err="signing in as $CI_USER left no session — check the identity exists in this
673	plane's issuer and that CI_PASSWORD_FILE holds its current password."	706	plane's issuer and that CI_PASSWORD_FILE holds its current password."
		707	fi
		708	[[ -n "$signin_err" ]] \|\| break
		709	[[ "$signin_attempt" -lt "$SIGNIN_ATTEMPTS" ]] \|\|
		710	fail "$signin_err
		711	This was attempt $signin_attempt of $SIGNIN_ATTEMPTS, over $(( (SIGNIN_ATTEMPTS - 1) * SIGNIN_RETRY_S ))s, so the issuer has
		712	had time to settle after the stage-7 roll and the answer is not changing."
		713	echo " sign-in attempt $signin_attempt of $SIGNIN_ATTEMPTS did not take — the issuer and the server were rolled"
		714	echo " seconds ago and a plane still starting cannot hand out a session; retrying in ${SIGNIN_RETRY_S}s"
		715	signin_attempt=$((signin_attempt + 1))
		716	sleep "$SIGNIN_RETRY_S"
		717	done
674	# The session in the jar IS the credential from here: the API takes the	718	# The session in the jar IS the credential from here: the API takes the
675	# console cookie exactly where it takes a bearer token, so this stage needs	719	# console cookie exactly where it takes a bearer token, so this stage needs
676	# no PAT of its own and leaves no token behind it.	720	# no PAT of its own and leaves no token behind it.
@@ -744,12 +788,27 @@ if [[ "$FROM" -le 8 && "$HAVE_CREDENTIAL" == "1" ]]; then
744	continue	788	continue
745	fi	789	fi
746		790
747	# A dark host blocks nothing. It cannot be handed an offer — the plane	791	# A host that is STILL dark once the grace has passed blocks nothing. It
748	# refuses one outright — and the smoke cannot place a VM on it either,	792	# cannot be handed an offer — the plane refuses one outright — and the
749	# so waiting for it would hold a release for a machine that has no part	793	# smoke cannot place a VM on it either, so waiting for it would hold a
750	# in proving it. It comes back to a fleet a release ahead of it and the	794	# release for a machine that has no part in proving it. It comes back to
751	# console offers it the upgrade then.	795	# a fleet a release ahead of it and the console offers it the upgrade
		796	# then.
		797	#
		798	# Inside the grace, though, dark means nothing at all. Stage 7 rolled the
		799	# server moments ago and every agent's sync session went with it, so the
		800	# first listing after it shows a fleet that is entirely dark. Writing
		801	# hosts off on that listing is how a release shipped with ZERO converged
		802	# hosts and still exited green — twice, v0.0.6 and v0.0.7 on prod, both
		803	# recovered by re-running --from 8 once the agents were back. So a host
		804	# that is dark this early is waited on exactly like one with an offer
		805	# outstanding: the loop polls on, and when it reconnects it flows down
		806	# this same list and is offered $TAG like any other host.
752	if [[ "$online" != "true" ]]; then	807	if [[ "$online" != "true" ]]; then
		808	if [[ $((SECONDS - started)) -lt $DARK_GRACE_S ]]; then
		809	waiting="$waiting $name (dark; agents reconnect after the stage-7 roll)"
		810	continue
		811	fi
753	case "$skipped" in	812	case "$skipped" in
754	" $id ") ;;	813	" $id ") ;;
755	*)	814	*)
@@ -922,11 +981,19 @@ if [[ "$FROM" -le 9 && "$SKIP_SMOKE" != "1" ]]; then
922	# and authenticates with nothing in front of it.	981	# and authenticates with nothing in front of it.
923	SMOKE_MCP_URL="${SMOKE_MCP_URL:-https://$CONSOLE_HOST https://$API_HOST}"	982	SMOKE_MCP_URL="${SMOKE_MCP_URL:-https://$CONSOLE_HOST https://$API_HOST}"
924	SMOKE_USER_CA_FILE="${SMOKE_USER_CA_FILE:-$HOME/eitri-deploy/$TARGET/smoke_user_ca}"	983	SMOKE_USER_CA_FILE="${SMOKE_USER_CA_FILE:-$HOME/eitri-deploy/$TARGET/smoke_user_ca}"
		984	# SMOKE_EXPECT_AGENT_VERSION is what makes this a proof OF $TAG rather than a
		985	# proof of whatever the fleet happens to be running: the smoke refuses to
		986	# place its VM on a host whose agent is not the release being shipped. A ship
		987	# has twice reached this stage with an unconverged fleet, and the smoke passed
		988	# both times — every leg it proves, a release-old agent satisfies just as well,
		989	# so the run exited green with the plane at $TAG and the hosts a release
		990	# behind. Stage 8 is what should prevent that; this is what catches it.
925	smoke_env=(	991	smoke_env=(
926	SERVER_URL="https://$CONSOLE_HOST"	992	SERVER_URL="https://$CONSOLE_HOST"
927	SMOKE_GATE="$GATE_HOST:$GATE_PORT"	993	SMOKE_GATE="$GATE_HOST:$GATE_PORT"
928	SMOKE_MCP_URL="$SMOKE_MCP_URL"	994	SMOKE_MCP_URL="$SMOKE_MCP_URL"
929	SMOKE_USER_CA_FILE="$SMOKE_USER_CA_FILE"	995	SMOKE_USER_CA_FILE="$SMOKE_USER_CA_FILE"
		996	SMOKE_EXPECT_AGENT_VERSION="$TAG"
930	)	997	)
931	if [[ -n "${CI_USER:-}" ]]; then	998	if [[ -n "${CI_USER:-}" ]]; then
932	: "${CI_PASSWORD_FILE:?set it alongside CI_USER in $SHIP_ENV}"	999	: "${CI_PASSWORD_FILE:?set it alongside CI_USER in $SHIP_ENV}"