a73x

82572684

fix(agent): declare disk images raw so the GPT stays writable

a73x   2026-07-30 12:19

Commit message
fix(agent): declare disk images raw so the GPT stays writable

Autodetected raw makes cloud-hypervisor disable sector 0 writes: the first
boot's growpart then grows the partition only in the kernel's memory, and
the guest dies in initramfs at its first power cycle. image_type=raw on
both disk entries keeps the partition table writable.

The smoke now power-cycles its throwaway VM — stop, start, fresh boot
proof, gate SSH again — because a first boot can look green on state that
never reached the disk.
internal/agent/cloudhv/cloudhv.go
Old New
@@ -91,9 +91,14 @@ func (p *Provisioner) buildArgs(spec state.VMSpec) []string {
91 "--kernel", p.firmware, 91 "--kernel", p.firmware,
92 "--cpus", fmt.Sprintf("boot=%d", spec.VCPUs), 92 "--cpus", fmt.Sprintf("boot=%d", spec.VCPUs),
93 "--memory", fmt.Sprintf("size=%dM", spec.MemMB), 93 "--memory", fmt.Sprintf("size=%dM", spec.MemMB),
94 // image_type=raw is load-bearing, not decoration: autodetected raw
95 // makes CH DISABLE SECTOR 0 WRITES, so the first boot's growpart
96 // rewrites the partition table only in memory and the guest dies in
97 // initramfs at its first power cycle. Declared raw keeps the GPT
98 // writable. (Autodetection is also deprecated in CH v53.)
94 "--disk", 99 "--disk",
95 fmt.Sprintf("path=%s", p.st.DiskPath(vmID)), 100 fmt.Sprintf("path=%s,image_type=raw", p.st.DiskPath(vmID)),
96 fmt.Sprintf("path=%s,readonly=on", p.st.SeedPath(vmID)), 101 fmt.Sprintf("path=%s,image_type=raw,readonly=on", p.st.SeedPath(vmID)),
97 "--net", fmt.Sprintf("tap=%s,mac=%s", tap, mac), 102 "--net", fmt.Sprintf("tap=%s,mac=%s", tap, mac),
98 "--serial", fmt.Sprintf("socket=%s", p.st.SerialSocketPath(vmID)), 103 "--serial", fmt.Sprintf("socket=%s", p.st.SerialSocketPath(vmID)),
99 "--console", "off", 104 "--console", "off",
internal/agent/cloudhv/cloudhv_test.go
Old New
@@ -39,6 +39,21 @@ func TestBuildArgs(t *testing.T) {
39 assert.Contains(t, joined, "tap=eit-vm1,mac="+state.MAC("vm1")) 39 assert.Contains(t, joined, "tap=eit-vm1,mac="+state.MAC("vm1"))
40 } 40 }
41 41
42 // TestBuildArgsDeclaresRawImageType pins image_type=raw on BOTH disk entries.
43 // Left to autodetection, CH treats "raw" as a guess and DISABLES SECTOR 0
44 // WRITES — so the first boot's growpart rewrites the partition table only in
45 // memory, and the VM lands in initramfs emergency mode at its first power
46 // cycle (on-disk GPT still image-sized, filesystem inside already grown).
47 func TestBuildArgsDeclaresRawImageType(t *testing.T) {
48 st, err := state.Open(t.TempDir())
49 require.NoError(t, err)
50 p := New(st, "ch", "fw", nil)
51 args := p.buildArgs(state.VMSpec{VMID: "vm1", VCPUs: 1, MemMB: 512})
52 joined := strings.Join(args, " ")
53 assert.Contains(t, joined, st.DiskPath("vm1")+",image_type=raw")
54 assert.Contains(t, joined, st.SeedPath("vm1")+",image_type=raw,readonly=on")
55 }
56
42 func TestBuildArgsUsesSerialSocket(t *testing.T) { 57 func TestBuildArgsUsesSerialSocket(t *testing.T) {
43 st, err := state.Open(t.TempDir()) 58 st, err := state.Open(t.TempDir())
44 require.NoError(t, err) 59 require.NoError(t, err)
internal/server/api/client/client.go
Old New
@@ -31,6 +31,7 @@ type (
31 VM = types.VM 31 VM = types.VM
32 CreateVMRequest = types.CreateVMRequest 32 CreateVMRequest = types.CreateVMRequest
33 CreateVMResponse = types.CreateVMResponse 33 CreateVMResponse = types.CreateVMResponse
34 PatchVMRequest = types.PatchVMRequest
34 Me = types.Me 35 Me = types.Me
35 CreateAPITokenResponse = types.CreateAPITokenResponse 36 CreateAPITokenResponse = types.CreateAPITokenResponse
36 UserCA = types.UserCA 37 UserCA = types.UserCA
@@ -138,6 +139,12 @@ func (c *Client) DeleteVM(ctx context.Context, id string) error {
138 return c.do(ctx, http.MethodDelete, "/api/v1/vms/"+url.PathEscape(id), nil, nil) 139 return c.do(ctx, http.MethodDelete, "/api/v1/vms/"+url.PathEscape(id), nil, nil)
139 } 140 }
140 141
142 // PatchVM sets the VM's desired power state ("running" or "stopped"); the
143 // agent actuates it asynchronously — poll ActualPower for the outcome.
144 func (c *Client) PatchVM(ctx context.Context, id, powerState string) error {
145 return c.do(ctx, http.MethodPatch, "/api/v1/vms/"+url.PathEscape(id), PatchVMRequest{PowerState: powerState}, nil)
146 }
147
141 // Me returns the signed-in identity (tenant handle + bound email) for the 148 // Me returns the signed-in identity (tenant handle + bound email) for the
142 // credential this client carries. It takes no context — the consumers (smoke 149 // credential this client carries. It takes no context — the consumers (smoke
143 // gate, CLI) call it as a quick synchronous probe; the do timeout bounds it. 150 // gate, CLI) call it as a quick synchronous probe; the do timeout bounds it.
internal/server/api/client/client_test.go
Old New
@@ -145,6 +145,22 @@ func TestDeleteVMEscapesID(t *testing.T) {
145 } 145 }
146 } 146 }
147 147
148 func TestPatchVM(t *testing.T) {
149 var cap capture
150 srv := serve(t, &cap, http.StatusOK, "{}")
151 c := &client.Client{BaseURL: srv.URL, Token: "tok"}
152
153 if err := c.PatchVM(context.Background(), "vm-123", "stopped"); err != nil {
154 t.Fatalf("PatchVM: %v", err)
155 }
156 if cap.method != http.MethodPatch || cap.path != "/api/v1/vms/vm-123" {
157 t.Errorf("request = %s %s, want PATCH /api/v1/vms/vm-123", cap.method, cap.path)
158 }
159 if !strings.Contains(string(cap.body), `"power_state":"stopped"`) {
160 t.Errorf("body = %s, want power_state stopped", cap.body)
161 }
162 }
163
148 func TestNoAuthHeaderWhenTokenEmpty(t *testing.T) { 164 func TestNoAuthHeaderWhenTokenEmpty(t *testing.T) {
149 var cap capture 165 var cap capture
150 srv := serve(t, &cap, http.StatusOK, `[]`) 166 srv := serve(t, &cap, http.StatusOK, `[]`)
internal/smoke/scenario.go
Old New
@@ -32,6 +32,38 @@ func bootProofCommand(agentStateDir, vmID string) string {
32 return fmt.Sprintf(`sudo cat '%s/vms/%s/serial.log' 2>/dev/null | tr -cd '\11\12\15\40-\176'`, agentStateDir, vmID) 32 return fmt.Sprintf(`sudo cat '%s/vms/%s/serial.log' 2>/dev/null | tr -cd '\11\12\15\40-\176'`, agentStateDir, vmID)
33 } 33 }
34 34
35 // truncateSerialCommand empties a VM's serial log on the agent host. The log
36 // deliberately survives VM restarts, so a post-restart boot proof would pass
37 // on the FIRST boot's bytes unless it starts from an empty file.
38 func truncateSerialCommand(agentStateDir, vmID string) string {
39 return fmt.Sprintf(`sudo truncate -s 0 '%s/vms/%s/serial.log'`, agentStateDir, vmID)
40 }
41
42 // proveBoot polls the VM's serial log over SSH until it shows a userspace
43 // boot (login prompt), a panic/root-mount failure (immediate FAIL), or the
44 // deadline passes. phase names which boot is being proven in the failure
45 // messages ("first boot", "after power cycle").
46 func proveBoot(ctx context.Context, cfg Config, vmID string, runSSH sshFunc, now func() time.Time, sleep func(time.Duration), phase string) error {
47 remoteCmd := bootProofCommand(cfg.AgentStateDir, vmID)
48 err := pollLoop(ctx, now, sleep, 180*time.Second, 6*time.Second, func() (bool, error) {
49 serial, sshErr := runSSH(ctx, remoteCmd)
50 if sshErr != nil {
51 // Match the bash scenario's "|| true": an SSH hiccup mid-boot is
52 // not fatal on its own, just an empty-serial iteration.
53 serial = ""
54 }
55 booted, panicked := classifySerial(serial)
56 if panicked {
57 return false, fmt.Errorf("FAIL: guest panic / root-mount failure in serial log (%s)", phase)
58 }
59 return booted, nil
60 })
61 if errors.Is(err, errPollTimeout) {
62 return fmt.Errorf("FAIL: no userspace boot evidence in serial within 180s (%s)", phase)
63 }
64 return err
65 }
66
35 // vmAPI is the subset of the shared API client the scenario needs. Declaring 67 // vmAPI is the subset of the shared API client the scenario needs. Declaring
36 // it lets tests supply a fake instead of a real HTTP-backed *client.Client. 68 // it lets tests supply a fake instead of a real HTTP-backed *client.Client.
37 type vmAPI interface { 69 type vmAPI interface {
@@ -39,6 +71,7 @@ type vmAPI interface {
39 CreateVM(ctx context.Context, req client.CreateVMRequest) (client.CreateVMResponse, error) 71 CreateVM(ctx context.Context, req client.CreateVMRequest) (client.CreateVMResponse, error)
40 ListVMs(ctx context.Context) ([]client.VM, error) 72 ListVMs(ctx context.Context) ([]client.VM, error)
41 DeleteVM(ctx context.Context, id string) error 73 DeleteVM(ctx context.Context, id string) error
74 PatchVM(ctx context.Context, id, powerState string) error
42 } 75 }
43 76
44 // getVM finds the VM with the given id in the current listing. The bool 77 // getVM finds the VM with the given id in the current listing. The bool
@@ -142,33 +175,65 @@ func runScenario(ctx context.Context, cfg Config, vmName string, c vmAPI, runSSH
142 } 175 }
143 coldStart := now().Sub(start) 176 coldStart := now().Sub(start)
144 177
145 remoteCmd := bootProofCommand(cfg.AgentStateDir, vmID) 178 if err := proveBoot(ctx, cfg, vmID, runSSH, now, sleep, "first boot"); err != nil {
146 err = pollLoop(ctx, now, sleep, 180*time.Second, 6*time.Second, func() (bool, error) { 179 return "", err
147 serial, sshErr := runSSH(ctx, remoteCmd) 180 }
148 if sshErr != nil { 181
149 // Match the bash scenario's "|| true": an SSH hiccup mid-boot is 182 gateOK := false
150 // not fatal on its own, just an empty-serial iteration. 183 if gate != nil {
151 serial = "" 184 if err := gate.exec(ctx, vmName); err != nil {
185 return "", err
152 } 186 }
153 booted, panicked := classifySerial(serial) 187 gateOK = true
154 if panicked { 188 }
155 return false, errors.New("FAIL: guest panic / root-mount failure in serial log") 189
190 // Power cycle: prove the guest comes BACK. A first boot runs on the
191 // kernel's in-memory partition table; only a stop→start proves the
192 // on-disk GPT survived growpart. The sector-0 regression hid exactly
193 // here — every first boot green, every persistent VM lost at its first
194 // reboot.
195 if err := c.PatchVM(ctx, vmID, "stopped"); err != nil {
196 return "", fmt.Errorf("patch vm stopped: %w", err)
197 }
198 err = pollLoop(ctx, now, sleep, 120*time.Second, 5*time.Second, func() (bool, error) {
199 vm, present, err := getVM(ctx, c, vmID)
200 if err != nil {
201 return false, fmt.Errorf("poll vm stopped: %w", err)
156 } 202 }
157 return booted, nil 203 return present && vm.ActualPower == "stopped", nil
158 }) 204 })
159 if err != nil { 205 if err != nil {
160 if errors.Is(err, errPollTimeout) { 206 if errors.Is(err, errPollTimeout) {
161 return "", errors.New("FAIL: no userspace boot evidence in serial within 180s") 207 return "", errors.New("FAIL: VM did not power off within 120s of the power cycle's stop")
162 } 208 }
163 return "", err 209 return "", err
164 } 210 }
165 211 if _, err := runSSH(ctx, truncateSerialCommand(cfg.AgentStateDir, vmID)); err != nil {
166 gateOK := false 212 return "", fmt.Errorf("truncate serial log before reboot proof: %w", err)
213 }
214 if err := c.PatchVM(ctx, vmID, "running"); err != nil {
215 return "", fmt.Errorf("patch vm running: %w", err)
216 }
217 err = pollLoop(ctx, now, sleep, 120*time.Second, 5*time.Second, func() (bool, error) {
218 vm, present, err := getVM(ctx, c, vmID)
219 if err != nil {
220 return false, fmt.Errorf("poll vm restarted: %w", err)
221 }
222 return present && vm.ActualPower == "running", nil
223 })
224 if err != nil {
225 if errors.Is(err, errPollTimeout) {
226 return "", errors.New("FAIL: VM did not power on within 120s of the power cycle's start")
227 }
228 return "", err
229 }
230 if err := proveBoot(ctx, cfg, vmID, runSSH, now, sleep, "after power cycle"); err != nil {
231 return "", err
232 }
167 if gate != nil { 233 if gate != nil {
168 if err := gate.exec(ctx, vmName); err != nil { 234 if err := gate.exec(ctx, vmName); err != nil {
169 return "", err 235 return "", fmt.Errorf("gate SSH after power cycle: %w", err)
170 } 236 }
171 gateOK = true
172 } 237 }
173 238
174 if err := c.DeleteVM(ctx, vmID); err != nil { 239 if err := c.DeleteVM(ctx, vmID); err != nil {
@@ -188,7 +253,7 @@ func runScenario(ctx context.Context, cfg Config, vmName string, c vmAPI, runSSH
188 return "", err 253 return "", err
189 } 254 }
190 255
191 msg := fmt.Sprintf("SMOKE COMPLETE — booted under UEFI, cold_start=%ds, reaped OK", int64(coldStart.Seconds())) 256 msg := fmt.Sprintf("SMOKE COMPLETE — booted under UEFI, cold_start=%ds, reboot: ok, reaped OK", int64(coldStart.Seconds()))
192 if gateOK { 257 if gateOK {
193 msg += ", gate SSH: ok" 258 msg += ", gate SSH: ok"
194 } 259 }
internal/smoke/scenario_test.go
Old New
@@ -54,12 +54,14 @@ type fakeClock struct{ t time.Time }
54 func (c *fakeClock) now() time.Time { return c.t } 54 func (c *fakeClock) now() time.Time { return c.t }
55 func (c *fakeClock) sleep(d time.Duration) { c.t = c.t.Add(d) } 55 func (c *fakeClock) sleep(d time.Duration) { c.t = c.t.Add(d) }
56 56
57 // testAPI implements vmAPI by delegating to per-test closures. 57 // testAPI implements vmAPI by delegating to per-test closures. patchVMFunc
58 // defaults to accepting any power-state change when nil.
58 type testAPI struct { 59 type testAPI struct {
59 listHostsFunc func(ctx context.Context) ([]client.Host, error) 60 listHostsFunc func(ctx context.Context) ([]client.Host, error)
60 createVMFunc func(ctx context.Context, req client.CreateVMRequest) (client.CreateVMResponse, error) 61 createVMFunc func(ctx context.Context, req client.CreateVMRequest) (client.CreateVMResponse, error)
61 listVMsFunc func(ctx context.Context) ([]client.VM, error) 62 listVMsFunc func(ctx context.Context) ([]client.VM, error)
62 deleteVMFunc func(ctx context.Context, id string) error 63 deleteVMFunc func(ctx context.Context, id string) error
64 patchVMFunc func(ctx context.Context, id, powerState string) error
63 } 65 }
64 66
65 func (a *testAPI) ListHosts(ctx context.Context) ([]client.Host, error) { 67 func (a *testAPI) ListHosts(ctx context.Context) ([]client.Host, error) {
@@ -72,6 +74,12 @@ func (a *testAPI) ListVMs(ctx context.Context) ([]client.VM, error) {
72 return a.listVMsFunc(ctx) 74 return a.listVMsFunc(ctx)
73 } 75 }
74 func (a *testAPI) DeleteVM(ctx context.Context, id string) error { return a.deleteVMFunc(ctx, id) } 76 func (a *testAPI) DeleteVM(ctx context.Context, id string) error { return a.deleteVMFunc(ctx, id) }
77 func (a *testAPI) PatchVM(ctx context.Context, id, powerState string) error {
78 if a.patchVMFunc == nil {
79 return nil
80 }
81 return a.patchVMFunc(ctx, id, powerState)
82 }
75 83
76 func baseCfg() Config { 84 func baseCfg() Config {
77 return Config{AgentStateDir: "/var/lib/eitri-agent", AgentUserHost: "ubuntu@10.0.0.5", AgentPort: 22} 85 return Config{AgentStateDir: "/var/lib/eitri-agent", AgentUserHost: "ubuntu@10.0.0.5", AgentPort: 22}
@@ -83,6 +91,8 @@ func noopReadPubKey() string { return "ssh-ed25519 AAAAfake test@smoke" }
83 91
84 func TestRunScenarioSuccess(t *testing.T) { 92 func TestRunScenarioSuccess(t *testing.T) {
85 listVMsCalls := 0 93 listVMsCalls := 0
94 power := "running" // actual power the fake reports; PatchVM moves it
95 deleted := false
86 api := &testAPI{ 96 api := &testAPI{
87 listHostsFunc: func(ctx context.Context) ([]client.Host, error) { 97 listHostsFunc: func(ctx context.Context) ([]client.Host, error) {
88 return []client.Host{{ID: "host-1"}}, nil 98 return []client.Host{{ID: "host-1"}}, nil
@@ -96,30 +106,43 @@ func TestRunScenarioSuccess(t *testing.T) {
96 listVMsFunc: func(ctx context.Context) ([]client.VM, error) { 106 listVMsFunc: func(ctx context.Context) ([]client.VM, error) {
97 listVMsCalls++ 107 listVMsCalls++
98 switch { 108 switch {
109 case deleted:
110 // Reap poll: absent from the first check.
111 return nil, nil
99 case listVMsCalls < 3: 112 case listVMsCalls < 3:
100 return []client.VM{{ID: "vm-1", Phase: "booting"}}, nil 113 return []client.VM{{ID: "vm-1", Phase: "booting"}}, nil
101 case listVMsCalls == 3:
102 return []client.VM{{ID: "vm-1", Phase: "ready", AssignedIP: "10.0.0.9"}}, nil
103 default: 114 default:
104 // Reap poll: absent from the first check. 115 return []client.VM{{ID: "vm-1", Phase: "ready", AssignedIP: "10.0.0.9", ActualPower: power}}, nil
105 return nil, nil
106 } 116 }
107 }, 117 },
118 patchVMFunc: func(ctx context.Context, id, powerState string) error {
119 if id != "vm-1" {
120 t.Errorf("patchVM id = %q, want vm-1", id)
121 }
122 power = powerState // the fake actuates instantly
123 return nil
124 },
108 deleteVMFunc: func(ctx context.Context, id string) error { 125 deleteVMFunc: func(ctx context.Context, id string) error {
109 if id != "vm-1" { 126 if id != "vm-1" {
110 t.Errorf("deleteVM id = %q, want vm-1", id) 127 t.Errorf("deleteVM id = %q, want vm-1", id)
111 } 128 }
129 deleted = true
112 return nil 130 return nil
113 }, 131 },
114 } 132 }
115 133
116 sshCalls := 0 134 sshCalls := 0
135 sawTruncate := false
117 runSSH := func(ctx context.Context, remoteCmd string) (string, error) { 136 runSSH := func(ctx context.Context, remoteCmd string) (string, error) {
118 sshCalls++ 137 sshCalls++
119 if !strings.Contains(remoteCmd, "vm-1") { 138 if !strings.Contains(remoteCmd, "vm-1") {
120 t.Errorf("remoteCmd = %q, want it to reference vm-1", remoteCmd) 139 t.Errorf("remoteCmd = %q, want it to reference vm-1", remoteCmd)
121 } 140 }
122 if sshCalls < 2 { 141 if strings.Contains(remoteCmd, "truncate") {
142 sawTruncate = true
143 return "", nil
144 }
145 if !sawTruncate && sshCalls < 2 {
123 return "[ 0.1] Booting Linux...", nil 146 return "[ 0.1] Booting Linux...", nil
124 } 147 }
125 return "Ubuntu 24.04 LTS ubuntu-vm login: ", nil 148 return "Ubuntu 24.04 LTS ubuntu-vm login: ", nil
@@ -136,6 +159,61 @@ func TestRunScenarioSuccess(t *testing.T) {
136 if !strings.Contains(msg, "cold_start=") { 159 if !strings.Contains(msg, "cold_start=") {
137 t.Errorf("message = %q, want cold_start=", msg) 160 t.Errorf("message = %q, want cold_start=", msg)
138 } 161 }
162 if !strings.Contains(msg, "reboot: ok") {
163 t.Errorf("message = %q, want reboot: ok", msg)
164 }
165 if !sawTruncate {
166 t.Error("serial log was never truncated before the reboot proof — proof could pass on first-boot bytes")
167 }
168 }
169
170 // TestRunScenarioRebootDeathFails pins the regression that motivated the
171 // power-cycle leg: a VM whose second boot never reaches userspace (stale
172 // on-disk GPT → initramfs emergency mode) must FAIL the smoke, not pass on
173 // its green first boot.
174 func TestRunScenarioRebootDeathFails(t *testing.T) {
175 power := "running"
176 api := &testAPI{
177 listHostsFunc: func(ctx context.Context) ([]client.Host, error) {
178 return []client.Host{{ID: "host-1"}}, nil
179 },
180 createVMFunc: func(ctx context.Context, req client.CreateVMRequest) (client.CreateVMResponse, error) {
181 return client.CreateVMResponse{ID: "vm-1"}, nil
182 },
183 listVMsFunc: func(ctx context.Context) ([]client.VM, error) {
184 return []client.VM{{ID: "vm-1", Phase: "ready", AssignedIP: "10.0.0.9", ActualPower: power}}, nil
185 },
186 patchVMFunc: func(ctx context.Context, id, powerState string) error {
187 power = powerState
188 return nil
189 },
190 deleteVMFunc: func(ctx context.Context, id string) error {
191 t.Fatal("DeleteVM must not be called when the reboot proof failed")
192 return nil
193 },
194 }
195
196 sawTruncate := false
197 runSSH := func(ctx context.Context, remoteCmd string) (string, error) {
198 if strings.Contains(remoteCmd, "truncate") {
199 sawTruncate = true
200 return "", nil
201 }
202 if sawTruncate {
203 // Post-reboot serial: emergency shell, never a login prompt.
204 return "Press Enter for system maintenance", nil
205 }
206 return "Ubuntu 24.04 LTS ubuntu-vm login: ", nil
207 }
208
209 clock := &fakeClock{t: time.Unix(0, 0)}
210 _, err := runScenario(context.Background(), baseCfg(), "smoke-test", api, runSSH, nil, clock.now, clock.sleep, noopReadPubKey)
211 if err == nil {
212 t.Fatal("runScenario: want error for a VM that never came back, got nil")
213 }
214 if !strings.Contains(err.Error(), "power cycle") {
215 t.Errorf("error = %q, want it to mention the power cycle", err.Error())
216 }
139 } 217 }
140 218
141 // --- runScenario: serial panic ----------------------------------------- 219 // --- runScenario: serial panic -----------------------------------------
@@ -209,6 +287,7 @@ func TestRunScenarioNeverReadyTimesOut(t *testing.T) {
209 // --- runScenario: never-reaped timeout --------------------------------- 287 // --- runScenario: never-reaped timeout ---------------------------------
210 288
211 func TestRunScenarioNeverReapedTimesOut(t *testing.T) { 289 func TestRunScenarioNeverReapedTimesOut(t *testing.T) {
290 power := "running"
212 api := &testAPI{ 291 api := &testAPI{
213 listHostsFunc: func(ctx context.Context) ([]client.Host, error) { 292 listHostsFunc: func(ctx context.Context) ([]client.Host, error) {
214 return []client.Host{{ID: "host-1"}}, nil 293 return []client.Host{{ID: "host-1"}}, nil
@@ -218,7 +297,11 @@ func TestRunScenarioNeverReapedTimesOut(t *testing.T) {
218 }, 297 },
219 listVMsFunc: func(ctx context.Context) ([]client.VM, error) { 298 listVMsFunc: func(ctx context.Context) ([]client.VM, error) {
220 // Always present, even after delete — models a stuck reap. 299 // Always present, even after delete — models a stuck reap.
221 return []client.VM{{ID: "vm-1", Phase: "ready", AssignedIP: "10.0.0.9"}}, nil 300 return []client.VM{{ID: "vm-1", Phase: "ready", AssignedIP: "10.0.0.9", ActualPower: power}}, nil
301 },
302 patchVMFunc: func(ctx context.Context, id, powerState string) error {
303 power = powerState
304 return nil
222 }, 305 },
223 deleteVMFunc: func(ctx context.Context, id string) error { return nil }, 306 deleteVMFunc: func(ctx context.Context, id string) error { return nil },
224 } 307 }
@@ -268,6 +351,8 @@ func TestRunScenarioNoHosts(t *testing.T) {
268 func happyPathAPI(t *testing.T, calls *[]string) *testAPI { 351 func happyPathAPI(t *testing.T, calls *[]string) *testAPI {
269 t.Helper() 352 t.Helper()
270 listVMsCalls := 0 353 listVMsCalls := 0
354 power := "running"
355 deleted := false
271 return &testAPI{ 356 return &testAPI{
272 listHostsFunc: func(ctx context.Context) ([]client.Host, error) { 357 listHostsFunc: func(ctx context.Context) ([]client.Host, error) {
273 return []client.Host{{ID: "host-1"}}, nil 358 return []client.Host{{ID: "host-1"}}, nil
@@ -279,15 +364,22 @@ func happyPathAPI(t *testing.T, calls *[]string) *testAPI {
279 listVMsFunc: func(ctx context.Context) ([]client.VM, error) { 364 listVMsFunc: func(ctx context.Context) ([]client.VM, error) {
280 listVMsCalls++ 365 listVMsCalls++
281 switch { 366 switch {
367 case deleted:
368 return nil, nil
282 case listVMsCalls < 3: 369 case listVMsCalls < 3:
283 return []client.VM{{ID: "vm-1", Phase: "booting"}}, nil 370 return []client.VM{{ID: "vm-1", Phase: "booting"}}, nil
284 case listVMsCalls == 3:
285 return []client.VM{{ID: "vm-1", Phase: "ready", AssignedIP: "10.0.0.9"}}, nil
286 default: 371 default:
287 return nil, nil 372 return []client.VM{{ID: "vm-1", Phase: "ready", AssignedIP: "10.0.0.9", ActualPower: power}}, nil
288 } 373 }
289 }, 374 },
290 deleteVMFunc: func(ctx context.Context, id string) error { return nil }, 375 patchVMFunc: func(ctx context.Context, id, powerState string) error {
376 power = powerState
377 return nil
378 },
379 deleteVMFunc: func(ctx context.Context, id string) error {
380 deleted = true
381 return nil
382 },
291 } 383 }
292 } 384 }
293 385
@@ -328,7 +420,9 @@ func TestRunScenarioGateRegistersBeforeCreateAndExecsAfterBoot(t *testing.T) {
328 t.Errorf("message = %q, want it to mention gate SSH: ok", msg) 420 t.Errorf("message = %q, want it to mention gate SSH: ok", msg)
329 } 421 }
330 422
331 want := []string{"register", "createVM", "exec"} 423 // exec appears twice: once after the first boot proof, once after the
424 // power-cycle proof — SSH through the gate must survive a reboot too.
425 want := []string{"register", "createVM", "exec", "exec"}
332 if len(calls) != len(want) { 426 if len(calls) != len(want) {
333 t.Fatalf("call order = %v, want %v", calls, want) 427 t.Fatalf("call order = %v, want %v", calls, want)
334 } 428 }