1ca7b3d5
vms: a host that says nothing cannot vouch for its guests
a73x 2026-09-05 17:43
Commit message
internal/server/api/api.go
| Old | New | ||
|---|---|---|---|
| @@ -556,7 +556,7 @@ func (a *API) handleListHosts(w http.ResponseWriter, r *http.Request) { | |||
| 556 | 556 | ||
| 557 | // --- VMs --- | 557 | // --- VMs --- |
| 558 | 558 | ||
| 559 | func toVMResponse(vm store.VM, actualPower, phase, statusDetail string, destroyAt int64) types.VM { | 559 | func toVMResponse(vm store.VM, actualPower, phase, statusDetail string, destroyAt int64, hostOnline bool) types.VM { |
| 560 | return types.VM{ | 560 | return types.VM{ |
| 561 | ID: vm.ID, | 561 | ID: vm.ID, |
| 562 | HostID: vm.HostID, | 562 | HostID: vm.HostID, |
| @@ -577,7 +577,7 @@ func toVMResponse(vm store.VM, actualPower, phase, statusDetail string, destroyA | |||
| 577 | Phase: phase, | 577 | Phase: phase, |
| 578 | StatusDetail: statusDetail, | 578 | StatusDetail: statusDetail, |
| 579 | DestroyAt: destroyAt, | 579 | DestroyAt: destroyAt, |
| 580 | Lifecycle: deriveLifecycle(vm, actualPower, phase), | 580 | Lifecycle: deriveLifecycle(vm, actualPower, phase, hostOnline), |
| 581 | InjectedKey: injectedKey(vm), | 581 | InjectedKey: injectedKey(vm), |
| 582 | TrustedCAs: trustedCAs(vm), | 582 | TrustedCAs: trustedCAs(vm), |
| 583 | } | 583 | } |
| @@ -589,10 +589,32 @@ func toVMResponse(vm store.VM, actualPower, phase, statusDetail string, destroyA | |||
| 589 | // web/src/lib/fleet.svelte.ts) displays it directly, keeping only a small | 589 | // web/src/lib/fleet.svelte.ts) displays it directly, keeping only a small |
| 590 | // defensive fallback for malformed or missing snapshots rather than | 590 | // defensive fallback for malformed or missing snapshots rather than |
| 591 | // re-deriving the value itself. | 591 | // re-deriving the value itself. |
| 592 | func deriveLifecycle(vm store.VM, actualPower, phase string) string { | 592 | // |
| 593 | // hostOnline is why "unreachable" exists. Every other word here is an | ||
| 594 | // observation, and the durable row holds the LAST one its host made: nothing | ||
| 595 | // rewrites status or power_state when an agent goes away, so those columns | ||
| 596 | // keep saying "ready"/"running" indefinitely after the machine under them is | ||
| 597 | // switched off. Reading them on a dark host would answer a question the plane | ||
| 598 | // cannot answer — and it is the answer an operator acts on, so it has to be | ||
| 599 | // the true one. See TestAVMOnADarkHostReadsAsUnreachable. | ||
| 600 | // | ||
| 601 | // This is a projection, not a transition: the VM row is untouched, nothing is | ||
| 602 | // reconciled, and the first report from a returning host restores every word | ||
| 603 | // (TestADarkHostsVMRecoversItsLifecycleWhenTheHostComesBack). A server that has | ||
| 604 | // just restarted holds an empty registry, so the whole fleet reads unreachable | ||
| 605 | // until each agent's first report lands — the same brief, self-healing silence | ||
| 606 | // the create-path preconditions already tolerate, and honest while it lasts. | ||
| 607 | func deriveLifecycle(vm store.VM, actualPower, phase string, hostOnline bool) string { | ||
| 608 | // The tombstone outranks unreachability because it is the plane's OWN | ||
| 609 | // intent rather than something a host told it: the delete is recorded here, | ||
| 610 | // it stays true while the host is away, and the console owes the operator | ||
| 611 | // the restore affordance for exactly as long as it does. | ||
| 593 | if vm.DeletedAt != nil { | 612 | if vm.DeletedAt != nil { |
| 594 | return "deleting" | 613 | return "deleting" |
| 595 | } | 614 | } |
| 615 | if !hostOnline { | ||
| 616 | return "unreachable" | ||
| 617 | } | ||
| 596 | if phase == "" { | 618 | if phase == "" { |
| 597 | phase = vm.Status | 619 | phase = vm.Status |
| 598 | } | 620 | } |
| @@ -632,7 +654,21 @@ func (a *API) buildVMResponses(vms []store.VM, states map[string]regState) []typ | |||
| 632 | for i, vm := range vms { | 654 | for i, vm := range vms { |
| 633 | var actualPower, phase, statusDetail string | 655 | var actualPower, phase, statusDetail string |
| 634 | var destroyAt int64 | 656 | var destroyAt int64 |
| 635 | if rs := states[vm.HostID]; rs.ok { | 657 | // Online, not merely present: a host whose registry entry has gone |
| 658 | // stale is as silent as one that never connected, and its last report | ||
| 659 | // is as much a memory. Both must read the same way — see | ||
| 660 | // deriveLifecycle. | ||
| 661 | // | ||
| 662 | // The gate covers the whole merge, not just the lifecycle word, so the | ||
| 663 | // response cannot contradict itself: everything below is an | ||
| 664 | // OBSERVATION (what power the host saw, what phase it was in, what it | ||
| 665 | // said it was doing, when it started the destroy clock), and a host | ||
| 666 | // that is not reporting is not observing. Serving `unreachable` beside | ||
| 667 | // a phase and a power would hand a client the same stale answer the | ||
| 668 | // status just refused to give, one field further down. | ||
| 669 | rs := states[vm.HostID] | ||
| 670 | hostOnline := rs.ok && rs.st.Online | ||
| 671 | if hostOnline { | ||
| 636 | for _, av := range rs.st.Report.VMs { | 672 | for _, av := range rs.st.Report.VMs { |
| 637 | if av.VMID == vm.ID { | 673 | if av.VMID == vm.ID { |
| 638 | actualPower = av.PowerState | 674 | actualPower = av.PowerState |
| @@ -648,7 +684,7 @@ func (a *API) buildVMResponses(vms []store.VM, states map[string]regState) []typ | |||
| 648 | } | 684 | } |
| 649 | } | 685 | } |
| 650 | } | 686 | } |
| 651 | out[i] = toVMResponse(vm, actualPower, phase, statusDetail, destroyAt) | 687 | out[i] = toVMResponse(vm, actualPower, phase, statusDetail, destroyAt, hostOnline) |
| 652 | } | 688 | } |
| 653 | return out | 689 | return out |
| 654 | } | 690 | } |
internal/server/api/api_test.go
| Old | New | ||
|---|---|---|---|
| @@ -1185,28 +1185,47 @@ func TestDeriveLifecycle(t *testing.T) { | |||
| 1185 | vm store.VM | 1185 | vm store.VM |
| 1186 | actualPower string | 1186 | actualPower string |
| 1187 | phase string | 1187 | phase string |
| 1188 | hostOnline bool | ||
| 1188 | want string | 1189 | want string |
| 1189 | }{ | 1190 | }{ |
| 1190 | {"tombstone wins over everything", | 1191 | {"tombstone wins over everything", |
| 1191 | store.VM{Status: "ready", PowerState: "running", DeletedAt: &deleted}, "running", "ready", "deleting"}, | 1192 | store.VM{Status: "ready", PowerState: "running", DeletedAt: &deleted}, "running", "ready", true, "deleting"}, |
| 1192 | {"failed phase", | 1193 | {"failed phase", |
| 1193 | store.VM{Status: "ready", PowerState: "running"}, "stopped", "failed", "failed"}, | 1194 | store.VM{Status: "ready", PowerState: "running"}, "stopped", "failed", true, "failed"}, |
| 1194 | {"still creating (live phase)", | 1195 | {"still creating (live phase)", |
| 1195 | store.VM{Status: "ready", PowerState: "running"}, "", "creating", "creating"}, | 1196 | store.VM{Status: "ready", PowerState: "running"}, "", "creating", true, "creating"}, |
| 1196 | {"empty phase falls back to status=creating", | 1197 | {"empty phase falls back to status=creating", |
| 1197 | store.VM{Status: "creating", PowerState: "running"}, "", "", "creating"}, | 1198 | store.VM{Status: "creating", PowerState: "running"}, "", "", true, "creating"}, |
| 1198 | {"ready phase but not running -> stopped", | 1199 | {"ready phase but not running -> stopped", |
| 1199 | store.VM{Status: "ready", PowerState: "stopped"}, "stopped", "ready", "stopped"}, | 1200 | store.VM{Status: "ready", PowerState: "stopped"}, "stopped", "ready", true, "stopped"}, |
| 1200 | {"desired running but agent reports stopped -> stopped", | 1201 | {"desired running but agent reports stopped -> stopped", |
| 1201 | store.VM{Status: "ready", PowerState: "running"}, "stopped", "ready", "stopped"}, | 1202 | store.VM{Status: "ready", PowerState: "running"}, "stopped", "ready", true, "stopped"}, |
| 1202 | {"running + ready -> ready", | 1203 | {"running + ready -> ready", |
| 1203 | store.VM{Status: "ready", PowerState: "running"}, "running", "ready", "ready"}, | 1204 | store.VM{Status: "ready", PowerState: "running"}, "running", "ready", true, "ready"}, |
| 1204 | {"no live phase, desired running falls back to power_state", | 1205 | {"no live phase, desired running falls back to power_state", |
| 1205 | store.VM{Status: "ready", PowerState: "running"}, "", "ready", "ready"}, | 1206 | store.VM{Status: "ready", PowerState: "running"}, "", "ready", true, "ready"}, |
| 1207 | |||
| 1208 | // A dark host observes nothing. Every word below the tombstone is an | ||
| 1209 | // observation the agent would have made, and the last one it made is | ||
| 1210 | // what the durable row still holds — so serving it would be reporting | ||
| 1211 | // a guess as a fact. The VM row is untouched; only the read says so. | ||
| 1212 | {"a running VM on a dark host is not known to be running", | ||
| 1213 | store.VM{Status: "ready", PowerState: "running"}, "", "", false, "unreachable"}, | ||
| 1214 | {"a stopped VM on a dark host is not known to be stopped", | ||
| 1215 | store.VM{Status: "ready", PowerState: "stopped"}, "", "", false, "unreachable"}, | ||
| 1216 | {"a create on a dark host is not known to be progressing", | ||
| 1217 | store.VM{Status: "creating", PowerState: "running"}, "", "", false, "unreachable"}, | ||
| 1218 | {"a failed VM on a dark host is not re-confirmed failed", | ||
| 1219 | store.VM{Status: "failed", PowerState: "stopped"}, "", "", false, "unreachable"}, | ||
| 1220 | // The tombstone is the plane's OWN intent, not something the host told | ||
| 1221 | // it, so it outranks unreachability: the console still owes the | ||
| 1222 | // operator the restore affordance while the host is away. | ||
| 1223 | {"a tombstone still reads as deleting on a dark host", | ||
| 1224 | store.VM{Status: "ready", PowerState: "running", DeletedAt: &deleted}, "", "", false, "deleting"}, | ||
| 1206 | } | 1225 | } |
| 1207 | for _, tc := range cases { | 1226 | for _, tc := range cases { |
| 1208 | t.Run(tc.name, func(t *testing.T) { | 1227 | t.Run(tc.name, func(t *testing.T) { |
| 1209 | assert.Equal(t, tc.want, deriveLifecycle(tc.vm, tc.actualPower, tc.phase)) | 1228 | assert.Equal(t, tc.want, deriveLifecycle(tc.vm, tc.actualPower, tc.phase, tc.hostOnline)) |
| 1210 | }) | 1229 | }) |
| 1211 | } | 1230 | } |
| 1212 | } | 1231 | } |
internal/server/api/narration_test.go
| Old | New | ||
|---|---|---|---|
| @@ -5,6 +5,7 @@ import ( | |||
| 5 | "testing" | 5 | "testing" |
| 6 | 6 | ||
| 7 | "github.com/a73x/eitri/internal/server/registry" | 7 | "github.com/a73x/eitri/internal/server/registry" |
| 8 | "github.com/a73x/eitri/internal/server/store" | ||
| 8 | "github.com/stretchr/testify/assert" | 9 | "github.com/stretchr/testify/assert" |
| 9 | "github.com/stretchr/testify/require" | 10 | "github.com/stretchr/testify/require" |
| 10 | ) | 11 | ) |
| @@ -132,3 +133,104 @@ func TestAnUncountedExposureServesNoCounters(t *testing.T) { | |||
| 132 | assert.Equal(t, "pending", list[1]["state"]) | 133 | assert.Equal(t, "pending", list[1]["state"]) |
| 133 | assert.Nil(t, list[1]["sessions"], "an unreported exposure has no counters either") | 134 | assert.Nil(t, list[1]["sessions"], "an unreported exposure has no counters either") |
| 134 | } | 135 | } |
| 136 | |||
| 137 | // TestAVMOnADarkHostReadsAsUnreachable is the whole point of the lifecycle | ||
| 138 | // projection: the durable row is the last thing a host said, and once that host | ||
| 139 | // stops saying anything the row is a memory, not a status. A VM the plane | ||
| 140 | // recorded as running keeps that power_state forever — nothing rewrites it when | ||
| 141 | // the agent goes away — so a read that trusted it would tell an operator their | ||
| 142 | // guest is up while the machine under it is off. | ||
| 143 | // | ||
| 144 | // The registry has no entry at all for this host (it enrolled, it never | ||
| 145 | // reported), which is the shape an operator hits by placing a VM on the wrong | ||
| 146 | // host: the create is accepted as desired state, and then nothing happens. | ||
| 147 | // "unreachable" is what the plane actually knows. | ||
| 148 | func TestAVMOnADarkHostReadsAsUnreachable(t *testing.T) { | ||
| 149 | ts, st, _, _, _ := newServer(t) | ||
| 150 | host := enroll(t, ts) | ||
| 151 | vmID := createTestVM(t, ts, host["host_id"], "web-1") | ||
| 152 | |||
| 153 | // Drive the durable row to the state a host that HAD been reporting would | ||
| 154 | // have left behind: a settled, running guest. Nothing reports it now. | ||
| 155 | _, err := st.RecordVMStatus(vmID, host["host_id"], "ready", "", "10.77.1.5") | ||
| 156 | require.NoError(t, err) | ||
| 157 | require.NoError(t, st.SetVMPower(vmID, "running")) | ||
| 158 | |||
| 159 | vm := vmByID(t, ts.URL, vmID) | ||
| 160 | assert.Equal(t, "unreachable", vm["lifecycle"], | ||
| 161 | "a host that says nothing cannot vouch for its guests") | ||
| 162 | assert.Equal(t, "", vm["actual_power"], "nothing observed this VM's power") | ||
| 163 | assert.Equal(t, "", vm["phase"], "nothing observed this VM's phase") | ||
| 164 | } | ||
| 165 | |||
| 166 | // TestADarkHostsVMRecoversItsLifecycleWhenTheHostComesBack is the other half: | ||
| 167 | // unreachable is a read-time projection over live registry state, not a state | ||
| 168 | // the VM was moved into, so the VM needs no repair when its host reconnects — | ||
| 169 | // the very first report puts every word back. | ||
| 170 | func TestADarkHostsVMRecoversItsLifecycleWhenTheHostComesBack(t *testing.T) { | ||
| 171 | ts, st, _, reg, _ := newServer(t) | ||
| 172 | host := enroll(t, ts) | ||
| 173 | vmID := createTestVM(t, ts, host["host_id"], "web-1") | ||
| 174 | _, err := st.RecordVMStatus(vmID, host["host_id"], "ready", "", "10.77.1.5") | ||
| 175 | require.NoError(t, err) | ||
| 176 | require.NoError(t, st.SetVMPower(vmID, "running")) | ||
| 177 | |||
| 178 | require.Equal(t, "unreachable", vmByID(t, ts.URL, vmID)["lifecycle"]) | ||
| 179 | |||
| 180 | reg.UpdateReport(host["host_id"], registry.Report{ | ||
| 181 | VMs: []registry.VMStatus{{VMID: vmID, PowerState: "running", Phase: "ready"}}, | ||
| 182 | }) | ||
| 183 | |||
| 184 | vm := vmByID(t, ts.URL, vmID) | ||
| 185 | assert.Equal(t, "ready", vm["lifecycle"], "one report is the whole repair") | ||
| 186 | assert.Equal(t, "running", vm["actual_power"]) | ||
| 187 | } | ||
| 188 | |||
| 189 | // TestAStaleHostsObservationsAreNotServed covers the second way a host goes | ||
| 190 | // dark, which the response has to treat identically to the first: the registry | ||
| 191 | // still HOLDS an entry — the host connected and reported, then went quiet past | ||
| 192 | // the online window — so every observation it last made is still sitting there, | ||
| 193 | // ready to be served as though it were current. | ||
| 194 | // | ||
| 195 | // It is asserted against buildVMResponses directly because the distinction is | ||
| 196 | // registry-clock-dependent (a real host would have to be left alone for | ||
| 197 | // OnlineWindow), and this is the merge that decides it. | ||
| 198 | func TestAStaleHostsObservationsAreNotServed(t *testing.T) { | ||
| 199 | vm := store.VM{ID: "vm1", HostID: "h1", Status: "ready", PowerState: "running"} | ||
| 200 | // Everything a host that WAS reporting leaves behind, with Online false — | ||
| 201 | // exactly what registry.Get derives once the last report ages out. | ||
| 202 | stale := regState{ok: true, st: registry.HostState{ | ||
| 203 | Online: false, | ||
| 204 | Report: registry.Report{VMs: []registry.VMStatus{{ | ||
| 205 | VMID: "vm1", PowerState: "running", Phase: "ready", | ||
| 206 | StatusDetail: "booting", | ||
| 207 | }}}, | ||
| 208 | }} | ||
| 209 | |||
| 210 | out := (&API{}).buildVMResponses([]store.VM{vm}, map[string]regState{"h1": stale}) | ||
| 211 | require.Len(t, out, 1) | ||
| 212 | assert.Equal(t, "unreachable", out[0].Lifecycle) | ||
| 213 | assert.Empty(t, out[0].ActualPower, "a host that stopped reporting stopped observing") | ||
| 214 | assert.Empty(t, out[0].Phase, "the last phase it saw is a memory, not a status") | ||
| 215 | assert.Empty(t, out[0].StatusDetail, "and so is the last thing it said it was doing") | ||
| 216 | } | ||
| 217 | |||
| 218 | // TestAnOnlineHostsObservationsStillReachTheWire is the control for the test | ||
| 219 | // above: the gate must be the host's silence and nothing else. | ||
| 220 | func TestAnOnlineHostsObservationsStillReachTheWire(t *testing.T) { | ||
| 221 | vm := store.VM{ID: "vm1", HostID: "h1", Status: "ready", PowerState: "running"} | ||
| 222 | live := regState{ok: true, st: registry.HostState{ | ||
| 223 | Online: true, | ||
| 224 | Report: registry.Report{VMs: []registry.VMStatus{{ | ||
| 225 | VMID: "vm1", PowerState: "running", Phase: "ready", | ||
| 226 | StatusDetail: "booting", | ||
| 227 | }}}, | ||
| 228 | }} | ||
| 229 | |||
| 230 | out := (&API{}).buildVMResponses([]store.VM{vm}, map[string]regState{"h1": live}) | ||
| 231 | require.Len(t, out, 1) | ||
| 232 | assert.Equal(t, "ready", out[0].Lifecycle) | ||
| 233 | assert.Equal(t, "running", out[0].ActualPower) | ||
| 234 | assert.Equal(t, "ready", out[0].Phase) | ||
| 235 | assert.Equal(t, "booting", out[0].StatusDetail) | ||
| 236 | } | ||
internal/server/api/types/types.go
| Old | New | ||
|---|---|---|---|
| @@ -147,8 +147,17 @@ type VM struct { | |||
| 147 | DestroyAt int64 `json:"destroy_at"` | 147 | DestroyAt int64 `json:"destroy_at"` |
| 148 | // Lifecycle rolls the axes above (deleted / phase / power) into one coarse | 148 | // Lifecycle rolls the axes above (deleted / phase / power) into one coarse |
| 149 | // word so every client agrees on "what is this VM doing" without | 149 | // word so every client agrees on "what is this VM doing" without |
| 150 | // re-deriving it: creating | ready | stopped | failed | deleting. A lossy | 150 | // re-deriving it: creating | ready | stopped | failed | deleting | |
| 151 | // read-time projection, never state — see TestLifecycleIsNeverReadByAControlLoop. | 151 | // unreachable. A lossy read-time projection, never state — see |
| 152 | // TestLifecycleIsNeverReadByAControlLoop. | ||
| 153 | // | ||
| 154 | // `unreachable` is the one word here that describes the PLANE rather than | ||
| 155 | // the guest: this VM's host is not reporting, so every observation below | ||
| 156 | // (phase, actual_power, status_detail) is absent and the durable columns | ||
| 157 | // hold only what that host last said before it went quiet. It is not a | ||
| 158 | // failure and nothing is reconciled — the VM may well be running fine on a | ||
| 159 | // machine that merely lost its uplink, and desired state still stands, so a | ||
| 160 | // create placed on a dark host is picked up when the host returns. | ||
| 152 | Lifecycle string `json:"lifecycle"` | 161 | Lifecycle string `json:"lifecycle"` |
| 153 | // InjectedKey describes the authorized key EITRI installed in this guest at | 162 | // InjectedKey describes the authorized key EITRI installed in this guest at |
| 154 | // create, or null when it installed none. It is a description, not the key: | 163 | // create, or null when it installed none. It is a description, not the key: |
internal/server/mcphttp/mcphttp_test.go
| Old | New | ||
|---|---|---|---|
| @@ -31,6 +31,7 @@ import ( | |||
| 31 | type fixture struct { | 31 | type fixture struct { |
| 32 | ts *httptest.Server | 32 | ts *httptest.Server |
| 33 | st *store.Store | 33 | st *store.Store |
| 34 | reg *registry.Registry | ||
| 34 | tenantA string | 35 | tenantA string |
| 35 | patA string | 36 | patA string |
| 36 | tenantB string | 37 | tenantB string |
| @@ -53,15 +54,16 @@ func newFixture(t *testing.T) fixture { | |||
| 53 | require.NoError(t, err) | 54 | require.NoError(t, err) |
| 54 | t.Cleanup(func() { st.Close() }) | 55 | t.Cleanup(func() { st.Close() }) |
| 55 | 56 | ||
| 57 | reg := registry.New(time.Now) | ||
| 56 | a := api.New(api.Config{ | 58 | a := api.New(api.Config{ |
| 57 | HostSecret: []byte("hostsecret"), | 59 | HostSecret: []byte("hostsecret"), |
| 58 | AdvertiseHTTP: "http://127.0.0.1:8080", | 60 | AdvertiseHTTP: "http://127.0.0.1:8080", |
| 59 | AdvertiseQUIC: "127.0.0.1:8443", | 61 | AdvertiseQUIC: "127.0.0.1:8443", |
| 60 | ServerCertSHA256: strings.Repeat("c", 64), | 62 | ServerCertSHA256: strings.Repeat("c", 64), |
| 61 | }, st, registry.New(time.Now), hub.New()) | 63 | }, st, reg, hub.New()) |
| 62 | t.Cleanup(a.Close) | 64 | t.Cleanup(a.Close) |
| 63 | 65 | ||
| 64 | f := fixture{st: st} | 66 | f := fixture{st: st, reg: reg} |
| 65 | f.tenantA, f.patA = newTenant(t, st, "alpha") | 67 | f.tenantA, f.patA = newTenant(t, st, "alpha") |
| 66 | f.tenantB, f.patB = newTenant(t, st, "beta") | 68 | f.tenantB, f.patB = newTenant(t, st, "beta") |
| 67 | 69 | ||
| @@ -102,7 +104,12 @@ func newTenant(t *testing.T, st *store.Store, handle string) (string, string) { | |||
| 102 | 104 | ||
| 103 | // seedVM gives a tenant one host and one ready VM, so vm_list has something to | 105 | // seedVM gives a tenant one host and one ready VM, so vm_list has something to |
| 104 | // filter and the exec tools get as far as trying to reach a guest. | 106 | // filter and the exec tools get as far as trying to reach a guest. |
| 105 | func seedVM(t *testing.T, st *store.Store, tenant, hostName, vmName string) { | 107 | // |
| 108 | // The report at the end is what makes the VM ready, not the row: a VM whose | ||
| 109 | // host is not reporting reads `unreachable` however settled its columns look | ||
| 110 | // (see deriveLifecycle), and every exec tool gates on `ready`. reg may be nil | ||
| 111 | // for a caller that only needs the rows. | ||
| 112 | func seedVM(t *testing.T, st *store.Store, reg *registry.Registry, tenant, hostName, vmName string) { | ||
| 106 | t.Helper() | 113 | t.Helper() |
| 107 | tok, err := st.CreateEnrollmentToken(tenant) | 114 | tok, err := st.CreateEnrollmentToken(tenant) |
| 108 | require.NoError(t, err) | 115 | require.NoError(t, err) |
| @@ -115,6 +122,10 @@ func seedVM(t *testing.T, st *store.Store, tenant, hostName, vmName string) { | |||
| 115 | VCPUs: 1, MemMB: 512, DiskGB: 5, PowerState: "running"})) | 122 | VCPUs: 1, MemMB: 512, DiskGB: 5, PowerState: "running"})) |
| 116 | _, err = st.RecordVMStatus(vmName+"-id", h.ID, "ready", "", "10.77.0.5") | 123 | _, err = st.RecordVMStatus(vmName+"-id", h.ID, "ready", "", "10.77.0.5") |
| 117 | require.NoError(t, err) | 124 | require.NoError(t, err) |
| 125 | if reg != nil { | ||
| 126 | reg.UpdateReport(h.ID, registry.Report{VMs: []registry.VMStatus{{ | ||
| 127 | VMID: vmName + "-id", PowerState: "running", Phase: "ready", IP: "10.77.0.5"}}}) | ||
| 128 | } | ||
| 118 | } | 129 | } |
| 119 | 130 | ||
| 120 | // connect opens a real MCP client against /mcp with pat as the bearer token. | 131 | // connect opens a real MCP client against /mcp with pat as the bearer token. |
| @@ -206,8 +217,8 @@ func TestMCPExposesTheFullToolset(t *testing.T) { | |||
| 206 | // design exists for: two PATs, two tenants, and neither sees the other's VMs. | 217 | // design exists for: two PATs, two tenants, and neither sees the other's VMs. |
| 207 | func TestVMListIsTenantIsolated(t *testing.T) { | 218 | func TestVMListIsTenantIsolated(t *testing.T) { |
| 208 | f := newFixture(t) | 219 | f := newFixture(t) |
| 209 | seedVM(t, f.st, f.tenantA, "host-alpha", "alpha-vm") | 220 | seedVM(t, f.st, f.reg, f.tenantA, "host-alpha", "alpha-vm") |
| 210 | seedVM(t, f.st, f.tenantB, "host-beta", "beta-vm") | 221 | seedVM(t, f.st, f.reg, f.tenantB, "host-beta", "beta-vm") |
| 211 | 222 | ||
| 212 | assert.Equal(t, []string{"alpha-vm"}, listVMNames(t, connect(t, f, f.patA))) | 223 | assert.Equal(t, []string{"alpha-vm"}, listVMNames(t, connect(t, f, f.patA))) |
| 213 | assert.Equal(t, []string{"beta-vm"}, listVMNames(t, connect(t, f, f.patB))) | 224 | assert.Equal(t, []string{"beta-vm"}, listVMNames(t, connect(t, f, f.patB))) |
| @@ -392,7 +403,7 @@ func callToolInto(t *testing.T, cs *mcp.ClientSession, params *mcp.CallToolParam | |||
| 392 | // by itself, so the message has to be a request with a recipe in it. | 403 | // by itself, so the message has to be a request with a recipe in it. |
| 393 | func TestExecWithoutADelegationRefusesInWords(t *testing.T) { | 404 | func TestExecWithoutADelegationRefusesInWords(t *testing.T) { |
| 394 | f := newFixture(t) | 405 | f := newFixture(t) |
| 395 | seedVM(t, f.st, f.tenantA, "host-alpha", "alpha-vm") | 406 | seedVM(t, f.st, f.reg, f.tenantA, "host-alpha", "alpha-vm") |
| 396 | cs := connect(t, f, f.patA) | 407 | cs := connect(t, f, f.patA) |
| 397 | 408 | ||
| 398 | res, err := cs.CallTool(t.Context(), &mcp.CallToolParams{ | 409 | res, err := cs.CallTool(t.Context(), &mcp.CallToolParams{ |
| @@ -412,11 +423,12 @@ func TestExecWithoutAJumpGateSaysSo(t *testing.T) { | |||
| 412 | st, err := store.Open(t.TempDir()+"/db", "10.77.0.0/16") | 423 | st, err := store.Open(t.TempDir()+"/db", "10.77.0.0/16") |
| 413 | require.NoError(t, err) | 424 | require.NoError(t, err) |
| 414 | t.Cleanup(func() { st.Close() }) | 425 | t.Cleanup(func() { st.Close() }) |
| 426 | reg := registry.New(time.Now) | ||
| 415 | a := api.New(api.Config{HostSecret: []byte("hostsecret"), ServerCertSHA256: strings.Repeat("c", 64)}, | 427 | a := api.New(api.Config{HostSecret: []byte("hostsecret"), ServerCertSHA256: strings.Repeat("c", 64)}, |
| 416 | st, registry.New(time.Now), hub.New()) | 428 | st, reg, hub.New()) |
| 417 | t.Cleanup(a.Close) | 429 | t.Cleanup(a.Close) |
| 418 | tenant, pat := newTenant(t, st, "alpha") | 430 | tenant, pat := newTenant(t, st, "alpha") |
| 419 | seedVM(t, st, tenant, "host-alpha", "alpha-vm") | 431 | seedVM(t, st, reg, tenant, "host-alpha", "alpha-vm") |
| 420 | 432 | ||
| 421 | root := http.NewServeMux() | 433 | root := http.NewServeMux() |
| 422 | root.Handle("/api/", a.Handler()) | 434 | root.Handle("/api/", a.Handler()) |
web/src/lib/fleet.svelte.ts
| Old | New | ||
|---|---|---|---|
| @@ -487,6 +487,11 @@ export function vmPhase(vm: VM): string { | |||
| 487 | /** vmPower is the power to display: the agent-observed power, falling back to | 487 | /** vmPower is the power to display: the agent-observed power, falling back to |
| 488 | * the desired power_state when no actual is reported yet. */ | 488 | * the desired power_state when no actual is reported yet. */ |
| 489 | export function vmPower(vm: VM): string { | 489 | export function vmPower(vm: VM): string { |
| 490 | // A dark host reports no power, and the columns that survive it are a | ||
| 491 | // memory: power_state is what the operator ASKED for, actual_power is | ||
| 492 | // blank because nothing observed it. Falling through to either would print | ||
| 493 | // "running" in the same row that says the plane cannot reach this VM. | ||
| 494 | if (vmStatus(vm) === 'unreachable') return '—'; | ||
| 490 | return vm.actual_power || vm.power_state; | 495 | return vm.actual_power || vm.power_state; |
| 491 | } | 496 | } |
| 492 | 497 | ||
| @@ -500,9 +505,18 @@ export function vmPower(vm: VM): string { | |||
| 500 | * | 505 | * |
| 501 | * An empty detail is the normal case in two situations that look identical | 506 | * An empty detail is the normal case in two situations that look identical |
| 502 | * here and should: a host with nothing to add, and a host running an agent old | 507 | * here and should: a host with nothing to add, and a host running an agent old |
| 503 | * enough that it never sends the field. Neither gets a placeholder. */ | 508 | * enough that it never sends the field. Neither gets a placeholder. |
| 509 | * | ||
| 510 | * `unreachable` is the exception that is not the host's words at all: the | ||
| 511 | * status is about the plane's reach rather than the guest, so alone it reads | ||
| 512 | * as something wrong with the VM. This names whose silence it is — and it | ||
| 513 | * REPLACES any status_detail the row still carries, because that sentence | ||
| 514 | * described work a host that has since gone quiet was doing, and presenting it | ||
| 515 | * now would be the same lie the status just refused to tell. */ | ||
| 504 | export function vmDetail(vm: VM): string { | 516 | export function vmDetail(vm: VM): string { |
| 505 | if (vmStatus(vm) !== 'creating') return ''; | 517 | const status = vmStatus(vm); |
| 518 | if (status === 'unreachable') return "eitri can't reach this VM's host"; | ||
| 519 | if (status !== 'creating') return ''; | ||
| 506 | return vm.status_detail ?? ''; | 520 | return vm.status_detail ?? ''; |
| 507 | } | 521 | } |
| 508 | 522 | ||
web/src/lib/fleet.test.ts
| Old | New | ||
|---|---|---|---|
| @@ -12,6 +12,8 @@ import { | |||
| 12 | upgradeStuck, | 12 | upgradeStuck, |
| 13 | upgradeStuckNote, | 13 | upgradeStuckNote, |
| 14 | vmDetail, | 14 | vmDetail, |
| 15 | vmPower, | ||
| 16 | vmPowerAction, | ||
| 15 | vmNetworkAddr, | 17 | vmNetworkAddr, |
| 16 | vmNetworkAddrHint, | 18 | vmNetworkAddrHint, |
| 17 | vmNetworkValue, | 19 | vmNetworkValue, |
| @@ -68,6 +70,14 @@ function vmCreating(status_detail: string): VM { | |||
| 68 | return { ...vmTrusting(null), lifecycle: 'creating', phase: 'creating', status_detail }; | 70 | return { ...vmTrusting(null), lifecycle: 'creating', phase: 'creating', status_detail }; |
| 69 | } | 71 | } |
| 70 | 72 | ||
| 73 | /** vmUnreachable is a VM whose host has stopped reporting. The durable columns | ||
| 74 | * still hold the last thing that host said — a settled, running guest — which | ||
| 75 | * is exactly why the server sends lifecycle `unreachable` over the top of | ||
| 76 | * them, and why nothing here may read them. */ | ||
| 77 | function vmUnreachable(): VM { | ||
| 78 | return { ...vmTrusting(null), lifecycle: 'unreachable', phase: '', actual_power: '' }; | ||
| 79 | } | ||
| 80 | |||
| 71 | /** exposure is one published port as its row sees it. sessions is null for a | 81 | /** exposure is one published port as its row sees it. sessions is null for a |
| 72 | * port nobody has counted: unreported, or served by an older agent. */ | 82 | * port nobody has counted: unreported, or served by an older agent. */ |
| 73 | function exposure(sessions: Exposure['sessions']): Exposure { | 83 | function exposure(sessions: Exposure['sessions']): Exposure { |
| @@ -469,6 +479,38 @@ describe('vmDetail', () => { | |||
| 469 | // past presented as the present. | 479 | // past presented as the present. |
| 470 | expect(vmDetail({ ...vmTrusting(null), status_detail: 'booting' })).toBe(''); | 480 | expect(vmDetail({ ...vmTrusting(null), status_detail: 'booting' })).toBe(''); |
| 471 | }); | 481 | }); |
| 482 | |||
| 483 | test('an unreachable VM explains itself', () => { | ||
| 484 | // `unreachable` is a word about the plane, not the guest, so on its own | ||
| 485 | // it reads as a fault of the VM. The sentence says whose silence it is. | ||
| 486 | expect(vmDetail(vmUnreachable())).toBe("eitri can't reach this VM's host"); | ||
| 487 | }); | ||
| 488 | |||
| 489 | test('an unreachable VM does not quote what its host last said', () => { | ||
| 490 | // The detail is the host's account of work in flight. A host that has | ||
| 491 | // gone silent is not doing that work, and may not have been for days. | ||
| 492 | expect(vmDetail({ ...vmUnreachable(), status_detail: 'downloading image' })).toBe( | ||
| 493 | "eitri can't reach this VM's host" | ||
| 494 | ); | ||
| 495 | }); | ||
| 496 | }); | ||
| 497 | |||
| 498 | describe('a VM whose host has gone dark', () => { | ||
| 499 | test('its power is not reported as the last thing its host saw', () => { | ||
| 500 | // power_state is desired state and survives the host; actual_power is an | ||
| 501 | // observation and is absent. Printing either beside `unreachable` would | ||
| 502 | // answer the one question the row just said it cannot. | ||
| 503 | expect(vmPower(vmUnreachable())).toBe('—'); | ||
| 504 | }); | ||
| 505 | |||
| 506 | test('it offers no power button', () => { | ||
| 507 | // A start/stop the plane cannot deliver is a lie in a button. | ||
| 508 | expect(vmPowerAction(vmUnreachable())).toBe(null); | ||
| 509 | }); | ||
| 510 | |||
| 511 | test('a reachable VM still reports its power normally', () => { | ||
| 512 | expect(vmPower(vmTrusting(null))).toBe('running'); | ||
| 513 | }); | ||
| 472 | }); | 514 | }); |
| 473 | 515 | ||
| 474 | describe('the two addresses a guest can have', () => { | 516 | describe('the two addresses a guest can have', () => { |