17485cac
feat(server): a host that cannot certify a guest is not a place to put one
a73x 2026-08-09 15:50
Commit message
docs/assumptions.md
| Old | New | ||
|---|---|---|---|
| @@ -407,6 +407,22 @@ than booting a guest clients would refuse. | |||
| 407 | connection, and both the snapshot and the report are marshalled and searched for | 407 | connection, and both the snapshot and the report are marshalled and searched for |
| 408 | private-key bytes. | 408 | private-key bytes. |
| 409 | 409 | ||
| 410 | ### A host that cannot certify a guest is not a place to put one | ||
| 411 | |||
| 412 | An agent from before v0.0.4 never reports a guest host key, so a VM it creates | ||
| 413 | can never be verified and is unreachable through the gate for the rest of its | ||
| 414 | life. The control plane reads the version the host reported and refuses the | ||
| 415 | create outright — and reads a version it cannot parse, or none at all, as too | ||
| 416 | old, because neither proves otherwise. Underpins refusing early: at create the | ||
| 417 | operator can still upgrade the agent, while at connect the only remedy left is | ||
| 418 | to recreate the VM. Only a connected host is judged, because only a connected | ||
| 419 | host has said anything: what a registry holds in memory about an offline one is | ||
| 420 | not evidence about the agent that will run the VM. | ||
| 421 | **Proven** in code: the create path refuses each shape of unprovable version | ||
| 422 | and names the upgrade endpoint, lets a host it has not heard from take the | ||
| 423 | create, and placement chosen by eitri skips such hosts and says why when that | ||
| 424 | leaves none. | ||
| 425 | |||
| 410 | ### What eitri signs with is encrypted where it rests | 426 | ### What eitri signs with is encrypted where it rests |
| 411 | 427 | ||
| 412 | Every piece of key material the server holds—the fleet's own host CA and gate | 428 | Every piece of key material the server holds—the fleet's own host CA and gate |
docs/shape.html
| Old | New | ||
|---|---|---|---|
| @@ -349,7 +349,8 @@ | |||
| 349 | "imports": [ | 349 | "imports": [ |
| 350 | "internal/gateclient", | 350 | "internal/gateclient", |
| 351 | "internal/random", | 351 | "internal/random", |
| 352 | "internal/server/api/client" | 352 | "internal/server/api/client", |
| 353 | "internal/server/release" | ||
| 353 | ] | 354 | ] |
| 354 | }, | 355 | }, |
| 355 | { | 356 | { |
docs/shape.json
| Old | New | ||
|---|---|---|---|
| @@ -298,7 +298,8 @@ | |||
| 298 | "imports": [ | 298 | "imports": [ |
| 299 | "internal/gateclient", | 299 | "internal/gateclient", |
| 300 | "internal/random", | 300 | "internal/random", |
| 301 | "internal/server/api/client" | 301 | "internal/server/api/client", |
| 302 | "internal/server/release" | ||
| 302 | ] | 303 | ] |
| 303 | }, | 304 | }, |
| 304 | { | 305 | { |
docs/upgrade.md
| Old | New | ||
|---|---|---|---|
| @@ -94,10 +94,17 @@ ignored by agents that predate them. | |||
| 94 | 94 | ||
| 95 | **One exception, from v0.0.4 on.** A guest's SSH host key is generated by the | 95 | **One exception, from v0.0.4 on.** A guest's SSH host key is generated by the |
| 96 | host that runs it, and an agent from before v0.0.4 does not know how to do | 96 | host that runs it, and an agent from before v0.0.4 does not know how to do |
| 97 | that. A VM created on such a host boots with a host key nothing has certified, | 97 | that. The server will not place a VM on such a host: the create is refused with |
| 98 | and `eitri ssh` refuses it at host verification with a clear error. Existing | 98 | the host's name, the version it reports, and the endpoint that upgrades it — |
| 99 | VMs are unaffected — they keep the host key and certificate they were built | 99 | nothing could verify a guest created there. A connected host that has reported |
| 100 | with. Upgrade the host's agent, then re-create any VM created in the meantime. | 100 | no version at all is read the same way, since nothing says otherwise. Only a |
| 101 | host that is connected is judged: one that is offline has told this server | ||
| 102 | nothing about the agent that will eventually run the VM, so the create is | ||
| 103 | accepted as desired state and the guest is verified — or refused — when it | ||
| 104 | boots. Placement chosen for you skips those hosts entirely. Existing VMs are unaffected — they | ||
| 105 | keep the host key and certificate they were built with; one created against an | ||
| 106 | older release with an uncertified key is refused by `eitri ssh` at host | ||
| 107 | verification, and re-creating it after the upgrade is the fix. | ||
| 101 | 108 | ||
| 102 | ## cloud-hypervisor | 109 | ## cloud-hypervisor |
| 103 | 110 | ||
internal/mcpserver/api_test.go
| Old | New | ||
|---|---|---|---|
| @@ -66,18 +66,85 @@ func TestDeleteVM(t *testing.T) { | |||
| 66 | require.NoError(t, c.DeleteVM(t.Context(), "abc")) | 66 | require.NoError(t, c.DeleteVM(t.Context(), "abc")) |
| 67 | } | 67 | } |
| 68 | 68 | ||
| 69 | func TestFirstOnlineHost(t *testing.T) { | 69 | func TestFirstEligibleHost(t *testing.T) { |
| 70 | c := fakeAPI(t, func(w http.ResponseWriter, r *http.Request) { | 70 | c := fakeAPI(t, func(w http.ResponseWriter, r *http.Request) { |
| 71 | json.NewEncoder(w).Encode([]map[string]any{ | 71 | json.NewEncoder(w).Encode([]map[string]any{ |
| 72 | {"id": "h0", "name": "down", "online": false}, | 72 | {"id": "h0", "name": "down", "online": false, "agent_version": "v0.0.5"}, |
| 73 | {"id": "h1", "name": "mewtwo", "online": true}, | 73 | {"id": "h1", "name": "mewtwo", "online": true, "agent_version": "v0.0.5"}, |
| 74 | }) | 74 | }) |
| 75 | }) | 75 | }) |
| 76 | h, err := c.FirstOnlineHost(t.Context()) | 76 | h, err := c.FirstEligibleHost(t.Context()) |
| 77 | require.NoError(t, err) | 77 | require.NoError(t, err) |
| 78 | assert.Equal(t, "h1", h.ID) | 78 | assert.Equal(t, "h1", h.ID) |
| 79 | } | 79 | } |
| 80 | 80 | ||
| 81 | // A host whose agent predates certified host keys is not a placement candidate: | ||
| 82 | // the control plane would refuse the create, so picking it only moves the | ||
| 83 | // failure later. The next host that can certify takes the VM instead. | ||
| 84 | func TestFirstEligibleHostSkipsPreCSRAgents(t *testing.T) { | ||
| 85 | c := fakeAPI(t, func(w http.ResponseWriter, r *http.Request) { | ||
| 86 | json.NewEncoder(w).Encode([]map[string]any{ | ||
| 87 | {"id": "h0", "name": "old", "online": true, "agent_version": "v0.0.3"}, | ||
| 88 | {"id": "h1", "name": "unstamped", "online": true, "agent_version": "dev"}, | ||
| 89 | {"id": "h2", "name": "quiet", "online": true}, | ||
| 90 | {"id": "h3", "name": "mewtwo", "online": true, "agent_version": "v0.0.4"}, | ||
| 91 | }) | ||
| 92 | }) | ||
| 93 | h, err := c.FirstEligibleHost(t.Context()) | ||
| 94 | require.NoError(t, err) | ||
| 95 | assert.Equal(t, "h3", h.ID) | ||
| 96 | } | ||
| 97 | |||
| 98 | // With every online host ruled out, the refusal names the reason and the fix — | ||
| 99 | // "no online hosts" would send the caller looking for hardware that is up. | ||
| 100 | func TestFirstEligibleHostReportsWhyNoneAreEligible(t *testing.T) { | ||
| 101 | c := fakeAPI(t, func(w http.ResponseWriter, r *http.Request) { | ||
| 102 | json.NewEncoder(w).Encode([]map[string]any{ | ||
| 103 | {"id": "h0", "name": "old", "online": true, "agent_version": "v0.0.3"}, | ||
| 104 | {"id": "h1", "name": "older", "online": true, "agent_version": "v0.0.1"}, | ||
| 105 | }) | ||
| 106 | }) | ||
| 107 | _, err := c.FirstEligibleHost(t.Context()) | ||
| 108 | require.Error(t, err) | ||
| 109 | assert.Contains(t, err.Error(), "2 online host(s)") | ||
| 110 | assert.Contains(t, err.Error(), "predate certified host keys (v0.0.4)") | ||
| 111 | assert.Contains(t, err.Error(), "upgrade-agent") | ||
| 112 | assert.NotContains(t, err.Error(), "no online hosts") | ||
| 113 | } | ||
| 114 | |||
| 115 | // A fleet with nothing up still says exactly that. | ||
| 116 | func TestFirstEligibleHostWithNothingOnline(t *testing.T) { | ||
| 117 | c := fakeAPI(t, func(w http.ResponseWriter, r *http.Request) { | ||
| 118 | json.NewEncoder(w).Encode([]map[string]any{ | ||
| 119 | {"id": "h0", "name": "down", "online": false, "agent_version": "v0.0.5"}, | ||
| 120 | }) | ||
| 121 | }) | ||
| 122 | _, err := c.FirstEligibleHost(t.Context()) | ||
| 123 | require.ErrorContains(t, err, "no online hosts") | ||
| 124 | } | ||
| 125 | |||
| 126 | // The control plane's create refusal is the whole story the model needs, so it | ||
| 127 | // must survive the trip through the client and out of vm_create — a bare "409" | ||
| 128 | // would leave it with nothing to act on. | ||
| 129 | func TestVMCreateSurfacesThePreCSRRefusal(t *testing.T) { | ||
| 130 | const refusal = "host mewtwo (h1) runs agent v0.0.3, which predates certified host keys (v0.0.4): " + | ||
| 131 | "a guest created there gets no certified host key. Upgrade that host's agent" | ||
| 132 | c := fakeAPI(t, func(w http.ResponseWriter, r *http.Request) { | ||
| 133 | if r.URL.Path == "/api/v1/hosts" { | ||
| 134 | json.NewEncoder(w).Encode([]map[string]any{ | ||
| 135 | {"id": "h1", "name": "mewtwo", "online": true, "agent_version": "v0.0.5"}, | ||
| 136 | }) | ||
| 137 | return | ||
| 138 | } | ||
| 139 | http.Error(w, refusal, http.StatusConflict) | ||
| 140 | }) | ||
| 141 | tools := &Tools{API: c} | ||
| 142 | _, err := tools.VMCreate(t.Context(), VMCreateIn{Host: "mewtwo"}) | ||
| 143 | require.Error(t, err) | ||
| 144 | assert.Contains(t, err.Error(), "predates certified host keys") | ||
| 145 | assert.Contains(t, err.Error(), "Upgrade that host's agent") | ||
| 146 | } | ||
| 147 | |||
| 81 | func TestAPIErrorSurfacesBodyNotToken(t *testing.T) { | 148 | func TestAPIErrorSurfacesBodyNotToken(t *testing.T) { |
| 82 | c := fakeAPI(t, func(w http.ResponseWriter, r *http.Request) { | 149 | c := fakeAPI(t, func(w http.ResponseWriter, r *http.Request) { |
| 83 | http.Error(w, "invalid name", http.StatusBadRequest) | 150 | http.Error(w, "invalid name", http.StatusBadRequest) |
internal/mcpserver/tools.go
| Old | New | ||
|---|---|---|---|
| @@ -13,6 +13,7 @@ import ( | |||
| 13 | "github.com/a73x/eitri/internal/gateclient" | 13 | "github.com/a73x/eitri/internal/gateclient" |
| 14 | "github.com/a73x/eitri/internal/random" | 14 | "github.com/a73x/eitri/internal/random" |
| 15 | "github.com/a73x/eitri/internal/server/api/client" | 15 | "github.com/a73x/eitri/internal/server/api/client" |
| 16 | "github.com/a73x/eitri/internal/server/release" | ||
| 16 | "golang.org/x/crypto/ssh" | 17 | "golang.org/x/crypto/ssh" |
| 17 | ) | 18 | ) |
| 18 | 19 | ||
| @@ -23,7 +24,7 @@ type api interface { | |||
| 23 | CreateVM(ctx context.Context, req client.CreateVMRequest) (client.CreateVMResponse, error) | 24 | CreateVM(ctx context.Context, req client.CreateVMRequest) (client.CreateVMResponse, error) |
| 24 | DeleteVM(ctx context.Context, id string) error | 25 | DeleteVM(ctx context.Context, id string) error |
| 25 | ListHosts(ctx context.Context) ([]client.Host, error) | 26 | ListHosts(ctx context.Context) ([]client.Host, error) |
| 26 | FirstOnlineHost(ctx context.Context) (client.Host, error) | 27 | FirstEligibleHost(ctx context.Context) (client.Host, error) |
| 27 | CreateExposure(ctx context.Context, vmID string, guestPort, hostPort int64) (client.Exposure, error) | 28 | CreateExposure(ctx context.Context, vmID string, guestPort, hostPort int64) (client.Exposure, error) |
| 28 | ListExposures(ctx context.Context, vmID string) ([]client.Exposure, error) | 29 | ListExposures(ctx context.Context, vmID string) ([]client.Exposure, error) |
| 29 | DeleteExposure(ctx context.Context, id string) error | 30 | DeleteExposure(ctx context.Context, id string) error |
| @@ -46,7 +47,7 @@ type Exec interface { | |||
| 46 | } | 47 | } |
| 47 | 48 | ||
| 48 | // API is the shared eitri API client plus the one piece of MCP placement | 49 | // API is the shared eitri API client plus the one piece of MCP placement |
| 49 | // policy the raw client doesn't carry: FirstOnlineHost. | 50 | // policy the raw client doesn't carry: FirstEligibleHost. |
| 50 | type API struct { | 51 | type API struct { |
| 51 | *client.Client | 52 | *client.Client |
| 52 | } | 53 | } |
| @@ -64,18 +65,36 @@ func (a API) RegisterUserCA(ctx context.Context, caLine, label string) error { | |||
| 64 | return c.UploadUserCA(ctx, "", caLine) | 65 | return c.UploadUserCA(ctx, "", caLine) |
| 65 | } | 66 | } |
| 66 | 67 | ||
| 67 | // FirstOnlineHost returns the first online host — the default placement | 68 | // FirstEligibleHost returns the first host a VM can be placed on — the default |
| 68 | // target when the caller doesn't name one. Ordering is server-defined; | 69 | // target when the caller doesn't name one. Eligible means online AND running an |
| 69 | // callers must not assume stability across calls. | 70 | // agent that certifies its guests' host keys: a VM created anywhere else is |
| 70 | func (a API) FirstOnlineHost(ctx context.Context) (client.Host, error) { | 71 | // unreachable through the gate, and the control plane refuses the create, so |
| 72 | // such a host is no more a candidate than one that is down. Ordering is | ||
| 73 | // server-defined; callers must not assume stability across calls. | ||
| 74 | // | ||
| 75 | // When every online host is ruled out that way, the refusal says so — "no | ||
| 76 | // online hosts" would send the caller looking for hardware that is in fact up | ||
| 77 | // and one upgrade away from usable. | ||
| 78 | func (a API) FirstEligibleHost(ctx context.Context) (client.Host, error) { | ||
| 71 | hosts, err := a.ListHosts(ctx) | 79 | hosts, err := a.ListHosts(ctx) |
| 72 | if err != nil { | 80 | if err != nil { |
| 73 | return client.Host{}, err | 81 | return client.Host{}, err |
| 74 | } | 82 | } |
| 83 | precsr := 0 | ||
| 75 | for _, h := range hosts { | 84 | for _, h := range hosts { |
| 76 | if h.Online { | 85 | if !h.Online { |
| 77 | return h, nil | 86 | continue |
| 78 | } | 87 | } |
| 88 | if !release.CertifiesGuestHostKeys(h.AgentVersion) { | ||
| 89 | precsr++ | ||
| 90 | continue | ||
| 91 | } | ||
| 92 | return h, nil | ||
| 93 | } | ||
| 94 | if precsr > 0 { | ||
| 95 | return client.Host{}, fmt.Errorf("no eligible hosts: %d online host(s) run agents that predate certified host keys (%s), "+ | ||
| 96 | "so a VM created on them could not be verified; upgrade an agent (the console's upgrade button, or "+ | ||
| 97 | "POST /api/v1/hosts/{id}/upgrade-agent) and try again", precsr, release.FirstCertifiedHostKeys) | ||
| 79 | } | 98 | } |
| 80 | return client.Host{}, errors.New("no online hosts") | 99 | return client.Host{}, errors.New("no online hosts") |
| 81 | } | 100 | } |
| @@ -117,7 +136,7 @@ func (t *Tools) waitTimeout() time.Duration { | |||
| 117 | 136 | ||
| 118 | type VMCreateIn struct { | 137 | type VMCreateIn struct { |
| 119 | Name string `json:"name,omitempty" jsonschema:"VM name (RFC-1123 label); default claude-<hex>"` | 138 | Name string `json:"name,omitempty" jsonschema:"VM name (RFC-1123 label); default claude-<hex>"` |
| 120 | Host string `json:"host,omitempty" jsonschema:"host name to place on; default first online host"` | 139 | Host string `json:"host,omitempty" jsonschema:"host name to place on; default first eligible host"` |
| 121 | VCPUs int64 `json:"vcpus,omitempty" jsonschema:"default 2"` | 140 | VCPUs int64 `json:"vcpus,omitempty" jsonschema:"default 2"` |
| 122 | MemMB int64 `json:"mem_mb,omitempty" jsonschema:"default 2048"` | 141 | MemMB int64 `json:"mem_mb,omitempty" jsonschema:"default 2048"` |
| 123 | DiskGB int64 `json:"disk_gb,omitempty" jsonschema:"default 20"` | 142 | DiskGB int64 `json:"disk_gb,omitempty" jsonschema:"default 20"` |
| @@ -164,7 +183,7 @@ func (t *Tools) VMCreate(ctx context.Context, in VMCreateIn) (VMCreateOut, error | |||
| 164 | } | 183 | } |
| 165 | req.HostID = hostID | 184 | req.HostID = hostID |
| 166 | } else { | 185 | } else { |
| 167 | h, err := t.API.FirstOnlineHost(ctx) | 186 | h, err := t.API.FirstEligibleHost(ctx) |
| 168 | if err != nil { | 187 | if err != nil { |
| 169 | return VMCreateOut{}, err | 188 | return VMCreateOut{}, err |
| 170 | } | 189 | } |
internal/mcpserver/tools_test.go
| Old | New | ||
|---|---|---|---|
| @@ -84,7 +84,7 @@ func (f *fakeToolsAPI) DeleteVM(ctx context.Context, id string) error { | |||
| 84 | func (f *fakeToolsAPI) ListHosts(ctx context.Context) ([]client.Host, error) { | 84 | func (f *fakeToolsAPI) ListHosts(ctx context.Context) ([]client.Host, error) { |
| 85 | return f.fleet(), nil | 85 | return f.fleet(), nil |
| 86 | } | 86 | } |
| 87 | func (f *fakeToolsAPI) FirstOnlineHost(ctx context.Context) (client.Host, error) { | 87 | func (f *fakeToolsAPI) FirstEligibleHost(ctx context.Context) (client.Host, error) { |
| 88 | for _, h := range f.fleet() { | 88 | for _, h := range f.fleet() { |
| 89 | if h.Online { | 89 | if h.Online { |
| 90 | return h, nil | 90 | return h, nil |
internal/server/api/api.go
| Old | New | ||
|---|---|---|---|
| @@ -781,6 +781,28 @@ func (a *API) handleCreateVM(w http.ResponseWriter, r *http.Request) { | |||
| 781 | return | 781 | return |
| 782 | } | 782 | } |
| 783 | 783 | ||
| 784 | // Certified-host-key precondition, the other half of the same story: a guest | ||
| 785 | // whose host key nothing signed is unreachable through the gate, and the | ||
| 786 | // signing depends on the agent that will run it. Refuse here, where the | ||
| 787 | // operator can still upgrade the host, rather than at connect time — by then | ||
| 788 | // the only remedy is to recreate the VM (see vmssh, which refuses the dial). | ||
| 789 | // | ||
| 790 | // Only a host that is connected and reporting has told this server what it | ||
| 791 | // runs. The registry is in-memory and filled by the agent's Hello, so every | ||
| 792 | // host in the fleet is briefly absent from it after a server restart, and a | ||
| 793 | // host that has gone quiet holds whatever it last said; neither is evidence | ||
| 794 | // about the agent that will pick this VM up. Judging them would answer a | ||
| 795 | // roll with "upgrade this host" about a host that is perfectly current — a | ||
| 796 | // wrong diagnosis at the one moment operators are watching. So an offline | ||
| 797 | // host takes the create exactly as it does for any other reason it cannot | ||
| 798 | // serve one right now: the row is desired state, the agent that materializes | ||
| 799 | // it may well be a newer one, and vmssh still refuses the dial if the guest | ||
| 800 | // it eventually boots has no certificate. | ||
| 801 | if st, ok := a.reg.Get(req.HostID); ok && st.Online && !release.CertifiesGuestHostKeys(st.AgentVersion) { | ||
| 802 | http.Error(w, precsrRefusal(host.Name, req.HostID, st.AgentVersion, a.URL(upgradeAgentPath(req.HostID))), http.StatusConflict) | ||
| 803 | return | ||
| 804 | } | ||
| 805 | |||
| 784 | // Install the SSH key into user-supplied cloud-init. When only one of the | 806 | // Install the SSH key into user-supplied cloud-init. When only one of the |
| 785 | // two is set the seed builder handles it (verbatim user-data, or the | 807 | // two is set the seed builder handles it (verbatim user-data, or the |
| 786 | // generated default template); it's the BOTH case that used to silently | 808 | // generated default template); it's the BOTH case that used to silently |
internal/server/api/api_test.go
| Old | New | ||
|---|---|---|---|
| @@ -14,6 +14,7 @@ import ( | |||
| 14 | "github.com/a73x/eitri/internal/joinblob" | 14 | "github.com/a73x/eitri/internal/joinblob" |
| 15 | "github.com/a73x/eitri/internal/server/hub" | 15 | "github.com/a73x/eitri/internal/server/hub" |
| 16 | "github.com/a73x/eitri/internal/server/registry" | 16 | "github.com/a73x/eitri/internal/server/registry" |
| 17 | "github.com/a73x/eitri/internal/server/release" | ||
| 17 | "github.com/a73x/eitri/internal/server/store" | 18 | "github.com/a73x/eitri/internal/server/store" |
| 18 | "github.com/stretchr/testify/assert" | 19 | "github.com/stretchr/testify/assert" |
| 19 | "github.com/stretchr/testify/require" | 20 | "github.com/stretchr/testify/require" |
| @@ -138,6 +139,7 @@ func enrollArch(t *testing.T, ts *httptest.Server, name, os, arch, prov string) | |||
| 138 | require.Equal(t, 201, resp.StatusCode) | 139 | require.Equal(t, 201, resp.StatusCode) |
| 139 | var out map[string]string | 140 | var out map[string]string |
| 140 | json.NewDecoder(resp.Body).Decode(&out) | 141 | json.NewDecoder(resp.Body).Decode(&out) |
| 142 | agentJoins(out["host_id"]) | ||
| 141 | return out | 143 | return out |
| 142 | } | 144 | } |
| 143 | 145 | ||
| @@ -271,6 +273,29 @@ func mintTestPAT(t *testing.T, st *store.Store) { | |||
| 271 | testPAT = secret | 273 | testPAT = secret |
| 272 | } | 274 | } |
| 273 | 275 | ||
| 276 | // testAgentVersion is what the enrol helpers report on a joining host's behalf: | ||
| 277 | // a release that certifies guest host keys, so a host these tests enrol is | ||
| 278 | // never one an operator would be told to upgrade. | ||
| 279 | const testAgentVersion = release.FirstCertifiedHostKeys | ||
| 280 | |||
| 281 | // testReg is the registry newServer built. Like testPAT it is package-level | ||
| 282 | // because the enrol helpers take only the server, and what an agent said about | ||
| 283 | // itself lives in the registry, not the store. | ||
| 284 | var testReg *registry.Registry | ||
| 285 | |||
| 286 | // agentJoins records the Hello a real agent sends the moment after it enrols: | ||
| 287 | // a host is in the fleet precisely because its agent connected and named its | ||
| 288 | // version. It stops there, at connected-but-not-yet-reporting, because several | ||
| 289 | // tests need an enrolled host whose agent is offline (the abandoned-tombstone | ||
| 290 | // sweep, for one). Tests that exercise the certified-host-key precondition | ||
| 291 | // bring their host online themselves with agentReports. | ||
| 292 | func agentJoins(hostID string) { testReg.SetAgentVersion(hostID, testAgentVersion) } | ||
| 293 | |||
| 294 | // agentReports is the first report that follows the Hello: it sets LastSeen, so | ||
| 295 | // registry.Get reads the host as online and the create path will judge what its | ||
| 296 | // agent said about itself. | ||
| 297 | func agentReports(hostID string) { testReg.UpdateReport(hostID, registry.Report{}) } | ||
| 298 | |||
| 274 | // newServer is the shared builder. It also returns the *API itself for tests | 299 | // newServer is the shared builder. It also returns the *API itself for tests |
| 275 | // that need post-construction wiring (SetConsoleDialer, SetCertMinter). | 300 | // that need post-construction wiring (SetConsoleDialer, SetCertMinter). |
| 276 | func newServer(t *testing.T) (*httptest.Server, *store.Store, *hub.Hub, *registry.Registry, *API) { | 301 | func newServer(t *testing.T) (*httptest.Server, *store.Store, *hub.Hub, *registry.Registry, *API) { |
| @@ -281,6 +306,7 @@ func newServer(t *testing.T) (*httptest.Server, *store.Store, *hub.Hub, *registr | |||
| 281 | seedTestTenant(t, st) | 306 | seedTestTenant(t, st) |
| 282 | h := hub.New() | 307 | h := hub.New() |
| 283 | reg := registry.New(time.Now) | 308 | reg := registry.New(time.Now) |
| 309 | testReg = reg | ||
| 284 | a := New(Config{ | 310 | a := New(Config{ |
| 285 | HostSecret: []byte("hostsecret"), | 311 | HostSecret: []byte("hostsecret"), |
| 286 | DefaultImages: map[string]DefaultImage{ | 312 | DefaultImages: map[string]DefaultImage{ |
| @@ -356,6 +382,16 @@ func do(t *testing.T, method, url, token string, body any) *http.Response { | |||
| 356 | 382 | ||
| 357 | func enroll(t *testing.T, ts *httptest.Server) map[string]string { | 383 | func enroll(t *testing.T, ts *httptest.Server) map[string]string { |
| 358 | t.Helper() | 384 | t.Helper() |
| 385 | out := enrollSilent(t, ts) | ||
| 386 | agentJoins(out["host_id"]) | ||
| 387 | return out // host_id, credential, bridge_cidr | ||
| 388 | } | ||
| 389 | |||
| 390 | // enrollSilent enrols a host whose agent never connects: the row is in the | ||
| 391 | // store and the registry has never heard of it. That is the state of every | ||
| 392 | // host in the fleet for a moment after the server restarts. | ||
| 393 | func enrollSilent(t *testing.T, ts *httptest.Server) map[string]string { | ||
| 394 | t.Helper() | ||
| 359 | resp := do(t, "POST", ts.URL+"/api/v1/enroll-tokens", testPAT, nil) | 395 | resp := do(t, "POST", ts.URL+"/api/v1/enroll-tokens", testPAT, nil) |
| 360 | require.Equal(t, 201, resp.StatusCode) | 396 | require.Equal(t, 201, resp.StatusCode) |
| 361 | var tok map[string]string | 397 | var tok map[string]string |
| @@ -479,6 +515,104 @@ func TestOneClickCreateFillsDefaultsAndPokesHub(t *testing.T) { | |||
| 479 | } | 515 | } |
| 480 | } | 516 | } |
| 481 | 517 | ||
| 518 | // TestCreateVMRefusesAHostThatCannotCertifyItsGuest pins the create-time half | ||
| 519 | // of the certified-host-key story. A guest whose host key nothing signed cannot | ||
| 520 | // be verified and is unreachable through the gate for the rest of its life, so | ||
| 521 | // the create is refused while an operator can still fix it by upgrading the | ||
| 522 | // agent — the refusal names the host, what it reported, and the endpoint that | ||
| 523 | // upgrades it. A version that cannot be read, and a connected host that has | ||
| 524 | // named none at all, are treated the same as an old one: neither proves the | ||
| 525 | // guest could be certified. | ||
| 526 | func TestCreateVMRefusesAHostThatCannotCertifyItsGuest(t *testing.T) { | ||
| 527 | for _, tc := range []struct{ name, version, want string }{ | ||
| 528 | {"an older release", "v0.0.3", "runs agent v0.0.3, which predates certified host keys (v0.0.4)"}, | ||
| 529 | {"an unstamped build", "dev", "runs agent dev, which predates certified host keys (v0.0.4)"}, | ||
| 530 | {"nothing reported", "", "has reported no agent version"}, | ||
| 531 | } { | ||
| 532 | t.Run(tc.name, func(t *testing.T) { | ||
| 533 | ts, st, _ := testServer(t) | ||
| 534 | out := enroll(t, ts) | ||
| 535 | testReg.SetAgentVersion(out["host_id"], tc.version) | ||
| 536 | agentReports(out["host_id"]) | ||
| 537 | |||
| 538 | resp := do(t, "POST", ts.URL+"/api/v1/vms", testPAT, | ||
| 539 | map[string]any{"host_id": out["host_id"], "name": "doomed"}) | ||
| 540 | require.Equal(t, 409, resp.StatusCode) | ||
| 541 | body, _ := io.ReadAll(resp.Body) | ||
| 542 | msg := string(body) | ||
| 543 | assert.Contains(t, msg, tc.want) | ||
| 544 | assert.Contains(t, msg, "host host-a ("+out["host_id"]+")", "the refusal must name the host to act on") | ||
| 545 | assert.Contains(t, msg, "/api/v1/hosts/"+out["host_id"]+"/upgrade-agent", "the refusal must name the fix") | ||
| 546 | |||
| 547 | vms, err := st.ListVMs() | ||
| 548 | require.NoError(t, err) | ||
| 549 | assert.Empty(t, vms, "a refused create must leave no row behind") | ||
| 550 | }) | ||
| 551 | } | ||
| 552 | } | ||
| 553 | |||
| 554 | // TestCreateVMOnACertifyingAgentIsUntouched is the other side of the same | ||
| 555 | // guard: every version at or past the floor — including a build described past | ||
| 556 | // it — places a VM exactly as before. | ||
| 557 | func TestCreateVMOnACertifyingAgentIsUntouched(t *testing.T) { | ||
| 558 | for _, version := range []string{"v0.0.4", "v0.0.5", "v0.0.5-2-gabc1234"} { | ||
| 559 | t.Run(version, func(t *testing.T) { | ||
| 560 | ts, _, _ := testServer(t) | ||
| 561 | out := enroll(t, ts) | ||
| 562 | testReg.SetAgentVersion(out["host_id"], version) | ||
| 563 | agentReports(out["host_id"]) | ||
| 564 | |||
| 565 | resp := do(t, "POST", ts.URL+"/api/v1/vms", testPAT, | ||
| 566 | map[string]any{"host_id": out["host_id"], "name": "fine"}) | ||
| 567 | assert.Equal(t, 201, resp.StatusCode) | ||
| 568 | }) | ||
| 569 | } | ||
| 570 | } | ||
| 571 | |||
| 572 | // TestCreateVMJudgesOnlyAConnectedHostsAgent pins who the certified-host-key | ||
| 573 | // refusal is allowed to accuse. The registry is in-memory and filled by the | ||
| 574 | // agent's Hello, so a host that is not connected right now has told this server | ||
| 575 | // process nothing: after a restart that is the whole fleet, and the empty | ||
| 576 | // version an absent entry carries must never be read as "too old" and answered | ||
| 577 | // with "upgrade this host" — the wrong diagnosis, delivered during a roll, about | ||
| 578 | // a host running the newest agent there is. A host that has gone quiet is the | ||
| 579 | // same: what it last said is not what the agent that eventually picks this VM up | ||
| 580 | // will be. Both take the create like any host that cannot serve one this second, | ||
| 581 | // and the connect-time refusal remains the backstop. | ||
| 582 | func TestCreateVMJudgesOnlyAConnectedHostsAgent(t *testing.T) { | ||
| 583 | t.Run("absent from the registry", func(t *testing.T) { | ||
| 584 | ts, st, _ := testServer(t) | ||
| 585 | out := enrollSilent(t, ts) | ||
| 586 | |||
| 587 | resp := do(t, "POST", ts.URL+"/api/v1/vms", testPAT, | ||
| 588 | map[string]any{"host_id": out["host_id"], "name": "pending"}) | ||
| 589 | body, _ := io.ReadAll(resp.Body) | ||
| 590 | require.Equal(t, 201, resp.StatusCode, "body: %s", body) | ||
| 591 | assert.NotContains(t, string(body), "upgrade-agent") | ||
| 592 | |||
| 593 | vms, err := st.ListVMs() | ||
| 594 | require.NoError(t, err) | ||
| 595 | assert.Len(t, vms, 1, "the create must land as desired state") | ||
| 596 | }) | ||
| 597 | |||
| 598 | t.Run("connected once, now quiet", func(t *testing.T) { | ||
| 599 | ts, st, _ := testServer(t) | ||
| 600 | out := enroll(t, ts) | ||
| 601 | // The last thing this host said was pre-CSR, and it has not reported | ||
| 602 | // since — stale memory, not a diagnosis. | ||
| 603 | testReg.SetAgentVersion(out["host_id"], "v0.0.3") | ||
| 604 | |||
| 605 | resp := do(t, "POST", ts.URL+"/api/v1/vms", testPAT, | ||
| 606 | map[string]any{"host_id": out["host_id"], "name": "pending"}) | ||
| 607 | body, _ := io.ReadAll(resp.Body) | ||
| 608 | require.Equal(t, 201, resp.StatusCode, "body: %s", body) | ||
| 609 | |||
| 610 | vms, err := st.ListVMs() | ||
| 611 | require.NoError(t, err) | ||
| 612 | assert.Len(t, vms, 1) | ||
| 613 | }) | ||
| 614 | } | ||
| 615 | |||
| 482 | func TestDeleteTombstones(t *testing.T) { | 616 | func TestDeleteTombstones(t *testing.T) { |
| 483 | ts, st, _ := testServer(t) | 617 | ts, st, _ := testServer(t) |
| 484 | out := enroll(t, ts) | 618 | out := enroll(t, ts) |
internal/server/api/decommission_api_test.go
| Old | New | ||
|---|---|---|---|
| @@ -24,13 +24,15 @@ func apiServer(t *testing.T) (*httptest.Server, *API, *store.Store) { | |||
| 24 | require.NoError(t, err) | 24 | require.NoError(t, err) |
| 25 | t.Cleanup(func() { st.Close() }) | 25 | t.Cleanup(func() { st.Close() }) |
| 26 | seedTestTenant(t, st) | 26 | seedTestTenant(t, st) |
| 27 | reg := registry.New(time.Now) | ||
| 28 | testReg = reg | ||
| 27 | a := New(Config{ | 29 | a := New(Config{ |
| 28 | HostSecret: []byte("hostsecret"), | 30 | HostSecret: []byte("hostsecret"), |
| 29 | DefaultImages: map[string]DefaultImage{"amd64": {URL: "http://img", SHA256: strings.Repeat("a", 64)}}, | 31 | DefaultImages: map[string]DefaultImage{"amd64": {URL: "http://img", SHA256: strings.Repeat("a", 64)}}, |
| 30 | AdvertiseHTTP: "http://127.0.0.1:8080", | 32 | AdvertiseHTTP: "http://127.0.0.1:8080", |
| 31 | AdvertiseQUIC: "127.0.0.1:8443", | 33 | AdvertiseQUIC: "127.0.0.1:8443", |
| 32 | ServerCertSHA256: strings.Repeat("c", 64), | 34 | ServerCertSHA256: strings.Repeat("c", 64), |
| 33 | }, st, registry.New(time.Now), hub.New()) | 35 | }, st, reg, hub.New()) |
| 34 | // BYO-CA precondition: VM create requires the tenant to have ≥1 registered | 36 | // BYO-CA precondition: VM create requires the tenant to have ≥1 registered |
| 35 | // SSH user CA. Seed the default tenant so VM-create tests reach the create | 37 | // SSH user CA. Seed the default tenant so VM-create tests reach the create |
| 36 | // path rather than the precondition (mirrors newServer in api_test.go). | 38 | // path rather than the precondition (mirrors newServer in api_test.go). |
internal/server/api/isolation_test.go
| Old | New | ||
|---|---|---|---|
| @@ -47,6 +47,7 @@ func enrollWith(t *testing.T, ts *httptest.Server, pat, name string) string { | |||
| 47 | require.Equal(t, 201, resp.StatusCode) | 47 | require.Equal(t, 201, resp.StatusCode) |
| 48 | var out map[string]string | 48 | var out map[string]string |
| 49 | require.NoError(t, json.NewDecoder(resp.Body).Decode(&out)) | 49 | require.NoError(t, json.NewDecoder(resp.Body).Decode(&out)) |
| 50 | agentJoins(out["host_id"]) | ||
| 50 | return out["host_id"] | 51 | return out["host_id"] |
| 51 | } | 52 | } |
| 52 | 53 | ||
internal/server/api/upgrade.go
| Old | New | ||
|---|---|---|---|
| @@ -3,11 +3,35 @@ package api | |||
| 3 | import ( | 3 | import ( |
| 4 | "database/sql" | 4 | "database/sql" |
| 5 | "errors" | 5 | "errors" |
| 6 | "fmt" | ||
| 6 | "net/http" | 7 | "net/http" |
| 7 | 8 | ||
| 8 | "github.com/a73x/eitri/internal/server/release" | 9 | "github.com/a73x/eitri/internal/server/release" |
| 9 | ) | 10 | ) |
| 10 | 11 | ||
| 12 | // upgradeAgentPath is the endpoint that offers one host's agent an upgrade — | ||
| 13 | // the route the console's upgrade button calls, and the fix every "this agent | ||
| 14 | // is too old" refusal names. | ||
| 15 | func upgradeAgentPath(hostID string) string { return "/api/v1/hosts/" + hostID + "/upgrade-agent" } | ||
| 16 | |||
| 17 | // precsrRefusal explains why a host cannot be given a VM: its agent predates | ||
| 18 | // certified guest host keys. The two readings — a version below the floor, and | ||
| 19 | // no reported version at all, which proves nothing either way — differ only in | ||
| 20 | // what is known, so they share the consequence and the fix. Both describe a | ||
| 21 | // host that is connected and reporting: silence from one that is up is an agent | ||
| 22 | // too old to name itself, which is why it reads the same as an old version. | ||
| 23 | func precsrRefusal(hostName, hostID, agentVersion, upgradeURL string) string { | ||
| 24 | known := fmt.Sprintf("host %s (%s) runs agent %s, which predates certified host keys (%s)", | ||
| 25 | hostName, hostID, agentVersion, release.FirstCertifiedHostKeys) | ||
| 26 | if agentVersion == "" { | ||
| 27 | known = fmt.Sprintf("host %s (%s) has reported no agent version, so nothing says it is new enough for certified host keys (%s)", | ||
| 28 | hostName, hostID, release.FirstCertifiedHostKeys) | ||
| 29 | } | ||
| 30 | return known + ": a guest created there gets no certified host key, so nothing could verify it and it would be " + | ||
| 31 | "unreachable through the gate. Upgrade that host's agent — the console's upgrade button, or POST " + | ||
| 32 | upgradeURL + " — then create the VM." | ||
| 33 | } | ||
| 34 | |||
| 11 | // handleUpgradeAgent records a pending self-upgrade offer for one host's agent | 35 | // handleUpgradeAgent records a pending self-upgrade offer for one host's agent |
| 12 | // and pokes its snapshot stream. The human is the rollout controller: nothing | 36 | // and pokes its snapshot stream. The human is the rollout controller: nothing |
| 13 | // upgrades without this per-host click, so a bad release stops at one host. | 37 | // upgrades without this per-host click, so a bad release stops at one host. |
internal/server/release/release.go
| Old | New | ||
|---|---|---|---|
| @@ -138,9 +138,40 @@ func Less(a, b string) bool { | |||
| 138 | if !oka || !okb { | 138 | if !oka || !okb { |
| 139 | return false | 139 | return false |
| 140 | } | 140 | } |
| 141 | for i := range pa { | 141 | return before(pa, pb) |
| 142 | if pa[i] != pb[i] { | 142 | } |
| 143 | return pa[i] < pb[i] | 143 | |
| 144 | // FirstCertifiedHostKeys is the release whose agent began generating each | ||
| 145 | // guest's SSH host key and submitting the public half for the control plane to | ||
| 146 | // sign. A VM created by anything older carries no host certificate and can | ||
| 147 | // never be issued one — the key it would certify was never reported — so | ||
| 148 | // nothing can verify that guest at connect time. | ||
| 149 | const FirstCertifiedHostKeys = "v0.0.4" | ||
| 150 | |||
| 151 | // CertifiesGuestHostKeys reports whether an agent at version v certifies the | ||
| 152 | // host keys of the guests it creates, ordering v against | ||
| 153 | // FirstCertifiedHostKeys by the same rule Less publishes. | ||
| 154 | // | ||
| 155 | // An unparsable version — "dev", a "-dirty" tree, a malformed tag — does not: | ||
| 156 | // nothing can be proven about a version that cannot be read, and this is the | ||
| 157 | // same conservative reading the upgrade path takes when it never offers such a | ||
| 158 | // build an upgrade. So is the empty version a host reports before it has said | ||
| 159 | // anything at all. | ||
| 160 | func CertifiesGuestHostKeys(v string) bool { | ||
| 161 | p, ok := parse(v) | ||
| 162 | if !ok { | ||
| 163 | return false | ||
| 164 | } | ||
| 165 | // The floor is a release tag, so it parses by construction (pinned by test). | ||
| 166 | floor, _ := parse(FirstCertifiedHostKeys) | ||
| 167 | return !before(p, floor) | ||
| 168 | } | ||
| 169 | |||
| 170 | // before compares two ordering tuples, the one place their fields are ranked. | ||
| 171 | func before(a, b [6]int) bool { | ||
| 172 | for i := range a { | ||
| 173 | if a[i] != b[i] { | ||
| 174 | return a[i] < b[i] | ||
| 144 | } | 175 | } |
| 145 | } | 176 | } |
| 146 | return false | 177 | return false |
internal/server/release/release_test.go
| Old | New | ||
|---|---|---|---|
| @@ -138,6 +138,45 @@ func TestLessLeavesUnstampedBuildsUnordered(t *testing.T) { | |||
| 138 | } | 138 | } |
| 139 | } | 139 | } |
| 140 | 140 | ||
| 141 | // TestCertifiesGuestHostKeys pins the capability question the create path asks | ||
| 142 | // of a host: does this agent generate a guest host key for the plane to sign? | ||
| 143 | // The floor is a release, so every build of it and everything after qualifies, | ||
| 144 | // while a pre-release of the floor does not — it is genuinely older. | ||
| 145 | func TestCertifiesGuestHostKeys(t *testing.T) { | ||
| 146 | yes := []string{"v0.0.4", "v0.0.4-2-gabc1234", "v0.0.5", "v0.1.0", "v1.0.0"} | ||
| 147 | for _, v := range yes { | ||
| 148 | if !CertifiesGuestHostKeys(v) { | ||
| 149 | t.Errorf("CertifiesGuestHostKeys(%q) = false, want true", v) | ||
| 150 | } | ||
| 151 | } | ||
| 152 | no := []string{"v0.0.3", "v0.0.1", "v0.0.4-pre.9", "v0.0.3-7-gabc1234"} | ||
| 153 | for _, v := range no { | ||
| 154 | if CertifiesGuestHostKeys(v) { | ||
| 155 | t.Errorf("CertifiesGuestHostKeys(%q) = true, want false", v) | ||
| 156 | } | ||
| 157 | } | ||
| 158 | } | ||
| 159 | |||
| 160 | // TestCertifiesGuestHostKeysRefusesTheUnreadable holds the conservative line: | ||
| 161 | // a version nothing can parse — including the empty one a host reports before | ||
| 162 | // it has said anything — proves nothing, so it is treated as too old, exactly | ||
| 163 | // as the upgrade path never offers such a build an upgrade. | ||
| 164 | func TestCertifiesGuestHostKeysRefusesTheUnreadable(t *testing.T) { | ||
| 165 | for _, v := range []string{"", "dev", "v0.0.9-dirty", "v0.0", "latest"} { | ||
| 166 | if CertifiesGuestHostKeys(v) { | ||
| 167 | t.Errorf("CertifiesGuestHostKeys(%q) = true, want false", v) | ||
| 168 | } | ||
| 169 | } | ||
| 170 | } | ||
| 171 | |||
| 172 | // The floor is spelled as a version, and every refusal quotes it; a typo would | ||
| 173 | // silently refuse (or admit) the whole fleet. | ||
| 174 | func TestFirstCertifiedHostKeysIsAReleaseTag(t *testing.T) { | ||
| 175 | if _, ok := parse(FirstCertifiedHostKeys); !ok { | ||
| 176 | t.Fatalf("FirstCertifiedHostKeys = %q, which does not parse as a version", FirstCertifiedHostKeys) | ||
| 177 | } | ||
| 178 | } | ||
| 179 | |||
| 141 | func TestRefreshParsesManifest(t *testing.T) { | 180 | func TestRefreshParsesManifest(t *testing.T) { |
| 142 | srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { | 181 | srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 143 | w.Write([]byte(`{"version":"v0.0.2","artifacts":{"eitri-agent":{"linux/amd64":{"url":"https://eitri.sh/dl/v0.0.2/a.tar.gz","sha256":"ab"}}}}`)) | 182 | w.Write([]byte(`{"version":"v0.0.2","artifacts":{"eitri-agent":{"linux/amd64":{"url":"https://eitri.sh/dl/v0.0.2/a.tar.gz","sha256":"ab"}}}}`)) |