a73x

a362e825

vmssh: the CA set is only blamed when the row proves it

a73x   2026-09-02 09:42

Commit message
vmssh: the CA set is only blamed when the row proves it

Every "unable to authenticate" from a guest was reported as the VM's frozen CA
set not naming the delegating CA, which sent an operator to re-delegate a
credential the row already trusted while the guest's own trust file was the
thing at fault.

handshakeError now argues from evidence: the fingerprint of the CA that signed
the certificate eitri offered, read off that certificate, and the VM's frozen
trusted_cas, threaded through vmLookup. The mismatch story is told only when
the set is known, readable whole, and excludes the delegating CA. Otherwise
the refusal says what the row does settle and names the guest-side causes: the
trust file or sshd drop-in missing or overwritten, a clock outside the
certificate's window, a principal mismatch; and that no tool on this seam can
look, since they all ride the SSH being refused. A row that recorded no set is
unknown, never a mismatch.

Patch: 3497ecbd3bcf2ed748d09b0f7285cb1044d9c5b5

internal/server/boot/sshgate.go
Old New
@@ -251,10 +251,30 @@ func vmLookup(st *store.Store) vmssh.VMLookup {
251 if err != nil { 251 if err != nil {
252 return vmssh.VM{}, false 252 return vmssh.VM{}, false
253 } 253 }
254 return vmssh.VM{HostID: hostID, VMID: vmID, HostCertified: vm.SSHHostCert != ""}, true 254 return vmssh.VM{
255 HostID: hostID, VMID: vmID,
256 HostCertified: vm.SSHHostCert != "",
257 TrustedCAFingerprints: frozenCAFingerprints(vm),
258 }, true
255 } 259 }
256 } 260 }
257 261
262 // frozenCAFingerprints is the CA set a VM was created to trust, as fingerprints
263 // — what a refusal compares a delegated certificate against. A row that
264 // recorded no set answers with none, which reads as "unknown" there. The
265 // authorized_keys lines stay in the row: this path explains trust, it does not
266 // serve it.
267 func frozenCAFingerprints(vm store.VM) []string {
268 if vm.TrustedCAs == nil {
269 return nil
270 }
271 out := make([]string, 0, len(vm.TrustedCAs))
272 for _, ca := range vm.TrustedCAs {
273 out = append(out, ca.Fingerprint)
274 }
275 return out
276 }
277
258 // revokedCert gates every cert auth against the revocation list. Fail-CLOSED 278 // revokedCert gates every cert auth against the revocation list. Fail-CLOSED
259 // for the single connection on a DB error: a store hiccup rejects THAT login 279 // for the single connection on a DB error: a store hiccup rejects THAT login
260 // (returns revoked=true) rather than fail-open (which would let a possibly- 280 // (returns revoked=true) rather than fail-open (which would let a possibly-
internal/server/boot/sshgate_test.go
Old New
@@ -324,3 +324,35 @@ func TestGateAddrNamesTheGateAsItsCertificateDoes(t *testing.T) {
324 }) 324 })
325 } 325 }
326 } 326 }
327
328 // TestVMLookupCarriesTheFrozenTrustSet: the server-side SSH path refuses with a
329 // story about the VM's CA set, so it needs the set itself. A row that recorded
330 // none must arrive empty rather than as a set that trusts nothing — the refusal
331 // reads those as different facts.
332 func TestVMLookupCarriesTheFrozenTrustSet(t *testing.T) {
333 s := newStore(t)
334 tenant, host := makeTenantHost(t, s, "sub-a", "alpha@x.com")
335 unrecorded := makeVM(t, s, host, "old")
336 require.NoError(t, s.RecordVMHostKey(unrecorded.ID, host.ID, "ssh-ed25519 AAAApub g", "cert-line"))
337
338 frozen := store.VM{
339 ID: "vm-web", HostID: host.ID, Name: "web",
340 ImageURL: "http://img", ImageSHA256: "abc",
341 VCPUs: 1, MemMB: 512, DiskGB: 5, PowerState: "running",
342 TrustedCAs: []store.TrustedCA{
343 {Label: "laptop", Fingerprint: "SHA256:aaa", AuthorizedKey: "ssh-ed25519 AAAAca laptop"},
344 {Label: "ci", Fingerprint: "SHA256:bbb", AuthorizedKey: "ssh-ed25519 AAAAci ci"},
345 },
346 }
347 require.NoError(t, s.CreateVM(frozen))
348
349 lookup := vmLookup(s)
350
351 got, ok := lookup(tenant, "web")
352 require.True(t, ok)
353 assert.Equal(t, []string{"SHA256:aaa", "SHA256:bbb"}, got.TrustedCAFingerprints)
354
355 got, ok = lookup(tenant, "old")
356 require.True(t, ok)
357 assert.Empty(t, got.TrustedCAFingerprints, "a row that recorded no set says nothing about what the guest trusts")
358 }
internal/server/vmssh/vmssh.go
Old New
@@ -21,6 +21,7 @@ import (
21 "fmt" 21 "fmt"
22 "io" 22 "io"
23 "net" 23 "net"
24 "slices"
24 "strings" 25 "strings"
25 "time" 26 "time"
26 27
@@ -34,8 +35,8 @@ type TCPDialer interface {
34 OpenTCP(ctx context.Context, hostID, vmID string, port uint32) (io.ReadWriteCloser, error) 35 OpenTCP(ctx context.Context, hostID, vmID string, port uint32) (io.ReadWriteCloser, error)
35 } 36 }
36 37
37 // VM is what a name resolves to: where the guest runs, and whether anything 38 // VM is what a name resolves to: where the guest runs, whether anything could
38 // could verify it on arrival. 39 // verify it on arrival, and the CA set it was built to trust.
39 type VM struct { 40 type VM struct {
40 HostID, VMID string 41 HostID, VMID string
41 // HostCertified reports that the control plane has signed this guest's host 42 // HostCertified reports that the control plane has signed this guest's host
@@ -43,6 +44,15 @@ type VM struct {
43 // certificate and can never acquire one without being recreated, so this is 44 // certificate and can never acquire one without being recreated, so this is
44 // a permanent property of that VM rather than a race to wait out. 45 // a permanent property of that VM rather than a race to wait out.
45 HostCertified bool 46 HostCertified bool
47 // TrustedCAFingerprints is the SHA256 fingerprint of every user CA frozen
48 // onto this VM at create, in OpenSSH's spelling. It is the DESIRED trust the
49 // agent was handed, not a report from the guest, which is why a guest can
50 // refuse a CA that appears here.
51 //
52 // Empty means no set was recorded — a row written before the column
53 // existed. That is "unknown", never "trusts nothing", and the refusal in
54 // handshakeError is entitled to claim nothing about the CA set from it.
55 TrustedCAFingerprints []string
46 } 56 }
47 57
48 // VMLookup resolves a bare VM name WITHIN tenant and re-checks ownership. It 58 // VMLookup resolves a bare VM name WITHIN tenant and re-checks ownership. It
@@ -132,7 +142,10 @@ func (d *Dialer) Dial(ctx context.Context, vmName string) (*ssh.Client, error) {
132 nc, chans, reqs, err := ssh.NewClientConn(pipeConn{pipe}, addr, conf) 142 nc, chans, reqs, err := ssh.NewClientConn(pipeConn{pipe}, addr, conf)
133 if err != nil { 143 if err != nil {
134 pipe.Close() 144 pipe.Close()
135 return nil, handshakeError(vmName, err) 145 return nil, handshakeError(vmName, err, certTrust{
146 delegatedCA: delegatedCAFingerprint(signer),
147 vmTrusts: vm.TrustedCAFingerprints,
148 })
136 } 149 }
137 return ssh.NewClient(nc, chans, reqs), nil 150 return ssh.NewClient(nc, chans, reqs), nil
138 } 151 }
@@ -180,16 +193,86 @@ func (d *Dialer) delegationsURL() string {
180 return d.DelegationsURL 193 return d.DelegationsURL
181 } 194 }
182 195
183 // handshakeError names the one failure a caller can do something about. A guest 196 // certTrust is the evidence a refusal may argue from: which CA signed the
197 // certificate eitri offered, and the CA set the VM froze at create. Either can
198 // be unknown, and an unknown one proves nothing about the other.
199 type certTrust struct {
200 delegatedCA string // SHA256 fingerprint of the CA that signed eitri's certificate
201 vmTrusts []string // SHA256 fingerprints frozen onto the VM; empty when the row recorded none
202 }
203
204 // excludesDelegatedCA reports that the VM's frozen set is known, complete, and
205 // does not name the delegating CA — the only footing from which the CA-set
206 // story below is true.
207 //
208 // An entry with no fingerprint is a CA whose stored line would not parse, so
209 // the set cannot be read whole and might well contain the delegating one. It
210 // blocks the claim rather than narrowing it.
211 func (t certTrust) excludesDelegatedCA() bool {
212 if t.delegatedCA == "" || len(t.vmTrusts) == 0 || slices.Contains(t.vmTrusts, "") {
213 return false
214 }
215 return !slices.Contains(t.vmTrusts, t.delegatedCA)
216 }
217
218 // handshakeError turns an authentication failure into the diagnosis the
219 // evidence supports, and no further.
220 //
221 // Two different faults reach it as the same "unable to authenticate". A guest
184 // bakes its CA set at create, so a delegation signed by a CA registered after 222 // bakes its CA set at create, so a delegation signed by a CA registered after
185 // that VM was made is refused by it — which reaches us as an ordinary 223 // that VM was made is refused for an ordering reason the caller fixes by
186 // authentication failure and would otherwise read as a mystery. 224 // re-delegating. A guest whose own trust file or sshd drop-in has gone refuses
187 func handshakeError(vmName string, err error) error { 225 // a certificate its CA set names, and re-delegating changes nothing there. The
188 if strings.Contains(err.Error(), "unable to authenticate") { 226 // VM's frozen set is what tells the two apart, so it is read before either is
227 // claimed: the ordering story requires a set that is known, readable whole, and
228 // without the delegating CA in it, and everything else points at the guest.
229 func handshakeError(vmName string, err error, trust certTrust) error {
230 if !strings.Contains(err.Error(), "unable to authenticate") {
231 return fmt.Errorf("vm %s ssh handshake: %w", vmName, err)
232 }
233 if trust.excludesDelegatedCA() {
189 return fmt.Errorf("vm %s refused eitri's certificate: it trusts the CA set it was created with, and that "+ 234 return fmt.Errorf("vm %s refused eitri's certificate: it trusts the CA set it was created with, and that "+
190 "set does not include the CA you delegated with. Delegate with a CA this VM trusts, or create a new VM", vmName) 235 "set does not include the CA you delegated with. Delegate with a CA this VM trusts, or create a new VM", vmName)
191 } 236 }
192 return fmt.Errorf("vm %s ssh handshake: %w", vmName, err) 237 // The refusal came from the guest and the CA set does not explain it, so
238 // the message names the things that do — all of them inside the guest,
239 // where nothing on this seam can look: every MCP tool that reaches a guest
240 // goes through the SSH being refused here.
241 return fmt.Errorf("vm %s refused eitri's certificate, and %s. Something inside the guest is refusing it: "+
242 "/etc/ssh/eitri_user_ca.pub or the sshd drop-in that names it missing or overwritten (an agent older than "+
243 "v0.0.8 let a BYO cloud-init carrying write_files replace the seed's), a guest clock outside the "+
244 "certificate's validity window, or a principal mismatch. No tool here can look: they all go through this "+
245 "same SSH. Open the guest's console and check it there", vmName, trust.setClause())
246 }
247
248 // setClause says what the VM's row does and does not settle, in the words the
249 // refusal above is built around. Naming the delegating CA gives an operator the
250 // fingerprint to compare against the VM's trusted_cas by eye.
251 func (t certTrust) setClause() string {
252 switch {
253 case t.delegatedCA == "":
254 return "nothing here can compare the certificate eitri offered against the set this VM was created with"
255 case len(t.vmTrusts) == 0:
256 return fmt.Sprintf("this VM recorded no trusted CA set, so nothing here can say whether it was created to "+
257 "trust the CA you delegated with (%s)", t.delegatedCA)
258 case slices.Contains(t.vmTrusts, ""):
259 return fmt.Sprintf("this VM's frozen CA set holds an entry nothing here can name, so it cannot say whether "+
260 "the CA you delegated with (%s) is one of them", t.delegatedCA)
261 default:
262 return fmt.Sprintf("the CA set it was created with does include the CA you delegated with (%s)", t.delegatedCA)
263 }
264 }
265
266 // delegatedCAFingerprint names the CA that signed the credential eitri is
267 // holding, read off that credential rather than looked up again — it is the
268 // certificate that was actually offered, so the two cannot disagree. Empty when
269 // the signer carries a bare key and there is no signing CA to name.
270 func delegatedCAFingerprint(s ssh.Signer) string {
271 cert, ok := s.PublicKey().(*ssh.Certificate)
272 if !ok || cert.SignatureKey == nil {
273 return ""
274 }
275 return ssh.FingerprintSHA256(cert.SignatureKey)
193 } 276 }
194 277
195 // keyEquals compares two SSH public keys by their wire encodings. 278 // keyEquals compares two SSH public keys by their wire encodings.
internal/server/vmssh/vmssh_test.go
Old New
@@ -173,13 +173,15 @@ func delegatedSigner(t *testing.T, ca ssh.Signer, principal string) ssh.Signer {
173 return cs 173 return cs
174 } 174 }
175 175
176 // lookupOne resolves exactly one name, for one tenant, to a certified VM. 176 // lookupOne resolves exactly one name, for one tenant, to a certified VM whose
177 func lookupOne(tenant, name, hostID, vmID string) VMLookup { 177 // row froze the CA fingerprints in trusted. No fingerprints stands for a row
178 // that recorded no set at all.
179 func lookupOne(tenant, name, hostID, vmID string, trusted ...string) VMLookup {
178 return func(gotTenant, gotName string) (VM, bool) { 180 return func(gotTenant, gotName string) (VM, bool) {
179 if gotTenant != tenant || gotName != name { 181 if gotTenant != tenant || gotName != name {
180 return VM{}, false 182 return VM{}, false
181 } 183 }
182 return VM{HostID: hostID, VMID: vmID, HostCertified: true}, true 184 return VM{HostID: hostID, VMID: vmID, HostCertified: true, TrustedCAFingerprints: trusted}, true
183 } 185 }
184 } 186 }
185 187
@@ -199,7 +201,7 @@ func newDialer(t *testing.T) (*Dialer, *fakeTunnel) {
199 Tenant: "acme", 201 Tenant: "acme",
200 VMUser: "ubuntu", 202 VMUser: "ubuntu",
201 TCP: tun, 203 TCP: tun,
202 Lookup: lookupOne("acme", "web-1", "h-1", "v-1"), 204 Lookup: lookupOne("acme", "web-1", "h-1", "v-1", ssh.FingerprintSHA256(tenantCA.PublicKey())),
203 Creds: fakeCreds{signer: delegatedSigner(t, tenantCA, "ubuntu"), hasCA: true}, 205 Creds: fakeCreds{signer: delegatedSigner(t, tenantCA, "ubuntu"), hasCA: true},
204 HostCA: hostCA.PublicKey(), 206 HostCA: hostCA.PublicKey(),
205 }, tun 207 }, tun
@@ -264,10 +266,13 @@ func TestDialRejectsAHostCertForAnotherVM(t *testing.T) {
264 // TestDialOnAVMThatDoesNotTrustTheDelegatedCA: the guest baked its CA set at 266 // TestDialOnAVMThatDoesNotTrustTheDelegatedCA: the guest baked its CA set at
265 // create, so a delegation signed by a CA registered later is refused by it. The 267 // create, so a delegation signed by a CA registered later is refused by it. The
266 // refusal has to say why, because "permission denied" reads as a key problem 268 // refusal has to say why, because "permission denied" reads as a key problem
267 // when it is an ordering one. 269 // when it is an ordering one — and the row proves the claim, so it is safe to
270 // make.
268 func TestDialOnAVMThatDoesNotTrustTheDelegatedCA(t *testing.T) { 271 func TestDialOnAVMThatDoesNotTrustTheDelegatedCA(t *testing.T) {
269 d, tun := newDialer(t) 272 d, tun := newDialer(t)
270 tun.userCA = newSigner(t).PublicKey() // the guest trusts some other CA 273 other := newSigner(t)
274 tun.userCA = other.PublicKey() // the guest trusts some other CA
275 d.Lookup = lookupOne("acme", "web-1", "h-1", "v-1", ssh.FingerprintSHA256(other.PublicKey()))
271 276
272 _, err := d.Dial(t.Context(), "web-1") 277 _, err := d.Dial(t.Context(), "web-1")
273 require.Error(t, err) 278 require.Error(t, err)
@@ -275,6 +280,20 @@ func TestDialOnAVMThatDoesNotTrustTheDelegatedCA(t *testing.T) {
275 assert.Contains(t, err.Error(), "the CA set it was created with") 280 assert.Contains(t, err.Error(), "the CA set it was created with")
276 } 281 }
277 282
283 // TestDialOnAVMThatDoesTrustTheDelegatedCA is the same refusal from the guest
284 // with the row disagreeing about the cause: the frozen set names the delegating
285 // CA, so the CA-set story is false and the message must point at the guest.
286 func TestDialOnAVMThatDoesTrustTheDelegatedCA(t *testing.T) {
287 d, tun := newDialer(t)
288 tun.userCA = newSigner(t).PublicKey() // the guest refuses, though its row says it should not
289
290 _, err := d.Dial(t.Context(), "web-1")
291 require.Error(t, err)
292 assert.Contains(t, err.Error(), "refused eitri's certificate")
293 assert.Contains(t, err.Error(), "does include the CA you delegated with")
294 assert.NotContains(t, err.Error(), "create a new VM")
295 }
296
278 // TestDialRefusesAForeignName: a name in another tenant answers exactly like a 297 // TestDialRefusesAForeignName: a name in another tenant answers exactly like a
279 // missing one — existence is never leaked across tenants. 298 // missing one — existence is never leaked across tenants.
280 func TestDialRefusesAForeignName(t *testing.T) { 299 func TestDialRefusesAForeignName(t *testing.T) {
@@ -418,3 +437,109 @@ func TestRefusalNamesADialableEndpoint(t *testing.T) {
418 require.Error(t, err) 437 require.Error(t, err)
419 assert.Contains(t, err.Error(), "POST /api/v1/delegations") 438 assert.Contains(t, err.Error(), "POST /api/v1/delegations")
420 } 439 }
440
441 // ── handshakeError ───────────────────────────────────────────────────────────
442
443 // fp builds a distinct SHA256 fingerprint string to compare by value. The
444 // handshake refusal only ever compares these as strings.
445 func fp(s string) string { return "SHA256:" + s }
446
447 // TestHandshakeErrorClaimsTheCASetOnlyWhenTheRowProvesIt: the frozen set is
448 // known and does not name the delegating CA. That is the one case where the
449 // ordering story is true, so it is the one case that tells it.
450 func TestHandshakeErrorClaimsTheCASetOnlyWhenTheRowProvesIt(t *testing.T) {
451 err := handshakeError("web-1", errors.New("ssh: unable to authenticate, attempted methods [none publickey]"),
452 certTrust{delegatedCA: fp("aaa"), vmTrusts: []string{fp("bbb"), fp("ccc")}})
453
454 require.Error(t, err)
455 assert.Contains(t, err.Error(), "vm web-1 refused eitri's certificate")
456 assert.Contains(t, err.Error(), "does not include the CA you delegated with")
457 assert.Contains(t, err.Error(), "create a new VM")
458 }
459
460 // TestHandshakeErrorPointsAtTheGuestWhenTheCAIsTrusted is the bug this branch
461 // exists for: a live VM whose frozen set DID name the delegating CA was still
462 // refusing, and the CA-set message sent the operator to re-delegate instead of
463 // at the guest.
464 func TestHandshakeErrorPointsAtTheGuestWhenTheCAIsTrusted(t *testing.T) {
465 err := handshakeError("web-1", errors.New("ssh: unable to authenticate, attempted methods [none publickey]"),
466 certTrust{delegatedCA: fp("aaa"), vmTrusts: []string{fp("aaa"), fp("bbb")}})
467
468 require.Error(t, err)
469 msg := err.Error()
470 assert.Contains(t, msg, "vm web-1 refused eitri's certificate")
471 assert.Contains(t, msg, "does include the CA you delegated with ("+fp("aaa")+")")
472 // The four things that make a guest refuse a certificate it should accept.
473 assert.Contains(t, msg, "/etc/ssh/eitri_user_ca.pub")
474 assert.Contains(t, msg, "sshd drop-in")
475 assert.Contains(t, msg, "clock")
476 assert.Contains(t, msg, "principal")
477 assert.Contains(t, msg, "console")
478 // And none of the advice that belongs to the other branch.
479 assert.NotContains(t, msg, "create a new VM")
480 assert.NotContains(t, msg, "does not include")
481 }
482
483 // TestHandshakeErrorOnAnUnrecordedTrustSet: a row written before the set was
484 // recorded proves nothing either way, so the refusal says so rather than
485 // guessing at the CA set.
486 func TestHandshakeErrorOnAnUnrecordedTrustSet(t *testing.T) {
487 err := handshakeError("web-1", errors.New("ssh: unable to authenticate, attempted methods [none publickey]"),
488 certTrust{delegatedCA: fp("aaa")})
489
490 require.Error(t, err)
491 msg := err.Error()
492 assert.Contains(t, msg, "recorded no trusted CA set")
493 assert.Contains(t, msg, fp("aaa"))
494 assert.Contains(t, msg, "console")
495 assert.NotContains(t, msg, "does not include the CA you delegated with")
496 assert.NotContains(t, msg, "create a new VM")
497 }
498
499 // TestHandshakeErrorWillNotAccuseASetItCannotRead: a frozen entry whose stored
500 // line would not parse carries no fingerprint, and the CA it names could be the
501 // delegating one. An unnameable entry therefore blocks the mismatch claim.
502 func TestHandshakeErrorWillNotAccuseASetItCannotRead(t *testing.T) {
503 err := handshakeError("web-1", errors.New("ssh: unable to authenticate, attempted methods [none publickey]"),
504 certTrust{delegatedCA: fp("aaa"), vmTrusts: []string{"", fp("bbb")}})
505
506 require.Error(t, err)
507 assert.NotContains(t, err.Error(), "does not include the CA you delegated with")
508 assert.Contains(t, err.Error(), "console")
509 }
510
511 // TestHandshakeErrorWithNoDelegatedFingerprint: nothing to compare is not a
512 // mismatch either, and the message must not print an empty fingerprint at the
513 // reader.
514 func TestHandshakeErrorWithNoDelegatedFingerprint(t *testing.T) {
515 err := handshakeError("web-1", errors.New("ssh: unable to authenticate, attempted methods [none publickey]"),
516 certTrust{vmTrusts: []string{fp("bbb")}})
517
518 require.Error(t, err)
519 assert.NotContains(t, err.Error(), "does not include the CA you delegated with")
520 assert.NotContains(t, err.Error(), "(SHA256:)")
521 assert.Contains(t, err.Error(), "console")
522 }
523
524 // TestHandshakeErrorLeavesEveryOtherFailureAlone: only an authentication
525 // failure gets a story. Everything else is reported as what it was.
526 func TestHandshakeErrorLeavesEveryOtherFailureAlone(t *testing.T) {
527 cause := errors.New("ssh: handshake failed: knownhosts: key mismatch")
528 err := handshakeError("web-1", cause, certTrust{delegatedCA: fp("aaa"), vmTrusts: []string{fp("bbb")}})
529
530 require.Error(t, err)
531 assert.Contains(t, err.Error(), "vm web-1 ssh handshake")
532 assert.ErrorIs(t, err, cause)
533 assert.NotContains(t, err.Error(), "refused eitri's certificate")
534 }
535
536 // TestDelegatedCAFingerprintNamesTheSigningCA: the fingerprint the refusal
537 // prints is read off the credential eitri actually offered, so it cannot drift
538 // from what was sent.
539 func TestDelegatedCAFingerprintNamesTheSigningCA(t *testing.T) {
540 ca := newSigner(t)
541 assert.Equal(t, ssh.FingerprintSHA256(ca.PublicKey()), delegatedCAFingerprint(delegatedSigner(t, ca, "ubuntu")))
542
543 // A signer holding a bare key names no CA, and the refusal must not invent one.
544 assert.Empty(t, delegatedCAFingerprint(newSigner(t)))
545 }