internal/server/vmssh/vmssh_test.go
Ref: Size: 20.0 KiB History
package vmssh
import (
"bytes"
"context"
"crypto/ed25519"
"crypto/rand"
"errors"
"io"
"net"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/crypto/ssh"
)
// ── scaffolding ──────────────────────────────────────────────────────────────
func newSigner(t *testing.T) ssh.Signer {
t.Helper()
_, priv, err := ed25519.GenerateKey(rand.Reader)
require.NoError(t, err)
s, err := ssh.NewSignerFromSigner(priv)
require.NoError(t, err)
return s
}
// hostCertSigner builds a host-cert-backed signer for principal, signed by ca —
// what a VM presents once eitri has minted its host certificate.
func hostCertSigner(t *testing.T, ca ssh.Signer, principal string) ssh.Signer {
t.Helper()
hostKey := newSigner(t)
cert := &ssh.Certificate{
Key: hostKey.PublicKey(),
CertType: ssh.HostCert,
ValidPrincipals: []string{principal},
ValidBefore: ssh.CertTimeInfinity,
}
require.NoError(t, cert.SignCert(rand.Reader, ca))
cs, err := ssh.NewCertSigner(cert, hostKey)
require.NoError(t, err)
return cs
}
// caUserAuth mirrors a guest's TrustedUserCAKeys policy: only certificates
// signed by the tenant's CA, and the principal must match the login user.
func caUserAuth(ca ssh.PublicKey) func(ssh.ConnMetadata, ssh.PublicKey) (*ssh.Permissions, error) {
checker := &ssh.CertChecker{
IsUserAuthority: func(auth ssh.PublicKey) bool {
return auth != nil && ca != nil && bytes.Equal(auth.Marshal(), ca.Marshal())
},
}
return func(meta ssh.ConnMetadata, key ssh.PublicKey) (*ssh.Permissions, error) {
cert, ok := key.(*ssh.Certificate)
if !ok || cert.CertType != ssh.UserCert {
return nil, errors.New("only user certificates are accepted")
}
if !checker.IsUserAuthority(cert.SignatureKey) {
return nil, errors.New("certificate not signed by a trusted CA")
}
if err := checker.CheckCert(meta.User(), cert); err != nil {
return nil, err
}
return &ssh.Permissions{}, nil
}
}
// fakeTunnel is a TCPDialer whose pipe is served by an in-memory sshd. It
// records the (hostID, vmID, port) it was asked for, which is how the tests see
// what the dialer resolved.
type fakeTunnel struct {
hostSigner ssh.Signer
userCA ssh.PublicKey
execOut string
openErr error
hostID string
vmID string
port uint32
}
// OpenTCP hands back a byte pipe to an sshd standing in for the guest. It is
// backed by a loopback socket rather than net.Pipe: an SSH handshake opens with
// both ends writing their version string, which an unbuffered pipe deadlocks on.
func (f *fakeTunnel) OpenTCP(_ context.Context, hostID, vmID string, port uint32) (io.ReadWriteCloser, error) {
f.hostID, f.vmID, f.port = hostID, vmID, port
if f.openErr != nil {
return nil, f.openErr
}
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
return nil, err
}
conf := &ssh.ServerConfig{PublicKeyCallback: caUserAuth(f.userCA)}
conf.AddHostKey(f.hostSigner)
go func() {
defer ln.Close()
nc, err := ln.Accept()
if err != nil {
return
}
serveGuest(nc, conf, f.execOut)
}()
return net.Dial("tcp", ln.Addr().String())
}
func serveGuest(nc net.Conn, conf *ssh.ServerConfig, out string) {
sc, chans, reqs, err := ssh.NewServerConn(nc, conf)
if err != nil {
nc.Close()
return
}
defer sc.Close()
go ssh.DiscardRequests(reqs)
for newCh := range chans {
if newCh.ChannelType() != "session" {
newCh.Reject(ssh.UnknownChannelType, "only session")
continue
}
ch, chReqs, err := newCh.Accept()
if err != nil {
continue
}
go func() {
defer ch.Close()
for req := range chReqs {
if req.Type == "exec" {
req.Reply(true, nil)
io.WriteString(ch, out)
ch.SendRequest("exit-status", false, ssh.Marshal(struct{ Status uint32 }{0}))
return
}
if req.WantReply {
req.Reply(false, nil)
}
}
}()
}
}
// fakeCreds answers what the tenant has lent eitri, from fixed values.
type fakeCreds struct {
signer ssh.Signer
hasCA bool
err error
}
func (f fakeCreds) Delegated(string) (ssh.Signer, bool) { return f.signer, f.signer != nil }
func (f fakeCreds) TenantHasUserCA(string) (bool, error) {
if f.err != nil {
return false, f.err
}
return f.hasCA, nil
}
// delegatedSigner builds the credential a completed delegation leaves eitri
// with: an ephemeral key plus a user certificate its tenant's CA signed.
func delegatedSigner(t *testing.T, ca ssh.Signer, principal string) ssh.Signer {
t.Helper()
key := newSigner(t)
cert := &ssh.Certificate{
Key: key.PublicKey(),
CertType: ssh.UserCert,
KeyId: "eitri-delegation",
ValidPrincipals: []string{principal},
ValidBefore: ssh.CertTimeInfinity,
}
require.NoError(t, cert.SignCert(rand.Reader, ca))
cs, err := ssh.NewCertSigner(cert, key)
require.NoError(t, err)
return cs
}
// lookupOne resolves exactly one name, for one tenant, to a certified VM whose
// row froze the CA fingerprints in trusted. No fingerprints stands for a row
// that recorded no set at all.
func lookupOne(tenant, name, hostID, vmID string, trusted ...string) VMLookup {
return func(gotTenant, gotName string) (VM, bool) {
if gotTenant != tenant || gotName != name {
return VM{}, false
}
return VM{HostID: hostID, VMID: vmID, HostCertified: true, TrustedCAFingerprints: trusted}, true
}
}
// newDialer wires a dialer whose tenant has delegated a live credential,
// against a guest that trusts the CA behind it.
func newDialer(t *testing.T) (*Dialer, *fakeTunnel) {
t.Helper()
hostCA := newSigner(t)
tenantCA := newSigner(t)
tun := &fakeTunnel{
hostSigner: hostCertSigner(t, hostCA, "acme.web-1"),
userCA: tenantCA.PublicKey(),
execOut: "hi\n",
}
return &Dialer{
Tenant: "acme",
VMUser: "ubuntu",
TCP: tun,
Lookup: lookupOne("acme", "web-1", "h-1", "v-1", ssh.FingerprintSHA256(tenantCA.PublicKey())),
Creds: fakeCreds{signer: delegatedSigner(t, tenantCA, "ubuntu"), hasCA: true},
HostCA: hostCA.PublicKey(),
}, tun
}
// ── tests ────────────────────────────────────────────────────────────────────
// TestDialReachesTheGuest is the whole path: resolve, tunnel to port 22,
// authenticate with the delegated credential, verify the guest's host
// certificate under its namespaced name, run a command.
func TestDialReachesTheGuest(t *testing.T) {
d, tun := newDialer(t)
client, err := d.Dial(t.Context(), "web-1")
require.NoError(t, err)
defer client.Close()
assert.Equal(t, "h-1", tun.hostID)
assert.Equal(t, "v-1", tun.vmID)
assert.Equal(t, uint32(22), tun.port)
sess, err := client.NewSession()
require.NoError(t, err)
defer sess.Close()
out, err := sess.Output("echo hi")
require.NoError(t, err)
assert.Equal(t, "hi\n", string(out))
}
// TestConnectNameIsNamespaced pins the name a guest's host certificate carries.
func TestConnectNameIsNamespaced(t *testing.T) {
d, _ := newDialer(t)
name, err := d.ConnectName(t.Context(), "web-1")
require.NoError(t, err)
assert.Equal(t, "acme.web-1", name)
}
// TestDialRejectsAForeignHostCA: a guest whose host certificate was signed by
// some other CA is not this fleet's guest.
func TestDialRejectsAForeignHostCA(t *testing.T) {
d, tun := newDialer(t)
tun.hostSigner = hostCertSigner(t, newSigner(t), "acme.web-1")
_, err := d.Dial(t.Context(), "web-1")
require.Error(t, err)
assert.Contains(t, err.Error(), "vm web-1 ssh handshake")
}
// TestDialRejectsAHostCertForAnotherVM: the certificate must name THIS VM, or a
// tunnel pointed at the wrong guest would go unnoticed.
func TestDialRejectsAHostCertForAnotherVM(t *testing.T) {
d, tun := newDialer(t)
hostCA := newSigner(t)
d.HostCA = hostCA.PublicKey()
tun.hostSigner = hostCertSigner(t, hostCA, "acme.other-vm")
_, err := d.Dial(t.Context(), "web-1")
require.Error(t, err)
assert.Contains(t, err.Error(), "vm web-1 ssh handshake")
}
// TestDialOnAVMThatDoesNotTrustTheDelegatedCA: the guest baked its CA set at
// create, so a delegation signed by a CA registered later is refused by it. The
// refusal has to say why, because "permission denied" reads as a key problem
// when it is an ordering one — and the row proves the claim, so it is safe to
// make.
func TestDialOnAVMThatDoesNotTrustTheDelegatedCA(t *testing.T) {
d, tun := newDialer(t)
other := newSigner(t)
tun.userCA = other.PublicKey() // the guest trusts some other CA
d.Lookup = lookupOne("acme", "web-1", "h-1", "v-1", ssh.FingerprintSHA256(other.PublicKey()))
_, err := d.Dial(t.Context(), "web-1")
require.Error(t, err)
assert.Contains(t, err.Error(), "refused eitri's certificate")
assert.Contains(t, err.Error(), "the CA set it was created with")
}
// TestDialOnAVMThatDoesTrustTheDelegatedCA is the same refusal from the guest
// with the row disagreeing about the cause: the frozen set names the delegating
// CA, so the CA-set story is false and the message must point at the guest.
func TestDialOnAVMThatDoesTrustTheDelegatedCA(t *testing.T) {
d, tun := newDialer(t)
tun.userCA = newSigner(t).PublicKey() // the guest refuses, though its row says it should not
_, err := d.Dial(t.Context(), "web-1")
require.Error(t, err)
assert.Contains(t, err.Error(), "refused eitri's certificate")
assert.Contains(t, err.Error(), "does include the CA you delegated with")
assert.NotContains(t, err.Error(), "create a new VM")
}
// TestDialRefusesAForeignName: a name in another tenant answers exactly like a
// missing one — existence is never leaked across tenants.
func TestDialRefusesAForeignName(t *testing.T) {
d, _ := newDialer(t)
d.Lookup = lookupOne("other-tenant", "web-1", "h-1", "v-1")
_, err := d.Dial(t.Context(), "web-1")
require.Error(t, err)
assert.Contains(t, err.Error(), `no VM named "web-1"`)
// A name that does not exist at all produces the identical message.
d.Lookup = func(string, string) (VM, bool) { return VM{}, false }
_, missing := d.Dial(t.Context(), "web-1")
require.Error(t, missing)
assert.Equal(t, err.Error(), missing.Error())
}
// TestDialWithNoDelegation is the refusal that matters most: eitri cannot get
// access on its own, so this message is a request, and it has to carry the
// whole recipe.
func TestDialWithNoDelegation(t *testing.T) {
d, _ := newDialer(t)
d.Creds = fakeCreds{hasCA: true}
_, err := d.Dial(t.Context(), "web-1")
require.Error(t, err)
msg := err.Error()
assert.Contains(t, msg, `no live SSH delegation for tenant "acme"`)
assert.Contains(t, msg, "eitri holds no signing key")
assert.Contains(t, msg, "delegate_begin")
assert.Contains(t, msg, "ssh-keygen -s <your-ca> -I eitri-delegation -n ubuntu")
assert.Contains(t, msg, "delegate_complete")
assert.Contains(t, msg, "a control-plane restart drops it")
}
// TestDialWithNoRegisteredCAAtAll adds the step before the recipe: there is
// nothing for a delegated certificate to chain to yet.
func TestDialWithNoRegisteredCAAtAll(t *testing.T) {
d, _ := newDialer(t)
d.Creds = fakeCreds{}
_, err := d.Dial(t.Context(), "web-1")
require.Error(t, err)
assert.Contains(t, err.Error(), "no registered SSH user CA at all")
assert.Contains(t, err.Error(), "eitri ca upload")
// And still the recipe, because that is the step after it.
assert.Contains(t, err.Error(), "delegate_begin")
}
// TestAnExpiredDelegationReadsAsNone: Delegated already dropped it, and the
// same message is the correct one — it says how to renew.
func TestAnExpiredDelegationReadsAsNone(t *testing.T) {
d, _ := newDialer(t)
d.Creds = fakeCreds{hasCA: true} // an expired entry reports itself absent
_, err := d.Dial(t.Context(), "web-1")
require.Error(t, err)
assert.Contains(t, err.Error(), "no live SSH delegation")
}
// TestDialWithoutAJumpGate: no host CA means guests carry no host certificate,
// so there is nothing to verify and no safe connection to make.
func TestDialWithoutAJumpGate(t *testing.T) {
d, _ := newDialer(t)
d.HostCA = nil
_, err := d.Dial(t.Context(), "web-1")
require.Error(t, err)
assert.Contains(t, err.Error(), "no SSH CA configured")
}
// TestDialWhenTheTunnelRefuses surfaces an offline host as an unreachable VM
// rather than a CA problem.
func TestDialWhenTheTunnelRefuses(t *testing.T) {
d, tun := newDialer(t)
tun.openErr = errors.New("host offline")
_, err := d.Dial(t.Context(), "web-1")
require.Error(t, err)
assert.Contains(t, err.Error(), "vm web-1 unreachable")
}
// TestDialSurfacesACAStoreFailureWithoutDetail: a lookup failure must not turn
// into a message that describes the database.
func TestDialSurfacesACAStoreFailureWithoutDetail(t *testing.T) {
d, _ := newDialer(t)
d.Creds = fakeCreds{err: errors.New("disk on fire")}
_, err := d.Dial(t.Context(), "web-1")
require.Error(t, err)
assert.NotContains(t, err.Error(), "disk on fire")
}
// TestPipeConnDeadlinesRefuse pins the adapter's honesty: a caller that sets a
// deadline learns it did nothing, instead of trusting a silent no-op.
func TestPipeConnDeadlinesRefuse(t *testing.T) {
client, server := net.Pipe()
defer client.Close()
defer server.Close()
c := pipeConn{client}
assert.ErrorIs(t, c.SetDeadline(time.Now()), errors.ErrUnsupported)
assert.ErrorIs(t, c.SetReadDeadline(time.Now()), errors.ErrUnsupported)
assert.ErrorIs(t, c.SetWriteDeadline(time.Now()), errors.ErrUnsupported)
assert.Equal(t, "eitri-sync", c.LocalAddr().Network())
assert.Equal(t, "tunnel", c.RemoteAddr().String())
}
// TestDialRefusesAVMThatPredatesCertifiedHostKeys: a guest with no host
// certificate can never acquire one without being recreated, so the refusal is
// a lifecycle instruction and it happens BEFORE a dial. The raw handshake
// failure ("non-certificate host key") reads as a crypto problem and is a dead
// end for anyone trying to fix it.
func TestDialRefusesAVMThatPredatesCertifiedHostKeys(t *testing.T) {
d, tun := newDialer(t)
d.Lookup = func(string, string) (VM, bool) {
return VM{HostID: "h-1", VMID: "v-1", HostCertified: false}, true
}
_, err := d.Dial(t.Context(), "web-1")
require.Error(t, err)
assert.Contains(t, err.Error(), "vm web-1 predates certified host keys")
assert.Contains(t, err.Error(), "Upgrade that host's agent, then recreate the VM")
assert.Empty(t, tun.hostID, "nothing may be dialed for a VM that could never be verified")
}
// TestRefusalNamesADialableEndpoint: the REST API is not necessarily on the
// host the caller is talking to, so the recipe carries a URL, not a path.
func TestRefusalNamesADialableEndpoint(t *testing.T) {
d, _ := newDialer(t)
d.Creds = fakeCreds{hasCA: true}
d.DelegationsURL = "https://console.eitri.sh/api/v1/delegations"
_, err := d.Dial(t.Context(), "web-1")
require.Error(t, err)
assert.Contains(t, err.Error(), "POST https://console.eitri.sh/api/v1/delegations")
// Unconfigured, the recipe still reads — just less usefully.
d.DelegationsURL = ""
_, err = d.Dial(t.Context(), "web-1")
require.Error(t, err)
assert.Contains(t, err.Error(), "POST /api/v1/delegations")
}
// ── handshakeError ───────────────────────────────────────────────────────────
// fp builds a distinct SHA256 fingerprint string to compare by value. The
// handshake refusal only ever compares these as strings.
func fp(s string) string { return "SHA256:" + s }
// TestHandshakeErrorClaimsTheCASetOnlyWhenTheRowProvesIt: the frozen set is
// known and does not name the delegating CA. That is the one case where the
// ordering story is true, so it is the one case that tells it.
func TestHandshakeErrorClaimsTheCASetOnlyWhenTheRowProvesIt(t *testing.T) {
err := handshakeError("web-1", errors.New("ssh: unable to authenticate, attempted methods [none publickey]"),
certTrust{delegatedCA: fp("aaa"), vmTrusts: []string{fp("bbb"), fp("ccc")}})
require.Error(t, err)
assert.Contains(t, err.Error(), "vm web-1 refused eitri's certificate")
assert.Contains(t, err.Error(), "does not include the CA you delegated with")
assert.Contains(t, err.Error(), "create a new VM")
}
// TestHandshakeErrorPointsAtTheGuestWhenTheCAIsTrusted is the bug this branch
// exists for: a live VM whose frozen set DID name the delegating CA was still
// refusing, and the CA-set message sent the operator to re-delegate instead of
// at the guest.
func TestHandshakeErrorPointsAtTheGuestWhenTheCAIsTrusted(t *testing.T) {
err := handshakeError("web-1", errors.New("ssh: unable to authenticate, attempted methods [none publickey]"),
certTrust{delegatedCA: fp("aaa"), vmTrusts: []string{fp("aaa"), fp("bbb")}})
require.Error(t, err)
msg := err.Error()
assert.Contains(t, msg, "vm web-1 refused eitri's certificate")
assert.Contains(t, msg, "does include the CA you delegated with ("+fp("aaa")+")")
// The four things that make a guest refuse a certificate it should accept.
assert.Contains(t, msg, "/etc/ssh/eitri_user_ca.pub")
assert.Contains(t, msg, "sshd drop-in")
assert.Contains(t, msg, "clock")
assert.Contains(t, msg, "principal")
assert.Contains(t, msg, "console")
// And none of the advice that belongs to the other branch.
assert.NotContains(t, msg, "create a new VM")
assert.NotContains(t, msg, "does not include")
}
// TestHandshakeErrorOnAnUnrecordedTrustSet: a row written before the set was
// recorded proves nothing either way, so the refusal says so rather than
// guessing at the CA set.
func TestHandshakeErrorOnAnUnrecordedTrustSet(t *testing.T) {
err := handshakeError("web-1", errors.New("ssh: unable to authenticate, attempted methods [none publickey]"),
certTrust{delegatedCA: fp("aaa")})
require.Error(t, err)
msg := err.Error()
assert.Contains(t, msg, "recorded no trusted CA set")
assert.Contains(t, msg, fp("aaa"))
assert.Contains(t, msg, "console")
assert.NotContains(t, msg, "does not include the CA you delegated with")
assert.NotContains(t, msg, "create a new VM")
}
// TestHandshakeErrorWillNotAccuseASetItCannotRead: a frozen entry whose stored
// line would not parse carries no fingerprint, and the CA it names could be the
// delegating one. An unnameable entry therefore blocks the mismatch claim.
func TestHandshakeErrorWillNotAccuseASetItCannotRead(t *testing.T) {
err := handshakeError("web-1", errors.New("ssh: unable to authenticate, attempted methods [none publickey]"),
certTrust{delegatedCA: fp("aaa"), vmTrusts: []string{"", fp("bbb")}})
require.Error(t, err)
assert.NotContains(t, err.Error(), "does not include the CA you delegated with")
assert.Contains(t, err.Error(), "console")
}
// TestHandshakeErrorWithNoDelegatedFingerprint: nothing to compare is not a
// mismatch either, and the message must not print an empty fingerprint at the
// reader.
func TestHandshakeErrorWithNoDelegatedFingerprint(t *testing.T) {
err := handshakeError("web-1", errors.New("ssh: unable to authenticate, attempted methods [none publickey]"),
certTrust{vmTrusts: []string{fp("bbb")}})
require.Error(t, err)
assert.NotContains(t, err.Error(), "does not include the CA you delegated with")
assert.NotContains(t, err.Error(), "(SHA256:)")
assert.Contains(t, err.Error(), "console")
}
// TestHandshakeErrorLeavesEveryOtherFailureAlone: only an authentication
// failure gets a story. Everything else is reported as what it was.
func TestHandshakeErrorLeavesEveryOtherFailureAlone(t *testing.T) {
cause := errors.New("ssh: handshake failed: knownhosts: key mismatch")
err := handshakeError("web-1", cause, certTrust{delegatedCA: fp("aaa"), vmTrusts: []string{fp("bbb")}})
require.Error(t, err)
assert.Contains(t, err.Error(), "vm web-1 ssh handshake")
assert.ErrorIs(t, err, cause)
assert.NotContains(t, err.Error(), "refused eitri's certificate")
}
// TestDelegatedCAFingerprintNamesTheSigningCA: the fingerprint the refusal
// prints is read off the credential eitri actually offered, so it cannot drift
// from what was sent.
func TestDelegatedCAFingerprintNamesTheSigningCA(t *testing.T) {
ca := newSigner(t)
assert.Equal(t, ssh.FingerprintSHA256(ca.PublicKey()), delegatedCAFingerprint(delegatedSigner(t, ca, "ubuntu")))
// A signer holding a bare key names no CA, and the refusal must not invent one.
assert.Empty(t, delegatedCAFingerprint(newSigner(t)))
}