internal/server/vmssh/vmssh.go
Ref: Size: 13.9 KiB History
// Package vmssh reaches a tenant's VM from inside the control plane: it tunnels
// to the guest's sshd over the host's live sync connection and authenticates
// with the credential that tenant has delegated to eitri. It is the server-side
// half of the MCP exec seam; the client-side half goes through the jump gate.
//
// Taking the tunnel rather than the gate is one hop fewer and the same crypto:
// the guest's host certificate is verified under its <tenant>.<name> principal
// against the host CA, exactly as a client dialing through the gate verifies it.
// It also avoids the server having to resolve and dial its own public gate
// domain, which is unreachable from inside a pod whenever that name points at an
// external address, and absent entirely when the gate is off.
//
// Gate-side certificate revocation is deliberately not consulted here. It does
// not need to be: these certificates live minutes, are used by the process that
// minted them, and are never handed to anyone.
package vmssh
import (
"context"
"errors"
"fmt"
"io"
"net"
"slices"
"strings"
"time"
"github.com/a73x/eitri/internal/names"
"golang.org/x/crypto/ssh"
)
// TCPDialer opens a byte pipe to vmID:port on hostID. *syncsvc.Service satisfies
// it — the same seam the jump gate itself is built on.
type TCPDialer interface {
OpenTCP(ctx context.Context, hostID, vmID string, port uint32) (io.ReadWriteCloser, error)
}
// VM is what a name resolves to: where the guest runs, whether anything could
// verify it on arrival, and the CA set it was built to trust.
type VM struct {
HostID, VMID string
// HostCertified reports that the control plane has signed this guest's host
// key. A guest created by an agent that predates the key exchange has no
// certificate and can never acquire one without being recreated, so this is
// a permanent property of that VM rather than a race to wait out.
HostCertified bool
// TrustedCAFingerprints is the SHA256 fingerprint of every user CA frozen
// onto this VM at create, in OpenSSH's spelling. It is the DESIRED trust the
// agent was handed, not a report from the guest, which is why a guest can
// refuse a CA that appears here.
//
// Empty means no set was recorded — a row written before the column
// existed. That is "unknown", never "trusts nothing", and the refusal in
// handshakeError is entitled to claim nothing about the CA set from it.
TrustedCAFingerprints []string
}
// VMLookup resolves a bare VM name WITHIN tenant and re-checks ownership. It
// mirrors the gate's resolve+authorize pair: a name never resolves across
// tenants, and a tombstoned or foreign VM is indistinguishable from a missing
// one.
type VMLookup func(tenant, name string) (VM, bool)
// Credentials answers what eitri may authenticate as, for one tenant, right
// now. The two questions are separate because the answers call for different
// words: a tenant with a live delegation and one that has never registered a CA
// at all are told different things to do next.
type Credentials interface {
// Delegated returns this tenant's live delegated credential, or false when
// there is none — including one that has expired, which is the same
// situation and calls for the same next step.
Delegated(tenant string) (ssh.Signer, bool)
// TenantHasUserCA reports whether the tenant has registered any CA at all.
// Without one there is nothing a delegated certificate could chain to.
TenantHasUserCA(tenant string) (bool, error)
}
// Dialer reaches one tenant's VMs. It implements the MCP exec seam's VMDialer.
type Dialer struct {
Tenant string
VMUser string // guest login user, and therefore the certificate principal
TCP TCPDialer
Lookup VMLookup
Creds Credentials
// HostCA verifies each guest's host certificate. Nil means this control
// plane has no SSH CA at all (the jump gate is off), so VMs carry no host
// certificate and there is nothing to verify against.
HostCA ssh.PublicKey
// DelegationsURL is where a caller POSTs to start a delegation, as a full
// URL. The refusal below is the only instruction most callers get, and the
// REST API does not necessarily live on the host they are talking to.
DelegationsURL string
}
// ConnectName returns the VM's <tenant>.<name> connect name — the form the gate
// resolves and the VM's host certificate names.
func (d *Dialer) ConnectName(_ context.Context, vmName string) (string, error) {
return names.ConnectName(d.Tenant, vmName), nil
}
// Dial opens an authenticated SSH connection to the VM named vmName. The caller
// must Close the returned client.
func (d *Dialer) Dial(ctx context.Context, vmName string) (*ssh.Client, error) {
if d.HostCA == nil {
return nil, errors.New("remote exec is unavailable: this control plane has no SSH CA configured")
}
signer, err := d.signer()
if err != nil {
return nil, err
}
vm, ok := d.Lookup(d.Tenant, vmName)
if !ok {
return nil, fmt.Errorf("no VM named %q", vmName)
}
// Refuse before dialing. This guest has no host certificate and cannot be
// issued one — its key was never reported — so the handshake would fail on
// an unverifiable host key, which reads as a crypto problem rather than the
// lifecycle one it is.
if !vm.HostCertified {
return nil, fmt.Errorf("vm %s predates certified host keys: it was created by an agent older than the "+
"host-key exchange, so nothing can verify it. Upgrade that host's agent, then recreate the VM", vmName)
}
pipe, err := d.TCP.OpenTCP(ctx, vm.HostID, vm.VMID, 22)
if err != nil {
return nil, fmt.Errorf("vm %s unreachable: %w", vmName, err)
}
// The address is both the tunnel's far end and the name the guest's host
// certificate is checked under — its principal is exactly this.
addr := d.Tenant + "." + vmName + ":22"
checker := &ssh.CertChecker{
IsHostAuthority: func(auth ssh.PublicKey, _ string) bool {
return keyEquals(auth, d.HostCA)
},
}
conf := &ssh.ClientConfig{
User: d.VMUser,
Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)},
HostKeyCallback: checker.CheckHostKey,
Timeout: 15 * time.Second,
}
nc, chans, reqs, err := ssh.NewClientConn(pipeConn{pipe}, addr, conf)
if err != nil {
pipe.Close()
return nil, handshakeError(vmName, err, certTrust{
delegatedCA: delegatedCAFingerprint(signer),
vmTrusts: vm.TrustedCAFingerprints,
})
}
return ssh.NewClient(nc, chans, reqs), nil
}
// signer returns the credential this tenant has delegated, or explains how to
// delegate one. The wording is the feature: eitri cannot obtain access on its
// own, so a refusal here is a request, and it has to be one a caller can act on
// without reading the docs.
func (d *Dialer) signer() (ssh.Signer, error) {
if signer, ok := d.Creds.Delegated(d.Tenant); ok {
return signer, nil
}
var prefix string
has, err := d.Creds.TenantHasUserCA(d.Tenant)
if err != nil {
return nil, errors.New("looking up this tenant's SSH CAs failed")
}
if !has {
prefix = "this tenant has no registered SSH user CA at all — upload one first (`eitri ca upload`), " +
"or your guests will trust nothing you sign.\n\n"
}
// ST1005 wants a one-line lowercase fragment. This message is the feature:
// eitri cannot obtain access on its own, so the refusal IS the request, and
// it has to carry the whole recipe or the caller is left guessing.
//nolint:staticcheck // deliberately multi-line, caller-facing prose
return nil, fmt.Errorf(`%sno live SSH delegation for tenant %q.
eitri holds no signing key — you delegate access to it, and it expires.
1. call delegate_begin (or POST %s) for a public key
2. sign it with your own CA:
ssh-keygen -s <your-ca> -I eitri-delegation -n %s -V +8h eitri-delegation.pub
3. return eitri-delegation-cert.pub via delegate_complete
A delegation lives in memory only: a control-plane restart drops it, and you
delegate again.`, prefix, d.Tenant, d.delegationsURL(), d.VMUser)
}
// delegationsURL is the configured endpoint, or the bare path when a plane has
// not been told its own public URL. Never empty, so the recipe always reads.
func (d *Dialer) delegationsURL() string {
if d.DelegationsURL == "" {
return "/api/v1/delegations"
}
return d.DelegationsURL
}
// certTrust is the evidence a refusal may argue from: which CA signed the
// certificate eitri offered, and the CA set the VM froze at create. Either can
// be unknown, and an unknown one proves nothing about the other.
type certTrust struct {
delegatedCA string // SHA256 fingerprint of the CA that signed eitri's certificate
vmTrusts []string // SHA256 fingerprints frozen onto the VM; empty when the row recorded none
}
// excludesDelegatedCA reports that the VM's frozen set is known, complete, and
// does not name the delegating CA — the only footing from which the CA-set
// story below is true.
//
// An entry with no fingerprint is a CA whose stored line would not parse, so
// the set cannot be read whole and might well contain the delegating one. It
// blocks the claim rather than narrowing it.
func (t certTrust) excludesDelegatedCA() bool {
if t.delegatedCA == "" || len(t.vmTrusts) == 0 || slices.Contains(t.vmTrusts, "") {
return false
}
return !slices.Contains(t.vmTrusts, t.delegatedCA)
}
// handshakeError turns an authentication failure into the diagnosis the
// evidence supports, and no further.
//
// Two different faults reach it as the same "unable to authenticate". A guest
// bakes its CA set at create, so a delegation signed by a CA registered after
// that VM was made is refused for an ordering reason the caller fixes by
// re-delegating. A guest whose own trust file or sshd drop-in has gone refuses
// a certificate its CA set names, and re-delegating changes nothing there. The
// VM's frozen set is what tells the two apart, so it is read before either is
// claimed: the ordering story requires a set that is known, readable whole, and
// without the delegating CA in it, and everything else points at the guest.
func handshakeError(vmName string, err error, trust certTrust) error {
if !strings.Contains(err.Error(), "unable to authenticate") {
return fmt.Errorf("vm %s ssh handshake: %w", vmName, err)
}
if trust.excludesDelegatedCA() {
return fmt.Errorf("vm %s refused eitri's certificate: it trusts the CA set it was created with, and that "+
"set does not include the CA you delegated with. Delegate with a CA this VM trusts, or create a new VM", vmName)
}
// The refusal came from the guest and the CA set does not explain it, so
// the message names the things that do — all of them inside the guest,
// where nothing on this seam can look: every MCP tool that reaches a guest
// goes through the SSH being refused here.
return fmt.Errorf("vm %s refused eitri's certificate, and %s. Something inside the guest is refusing it: "+
"/etc/ssh/eitri_user_ca.pub or the sshd drop-in that names it missing or overwritten (an agent older than "+
"v0.0.8 let a BYO cloud-init carrying write_files replace the seed's), a guest clock outside the "+
"certificate's validity window, or a principal mismatch. No tool here can look: they all go through this "+
"same SSH. Open the guest's console and check it there", vmName, trust.setClause())
}
// setClause says what the VM's row does and does not settle, in the words the
// refusal above is built around. Naming the delegating CA gives an operator the
// fingerprint to compare against the VM's trusted_cas by eye.
func (t certTrust) setClause() string {
switch {
case t.delegatedCA == "":
return "nothing here can compare the certificate eitri offered against the set this VM was created with"
case len(t.vmTrusts) == 0:
return fmt.Sprintf("this VM recorded no trusted CA set, so nothing here can say whether it was created to "+
"trust the CA you delegated with (%s)", t.delegatedCA)
case slices.Contains(t.vmTrusts, ""):
return fmt.Sprintf("this VM's frozen CA set holds an entry nothing here can name, so it cannot say whether "+
"the CA you delegated with (%s) is one of them", t.delegatedCA)
default:
return fmt.Sprintf("the CA set it was created with does include the CA you delegated with (%s)", t.delegatedCA)
}
}
// delegatedCAFingerprint names the CA that signed the credential eitri is
// holding, read off that credential rather than looked up again — it is the
// certificate that was actually offered, so the two cannot disagree. Empty when
// the signer carries a bare key and there is no signing CA to name.
func delegatedCAFingerprint(s ssh.Signer) string {
cert, ok := s.PublicKey().(*ssh.Certificate)
if !ok || cert.SignatureKey == nil {
return ""
}
return ssh.FingerprintSHA256(cert.SignatureKey)
}
// keyEquals compares two SSH public keys by their wire encodings.
func keyEquals(a, b ssh.PublicKey) bool {
return a != nil && b != nil && string(a.Marshal()) == string(b.Marshal())
}
// pipeConn adapts the tunnel's byte pipe to the net.Conn that ssh.NewClientConn
// requires. The addresses are stubs — the SSH client only ever reports them —
// and the deadline methods refuse rather than lying: the client sets none of
// them, and a silent no-op deadline would be a trap for the next caller.
type pipeConn struct{ rwc io.ReadWriteCloser }
func (c pipeConn) Read(p []byte) (int, error) { return c.rwc.Read(p) }
func (c pipeConn) Write(p []byte) (int, error) { return c.rwc.Write(p) }
func (c pipeConn) Close() error { return c.rwc.Close() }
func (c pipeConn) LocalAddr() net.Addr { return tunnelAddr{} }
func (c pipeConn) RemoteAddr() net.Addr { return tunnelAddr{} }
func (c pipeConn) SetDeadline(time.Time) error { return errors.ErrUnsupported }
func (c pipeConn) SetReadDeadline(time.Time) error { return errors.ErrUnsupported }
func (c pipeConn) SetWriteDeadline(time.Time) error { return errors.ErrUnsupported }
// tunnelAddr names the sync tunnel in the two places net.Conn insists on an
// address. There is no socket underneath, so there is no address to report.
type tunnelAddr struct{}
func (tunnelAddr) Network() string { return "eitri-sync" }
func (tunnelAddr) String() string { return "tunnel" }