internal/server/boot/sshgate.go
Ref: Size: 14.9 KiB History
package boot
import (
"encoding/json"
"errors"
"fmt"
"log/slog"
"net"
"sync/atomic"
"time"
"github.com/a73x/eitri/internal/server/api"
serverconfig "github.com/a73x/eitri/internal/server/config"
"github.com/a73x/eitri/internal/server/sshca"
"github.com/a73x/eitri/internal/server/sshgate"
"github.com/a73x/eitri/internal/server/store"
"github.com/a73x/eitri/internal/server/syncsvc"
"github.com/a73x/eitri/internal/server/vmssh"
"golang.org/x/crypto/ssh"
)
// sshGateSetup carries the jump-gate state from config-time setup to the later
// wiring points in startup (API cert minters, sync snapshot, gate listener).
// A nil *sshGateSetup means the gate is OFF: every method is a no-op on a nil
// receiver, so run holds one value instead of repeating `!= nil` guards.
type sshGateSetup struct {
ca *sshca.CA
listen string // cfg.SSHListen
domain string // cfg.SSHGateDomain
}
// setupSSHGate wires the SSH jump gate (§B): OFF unless ssh_listen is set
// (returns nil). When enabled, load or create the persistent host CA + gate
// host key (0600, sealed under kek, never logged). A key file that will not
// open is a startup failure: sshca never regenerates over one, because a fresh
// host CA would invalidate every pin and every VM's host certificate at once.
//
// A gate that is on must also be able to name itself, or it does not boot: see
// the refusal below. That is what lets the rest of the system assume a running
// plane with a gate publishes a real address for it.
func setupSSHGate(cfg serverconfig.Config, kek []byte) (*sshGateSetup, error) {
if cfg.SSHListen == "" {
return nil, nil
}
if cfg.SSHCAKey == "" || cfg.SSHHostKey == "" {
return nil, errors.New("ssh_ca_key and ssh_host_key are required when ssh_listen is set")
}
g := &sshGateSetup{listen: cfg.SSHListen, domain: cfg.SSHGateDomain}
// The config is internally consistent and still serves nobody. gateAddr
// publishing nothing is the symptom; the cause is that the gate has no name,
// and gateDomain therefore puts "localhost" on its host certificate — which
// every client verifies against the name it dialed, so no value of EITRI_GATE
// reaches this gate from another machine. Stop here rather than run one only
// this box can use.
if g.gateAddr() == "" {
return nil, fmt.Errorf("ssh_gate_domain is required when ssh_listen binds every interface (got %q): a bind says which interfaces to accept on, not what to call the machine, so the gate has no name to advertise and its host certificate would name localhost, which every remote client refuses against the name it dialed. Set ssh_gate_domain to the name clients dial, or bind one concrete address (127.0.0.1 for a plane only its own machine reaches)", cfg.SSHListen)
}
sshGate, err := sshca.New(cfg.SSHCAKey, cfg.SSHHostKey, kek)
if err != nil {
return nil, fmt.Errorf("ssh ca: %w", err)
}
g.ca = sshGate
// Log the HOST CA identity operators pin via @cert-authority for gate + VM
// host verification. Only the *public* key is ever logged (private material
// never is). eitri holds no user CA — those are BYO per-tenant.
slog.Info("ssh jump gate configured", "listen", cfg.SSHListen,
"host_ca", string(sshGate.HostCAAuthorizedKey()))
// The gate listener itself is started later (startListener), once
// syncsvc.Service (the tunnel dialer) exists.
return g, nil
}
// wireAPI publishes what a client needs to reach a guest: the HOST CA pubkey
// via GET /api/v1/ssh-ca, so it can pin `@cert-authority` and verify the gate
// and every VM by certificate, and the gate's own address via GET /api/v1/me,
// so it knows what to hop through. eitri never mints user certs — user CAs are
// BYO per-tenant (uploaded, never held here). Left unwired when the gate is
// off, so the ssh-ca endpoint 404s and /me names no gate.
func (g *sshGateSetup) wireAPI(a *api.API) {
if g == nil {
return
}
a.SetSSHCAAuthorizedKey(string(g.ca.HostCAAuthorizedKey()))
a.SetSSHGate(g.gateAddr())
}
// gateDomain is the hostname clients dial the gate as, and therefore the one
// principal on its host certificate: the configured ssh_gate_domain, else the
// host part of ssh_listen, else localhost. That last rung is the reason a
// wildcard bind with no domain refuses to boot — a certificate has to name
// something, and "localhost" is a name no remote client can dial.
func (g *sshGateSetup) gateDomain() string {
if g.domain != "" {
return g.domain
}
if h, _, err := net.SplitHostPort(g.listen); err == nil && h != "" {
return h
}
return "localhost"
}
// gateAddr is the full address a client dials for the gate hop — the value
// EITRI_GATE takes and the one the console prints in its connect recipes.
//
// The host part is the configured ssh_gate_domain, else the host ssh_listen
// binds. A wildcard or empty bind (0.0.0.0, ::, ":2222") is where that stops:
// it says which interfaces to accept on, not what to call the machine, and a
// recipe built from one dials the wrong host from everywhere but this one. It
// answers "" there — and setupSSHGate reads that empty answer as a refusal to
// boot, so a running plane with a gate on always has an address to publish.
//
// Whatever it returns, the host part is gateDomain and nothing else: a client
// verifies the name it dialed against that certificate, so any other spelling
// of the same machine is a hard verification failure, by design.
func (g *sshGateSetup) gateAddr() string {
host, port := g.domain, "22"
bindHost, bindPort, err := net.SplitHostPort(g.listen)
if err == nil && bindPort != "" {
port = bindPort
}
if host == "" {
if err != nil || unspecifiedHost(bindHost) {
return ""
}
host = bindHost
}
return net.JoinHostPort(host, port)
}
// unspecifiedHost reports whether h names every interface rather than one host.
func unspecifiedHost(h string) bool {
ip := net.ParseIP(h)
return h == "" || (ip != nil && ip.IsUnspecified())
}
// wireSync gives the sync service the one thing it needs from the gate: the
// ability to sign a certificate for a host key a guest's host generated. Left
// unwired when the gate is off, which is how a host learns there is no
// certificate coming and boots its guests uncertified.
func (g *sshGateSetup) wireSync(svc *syncsvc.Service) {
if g == nil {
return
}
svc.SetHostCertSigner(guestHostCertSigner{ca: g.ca.HostCA()})
}
// guestHostCertSigner signs guest host certificates with the fleet's host CA.
// It holds a CA signer and a signing rule and nothing else — the principal is
// decided by the caller, which is the control plane reading the VM's own row.
type guestHostCertSigner struct{ ca ssh.Signer }
func (s guestHostCertSigner) SignHostCert(pub ssh.PublicKey, principal string) (string, error) {
cert, err := sshca.SignHostCert(s.ca, pub, []string{principal}, principal, time.Now(), sshca.HostCertTTL)
if err != nil {
return "", err
}
return string(ssh.MarshalAuthorizedKey(cert)), nil
}
// hostCAPublicKey returns the HOST CA public key that certifies every VM's host
// key, for callers that verify a guest themselves rather than through the gate.
// Nil when the gate is off — there is then no CA, and no VM carries a host cert.
func (g *sshGateSetup) hostCAPublicKey() ssh.PublicKey {
if g == nil {
return nil
}
return g.ca.HostCA().PublicKey()
}
// startListener starts the SSH jump gate listener: when enabled, front
// `ssh -J gate ubuntu@<vm>` with the hardened bastion. It resolves VM names
// against the store, tunnels port 22 through the sync connection (svc.OpenTCP),
// and trusts only certs signed by a registered tenant user CA. A failed bind or
// cert-sign is fatal (returned to run, like QUIC/HTTP); a serve failure after
// bind lands on fatal so a dead gate does not run silently.
func (g *sshGateSetup) startListener(st *store.Store, svc *syncsvc.Service, fatal chan<- error) error {
if g == nil {
return nil
}
// The gate host cert's principal is the name clients dial — the same one
// /me hands them, since a client verifies what it dialed against this cert.
principal := g.gateDomain()
slog.Info("ssh gate host cert", "principal", principal)
// Sign a long-lived HOST cert for the gate's own host key and present THAT
// (via a cert signer) instead of the bare key, so a client verifying with
// `@cert-authority` accepts the gate on first connect — no TOFU window.
gateCert, err := sshca.SignHostCert(g.ca.HostCA(), g.ca.HostKey().PublicKey(),
[]string{principal}, "eitri-gate", time.Now(), sshca.HostCertTTL)
if err != nil {
return fmt.Errorf("sign gate host cert: %w", err)
}
gateHostSigner, err := ssh.NewCertSigner(gateCert, g.ca.HostKey())
if err != nil {
return fmt.Errorf("gate host cert signer: %w", err)
}
gate := sshgate.New(gateHostSigner, userCALookup(st), resolveVM(st), authorizeVM(st), svc.OpenTCP,
revokedCert(st), gateAudit(st))
ln, err := net.Listen("tcp", g.listen)
if err != nil {
return fmt.Errorf("ssh gate listen: %w", err)
}
go func() {
slog.Info("ssh jump gate listening", "addr", g.listen)
// Serve returns only when the listener fails, never nil.
err := gate.Serve(ln)
fatal <- fmt.Errorf("ssh gate serve: %w", err)
}()
return nil
}
// resolveVM is tenant-scoped: the bare name is looked up WITHIN the connection's
// tenant only (VMByTenantName), so a name never resolves across tenants. A
// lookup error (unknown or tombstoned VM) reports ok=false rather than tunneling
// to a dead or foreign VM.
func resolveVM(st *store.Store) sshgate.Resolver {
return func(tenant, name string) (hostID, vmID string, ok bool) {
vm, err := st.VMByTenantName(tenant, name)
if err != nil {
return "", "", false
}
return vm.HostID, vm.ID, true
}
}
// authorizeVM re-reads the VM row and requires tenant equality: the tenant that
// the connection's user cert resolved to (via its per-tenant CA) must own the
// VM, or the connection is refused. A tombstoned VM (DeletedAt set) or a store
// error is refused too.
func authorizeVM(st *store.Store) sshgate.Authorizer {
return func(tenant, vmID string) bool {
vm, err := st.GetVM(vmID)
return err == nil && vm.DeletedAt == nil && vm.Tenant == tenant
}
}
// vmLookup pairs resolveVM with authorizeVM for callers that reach a VM without
// the gate in front of them — the /mcp handler's server-side SSH. It is the same
// two checks in the same order: resolve the name within the tenant only, then
// re-read the row and require it to still be that tenant's and still be alive.
func vmLookup(st *store.Store) vmssh.VMLookup {
resolve, authorize := resolveVM(st), authorizeVM(st)
return func(tenant, name string) (vmssh.VM, bool) {
hostID, vmID, ok := resolve(tenant, name)
if !ok || !authorize(tenant, vmID) {
return vmssh.VM{}, false
}
// The row is read again for the certificate rather than carried out of
// resolve: authorize has just re-read it, and a VM that lost its row
// between the two is one we must not claim to have verified.
vm, err := st.GetVM(vmID)
if err != nil {
return vmssh.VM{}, false
}
return vmssh.VM{
HostID: hostID, VMID: vmID,
HostCertified: vm.SSHHostCert != "",
TrustedCAFingerprints: frozenCAFingerprints(vm),
}, true
}
}
// frozenCAFingerprints is the CA set a VM was created to trust, as fingerprints
// — what a refusal compares a delegated certificate against. A row that
// recorded no set answers with none, which reads as "unknown" there. The
// authorized_keys lines stay in the row: this path explains trust, it does not
// serve it.
func frozenCAFingerprints(vm store.VM) []string {
if vm.TrustedCAs == nil {
return nil
}
out := make([]string, 0, len(vm.TrustedCAs))
for _, ca := range vm.TrustedCAs {
out = append(out, ca.Fingerprint)
}
return out
}
// revokedCert gates every cert auth against the revocation list. Fail-CLOSED
// for the single connection on a DB error: a store hiccup rejects THAT login
// (returns revoked=true) rather than fail-open (which would let a possibly-
// revoked cert through) or fail-the-whole-gate (which a global close would
// amount to, DoSing every login on any transient error).
func revokedCert(st *store.Store) sshgate.Revoker {
return func(tenant string, serial uint64) bool {
revoked, err := st.IsSSHCertRevoked(tenant, serial)
if err != nil {
slog.Error("ssh cert revocation lookup failed; rejecting connection", "err", err)
return true
}
return revoked
}
}
// userCALookup trusts the DB-registered set of tenant user CAs and stamps each
// connection with the tenant that registered the signing CA. It looks up by the
// SAME canonical authorized_keys line the store persists (ca_pubkey), so the
// bytes agree. A lookup error fails closed (rejects the cert).
func userCALookup(st *store.Store) sshgate.UserCALookup {
return func(pub ssh.PublicKey) (string, bool) {
tenant, ok, err := st.TenantForUserCA(sshca.AuthorizedKeyLine(pub))
if err != nil {
slog.Error("tenant user-ca lookup failed; rejecting", "err", err)
return "", false
}
return tenant, ok
}
}
// gateAudit files gate events in the same append-only log the API writes to, in
// the same (tenant, action, detail-JSON) shape — so a tenant's trail reads as
// one story rather than two, and `GET /api/v1/audit` shows who reached which VM
// without a second place to look.
//
// An event the gate could not attribute (a refused login carries no tenant) is
// filed under the system scope, exactly as a denied enrollment is.
//
// Writes happen on ONE background goroutine behind a bounded queue, because the
// gate calls this on a live connection's goroutine and the store serializes
// every write onto a single connection shared with syncsvc and the API. Writing
// inline would put a database round trip between a client and its tunnel, and
// would stall logins under exactly the load that makes an audit trail worth
// having.
//
// A full queue DROPS, and says so. The alternative is blocking the gate, and an
// audit trail that can wedge the thing it audits is worse than one with a
// counted gap in it.
func gateAudit(st *store.Store) sshgate.Audit {
type event struct {
tenant, action string
detail map[string]string
}
q := make(chan event, gateAuditQueue)
go func() {
for e := range q {
raw, _ := json.Marshal(e.detail)
if err := st.AppendAudit(e.tenant, e.action, string(raw)); err != nil {
slog.Warn("gate audit append failed", "action", e.action, "err", err)
}
}
}()
var dropped atomic.Int64
return func(tenant, action string, detail map[string]string) {
if tenant == "" {
tenant = store.SystemTenant
}
select {
case q <- event{tenant, action, detail}:
default:
// Count rather than log per drop: the situation that fills this queue
// is the one where another log line per event is also the problem.
if n := dropped.Add(1); n == 1 || n%gateAuditQueue == 0 {
slog.Warn("gate audit queue full; events dropped", "dropped_total", n)
}
}
}
}
// gateAuditQueue bounds the audit backlog the gate may build up. Deep enough to
// absorb a burst of real logins, shallow enough that a flood is dropped rather
// than buffered into memory.
const gateAuditQueue = 256