internal/smoke/gatecheck.go

Ref: Size: 4.9 KiB History
package smoke

import (
	"context"
	"errors"
	"fmt"
	"strings"
	"time"

	"github.com/a73x/eitri/internal/gateclient"
	"golang.org/x/crypto/ssh"
)

// realGateHooks builds the live SSH-CA gate steps: register the smoke user CA
// with the tenant, and reach the guest through the gate to prove access. The
// tenant is the operator PAT's own tenant (derived via Me() by the caller), so
// the CA registration and connect names match the fleet's real partition.
func realGateHooks(cfg Config, tenant string, ca gateclient.CertAuthority, userCA ssh.Signer, now func() time.Time, sleep func(time.Duration)) *gateHooks {
	auth := gateclient.NewGateAuth(ca, userCA, tenant, cfg.SmokeVMUser, now)
	return &gateHooks{
		register: func(ctx context.Context) error { return auth.Register(ctx) },
		exec:     func(ctx context.Context, vmName string) error { return gateExec(ctx, cfg, auth, vmName, now, sleep) },
		run: func(ctx context.Context, vmName, cmd string) (string, error) {
			return gateRun(ctx, cfg, auth, vmName, cmd, now, sleep)
		},
	}
}

// gateExec proves the guest is reachable through the SSH-CA gate, and that the
// seed the guest booted from survived the tenant cloud-init it was created with.
// It retries dial+login over the guest's pre-sshd boot window (a dial or session
// error mid-boot is expected, not fatal) until it logs in as cfg.SmokeVMUser,
// confirms `id -un` echoes that same user, and reads back a guest where both
// sides of the cloud-init merge landed — or the 120s deadline expires.
//
// The login itself is half the merge proof: a CA-signed certificate is accepted
// only because the seed's write_files put the trust file and the sshd drop-in on
// the guest, which is exactly what a tenant `write_files` used to replace. The
// probe supplies the other half, the root the seed's runcmd grew.
//
// The probe is inside the retry rather than after it because cloud-init writes
// guest files before it runs runcmd, so a login can land in the window between
// the two: a root that is not grown YET is an unfinished boot, not a verdict.
// Only the deadline decides, and lastErr carries the reason it ran out.
func gateExec(ctx context.Context, cfg Config, auth gateclient.Credentials, vmName string, now func() time.Time, sleep func(time.Duration)) error {
	var lastErr error
	err := pollLoop(ctx, now, sleep, 120*time.Second, 5*time.Second, func() (bool, error) {
		client, dialErr := gateclient.Dial(ctx, gateclient.DialConfig{
			Gate: cfg.SmokeGate,
			Auth: auth,
		}, vmName)
		if dialErr != nil {
			lastErr = dialErr
			return false, nil
		}
		defer client.Close()

		out, runErr := runGuestCommand(client, "id -un")
		if runErr != nil {
			lastErr = runErr
			return false, nil
		}

		got := strings.TrimSpace(out)
		if got != cfg.SmokeVMUser {
			return false, fmt.Errorf("gate SSH logged into %q as %q, want %q", vmName, got, cfg.SmokeVMUser)
		}

		probe, probeErr := runGuestCommand(client, seedProbeCmd)
		if probeErr != nil {
			lastErr = probeErr
			return false, nil
		}
		if seedErr := proveSeedSurvivedBYO(probe); seedErr != nil {
			lastErr = seedErr
			return false, nil
		}
		return true, nil
	})
	if err != nil {
		if errors.Is(err, errPollTimeout) {
			return fmt.Errorf("FAIL: guest %q did not pass the gate proof within 120s: %w", vmName, lastErr)
		}
		return err
	}
	return nil
}

// gateRun runs one command inside a guest through the SSH-CA gate and returns
// what it printed. Like gateExec it retries the DIAL over the guest's pre-sshd
// boot window, since a leg that reaches a fresh guest is racing sshd's start.
// The command itself is not retried: what the callers run is not idempotent —
// a second mount of an already-mounted volume fails on its own — so a command
// that ran and failed is the answer, not a reason to try again.
func gateRun(ctx context.Context, cfg Config, auth gateclient.Credentials, vmName, cmd string, now func() time.Time, sleep func(time.Duration)) (string, error) {
	var out string
	var lastErr error
	err := pollLoop(ctx, now, sleep, 120*time.Second, 5*time.Second, func() (bool, error) {
		client, dialErr := gateclient.Dial(ctx, gateclient.DialConfig{
			Gate: cfg.SmokeGate,
			Auth: auth,
		}, vmName)
		if dialErr != nil {
			lastErr = dialErr
			return false, nil
		}
		defer client.Close()

		got, runErr := runGuestCommand(client, cmd)
		if runErr != nil {
			return false, runErr
		}
		out = got
		return true, nil
	})
	if errors.Is(err, errPollTimeout) {
		return "", fmt.Errorf("could not reach guest %q through the gate within 120s: %w", vmName, lastErr)
	}
	return out, err
}

// runGuestCommand runs cmd in a new session on client and returns its stdout.
func runGuestCommand(client *ssh.Client, cmd string) (string, error) {
	session, err := client.NewSession()
	if err != nil {
		return "", fmt.Errorf("open ssh session: %w", err)
	}
	defer session.Close()

	out, err := session.Output(cmd)
	if err != nil {
		return "", fmt.Errorf("run %q: %w", cmd, err)
	}
	return string(out), nil
}