internal/smoke/volume.go
Ref: Size: 8.6 KiB History
package smoke
import (
"context"
"errors"
"fmt"
"os"
"strings"
"time"
"github.com/a73x/eitri/internal/server/api/client"
)
// volumeMarker is what the first VM writes to the volume and the second one
// must read back. Nothing about it is special except that an empty disk cannot
// produce it: a freshly created volume answers with nothing, a recycled one
// with somebody else's bytes, and both are failures.
const volumeMarker = "eitri-smoke"
// volumeClaimSizeGB is the smallest claim worth making. The leg proves the
// data survives its VM, not that any particular size can be allocated.
const volumeClaimSizeGB = 1
// volumeDevice is where the first attached claim lands in the guest — after
// the root and seed disks, so the first (and here only) claim is /dev/vdc.
const volumeDevice = "/dev/vdc"
// writeMarkerCmd formats the volume and leaves the marker on it, unmounting
// before it returns so every byte is on the disk and not in the guest's page
// cache when the VM that wrote it is destroyed.
const writeMarkerCmd = "sudo mkfs.ext4 -q -F " + volumeDevice +
" && sudo mount " + volumeDevice + " /mnt" +
" && echo " + volumeMarker + " | sudo tee /mnt/marker >/dev/null" +
" && sudo umount /mnt"
// readMarkerCmd mounts the same volume in a VM that never wrote to it and
// reads back what the previous one left.
const readMarkerCmd = "sudo mount " + volumeDevice + " /mnt && cat /mnt/marker"
// proveVolume proves durable storage outlives the VM it was attached to: claim
// a volume, attach it to a VM that writes a marker on it, destroy that VM
// entirely, attach the same claim to a second VM, and read the marker back. A
// volume that answers with anything else — an empty disk, somebody else's data
// — fails the leg, because a claim that cannot carry a byte past the guest
// that wrote it is not durable storage.
//
// The guest commands go through the SSH-CA gate, so the leg only runs where the
// gate is configured. Whatever the verdict, both VMs and the claim are gone
// before it returns: this runs against a live fleet.
func proveVolume(ctx context.Context, c vmAPI, hostID, vmName string, gate *gateHooks, readPubKey func() string, now func() time.Time, sleep func(time.Duration)) error {
claim, err := c.CreateVolumeClaim(ctx, vmName+"-vol", volumeClaimSizeGB)
if err != nil {
return fmt.Errorf("create volume claim: %w", err)
}
// VMs this leg still has to clean up. A VM holds its claim until the agent
// hard-destroys it — minutes after the delete is accepted, once the
// tombstone grace runs out — so every path here waits that reap out before
// it tries to delete the claim. A cleanup that gave up sooner would leave
// the claim, and the storage behind it, on the fleet on every run.
var holders []volumeVM
defer func() {
// Cleanup outlives a cancelled ctx: leaving a VM and a paid-for volume
// on a live plane is worse than the failure that got us here. A cleanup
// that itself fails says so on stderr — the leg's own verdict, above,
// stands either way.
out := context.WithoutCancel(ctx)
for _, h := range holders {
if _, err := deleteVMAndWaitReaped(out, c, h.id, h.name, claim.ID, now, sleep); err != nil {
fmt.Fprintf(os.Stderr, "eitri-smoke: volume cleanup: %v\n", err)
}
}
if err := deleteClaimWhenReleased(out, c, claim.ID, now, sleep); err != nil {
fmt.Fprintf(os.Stderr, "eitri-smoke: volume claim %s left behind: %v\n", claim.ID, err)
}
}()
writerName := vmName + "-v1"
writerID, err := createVolumeVM(ctx, c, hostID, writerName, claim.ID, readPubKey, now, sleep)
if writerID != "" {
holders = append(holders, volumeVM{id: writerID, name: writerName})
}
if err != nil {
return err
}
if _, err := gate.run(ctx, writerName, writeMarkerCmd); err != nil {
return fmt.Errorf("FAIL: could not write the marker to the volume on %s: %w", writerName, err)
}
// Destroy the writer outright — not a reboot, not a detach. The claim is
// only proven durable if it survives the guest, the disk image, and the
// record of the VM that made it.
deleted, err := deleteVMAndWaitReaped(ctx, c, writerID, writerName, claim.ID, now, sleep)
if deleted {
holders = drop(holders, writerID)
}
if err != nil {
return err
}
readerName := vmName + "-v2"
readerID, err := createVolumeVM(ctx, c, hostID, readerName, claim.ID, readPubKey, now, sleep)
if readerID != "" {
holders = append(holders, volumeVM{id: readerID, name: readerName})
}
if err != nil {
return err
}
out, err := gate.run(ctx, readerName, readMarkerCmd)
if err != nil {
return fmt.Errorf("FAIL: could not read the marker back from the volume on %s: %w", readerName, err)
}
if got := strings.TrimSpace(out); got != volumeMarker {
return fmt.Errorf("FAIL: the volume reattached to %s answered %q, want %q — the data did not survive the VM that wrote it",
readerName, truncateBanner(got), volumeMarker)
}
// The reader goes here rather than in the cleanup above only so a failure
// deleting it is the leg's verdict rather than a line on stderr; either way
// the claim cannot be deleted until this reap finishes.
deleted, err = deleteVMAndWaitReaped(ctx, c, readerID, readerName, claim.ID, now, sleep)
if deleted {
holders = drop(holders, readerID)
}
return err
}
// volumeVM is one of the leg's VMs: the id to act on, the name to report.
type volumeVM struct{ id, name string }
// createVolumeVM creates one VM with the claim attached and waits for it to be
// ready. It returns the VM's id even when the wait fails, so the caller can
// still clean up a VM the fleet did create.
func createVolumeVM(ctx context.Context, c vmAPI, hostID, name, claimID string, readPubKey func() string, now func() time.Time, sleep func(time.Duration)) (string, error) {
created, err := c.CreateVM(ctx, client.CreateVMRequest{
HostID: hostID,
Name: name,
SSHAuthorizedKey: readPubKey(),
VolumeClaims: []string{claimID},
})
if err != nil {
return "", fmt.Errorf("create vm %s with claim %s: %w", name, claimID, err)
}
var lastPhase string
err = pollLoop(ctx, now, sleep, 600*time.Second, 5*time.Second, func() (bool, error) {
vm, _, err := getVM(ctx, c, created.ID)
if err != nil {
return false, fmt.Errorf("poll vm ready: %w", err)
}
lastPhase = vm.Phase
return vm.Phase == "ready" && vm.AssignedIP != "", nil
})
if errors.Is(err, errPollTimeout) {
return created.ID, fmt.Errorf("FAIL: VM %s (with a volume attached) not ready within 600s (phase=%s)", name, lastPhase)
}
return created.ID, err
}
// deleteVMAndWaitReaped deletes a VM and waits until the fleet stops listing
// it — the moment the claim it holds becomes attachable again. The window
// matches the scenario's reap poll: it must outlast the agent's tombstone
// grace, which a fielded plane runs at five minutes.
//
// deleted reports whether the DELETE itself was accepted, and it is true even
// when the reap then times out, so a caller never re-issues a delete that
// worked and never blames one that did.
func deleteVMAndWaitReaped(ctx context.Context, c vmAPI, vmID, name, claimID string, now func() time.Time, sleep func(time.Duration)) (deleted bool, err error) {
if err := c.DeleteVM(ctx, vmID); err != nil {
return false, fmt.Errorf("delete vm %s: %w", name, err)
}
err = pollLoop(ctx, now, sleep, 7*time.Minute, 5*time.Second, func() (bool, error) {
_, present, err := getVM(ctx, c, vmID)
if err != nil {
return false, fmt.Errorf("poll vm reaped: %w", err)
}
return !present, nil
})
if errors.Is(err, errPollTimeout) {
return true, fmt.Errorf("FAIL: VM %s (%s) was deleted but not reaped within 7m; it still holds claim %s",
name, vmID, claimID)
}
return true, err
}
// deleteClaimWhenReleased deletes a claim once nothing holds it. Its callers
// have already waited for the reap of every VM that did, so this is a short
// retry over the tick between a row disappearing and the plane agreeing it has
// — NOT the wait for the reap itself, which no 30-second poll could outlast.
func deleteClaimWhenReleased(ctx context.Context, c vmAPI, claimID string, now func() time.Time, sleep func(time.Duration)) error {
var lastErr error
err := pollLoop(ctx, now, sleep, 30*time.Second, 3*time.Second, func() (bool, error) {
if derr := c.DeleteVolumeClaim(ctx, claimID); derr != nil {
lastErr = derr
return false, nil
}
return true, nil
})
if errors.Is(err, errPollTimeout) {
return fmt.Errorf("still refused 30s after its VMs were reaped: %w", lastErr)
}
return err
}
// drop removes the VM with this id, keeping the rest in order.
func drop(vms []volumeVM, id string) []volumeVM {
kept := vms[:0]
for _, vm := range vms {
if vm.id != id {
kept = append(kept, vm)
}
}
return kept
}