internal/smoke/volume_test.go
Ref: Size: 12.5 KiB History
package smoke
import (
"context"
"errors"
"fmt"
"strings"
"testing"
"time"
"github.com/a73x/eitri/internal/server/api/client"
)
// volumeFleet is a small stateful fake fleet for the volume leg. It models the
// one property the leg has to get right: a deleted VM is NOT gone. It keeps
// its row — and with it the claim it holds — for reapAfter, the agent's
// tombstone grace, and the claim is refused for every second of that. A fake
// that reaped instantly would call a leg that leaks storage on a live fleet
// green.
type volumeFleet struct {
t *testing.T
clock *fakeClock
events []string // what the leg asked for, in order
created []client.CreateVMRequest // every VM create, in order
live map[string]string // live vm id -> name
dying map[string]dyingVM // deleted, still listed until reaped
next int
claimID string
// refusals is how many delete-claim attempts are refused after the last VM
// row is gone — the tick between the reap and the plane agreeing to it.
refusals int
claimDeletes int
// reapAfter is how long a deleted VM keeps its row. A fielded plane runs
// the agent's five-minute tombstone grace, so the fake does too.
reapAfter time.Duration
}
// dyingVM is a deleted VM still in its tombstone grace.
type dyingVM struct {
name string
destroyAt time.Time
}
func newVolumeFleet(t *testing.T, clock *fakeClock) *volumeFleet {
t.Helper()
return &volumeFleet{
t: t,
clock: clock,
live: map[string]string{},
dying: map[string]dyingVM{},
claimID: "claim-1",
reapAfter: 5 * time.Minute,
}
}
// reap drops the rows whose tombstone grace has run out. It is what every
// listing and every claim delete consults, so "is it gone yet" has exactly one
// answer in this fake.
func (f *volumeFleet) reap() {
for id, vm := range f.dying {
if !f.clock.now().Before(vm.destroyAt) {
delete(f.dying, id)
}
}
}
func (f *volumeFleet) api() *testAPI {
return &testAPI{
createVolumeClaimFunc: func(ctx context.Context, name string, sizeGB int64) (client.VolumeClaim, error) {
f.events = append(f.events, "claim:"+name)
return client.VolumeClaim{ID: f.claimID, Name: name, SizeGB: sizeGB}, nil
},
deleteVolumeClaimFunc: func(ctx context.Context, id string) error {
if id != f.claimID {
f.t.Errorf("DeleteVolumeClaim id = %q, want %q", id, f.claimID)
}
f.claimDeletes++
f.reap()
// A VM in its tombstone grace still holds the claim: the row is
// there, the file behind it is still attached to that VM.
if len(f.live)+len(f.dying) > 0 {
return fmt.Errorf("409: claim %s is attached to a VM", id)
}
if f.claimDeletes <= f.refusals {
return fmt.Errorf("409: claim %s is still attached", id)
}
f.events = append(f.events, "claim-delete")
return nil
},
createVMFunc: func(ctx context.Context, req client.CreateVMRequest) (client.CreateVMResponse, error) {
f.next++
id := fmt.Sprintf("vm-%d", f.next)
f.created = append(f.created, req)
f.live[id] = req.Name
f.events = append(f.events, "vm:"+req.Name)
return client.CreateVMResponse{ID: id}, nil
},
listVMsFunc: func(ctx context.Context) ([]client.VM, error) {
f.reap()
var out []client.VM
for id, name := range f.live {
out = append(out, client.VM{ID: id, Name: name, Phase: "ready", AssignedIP: "10.0.0.9"})
}
for id, vm := range f.dying {
out = append(out, client.VM{ID: id, Name: vm.name, Phase: "ready", AssignedIP: "10.0.0.9", Deleted: true})
}
return out, nil
},
deleteVMFunc: func(ctx context.Context, id string) error {
name, ok := f.live[id]
if !ok {
// A delete re-issued against a row that is already deleted is
// recorded before it is refused, so a test can see the wasted
// call and the 404 it earns.
if vm, dying := f.dying[id]; dying {
f.events = append(f.events, "delete:"+vm.name)
return fmt.Errorf("404: VM %s is already deleted", id)
}
return fmt.Errorf("404: no VM %s", id)
}
f.events = append(f.events, "delete:"+name)
delete(f.live, id)
f.dying[id] = dyingVM{name: name, destroyAt: f.clock.now().Add(f.reapAfter)}
return nil
},
}
}
// markerGate answers the way a working guest does: the write command says
// nothing, and the read command hands the marker back.
func markerGate(record *[]string, marker string, writeErr error) *gateHooks {
return &gateHooks{
run: func(ctx context.Context, vmName, cmd string) (string, error) {
*record = append(*record, vmName)
if strings.Contains(cmd, "mkfs") {
return "", writeErr
}
return marker + "\n", nil
},
}
}
func TestProveVolumeCarriesAMarkerPastTheVMThatWroteIt(t *testing.T) {
clock := &fakeClock{t: time.Unix(0, 0)}
fleet := newVolumeFleet(t, clock)
var ran []string
gate := markerGate(&ran, volumeMarker, nil)
if err := proveVolume(context.Background(), fleet.api(), "host-1", "smoke-test", gate, noopReadPubKey, clock.now, clock.sleep); err != nil {
t.Fatalf("proveVolume: %v", err)
}
// The claim is storage the VMs attach to, so it exists before either of
// them: a claim made after the fact could not be the disk they booted with.
want := []string{
"claim:smoke-test-vol",
"vm:smoke-test-v1",
"delete:smoke-test-v1",
"vm:smoke-test-v2",
"delete:smoke-test-v2",
"claim-delete",
}
if strings.Join(fleet.events, ",") != strings.Join(want, ",") {
t.Fatalf("fleet calls = %v, want %v", fleet.events, want)
}
if len(fleet.created) != 2 {
t.Fatalf("VM creates = %d, want 2", len(fleet.created))
}
for i, req := range fleet.created {
if len(req.VolumeClaims) != 1 || req.VolumeClaims[0] != "claim-1" {
t.Errorf("VM %d VolumeClaims = %v, want [claim-1] — both VMs must name the same claim", i+1, req.VolumeClaims)
}
if req.HostID != "host-1" {
t.Errorf("VM %d HostID = %q, want host-1 — the claim is bound to one host", i+1, req.HostID)
}
if req.SSHAuthorizedKey != noopReadPubKey() {
t.Errorf("VM %d SSHAuthorizedKey = %q, want the local key", i+1, req.SSHAuthorizedKey)
}
}
if len(ran) != 2 || ran[0] != "smoke-test-v1" || ran[1] != "smoke-test-v2" {
t.Errorf("guest commands ran on %v, want [smoke-test-v1 smoke-test-v2]", ran)
}
}
// TestProveVolumeFailsWhenTheMarkerComesBackWrong is the leg's whole point: a
// volume that reattaches but answers with something else did not carry the
// data across, and the failure quotes what it did answer with.
func TestProveVolumeFailsWhenTheMarkerComesBackWrong(t *testing.T) {
clock := &fakeClock{t: time.Unix(0, 0)}
fleet := newVolumeFleet(t, clock)
var ran []string
gate := markerGate(&ran, "a-fresh-empty-disk", nil)
err := proveVolume(context.Background(), fleet.api(), "host-1", "smoke-test", gate, noopReadPubKey, clock.now, clock.sleep)
if err == nil {
t.Fatal("proveVolume: want an error when the marker came back wrong, got nil")
}
if !strings.Contains(err.Error(), "a-fresh-empty-disk") {
t.Errorf("error = %q, want it to quote what the volume answered with", err.Error())
}
if !strings.Contains(err.Error(), volumeMarker) {
t.Errorf("error = %q, want it to name the marker it wanted", err.Error())
}
// The failing VM and the storage behind it are still cleaned up.
if len(fleet.live) != 0 {
t.Errorf("live VMs = %v, want none — the leg deletes its VMs on the failure path too", fleet.live)
}
if fleet.claimDeletes == 0 {
t.Error("the claim was never deleted; a failed leg must not leave storage behind")
}
}
// TestProveVolumeCleansUpWhenTheGuestCommandFails covers the earlier failure
// point: the first VM is up but cannot write, so that VM and the claim both go.
func TestProveVolumeCleansUpWhenTheGuestCommandFails(t *testing.T) {
clock := &fakeClock{t: time.Unix(0, 0)}
fleet := newVolumeFleet(t, clock)
var ran []string
gate := markerGate(&ran, volumeMarker, errors.New("mkfs: no such device"))
err := proveVolume(context.Background(), fleet.api(), "host-1", "smoke-test", gate, noopReadPubKey, clock.now, clock.sleep)
if err == nil {
t.Fatal("proveVolume: want an error when the guest could not write the marker, got nil")
}
if !strings.Contains(err.Error(), "no such device") {
t.Errorf("error = %q, want it to carry the guest's own complaint", err.Error())
}
if len(fleet.created) != 1 {
t.Errorf("VM creates = %d, want 1 — the second VM proves nothing once the first never wrote", len(fleet.created))
}
want := []string{"claim:smoke-test-vol", "vm:smoke-test-v1", "delete:smoke-test-v1", "claim-delete"}
if strings.Join(fleet.events, ",") != strings.Join(want, ",") {
t.Errorf("fleet calls = %v, want %v", fleet.events, want)
}
}
// TestProveVolumeFreesTheClaimAfterTheTombstoneGrace is the leak guard. A VM
// holds its claim until the agent hard-destroys it, minutes after the delete
// is accepted, so a leg that deletes the claim on the way out without waiting
// that grace out leaves a claim and the storage behind it on a live fleet —
// every single run. The proof that it waited is the clock: two full graces of
// virtual time passed, one per VM.
func TestProveVolumeFreesTheClaimAfterTheTombstoneGrace(t *testing.T) {
clock := &fakeClock{t: time.Unix(0, 0)}
fleet := newVolumeFleet(t, clock)
start := clock.now()
var ran []string
gate := markerGate(&ran, volumeMarker, nil)
if err := proveVolume(context.Background(), fleet.api(), "host-1", "smoke-test", gate, noopReadPubKey, clock.now, clock.sleep); err != nil {
t.Fatalf("proveVolume: %v", err)
}
if fleet.events[len(fleet.events)-1] != "claim-delete" {
t.Fatalf("fleet calls = %v, want the claim actually deleted at the end", fleet.events)
}
if len(fleet.live) != 0 || len(fleet.dying) != 0 {
t.Errorf("rows left = %v / %v, want none", fleet.live, fleet.dying)
}
if waited := clock.now().Sub(start); waited < 2*fleet.reapAfter {
t.Errorf("leg waited %s, want at least %s — one tombstone grace per VM before the claim can go",
waited, 2*fleet.reapAfter)
}
}
// TestProveVolumeRetriesTheClaimDeleteAfterTheReap: the plane can still refuse
// for a tick after the row is gone, so the delete is retried rather than
// reported as storage the leg never actually left behind.
func TestProveVolumeRetriesTheClaimDeleteAfterTheReap(t *testing.T) {
clock := &fakeClock{t: time.Unix(0, 0)}
fleet := newVolumeFleet(t, clock)
fleet.refusals = 2
var ran []string
gate := markerGate(&ran, volumeMarker, nil)
if err := proveVolume(context.Background(), fleet.api(), "host-1", "smoke-test", gate, noopReadPubKey, clock.now, clock.sleep); err != nil {
t.Fatalf("proveVolume: %v", err)
}
if fleet.claimDeletes != 3 {
t.Errorf("DeleteVolumeClaim attempts = %d, want 3 — two refusals then the delete that lands", fleet.claimDeletes)
}
if fleet.events[len(fleet.events)-1] != "claim-delete" {
t.Errorf("fleet calls = %v, want the claim deleted last", fleet.events)
}
}
// TestProveVolumeFailsWhenTheFirstVMIsNeverReaped: the second VM cannot attach
// a claim the first one still holds, so a VM that will not go away fails the
// leg here rather than as a mystery 409 on the next create. The delete itself
// was accepted — the fleet simply never finished it — so the message says
// that, and the cleanup does not re-issue a delete that already worked.
func TestProveVolumeFailsWhenTheFirstVMIsNeverReaped(t *testing.T) {
clock := &fakeClock{t: time.Unix(0, 0)}
fleet := newVolumeFleet(t, clock)
fleet.reapAfter = 24 * time.Hour // accepted, never finished
var ran []string
gate := markerGate(&ran, volumeMarker, nil)
err := proveVolume(context.Background(), fleet.api(), "host-1", "smoke-test", gate, noopReadPubKey, clock.now, clock.sleep)
if err == nil {
t.Fatal("proveVolume: want an error when the first VM was never reaped, got nil")
}
if !strings.Contains(err.Error(), "smoke-test-v1") {
t.Errorf("error = %q, want it to name the VM that would not go away", err.Error())
}
if !strings.Contains(err.Error(), "deleted but not reaped") {
t.Errorf("error = %q, want it to say the delete was accepted and the reap never finished", err.Error())
}
if !strings.Contains(err.Error(), "claim-1") {
t.Errorf("error = %q, want it to name the claim the VM is still holding", err.Error())
}
if len(fleet.created) != 1 {
t.Errorf("VM creates = %d, want 1 — the second VM is never created", len(fleet.created))
}
// One delete, not two: the cleanup must not re-issue a delete that was
// accepted, or the 404 it earns would blame a delete that worked.
deletes := 0
for _, e := range fleet.events {
if strings.HasPrefix(e, "delete:") {
deletes++
}
}
if deletes != 1 {
t.Errorf("DeleteVM calls = %d, want 1 — the accepted delete is not re-issued on the way out", deletes)
}
}