a73x

internal/agent/reconcile/volumes_test.go

Ref:   Size: 23.1 KiB   History

package reconcile

import (
	"crypto/sha256"
	"encoding/hex"
	"os"
	"path/filepath"
	"syscall"
	"testing"
	"time"

	"github.com/a73x/eitri/internal/pb"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

// volSnap is snap's twin for the other half of desired state: the FULL set of
// volumes this host is meant to hold.
func volSnap(epoch uint64, vols ...*pb.VolumeSpec) *pb.Snapshot {
	return &pb.Snapshot{Epoch: epoch, Volumes: vols}
}

// volID turns a short name a test can read into an id the agent will accept:
// the 32 hex characters random.Hex(16) mints. Derived from the name so it is
// stable across a test's ticks, and so two names never collide.
func volID(name string) string {
	sum := sha256.Sum256([]byte(name))
	return hex.EncodeToString(sum[:16])
}

func volSpec(name string, sizeGB int64) *pb.VolumeSpec {
	return &pb.VolumeSpec{VolumeId: volID(name), SizeGb: sizeGB}
}

func deadVol(name string, sizeGB int64) *pb.VolumeSpec {
	v := volSpec(name, sizeGB)
	v.Tombstoned = true
	return v
}

func withVolumes(names ...string) func(*pb.VMSpec) {
	ids := make([]string, len(names))
	for i, n := range names {
		ids[i] = volID(n)
	}
	return func(v *pb.VMSpec) { v.VolumeIds = ids }
}

func findVolume(rep *pb.Report, name string) *pb.VolumeStatus {
	for _, v := range rep.Volumes {
		if v.VolumeId == volID(name) {
			return v
		}
	}
	return nil
}

// The fixture's volume paths, by the short name the test used.
func (f *fixture) volDir(name string) string    { return f.st.VolumeDir(volID(name)) }
func (f *fixture) volPath(name string) string   { return f.st.VolumePath(volID(name)) }
func (f *fixture) volMarker(name string) string { return f.st.VolumeTombstonePath(volID(name)) }

// ageMarker backdates a volume's tombstone marker so the reclaim grace is
// measured against the fixture's clock rather than the wall clock the file
// system stamped it with.
func (f *fixture) ageMarker(t *testing.T, name string, age time.Duration) {
	t.Helper()
	at := f.now.Add(-age)
	require.NoError(t, os.Chtimes(f.volMarker(name), at, at))
}

func TestVolumeIsCreatedSparseAndReported(t *testing.T) {
	f := setup(t)
	rep := f.step(volSnap(1, volSpec("v1", 2)))

	require.Len(t, rep.Volumes, 1)
	assert.Equal(t, volID("v1"), rep.Volumes[0].VolumeId)
	assert.True(t, rep.Volumes[0].Present)
	assert.EqualValues(t, 2, rep.Volumes[0].SizeGb)

	fi, err := os.Stat(f.volPath("v1"))
	require.NoError(t, err)
	assert.EqualValues(t, 2<<30, fi.Size())
	if sparseFS(t, f.st.VolumesDir()) {
		assert.Less(t, allocatedBytes(t, f.volPath("v1")), int64(1<<20),
			"a 2 GiB volume must not cost 2 GiB the moment it is asked for")
	}

	_, err = os.Stat(f.volPath("v1") + ".partial")
	assert.True(t, os.IsNotExist(err), "the temporary is renamed into place, never left beside it")
}

// TestVolumeIsNeverResized: the file is the guest's block device and a
// filesystem sits on it, so the agent reports what it found rather than
// growing what the control plane now asks for.
func TestVolumeIsNeverResized(t *testing.T) {
	f := setup(t)
	f.step(volSnap(1, volSpec("v1", 2)))
	rep := f.step(volSnap(2, volSpec("v1", 5)))

	fi, err := os.Stat(f.volPath("v1"))
	require.NoError(t, err)
	assert.EqualValues(t, 2<<30, fi.Size(), "a guest filesystem sits on it")
	require.Len(t, rep.Volumes, 1)
	assert.EqualValues(t, 2, rep.Volumes[0].SizeGb, "reported as found, not as asked")
}

// TestVolumeConvergenceIsIdempotent: a repeated snapshot must not re-create a
// file that is already there, which is what would silently discard a guest's
// data on every tick.
func TestVolumeConvergenceIsIdempotent(t *testing.T) {
	f := setup(t)
	f.step(volSnap(1, volSpec("v1", 1)))
	require.NoError(t, os.WriteFile(f.volPath("v1"), []byte("a guest wrote this"), 0o600))

	rep := f.step(volSnap(2, volSpec("v1", 1)))
	require.Len(t, rep.Volumes, 1)
	assert.True(t, rep.Volumes[0].Present)
	data, err := os.ReadFile(f.volPath("v1"))
	require.NoError(t, err)
	assert.Equal(t, "a guest wrote this", string(data), "an existing volume is left exactly as it is")
}

func TestTombstonedVolumeIsDeletedAfterGrace(t *testing.T) {
	f := setup(t)
	f.step(volSnap(1, volSpec("v1", 1)))

	rep := f.step(volSnap(2, deadVol("v1", 1)))
	require.Len(t, rep.Volumes, 1)
	assert.True(t, rep.Volumes[0].Present, "still within grace")
	_, err := os.Stat(f.volPath("v1"))
	assert.NoError(t, err)
	_, err = os.Stat(f.volMarker("v1"))
	require.NoError(t, err, "the marker that starts the grace is written on the first tombstoned tick")

	f.ageMarker(t, "v1", 6*time.Minute) // past the fixture's 5m TombstoneGrace
	rep = f.step(volSnap(3, deadVol("v1", 1)))
	require.Len(t, rep.Volumes, 1)
	assert.False(t, rep.Volumes[0].Present, "reported gone so the control plane can reap the row")
	_, err = os.Stat(f.volDir("v1"))
	assert.True(t, os.IsNotExist(err), "the whole directory goes, marker included")
}

// TestTombstoneGraceIsMeasuredFromTheFirstMarker: the marker is written once
// and never refreshed, so a volume tombstoned minutes ago is not given a fresh
// grace by every tick that re-states the tombstone.
func TestTombstoneGraceIsMeasuredFromTheFirstMarker(t *testing.T) {
	f := setup(t)
	f.step(volSnap(1, volSpec("v1", 1)))
	f.step(volSnap(2, deadVol("v1", 1)))
	f.ageMarker(t, "v1", 4*time.Minute)

	rep := f.step(volSnap(3, deadVol("v1", 1)))
	assert.True(t, rep.Volumes[0].Present, "4 minutes into a 5 minute grace")

	f.now = f.now.Add(2 * time.Minute) // the clock moves, the marker does not
	rep = f.step(volSnap(4, deadVol("v1", 1)))
	assert.False(t, rep.Volumes[0].Present, "the grace ran from the first marker, not from this tick")
}

// TestTombstonedVolumeThisHostNeverHadIsReportedGone: nothing to reclaim, so
// nothing is written — and the control plane hears "gone" at once instead of
// waiting out a grace on a file that does not exist.
func TestTombstonedVolumeThisHostNeverHadIsReportedGone(t *testing.T) {
	f := setup(t)
	rep := f.step(volSnap(1, deadVol("ghost", 1)))
	require.Len(t, rep.Volumes, 1)
	assert.Equal(t, volID("ghost"), rep.Volumes[0].VolumeId)
	assert.False(t, rep.Volumes[0].Present)
	_, err := os.Stat(f.volDir("ghost"))
	assert.True(t, os.IsNotExist(err), "no directory is made for a volume there is nothing to reclaim of")
}

// The rule that matters: a file the snapshot does not name is reported and
// kept. One server bug must not delete user data.
func TestUnknownVolumeIsReportedNeverDeleted(t *testing.T) {
	f := setup(t)
	f.step(volSnap(1, volSpec("v1", 1)))

	rep := f.step(volSnap(2))
	require.Len(t, rep.Volumes, 1)
	assert.Equal(t, volID("v1"), rep.Volumes[0].VolumeId)
	assert.True(t, rep.Volumes[0].Present)
	assert.EqualValues(t, 1, rep.Volumes[0].SizeGb)
	_, err := os.Stat(f.volPath("v1"))
	assert.NoError(t, err, "an orphan is evidence of a bug somewhere, never a licence to delete")
}

// TestEverySnapshotVolumeGetsAStatusRow pins the contract the server's reap
// rests on: it reads an omitted volume as gone, so a volume the agent could
// not make must say so in words rather than by silence.
func TestEverySnapshotVolumeGetsAStatusRow(t *testing.T) {
	f := setup(t)
	rep := f.step(volSnap(1, volSpec("ok", 1), volSpec("nonsense", 0)))

	require.Len(t, rep.Volumes, 2)
	assert.True(t, findVolume(rep, "ok").GetPresent())
	bad := findVolume(rep, "nonsense")
	require.NotNil(t, bad, "a volume that could not be created still gets a row")
	assert.False(t, bad.GetPresent())
}

// TestVolumeConvergenceSurvivesAnUnusableVolumesDirectory: the state directory
// can be wrong in ways nothing here can fix — a file where the volumes
// directory belongs. Every volume the snapshot named still gets a row, and the
// row errs toward present: this host cannot see the file, which is not the
// same as knowing it is gone.
func TestVolumeConvergenceSurvivesAnUnusableVolumesDirectory(t *testing.T) {
	f := setup(t)
	require.NoError(t, os.RemoveAll(f.st.VolumesDir()))
	require.NoError(t, os.WriteFile(f.st.VolumesDir(), []byte("not a directory"), 0o600))

	rep := f.step(volSnap(1, volSpec("v1", 1)))
	require.Len(t, rep.Volumes, 1)
	assert.Equal(t, volID("v1"), rep.Volumes[0].VolumeId)
	assert.True(t, rep.Volumes[0].Present, "a volume this host cannot look at is not a volume it may report gone")

	data, err := os.ReadFile(f.st.VolumesDir())
	require.NoError(t, err)
	assert.Equal(t, "not a directory", string(data), "and nothing was written over it")
}

// TestUnreadableVolumeIsNotRecreatedOverTheTopOfItself is the rule statVolume
// exists for: create ends in a rename over whatever is at disk.raw, so a stat
// that fails for any reason other than "no such file" must NOT be read as an
// absent volume. The guest's data is on the other side of that rename.
func TestUnreadableVolumeIsNotRecreatedOverTheTopOfItself(t *testing.T) {
	requireNonRoot(t)
	f := setup(t)
	f.step(volSnap(1, volSpec("v1", 1)))
	require.NoError(t, os.WriteFile(f.volPath("v1"), []byte("a guest wrote this"), 0o600))
	chmodForTest(t, f.volDir("v1"), 0o000) // the file cannot even be looked at

	rep := f.step(volSnap(2, volSpec("v1", 1)))
	require.Len(t, rep.Volumes, 1)
	assert.True(t, rep.Volumes[0].Present, "unreadable is not absent")

	require.NoError(t, os.Chmod(f.volDir("v1"), 0o700))
	_, err := os.Stat(f.volPath("v1") + ".partial")
	assert.True(t, os.IsNotExist(err), "no create was even attempted")
	data, err := os.ReadFile(f.volPath("v1"))
	require.NoError(t, err)
	assert.Equal(t, "a guest wrote this", string(data), "and the guest's bytes are untouched")
}

// TestTornTemporaryThatCannotBeReplacedIsReported: .partial is where a killed
// create leaves its mess. One that cannot be cleared makes the volume absent,
// which is what the report then says.
func TestTornTemporaryThatCannotBeReplacedIsReported(t *testing.T) {
	f := setup(t)
	tmp := f.volPath("v1") + ".partial"
	require.NoError(t, os.MkdirAll(filepath.Join(tmp, "occupied"), 0o700))

	rep := f.step(volSnap(1, volSpec("v1", 1)))
	require.Len(t, rep.Volumes, 1)
	assert.False(t, rep.Volumes[0].Present)
	_, err := os.Stat(f.volPath("v1"))
	assert.True(t, os.IsNotExist(err), "nothing is renamed into place from a temporary that was never written")
}

// TestTombstonedVolumeIsKeptWhenTheMarkerCannotBeWritten: the grace is a file,
// and a grace this host cannot start is not permission to delete.
func TestTombstonedVolumeIsKeptWhenTheMarkerCannotBeWritten(t *testing.T) {
	requireNonRoot(t)
	f := setup(t)
	f.step(volSnap(1, volSpec("v1", 1)))
	chmodForTest(t, f.volDir("v1"), 0o500)

	rep := f.step(volSnap(2, deadVol("v1", 1)))
	require.Len(t, rep.Volumes, 1)
	assert.True(t, rep.Volumes[0].Present)
	_, err := os.Stat(f.volPath("v1"))
	assert.NoError(t, err)
}

// TestTombstonedVolumeIsKeptWhenItCannotBeRemoved: the row must outlive the
// bytes, never the other way round — reporting a volume gone while its file is
// still here would have the control plane forget a disk this host still holds.
func TestTombstonedVolumeIsKeptWhenItCannotBeRemoved(t *testing.T) {
	requireNonRoot(t)
	f := setup(t)
	f.step(volSnap(1, volSpec("v1", 1)))
	f.step(volSnap(2, deadVol("v1", 1)))
	f.ageMarker(t, "v1", 6*time.Minute)
	chmodForTest(t, f.volDir("v1"), 0o500)

	rep := f.step(volSnap(3, deadVol("v1", 1)))
	require.Len(t, rep.Volumes, 1)
	assert.True(t, rep.Volumes[0].Present, "still present, so still reported present")
	_, err := os.Stat(f.volPath("v1"))
	assert.NoError(t, err)
}

// TestPartialVolumeIsRecreated: a create killed mid-truncate leaves a
// .partial, never a short file at the real path, so the next tick simply
// makes the volume.
func TestPartialVolumeIsRecreated(t *testing.T) {
	f := setup(t)
	require.NoError(t, os.MkdirAll(f.volDir("v1"), 0o700))
	require.NoError(t, os.WriteFile(f.volPath("v1")+".partial", []byte("torn"), 0o600))

	rep := f.step(volSnap(1, volSpec("v1", 1)))
	fi, err := os.Stat(f.volPath("v1"))
	require.NoError(t, err)
	assert.EqualValues(t, 1<<30, fi.Size())
	assert.True(t, rep.Volumes[0].Present)
	_, err = os.Stat(f.volPath("v1") + ".partial")
	assert.True(t, os.IsNotExist(err), "the torn temporary is replaced, not stepped around")
}

// Volumes are materialised before the VM that needs them is dispatched, in
// the same Step, so a VM and its volume arriving together boot first try.
//
// The first half is the deterministic one: Step converges volumes on its own
// goroutine, so the file and its row exist the moment Step returns, with no
// worker having had to run. The second half is what the ordering buys — the
// backend found the device already there.
func TestVolumesAreMaterialisedBeforeVMsDispatch(t *testing.T) {
	f := setup(t)
	s := volSnap(1, volSpec("v1", 1))
	s.Vms = []*pb.VMSpec{vm("vm1", withVolumes("v1"))}

	rep := f.eng.Step(t.Context(), s)
	_, err := os.Stat(f.volPath("v1"))
	require.NoError(t, err, "the volume is a file by the time Step returns, not when a worker gets to it")
	require.Len(t, rep.Volumes, 1)
	assert.True(t, rep.Volumes[0].Present)

	f.eng.manager().waitIdle()
	require.Equal(t, "ready", findVM(f.aggregateNow(1), "vm1").GetPhase())
	assert.Equal(t, []string{volID("v1")}, f.prov.bootedSpec["vm1"].VolumeIDs, "the backend is told which files to attach")
	assert.True(t, f.prov.volumesAtBoot["vm1"], "and the file was on disk before Boot was called")
}

// A VM whose volume is absent fails legibly rather than booting bare.
func TestVMWithMissingVolumeFailsLegibly(t *testing.T) {
	f := setup(t)
	rep := f.step(snap(1, vm("vm1", withVolumes("ghost"))))

	row := findVM(rep, "vm1")
	require.NotNil(t, row)
	assert.Equal(t, "failed", row.Phase, "permanent: nothing on this host heals a volume that was never placed")
	assert.Contains(t, row.LastError, "volume "+volID("ghost")+" not materialized on this host")
	assert.Empty(t, f.prov.booted, "never booted")

	f.step(snap(2, vm("vm1", withVolumes("ghost"))))
	assert.Empty(t, f.prov.booted, "and no later tick boots it either")
}

// A guest that already exists is held to the same rule as a new one: powering
// it back on without the volume it was created with would hand it a filesystem
// that is simply not there, and a guest cannot report that upward.
//
// The volume is removed while the snapshot no longer names it, which is exactly
// what a host holds after the control plane moved the volume elsewhere: nothing
// recreates the file, and the VM still asks for it.
func TestPoweringOnWithoutItsVolumeFailsRatherThanBootsBare(t *testing.T) {
	f := setup(t)
	s := volSnap(1, volSpec("v1", 1))
	s.Vms = []*pb.VMSpec{vm("vm1", withVolumes("v1"))}
	f.step(s)
	require.Equal(t, []string{"vm1"}, f.prov.booted)

	s2 := volSnap(2, volSpec("v1", 1))
	s2.Vms = []*pb.VMSpec{vm("vm1", withVolumes("v1"), stopped)}
	f.step(s2)
	require.False(t, f.prov.Running("vm1"), "stopped first, so the next tick is a power-on and not a create")

	require.NoError(t, os.RemoveAll(f.volDir("v1")))

	rep := f.step(snap(3, vm("vm1", withVolumes("v1"))))

	row := findVM(rep, "vm1")
	require.NotNil(t, row)
	assert.Equal(t, "failed", row.Phase)
	assert.Contains(t, row.LastError, "volume "+volID("v1")+" not materialized on this host")
	assert.Equal(t, []string{"vm1"}, f.prov.booted, "booted once, at create — never a second time without its device")
}

// The same rule on the other converge path: a host reboot loses every guest,
// and the restart that brings them back is a Boot like any other.
func TestAHostRebootDoesNotBringAGuestBackWithoutItsVolume(t *testing.T) {
	f := setup(t)
	s := volSnap(1, volSpec("v1", 1))
	s.Vms = []*pb.VMSpec{vm("vm1", withVolumes("v1"))}
	f.step(s)
	require.Equal(t, []string{"vm1"}, f.prov.booted)

	require.NoError(t, os.RemoveAll(f.volDir("v1")))
	f.prov.booted = nil
	f.boot = "boot-2" // the host rebooted: the guest is lost and would be booted again
	f.prov.running["vm1"] = false

	rep := f.step(snap(2, vm("vm1", withVolumes("v1"))))

	row := findVM(rep, "vm1")
	require.NotNil(t, row)
	assert.Equal(t, "failed", row.Phase)
	assert.Contains(t, row.LastError, "volume "+volID("v1")+" not materialized on this host")
	assert.Empty(t, f.prov.booted, "a lost guest comes back with its devices or not at all")
}

// TestVolumeIDsSurviveAnAgentRestart: the ids are desired state on disk, so a
// restarted agent re-attaches the same devices in the same order without
// waiting for a snapshot to tell it again.
func TestVolumeIDsSurviveAnAgentRestart(t *testing.T) {
	f := setup(t)
	s := volSnap(1, volSpec("va", 1), volSpec("vb", 1))
	s.Vms = []*pb.VMSpec{vm("vm1", withVolumes("vb", "va"))}
	f.step(s)

	recs, err := f.st.LoadVMs()
	require.NoError(t, err)
	assert.Equal(t, []string{volID("vb"), volID("va")}, recs["vm1"].Spec.VolumeIDs, "attachment order is persisted")

	f.restart(t)
	f.prov.booted = nil
	f.boot = "boot-2" // the host rebooted with it, so the guest is lost and comes back
	f.prov.running["vm1"] = false
	s2 := volSnap(2, volSpec("va", 1), volSpec("vb", 1))
	s2.Vms = []*pb.VMSpec{vm("vm1", withVolumes("vb", "va"))}
	f.step(s2)

	require.Equal(t, []string{"vm1"}, f.prov.booted)
	assert.Equal(t, []string{volID("vb"), volID("va")}, f.prov.bootedSpec["vm1"].VolumeIDs)
}

// TestEditingTheVolumeListResetsCreateAttempts is Equal doing the job == did:
// binding a volume is an edit, so a VM that spent its retry budget under the
// old spec gets a fresh one.
func TestEditingTheVolumeListResetsCreateAttempts(t *testing.T) {
	f := setup(t)
	f.prov.prepErr = assert.AnError
	for range 3 {
		f.step(snap(1, vm("vm1")))
	}
	f.prov.prepErr = nil

	s := volSnap(2, volSpec("v1", 1))
	s.Vms = []*pb.VMSpec{vm("vm1", withVolumes("v1"))}
	rep := f.step(s)
	assert.Equal(t, []string{"vm1"}, f.prov.prepared, "a newly bound volume is a spec edit")
	assert.Equal(t, "ready", findVM(rep, "vm1").GetPhase())
}

// TestFencedSnapshotReportsVolumesWithoutTouchingThem: the fence path acts on
// nothing, volumes included — but it must still SAY what is on this host. The
// control plane reaps a tombstoned volume on a report that omits it, so a
// fenced report naming none would read as a host that has none.
func TestFencedSnapshotReportsVolumesWithoutTouchingThem(t *testing.T) {
	f := setup(t)
	f.step(volSnap(5, volSpec("v1", 2), volSpec("v2", 1)))
	f.step(volSnap(6, volSpec("v1", 2), deadVol("v2", 1))) // v2 is on its way out
	require.NoError(t, os.WriteFile(f.volPath("v1"), []byte("a guest wrote this"), 0o600))

	rep := f.step(volSnap(3, volSpec("v3", 1)))

	require.True(t, rep.FenceViolation)
	require.Len(t, rep.Volumes, 2, "both volumes on disk are reported, the tombstoned one included")
	assert.True(t, findVolume(rep, "v1").GetPresent())
	assert.True(t, findVolume(rep, "v2").GetPresent())
	assert.Nil(t, findVolume(rep, "v3"), "a stale snapshot's volume is not conjured into the report")

	_, err := os.Stat(f.volPath("v3"))
	assert.True(t, os.IsNotExist(err), "a stale snapshot must not materialise a file")
	data, err := os.ReadFile(f.volPath("v1"))
	require.NoError(t, err)
	assert.Equal(t, "a guest wrote this", string(data), "nor touch one that is here")

	f.now = f.now.Add(time.Hour) // long past the grace v2's marker started
	rep = f.step(volSnap(3, volSpec("v3", 1)))
	require.True(t, rep.FenceViolation)
	assert.True(t, findVolume(rep, "v2").GetPresent(), "and the fence path reclaims nothing, whatever the clock says")
}

// TestAVolumeIDIsNotAPath: volume paths are the ones this package hands to
// os.RemoveAll, so an id is checked before it is joined into one. Without that
// check a single malformed snapshot aims a reclaim at a live VM's directory.
func TestAVolumeIDIsNotAPath(t *testing.T) {
	f := setup(t)
	f.step(snap(1, vm("vm1")))
	require.NoError(t, os.WriteFile(f.st.DiskPath("vm1"), []byte("a running guest's root disk"), 0o600))

	traversal := "../vms/vm1"
	rep := f.step(volSnap(2, &pb.VolumeSpec{VolumeId: traversal, SizeGb: 1, Tombstoned: true}))

	require.Len(t, rep.Volumes, 1)
	assert.Equal(t, traversal, rep.Volumes[0].VolumeId, "answered for, so the control plane is not left waiting")
	assert.False(t, rep.Volumes[0].Present)

	_, err := os.Stat(filepath.Join(f.st.VMDir("vm1"), "tombstoned"))
	assert.True(t, os.IsNotExist(err), "no reclaim clock is started over a VM's own directory")
	data, err := os.ReadFile(f.st.DiskPath("vm1"))
	require.NoError(t, err)
	assert.Equal(t, "a running guest's root disk", string(data), "and the guest's root disk is still there")
}

// TestVolumeIDsThatAreNotIDsAreRefused covers the rest of the alphabet: what
// the control plane mints is random.Hex(16), and nothing else is joined into a
// path at all.
func TestVolumeIDsThatAreNotIDsAreRefused(t *testing.T) {
	f := setup(t)
	bad := []string{
		"",
		"v1",
		"../../etc/passwd",
		"deadbeefdeadbeefdeadbeefdeadbee",   // 31: one short
		"deadbeefdeadbeefdeadbeefdeadbeeff", // 33: one long
		"DEADBEEFDEADBEEFDEADBEEFDEADBEEF",  // hex, but not the case Hex writes
		"deadbeef-deadbeef-deadbeef-dead",
	}
	var specs []*pb.VolumeSpec
	for _, id := range bad {
		specs = append(specs, &pb.VolumeSpec{VolumeId: id, SizeGb: 1})
	}

	rep := f.step(volSnap(1, specs...))

	require.Len(t, rep.Volumes, len(bad), "every id gets an answer, however wrong it was")
	for i, v := range rep.Volumes {
		assert.Equal(t, bad[i], v.VolumeId)
		assert.False(t, v.Present, "%q must not be treated as a volume", bad[i])
	}
	entries, err := os.ReadDir(f.st.VolumesDir())
	require.NoError(t, err)
	assert.Empty(t, entries, "nothing was created for any of them")
}

// TestSnapshotBelowTheFloorTouchesNoVolume: an agent that cannot fully read a
// snapshot acts on no part of it. The server reaps a tombstoned volume on an
// omitted row only for an agent that reports volumes at all, and this one has
// just said it is too old to.
func TestSnapshotBelowTheFloorTouchesNoVolume(t *testing.T) {
	f := setup(t)
	f.eng.AgentVersion = "v0.0.6"
	s := volSnap(1, volSpec("v1", 1))
	s.MinAgentVersion = "v0.0.7"
	s.Vms = []*pb.VMSpec{vm("vm1", withVolumes("v1"))}

	rep := f.eng.Step(t.Context(), s)
	assert.Empty(t, rep.Volumes)
	assert.Equal(t, "failed", findVM(rep, "vm1").GetPhase())
	_, err := os.Stat(f.volPath("v1"))
	assert.True(t, os.IsNotExist(err))
}

// requireNonRoot skips a test that makes a directory unwritable to prove what
// the agent does when it cannot write. Root is not refused by permission bits,
// so under root the test would prove the opposite of what it says.
func requireNonRoot(t *testing.T) {
	t.Helper()
	if os.Geteuid() == 0 {
		t.Skip("running as root: permission bits refuse nothing")
	}
}

// chmodForTest makes a directory unwritable and puts it back afterwards, so
// t.TempDir's own cleanup can still remove it.
func chmodForTest(t *testing.T, dir string, mode os.FileMode) {
	t.Helper()
	require.NoError(t, os.Chmod(dir, mode))
	t.Cleanup(func() { _ = os.Chmod(dir, 0o700) })
}

// allocatedBytes reports how much disk a file actually occupies, as opposed to
// the size it claims. st_blocks is in 512-byte units by POSIX definition,
// whatever the filesystem's own block size is.
func allocatedBytes(t *testing.T, path string) int64 {
	t.Helper()
	fi, err := os.Stat(path)
	require.NoError(t, err)
	st, ok := fi.Sys().(*syscall.Stat_t)
	require.True(t, ok, "no syscall.Stat_t for %s", path)
	return st.Blocks * 512
}

// sparseFS reports whether dir's filesystem holds holes at all. Without the
// probe, asserting sparseness asserts something about the machine the test ran
// on rather than about the code.
func sparseFS(t *testing.T, dir string) bool {
	t.Helper()
	p := filepath.Join(dir, "sparse-probe")
	f, err := os.Create(p)
	require.NoError(t, err)
	require.NoError(t, f.Truncate(8<<20))
	require.NoError(t, f.Close())
	alloc := allocatedBytes(t, p)
	require.NoError(t, os.Remove(p))
	return alloc <= 1<<20
}