internal/server/store/volumes_test.go
Ref: Size: 14.9 KiB History
package store
import (
"database/sql"
"regexp"
"sort"
"testing"
"github.com/stretchr/testify/require"
)
// enrollSecondHost gives the fleet a host that is not enrollHost's, so a claim
// bound on one host can be asked for from the other.
func enrollSecondHost(t *testing.T, s *Store) Host {
t.Helper()
tok, err := s.CreateEnrollmentToken(testTenant)
require.NoError(t, err)
h, err := s.RedeemEnrollmentToken(tok, EnrollFacts{Name: "host-b", OS: "linux", Arch: "amd64", Provisioner: "cloudhv", Remote: ""})
require.NoError(t, err)
return h
}
func testVM(id, host string, claims ...string) VM {
return VM{ID: id, HostID: host, Name: id, ImageURL: "u", ImageSHA256: "s",
VCPUs: 1, MemMB: 1, DiskGB: 1, PowerState: "running", VolumeClaimIDs: claims}
}
func TestVolumeClaimLifecycle(t *testing.T) {
st := newStore(t)
hostID := enrollHost(t, st).ID
c, err := st.CreateVolumeClaim(testTenant, "data", 5)
require.NoError(t, err)
require.Equal(t, "", c.BoundVolumeID, "a fresh claim is Pending")
_, err = st.CreateVolumeClaim(testTenant, "data", 5)
require.ErrorIs(t, err, ErrClaimNameTaken)
// First VM naming the claim binds it on its host.
require.NoError(t, st.CreateVM(testVM("vm1", hostID, c.ID)))
got, err := st.GetVolumeClaim(c.ID)
require.NoError(t, err)
require.NotEmpty(t, got.BoundVolumeID)
require.Equal(t, hostID, got.HostID)
require.Equal(t, "vm1", got.VMID)
vm, err := st.GetVM("vm1")
require.NoError(t, err)
require.Equal(t, []string{got.BoundVolumeID}, vm.VolumeIDs)
// Second VM: attached → refused naming the holder; nothing of it remains.
err = st.CreateVM(testVM("vm2", hostID, c.ID))
var attached *ClaimAttachedError
require.ErrorAs(t, err, &attached)
require.Equal(t, "vm1", attached.VMID)
require.ErrorIs(t, err, ErrClaimAttached)
_, err = st.GetVM("vm2")
require.ErrorIs(t, err, sql.ErrNoRows, "the refused create rolled back entirely")
// Delete while attached → refused.
require.ErrorIs(t, st.TombstoneVolumeClaim(c.ID), ErrClaimAttached)
// Destroy the VM: attachment gone, volume and claim untouched.
require.NoError(t, st.TombstoneVM("vm1"))
require.NoError(t, st.HardDeleteVM("vm1", hostID))
got, err = st.GetVolumeClaim(c.ID)
require.NoError(t, err)
require.Equal(t, "", got.VMID)
require.Equal(t, hostID, got.HostID, "the data still lives where it was placed")
// A VM on another host cannot take a bound claim.
other := enrollSecondHost(t, st).ID
err = st.CreateVM(testVM("vm3", other, c.ID))
var pinned *ClaimPinnedError
require.ErrorAs(t, err, &pinned)
require.Equal(t, hostID, pinned.HostID)
require.ErrorIs(t, err, ErrClaimPinned)
// Delete cascades claim → volume as tombstones; the host still sees it.
require.NoError(t, st.TombstoneVolumeClaim(c.ID))
vols, err := st.ListVolumesForHost(hostID)
require.NoError(t, err)
require.Len(t, vols, 1)
require.NotNil(t, vols[0].DeletedAt)
_, err = st.GetVolumeClaim(c.ID)
require.ErrorIs(t, err, sql.ErrNoRows, "a tombstoned claim reads as gone")
// Reap after the host reports it gone: both rows go.
volID := vols[0].ID
require.NoError(t, st.HardDeleteVolume(volID))
vols, err = st.ListVolumesForHost(hostID)
require.NoError(t, err)
require.Empty(t, vols)
require.ErrorIs(t, st.HardDeleteVolume(volID), sql.ErrNoRows)
}
// The reap needs the tenant of a claim that is already tombstoned, which is
// exactly the claim GetVolumeClaim refuses to answer for.
func TestGetVolumeClaimAnyReadsATombstonedClaim(t *testing.T) {
st := newStore(t)
c, err := st.CreateVolumeClaim(testTenant, "data", 5)
require.NoError(t, err)
require.NoError(t, st.TombstoneVolumeClaim(c.ID))
_, err = st.GetVolumeClaim(c.ID)
require.ErrorIs(t, err, sql.ErrNoRows)
got, err := st.GetVolumeClaimAny(c.ID)
require.NoError(t, err)
require.Equal(t, testTenant, got.Tenant)
require.NotNil(t, got.DeletedAt)
_, err = st.GetVolumeClaimAny("nope")
require.ErrorIs(t, err, sql.ErrNoRows)
}
func TestCreateVMRefusesForeignOrMissingClaim(t *testing.T) {
st := newStore(t)
hostID := enrollHost(t, st).ID
other, err := st.CreateTenantForIdentity("https://test-issuer", "other-subject", "other@test.local")
require.NoError(t, err)
c, err := st.CreateVolumeClaim(other.ID, "data", 5)
require.NoError(t, err)
require.ErrorIs(t, st.CreateVM(testVM("vm1", hostID, c.ID)), ErrClaimNotFound,
"another tenant's claim is indistinguishable from none")
require.ErrorIs(t, st.CreateVM(testVM("vm1", hostID, "nope")), ErrClaimNotFound)
}
func TestCommittedOnHostCountsBoundVolumes(t *testing.T) {
st := newStore(t)
hostID := enrollHost(t, st).ID
c, err := st.CreateVolumeClaim(testTenant, "data", 7)
require.NoError(t, err)
vm := testVM("vm1", hostID, c.ID)
vm.DiskGB = 3
require.NoError(t, st.CreateVM(vm))
held, err := st.CommittedOnHost(hostID)
require.NoError(t, err)
require.EqualValues(t, 10, held.Live.DiskGB, "a bound volume is disk the host has committed, VM or no VM")
require.NoError(t, st.TombstoneVM("vm1"))
require.NoError(t, st.HardDeleteVM("vm1", hostID))
held, err = st.CommittedOnHost(hostID)
require.NoError(t, err)
require.EqualValues(t, 7, held.Live.DiskGB)
}
// Attachment order is the order the VM named its claims; the guest's device
// letters follow it.
func TestVMVolumeIDsKeepAttachmentOrder(t *testing.T) {
st := newStore(t)
hostID := enrollHost(t, st).ID
b, err := st.CreateVolumeClaim(testTenant, "b", 1)
require.NoError(t, err)
a, err := st.CreateVolumeClaim(testTenant, "a", 1)
require.NoError(t, err)
require.NoError(t, st.CreateVM(testVM("vm1", hostID, b.ID, a.ID)))
vm, err := st.GetVM("vm1")
require.NoError(t, err)
cb, err := st.GetVolumeClaim(b.ID)
require.NoError(t, err)
ca, err := st.GetVolumeClaim(a.ID)
require.NoError(t, err)
require.Equal(t, []string{cb.BoundVolumeID, ca.BoundVolumeID}, vm.VolumeIDs)
}
func TestListVolumeClaimsIsTenantScoped(t *testing.T) {
st := newStore(t)
hostID := enrollHost(t, st).ID
c, err := st.CreateVolumeClaim(testTenant, "data", 5)
require.NoError(t, err)
other, err := st.CreateTenantForIdentity("https://test-issuer", "other-subject", "other@test.local")
require.NoError(t, err)
_, err = st.CreateVolumeClaim(other.ID, "data", 5)
require.NoError(t, err, "the same name in another tenant is a different claim")
claims, err := st.ListVolumeClaims(testTenant)
require.NoError(t, err)
require.Len(t, claims, 1)
require.Equal(t, c.ID, claims[0].ID)
require.Equal(t, "", claims[0].HostID, "an unbound claim names no host")
require.NoError(t, st.CreateVM(testVM("vm1", hostID, c.ID)))
claims, err = st.ListVolumeClaims(testTenant)
require.NoError(t, err)
require.Len(t, claims, 1)
require.Equal(t, hostID, claims[0].HostID)
require.Equal(t, "vm1", claims[0].VMID)
// A tombstoned claim leaves the tenant's list.
require.NoError(t, st.TombstoneVM("vm1"))
require.NoError(t, st.HardDeleteVM("vm1", hostID))
require.NoError(t, st.TombstoneVolumeClaim(c.ID))
claims, err = st.ListVolumeClaims(testTenant)
require.NoError(t, err)
require.Empty(t, claims)
}
// A Pending claim has no bytes anywhere, so deleting it is bookkeeping only.
func TestTombstoneVolumeClaimUnbound(t *testing.T) {
st := newStore(t)
c, err := st.CreateVolumeClaim(testTenant, "data", 5)
require.NoError(t, err)
before, err := st.Epoch()
require.NoError(t, err)
require.NoError(t, st.TombstoneVolumeClaim(c.ID))
after, err := st.Epoch()
require.NoError(t, err)
require.Equal(t, before, after, "an unbound claim is on no host, so no agent has to re-snapshot")
require.ErrorIs(t, st.TombstoneVolumeClaim(c.ID), sql.ErrNoRows)
}
// countRows is the only way a test can prove a row is DELETED rather than
// tombstoned: every read above deliberately hides tombstones.
func countRows(t *testing.T, s *Store, query string, args ...any) int {
t.Helper()
var n int
require.NoError(t, s.db.QueryRow(query, args...).Scan(&n))
return n
}
// A host holding volumes is not a decommission that needs more time — nothing
// releases a volume but its claim's deletion, so the graceful path refuses and
// says which volumes are in the way.
func TestRemoveHostRefusesWhileVolumesRemain(t *testing.T) {
st := newStore(t)
hostID := enrollHost(t, st).ID
c, err := st.CreateVolumeClaim(testTenant, "data", 5)
require.NoError(t, err)
require.NoError(t, st.CreateVM(testVM("vm1", hostID, c.ID)))
require.NoError(t, st.TombstoneVM("vm1"))
require.NoError(t, st.HardDeleteVM("vm1", hostID))
// Live volume, no VMs left: drained of guests, still holding bytes.
err = st.RemoveHost(hostID)
require.ErrorIs(t, err, ErrHostHoldsVolumes)
require.Contains(t, err.Error(), "1 volume(s)")
// Tombstoning the claim does not release the host either: the file is on
// that disk until the agent acks the reclaim.
require.NoError(t, st.TombstoneVolumeClaim(c.ID))
require.ErrorIs(t, st.RemoveHost(hostID), ErrHostHoldsVolumes)
// Reaped: the host is finally removable.
vols, err := st.ListVolumesForHost(hostID)
require.NoError(t, err)
require.Len(t, vols, 1)
require.NoError(t, st.HardDeleteVolume(vols[0].ID))
require.NoError(t, st.RemoveHost(hostID))
}
// The ids this store mints are a WIRE CONTRACT, not a local detail: the agent's
// validVolumeID (internal/agent/reconcile/volumes.go) refuses any volume id that
// is not exactly 32 lowercase hex characters, because the id is joined into a
// path the agent hands to os.RemoveAll. A shorter id, an uppercase one, or a
// uuid with dashes would be refused on every host in the fleet and the volume
// would never materialize — so the shape is pinned on the minting side too.
func TestMintedIDsAreThirtyTwoLowercaseHex(t *testing.T) {
hex32 := regexp.MustCompile(`^[0-9a-f]{32}$`)
st := newStore(t)
hostID := enrollHost(t, st).ID
c, err := st.CreateVolumeClaim(testTenant, "data", 5)
require.NoError(t, err)
require.Regexp(t, hex32, c.ID, "claim id shape the agent's validVolumeID demands")
// bindClaims mints the volume id when the first VM places the claim.
require.NoError(t, st.CreateVM(testVM("vm1", hostID, c.ID)))
vols, err := st.ListVolumesForHost(hostID)
require.NoError(t, err)
require.Len(t, vols, 1)
require.Regexp(t, hex32, vols[0].ID, "volume id shape the agent's validVolumeID demands")
require.Equal(t, c.ID, vols[0].ClaimID)
}
// Force is for hardware that is gone, and it takes the disks with it.
func TestForceRemoveHostLosesItsVolumes(t *testing.T) {
st := newStore(t)
hostID := enrollHost(t, st).ID
live, err := st.CreateVolumeClaim(testTenant, "live", 5)
require.NoError(t, err)
doomed, err := st.CreateVolumeClaim(testTenant, "doomed", 5)
require.NoError(t, err)
require.NoError(t, st.CreateVM(testVM("vm1", hostID, live.ID, doomed.ID)))
// The doomed claim is already deleted and waiting on a reclaim ack that
// will never come.
require.NoError(t, st.TombstoneVM("vm1"))
require.NoError(t, st.HardDeleteVM("vm1", hostID))
require.NoError(t, st.TombstoneVolumeClaim(doomed.ID))
// Read the doomed volumes' ids while they still exist: after the force they
// are nowhere but the tally, which is the whole point of reporting them.
before, err := st.ListVolumesForHost(hostID)
require.NoError(t, err)
require.Len(t, before, 2)
wantIDs := []string{before[0].ID, before[1].ID}
sort.Strings(wantIDs)
gone, err := st.ForceRemoveHost(hostID)
require.NoError(t, err)
require.Equal(t, 0, gone.VMsPurged, "the VM was already reaped; the volumes were not")
// The tally is the receipt for the loss: both volumes, named, and only the
// LIVE claim counted as handed back — the tombstoned one was deleted, not
// returned to Pending.
require.Equal(t, 2, gone.VolumesDestroyed)
require.Equal(t, wantIDs, gone.VolumeIDs, "the destroyed volumes, in id order")
require.Equal(t, 1, gone.ClaimsUnbound, "the tombstoned claim went; only the live one is Pending again")
// The live claim survives as Pending: the request stands, the bytes do not.
got, err := st.GetVolumeClaim(live.ID)
require.NoError(t, err)
require.Equal(t, "", got.BoundVolumeID, "a claim whose host died is Pending again")
require.Equal(t, "", got.HostID)
// The tombstoned one is gone outright — no host is left to ack its reclaim.
require.Zero(t, countRows(t, st, `SELECT COUNT(*) FROM volume_claims WHERE id=?`, doomed.ID))
require.Zero(t, countRows(t, st, `SELECT COUNT(*) FROM volumes WHERE host_id=?`, hostID))
// And the freed claim can be placed again, elsewhere.
other := enrollSecondHost(t, st).ID
require.NoError(t, st.CreateVM(testVM("vm2", other, live.ID)))
got, err = st.GetVolumeClaim(live.ID)
require.NoError(t, err)
require.Equal(t, other, got.HostID)
}
// Force must also work while the VMs are still there — the usual dead-hardware
// case, where nothing drained at all.
func TestForceRemoveHostPurgesVMsHoldingVolumes(t *testing.T) {
st := newStore(t)
hostID := enrollHost(t, st).ID
c, err := st.CreateVolumeClaim(testTenant, "data", 5)
require.NoError(t, err)
require.NoError(t, st.CreateVM(testVM("vm1", hostID, c.ID)))
vols, err := st.ListVolumesForHost(hostID)
require.NoError(t, err)
require.Len(t, vols, 1)
gone, err := st.ForceRemoveHost(hostID)
require.NoError(t, err)
require.Equal(t, 1, gone.VMsPurged)
require.Equal(t, 1, gone.VolumesDestroyed)
require.Equal(t, 1, gone.ClaimsUnbound, "the claim is Pending again, on no host")
require.Equal(t, []string{vols[0].ID}, gone.VolumeIDs)
got, err := st.GetVolumeClaim(c.ID)
require.NoError(t, err)
require.Equal(t, "", got.BoundVolumeID)
require.Equal(t, "", got.VMID, "the attachment went with the VM")
require.Zero(t, countRows(t, st, `SELECT COUNT(*) FROM volumes WHERE host_id=?`, hostID))
}
// A refusal has to name what is in the way — the API turns these into the 409
// the caller reads.
func TestClaimErrorsNameWhatIsInTheWay(t *testing.T) {
require.Equal(t, "volume claim c1 is attached to vm vm1",
(&ClaimAttachedError{ClaimID: "c1", VMID: "vm1"}).Error())
require.Equal(t, "volume claim c1 is bound to host h1",
(&ClaimPinnedError{ClaimID: "c1", HostID: "h1"}).Error())
}
// One bad claim takes the whole create down, including the claims already
// bound earlier in the same list: a VM either gets all its volumes or none.
func TestCreateVMBindsAllClaimsOrNone(t *testing.T) {
st := newStore(t)
hostID := enrollHost(t, st).ID
good, err := st.CreateVolumeClaim(testTenant, "good", 5)
require.NoError(t, err)
require.ErrorIs(t, st.CreateVM(testVM("vm1", hostID, good.ID, "nope")), ErrClaimNotFound)
got, err := st.GetVolumeClaim(good.ID)
require.NoError(t, err)
require.Equal(t, "", got.BoundVolumeID, "the first claim's binding rolled back with the create")
vols, err := st.ListVolumesForHost(hostID)
require.NoError(t, err)
require.Empty(t, vols, "no volume was placed on the host")
}
func TestHardDeleteVolumeRefusesLiveVolume(t *testing.T) {
st := newStore(t)
hostID := enrollHost(t, st).ID
c, err := st.CreateVolumeClaim(testTenant, "data", 5)
require.NoError(t, err)
require.NoError(t, st.CreateVM(testVM("vm1", hostID, c.ID)))
got, err := st.GetVolumeClaim(c.ID)
require.NoError(t, err)
require.ErrorIs(t, st.HardDeleteVolume(got.BoundVolumeID), sql.ErrNoRows,
"a live volume is not reapable; the host still has the file")
}