internal/server/api/capacity_api_test.go
Ref: Size: 13.4 KiB History
package api
import (
"encoding/json"
"io"
"net/http"
"net/http/httptest"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/a73x/eitri/internal/server/registry"
)
// agentReportsCappedCapacity is the report of a host whose operator held some
// of it back: it advertises less than the machine it also measures. That gap is
// the only proof this server gets that a cap exists — and a cap is the only
// thing the agent will refuse a boot against — so it is the setup for every
// refusal below. The metrics are a machine an order of magnitude bigger than
// what it offers, which is what --max-mem-mb/--max-disk-gb produce.
func agentReportsCappedCapacity(hostID string, vcpus, memMB, diskGB int64) {
testReg.UpdateReport(hostID, registry.Report{
Capacity: registry.Capacity{VCPUs: vcpus, MemMB: memMB, DiskGB: diskGB},
Metrics: registry.Metrics{
MemUsedMB: memMB, MemAvailableMB: memMB * 10,
DiskUsedGB: diskGB, DiskFreeGB: diskGB * 10,
},
})
}
// agentReportsUncappedCapacity is the other host: no --max-* flags, so it
// advertises exactly the machine its own metrics describe. Nothing here says
// "limit" — an uncapped agent admits whatever it is asked — so nothing here may
// refuse a create.
func agentReportsUncappedCapacity(hostID string, vcpus, memMB, diskGB int64) {
testReg.UpdateReport(hostID, registry.Report{
Capacity: registry.Capacity{VCPUs: vcpus, MemMB: memMB, DiskGB: diskGB},
Metrics: registry.Metrics{
MemUsedMB: memMB / 4, MemAvailableMB: memMB - memMB/4,
DiskUsedGB: diskGB / 4, DiskFreeGB: diskGB - diskGB/4,
},
})
}
// createVM posts one create and returns the response, so a test reads as the
// sequence of placements it is really making.
func createVM(t *testing.T, ts *httptest.Server, hostID, name string, vcpus, memMB, diskGB int64) *http.Response {
t.Helper()
return do(t, "POST", ts.URL+"/api/v1/vms", testPAT, map[string]any{
"host_id": hostID, "name": name,
"vcpus": vcpus, "mem_mb": memMB, "disk_gb": diskGB,
})
}
func bodyOf(t *testing.T, resp *http.Response) string {
t.Helper()
b, err := io.ReadAll(resp.Body)
require.NoError(t, err)
return string(b)
}
// TestCreateVMRefusesAHostWithNoRoom pins the third refusal of this shape: the
// request is fine and the host has been told not to serve it. Each dimension
// binds on its own, and the refusal has to be actionable on its own terms —
// which dimension, what the VM asked for, what the host is already holding out
// of what it may hold.
func TestCreateVMRefusesAHostWithNoRoom(t *testing.T) {
for _, tc := range []struct {
name string
vcpus, memMB, diskGB int64
want string
}{
{"memory", 1, 8192, 10, "memory — needs 8192MB, already holds 2048 of 4096MB"},
{"disk", 1, 1024, 200, "disk — needs 200GB, already holds 10 of 100GB"},
} {
t.Run(tc.name, func(t *testing.T) {
ts, st, _ := testServer(t)
out := enroll(t, ts)
agentReportsCappedCapacity(out["host_id"], 6, 4096, 100)
// One VM already there, so the refusal has a "holds" to report.
require.Equal(t, 201, createVM(t, ts, out["host_id"], "sitting", 2, 2048, 10).StatusCode)
resp := createVM(t, ts, out["host_id"], "toobig", tc.vcpus, tc.memMB, tc.diskGB)
require.Equal(t, 409, resp.StatusCode)
msg := bodyOf(t, resp)
assert.Contains(t, msg, tc.want)
assert.Contains(t, msg, "host host-a ("+out["host_id"]+")", "the refusal must name the host")
assert.Contains(t, msg, "delete a VM on that host", "live VMs fill it: the remedy is to free one")
assert.Contains(t, msg, "create it on a host with room", "the refusal must name a way out")
vms, err := st.ListVMs()
require.NoError(t, err)
require.Len(t, vms, 1, "a refused create must leave no row behind")
})
}
}
// TestCreateVMCountsAPendingClaimAgainstTheDisk is the disk dimension's other
// half. A create that names a pending claim commits the host to that claim's
// bytes as surely as to the guest's root disk — the volume is placed by THIS
// create — so the two are judged together. Judging only disk_gb would admit a
// VM whose volume the host has no room for, and the tenant would find that out
// when the agent failed to materialize the file.
//
// A bound claim is not added: its volume is already on the host, so
// CommittedOnHost has counted it, and counting it again would refuse a VM for
// disk it is re-using rather than asking for.
func TestCreateVMCountsAPendingClaimAgainstTheDisk(t *testing.T) {
ts, _, _, reg, _ := newServer(t)
floorVolumes(t, "v0.0.7")
out := enroll(t, ts)
hostID := out["host_id"]
onlineAt(reg, hostID, "v0.0.7")
agentReportsCappedCapacity(hostID, 8, 8192, 100)
claimPost(t, ts, "data", 60)
resp := do(t, "POST", ts.URL+"/api/v1/vms", testPAT, map[string]any{
"host_id": hostID, "name": "with-volume", "vcpus": 1, "mem_mb": 1024, "disk_gb": 50,
"volume_claims": []string{"data"}})
require.Equal(t, 409, resp.StatusCode)
assert.Contains(t, bodyOf(t, resp), "disk — needs 110GB, already holds 0 of 100GB",
"the refusal must count the volume the create would place")
assert.Equal(t, 201, createVM(t, ts, hostID, "no-volume", 1, 1024, 50).StatusCode,
"the same VM without the claim fits")
}
// TestCreateVMRefusalNamesEveryBindingDimension: a VM too big in two ways
// should be resized once, not discovered twice.
func TestCreateVMRefusalNamesEveryBindingDimension(t *testing.T) {
ts, _, _ := testServer(t)
out := enroll(t, ts)
agentReportsCappedCapacity(out["host_id"], 6, 4096, 100)
resp := createVM(t, ts, out["host_id"], "huge", 1, 8192, 200)
require.Equal(t, 409, resp.StatusCode)
msg := bodyOf(t, resp)
assert.Contains(t, msg, "memory — needs 8192MB, already holds 0 of 4096MB")
assert.Contains(t, msg, "disk — needs 200GB, already holds 0 of 100GB")
}
// TestCreateVMFillsAHostExactly is the boundary: a declared capacity is a
// limit, not a threshold. A VM that fits the last of the host is placed, and
// the next one — however small — is refused, because there is nothing left.
func TestCreateVMFillsAHostExactly(t *testing.T) {
ts, _, _ := testServer(t)
out := enroll(t, ts)
agentReportsCappedCapacity(out["host_id"], 4, 4096, 40)
require.Equal(t, 201, createVM(t, ts, out["host_id"], "half", 1, 2048, 20).StatusCode)
require.Equal(t, 201, createVM(t, ts, out["host_id"], "rest", 1, 2048, 20).StatusCode,
"a VM that exactly fills the host still fits")
resp := createVM(t, ts, out["host_id"], "onemore", 1, 1, 1)
require.Equal(t, 409, resp.StatusCode)
assert.Contains(t, bodyOf(t, resp), "memory — needs 1MB, already holds 4096 of 4096MB")
}
// TestCreateVMNeverRefusesAnUncappedHost is the rule this whole preflight is
// bounded by: the agent's admission is the authority, and an agent with no
// --max-* flag admits whatever it is asked. A host that advertises the machine
// it measures has declared no limit, so its totals are not a wall — guest disks
// are sparse and guest memory is not preallocated, and running well past the
// paper numbers is what such a host is for. A courtesy check that refused here
// would block VMs the fleet is designed to run, and no operator could see why
// without an ssh session.
func TestCreateVMNeverRefusesAnUncappedHost(t *testing.T) {
ts, st, _ := testServer(t)
out := enroll(t, ts)
agentReportsUncappedCapacity(out["host_id"], 4, 4096, 100)
require.Equal(t, 201, createVM(t, ts, out["host_id"], "big", 8, 8192, 400).StatusCode,
"one VM may exceed the whole machine")
require.Equal(t, 201, createVM(t, ts, out["host_id"], "bigger", 8, 8192, 400).StatusCode,
"and so may the next one")
vms, err := st.ListVMs()
require.NoError(t, err)
assert.Len(t, vms, 2)
}
// TestCreateVMNeverJudgesVCPUs is the honest edge of the proof. A cap is proven
// by an advertisement below the machine's own measurements, and no report
// carries the machine's core count — so a --max-vcpus cap is invisible here and
// a vCPU overflow is never refused, even on a host proven capped in memory. The
// agent still enforces it at boot; this server just cannot say so first, and
// guessing would refuse every oversubscribed VM on the fleet.
func TestCreateVMNeverJudgesVCPUs(t *testing.T) {
ts, _, _ := testServer(t)
out := enroll(t, ts)
agentReportsCappedCapacity(out["host_id"], 2, 4096, 100)
resp := createVM(t, ts, out["host_id"], "manycpus", 64, 1024, 10)
assert.Equal(t, 201, resp.StatusCode)
}
// TestCreateVMJudgesOnlyAHostThatHasReportedCapacity: reported capacity is
// registry state. A host that has never connected, one that has said hello and
// nothing since, and one whose report carries no capacity at all are all hosts
// this server knows nothing about the size of — and a zero read as "no room"
// would refuse every create on the fleet for the moment after a restart. All
// three take the create as desired state; the agent's own admission check is
// the backstop.
func TestCreateVMJudgesOnlyAHostThatHasReportedCapacity(t *testing.T) {
t.Run("never connected", func(t *testing.T) {
ts, st, _ := testServer(t)
out := enrollSilent(t, ts)
require.Equal(t, 201, createVM(t, ts, out["host_id"], "pending", 64, 999999, 9999).StatusCode)
vms, _ := st.ListVMs()
assert.Len(t, vms, 1, "the create must land as desired state")
})
t.Run("said hello, never reported", func(t *testing.T) {
ts, dbst, _ := testServer(t)
out := enroll(t, ts) // a Hello and nothing since: the registry knows the
// agent's version and not one thing about the machine
require.Equal(t, 201, createVM(t, ts, out["host_id"], "pending", 64, 999999, 9999).StatusCode)
vms, _ := dbst.ListVMs()
assert.Len(t, vms, 1)
})
t.Run("online but reporting no capacity", func(t *testing.T) {
ts, dbst, _ := testServer(t)
out := enroll(t, ts)
agentReports(out["host_id"]) // a report with no capacity in it
require.Equal(t, 201, createVM(t, ts, out["host_id"], "unjudged", 64, 999999, 9999).StatusCode)
vms, _ := dbst.ListVMs()
assert.Len(t, vms, 1)
})
t.Run("capacity but no metrics to measure it against", func(t *testing.T) {
ts, dbst, _ := testServer(t)
out := enroll(t, ts)
testReg.UpdateReport(out["host_id"], registry.Report{
Capacity: registry.Capacity{VCPUs: 4, MemMB: 4096, DiskGB: 40}})
require.Equal(t, 201, createVM(t, ts, out["host_id"], "unjudged", 8, 99999, 999).StatusCode,
"without the machine's size, nothing proves that capacity is a limit")
vms, _ := dbst.ListVMs()
assert.Len(t, vms, 1)
})
}
// TestCreateVMCountsATombstonedVMUntilItIsReaped is the predicate the whole
// refusal rests on. A tombstone is a destroy in progress: the disk is still on
// the host and the guest may still be shutting down, so its resources are not
// free to promise to somebody else. Only the reap — the agent's ack, or the
// abandoned sweep — frees them, and it frees them by removing the row.
//
// It is also the case where the obvious remedy is a lie. The caller is being
// refused BY the VM they just deleted, so "delete a VM" would send them looking
// for one that is already gone; the refusal has to name the teardown and say to
// wait for it, because that wait ends by itself.
func TestCreateVMCountsATombstonedVMUntilItIsReaped(t *testing.T) {
ts, dbst, _ := testServer(t)
out := enroll(t, ts)
agentReportsCappedCapacity(out["host_id"], 4, 4096, 40)
resp := createVM(t, ts, out["host_id"], "first", 1, 4096, 40)
require.Equal(t, 201, resp.StatusCode)
var created map[string]string
require.NoError(t, json.NewDecoder(resp.Body).Decode(&created))
require.Equal(t, 204, do(t, "DELETE", ts.URL+"/api/v1/vms/"+created["id"], testPAT, nil).StatusCode)
refused := createVM(t, ts, out["host_id"], "second", 1, 4096, 40)
require.Equal(t, 409, refused.StatusCode, "a deleting VM still occupies the host")
msg := bodyOf(t, refused)
assert.Contains(t, msg, "memory — needs 4096MB, already holds 4096 of 4096MB, 4096MB of it by the teardown")
assert.Contains(t, msg, "1 VM deleted from that host is still being torn down",
"the refusal must name how many teardowns are holding the room")
assert.Contains(t, msg, "Wait for the teardown to finish", "the wait is the remedy")
assert.NotContains(t, msg, "delete a VM on that host",
"the caller already deleted one; sending them to delete another is the lie")
// The agent acks the destroy: the row goes, and with it the commitment.
require.NoError(t, dbst.HardDeleteVM(created["id"], out["host_id"]))
assert.Equal(t, 201, createVM(t, ts, out["host_id"], "second", 1, 4096, 40).StatusCode,
"a reaped VM holds nothing")
}
// TestCreateVMKeepsTheDeleteRemedyWhenLiveVMsFillTheHost is the other half of
// the pair: a teardown in flight does not soften a host that is full without
// it. The VMs holding the room are really there, so the remedy is really to
// free one.
func TestCreateVMKeepsTheDeleteRemedyWhenLiveVMsFillTheHost(t *testing.T) {
ts, _, _ := testServer(t)
out := enroll(t, ts)
agentReportsCappedCapacity(out["host_id"], 8, 8192, 80)
require.Equal(t, 201, createVM(t, ts, out["host_id"], "living", 1, 6144, 60).StatusCode)
dying := createVM(t, ts, out["host_id"], "dying", 1, 2048, 20)
require.Equal(t, 201, dying.StatusCode)
var created map[string]string
require.NoError(t, json.NewDecoder(dying.Body).Decode(&created))
require.Equal(t, 204, do(t, "DELETE", ts.URL+"/api/v1/vms/"+created["id"], testPAT, nil).StatusCode)
refused := createVM(t, ts, out["host_id"], "next", 1, 4096, 40)
require.Equal(t, 409, refused.StatusCode)
msg := bodyOf(t, refused)
assert.Contains(t, msg, "delete a VM on that host",
"the live VMs alone leave no room: waiting for the teardown will not help")
assert.NotContains(t, msg, "still being torn down")
}