internal/server/api/capacity.go

Ref: main v0.0.5   Size: 7.1 KiB   History

package api

import (
	"fmt"
	"strings"

	"github.com/a73x/eitri/internal/server/registry"
	"github.com/a73x/eitri/internal/server/store"
)

// declaredLimit answers the only question this refusal may rest on: is the
// number this host advertised for a dimension a limit its operator declared, or
// is it just the size of the machine? It returns the limit when the report
// proves the first, and 0 — "nothing proven, do not judge" — otherwise.
//
// The proof is arithmetic on the host's own report. An agent advertises the
// machine's totals clamped to its --max-vcpus/--max-mem-mb/--max-disk-gb flags
// (syncclient.advertisedCapacity), and the clamp only ever lowers, so an
// advertisement BELOW the machine's total can only be a configured cap. The
// machine's total is not on the wire as such, but the metrics in the same
// report imply it: the agent computes used as total-minus-available, so
// mem_used_mb + mem_available_mb is exactly the machine's memory, and
// disk_used_gb + disk_free_gb is the state dir's filesystem minus its reserved
// blocks — an UNDERCOUNT, which is the safe direction here (it can only fail to
// prove a cap, never invent one).
//
// This is a coupling to hostinfo.readMetrics on both platforms, and an
// invariant test there guards it (TestMetricsImplyTheCapacityTheSameProbeSees).
// If that identity is ever broken the failure is a refusal that stops firing,
// not one that fires wrongly.
//
// Everything unprovable reads as uncapped, and the list is worth naming: vCPUs
// always (no report carries the machine's core count, so a --max-vcpus cap is
// invisible here), a dimension whose probe failed and reports zeroes, and a cap
// set at exactly the machine's total. Each of those falls back to the world
// before this preflight existed — the agent refuses at boot and the VM sits
// failed — which is the honest cost of not refusing a VM the host would have
// run. See overCapacityRefusal for why that asymmetry is the whole design.
func declaredLimit(advertised, machineTotal int64) int64 {
	if advertised <= 0 || machineTotal <= 0 || advertised >= machineTotal {
		return 0
	}
	return advertised
}

// dimension is one resource under judgement: what the VM asked for, what the
// host is holding (split live vs. still-being-torn-down), and the limit — 0
// when this dimension is not judged at all.
type dimension struct {
	noun, unit          string
	want, live, pending int64
	limit               int64
}

func (d dimension) overWithPending() bool { return d.limit > 0 && d.live+d.pending+d.want > d.limit }
func (d dimension) overLiveAlone() bool   { return d.limit > 0 && d.live+d.want > d.limit }

// held phrases the binding numbers the way an operator has to act on them:
// what was asked for, what is already there, out of what.
func (d dimension) held() string {
	return fmt.Sprintf("%s — needs %d%s, already holds %d of %d%s",
		d.noun, d.want, d.unit, d.live+d.pending, d.limit, d.unit)
}

// heldWithPending is the same line for a refusal a teardown is tipping: the
// share held by VMs on their way out is the part that is about to go away.
func (d dimension) heldWithPending() string {
	return fmt.Sprintf("%s, %d%s of it by the teardown", d.held(), d.pending, d.unit)
}

// overCapacityRefusal explains why a host cannot be given a VM: it is already
// holding as much as its operator said it may. It returns "" when the VM fits,
// and — deliberately — whenever this server cannot prove the host would refuse.
//
// The answer exists at create — the host's capacity and everything already on
// it are both known here — so the caller learns it in milliseconds instead of
// watching a VM sit `creating` and then `failed` because the agent reached the
// same conclusion minutes later, at the far end of an image download.
//
// The agent's admission is the AUTHORITY; this is a courtesy that runs first,
// and a courtesy must never be stricter than the authority it anticipates. The
// agent refuses a boot only against a configured cap (reconcile.quotaCheckLocked
// — with no --max-* flag it admits whatever it is asked, and that is the design:
// guest disks are sparse and memory is not preallocated, so a host deliberately
// runs more than it has on paper). So this refuses only where declaredLimit can
// prove a cap exists, and stays silent everywhere else. Refusing a VM the host
// would have run is the one failure with no remedy short of an ssh session;
// missing one it will refuse costs the caller the wait it always used to cost.
//
// Every judged dimension that binds is named: a VM that is too big in two ways
// should be resized once, not twice.
func overCapacityRefusal(hostName, hostID string, want store.Alloc, held store.Commitment, rep registry.Report) string {
	m := rep.Metrics
	dims := []dimension{
		{noun: "vcpus", want: want.VCPUs, live: held.Live.VCPUs, pending: held.Pending.VCPUs,
			limit: declaredLimit(rep.Capacity.VCPUs, 0)},
		{noun: "memory", unit: "MB", want: want.MemMB, live: held.Live.MemMB, pending: held.Pending.MemMB,
			limit: declaredLimit(rep.Capacity.MemMB, m.MemUsedMB+m.MemAvailableMB)},
		{noun: "disk", unit: "GB", want: want.DiskGB, live: held.Live.DiskGB, pending: held.Pending.DiskGB,
			limit: declaredLimit(rep.Capacity.DiskGB, m.DiskUsedGB+m.DiskFreeGB)},
	}

	var binding []dimension
	liveAloneFits := true
	for _, d := range dims {
		if d.overWithPending() {
			binding = append(binding, d)
		}
		if d.overLiveAlone() {
			liveAloneFits = false
		}
	}
	if len(binding) == 0 {
		return ""
	}

	// A destroy in flight is the reason, and "delete a VM" would be a lie: the
	// caller may well have just deleted one, and is being refused by the very
	// resources that delete has not finished releasing. Say what is actually
	// happening and give the remedy that matches it — the wait ends by itself.
	if liveAloneFits {
		var lines []string
		for _, d := range binding {
			lines = append(lines, d.heldWithPending())
		}
		return fmt.Sprintf("host %s (%s) has no room for this VM yet: %s. %s, and a deleted VM holds its "+
			"vcpus, memory and disk until its host finishes tearing it down — the VMs still running there "+
			"would leave room for this one. Wait for the teardown to finish and create it again, ask for "+
			"less, or create it on a host with room.",
			hostName, hostID, strings.Join(lines, "; "), pendingTeardowns(held.PendingVMs))
	}

	var lines []string
	for _, d := range binding {
		lines = append(lines, d.held())
	}
	return fmt.Sprintf("host %s (%s) has no room for this VM: %s. That is a limit its operator set below "+
		"what the machine has, so the agent there would refuse this VM at boot and it would sit failed. "+
		"Ask for less, delete a VM on that host to free what it holds, or create it on a host with room.",
		hostName, hostID, strings.Join(lines, "; "))
}

// pendingTeardowns names the deletes still in flight, counted, so the caller
// can tell "the one I just deleted" from "somebody else is clearing the host".
func pendingTeardowns(n int) string {
	if n == 1 {
		return "1 VM deleted from that host is still being torn down"
	}
	return fmt.Sprintf("%d VMs deleted from that host are still being torn down", n)
}