internal/agent/reconcile/volumes.go
Ref: Size: 9.1 KiB History
package reconcile
import (
"errors"
"fmt"
"log/slog"
"os"
"path/filepath"
"github.com/a73x/eitri/internal/agent/permanent"
"github.com/a73x/eitri/internal/agent/state"
"github.com/a73x/eitri/internal/pb"
)
// reconcileVolumes drives the volumes directory toward the snapshot and
// reports every volume id found there. It runs BEFORE VM dispatch so a VM's
// disks exist when its backend lists them, and synchronously, because a sparse
// truncate costs nothing — there is no download and no copy to keep off the
// heartbeat's path.
//
// The rule that matters: a file the snapshot does not name is reported and
// KEPT. A VM that vanishes from a snapshot is eventually reclaimed; a volume
// never is, by automation. Only an explicit tombstone deletes data, because
// the file is the only copy of whatever a guest wrote to it.
//
// Every volume the snapshot names gets a row, present or not: the control
// plane reads an omitted volume as gone, so silence about one it asked for
// would reap the row out from under a file that is still here.
func (e *Engine) reconcileVolumes(snap *pb.Snapshot) []*pb.VolumeStatus {
var out []*pb.VolumeStatus
seen := make(map[string]bool, len(snap.GetVolumes()))
for _, spec := range snap.GetVolumes() {
id := spec.GetVolumeId()
if !validVolumeID(id) {
// Refused before the id is joined into a path. Reported absent, so
// the control plane hears an answer for every volume it named.
slog.Warn("reconcile: refusing a volume id that is not one", "volume", id)
out = append(out, &pb.VolumeStatus{VolumeId: id, Present: false})
continue
}
seen[id] = true
out = append(out, e.convergeVolume(id, spec))
}
return append(out, e.reportVolumes(seen)...) // orphans: reported, kept
}
// reportVolumes lists what is in the volumes directory, skipping the ids in
// skip, and TOUCHES NOTHING: it stats, it does not create, mark or delete. The
// fence path reports on state without acting on it, and calls this on its own.
//
// Directory names are not run through validVolumeID: they came from this
// agent's own writes, a filesystem cannot hold a separator in one, and nothing
// here does more than stat them. An orphan with a strange name is still an
// orphan the operator should be told about.
func (e *Engine) reportVolumes(skip map[string]bool) []*pb.VolumeStatus {
entries, err := os.ReadDir(e.St.VolumesDir())
if err != nil {
// Not fatal: a caller that converged volumes has already reported them.
// What is lost is the orphan report, so say so — an operator chasing a
// volume that is in neither the fleet nor the host has nothing else to
// go on.
slog.Warn("reconcile: cannot list the volumes directory", "err", err)
return nil
}
var out []*pb.VolumeStatus
for _, ent := range entries {
if !ent.IsDir() || skip[ent.Name()] {
continue
}
if st := e.statVolume(ent.Name()); st != nil {
out = append(out, st)
}
}
return out
}
// volumesMaterialized refuses a boot whose volumes are not all files on this
// host. It guards EVERY call to Prov.Boot — the create, the restart after a
// host reboot, and a plain power-on — because a guest handed a device that is
// not there finds a filesystem missing and has no way to report that upward:
// the VM comes up "ready" with a mount point full of nothing.
//
// Permanent: nothing on this host heals a volume the control plane never placed
// here, so spending three ticks finding that out helps nobody. Re-placing it
// edits the spec, which hands the VM a fresh budget.
func (e *Engine) volumesMaterialized(spec state.VMSpec) error {
for _, id := range spec.VolumeIDs {
if _, err := os.Stat(e.St.VolumePath(id)); err != nil {
return permanent.Errorf("volume %s not materialized on this host", id)
}
}
return nil
}
// validVolumeID reports whether id is what the control plane mints for a
// volume: random.Hex(16), so 32 lowercase hex characters. It is checked before
// the id is joined into a path, because a volume path is the one this package
// hands to os.RemoveAll — an id carrying a separator or a ".." would let a
// malformed snapshot aim a reclaim at a directory that is not a volume, a VM's
// own among them. Empty, wrong length and wrong alphabet all fail here.
func validVolumeID(id string) bool {
if len(id) != 32 {
return false
}
for _, c := range id {
if (c < '0' || c > '9') && (c < 'a' || c > 'f') {
return false
}
}
return true
}
// statVolume reports a volume as found on disk, nil ONLY when there is no file.
// Size is what the file claims, which is the size the guest's filesystem was
// made against.
//
// An error that is not "does not exist" reports the volume PRESENT, with
// whatever size could be read. Absence is what sends convergeVolume off to
// create the file, and creating ends in a rename over whatever is at that path
// — so a permission error, an I/O error or a bad symlink read as "missing"
// would destroy the volume it failed to look at. Only ErrNotExist is evidence
// of absence; everything else is evidence of nothing.
func (e *Engine) statVolume(id string) *pb.VolumeStatus {
fi, err := os.Stat(e.St.VolumePath(id))
switch {
case errors.Is(err, os.ErrNotExist):
return nil
case err != nil:
slog.Warn("reconcile: cannot stat volume; assuming it is there", "volume", id, "err", err)
return &pb.VolumeStatus{VolumeId: id, Present: true}
}
return &pb.VolumeStatus{VolumeId: id, Present: true, SizeGb: fi.Size() >> 30}
}
// convergeVolume drives ONE volume toward its spec and reports what is on disk
// afterwards.
func (e *Engine) convergeVolume(id string, spec *pb.VolumeSpec) *pb.VolumeStatus {
if spec.GetTombstoned() {
return e.reclaimVolume(id)
}
if st := e.statVolume(id); st != nil {
// NEVER resized, in either direction: a guest filesystem sits on this
// file, so growing it is the control plane's job to do through the
// guest and shrinking it destroys data. Reported as found.
return st
}
if err := createSparse(e.St.VolumePath(id), spec.GetSizeGb()); err != nil {
// Reported absent rather than skipped. The next tick retries; until one
// succeeds the control plane can see that this host owes a file.
slog.Warn("reconcile: create volume", "volume", id, "err", err)
return &pb.VolumeStatus{VolumeId: id, Present: false}
}
return e.statVolume(id)
}
// reclaimVolume deletes a tombstoned volume's directory once TombstoneGrace has
// passed since the marker was first written, and reports what is left.
//
// The grace is deliberately measured from a marker on disk rather than from the
// tick that noticed: the tombstone is restated in every snapshot, so timing it
// from the snapshot would restart the clock forever. The marker's mtime is a
// wall-clock fact and Engine.Now is the loop's clock — the same clock in
// production, and injectable here for the same reason the quarantine grace is.
func (e *Engine) reclaimVolume(id string) *pb.VolumeStatus {
st := e.statVolume(id)
if st == nil {
// Nothing to reclaim — already deleted, or never materialised on this
// host. Say so at once so the control plane can reap the row, and leave
// no directory behind for a file that does not exist.
return &pb.VolumeStatus{VolumeId: id, Present: false}
}
marker := e.St.VolumeTombstonePath(id)
fi, err := os.Stat(marker)
if errors.Is(err, os.ErrNotExist) {
if err = os.WriteFile(marker, nil, 0o600); err == nil {
fi, err = os.Stat(marker)
}
}
if err != nil {
// A clock this host cannot read is not permission to delete. Keep the
// file and retry next tick.
slog.Warn("reconcile: volume tombstone marker", "volume", id, "err", err)
return st
}
if e.Now().Sub(fi.ModTime()) < e.TombstoneGrace {
return st // still in grace: the delete is undoable until it is not
}
// Grace expired: the whole directory goes, marker included. A failure keeps
// the file — the level-triggered loop retries, and the control plane must
// not reap the row while the bytes are still here.
if err := os.RemoveAll(e.St.VolumeDir(id)); err != nil {
slog.Warn("reconcile: reclaim volume", "volume", id, "err", err)
return st
}
return &pb.VolumeStatus{VolumeId: id, Present: false}
}
// createSparse makes a sizeGB sparse file at path via .partial + rename, the
// shape PrepareRootDisk uses, so a crash mid-create leaves no torn file at the
// final path — and the file at the final path is therefore always the full
// size a guest was promised.
func createSparse(path string, sizeGB int64) error {
// Bounds, not policy: the control plane validates what a tenant may ask
// for. This refuses what cannot be a volume at all, so a zero or a garbage
// size fails here instead of producing an empty block device a guest would
// mount.
if sizeGB < 1 || sizeGB > 1<<20 {
return fmt.Errorf("size_gb %d out of range", sizeGB)
}
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
return err
}
tmp := path + ".partial"
_ = os.Remove(tmp) // a previous attempt's torn temporary, if any
f, err := os.OpenFile(tmp, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
if err != nil {
return err
}
if err := f.Truncate(sizeGB << 30); err != nil {
_ = f.Close()
_ = os.Remove(tmp)
return err
}
if err := f.Close(); err != nil {
_ = os.Remove(tmp)
return err
}
return os.Rename(tmp, path)
}