39c407f2 — eitri

RETRO.md

Old		New
@@ -29,6 +29,10 @@ One line per push to `main`: what slowed the work down. Enforced by
29	a tree that no longer existed. Proving the rest was comments-only needed a	29	a tree that no longer existed. Proving the rest was comments-only needed a
30	purpose-built token-stream comparator; go/printer reflows, so a diff that	30	purpose-built token-stream comparator; go/printer reflows, so a diff that
31	round-trips through it reads as changed when nothing was.	31	round-trips through it reads as changed when nothing was.
		32	- Volumes: the plan specified a schema FK (`volumes.host_id → hosts`) that
		33	broke host removal, and a 120s smoke cleanup window against a 5-minute reap
		34	grace; both were only caught by review against the live contracts, not by
		35	the task tests.
32		36
33	## 2026-08-23	37	## 2026-08-23
34		38

docs/assumptions.md

Old		New
@@ -731,3 +731,15 @@ document the first time the part handler reads it and the stripped result is
731	what gets written, so the later user-data-versus-vendor-data merge never sees	731	what gets written, so the later user-data-versus-vendor-data merge never sees
732	one and its first-wins default stands. Vendor-data was dropped from the seed	732	one and its first-wins default stands. Vendor-data was dropped from the seed
733	entirely rather than left as a second mechanism.	733	entirely rather than left as a second mechanism.
		734
		735	### A volume is exactly as durable as one host's disk
		736
		737	A volume is a raw sparse file on the host that first materialized it. There is
		738	no replication, no snapshot, and no migration; `e698610c` (VMs cannot move
		739	between hosts) now also means volumes cannot. A host that loses its disk loses
		740	every volume on it. Force-removing a host is the one deliberate path that takes
		741	its volumes with it — their claims return to pending rather than staying bound
		742	to a host that is gone.
		743	Falsifier: a volume readable from any host other than the one it was
		744	created on.
		745

docs/openapi.json

@@ @@ -151,6 +151,12 @@ @@
+          },
+          "vcpus": {
+            "type": "integer"
+          },
+          "volume_claims": {
+            "items": {
+              "type": "string"
+            },
+            "type": "array"
+        },
+        "type": "object"
@@ @@ -170,6 +176,17 @@ @@
+        ],
+        "type": "object"
+      },
+      "CreateVolumeClaimRequest": {
+        "properties": {
+          "name": {
+            "type": "string"
+          },
+          "size_gb": {
+            "type": "integer"
+          }
+        },
+        "type": "object"
+      },
+      "Delegation": {
+        "properties": {
+          "ca_fingerprint": {
@@ @@ -862,6 +879,48 @@ @@
+          "vcpus"
+        ],
+        "type": "object"
+      },
+      "VolumeClaim": {
+        "properties": {
+          "created_at": {
+            "format": "date-time",
+            "type": "string"
+          },
+          "host_id": {
+            "type": "string"
+          },
+          "id": {
+            "type": "string"
+          },
+          "name": {
+            "type": "string"
+          },
+          "present": {
+            "type": [
+              "boolean",
+              "null"
+            ]
+          },
+          "size_gb": {
+            "type": "integer"
+          },
+          "status": {
+            "type": "string"
+          },
+          "vm_id": {
+            "type": "string"
+          }
+        },
+        "required": [
+          "created_at",
+          "host_id",
+          "id",
+          "name",
+          "size_gb",
+          "status",
+          "vm_id"
+        ],
+        "type": "object"
+    },
+    "securitySchemes": {
@@ @@ -2198,6 +2257,156 @@ @@
+        ],
+        "summary": "Un-tombstone a VM still within the teardown grace window; the agent re-adopts the guest."
+    },
+    "/api/v1/volume-claims": {
+      "get": {
+        "responses": {
+          "200": {
+            "content": {
+              "application/json": {
+                "schema": {
+                  "items": {
+                    "$ref": "#/components/schemas/VolumeClaim"
+                  },
+                  "type": "array"
+                }
+              }
+            },
+            "description": "success"
+          },
+          "default": {
+            "content": {
+              "text/plain": {
+                "schema": {
+                  "type": "string"
+                }
+              }
+            },
+            "description": "error (plain text)"
+          }
+        },
+        "security": [
+          {
+            "patToken": []
+          }
+        ],
+        "summary": "List the tenant's claims: where each is bound, which VM holds it, and whether its host has the file."
+      },
+      "post": {
+        "requestBody": {
+          "content": {
+            "application/json": {
+              "schema": {
+                "$ref": "#/components/schemas/CreateVolumeClaimRequest"
+              }
+            }
+          },
+          "required": true
+        },
+        "responses": {
+          "201": {
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/VolumeClaim"
+                }
+              }
+            },
+            "description": "success"
+          },
+          "default": {
+            "content": {
+              "text/plain": {
+                "schema": {
+                  "type": "string"
+                }
+              }
+            },
+            "description": "error (plain text)"
+          }
+        },
+        "security": [
+          {
+            "patToken": []
+          }
+        ],
+        "summary": "Claim durable storage. Pending until the first VM naming it is created; that VM's host then holds the bytes, and every later VM using the claim is placed there."
+      }
+    },
+    "/api/v1/volume-claims/{id}": {
+      "delete": {
+        "parameters": [
+          {
+            "in": "path",
+            "name": "id",
+            "required": true,
+            "schema": {
+              "type": "string"
+            }
+          }
+        ],
+        "responses": {
+          "204": {
+            "description": "success"
+          },
+          "default": {
+            "content": {
+              "text/plain": {
+                "schema": {
+                  "type": "string"
+                }
+              }
+            },
+            "description": "error (plain text)"
+          }
+        },
+        "security": [
+          {
+            "patToken": []
+          }
+        ],
+        "summary": "Delete a claim and the data behind it. Refused (409) while a VM holds it."
+      },
+      "get": {
+        "parameters": [
+          {
+            "in": "path",
+            "name": "id",
+            "required": true,
+            "schema": {
+              "type": "string"
+            }
+          }
+        ],
+        "responses": {
+          "200": {
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/VolumeClaim"
+                }
+              }
+            },
+            "description": "success"
+          },
+          "default": {
+            "content": {
+              "text/plain": {
+                "schema": {
+                  "type": "string"
+                }
+              }
+            },
+            "description": "error (plain text)"
+          }
+        },
+        "security": [
+          {
+            "patToken": []
+          }
+        ],
+        "summary": "One claim."
+      }

docs/shape.html

Old		New
@@ -236,6 +236,7 @@
236	"plane": "data",	236	"plane": "data",
237	"synopsis": "Package reconcile implements the agent's level-triggered reconcile loop.",	237	"synopsis": "Package reconcile implements the agent's level-triggered reconcile loop.",
238	"imports": [	238	"imports": [
		239	"internal/agent/permanent",
239	"internal/agent/seed",	240	"internal/agent/seed",
240	"internal/agent/state",	241	"internal/agent/state",
241	"internal/pb",	242	"internal/pb",
@@ -581,6 +582,7 @@
581	"internal/server/hosttoken",	582	"internal/server/hosttoken",
582	"internal/server/hub",	583	"internal/server/hub",
583	"internal/server/registry",	584	"internal/server/registry",
		585	"internal/server/release",
584	"internal/server/store",	586	"internal/server/store",
585	"internal/transport"	587	"internal/transport"
586	]	588	]

docs/shape.json

Old		New
@@ -185,6 +185,7 @@
185	"plane": "data",	185	"plane": "data",
186	"synopsis": "Package reconcile implements the agent's level-triggered reconcile loop.",	186	"synopsis": "Package reconcile implements the agent's level-triggered reconcile loop.",
187	"imports": [	187	"imports": [
		188	"internal/agent/permanent",
188	"internal/agent/seed",	189	"internal/agent/seed",
189	"internal/agent/state",	190	"internal/agent/state",
190	"internal/pb",	191	"internal/pb",
@@ -530,6 +531,7 @@
530	"internal/server/hosttoken",	531	"internal/server/hosttoken",
531	"internal/server/hub",	532	"internal/server/hub",
532	"internal/server/registry",	533	"internal/server/registry",
		534	"internal/server/release",
533	"internal/server/store",	535	"internal/server/store",
534	"internal/transport"	536	"internal/transport"
535	]	537	]

docs/volumes.md

@@ @@ -0,0 +1,63 @@ @@
+<!-- DRAFT: voice not yet passed by the author -->
+# Volumes
+*Durable block storage that outlives the VM it is attached to*
+A guest's root disk dies with the VM. A volume is where state survives past
+`DELETE /vms/{id}` — a checked-out repo, a database, a build cache.
+## Claim storage
+    POST /api/v1/volume-claims {"name": "...", "size_gb": ...}
+Returns a claim, status `pending`.
+## Attach at create
+Name the claim on `POST /api/v1/vms`:
+    POST /api/v1/vms {"volume_claims": ["<name or id>", ...]}
+The first VM naming a pending claim binds it to that VM's host. Every later
+VM using the same claim must be placed on that host — naming a different one
+is refused with 409.
+## In the guest
+Devices come in a fixed order: root is `/dev/vda`, the cloud-init seed is
+`/dev/vdb`, the first volume is `/dev/vdc`, then the rest in the order named.
+A volume is a raw block device. The guest partitions, formats and mounts it —
+eitri never touches the bytes:
+```sh
+sudo mkfs.ext4 /dev/vdc && sudo mount /dev/vdc /mnt
+```
+## Lifecycle
+Deleting the VM frees the claim once the VM is reaped; the data stays, and
+the next VM naming the claim sees it.
+    GET /api/v1/volume-claims
+Shows each claim's status, `host_id`, `vm_id` and `present`.
+    DELETE /api/v1/volume-claims/{id}
+Deletes the claim and its data. Refused with 409 while a VM holds it. Size is
+fixed once a claim is bound.
+## Hosts
+A host cannot be removed while it holds volumes — delete their claims first.
+Force-removing a host destroys its volumes and returns their claims to
+pending.
+Volumes need an agent at or above `v0.0.8-pre.1`. An older agent's host
+refuses a volume-bearing create with an upgrade link.
+## Related
+- [quickstart](quickstart.md)
+- [networking](networking.md)

internal/agent/cloudhv/cloudhv.go

Old		New
@@ -124,13 +124,19 @@ func BootstrapDest(chBin string) string {
124	return filepath.Join("/usr/local/bin", chBin)	124	return filepath.Join("/usr/local/bin", chBin)
125	}	125	}
126		126
127	// disks returns the VM's block devices in attachment order: the root disk	127	// disks returns the VM's block devices in attachment order: the root disk, the
128	// first, then the read-only cloud-init seed. User-attached volumes append here.	128	// read-only cloud-init seed, then every volume in spec order — so the first
		129	// volume the tenant attached is /dev/vdc, and stays /dev/vdc across reboots.
		130	// Volumes are writable: the guest owns what is on them.
129	func (p *Provisioner) disks(spec state.VMSpec) []state.Disk {	131	func (p *Provisioner) disks(spec state.VMSpec) []state.Disk {
130	return []state.Disk{	132	out := []state.Disk{
131	{Path: p.st.DiskPath(spec.VMID)},	133	{Path: p.st.DiskPath(spec.VMID)},
132	{Path: p.st.SeedPath(spec.VMID), ReadOnly: true},	134	{Path: p.st.SeedPath(spec.VMID), ReadOnly: true},
133	}	135	}
		136	for _, id := range spec.VolumeIDs {
		137	out = append(out, state.Disk{Path: p.st.VolumePath(id)})
		138	}
		139	return out
134	}	140	}
135		141
136	// buildArgs returns the cloud-hypervisor command-line arguments for spec.	142	// buildArgs returns the cloud-hypervisor command-line arguments for spec.

internal/agent/cloudhv/cloudhv_test.go

Old		New
@@ -75,10 +75,14 @@ func TestBuildArgsDeclaresRawImageType(t *testing.T) {
75	st, err := state.Open(t.TempDir())	75	st, err := state.Open(t.TempDir())
76	require.NoError(t, err)	76	require.NoError(t, err)
77	p := New(st, "ch", "fw", nil, newFakeNet())	77	p := New(st, "ch", "fw", nil, newFakeNet())
78	args := p.buildArgs(state.VMSpec{VMID: "vm1", VCPUs: 1, MemMB: 512})	78	args := p.buildArgs(state.VMSpec{VMID: "vm1", VCPUs: 1, MemMB: 512, VolumeIDs: []string{"vb"}})
79	joined := strings.Join(args, " ")	79	joined := strings.Join(args, " ")
80	assert.Contains(t, joined, st.DiskPath("vm1")+",image_type=raw")	80	assert.Contains(t, joined, st.DiskPath("vm1")+",image_type=raw")
81	assert.Contains(t, joined, st.SeedPath("vm1")+",image_type=raw,readonly=on")	81	assert.Contains(t, joined, st.SeedPath("vm1")+",image_type=raw,readonly=on")
		82	// A volume is a raw file like the others, and writable: the sector-0 trap
		83	// costs a volume its partition table just as surely as it costs the root disk.
		84	assert.Contains(t, joined, "path="+st.VolumePath("vb")+",image_type=raw")
		85	assert.NotContains(t, joined, st.VolumePath("vb")+",image_type=raw,readonly=on")
82	}	86	}
83		87
84	// TestDisksPutsRootFirstAndSeedReadOnly pins the attachment order: index 0	88	// TestDisksPutsRootFirstAndSeedReadOnly pins the attachment order: index 0
@@ -111,6 +115,25 @@ func TestDisksPutsRootFirstAndSeedReadOnly(t *testing.T) {
111	}	115	}
112	}	116	}
113		117
		118	// TestDisksAppendVolumesAfterSeedInOrder pins the rest of the guest ABI: a
		119	// volume never displaces the root disk or the seed, and the spec's order is the
		120	// device order, so the first volume a tenant attached is /dev/vdc on every boot.
		121	func TestDisksAppendVolumesAfterSeedInOrder(t *testing.T) {
		122	st, err := state.Open(t.TempDir())
		123	require.NoError(t, err)
		124	p := New(st, "cloud-hypervisor", "/fw/CLOUDHV.fd", nil, newFakeNet())
		125
		126	disks := p.disks(state.VMSpec{VMID: "vm-1", VolumeIDs: []string{"vb", "va"}})
		127
		128	require.Len(t, disks, 4)
		129	assert.Equal(t, st.DiskPath("vm-1"), disks[0].Path)
		130	assert.Equal(t, st.SeedPath("vm-1"), disks[1].Path)
		131	assert.Equal(t, st.VolumePath("vb"), disks[2].Path, "spec order, not sorted: the first volume is /dev/vdc")
		132	assert.Equal(t, st.VolumePath("va"), disks[3].Path)
		133	assert.False(t, disks[2].ReadOnly, "a volume is the guest's to write to")
		134	assert.False(t, disks[3].ReadOnly)
		135	}
		136
114	func TestBuildArgsUsesSerialSocket(t *testing.T) {	137	func TestBuildArgsUsesSerialSocket(t *testing.T) {
115	st, err := state.Open(t.TempDir())	138	st, err := state.Open(t.TempDir())
116	require.NoError(t, err)	139	require.NoError(t, err)

internal/agent/reconcile/reconcile.go

Old		New
@@ -22,6 +22,7 @@ import (
22	"errors"	22	"errors"
23	"fmt"	23	"fmt"
24	"log/slog"	24	"log/slog"
		25	"slices"
25	"strings"	26	"strings"
26	"sync"	27	"sync"
27	"time"	28	"time"
@@ -267,6 +268,13 @@ func (e Engine) Step(ctx context.Context, snap pb.Snapshot) *pb.Report {
267	return e.refuseSnapshot(snap, floor)	268	return e.refuseSnapshot(snap, floor)
268	}	269	}
269		270
		271	// ── 3. Volumes ───────────────────────────────────────────────────────────
		272	// Synchronously, and BEFORE dispatch: a VM's volumes must be files by the
		273	// time its backend is handed them, so a VM and its volumes arriving in the
		274	// same snapshot boot on the first try rather than the second. Cheap enough
		275	// to stay on the heartbeat's path — a sparse truncate, no copy.
		276	volumes := e.reconcileVolumes(snap)
		277
270	// One record scan serves the whole tick: dispatch slices it into assignments	278	// One record scan serves the whole tick: dispatch slices it into assignments
271	// and aggregate acks destroys against it. Reading it twice cost a second	279	// and aggregate acks destroys against it. Reading it twice cost a second
272	// full state-dir scan on the one blocking path in Step, microseconds after	280	// full state-dir scan on the one blocking path in Step, microseconds after
@@ -278,7 +286,7 @@ func (e Engine) Step(ctx context.Context, snap pb.Snapshot) *pb.Report {
278	// ctx would cancel them the instant the report went out. VMTimeout bounds	286	// ctx would cancel them the instant the report went out. VMTimeout bounds
279	// each pass instead (see reconcileOne).	287	// each pass instead (see reconcileOne).
280	e.dispatch(snap, recs) //nolint:contextcheck // a pass deliberately outlives its Step	288	e.dispatch(snap, recs) //nolint:contextcheck // a pass deliberately outlives its Step
281	return e.aggregate(snap.Epoch, recs)	289	return e.aggregate(snap.Epoch, recs, volumes)
282	}	290	}
283		291
284	// dispatch hands every VM its slice of this snapshot and reaps the workers for	292	// dispatch hands every VM its slice of this snapshot and reaps the workers for
@@ -312,8 +320,13 @@ func (e Engine) dispatch(snap pb.Snapshot, recs map[string]state.Record) {
312	//	320	//
313	// recs is the record view the destroy ack is computed against, with the caller	321	// recs is the record view the destroy ack is computed against, with the caller
314	// choosing how fresh it is (see ackDestroyed).	322	// choosing how fresh it is (see ackDestroyed).
315	func (e Engine) aggregate(epoch uint64, recs map[string]state.Record) pb.Report {	323	//
316	rep := &pb.Report{LastSeenEpoch: epoch}	324	// volumes is this tick's volume convergence, threaded in for the same reason
		325	// epoch is: the work was done in Step, and re-deriving it here would converge
		326	// the volumes a second time per report. Nil is a caller that converged none —
		327	// the paths that report on state without acting on it.
		328	func (e Engine) aggregate(epoch uint64, recs map[string]state.Record, volumes []pb.VolumeStatus) *pb.Report {
		329	rep := &pb.Report{LastSeenEpoch: epoch, Volumes: volumes}
317	m := e.manager()	330	m := e.manager()
318	m.collect(rep)	331	m.collect(rep)
319	e.ackDestroyed(rep, m.tombstones(), recs)	332	e.ackDestroyed(rep, m.tombstones(), recs)
@@ -333,6 +346,15 @@ func (e Engine) aggregate(epoch uint64, recs map[string]state.Record) pb.Repor
333	// Tombstoned VMs are skipped: they are already deleted, nothing will act on	346	// Tombstoned VMs are skipped: they are already deleted, nothing will act on
334	// them again, and hanging an upgrade-the-agent error on a VM on its way out	347	// them again, and hanging an upgrade-the-agent error on a VM on its way out
335	// would leave a spurious failure the operator cannot clear.	348	// would leave a spurious failure the operator cannot clear.
		349	//
		350	// VOLUMES: this report deliberately carries none, and that is safe only
		351	// because of who reaches it. The control plane reaps a tombstoned volume when
		352	// a VOLUMES-CAPABLE agent's report omits it, and an agent refusing a snapshot
		353	// is by definition below that snapshot's floor — which, today, any snapshot
		354	// carrying volumes sets to the release that introduced them. If a later
		355	// feature ever raises a floor above a version that already reports volumes,
		356	// this refusal must sweep and report them (see Engine.reportVolumes) or the
		357	// refusal will read upward as "every tombstoned volume on this host is gone".
336	func (e Engine) refuseSnapshot(snap pb.Snapshot, floor string) *pb.Report {	358	func (e Engine) refuseSnapshot(snap pb.Snapshot, floor string) *pb.Report {
337	rep := &pb.Report{LastSeenEpoch: snap.Epoch}	359	rep := &pb.Report{LastSeenEpoch: snap.Epoch}
338	reason := fmt.Sprintf("agent %s is below this snapshot's floor %s; upgrade the agent", e.AgentVersion, floor)	360	reason := fmt.Sprintf("agent %s is below this snapshot's floor %s; upgrade the agent", e.AgentVersion, floor)
@@ -346,10 +368,17 @@ func (e Engine) refuseSnapshot(snap pb.Snapshot, floor string) *pb.Report {
346	}	368	}
347		369
348	// fenceReport is the read-only report returned for a stale snapshot: current	370	// fenceReport is the read-only report returned for a stale snapshot: current
349	// actual state, derived entirely from persisted records, with no mutation and	371	// actual state, derived entirely from persisted records and a stat of the
350	// no dispatch.	372	// volumes directory, with no mutation and no dispatch.
		373	//
		374	// The volume sweep is a report, not a convergence: no file is made, no
		375	// tombstone marker written, nothing reclaimed. It is here because a report
		376	// that named no volumes would be read upward as this host having none — and
		377	// the control plane reaps a tombstoned volume on exactly that silence. The
		378	// server refuses to reap off a fenced report as well; the two halves are
		379	// independent, and each is worth having on its own.
351	func (e Engine) fenceReport(currentEpoch uint64) pb.Report {	380	func (e Engine) fenceReport(currentEpoch uint64) pb.Report {
352	rep := &pb.Report{FenceViolation: true, LastSeenEpoch: currentEpoch}	381	rep := &pb.Report{FenceViolation: true, LastSeenEpoch: currentEpoch, Volumes: e.reportVolumes(nil)}
353	recs, err := e.St.LoadVMs()	382	recs, err := e.St.LoadVMs()
354	if err != nil {	383	if err != nil {
355	return rep	384	return rep
@@ -653,8 +682,9 @@ func (e Engine) create(ctx context.Context, d pb.VMSpec, rec state.Record, ok
653		682
654	spec := specFromWire(d)	683	spec := specFromWire(d)
655		684
656	// VMSpec is fully comparable (all fields are strings/ints/bool), so == is safe.	685	// VMSpec stopped being ==-comparable when it grew a volume list; Equal is
657	if ok && spec != rec.Spec {	686	// what == was, order of the volumes included.
		687	if ok && !spec.Equal(rec.Spec) {
658	rec.CreateAttempts = 0	688	rec.CreateAttempts = 0
659	rec.LastError = ""	689	rec.LastError = ""
660	}	690	}
@@ -784,6 +814,11 @@ func (e Engine) create(ctx context.Context, d pb.VMSpec, rec state.Record, ok
784		814
785	// Boot if desired running.	815	// Boot if desired running.
786	if d.PowerState == "running" {	816	if d.PowerState == "running" {
		817	if err := e.volumesMaterialized(rec.Spec); err != nil {
		818	e.failCreate(ctx, rec, err, res)
		819	return
		820	}
		821
787	say("booting")	822	say("booting")
788	if err := e.Prov.Boot(ctx, d.VmId, rec.Spec); err != nil {	823	if err := e.Prov.Boot(ctx, d.VmId, rec.Spec); err != nil {
789	e.failCreate(ctx, rec, err, res)	824	e.failCreate(ctx, rec, err, res)
@@ -805,12 +840,14 @@ func (e Engine) create(ctx context.Context, d pb.VMSpec, rec state.Record, ok
805	res.report(d.VmId, recAddrs(rec), powerState, "ready", "")	840	res.report(d.VmId, recAddrs(rec), powerState, "ready", "")
806	}	841	}
807		842
808	// permanent reports whether err (anywhere in its chain) carries the	843	// isPermanent reports whether err (anywhere in its chain) carries the
809	// consumer-owned permanence marker — the provisioner's way of saying no	844	// consumer-owned permanence marker — the provisioner's way of saying no
810	// retry can ever succeed (e.g. the disk-shrink guard). Consumer-side	845	// retry can ever succeed (e.g. the disk-shrink guard). Consumer-side
811	// interface per the R5 convention: reconcile declares it; the producers	846	// interface per the R5 convention: reconcile declares it; the producers
812	// (cloudhv, vfkit, imagecache, netenv) mint one via internal/agent/permanent.	847	// (cloudhv, vfkit, imagecache, netenv) mint one via internal/agent/permanent,
813	func permanent(err error) bool {	848	// as does this package's own volumesMaterialized, which knows a missing volume
		849	// is not something this host can heal.
		850	func isPermanent(err error) bool {
814	var p interface{ Permanent() bool }	851	var p interface{ Permanent() bool }
815	return errors.As(err, &p) && p.Permanent()	852	return errors.As(err, &p) && p.Permanent()
816	}	853	}
@@ -822,7 +859,7 @@ func permanent(err error) bool {
822	func (e Engine) failCreate(ctx context.Context, rec state.Record, err error, res vmResult) {	859	func (e Engine) failCreate(ctx context.Context, rec state.Record, err error, res vmResult) {
823	if ctx.Err() != nil {	860	if ctx.Err() != nil {
824	rec.CreateAttempts-- // refund: this VM's pass died mid-operation	861	rec.CreateAttempts-- // refund: this VM's pass died mid-operation
825	} else if permanent(err) {	862	} else if isPermanent(err) {
826	rec.CreateAttempts = e.MaxCreateAttempts // terminal now; retry cannot succeed	863	rec.CreateAttempts = e.MaxCreateAttempts // terminal now; retry cannot succeed
827	}	864	}
828	rec.LastError = err.Error()	865	rec.LastError = err.Error()
@@ -871,7 +908,11 @@ func (e Engine) converge(ctx context.Context, d pb.VMSpec, rec state.Record, r
871	if d.PowerState == "running" {	908	if d.PowerState == "running" {
872	// Restart: the backend re-attaches the VM to the host network as	909	// Restart: the backend re-attaches the VM to the host network as
873	// part of Boot, which is what rebuilds a tap that did not survive	910	// part of Boot, which is what rebuilds a tap that did not survive
874	// the host reboot.	911	// the host reboot. Its volumes have to survive too.
		912	if err := e.volumesMaterialized(rec.Spec); err != nil {
		913	e.failConverge(rec, err, res)
		914	return
		915	}
875	if err := e.Prov.Boot(ctx, d.VmId, rec.Spec); err != nil {	916	if err := e.Prov.Boot(ctx, d.VmId, rec.Spec); err != nil {
876	e.failConverge(rec, err, res)	917	e.failConverge(rec, err, res)
877	return	918	return
@@ -895,6 +936,10 @@ func (e Engine) converge(ctx context.Context, d pb.VMSpec, rec state.Record, r
895	// Not lost: drive power state.	936	// Not lost: drive power state.
896	if d.PowerState == "running" && !running {	937	if d.PowerState == "running" && !running {
897	// Start the VM. Boot re-attaches the network, idempotently.	938	// Start the VM. Boot re-attaches the network, idempotently.
		939	if err := e.volumesMaterialized(rec.Spec); err != nil {
		940	e.failConverge(rec, err, res)
		941	return
		942	}
898	if err := e.Prov.Boot(ctx, d.VmId, rec.Spec); err != nil {	943	if err := e.Prov.Boot(ctx, d.VmId, rec.Spec); err != nil {
899	e.failConverge(rec, err, res)	944	e.failConverge(rec, err, res)
900	return	945	return
@@ -1053,8 +1098,11 @@ func specFromWire(d *pb.VMSpec) state.VMSpec {
1053	CloudInit: d.CloudInit,	1098	CloudInit: d.CloudInit,
1054	SSHAuthorizedKey: d.SshAuthorizedKey,	1099	SSHAuthorizedKey: d.SshAuthorizedKey,
1055	Network: d.Network,	1100	Network: d.Network,
1056	VCPUs: d.Vcpus,	1101	// Cloned: the record outlives the snapshot it came from, and a worker
1057	MemMB: d.MemMb,	1102	// holds its assignment well past the Step that delivered it.
1058	DiskGB: d.DiskGb,	1103	VolumeIDs: slices.Clone(d.VolumeIds),
		1104	VCPUs: d.Vcpus,
		1105	MemMB: d.MemMb,
		1106	DiskGB: d.DiskGb,
1059	}	1107	}
1060	}	1108	}

internal/agent/reconcile/reconcile_test.go

Old		New
@@ -29,17 +29,29 @@ import (
29	// addressing is only observable through the seam, so the fake has to own it for	29	// addressing is only observable through the seam, so the fake has to own it for
30	// these tests to mean anything.	30	// these tests to mean anything.
31	type fakeProv struct {	31	type fakeProv struct {
32	mu sync.Mutex	32	// st is the same state store the engine writes through, so the fake can
33	running map[string]bool	33	// see the host's disk exactly as a real backend does at Boot.
34	prepCalls int // total PrepareRootDisk invocations, including failed ones	34	st *state.Store
35	prepared []string	35
36	booted []string	36	mu sync.Mutex
37	shutdown []string	37	running map[string]bool
38	destroyed []string	38	prepCalls int // total PrepareRootDisk invocations, including failed ones
39	prepErr error	39	prepared []string
40	bootErr error // one-shot: consumed and cleared on first Boot call	40	booted []string
41	destroyErr error // sticky: every Destroy fails until it is cleared	41	// bootedSpec is the spec each VM was last booted with. Attachment is part
42	preflightErr error // sticky: the backend refuses this host outright	42	// of Boot on both real backends, so what a guest ends up holding is only
		43	// observable here.
		44	bootedSpec map[string]state.VMSpec
		45	// volumesAtBoot records, per VM, whether every volume its spec named was
		46	// already a file when Boot was called. The ordering that makes a volume
		47	// usable is not visible from the spec alone.
		48	volumesAtBoot map[string]bool
		49	shutdown []string
		50	destroyed []string
		51	prepErr error
		52	bootErr error // one-shot: consumed and cleared on first Boot call
		53	destroyErr error // sticky: every Destroy fails until it is cleared
		54	preflightErr error // sticky: the backend refuses this host outright
43		55
44	cidr string	56	cidr string
45	addrs map[string]string // vmID -> ip (sticky, mirrors the DHCP table)	57	addrs map[string]string // vmID -> ip (sticky, mirrors the DHCP table)
@@ -58,12 +70,15 @@ type fakeProv struct {
58	lateAddress bool	70	lateAddress bool
59	}	71	}
60		72
61	func newFakeProv() *fakeProv {	73	func newFakeProv(st state.Store) fakeProv {
62	return &fakeProv{	74	return &fakeProv{
63	running: map[string]bool{},	75	st: st,
64	addrs: map[string]string{},	76	running: map[string]bool{},
65	netAddrs: map[string]string{},	77	addrs: map[string]string{},
66	cidr: "10.77.1.0/24",	78	netAddrs: map[string]string{},
		79	bootedSpec: map[string]state.VMSpec{},
		80	volumesAtBoot: map[string]bool{},
		81	cidr: "10.77.1.0/24",
67	}	82	}
68	}	83	}
69		84
@@ -84,9 +99,11 @@ func (f *fakeProv) PrepareRootDisk(_ context.Context, s state.VMSpec, _ string)
84	return nil	99	return nil
85	}	100	}
86		101
87	func (f *fakeProv) Boot(_ context.Context, id string, _ state.VMSpec) error {	102	func (f *fakeProv) Boot(_ context.Context, id string, spec state.VMSpec) error {
88	f.mu.Lock()	103	f.mu.Lock()
89	defer f.mu.Unlock()	104	defer f.mu.Unlock()
		105	f.bootedSpec[id] = spec
		106	f.volumesAtBoot[id] = f.volumesOnDisk(spec)
90	if f.bootErr != nil {	107	if f.bootErr != nil {
91	err := f.bootErr	108	err := f.bootErr
92	f.bootErr = nil // one-shot: clear after first use	109	f.bootErr = nil // one-shot: clear after first use
@@ -102,6 +119,19 @@ func (f *fakeProv) Boot(_ context.Context, id string, _ state.VMSpec) error {
102	return nil	119	return nil
103	}	120	}
104		121
		122	// volumesOnDisk reports whether every volume this spec names is a file already.
		123	// A real backend hands each path to the hypervisor as a block device, so a
		124	// missing one is a guest booting without the disk its owner is about to write
		125	// to — which is what makes "materialise before dispatch" a testable claim.
		126	func (f *fakeProv) volumesOnDisk(spec state.VMSpec) bool {
		127	for _, id := range spec.VolumeIDs {
		128	if _, err := os.Stat(f.st.VolumePath(id)); err != nil {
		129	return false
		130	}
		131	}
		132	return true
		133	}
		134
105	// attachLocked gives id a sticky address, mirroring netenv: a VM that already	135	// attachLocked gives id a sticky address, mirroring netenv: a VM that already
106	// holds one keeps it, otherwise one is allocated over the set already handed	136	// holds one keeps it, otherwise one is allocated over the set already handed
107	// out. Serialized by the fake's own lock, exactly as the real backend's DHCP	137	// out. Serialized by the fake's own lock, exactly as the real backend's DHCP
@@ -205,7 +235,7 @@ func setup(t testing.T) fixture {
205	t.Helper()	235	t.Helper()
206	st, err := state.Open(t.TempDir())	236	st, err := state.Open(t.TempDir())
207	require.NoError(t, err)	237	require.NoError(t, err)
208	f := &fixture{st: st, prov: newFakeProv(), now: time.Unix(1_700_000_000, 0), boot: "boot-1"}	238	f := &fixture{st: st, prov: newFakeProv(st), now: time.Unix(1_700_000_000, 0), boot: "boot-1"}
209	f.eng = f.newEngine()	239	f.eng = f.newEngine()
210	t.Cleanup(f.eng.Stop)	240	t.Cleanup(f.eng.Stop)
211	return f	241	return f
@@ -253,7 +283,12 @@ func (f fixture) step(s pb.Snapshot) *pb.Report {
253	return rep	283	return rep
254	}	284	}
255	f.eng.manager().waitIdle()	285	f.eng.manager().waitIdle()
256	return f.aggregateNow(s.Epoch)	286	settled := f.aggregateNow(s.Epoch)
		287	// Volumes converge once per tick, inside Step and before dispatch. Carry
		288	// that tick's rows onto the settled report rather than reconciling them a
		289	// second time, which is not what a tick does.
		290	settled.Volumes = rep.Volumes
		291	return settled
257	}	292	}
258		293
259	// aggregateNow re-reads records and builds the report for epoch, the way Step	294	// aggregateNow re-reads records and builds the report for epoch, the way Step
@@ -261,7 +296,7 @@ func (f fixture) step(s pb.Snapshot) *pb.Report {
261	// the just-finished passes have already changed.	296	// the just-finished passes have already changed.
262	func (f fixture) aggregateNow(epoch uint64) pb.Report {	297	func (f fixture) aggregateNow(epoch uint64) pb.Report {
263	recs, _ := f.st.LoadVMs()	298	recs, _ := f.st.LoadVMs()
264	return f.eng.aggregate(epoch, recs)	299	return f.eng.aggregate(epoch, recs, nil)
265	}	300	}
266		301
267	func snap(epoch uint64, vms ...pb.VMSpec) pb.Snapshot {	302	func snap(epoch uint64, vms ...pb.VMSpec) pb.Snapshot {

internal/agent/reconcile/volumes.go

@@ @@ -0,0 +1,231 @@ @@
+package reconcile
+import (
+	"errors"
+	"fmt"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"github.com/a73x/eitri/internal/agent/permanent"
+	"github.com/a73x/eitri/internal/agent/state"
+	"github.com/a73x/eitri/internal/pb"
+)
+// reconcileVolumes drives the volumes directory toward the snapshot and
+// reports every volume id found there. It runs BEFORE VM dispatch so a VM's
+// disks exist when its backend lists them, and synchronously, because a sparse
+// truncate costs nothing — there is no download and no copy to keep off the
+// heartbeat's path.
+//
+// The rule that matters: a file the snapshot does not name is reported and
+// KEPT. A VM that vanishes from a snapshot is eventually reclaimed; a volume
+// never is, by automation. Only an explicit tombstone deletes data, because
+// the file is the only copy of whatever a guest wrote to it.
+//
+// Every volume the snapshot names gets a row, present or not: the control
+// plane reads an omitted volume as gone, so silence about one it asked for
+// would reap the row out from under a file that is still here.
+func (e *Engine) reconcileVolumes(snap *pb.Snapshot) []*pb.VolumeStatus {
+	var out []*pb.VolumeStatus
+	seen := make(map[string]bool, len(snap.GetVolumes()))
+	for _, spec := range snap.GetVolumes() {
+		id := spec.GetVolumeId()
+		if !validVolumeID(id) {
+			// Refused before the id is joined into a path. Reported absent, so
+			// the control plane hears an answer for every volume it named.
+			slog.Warn("reconcile: refusing a volume id that is not one", "volume", id)
+			out = append(out, &pb.VolumeStatus{VolumeId: id, Present: false})
+			continue
+		}
+		seen[id] = true
+		out = append(out, e.convergeVolume(id, spec))
+	}
+	return append(out, e.reportVolumes(seen)...) // orphans: reported, kept
+}
+// reportVolumes lists what is in the volumes directory, skipping the ids in
+// skip, and TOUCHES NOTHING: it stats, it does not create, mark or delete. The
+// fence path reports on state without acting on it, and calls this on its own.
+//
+// Directory names are not run through validVolumeID: they came from this
+// agent's own writes, a filesystem cannot hold a separator in one, and nothing
+// here does more than stat them. An orphan with a strange name is still an
+// orphan the operator should be told about.
+func (e *Engine) reportVolumes(skip map[string]bool) []*pb.VolumeStatus {
+	entries, err := os.ReadDir(e.St.VolumesDir())
+	if err != nil {
+		// Not fatal: a caller that converged volumes has already reported them.
+		// What is lost is the orphan report, so say so — an operator chasing a
+		// volume that is in neither the fleet nor the host has nothing else to
+		// go on.
+		slog.Warn("reconcile: cannot list the volumes directory", "err", err)
+		return nil
+	}
+	var out []*pb.VolumeStatus
+	for _, ent := range entries {
+		if !ent.IsDir() || skip[ent.Name()] {
+			continue
+		}
+		if st := e.statVolume(ent.Name()); st != nil {
+			out = append(out, st)
+		}
+	}
+	return out
+}
+// volumesMaterialized refuses a boot whose volumes are not all files on this
+// host. It guards EVERY call to Prov.Boot — the create, the restart after a
+// host reboot, and a plain power-on — because a guest handed a device that is
+// not there finds a filesystem missing and has no way to report that upward:
+// the VM comes up "ready" with a mount point full of nothing.
+//
+// Permanent: nothing on this host heals a volume the control plane never placed
+// here, so spending three ticks finding that out helps nobody. Re-placing it
+// edits the spec, which hands the VM a fresh budget.
+func (e *Engine) volumesMaterialized(spec state.VMSpec) error {
+	for _, id := range spec.VolumeIDs {
+		if _, err := os.Stat(e.St.VolumePath(id)); err != nil {
+			return permanent.Errorf("volume %s not materialized on this host", id)
+		}
+	}
+	return nil
+}
+// validVolumeID reports whether id is what the control plane mints for a
+// volume: random.Hex(16), so 32 lowercase hex characters. It is checked before
+// the id is joined into a path, because a volume path is the one this package
+// hands to os.RemoveAll — an id carrying a separator or a ".." would let a
+// malformed snapshot aim a reclaim at a directory that is not a volume, a VM's
+// own among them. Empty, wrong length and wrong alphabet all fail here.
+func validVolumeID(id string) bool {
+	if len(id) != 32 {
+		return false
+	}
+	for _, c := range id {
+		if (c < '0' || c > '9') && (c < 'a' || c > 'f') {
+			return false
+		}
+	}
+	return true
+}
+// statVolume reports a volume as found on disk, nil ONLY when there is no file.
+// Size is what the file claims, which is the size the guest's filesystem was
+// made against.
+//
+// An error that is not "does not exist" reports the volume PRESENT, with
+// whatever size could be read. Absence is what sends convergeVolume off to
+// create the file, and creating ends in a rename over whatever is at that path
+// — so a permission error, an I/O error or a bad symlink read as "missing"
+// would destroy the volume it failed to look at. Only ErrNotExist is evidence
+// of absence; everything else is evidence of nothing.
+func (e *Engine) statVolume(id string) *pb.VolumeStatus {
+	fi, err := os.Stat(e.St.VolumePath(id))
+	switch {
+	case errors.Is(err, os.ErrNotExist):
+		return nil
+	case err != nil:
+		slog.Warn("reconcile: cannot stat volume; assuming it is there", "volume", id, "err", err)
+		return &pb.VolumeStatus{VolumeId: id, Present: true}
+	}
+	return &pb.VolumeStatus{VolumeId: id, Present: true, SizeGb: fi.Size() >> 30}
+}
+// convergeVolume drives ONE volume toward its spec and reports what is on disk
+// afterwards.
+func (e *Engine) convergeVolume(id string, spec *pb.VolumeSpec) *pb.VolumeStatus {
+	if spec.GetTombstoned() {
+		return e.reclaimVolume(id)
+	}
+	if st := e.statVolume(id); st != nil {
+		// NEVER resized, in either direction: a guest filesystem sits on this
+		// file, so growing it is the control plane's job to do through the
+		// guest and shrinking it destroys data. Reported as found.
+		return st
+	}
+	if err := createSparse(e.St.VolumePath(id), spec.GetSizeGb()); err != nil {
+		// Reported absent rather than skipped. The next tick retries; until one
+		// succeeds the control plane can see that this host owes a file.
+		slog.Warn("reconcile: create volume", "volume", id, "err", err)
+		return &pb.VolumeStatus{VolumeId: id, Present: false}
+	}
+	return e.statVolume(id)
+}
+// reclaimVolume deletes a tombstoned volume's directory once TombstoneGrace has
+// passed since the marker was first written, and reports what is left.
+//
+// The grace is deliberately measured from a marker on disk rather than from the
+// tick that noticed: the tombstone is restated in every snapshot, so timing it
+// from the snapshot would restart the clock forever. The marker's mtime is a
+// wall-clock fact and Engine.Now is the loop's clock — the same clock in
+// production, and injectable here for the same reason the quarantine grace is.
+func (e *Engine) reclaimVolume(id string) *pb.VolumeStatus {
+	st := e.statVolume(id)
+	if st == nil {
+		// Nothing to reclaim — already deleted, or never materialised on this
+		// host. Say so at once so the control plane can reap the row, and leave
+		// no directory behind for a file that does not exist.
+		return &pb.VolumeStatus{VolumeId: id, Present: false}
+	}
+	marker := e.St.VolumeTombstonePath(id)
+	fi, err := os.Stat(marker)
+	if errors.Is(err, os.ErrNotExist) {
+		if err = os.WriteFile(marker, nil, 0o600); err == nil {
+			fi, err = os.Stat(marker)
+		}
+	}
+	if err != nil {
+		// A clock this host cannot read is not permission to delete. Keep the
+		// file and retry next tick.
+		slog.Warn("reconcile: volume tombstone marker", "volume", id, "err", err)
+		return st
+	}
+	if e.Now().Sub(fi.ModTime()) < e.TombstoneGrace {
+		return st // still in grace: the delete is undoable until it is not
+	}
+	// Grace expired: the whole directory goes, marker included. A failure keeps
+	// the file — the level-triggered loop retries, and the control plane must
+	// not reap the row while the bytes are still here.
+	if err := os.RemoveAll(e.St.VolumeDir(id)); err != nil {
+		slog.Warn("reconcile: reclaim volume", "volume", id, "err", err)
+		return st
+	}
+	return &pb.VolumeStatus{VolumeId: id, Present: false}
+}
+// createSparse makes a sizeGB sparse file at path via .partial + rename, the
+// shape PrepareRootDisk uses, so a crash mid-create leaves no torn file at the
+// final path — and the file at the final path is therefore always the full
+// size a guest was promised.
+func createSparse(path string, sizeGB int64) error {
+	// Bounds, not policy: the control plane validates what a tenant may ask
+	// for. This refuses what cannot be a volume at all, so a zero or a garbage
+	// size fails here instead of producing an empty block device a guest would
+	// mount.
+	if sizeGB < 1 || sizeGB > 1<<20 {
+		return fmt.Errorf("size_gb %d out of range", sizeGB)
+	}
+	if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
+		return err
+	}
+	tmp := path + ".partial"
+	_ = os.Remove(tmp) // a previous attempt's torn temporary, if any
+	f, err := os.OpenFile(tmp, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
+	if err != nil {
+		return err
+	}
+	if err := f.Truncate(sizeGB << 30); err != nil {
+		_ = f.Close()
+		_ = os.Remove(tmp)
+		return err
+	}
+	if err := f.Close(); err != nil {
+		_ = os.Remove(tmp)
+		return err
+	}
+	return os.Rename(tmp, path)
+}

internal/agent/reconcile/volumes_test.go

@@ @@ -0,0 +1,590 @@ @@
+package reconcile
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"os"
+	"path/filepath"
+	"syscall"
+	"testing"
+	"time"
+	"github.com/a73x/eitri/internal/pb"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+// volSnap is snap's twin for the other half of desired state: the FULL set of
+// volumes this host is meant to hold.
+func volSnap(epoch uint64, vols ...*pb.VolumeSpec) *pb.Snapshot {
+	return &pb.Snapshot{Epoch: epoch, Volumes: vols}
+}
+// volID turns a short name a test can read into an id the agent will accept:
+// the 32 hex characters random.Hex(16) mints. Derived from the name so it is
+// stable across a test's ticks, and so two names never collide.
+func volID(name string) string {
+	sum := sha256.Sum256([]byte(name))
+	return hex.EncodeToString(sum[:16])
+}
+func volSpec(name string, sizeGB int64) *pb.VolumeSpec {
+	return &pb.VolumeSpec{VolumeId: volID(name), SizeGb: sizeGB}
+}
+func deadVol(name string, sizeGB int64) *pb.VolumeSpec {
+	v := volSpec(name, sizeGB)
+	v.Tombstoned = true
+	return v
+}
+func withVolumes(names ...string) func(*pb.VMSpec) {
+	ids := make([]string, len(names))
+	for i, n := range names {
+		ids[i] = volID(n)
+	}
+	return func(v *pb.VMSpec) { v.VolumeIds = ids }
+}
+func findVolume(rep *pb.Report, name string) *pb.VolumeStatus {
+	for _, v := range rep.Volumes {
+		if v.VolumeId == volID(name) {
+			return v
+		}
+	}
+	return nil
+}
+// The fixture's volume paths, by the short name the test used.
+func (f *fixture) volDir(name string) string    { return f.st.VolumeDir(volID(name)) }
+func (f *fixture) volPath(name string) string   { return f.st.VolumePath(volID(name)) }
+func (f *fixture) volMarker(name string) string { return f.st.VolumeTombstonePath(volID(name)) }
+// ageMarker backdates a volume's tombstone marker so the reclaim grace is
+// measured against the fixture's clock rather than the wall clock the file
+// system stamped it with.
+func (f *fixture) ageMarker(t *testing.T, name string, age time.Duration) {
+	t.Helper()
+	at := f.now.Add(-age)
+	require.NoError(t, os.Chtimes(f.volMarker(name), at, at))
+}
+func TestVolumeIsCreatedSparseAndReported(t *testing.T) {
+	f := setup(t)
+	rep := f.step(volSnap(1, volSpec("v1", 2)))
+	require.Len(t, rep.Volumes, 1)
+	assert.Equal(t, volID("v1"), rep.Volumes[0].VolumeId)
+	assert.True(t, rep.Volumes[0].Present)
+	assert.EqualValues(t, 2, rep.Volumes[0].SizeGb)
+	fi, err := os.Stat(f.volPath("v1"))
+	require.NoError(t, err)
+	assert.EqualValues(t, 2<<30, fi.Size())
+	if sparseFS(t, f.st.VolumesDir()) {
+		assert.Less(t, allocatedBytes(t, f.volPath("v1")), int64(1<<20),
+			"a 2 GiB volume must not cost 2 GiB the moment it is asked for")
+	}
+	_, err = os.Stat(f.volPath("v1") + ".partial")
+	assert.True(t, os.IsNotExist(err), "the temporary is renamed into place, never left beside it")
+}
+// TestVolumeIsNeverResized: the file is the guest's block device and a
+// filesystem sits on it, so the agent reports what it found rather than
+// growing what the control plane now asks for.
+func TestVolumeIsNeverResized(t *testing.T) {
+	f := setup(t)
+	f.step(volSnap(1, volSpec("v1", 2)))
+	rep := f.step(volSnap(2, volSpec("v1", 5)))
+	fi, err := os.Stat(f.volPath("v1"))
+	require.NoError(t, err)
+	assert.EqualValues(t, 2<<30, fi.Size(), "a guest filesystem sits on it")
+	require.Len(t, rep.Volumes, 1)
+	assert.EqualValues(t, 2, rep.Volumes[0].SizeGb, "reported as found, not as asked")
+}
+// TestVolumeConvergenceIsIdempotent: a repeated snapshot must not re-create a
+// file that is already there, which is what would silently discard a guest's
+// data on every tick.
+func TestVolumeConvergenceIsIdempotent(t *testing.T) {
+	f := setup(t)
+	f.step(volSnap(1, volSpec("v1", 1)))
+	require.NoError(t, os.WriteFile(f.volPath("v1"), []byte("a guest wrote this"), 0o600))
+	rep := f.step(volSnap(2, volSpec("v1", 1)))
+	require.Len(t, rep.Volumes, 1)
+	assert.True(t, rep.Volumes[0].Present)
+	data, err := os.ReadFile(f.volPath("v1"))
+	require.NoError(t, err)
+	assert.Equal(t, "a guest wrote this", string(data), "an existing volume is left exactly as it is")
+}
+func TestTombstonedVolumeIsDeletedAfterGrace(t *testing.T) {
+	f := setup(t)
+	f.step(volSnap(1, volSpec("v1", 1)))
+	rep := f.step(volSnap(2, deadVol("v1", 1)))
+	require.Len(t, rep.Volumes, 1)
+	assert.True(t, rep.Volumes[0].Present, "still within grace")
+	_, err := os.Stat(f.volPath("v1"))
+	assert.NoError(t, err)
+	_, err = os.Stat(f.volMarker("v1"))
+	require.NoError(t, err, "the marker that starts the grace is written on the first tombstoned tick")
+	f.ageMarker(t, "v1", 6*time.Minute) // past the fixture's 5m TombstoneGrace
+	rep = f.step(volSnap(3, deadVol("v1", 1)))
+	require.Len(t, rep.Volumes, 1)
+	assert.False(t, rep.Volumes[0].Present, "reported gone so the control plane can reap the row")
+	_, err = os.Stat(f.volDir("v1"))
+	assert.True(t, os.IsNotExist(err), "the whole directory goes, marker included")
+}
+// TestTombstoneGraceIsMeasuredFromTheFirstMarker: the marker is written once
+// and never refreshed, so a volume tombstoned minutes ago is not given a fresh
+// grace by every tick that re-states the tombstone.
+func TestTombstoneGraceIsMeasuredFromTheFirstMarker(t *testing.T) {
+	f := setup(t)
+	f.step(volSnap(1, volSpec("v1", 1)))
+	f.step(volSnap(2, deadVol("v1", 1)))
+	f.ageMarker(t, "v1", 4*time.Minute)
+	rep := f.step(volSnap(3, deadVol("v1", 1)))
+	assert.True(t, rep.Volumes[0].Present, "4 minutes into a 5 minute grace")
+	f.now = f.now.Add(2 * time.Minute) // the clock moves, the marker does not
+	rep = f.step(volSnap(4, deadVol("v1", 1)))
+	assert.False(t, rep.Volumes[0].Present, "the grace ran from the first marker, not from this tick")
+}
+// TestTombstonedVolumeThisHostNeverHadIsReportedGone: nothing to reclaim, so
+// nothing is written — and the control plane hears "gone" at once instead of
+// waiting out a grace on a file that does not exist.
+func TestTombstonedVolumeThisHostNeverHadIsReportedGone(t *testing.T) {
+	f := setup(t)
+	rep := f.step(volSnap(1, deadVol("ghost", 1)))
+	require.Len(t, rep.Volumes, 1)
+	assert.Equal(t, volID("ghost"), rep.Volumes[0].VolumeId)
+	assert.False(t, rep.Volumes[0].Present)
+	_, err := os.Stat(f.volDir("ghost"))
+	assert.True(t, os.IsNotExist(err), "no directory is made for a volume there is nothing to reclaim of")
+}
+// The rule that matters: a file the snapshot does not name is reported and
+// kept. One server bug must not delete user data.
+func TestUnknownVolumeIsReportedNeverDeleted(t *testing.T) {
+	f := setup(t)
+	f.step(volSnap(1, volSpec("v1", 1)))
+	rep := f.step(volSnap(2))
+	require.Len(t, rep.Volumes, 1)
+	assert.Equal(t, volID("v1"), rep.Volumes[0].VolumeId)
+	assert.True(t, rep.Volumes[0].Present)
+	assert.EqualValues(t, 1, rep.Volumes[0].SizeGb)
+	_, err := os.Stat(f.volPath("v1"))
+	assert.NoError(t, err, "an orphan is evidence of a bug somewhere, never a licence to delete")
+}
+// TestEverySnapshotVolumeGetsAStatusRow pins the contract the server's reap
+// rests on: it reads an omitted volume as gone, so a volume the agent could
+// not make must say so in words rather than by silence.
+func TestEverySnapshotVolumeGetsAStatusRow(t *testing.T) {
+	f := setup(t)
+	rep := f.step(volSnap(1, volSpec("ok", 1), volSpec("nonsense", 0)))
+	require.Len(t, rep.Volumes, 2)
+	assert.True(t, findVolume(rep, "ok").GetPresent())
+	bad := findVolume(rep, "nonsense")
+	require.NotNil(t, bad, "a volume that could not be created still gets a row")
+	assert.False(t, bad.GetPresent())
+}
+// TestVolumeConvergenceSurvivesAnUnusableVolumesDirectory: the state directory
+// can be wrong in ways nothing here can fix — a file where the volumes
+// directory belongs. Every volume the snapshot named still gets a row, and the
+// row errs toward present: this host cannot see the file, which is not the
+// same as knowing it is gone.
+func TestVolumeConvergenceSurvivesAnUnusableVolumesDirectory(t *testing.T) {
+	f := setup(t)
+	require.NoError(t, os.RemoveAll(f.st.VolumesDir()))
+	require.NoError(t, os.WriteFile(f.st.VolumesDir(), []byte("not a directory"), 0o600))
+	rep := f.step(volSnap(1, volSpec("v1", 1)))
+	require.Len(t, rep.Volumes, 1)
+	assert.Equal(t, volID("v1"), rep.Volumes[0].VolumeId)
+	assert.True(t, rep.Volumes[0].Present, "a volume this host cannot look at is not a volume it may report gone")
+	data, err := os.ReadFile(f.st.VolumesDir())
+	require.NoError(t, err)
+	assert.Equal(t, "not a directory", string(data), "and nothing was written over it")
+}
+// TestUnreadableVolumeIsNotRecreatedOverTheTopOfItself is the rule statVolume
+// exists for: create ends in a rename over whatever is at disk.raw, so a stat
+// that fails for any reason other than "no such file" must NOT be read as an
+// absent volume. The guest's data is on the other side of that rename.
+func TestUnreadableVolumeIsNotRecreatedOverTheTopOfItself(t *testing.T) {
+	requireNonRoot(t)
+	f := setup(t)
+	f.step(volSnap(1, volSpec("v1", 1)))
+	require.NoError(t, os.WriteFile(f.volPath("v1"), []byte("a guest wrote this"), 0o600))
+	chmodForTest(t, f.volDir("v1"), 0o000) // the file cannot even be looked at
+	rep := f.step(volSnap(2, volSpec("v1", 1)))
+	require.Len(t, rep.Volumes, 1)
+	assert.True(t, rep.Volumes[0].Present, "unreadable is not absent")
+	require.NoError(t, os.Chmod(f.volDir("v1"), 0o700))
+	_, err := os.Stat(f.volPath("v1") + ".partial")
+	assert.True(t, os.IsNotExist(err), "no create was even attempted")
+	data, err := os.ReadFile(f.volPath("v1"))
+	require.NoError(t, err)
+	assert.Equal(t, "a guest wrote this", string(data), "and the guest's bytes are untouched")
+}
+// TestTornTemporaryThatCannotBeReplacedIsReported: .partial is where a killed
+// create leaves its mess. One that cannot be cleared makes the volume absent,
+// which is what the report then says.
+func TestTornTemporaryThatCannotBeReplacedIsReported(t *testing.T) {
+	f := setup(t)
+	tmp := f.volPath("v1") + ".partial"
+	require.NoError(t, os.MkdirAll(filepath.Join(tmp, "occupied"), 0o700))
+	rep := f.step(volSnap(1, volSpec("v1", 1)))
+	require.Len(t, rep.Volumes, 1)
+	assert.False(t, rep.Volumes[0].Present)
+	_, err := os.Stat(f.volPath("v1"))
+	assert.True(t, os.IsNotExist(err), "nothing is renamed into place from a temporary that was never written")
+}
+// TestTombstonedVolumeIsKeptWhenTheMarkerCannotBeWritten: the grace is a file,
+// and a grace this host cannot start is not permission to delete.
+func TestTombstonedVolumeIsKeptWhenTheMarkerCannotBeWritten(t *testing.T) {
+	requireNonRoot(t)
+	f := setup(t)
+	f.step(volSnap(1, volSpec("v1", 1)))
+	chmodForTest(t, f.volDir("v1"), 0o500)
+	rep := f.step(volSnap(2, deadVol("v1", 1)))
+	require.Len(t, rep.Volumes, 1)
+	assert.True(t, rep.Volumes[0].Present)
+	_, err := os.Stat(f.volPath("v1"))
+	assert.NoError(t, err)
+}
+// TestTombstonedVolumeIsKeptWhenItCannotBeRemoved: the row must outlive the
+// bytes, never the other way round — reporting a volume gone while its file is
+// still here would have the control plane forget a disk this host still holds.
+func TestTombstonedVolumeIsKeptWhenItCannotBeRemoved(t *testing.T) {
+	requireNonRoot(t)
+	f := setup(t)
+	f.step(volSnap(1, volSpec("v1", 1)))
+	f.step(volSnap(2, deadVol("v1", 1)))
+	f.ageMarker(t, "v1", 6*time.Minute)
+	chmodForTest(t, f.volDir("v1"), 0o500)
+	rep := f.step(volSnap(3, deadVol("v1", 1)))
+	require.Len(t, rep.Volumes, 1)
+	assert.True(t, rep.Volumes[0].Present, "still present, so still reported present")
+	_, err := os.Stat(f.volPath("v1"))
+	assert.NoError(t, err)
+}
+// TestPartialVolumeIsRecreated: a create killed mid-truncate leaves a
+// .partial, never a short file at the real path, so the next tick simply
+// makes the volume.
+func TestPartialVolumeIsRecreated(t *testing.T) {
+	f := setup(t)
+	require.NoError(t, os.MkdirAll(f.volDir("v1"), 0o700))
+	require.NoError(t, os.WriteFile(f.volPath("v1")+".partial", []byte("torn"), 0o600))
+	rep := f.step(volSnap(1, volSpec("v1", 1)))
+	fi, err := os.Stat(f.volPath("v1"))
+	require.NoError(t, err)
+	assert.EqualValues(t, 1<<30, fi.Size())
+	assert.True(t, rep.Volumes[0].Present)
+	_, err = os.Stat(f.volPath("v1") + ".partial")
+	assert.True(t, os.IsNotExist(err), "the torn temporary is replaced, not stepped around")
+}
+// Volumes are materialised before the VM that needs them is dispatched, in
+// the same Step, so a VM and its volume arriving together boot first try.
+//
+// The first half is the deterministic one: Step converges volumes on its own
+// goroutine, so the file and its row exist the moment Step returns, with no
+// worker having had to run. The second half is what the ordering buys — the
+// backend found the device already there.
+func TestVolumesAreMaterialisedBeforeVMsDispatch(t *testing.T) {
+	f := setup(t)
+	s := volSnap(1, volSpec("v1", 1))
+	s.Vms = []*pb.VMSpec{vm("vm1", withVolumes("v1"))}
+	rep := f.eng.Step(t.Context(), s)
+	_, err := os.Stat(f.volPath("v1"))
+	require.NoError(t, err, "the volume is a file by the time Step returns, not when a worker gets to it")
+	require.Len(t, rep.Volumes, 1)
+	assert.True(t, rep.Volumes[0].Present)
+	f.eng.manager().waitIdle()
+	require.Equal(t, "ready", findVM(f.aggregateNow(1), "vm1").GetPhase())
+	assert.Equal(t, []string{volID("v1")}, f.prov.bootedSpec["vm1"].VolumeIDs, "the backend is told which files to attach")
+	assert.True(t, f.prov.volumesAtBoot["vm1"], "and the file was on disk before Boot was called")
+}
+// A VM whose volume is absent fails legibly rather than booting bare.
+func TestVMWithMissingVolumeFailsLegibly(t *testing.T) {
+	f := setup(t)
+	rep := f.step(snap(1, vm("vm1", withVolumes("ghost"))))
+	row := findVM(rep, "vm1")
+	require.NotNil(t, row)
+	assert.Equal(t, "failed", row.Phase, "permanent: nothing on this host heals a volume that was never placed")
+	assert.Contains(t, row.LastError, "volume "+volID("ghost")+" not materialized on this host")
+	assert.Empty(t, f.prov.booted, "never booted")
+	f.step(snap(2, vm("vm1", withVolumes("ghost"))))
+	assert.Empty(t, f.prov.booted, "and no later tick boots it either")
+}
+// A guest that already exists is held to the same rule as a new one: powering
+// it back on without the volume it was created with would hand it a filesystem
+// that is simply not there, and a guest cannot report that upward.
+//
+// The volume is removed while the snapshot no longer names it, which is exactly
+// what a host holds after the control plane moved the volume elsewhere: nothing
+// recreates the file, and the VM still asks for it.
+func TestPoweringOnWithoutItsVolumeFailsRatherThanBootsBare(t *testing.T) {
+	f := setup(t)
+	s := volSnap(1, volSpec("v1", 1))
+	s.Vms = []*pb.VMSpec{vm("vm1", withVolumes("v1"))}
+	f.step(s)
+	require.Equal(t, []string{"vm1"}, f.prov.booted)
+	s2 := volSnap(2, volSpec("v1", 1))
+	s2.Vms = []*pb.VMSpec{vm("vm1", withVolumes("v1"), stopped)}
+	f.step(s2)
+	require.False(t, f.prov.Running("vm1"), "stopped first, so the next tick is a power-on and not a create")
+	require.NoError(t, os.RemoveAll(f.volDir("v1")))
+	rep := f.step(snap(3, vm("vm1", withVolumes("v1"))))
+	row := findVM(rep, "vm1")
+	require.NotNil(t, row)
+	assert.Equal(t, "failed", row.Phase)
+	assert.Contains(t, row.LastError, "volume "+volID("v1")+" not materialized on this host")
+	assert.Equal(t, []string{"vm1"}, f.prov.booted, "booted once, at create — never a second time without its device")
+}
+// The same rule on the other converge path: a host reboot loses every guest,
+// and the restart that brings them back is a Boot like any other.
+func TestAHostRebootDoesNotBringAGuestBackWithoutItsVolume(t *testing.T) {
+	f := setup(t)
+	s := volSnap(1, volSpec("v1", 1))
+	s.Vms = []*pb.VMSpec{vm("vm1", withVolumes("v1"))}
+	f.step(s)
+	require.Equal(t, []string{"vm1"}, f.prov.booted)
+	require.NoError(t, os.RemoveAll(f.volDir("v1")))
+	f.prov.booted = nil
+	f.boot = "boot-2" // the host rebooted: the guest is lost and would be booted again
+	f.prov.running["vm1"] = false
+	rep := f.step(snap(2, vm("vm1", withVolumes("v1"))))
+	row := findVM(rep, "vm1")
+	require.NotNil(t, row)
+	assert.Equal(t, "failed", row.Phase)
+	assert.Contains(t, row.LastError, "volume "+volID("v1")+" not materialized on this host")
+	assert.Empty(t, f.prov.booted, "a lost guest comes back with its devices or not at all")
+}
+// TestVolumeIDsSurviveAnAgentRestart: the ids are desired state on disk, so a
+// restarted agent re-attaches the same devices in the same order without
+// waiting for a snapshot to tell it again.
+func TestVolumeIDsSurviveAnAgentRestart(t *testing.T) {
+	f := setup(t)
+	s := volSnap(1, volSpec("va", 1), volSpec("vb", 1))
+	s.Vms = []*pb.VMSpec{vm("vm1", withVolumes("vb", "va"))}
+	f.step(s)
+	recs, err := f.st.LoadVMs()
+	require.NoError(t, err)
+	assert.Equal(t, []string{volID("vb"), volID("va")}, recs["vm1"].Spec.VolumeIDs, "attachment order is persisted")
+	f.restart(t)
+	f.prov.booted = nil
+	f.boot = "boot-2" // the host rebooted with it, so the guest is lost and comes back
+	f.prov.running["vm1"] = false
+	s2 := volSnap(2, volSpec("va", 1), volSpec("vb", 1))
+	s2.Vms = []*pb.VMSpec{vm("vm1", withVolumes("vb", "va"))}
+	f.step(s2)
+	require.Equal(t, []string{"vm1"}, f.prov.booted)
+	assert.Equal(t, []string{volID("vb"), volID("va")}, f.prov.bootedSpec["vm1"].VolumeIDs)
+}
+// TestEditingTheVolumeListResetsCreateAttempts is Equal doing the job == did:
+// binding a volume is an edit, so a VM that spent its retry budget under the
+// old spec gets a fresh one.
+func TestEditingTheVolumeListResetsCreateAttempts(t *testing.T) {
+	f := setup(t)
+	f.prov.prepErr = assert.AnError
+	for range 3 {
+		f.step(snap(1, vm("vm1")))
+	}
+	f.prov.prepErr = nil
+	s := volSnap(2, volSpec("v1", 1))
+	s.Vms = []*pb.VMSpec{vm("vm1", withVolumes("v1"))}
+	rep := f.step(s)
+	assert.Equal(t, []string{"vm1"}, f.prov.prepared, "a newly bound volume is a spec edit")
+	assert.Equal(t, "ready", findVM(rep, "vm1").GetPhase())
+}
+// TestFencedSnapshotReportsVolumesWithoutTouchingThem: the fence path acts on
+// nothing, volumes included — but it must still SAY what is on this host. The
+// control plane reaps a tombstoned volume on a report that omits it, so a
+// fenced report naming none would read as a host that has none.
+func TestFencedSnapshotReportsVolumesWithoutTouchingThem(t *testing.T) {
+	f := setup(t)
+	f.step(volSnap(5, volSpec("v1", 2), volSpec("v2", 1)))
+	f.step(volSnap(6, volSpec("v1", 2), deadVol("v2", 1))) // v2 is on its way out
+	require.NoError(t, os.WriteFile(f.volPath("v1"), []byte("a guest wrote this"), 0o600))
+	rep := f.step(volSnap(3, volSpec("v3", 1)))
+	require.True(t, rep.FenceViolation)
+	require.Len(t, rep.Volumes, 2, "both volumes on disk are reported, the tombstoned one included")
+	assert.True(t, findVolume(rep, "v1").GetPresent())
+	assert.True(t, findVolume(rep, "v2").GetPresent())
+	assert.Nil(t, findVolume(rep, "v3"), "a stale snapshot's volume is not conjured into the report")
+	_, err := os.Stat(f.volPath("v3"))
+	assert.True(t, os.IsNotExist(err), "a stale snapshot must not materialise a file")
+	data, err := os.ReadFile(f.volPath("v1"))
+	require.NoError(t, err)
+	assert.Equal(t, "a guest wrote this", string(data), "nor touch one that is here")
+	f.now = f.now.Add(time.Hour) // long past the grace v2's marker started
+	rep = f.step(volSnap(3, volSpec("v3", 1)))
+	require.True(t, rep.FenceViolation)
+	assert.True(t, findVolume(rep, "v2").GetPresent(), "and the fence path reclaims nothing, whatever the clock says")
+}
+// TestAVolumeIDIsNotAPath: volume paths are the ones this package hands to
+// os.RemoveAll, so an id is checked before it is joined into one. Without that
+// check a single malformed snapshot aims a reclaim at a live VM's directory.
+func TestAVolumeIDIsNotAPath(t *testing.T) {
+	f := setup(t)
+	f.step(snap(1, vm("vm1")))
+	require.NoError(t, os.WriteFile(f.st.DiskPath("vm1"), []byte("a running guest's root disk"), 0o600))
+	traversal := "../vms/vm1"
+	rep := f.step(volSnap(2, &pb.VolumeSpec{VolumeId: traversal, SizeGb: 1, Tombstoned: true}))
+	require.Len(t, rep.Volumes, 1)
+	assert.Equal(t, traversal, rep.Volumes[0].VolumeId, "answered for, so the control plane is not left waiting")
+	assert.False(t, rep.Volumes[0].Present)
+	_, err := os.Stat(filepath.Join(f.st.VMDir("vm1"), "tombstoned"))
+	assert.True(t, os.IsNotExist(err), "no reclaim clock is started over a VM's own directory")
+	data, err := os.ReadFile(f.st.DiskPath("vm1"))
+	require.NoError(t, err)
+	assert.Equal(t, "a running guest's root disk", string(data), "and the guest's root disk is still there")
+}
+// TestVolumeIDsThatAreNotIDsAreRefused covers the rest of the alphabet: what
+// the control plane mints is random.Hex(16), and nothing else is joined into a
+// path at all.
+func TestVolumeIDsThatAreNotIDsAreRefused(t *testing.T) {
+	f := setup(t)
+	bad := []string{
+		"",
+		"v1",
+		"../../etc/passwd",
+		"deadbeefdeadbeefdeadbeefdeadbee",   // 31: one short
+		"deadbeefdeadbeefdeadbeefdeadbeeff", // 33: one long
+		"DEADBEEFDEADBEEFDEADBEEFDEADBEEF",  // hex, but not the case Hex writes
+		"deadbeef-deadbeef-deadbeef-dead",
+	}
+	var specs []*pb.VolumeSpec
+	for _, id := range bad {
+		specs = append(specs, &pb.VolumeSpec{VolumeId: id, SizeGb: 1})
+	}
+	rep := f.step(volSnap(1, specs...))
+	require.Len(t, rep.Volumes, len(bad), "every id gets an answer, however wrong it was")
+	for i, v := range rep.Volumes {
+		assert.Equal(t, bad[i], v.VolumeId)
+		assert.False(t, v.Present, "%q must not be treated as a volume", bad[i])
+	}
+	entries, err := os.ReadDir(f.st.VolumesDir())
+	require.NoError(t, err)
+	assert.Empty(t, entries, "nothing was created for any of them")
+}
+// TestSnapshotBelowTheFloorTouchesNoVolume: an agent that cannot fully read a
+// snapshot acts on no part of it. The server reaps a tombstoned volume on an
+// omitted row only for an agent that reports volumes at all, and this one has
+// just said it is too old to.
+func TestSnapshotBelowTheFloorTouchesNoVolume(t *testing.T) {
+	f := setup(t)
+	f.eng.AgentVersion = "v0.0.6"
+	s := volSnap(1, volSpec("v1", 1))
+	s.MinAgentVersion = "v0.0.7"
+	s.Vms = []*pb.VMSpec{vm("vm1", withVolumes("v1"))}
+	rep := f.eng.Step(t.Context(), s)
+	assert.Empty(t, rep.Volumes)
+	assert.Equal(t, "failed", findVM(rep, "vm1").GetPhase())
+	_, err := os.Stat(f.volPath("v1"))
+	assert.True(t, os.IsNotExist(err))
+}
+// requireNonRoot skips a test that makes a directory unwritable to prove what
+// the agent does when it cannot write. Root is not refused by permission bits,
+// so under root the test would prove the opposite of what it says.
+func requireNonRoot(t *testing.T) {
+	t.Helper()
+	if os.Geteuid() == 0 {
+		t.Skip("running as root: permission bits refuse nothing")
+	}
+}
+// chmodForTest makes a directory unwritable and puts it back afterwards, so
+// t.TempDir's own cleanup can still remove it.
+func chmodForTest(t *testing.T, dir string, mode os.FileMode) {
+	t.Helper()
+	require.NoError(t, os.Chmod(dir, mode))
+	t.Cleanup(func() { _ = os.Chmod(dir, 0o700) })
+}
+// allocatedBytes reports how much disk a file actually occupies, as opposed to
+// the size it claims. st_blocks is in 512-byte units by POSIX definition,
+// whatever the filesystem's own block size is.
+func allocatedBytes(t *testing.T, path string) int64 {
+	t.Helper()
+	fi, err := os.Stat(path)
+	require.NoError(t, err)
+	st, ok := fi.Sys().(*syscall.Stat_t)
+	require.True(t, ok, "no syscall.Stat_t for %s", path)
+	return st.Blocks * 512
+}
+// sparseFS reports whether dir's filesystem holds holes at all. Without the
+// probe, asserting sparseness asserts something about the machine the test ran
+// on rather than about the code.
+func sparseFS(t *testing.T, dir string) bool {
+	t.Helper()
+	p := filepath.Join(dir, "sparse-probe")
+	f, err := os.Create(p)
+	require.NoError(t, err)
+	require.NoError(t, f.Truncate(8<<20))
+	require.NoError(t, f.Close())
+	alloc := allocatedBytes(t, p)
+	require.NoError(t, os.Remove(p))
+	return alloc <= 1<<20
+}

internal/agent/state/state.go

Old		New
@@ -9,6 +9,7 @@ import (
9	"fmt"	9	"fmt"
10	"os"	10	"os"
11	"path/filepath"	11	"path/filepath"
		12	"reflect"
12	"strconv"	13	"strconv"
13	"strings"	14	"strings"
14	"time"	15	"time"
@@ -24,10 +25,35 @@ type VMSpec struct {
24	// record: it is desired state the control plane sent, and it must survive	25	// record: it is desired state the control plane sent, and it must survive
25	// an agent restart so the replay can rebuild the VM's in-memory discovery	26	// an agent restart so the replay can rebuild the VM's in-memory discovery
26	// state.	27	// state.
27	Network string	28	Network string
		29	// VolumeIDs are the bound volumes to attach after root and seed, in this
		30	// order — the first is /dev/vdc. Desired state that must survive an agent
		31	// restart, like Network: a restarted agent re-attaches the same devices in
		32	// the same order without waiting to be told again.
		33	VolumeIDs []string
28	VCPUs, MemMB, DiskGB int64	34	VCPUs, MemMB, DiskGB int64
29	}	35	}
30		36
		37	// Equal is what == was before VolumeIDs made the struct uncomparable. Order is
		38	// part of the answer: the first volume is /dev/vdc, so a reordered list is a
		39	// different guest and reconcile treats it as an edit.
		40	//
		41	// DeepEqual rather than a hand-written field list, because the caller that
		42	// matters is the one deciding whether a user edited a VM (reconcile.create,
		43	// which hands an edited spec a fresh retry budget). A field added to this
		44	// struct and forgotten in a field list would make edits to it invisible, and
		45	// nothing would fail to say so. An empty list and no list are the same guest,
		46	// so both normalise to nil first — s and o are copies, so this is local.
		47	func (s VMSpec) Equal(o VMSpec) bool {
		48	if len(s.VolumeIDs) == 0 {
		49	s.VolumeIDs = nil
		50	}
		51	if len(o.VolumeIDs) == 0 {
		52	o.VolumeIDs = nil
		53	}
		54	return reflect.DeepEqual(s, o)
		55	}
		56
31	// Disk is one block device attached to a VM, in attachment order. Index 0 is	57	// Disk is one block device attached to a VM, in attachment order. Index 0 is
32	// the root disk (/dev/vda) — both cloud-hypervisor and vfkit order by argument	58	// the root disk (/dev/vda) — both cloud-hypervisor and vfkit order by argument
33	// position and treat the first as root.	59	// position and treat the first as root.
@@ -73,7 +99,7 @@ type Store struct{ dir string }
73	// Open initialises the state directory, creating required subdirectories if	99	// Open initialises the state directory, creating required subdirectories if
74	// they do not already exist. Returns a ready-to-use *Store.	100	// they do not already exist. Returns a ready-to-use *Store.
75	func Open(dir string) (*Store, error) {	101	func Open(dir string) (*Store, error) {
76	for _, sub := range []string{dir, filepath.Join(dir, "vms"), filepath.Join(dir, "images")} {	102	for _, sub := range []string{dir, filepath.Join(dir, "vms"), filepath.Join(dir, "images"), filepath.Join(dir, "volumes")} {
77	if err := os.MkdirAll(sub, 0o700); err != nil {	103	if err := os.MkdirAll(sub, 0o700); err != nil {
78	return nil, err	104	return nil, err
79	}	105	}
@@ -84,6 +110,26 @@ func Open(dir string) (*Store, error) {
84	// ImagesDir returns the path where downloaded images are cached.	110	// ImagesDir returns the path where downloaded images are cached.
85	func (s *Store) ImagesDir() string { return filepath.Join(s.dir, "images") }	111	func (s *Store) ImagesDir() string { return filepath.Join(s.dir, "images") }
86		112
		113	// VolumesDir holds one directory per volume, BESIDE vms/ and never inside a
		114	// VM's: a VM's directory is removed on destroy, a volume's is not. That
		115	// separation is the durability.
		116	func (s *Store) VolumesDir() string { return filepath.Join(s.dir, "volumes") }
		117
		118	// VolumeDir returns the per-volume directory for the given volume id.
		119	func (s *Store) VolumeDir(id string) string { return filepath.Join(s.VolumesDir(), id) }
		120
		121	// VolumePath returns the path of the volume's backing file — the block device
		122	// a guest is handed.
		123	func (s *Store) VolumePath(id string) string { return filepath.Join(s.VolumeDir(id), "disk.raw") }
		124
		125	// VolumeTombstonePath is the marker whose mtime starts the reclaim grace. It
		126	// is written once, on the first tick that sees the tombstone, and never
		127	// refreshed: the grace runs from the deletion, not from the latest snapshot
		128	// that restates it.
		129	func (s *Store) VolumeTombstonePath(id string) string {
		130	return filepath.Join(s.VolumeDir(id), "tombstoned")
		131	}
		132
87	// VMDir returns the per-VM directory for the given vmID.	133	// VMDir returns the per-VM directory for the given vmID.
88	func (s *Store) VMDir(vmID string) string { return filepath.Join(s.dir, "vms", vmID) }	134	func (s *Store) VMDir(vmID string) string { return filepath.Join(s.dir, "vms", vmID) }
89		135

internal/agent/state/state_test.go

@@ @@ -3,6 +3,7 @@ package state @@
+import (
+	"os"
+	"path/filepath"
+	"reflect"
+	"strconv"
+	"strings"
+	"testing"
+	require.True(t, ok)
+	assert.Equal(t, good, id.ServerCertSHA256)
+// TestVMSpecEqual pins the replacement for ==: VolumeIDs made the struct
+// uncomparable, and reconcile decides whether a user edited a VM from this
+// answer. Order is part of the spec — the first volume is /dev/vdc — so a
+// reorder is an edit.
+func TestVMSpecEqual(t *testing.T) {
+	base := VMSpec{VMID: "vm1", Name: "a", ImageURL: "u", ImageSHA256: "s",
+		Network: "lan", VCPUs: 2, MemMB: 2048, DiskGB: 10, VolumeIDs: []string{"va", "vb"}}
+	same := base
+	same.VolumeIDs = []string{"va", "vb"}
+	assert.True(t, base.Equal(same), "an identical spec compares equal through a fresh slice")
+	reordered := base
+	reordered.VolumeIDs = []string{"vb", "va"}
+	assert.False(t, base.Equal(reordered), "attachment order is part of the spec")
+	fewer := base
+	fewer.VolumeIDs = []string{"va"}
+	assert.False(t, base.Equal(fewer))
+	none := base
+	none.VolumeIDs = nil
+	assert.False(t, base.Equal(none))
+	assert.True(t, VMSpec{VMID: "vm1"}.Equal(VMSpec{VMID: "vm1"}), "no volumes either side is still equal")
+	// Every field, found by reflection rather than listed by hand: a field
+	// added to VMSpec and missed by Equal would make edits to it invisible to
+	// the retry budget, and a hand-written list here would miss it in exactly
+	// the same way.
+	rt := reflect.TypeOf(base)
+	for i := range rt.NumField() {
+		other := base
+		other.VolumeIDs = []string{"va", "vb"}
+		fv := reflect.ValueOf(&other).Elem().Field(i)
+		switch fv.Kind() {
+		case reflect.String:
+			fv.SetString("edited")
+		case reflect.Int64:
+			fv.SetInt(99)
+		case reflect.Slice:
+			fv.Set(reflect.ValueOf([]string{"edited"}))
+		default:
+			t.Fatalf("field %s has kind %s: teach this test how to edit it, then check Equal notices",
+				rt.Field(i).Name, fv.Kind())
+		}
+		assert.False(t, base.Equal(other), "an edit to %s must not compare equal", rt.Field(i).Name)
+	}
+}
+// TestVolumePathsSitBesideTheVMs pins the layout the durability rests on: a
+// volume's directory is not inside any VM's, so destroying a VM cannot take a
+// volume with it.
+func TestVolumePathsSitBesideTheVMs(t *testing.T) {
+	s := open(t)
+	assert.Equal(t, filepath.Join(s.dir, "volumes"), s.VolumesDir())
+	assert.Equal(t, filepath.Join(s.VolumesDir(), "v1"), s.VolumeDir("v1"))
+	assert.Equal(t, filepath.Join(s.VolumeDir("v1"), "disk.raw"), s.VolumePath("v1"))
+	assert.Equal(t, filepath.Join(s.VolumeDir("v1"), "tombstoned"), s.VolumeTombstonePath("v1"))
+	require.NoError(t, s.SaveVM(Record{Spec: VMSpec{VMID: "vm1", VolumeIDs: []string{"v1"}}}))
+	require.NoError(t, os.MkdirAll(s.VolumeDir("v1"), 0o700))
+	require.NoError(t, os.WriteFile(s.VolumePath("v1"), []byte("data"), 0o600))
+	require.NoError(t, s.DeleteVM("vm1"))
+	_, err := os.Stat(s.VolumePath("v1"))
+	assert.NoError(t, err, "destroying a VM must not take its volumes with it")
+}
+// TestOpenCreatesVolumesDir: the orphan scan reads this directory every tick,
+// so a fresh agent must not have to create it first.
+func TestOpenCreatesVolumesDir(t *testing.T) {
+	s := open(t)
+	fi, err := os.Stat(s.VolumesDir())
+	require.NoError(t, err)
+	assert.True(t, fi.IsDir())
+}
+// TestVolumeIDsSurviveAReopen: the bound volumes are desired state, so they
+// must be on disk like Network — a restarted agent still knows which files to
+// attach.
+func TestVolumeIDsSurviveAReopen(t *testing.T) {
+	dir := t.TempDir()
+	s, _ := Open(dir)
+	require.NoError(t, s.SaveVM(Record{Spec: VMSpec{VMID: "vm1", VolumeIDs: []string{"vb", "va"}}}))
+	s2, _ := Open(dir)
+	got, _, err := s2.Get("vm1")
+	require.NoError(t, err)
+	assert.Equal(t, []string{"vb", "va"}, got.Spec.VolumeIDs, "in attachment order")
+}

internal/agent/vfkit/vfkit.go

Old		New
@@ -163,20 +163,26 @@ func (p *Provisioner) FailureReason(vmID string) string { return hyperlog.Reason
163	// can hand ConsoleSource the same path Boot writes.	163	// can hand ConsoleSource the same path Boot writes.
164	func (p *Provisioner) SocketPath(vmID string) string { return p.sockPath(vmID) }	164	func (p *Provisioner) SocketPath(vmID string) string { return p.sockPath(vmID) }
165		165
166	// disks returns the VM's block devices in attachment order: the root disk	166	// disks returns the VM's block devices in attachment order: the root disk, the
167	// first, then the cloud-init seed. Both files must already exist when vfkit	167	// cloud-init seed, then every volume in spec order — so the first volume the
168	// starts — it opens them while parsing its arguments, before any VM is built —	168	// tenant attached is /dev/vdc, and stays /dev/vdc across reboots. Every one of
169	// which they do: reconcile prepares the disk and writes the seed before Boot.	169	// these files must already exist when vfkit starts — it opens them while
		170	// parsing its arguments, before any VM is built — which they do: reconcile
		171	// prepares the disk, writes the seed and materialises the volumes before Boot.
170	//	172	//
171	// vfkit's virtio-blk has no read-only option	173	// vfkit's virtio-blk has no read-only option
172	// — unlike cloud-hypervisor's — so state.Disk.ReadOnly is dropped here rather	174	// — unlike cloud-hypervisor's — so state.Disk.ReadOnly is dropped here rather
173	// than honored. Nothing rests on it: the seed is a per-VM file, and cloud-init	175	// than honored. Nothing rests on it: the seed is a per-VM file, and cloud-init
174	// mounts it read-only from inside the guest regardless.	176	// mounts it read-only from inside the guest regardless.
175	func (p *Provisioner) disks(spec state.VMSpec) []state.Disk {	177	func (p *Provisioner) disks(spec state.VMSpec) []state.Disk {
176	return []state.Disk{	178	out := []state.Disk{
177	{Path: p.st.DiskPath(spec.VMID)},	179	{Path: p.st.DiskPath(spec.VMID)},
178	{Path: p.st.SeedPath(spec.VMID), ReadOnly: true},	180	{Path: p.st.SeedPath(spec.VMID), ReadOnly: true},
179	}	181	}
		182	for _, id := range spec.VolumeIDs {
		183	out = append(out, state.Disk{Path: p.st.VolumePath(id)})
		184	}
		185	return out
180	}	186	}
181		187
182	func (p *Provisioner) buildArgs(spec state.VMSpec, createVarStore bool) []string {	188	func (p *Provisioner) buildArgs(spec state.VMSpec, createVarStore bool) []string {

internal/agent/vfkit/vfkit_test.go

@@ @@ -80,6 +80,32 @@ func TestBuildArgsPutsTheRootDiskFirst(t *testing.T) { @@
+	assert.Contains(t, blk[1], "seed.iso")
+// TestDisksAppendVolumesAfterSeedInOrder pins the rest of the guest ABI: a
+// volume never displaces the root disk or the seed, and the spec's order is the
+// device order, so the first volume a tenant attached is /dev/vdc on every boot.
+func TestDisksAppendVolumesAfterSeedInOrder(t *testing.T) {
+	p := newTestProv(t, nil)
+	st := p.st
+	spec := testSpec()
+	spec.VolumeIDs = []string{"vb", "va"}
+	disks := p.disks(spec)
+	require.Len(t, disks, 4)
+	assert.Equal(t, st.DiskPath("vm-1"), disks[0].Path)
+	assert.Equal(t, st.SeedPath("vm-1"), disks[1].Path)
+	assert.Equal(t, st.VolumePath("vb"), disks[2].Path, "spec order, not sorted: the first volume is /dev/vdc")
+	assert.Equal(t, st.VolumePath("va"), disks[3].Path)
+	assert.False(t, disks[2].ReadOnly, "a volume is the guest's to write to")
+	assert.False(t, disks[3].ReadOnly)
+	args := p.buildArgs(spec, false)
+	joined := strings.Join(args, " ")
+	assert.Contains(t, joined, "virtio-blk,path="+st.VolumePath("vb"))
+	assert.Less(t, strings.Index(joined, st.SeedPath("vm-1")), strings.Index(joined, st.VolumePath("vb")),
+		"vfkit maps block devices by argument position, so the volumes come last")
+}
+func TestBuildArgsCreatesTheVariableStoreOnlyWhenThereIsNone(t *testing.T) {
+	p := newTestProv(t, nil)

internal/pb/sync.pb.go

Old		New
@@ -832,7 +832,8 @@ type Report struct {
832	// answer leaves the fleet's record alone rather than erasing it. In the	832	// answer leaves the fleet's record alone rather than erasing it. In the
833	// report rather than Hello for the same reason guest_cidr is: it can change	833	// report rather than Hello for the same reason guest_cidr is: it can change
834	// while an agent stays connected.	834	// while an agent stays connected.
835	HostUplinkAddr string `protobuf:"bytes,10,opt,name=host_uplink_addr,json=hostUplinkAddr,proto3" json:"host_uplink_addr,omitempty"`	835	HostUplinkAddr string `protobuf:"bytes,10,opt,name=host_uplink_addr,json=hostUplinkAddr,proto3" json:"host_uplink_addr,omitempty"`
		836	Volumes []*VolumeStatus `protobuf:"bytes,11,rep,name=volumes,proto3" json:"volumes,omitempty"`
836	unknownFields protoimpl.UnknownFields	837	unknownFields protoimpl.UnknownFields
837	sizeCache protoimpl.SizeCache	838	sizeCache protoimpl.SizeCache
838	}	839	}
@@ -937,6 +938,13 @@ func (x *Report) GetHostUplinkAddr() string {
937	return ""	938	return ""
938	}	939	}
939		940
		941	func (x Report) GetVolumes() []VolumeStatus {
		942	if x != nil {
		943	return x.Volumes
		944	}
		945	return nil
		946	}
		947
940	// VMSpec is the half of a VM the control plane owns. See VMStatus.	948	// VMSpec is the half of a VM the control plane owns. See VMStatus.
941	type VMSpec struct {	949	type VMSpec struct {
942	state protoimpl.MessageState `protogen:"open.v1"`	950	state protoimpl.MessageState `protogen:"open.v1"`
@@ -977,7 +985,11 @@ type VMSpec struct {
977	// this field know). A name is only ever placed on a host that advertised	985	// this field know). A name is only ever placed on a host that advertised
978	// it (see Hello.host_networks); an agent that has the name but no longer	986	// it (see Hello.host_networks); an agent that has the name but no longer
979	// the configuration fails the VM legibly rather than silently NAT-ing it.	987	// the configuration fails the VM legibly rather than silently NAT-ing it.
980	Network string `protobuf:"bytes,20,opt,name=network,proto3" json:"network,omitempty"`	988	Network string `protobuf:"bytes,20,opt,name=network,proto3" json:"network,omitempty"`
		989	// volume_ids are the bound volumes to attach after root and seed, in this
		990	// order — the first is /dev/vdc. Every id names a VolumeSpec in the same
		991	// snapshot.
		992	VolumeIds []string `protobuf:"bytes,21,rep,name=volume_ids,json=volumeIds,proto3" json:"volume_ids,omitempty"`
981	unknownFields protoimpl.UnknownFields	993	unknownFields protoimpl.UnknownFields
982	sizeCache protoimpl.SizeCache	994	sizeCache protoimpl.SizeCache
983	}	995	}
@@ -1124,6 +1136,13 @@ func (x *VMSpec) GetNetwork() string {
1124	return ""	1136	return ""
1125	}	1137	}
1126		1138
		1139	func (x *VMSpec) GetVolumeIds() []string {
		1140	if x != nil {
		1141	return x.VolumeIds
		1142	}
		1143	return nil
		1144	}
		1145
1127	// Snapshot is the FULL spec for one host; the agent converges toward it.	1146	// Snapshot is the FULL spec for one host; the agent converges toward it.
1128	type Snapshot struct {	1147	type Snapshot struct {
1129	state protoimpl.MessageState `protogen:"open.v1"`	1148	state protoimpl.MessageState `protogen:"open.v1"`
@@ -1131,7 +1150,7 @@ type Snapshot struct {
1131	Vms []*VMSpec `protobuf:"bytes,2,rep,name=vms,proto3" json:"vms,omitempty"` // FULL set for this host, including tombstoned	1150	Vms []*VMSpec `protobuf:"bytes,2,rep,name=vms,proto3" json:"vms,omitempty"` // FULL set for this host, including tombstoned
1132	AgentUpgrade *AgentUpgrade `protobuf:"bytes,3,opt,name=agent_upgrade,json=agentUpgrade,proto3" json:"agent_upgrade,omitempty"` // optional operator-initiated agent self-upgrade	1151	AgentUpgrade *AgentUpgrade `protobuf:"bytes,3,opt,name=agent_upgrade,json=agentUpgrade,proto3" json:"agent_upgrade,omitempty"` // optional operator-initiated agent self-upgrade
1133	Exposures []*ExposureSpec `protobuf:"bytes,4,rep,name=exposures,proto3" json:"exposures,omitempty"` // FULL set for this host	1152	Exposures []*ExposureSpec `protobuf:"bytes,4,rep,name=exposures,proto3" json:"exposures,omitempty"` // FULL set for this host
1134	// 5 is taken by volumes in the next change.	1153	Volumes []*VolumeSpec `protobuf:"bytes,5,rep,name=volumes,proto3" json:"volumes,omitempty"` // FULL set for this host
1135	// min_agent_version is the lowest agent release that understands every	1154	// min_agent_version is the lowest agent release that understands every
1136	// field in this snapshot. An agent below it fails every VM here with a	1155	// field in this snapshot. An agent below it fails every VM here with a
1137	// legible reason rather than materialising a spec it only half-reads.	1156	// legible reason rather than materialising a spec it only half-reads.
@@ -1199,6 +1218,13 @@ func (x Snapshot) GetExposures() []ExposureSpec {
1199	return nil	1218	return nil
1200	}	1219	}
1201		1220
		1221	func (x Snapshot) GetVolumes() []VolumeSpec {
		1222	if x != nil {
		1223	return x.Volumes
		1224	}
		1225	return nil
		1226	}
		1227
1202	func (x *Snapshot) GetMinAgentVersion() string {	1228	func (x *Snapshot) GetMinAgentVersion() string {
1203	if x != nil {	1229	if x != nil {
1204	return x.MinAgentVersion	1230	return x.MinAgentVersion
@@ -1718,6 +1744,132 @@ func (x *ExposureSessions) GetDropped() int64 {
1718	return 0	1744	return 0
1719	}	1745	}
1720		1746
		1747	// VolumeSpec is one volume the control plane has placed on this host. The
		1748	// agent keeps a sparse raw file for it at volumes/<volume_id>/disk.raw, beside
		1749	// vms/ and never inside a VM's directory: a VM's directory dies with the VM,
		1750	// a volume's does not. Snapshot.volumes is the FULL set for the host.
		1751	type VolumeSpec struct {
		1752	state protoimpl.MessageState `protogen:"open.v1"`
		1753	VolumeId string `protobuf:"bytes,1,opt,name=volume_id,json=volumeId,proto3" json:"volume_id,omitempty"`
		1754	SizeGb int64 `protobuf:"varint,2,opt,name=size_gb,json=sizeGb,proto3" json:"size_gb,omitempty"`
		1755	Tombstoned bool `protobuf:"varint,3,opt,name=tombstoned,proto3" json:"tombstoned,omitempty"` // delete the file after grace; set only once no VM references it
		1756	unknownFields protoimpl.UnknownFields
		1757	sizeCache protoimpl.SizeCache
		1758	}
		1759
		1760	func (x *VolumeSpec) Reset() {
		1761	*x = VolumeSpec{}
		1762	mi := &file_proto_eitri_v1_sync_proto_msgTypes[19]
		1763	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
		1764	ms.StoreMessageInfo(mi)
		1765	}
		1766
		1767	func (x *VolumeSpec) String() string {
		1768	return protoimpl.X.MessageStringOf(x)
		1769	}
		1770
		1771	func (*VolumeSpec) ProtoMessage() {}
		1772
		1773	func (x *VolumeSpec) ProtoReflect() protoreflect.Message {
		1774	mi := &file_proto_eitri_v1_sync_proto_msgTypes[19]
		1775	if x != nil {
		1776	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
		1777	if ms.LoadMessageInfo() == nil {
		1778	ms.StoreMessageInfo(mi)
		1779	}
		1780	return ms
		1781	}
		1782	return mi.MessageOf(x)
		1783	}
		1784
		1785	// Deprecated: Use VolumeSpec.ProtoReflect.Descriptor instead.
		1786	func (*VolumeSpec) Descriptor() ([]byte, []int) {
		1787	return file_proto_eitri_v1_sync_proto_rawDescGZIP(), []int{19}
		1788	}
		1789
		1790	func (x *VolumeSpec) GetVolumeId() string {
		1791	if x != nil {
		1792	return x.VolumeId
		1793	}
		1794	return ""
		1795	}
		1796
		1797	func (x *VolumeSpec) GetSizeGb() int64 {
		1798	if x != nil {
		1799	return x.SizeGb
		1800	}
		1801	return 0
		1802	}
		1803
		1804	func (x *VolumeSpec) GetTombstoned() bool {
		1805	if x != nil {
		1806	return x.Tombstoned
		1807	}
		1808	return false
		1809	}
		1810
		1811	// VolumeStatus is what the host found on its disk for one volume id —
		1812	// including ids the snapshot did not name, which are reported and kept.
		1813	type VolumeStatus struct {
		1814	state protoimpl.MessageState `protogen:"open.v1"`
		1815	VolumeId string `protobuf:"bytes,1,opt,name=volume_id,json=volumeId,proto3" json:"volume_id,omitempty"`
		1816	Present bool `protobuf:"varint,2,opt,name=present,proto3" json:"present,omitempty"`
		1817	SizeGb int64 `protobuf:"varint,3,opt,name=size_gb,json=sizeGb,proto3" json:"size_gb,omitempty"` // as found, not as asked: a guest filesystem sits on it, so it is never resized
		1818	unknownFields protoimpl.UnknownFields
		1819	sizeCache protoimpl.SizeCache
		1820	}
		1821
		1822	func (x *VolumeStatus) Reset() {
		1823	*x = VolumeStatus{}
		1824	mi := &file_proto_eitri_v1_sync_proto_msgTypes[20]
		1825	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
		1826	ms.StoreMessageInfo(mi)
		1827	}
		1828
		1829	func (x *VolumeStatus) String() string {
		1830	return protoimpl.X.MessageStringOf(x)
		1831	}
		1832
		1833	func (*VolumeStatus) ProtoMessage() {}
		1834
		1835	func (x *VolumeStatus) ProtoReflect() protoreflect.Message {
		1836	mi := &file_proto_eitri_v1_sync_proto_msgTypes[20]
		1837	if x != nil {
		1838	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
		1839	if ms.LoadMessageInfo() == nil {
		1840	ms.StoreMessageInfo(mi)
		1841	}
		1842	return ms
		1843	}
		1844	return mi.MessageOf(x)
		1845	}
		1846
		1847	// Deprecated: Use VolumeStatus.ProtoReflect.Descriptor instead.
		1848	func (*VolumeStatus) Descriptor() ([]byte, []int) {
		1849	return file_proto_eitri_v1_sync_proto_rawDescGZIP(), []int{20}
		1850	}
		1851
		1852	func (x *VolumeStatus) GetVolumeId() string {
		1853	if x != nil {
		1854	return x.VolumeId
		1855	}
		1856	return ""
		1857	}
		1858
		1859	func (x *VolumeStatus) GetPresent() bool {
		1860	if x != nil {
		1861	return x.Present
		1862	}
		1863	return false
		1864	}
		1865
		1866	func (x *VolumeStatus) GetSizeGb() int64 {
		1867	if x != nil {
		1868	return x.SizeGb
		1869	}
		1870	return 0
		1871	}
		1872
1721	var File_proto_eitri_v1_sync_proto protoreflect.FileDescriptor	1873	var File_proto_eitri_v1_sync_proto protoreflect.FileDescriptor
1722		1874
1723	const file_proto_eitri_v1_sync_proto_rawDesc = "" +	1875	const file_proto_eitri_v1_sync_proto_rawDesc = "" +
@@ -1790,7 +1942,7 @@ const file_proto_eitri_v1_sync_proto_rawDesc = "" +
1790	"\x04name\x18\x02 \x01(\tR\x04name\x12\x1f\n" +	1942	"\x04name\x18\x02 \x01(\tR\x04name\x12\x1f\n" +
1791	"\vvmspec_json\x18\x03 \x01(\fR\n" +	1943	"\vvmspec_json\x18\x03 \x01(\fR\n" +
1792	"vmspecJson\x12&\n" +	1944	"vmspecJson\x12&\n" +
1793	"\x0fdestroy_at_unix\x18\x04 \x01(\x03R\rdestroyAtUnix\"\xba\x03\n" +	1945	"\x0fdestroy_at_unix\x18\x04 \x01(\x03R\rdestroyAtUnix\"\xec\x03\n" +
1794	"\x06Report\x12$\n" +	1946	"\x06Report\x12$\n" +
1795	"\x03vms\x18\x01 \x03(\v2\x12.eitri.v1.VMStatusR\x03vms\x12\x1c\n" +	1947	"\x03vms\x18\x01 \x03(\v2\x12.eitri.v1.VMStatusR\x03vms\x12\x1c\n" +
1796	"\tdestroyed\x18\x02 \x03(\tR\tdestroyed\x129\n" +	1948	"\tdestroyed\x18\x02 \x03(\tR\tdestroyed\x129\n" +
@@ -1803,7 +1955,8 @@ const file_proto_eitri_v1_sync_proto_rawDesc = "" +
1803	"guest_cidr\x18\b \x01(\tR\tguestCidr\x126\n" +	1955	"guest_cidr\x18\b \x01(\tR\tguestCidr\x126\n" +
1804	"\texposures\x18\t \x03(\v2\x18.eitri.v1.ExposureStatusR\texposures\x12(\n" +	1956	"\texposures\x18\t \x03(\v2\x18.eitri.v1.ExposureStatusR\texposures\x12(\n" +
1805	"\x10host_uplink_addr\x18\n" +	1957	"\x10host_uplink_addr\x18\n" +
1806	" \x01(\tR\x0ehostUplinkAddr\"\xb9\x04\n" +	1958	" \x01(\tR\x0ehostUplinkAddr\x120\n" +
		1959	"\avolumes\x18\v \x03(\v2\x16.eitri.v1.VolumeStatusR\avolumes\"\xd8\x04\n" +
1807	"\x06VMSpec\x12\x13\n" +	1960	"\x06VMSpec\x12\x13\n" +
1808	"\x05vm_id\x18\x01 \x01(\tR\x04vmId\x12\x12\n" +	1961	"\x05vm_id\x18\x01 \x01(\tR\x04vmId\x12\x12\n" +
1809	"\x04name\x18\x02 \x01(\tR\x04name\x12\x1b\n" +	1962	"\x04name\x18\x02 \x01(\tR\x04name\x12\x1b\n" +
@@ -1827,12 +1980,15 @@ const file_proto_eitri_v1_sync_proto_rawDesc = "" +
1827	"\rssh_host_cert\x18\x11 \x01(\tR\vsshHostCert\x12<\n" +	1980	"\rssh_host_cert\x18\x11 \x01(\tR\vsshHostCert\x12<\n" +
1828	"\x1bssh_user_ca_authorized_keys\x18\x12 \x03(\tR\x17sshUserCaAuthorizedKeys\x12,\n" +	1981	"\x1bssh_user_ca_authorized_keys\x18\x12 \x03(\tR\x17sshUserCaAuthorizedKeys\x12,\n" +
1829	"\x12host_cert_required\x18\x13 \x01(\bR\x10hostCertRequired\x12\x18\n" +	1982	"\x12host_cert_required\x18\x13 \x01(\bR\x10hostCertRequired\x12\x18\n" +
1830	"\anetwork\x18\x14 \x01(\tR\anetworkJ\x04\b\r\x10\x0eJ\x04\b\x0e\x10\x0fJ\x04\b\x0f\x10\x10J\x04\b\x10\x10\x11R\x10ssh_host_key_pem\"\xe3\x01\n" +	1983	"\anetwork\x18\x14 \x01(\tR\anetwork\x12\x1d\n" +
		1984	"\n" +
		1985	"volume_ids\x18\x15 \x03(\tR\tvolumeIdsJ\x04\b\r\x10\x0eJ\x04\b\x0e\x10\x0fJ\x04\b\x0f\x10\x10J\x04\b\x10\x10\x11R\x10ssh_host_key_pem\"\x93\x02\n" +
1831	"\bSnapshot\x12\x14\n" +	1986	"\bSnapshot\x12\x14\n" +
1832	"\x05epoch\x18\x01 \x01(\x04R\x05epoch\x12\"\n" +	1987	"\x05epoch\x18\x01 \x01(\x04R\x05epoch\x12\"\n" +
1833	"\x03vms\x18\x02 \x03(\v2\x10.eitri.v1.VMSpecR\x03vms\x12;\n" +	1988	"\x03vms\x18\x02 \x03(\v2\x10.eitri.v1.VMSpecR\x03vms\x12;\n" +
1834	"\ragent_upgrade\x18\x03 \x01(\v2\x16.eitri.v1.AgentUpgradeR\fagentUpgrade\x124\n" +	1989	"\ragent_upgrade\x18\x03 \x01(\v2\x16.eitri.v1.AgentUpgradeR\fagentUpgrade\x124\n" +
1835	"\texposures\x18\x04 \x03(\v2\x16.eitri.v1.ExposureSpecR\texposures\x12*\n" +	1990	"\texposures\x18\x04 \x03(\v2\x16.eitri.v1.ExposureSpecR\texposures\x12.\n" +
		1991	"\avolumes\x18\x05 \x03(\v2\x14.eitri.v1.VolumeSpecR\avolumes\x12*\n" +
1836	"\x11min_agent_version\x18\x06 \x01(\tR\x0fminAgentVersion\"R\n" +	1992	"\x11min_agent_version\x18\x06 \x01(\tR\x0fminAgentVersion\"R\n" +
1837	"\fAgentUpgrade\x12\x18\n" +	1993	"\fAgentUpgrade\x12\x18\n" +
1838	"\aversion\x18\x01 \x01(\tR\aversion\x12\x10\n" +	1994	"\aversion\x18\x01 \x01(\tR\aversion\x12\x10\n" +
@@ -1864,7 +2020,18 @@ const file_proto_eitri_v1_sync_proto_rawDesc = "" +
1864	"\x10ExposureSessions\x12\x16\n" +	2020	"\x10ExposureSessions\x12\x16\n" +
1865	"\x06active\x18\x01 \x01(\x03R\x06active\x12\x18\n" +	2021	"\x06active\x18\x01 \x01(\x03R\x06active\x12\x18\n" +
1866	"\arefused\x18\x02 \x01(\x03R\arefused\x12\x18\n" +	2022	"\arefused\x18\x02 \x01(\x03R\arefused\x12\x18\n" +
1867	"\adropped\x18\x03 \x01(\x03R\adroppedB&Z$github.com/a73x/eitri/internal/pb;pbb\x06proto3"	2023	"\adropped\x18\x03 \x01(\x03R\adropped\"b\n" +
		2024	"\n" +
		2025	"VolumeSpec\x12\x1b\n" +
		2026	"\tvolume_id\x18\x01 \x01(\tR\bvolumeId\x12\x17\n" +
		2027	"\asize_gb\x18\x02 \x01(\x03R\x06sizeGb\x12\x1e\n" +
		2028	"\n" +
		2029	"tombstoned\x18\x03 \x01(\bR\n" +
		2030	"tombstoned\"^\n" +
		2031	"\fVolumeStatus\x12\x1b\n" +
		2032	"\tvolume_id\x18\x01 \x01(\tR\bvolumeId\x12\x18\n" +
		2033	"\apresent\x18\x02 \x01(\bR\apresent\x12\x17\n" +
		2034	"\asize_gb\x18\x03 \x01(\x03R\x06sizeGbB&Z$github.com/a73x/eitri/internal/pb;pbb\x06proto3"
1868		2035
1869	var (	2036	var (
1870	file_proto_eitri_v1_sync_proto_rawDescOnce sync.Once	2037	file_proto_eitri_v1_sync_proto_rawDescOnce sync.Once
@@ -1878,7 +2045,7 @@ func file_proto_eitri_v1_sync_proto_rawDescGZIP() []byte {
1878	return file_proto_eitri_v1_sync_proto_rawDescData	2045	return file_proto_eitri_v1_sync_proto_rawDescData
1879	}	2046	}
1880		2047
1881	var file_proto_eitri_v1_sync_proto_msgTypes = make([]protoimpl.MessageInfo, 19)	2048	var file_proto_eitri_v1_sync_proto_msgTypes = make([]protoimpl.MessageInfo, 21)
1882	var file_proto_eitri_v1_sync_proto_goTypes = []any{	2049	var file_proto_eitri_v1_sync_proto_goTypes = []any{
1883	(*AgentMessage)(nil), // 0: eitri.v1.AgentMessage	2050	(*AgentMessage)(nil), // 0: eitri.v1.AgentMessage
1884	(*ServerMessage)(nil), // 1: eitri.v1.ServerMessage	2051	(*ServerMessage)(nil), // 1: eitri.v1.ServerMessage
@@ -1899,6 +2066,8 @@ var file_proto_eitri_v1_sync_proto_goTypes = []any{
1899	(*ExposureSpec)(nil), // 16: eitri.v1.ExposureSpec	2066	(*ExposureSpec)(nil), // 16: eitri.v1.ExposureSpec
1900	(*ExposureStatus)(nil), // 17: eitri.v1.ExposureStatus	2067	(*ExposureStatus)(nil), // 17: eitri.v1.ExposureStatus
1901	(*ExposureSessions)(nil), // 18: eitri.v1.ExposureSessions	2068	(*ExposureSessions)(nil), // 18: eitri.v1.ExposureSessions
		2069	(*VolumeSpec)(nil), // 19: eitri.v1.VolumeSpec
		2070	(*VolumeStatus)(nil), // 20: eitri.v1.VolumeStatus
1902	}	2071	}
1903	var file_proto_eitri_v1_sync_proto_depIdxs = []int32{	2072	var file_proto_eitri_v1_sync_proto_depIdxs = []int32{
1904	2, // 0: eitri.v1.AgentMessage.hello:type_name -> eitri.v1.Hello	2073	2, // 0: eitri.v1.AgentMessage.hello:type_name -> eitri.v1.Hello
@@ -1915,15 +2084,17 @@ var file_proto_eitri_v1_sync_proto_depIdxs = []int32{
1915	3, // 11: eitri.v1.Report.capacity:type_name -> eitri.v1.Capacity	2084	3, // 11: eitri.v1.Report.capacity:type_name -> eitri.v1.Capacity
1916	5, // 12: eitri.v1.Report.metrics:type_name -> eitri.v1.HostMetrics	2085	5, // 12: eitri.v1.Report.metrics:type_name -> eitri.v1.HostMetrics
1917	17, // 13: eitri.v1.Report.exposures:type_name -> eitri.v1.ExposureStatus	2086	17, // 13: eitri.v1.Report.exposures:type_name -> eitri.v1.ExposureStatus
1918	9, // 14: eitri.v1.Snapshot.vms:type_name -> eitri.v1.VMSpec	2087	20, // 14: eitri.v1.Report.volumes:type_name -> eitri.v1.VolumeStatus
1919	11, // 15: eitri.v1.Snapshot.agent_upgrade:type_name -> eitri.v1.AgentUpgrade	2088	9, // 15: eitri.v1.Snapshot.vms:type_name -> eitri.v1.VMSpec
1920	16, // 16: eitri.v1.Snapshot.exposures:type_name -> eitri.v1.ExposureSpec	2089	11, // 16: eitri.v1.Snapshot.agent_upgrade:type_name -> eitri.v1.AgentUpgrade
1921	18, // 17: eitri.v1.ExposureStatus.sessions:type_name -> eitri.v1.ExposureSessions	2090	16, // 17: eitri.v1.Snapshot.exposures:type_name -> eitri.v1.ExposureSpec
1922	18, // [18:18] is the sub-list for method output_type	2091	19, // 18: eitri.v1.Snapshot.volumes:type_name -> eitri.v1.VolumeSpec
1923	18, // [18:18] is the sub-list for method input_type	2092	18, // 19: eitri.v1.ExposureStatus.sessions:type_name -> eitri.v1.ExposureSessions
1924	18, // [18:18] is the sub-list for extension type_name	2093	20, // [20:20] is the sub-list for method output_type
1925	18, // [18:18] is the sub-list for extension extendee	2094	20, // [20:20] is the sub-list for method input_type
1926	0, // [0:18] is the sub-list for field type_name	2095	20, // [20:20] is the sub-list for extension type_name
		2096	20, // [20:20] is the sub-list for extension extendee
		2097	0, // [0:20] is the sub-list for field type_name
1927	}	2098	}
1928		2099
1929	func init() { file_proto_eitri_v1_sync_proto_init() }	2100	func init() { file_proto_eitri_v1_sync_proto_init() }
@@ -1948,7 +2119,7 @@ func file_proto_eitri_v1_sync_proto_init() {
1948	GoPackagePath: reflect.TypeOf(x{}).PkgPath(),	2119	GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
1949	RawDescriptor: unsafe.Slice(unsafe.StringData(file_proto_eitri_v1_sync_proto_rawDesc), len(file_proto_eitri_v1_sync_proto_rawDesc)),	2120	RawDescriptor: unsafe.Slice(unsafe.StringData(file_proto_eitri_v1_sync_proto_rawDesc), len(file_proto_eitri_v1_sync_proto_rawDesc)),
1950	NumEnums: 0,	2121	NumEnums: 0,
1951	NumMessages: 19,	2122	NumMessages: 21,
1952	NumExtensions: 0,	2123	NumExtensions: 0,
1953	NumServices: 0,	2124	NumServices: 0,
1954	},	2125	},

internal/server/api/api.go

Old		New
@@ -29,6 +29,13 @@ import (
29	"github.com/a73x/eitri/internal/version"	29	"github.com/a73x/eitri/internal/version"
30	)	30	)
31		31
		32	// volumesFeature is the agent floor a volume-bearing create is judged against.
		33	// It is a var, not the constant its siblings use, because the floor is a
		34	// RELEASE TAG: until volumes ship there is no agent above it, so every test
		35	// that exercises the admission has to name a floor its fixtures can clear.
		36	// Nothing but a test writes it.
		37	var volumesFeature = release.Volumes
		38
32	// DefaultImage is the image applied to one-click VM creates.	39	// DefaultImage is the image applied to one-click VM creates.
33	type DefaultImage struct {	40	type DefaultImage struct {
34	URL string	41	URL string
@@ -89,6 +96,13 @@ type API struct {
89	// wires it, which happens only when the jump gate is on; nil ⇒ the	96	// wires it, which happens only when the jump gate is on; nil ⇒ the
90	// delegation routes answer 503.	97	// delegation routes answer 503.
91	delegations *delegation.Keyring	98	delegations *delegation.Keyring
		99	// volumeStuck remembers, per decommissioning host, the volume count the
		100	// last "stuck on volumes" line reported. The sweep runs every two seconds
		101	// and a volume never drains itself, so logging the refusal every pass would
		102	// bury the log; logging only when the number CHANGES says it once when the
		103	// decommission stalls and once more each time the operator deletes a claim.
		104	// Touched only from StartBackground's goroutine.
		105	volumeStuck map[string]int
92	}	106	}
93		107
94	// URL renders an API path as a full URL a caller can actually dial, using the	108	// URL renders an API path as a full URL a caller can actually dial, using the
@@ -252,27 +266,65 @@ func (a *API) sweepAbandonedVMs(now time.Time) bool {
252		266
253	// sweepDecommissioned finalizes any decommissioning host with no VM rows left	267	// sweepDecommissioned finalizes any decommissioning host with no VM rows left
254	// (fully drained). Returns true if it removed at least one host.	268	// (fully drained). Returns true if it removed at least one host.
		269	//
		270	// One refusal is not like the others. An undrained VM is a decommission still
		271	// in progress and silence is correct — the next pass, two seconds later, is the
		272	// retry. A held volume is a decommission that will NEVER finish on its own: a
		273	// volume outlives its guests by design, so no amount of waiting releases it and
		274	// the host sits in `decommissioning` until an operator deletes the claims. That
		275	// one gets said out loud, and the map keeps it from being said 30 times a
		276	// minute forever.
255	func (a *API) sweepDecommissioned() bool {	277	func (a *API) sweepDecommissioned() bool {
256	hosts, err := a.st.ListHosts()	278	hosts, err := a.st.ListHosts()
257	if err != nil {	279	if err != nil {
258	return false	280	return false
259	}	281	}
260	removed := false	282	removed := false
		283	// Rebuilt every sweep, so a host that leaves (removed, or forced away)
		284	// takes its entry with it instead of leaking one per decommission.
		285	stuck := map[string]int{}
261	for _, h := range hosts {	286	for _, h := range hosts {
262	if h.Status != "decommissioning" {	287	if h.Status != "decommissioning" {
263	continue	288	continue
264	}	289	}
265	// RemoveHost re-counts VM rows in-transaction and refuses while any	290	// RemoveHost re-counts VM rows in-transaction and refuses while any
266	// remain, so a pre-check here would only save a wasted call on hosts	291	// remain, so a pre-check here would only save a wasted call on hosts
267	// that aren't yet drained — every RemoveHost error (drained or not) is	292	// that aren't yet drained — an undrained VM is handled by simply
268	// already handled by simply skipping to the next host.	293	// skipping to the next host.
269	if a.st.RemoveHost(h.ID) == nil {	294	err := a.st.RemoveHost(h.ID)
		295	switch {
		296	case err == nil:
270	removed = true	297	removed = true
		298	case errors.Is(err, store.ErrHostHoldsVolumes):
		299	n, ids := a.volumesOnHost(h.ID)
		300	prev, seen := a.volumeStuck[h.ID]
		301	stuck[h.ID] = n
		302	if !seen \|\| prev != n {
		303	slog.Warn("decommission is stuck: host still holds volumes",
		304	"host", h.ID, "volumes", n, "volume_ids", ids,
		305	"remedy", "delete the volume claims placed on this host")
		306	}
271	}	307	}
272	}	308	}
		309	a.volumeStuck = stuck
273	return removed	310	return removed
274	}	311	}
275		312
		313	// volumesOnHost reports how many volume rows a host still holds and names them,
		314	// for the stuck-decommission line. A read failure reports nothing rather than
		315	// inventing a number: the count in the log would be the only lie in it.
		316	func (a *API) volumesOnHost(hostID string) (int, string) {
		317	vols, err := a.st.ListVolumesForHost(hostID)
		318	if err != nil {
		319	return 0, ""
		320	}
		321	ids := make([]string, 0, len(vols))
		322	for _, v := range vols {
		323	ids = append(ids, v.ID)
		324	}
		325	return len(vols), strings.Join(ids, ",")
		326	}
		327
276	// userAuth resolves a request to a tenant principal: PAT bearer first, then	328	// userAuth resolves a request to a tenant principal: PAT bearer first, then
277	// the eitri_session console cookie. There are no other credentials and no	329	// the eitri_session console cookie. There are no other credentials and no
278	// per-route exceptions (spec §3); a request with neither gets 401 and the SPA	330	// per-route exceptions (spec §3); a request with neither gets 401 and the SPA
@@ -872,6 +924,28 @@ func (a API) handleCreateVM(w http.ResponseWriter, r http.Request) {
872	return	924	return
873	}	925	}
874		926
		927	// Volumes, the fifth refusal of this shape and the first that judges an
		928	// OFFLINE host (see refuseBelowFloor): an old agent ignores volume_ids
		929	// and boots the guest bare, so the data the tenant meant for the volume
		930	// lands on the root disk — the one thing they did not ask for.
		931	//
		932	// It runs before the capacity block because it feeds it: a pending claim's
		933	// bytes are disk this create commits, and the host has to have room for
		934	// both. The claims are read against the HOST's tenant, which the authz
		935	// gate above has already proven is the caller's own.
		936	claimIDs, claimGB, msg, code, err := a.resolveClaims(host.Tenant, req.VolumeClaims)
		937	if err != nil {
		938	http.Error(w, "internal error", http.StatusInternalServerError)
		939	return
		940	}
		941	if msg != "" {
		942	http.Error(w, msg, code)
		943	return
		944	}
		945	if len(claimIDs) > 0 && a.refuseBelowFloor(w, req.HostID, volumesFeature, true) {
		946	return
		947	}
		948
875	// Capacity precondition, the third refusal of this same shape: the request	949	// Capacity precondition, the third refusal of this same shape: the request
876	// is fine, and the host has been told not to serve it. A host whose operator	950	// is fine, and the host has been told not to serve it. A host whose operator
877	// capped it is already holding as much as it may hold cannot boot one more	951	// capped it is already holding as much as it may hold cannot boot one more
@@ -898,7 +972,10 @@ func (a API) handleCreateVM(w http.ResponseWriter, r http.Request) {
898	http.Error(w, "internal error", http.StatusInternalServerError)	972	http.Error(w, "internal error", http.StatusInternalServerError)
899	return	973	return
900	}	974	}
901	want := store.Alloc{VCPUs: req.VCPUs, MemMB: req.MemMB, DiskGB: req.DiskGB}	975	// claimGB is the PENDING claims only: a bound claim's volume already
		976	// sits in held (CommittedOnHost counts live volumes), so adding it
		977	// here would charge the host twice for disk this VM is re-using.
		978	want := store.Alloc{VCPUs: req.VCPUs, MemMB: req.MemMB, DiskGB: req.DiskGB + claimGB}
902	if msg := overCapacityRefusal(host.Name, req.HostID, want, held, hostState.Report); msg != "" {	979	if msg := overCapacityRefusal(host.Name, req.HostID, want, held, hostState.Report); msg != "" {
903	http.Error(w, msg, http.StatusConflict)	980	http.Error(w, msg, http.StatusConflict)
904	return	981	return
@@ -980,6 +1057,11 @@ func (a API) handleCreateVM(w http.ResponseWriter, r http.Request) {
980	DiskGB: req.DiskGB,	1057	DiskGB: req.DiskGB,
981	PowerState: req.PowerState,	1058	PowerState: req.PowerState,
982	Network: req.Network,	1059	Network: req.Network,
		1060
		1061	// Resolved to ids, in the order the caller named them, which is the
		1062	// order the guest will see the devices in. CreateVM binds them inside
		1063	// its own transaction — placement and binding are one commit.
		1064	VolumeClaimIDs: claimIDs,
983	}	1065	}
984		1066
985	// The row carries no host key. A guest's host key is generated by the host	1067	// The row carries no host key. A guest's host key is generated by the host
@@ -988,6 +1070,14 @@ func (a API) handleCreateVM(w http.ResponseWriter, r http.Request) {
988	// (see syncsvc.signAndRecordHostCert). Creating a VM therefore involves no	1070	// (see syncsvc.signAndRecordHostCert). Creating a VM therefore involves no
989	// key material at all, which is why there is nothing here to guard.	1071	// key material at all, which is why there is nothing here to guard.
990		1072
		1073	// The claim cases below are the same three the admission above already
		1074	// checks, re-answered from inside the transaction that is authoritative
		1075	// for them: the preflight read and this write are separate moments, and
		1076	// another create can attach a claim in between. That is the race the
		1077	// attachment table's unique index referees, and this is where its verdict
		1078	// is turned back into an answer.
		1079	var attached *store.ClaimAttachedError
		1080	var pinned *store.ClaimPinnedError
991	if err := a.st.CreateVM(vm); err != nil {	1081	if err := a.st.CreateVM(vm); err != nil {
992	switch {	1082	switch {
993	case errors.Is(err, store.ErrNameTaken):	1083	case errors.Is(err, store.ErrNameTaken):
@@ -996,13 +1086,21 @@ func (a API) handleCreateVM(w http.ResponseWriter, r http.Request) {
996	http.Error(w, "unknown host_id", http.StatusBadRequest)	1086	http.Error(w, "unknown host_id", http.StatusBadRequest)
997	case errors.Is(err, store.ErrHostNotEnrolled):	1087	case errors.Is(err, store.ErrHostNotEnrolled):
998	http.Error(w, "host is not accepting new VMs", http.StatusConflict)	1088	http.Error(w, "host is not accepting new VMs", http.StatusConflict)
		1089	case errors.Is(err, store.ErrClaimNotFound):
		1090	http.Error(w, "unknown volume claim", http.StatusNotFound)
		1091	case errors.As(err, &attached):
		1092	http.Error(w, "volume claim "+attached.ClaimID+" is attached to vm "+attached.VMID, http.StatusConflict)
		1093	case errors.As(err, &pinned):
		1094	http.Error(w, "volume claim "+pinned.ClaimID+" is bound to host "+pinned.HostID+
		1095	"; its data lives there, so a VM using it must be placed there", http.StatusConflict)
999	default:	1096	default:
1000	http.Error(w, "internal error", http.StatusInternalServerError)	1097	http.Error(w, "internal error", http.StatusInternalServerError)
1001	}	1098	}
1002	return	1099	return
1003	}	1100	}
1004		1101
1005	a.audit(host.Tenant, "vm.create", map[string]string{"vm_id": id, "name": req.Name, "host_id": req.HostID})	1102	a.audit(host.Tenant, "vm.create", map[string]string{"vm_id": id, "name": req.Name, "host_id": req.HostID,
		1103	"volume_claims": strings.Join(claimIDs, ",")})
1006	a.hub.Poke(req.HostID)	1104	a.hub.Poke(req.HostID)
1007	a.notif.notify()	1105	a.notif.notify()
1008	writeJSON(w, http.StatusCreated, types.CreateVMResponse{ID: id, Name: req.Name})	1106	writeJSON(w, http.StatusCreated, types.CreateVMResponse{ID: id, Name: req.Name})

internal/server/api/api_test.go

Old		New
@@ -987,7 +987,8 @@ func TestEnrollMintsGenerationCredential(t *testing.T) {
987	}	987	}
988		988
989	// TestAuditDetailKeysArePinned nails down the KEY SET of every audit detail an	989	// TestAuditDetailKeysArePinned nails down the KEY SET of every audit detail an
990	// action in api.go, tokens.go, and usercas.go emits. AuditEvent.Detail is a	990	// action in api.go, volumes.go, events.go, tokens.go, and usercas.go emits.
		991	// AuditEvent.Detail is a
991	// json.RawMessage, so the wire golden marshals it as opaque bytes and cannot	992	// json.RawMessage, so the wire golden marshals it as opaque bytes and cannot
992	// see inside — a rename like host_id→hostId would sail through every other	993	// see inside — a rename like host_id→hostId would sail through every other
993	// test. Here each action is driven for real and its emitted detail decoded, so	994	// test. Here each action is driven for real and its emitted detail decoded, so
@@ -1012,6 +1013,15 @@ func TestAuditDetailKeysArePinned(t *testing.T) {
1012	require.Equal(t, 204, do(t, "DELETE", ts.URL+"/api/v1/vms/"+vmID, testPAT, nil).StatusCode)	1013	require.Equal(t, 204, do(t, "DELETE", ts.URL+"/api/v1/vms/"+vmID, testPAT, nil).StatusCode)
1013	require.Equal(t, 204, do(t, "POST", ts.URL+"/api/v1/vms/"+vmID+"/restore", testPAT, nil).StatusCode)	1014	require.Equal(t, 204, do(t, "POST", ts.URL+"/api/v1/vms/"+vmID+"/restore", testPAT, nil).StatusCode)
1014		1015
		1016	// volumes.go: claim storage and give it back. The claim is never attached,
		1017	// so the delete is the ordinary path rather than the 409.
		1018	resp = do(t, "POST", ts.URL+"/api/v1/volume-claims", testPAT,
		1019	map[string]any{"name": "audited-claim", "size_gb": 5})
		1020	require.Equal(t, 201, resp.StatusCode)
		1021	var claim map[string]any
		1022	require.NoError(t, json.NewDecoder(resp.Body).Decode(&claim))
		1023	require.Equal(t, 204, do(t, "DELETE", ts.URL+"/api/v1/volume-claims/"+claim["id"].(string), testPAT, nil).StatusCode)
		1024
1015	// tokens.go: mint then revoke a PAT.	1025	// tokens.go: mint then revoke a PAT.
1016	resp = do(t, "POST", ts.URL+"/api/v1/tokens", testPAT,	1026	resp = do(t, "POST", ts.URL+"/api/v1/tokens", testPAT,
1017	map[string]any{"name": "audited-token", "ttl_seconds": 3600})	1027	map[string]any{"name": "audited-token", "ttl_seconds": 3600})
@@ -1027,6 +1037,12 @@ func TestAuditDetailKeysArePinned(t *testing.T) {
1027	"label": "audit-ca",	1037	"label": "audit-ca",
1028	}).StatusCode)	1038	}).StatusCode)
1029		1039
		1040	// events.go: the forced decommission, LAST because it destroys the host
		1041	// everything above ran on. Only the force shape is pinned — it is the one
		1042	// that reports what was lost, and the graceful path's row shares the action
		1043	// name, so newest-row-per-action can only hold one of the two.
		1044	require.Equal(t, 200, do(t, "DELETE", ts.URL+"/api/v1/hosts/"+hostID+"?force=true", testPAT, nil).StatusCode)
		1045
1030	rows, err := st.ListAudit(testTenant, 100)	1046	rows, err := st.ListAudit(testTenant, 100)
1031	require.NoError(t, err)	1047	require.NoError(t, err)
1032	// Newest row per action wins; the detail shape is one per action.	1048	// Newest row per action wins; the detail shape is one per action.
@@ -1046,10 +1062,20 @@ func TestAuditDetailKeysArePinned(t *testing.T) {
1046	}	1062	}
1047		1063
1048	want := map[string][]string{	1064	want := map[string][]string{
1049	"vm.create": {"vm_id", "name", "host_id"},	1065	// volume_claims is on every vm.create, empty when the VM asked for
1050	"vm.power": {"vm_id", "name", "power"},	1066	// none: "this guest was given no volumes" is a fact worth recording,
1051	"vm.delete": {"vm_id", "name"},	1067	// and a key that comes and goes is a shape no reader can rely on.
1052	"vm.restore": {"vm_id", "name"},	1068	"vm.create": {"vm_id", "name", "host_id", "volume_claims"},
		1069	"vm.power": {"vm_id", "name", "power"},
		1070	"vm.delete": {"vm_id", "name"},
		1071	"vm.restore": {"vm_id", "name"},
		1072	"volume_claim.create": {"claim_id", "name", "size_gb"},
		1073	"volume_claim.delete": {"claim_id", "name", "volume_id"},
		1074	// Force is the one path that destroys data on purpose: its row is the
		1075	// only surviving record of which volumes went with the hardware, so
		1076	// every count and the id list are part of the pinned shape.
		1077	"host.decommission": {"host_id", "remote", "force", "vms_purged",
		1078	"volumes_destroyed", "claims_unbound", "volume_ids"},
1053	"enroll-token.mint": {"remote", "token_hash_prefix"},	1079	"enroll-token.mint": {"remote", "token_hash_prefix"},
1054	"api-token.mint": {"token_id", "name"},	1080	"api-token.mint": {"token_id", "name"},
1055	"api-token.revoke": {"token_id"},	1081	"api-token.revoke": {"token_id"},

internal/server/api/capacity_api_test.go

@@ @@ -97,6 +97,36 @@ func TestCreateVMRefusesAHostWithNoRoom(t *testing.T) { @@
+// TestCreateVMCountsAPendingClaimAgainstTheDisk is the disk dimension's other
+// half. A create that names a pending claim commits the host to that claim's
+// bytes as surely as to the guest's root disk — the volume is placed by THIS
+// create — so the two are judged together. Judging only disk_gb would admit a
+// VM whose volume the host has no room for, and the tenant would find that out
+// when the agent failed to materialize the file.
+//
+// A bound claim is not added: its volume is already on the host, so
+// CommittedOnHost has counted it, and counting it again would refuse a VM for
+// disk it is re-using rather than asking for.
+func TestCreateVMCountsAPendingClaimAgainstTheDisk(t *testing.T) {
+	ts, _, _, reg, _ := newServer(t)
+	floorVolumes(t, "v0.0.7")
+	out := enroll(t, ts)
+	hostID := out["host_id"]
+	onlineAt(reg, hostID, "v0.0.7")
+	agentReportsCappedCapacity(hostID, 8, 8192, 100)
+	claimPost(t, ts, "data", 60)
+	resp := do(t, "POST", ts.URL+"/api/v1/vms", testPAT, map[string]any{
+		"host_id": hostID, "name": "with-volume", "vcpus": 1, "mem_mb": 1024, "disk_gb": 50,
+		"volume_claims": []string{"data"}})
+	require.Equal(t, 409, resp.StatusCode)
+	assert.Contains(t, bodyOf(t, resp), "disk — needs 110GB, already holds 0 of 100GB",
+		"the refusal must count the volume the create would place")
+	assert.Equal(t, 201, createVM(t, ts, hostID, "no-volume", 1, 1024, 50).StatusCode,
+		"the same VM without the claim fits")
+}
+// TestCreateVMRefusalNamesEveryBindingDimension: a VM too big in two ways
+// should be resized once, not discovered twice.
+func TestCreateVMRefusalNamesEveryBindingDimension(t *testing.T) {

internal/server/api/client/client.go

Old		New
@@ -40,6 +40,9 @@ type (
40	DelegationChallenge = types.DelegationChallenge	40	DelegationChallenge = types.DelegationChallenge
41	DelegationRequest = types.DelegationRequest	41	DelegationRequest = types.DelegationRequest
42	Delegation = types.Delegation	42	Delegation = types.Delegation
		43
		44	VolumeClaim = types.VolumeClaim
		45	CreateVolumeClaimRequest = types.CreateVolumeClaimRequest
43	)	46	)
44		47
45	// DefaultDiskGB is the wire contract's default disk size, re-exported for the	48	// DefaultDiskGB is the wire contract's default disk size, re-exported for the
@@ -180,6 +183,34 @@ func (c *Client) DeleteExposure(ctx context.Context, id string) error {
180	return c.do(ctx, http.MethodDelete, "/api/v1/exposures/"+url.PathEscape(id), nil, nil)	183	return c.do(ctx, http.MethodDelete, "/api/v1/exposures/"+url.PathEscape(id), nil, nil)
181	}	184	}
182		185
		186	// CreateVolumeClaim claims sizeGB of durable storage under name. The claim is
		187	// Pending — nothing is placed — until the first VM that names it is created,
		188	// which is what decides the host its bytes live on.
		189	func (c *Client) CreateVolumeClaim(ctx context.Context, name string, sizeGB int64) (VolumeClaim, error) {
		190	var out VolumeClaim
		191	return out, c.do(ctx, http.MethodPost, "/api/v1/volume-claims",
		192	CreateVolumeClaimRequest{Name: name, SizeGB: sizeGB}, &out)
		193	}
		194
		195	// ListVolumeClaims returns the caller tenant's claims, oldest first.
		196	func (c *Client) ListVolumeClaims(ctx context.Context) ([]VolumeClaim, error) {
		197	var out []VolumeClaim
		198	return out, c.do(ctx, http.MethodGet, "/api/v1/volume-claims", nil, &out)
		199	}
		200
		201	// GetVolumeClaim reads one claim by id: where it is bound, which VM holds it,
		202	// and whether its host has reported the file.
		203	func (c *Client) GetVolumeClaim(ctx context.Context, id string) (VolumeClaim, error) {
		204	var out VolumeClaim
		205	return out, c.do(ctx, http.MethodGet, "/api/v1/volume-claims/"+url.PathEscape(id), nil, &out)
		206	}
		207
		208	// DeleteVolumeClaim deletes a claim and the data behind it. Refused while a VM
		209	// holds it — delete that VM first.
		210	func (c *Client) DeleteVolumeClaim(ctx context.Context, id string) error {
		211	return c.do(ctx, http.MethodDelete, "/api/v1/volume-claims/"+url.PathEscape(id), nil, nil)
		212	}
		213
183	// Me returns the signed-in identity (tenant handle + bound email) for the	214	// Me returns the signed-in identity (tenant handle + bound email) for the
184	// credential this client carries. It takes no context — the consumers (smoke	215	// credential this client carries. It takes no context — the consumers (smoke
185	// gate, CLI) call it as a quick synchronous probe; the do timeout bounds it.	216	// gate, CLI) call it as a quick synchronous probe; the do timeout bounds it.

internal/server/api/client/client_test.go

@@ @@ -528,6 +528,88 @@ func TestDeleteExposure(t *testing.T) { @@
+func TestCreateVolumeClaim(t *testing.T) {
+	var cap capture
+	srv := serve(t, &cap, http.StatusCreated,
+		`{"id":"c-1","name":"project-data","size_gb":50,"status":"pending",`+
+			`"host_id":"","vm_id":"","present":null,"created_at":"2026-08-22T12:00:00Z"}`)
+	c := &client.Client{BaseURL: srv.URL, Token: "eitri_pat_x"}
+	got, err := c.CreateVolumeClaim(context.Background(), "project-data", 50)
+	if err != nil {
+		t.Fatalf("CreateVolumeClaim: %v", err)
+	}
+	if cap.method != http.MethodPost || cap.path != "/api/v1/volume-claims" {
+		t.Errorf("request = %s %s, want POST /api/v1/volume-claims", cap.method, cap.path)
+	}
+	var sent types.CreateVolumeClaimRequest
+	if err := json.Unmarshal(cap.body, &sent); err != nil {
+		t.Fatalf("decode sent body: %v", err)
+	}
+	if sent.Name != "project-data" || sent.SizeGB != 50 {
+		t.Errorf("sent = %+v, want project-data / 50GB", sent)
+	}
+	if got.ID != "c-1" || got.Status != "pending" {
+		t.Errorf("got = %+v, want the pending claim back", got)
+	}
+	// present is null on the wire and must decode to nil, not false: a client
+	// that flattens the two reports a missing disk for one nobody has looked at.
+	if got.Present != nil {
+		t.Errorf("present = %v, want nil for a claim no host has reported on", *got.Present)
+	}
+}
+func TestListVolumeClaims(t *testing.T) {
+	var cap capture
+	srv := serve(t, &cap, http.StatusOK,
+		`[{"id":"c-1","name":"data","size_gb":5,"status":"bound","host_id":"h-1","vm_id":"v-1","present":true}]`)
+	c := &client.Client{BaseURL: srv.URL, Token: "eitri_pat_x"}
+	got, err := c.ListVolumeClaims(context.Background())
+	if err != nil {
+		t.Fatalf("ListVolumeClaims: %v", err)
+	}
+	if cap.method != http.MethodGet || cap.path != "/api/v1/volume-claims" {
+		t.Errorf("request = %s %s, want GET /api/v1/volume-claims", cap.method, cap.path)
+	}
+	if len(got) != 1 || got[0].Status != "bound" || got[0].HostID != "h-1" {
+		t.Errorf("got = %+v, want one bound claim on h-1", got)
+	}
+	if got[0].Present == nil || !*got[0].Present {
+		t.Errorf("present = %v, want true", got[0].Present)
+	}
+}
+func TestGetVolumeClaimEscapesID(t *testing.T) {
+	var cap capture
+	srv := serve(t, &cap, http.StatusOK, `{"id":"c-1","name":"data","status":"pending"}`)
+	c := &client.Client{BaseURL: srv.URL, Token: "eitri_pat_x"}
+	got, err := c.GetVolumeClaim(context.Background(), "a b/c")
+	if err != nil {
+		t.Fatalf("GetVolumeClaim: %v", err)
+	}
+	if want := "/api/v1/volume-claims/a%20b%2Fc"; cap.method != http.MethodGet || cap.path != want {
+		t.Errorf("request = %s %s, want GET %s", cap.method, cap.path, want)
+	}
+	if got.ID != "c-1" {
+		t.Errorf("got = %+v, want claim c-1", got)
+	}
+}
+func TestDeleteVolumeClaim(t *testing.T) {
+	var cap capture
+	srv := serve(t, &cap, http.StatusNoContent, "")
+	c := &client.Client{BaseURL: srv.URL, Token: "eitri_pat_x"}
+	if err := c.DeleteVolumeClaim(context.Background(), "c-1"); err != nil {
+		t.Fatalf("DeleteVolumeClaim: %v", err)
+	}
+	if cap.method != http.MethodDelete || cap.path != "/api/v1/volume-claims/c-1" {
+		t.Errorf("request = %s %s, want DELETE /api/v1/volume-claims/c-1", cap.method, cap.path)
+	}
+}
+// The four delegation calls are one endpoint distinguished by method, so the
+// method is the thing worth pinning: getting it wrong would silently start or
+// end a delegation instead of reading one.

internal/server/api/decommission_api_test.go

Old		New
@@ -3,11 +3,13 @@ package api
3	import (	3	import (
4	"bufio"	4	"bufio"
5	"context"	5	"context"
		6	"log/slog"
6	"net/http"	7	"net/http"
7	"strings"	8	"strings"
8	"testing"	9	"testing"
9	"time"	10	"time"
10		11
		12	"github.com/a73x/eitri/internal/server/store"
11	"github.com/stretchr/testify/assert"	13	"github.com/stretchr/testify/assert"
12	"github.com/stretchr/testify/require"	14	"github.com/stretchr/testify/require"
13	)	15	)
@@ -52,6 +54,51 @@ func TestSweepLeavesHostWithVMs(t *testing.T) {
52	require.Len(t, hosts, 1)	54	require.Len(t, hosts, 1)
53	}	55	}
54		56
		57	// A decommission blocked on volumes never finishes on its own, so the sweep
		58	// has to say so — once, not thirty times a minute.
		59	func TestSweepSaysWhenAHostIsStuckOnVolumes(t *testing.T) {
		60	ts, st, _, _, a := newServer(t)
		61	out := enroll(t, ts)
		62	hostID := out["host_id"]
		63
		64	c, err := st.CreateVolumeClaim(testTenant, "data", 5)
		65	require.NoError(t, err)
		66	// Straight to the store: no HTTP surface names a claim yet.
		67	require.NoError(t, st.CreateVM(store.VM{
		68	ID: "vm-a", HostID: hostID, Name: "vm-a", ImageURL: "u", ImageSHA256: "s",
		69	VCPUs: 1, MemMB: 512, DiskGB: 1, PowerState: "running",
		70	VolumeClaimIDs: []string{c.ID},
		71	}))
		72
		73	do(t, "DELETE", ts.URL+"/api/v1/hosts/"+hostID, testPAT, nil)
		74
		75	var logged strings.Builder
		76	restore := slog.Default()
		77	slog.SetDefault(slog.New(slog.NewTextHandler(&logged, nil)))
		78	t.Cleanup(func() { slog.SetDefault(restore) })
		79
		80	// The VM is still there, so the first refusals are the ordinary undrained
		81	// kind and stay quiet.
		82	assert.False(t, a.sweepDecommissioned())
		83	assert.Empty(t, logged.String(), "an undrained VM is a decommission in progress, not a stuck one")
		84
		85	// Reap the VM the decommission already tombstoned: now only the volume is
		86	// in the way, and that one is terminal.
		87	require.NoError(t, st.HardDeleteVM("vm-a", hostID))
		88	assert.False(t, a.sweepDecommissioned(), "a host holding volumes must not be swept")
		89	first := logged.String()
		90	assert.Contains(t, first, "decommission is stuck")
		91	assert.Contains(t, first, "volumes=1")
		92
		93	// Every later pass finds the same count and stays silent.
		94	assert.False(t, a.sweepDecommissioned())
		95	assert.False(t, a.sweepDecommissioned())
		96	assert.Equal(t, first, logged.String(), "the stuck line is said once, not once per tick")
		97
		98	hosts := decodeJSONKeys(t, do(t, "GET", ts.URL+"/api/v1/hosts", testPAT, nil))
		99	require.Len(t, hosts, 1, "the host stays until its claims are deleted")
		100	}
		101
55	func TestEventsStreamSendsSnapshot(t *testing.T) {	102	func TestEventsStreamSendsSnapshot(t *testing.T) {
56	ts, _, _, _, _ := newServer(t)	103	ts, _, _, _, _ := newServer(t)
57	enroll(t, ts)	104	enroll(t, ts)

internal/server/api/decommission_poke_test.go

@@ @@ -1,6 +1,7 @@ @@
+package api
+import (
+	"encoding/json"
+	"net/http"
+	"testing"
+	"time"
@@ @@ -50,6 +51,49 @@ func TestForceDecommissionRemovesHostWithVMs(t *testing.T) { @@
+	assert.Empty(t, hosts, "force must remove the host immediately")
+// TestForceDecommissionAuditNamesWhatItDestroyed pins the receipt for the one
+// path that loses data on purpose. The volumes are deleted with the host and
+// their ids exist nowhere afterwards, so this audit row is the only record an
+// operator reconciling a lost host against backups has to work from.
+func TestForceDecommissionAuditNamesWhatItDestroyed(t *testing.T) {
+	ts, st, _, reg, _ := newServer(t)
+	floorVolumes(t, "v0.0.7")
+	out := enroll(t, ts)
+	hostID := out["host_id"]
+	onlineAt(reg, hostID, "v0.0.7")
+	resp := do(t, "POST", ts.URL+"/api/v1/volume-claims", testPAT,
+		map[string]any{"name": "payroll", "size_gb": 5})
+	require.Equal(t, http.StatusCreated, resp.StatusCode)
+	var claim struct {
+		ID string `json:"id"`
+	}
+	require.NoError(t, json.NewDecoder(resp.Body).Decode(&claim))
+	require.NotEmpty(t, claim.ID)
+	require.Equal(t, http.StatusCreated, do(t, "POST", ts.URL+"/api/v1/vms", testPAT,
+		map[string]any{"host_id": hostID, "name": "vm-a", "volume_claims": []string{claim.ID}}).StatusCode)
+	// The volume id as minted, read while the row still exists.
+	vols, err := st.ListVolumesForHost(hostID)
+	require.NoError(t, err)
+	require.Len(t, vols, 1)
+	require.Equal(t, http.StatusOK,
+		do(t, "DELETE", ts.URL+"/api/v1/hosts/"+hostID+"?force=true", testPAT, nil).StatusCode)
+	rows, err := st.ListAudit(testTenant, 10)
+	require.NoError(t, err)
+	require.NotEmpty(t, rows)
+	require.Equal(t, "host.decommission", rows[0].Action, "newest first")
+	var detail map[string]string
+	require.NoError(t, json.Unmarshal([]byte(rows[0].Detail), &detail))
+	assert.Equal(t, "true", detail["force"])
+	assert.Equal(t, "1", detail["vms_purged"])
+	assert.Equal(t, "1", detail["volumes_destroyed"])
+	assert.Equal(t, "1", detail["claims_unbound"])
+	assert.Equal(t, vols[0].ID, detail["volume_ids"], "the destroyed volume, named")
+}
+// TestCreateVMRejectedOnDecommissioningHost pins that a host mid-decommission
+// no longer accepts new VMs (previously only the FK was enforced, so a create
+// could land on a host being torn down).

internal/server/api/events.go

Old		New
@@ -7,6 +7,7 @@ import (
7	"fmt"	7	"fmt"
8	"net/http"	8	"net/http"
9	"strconv"	9	"strconv"
		10	"strings"
10	"time"	11	"time"
11		12
12	"github.com/a73x/eitri/internal/server/api/types"	13	"github.com/a73x/eitri/internal/server/api/types"
@@ -46,7 +47,7 @@ func (a API) handleDecommissionHost(w http.ResponseWriter, r http.Request) {
46	}	47	}
47		48
48	if forceParam(r) {	49	if forceParam(r) {
49	purged, err := a.st.ForceRemoveHost(id)	50	gone, err := a.st.ForceRemoveHost(id)
50	switch {	51	switch {
51	case errors.Is(err, sql.ErrNoRows):	52	case errors.Is(err, sql.ErrNoRows):
52	http.Error(w, "host not found", http.StatusNotFound)	53	http.Error(w, "host not found", http.StatusNotFound)
@@ -55,9 +56,16 @@ func (a API) handleDecommissionHost(w http.ResponseWriter, r http.Request) {
55	http.Error(w, "internal error", http.StatusInternalServerError)	56	http.Error(w, "internal error", http.StatusInternalServerError)
56	return	57	return
57	}	58	}
		59	// Force is the one path that destroys data on purpose, so its audit row
		60	// is the only record of WHAT it destroyed: the volumes are deleted here
		61	// and their ids exist nowhere afterwards. An operator reconciling a lost
		62	// host against backups has this row and nothing else.
58	a.audit(h.Tenant, "host.decommission", map[string]string{	63	a.audit(h.Tenant, "host.decommission", map[string]string{
59	"host_id": id, "remote": clientIP(r),	64	"host_id": id, "remote": clientIP(r),
60	"force": "true", "vms_purged": strconv.Itoa(purged),	65	"force": "true", "vms_purged": strconv.Itoa(gone.VMsPurged),
		66	"volumes_destroyed": strconv.Itoa(gone.VolumesDestroyed),
		67	"claims_unbound": strconv.Itoa(gone.ClaimsUnbound),
		68	"volume_ids": strings.Join(gone.VolumeIDs, ","),
61	})	69	})
62	if a.upgrader != nil {	70	if a.upgrader != nil {
63	a.upgrader.ClearAgentUpgrade(id)	71	a.upgrader.ClearAgentUpgrade(id)

internal/server/api/routes.go

@@ @@ -266,6 +266,32 @@ var routeTable = []Route{ @@
+		Doc:     "Revoke an exposure; its host closes the listener on the next converge.",
+		handler: (*API).handleDeleteExposure,
+	},
+	// Volume claims: durable raw block storage a tenant claims and attaches to
+	// a VM at create; the bytes outlive the VM. There is no update verb — size
+	// is immutable once bound, because a guest filesystem sits on it.
+	{
+		Method: "POST", Path: "/api/v1/volume-claims", Auth: AuthUser, Kind: KindJSON,
+		Request: (*types.CreateVolumeClaimRequest)(nil), Response: (*types.VolumeClaim)(nil), Success: http.StatusCreated,
+		Doc:     "Claim durable storage. Pending until the first VM naming it is created; that VM's host then holds the bytes, and every later VM using the claim is placed there.",
+		handler: (*API).handleCreateVolumeClaim,
+	},
+	{
+		Method: "GET", Path: "/api/v1/volume-claims", Auth: AuthUser, Kind: KindJSON,
+		Response: []types.VolumeClaim(nil), Success: http.StatusOK,
+		Doc:     "List the tenant's claims: where each is bound, which VM holds it, and whether its host has the file.",
+		handler: (*API).handleListVolumeClaims,
+	},
+	{
+		Method: "GET", Path: "/api/v1/volume-claims/{id}", Auth: AuthUser, Kind: KindJSON,
+		Response: (*types.VolumeClaim)(nil), Success: http.StatusOK,
+		Doc:     "One claim.",
+		handler: (*API).handleGetVolumeClaim,
+	},
+	{
+		Method: "DELETE", Path: "/api/v1/volume-claims/{id}", Auth: AuthUser, Kind: KindJSON, Success: http.StatusNoContent,
+		Doc:     "Delete a claim and the data behind it. Refused (409) while a VM holds it.",
+		handler: (*API).handleDeleteVolumeClaim,
+	},
+	// BYO per-tenant SSH user CAs: eitri stores only the CA pubkey and never
+	// holds a user signing key. Tenant-scoped (caller must act for {tenant}).

internal/server/api/routes_test.go

Old		New
@@ -34,7 +34,7 @@ func exemplarElem(t *testing.T, route Route, role string, v any) reflect.Type {
34	// `required` array for request schemas, so a type serving both roles would	34	// `required` array for request schemas, so a type serving both roles would
35	// get the wrong treatment on one of them).	35	// get the wrong treatment on one of them).
36	func TestRouteTable(t *testing.T) {	36	func TestRouteTable(t *testing.T) {
37	const wantRoutes = 35	37	const wantRoutes = 39
38	if len(routeTable) != wantRoutes {	38	if len(routeTable) != wantRoutes {
39	t.Fatalf("route table has %d entries, want %d — new endpoint? update this pin and cmd/eitri-apispec coverage together", len(routeTable), wantRoutes)	39	t.Fatalf("route table has %d entries, want %d — new endpoint? update this pin and cmd/eitri-apispec coverage together", len(routeTable), wantRoutes)
40	}	40	}

internal/server/api/testdata/create-vm-request.golden.json

Old		New
@@ -9,5 +9,9 @@
9	"vcpus": 4,	9	"vcpus": 4,
10	"mem_mb": 4096,	10	"mem_mb": 4096,
11	"disk_gb": 20,	11	"disk_gb": 20,
12	"network": "lan"	12	"network": "lan",
		13	"volume_claims": [
		14	"project-data",
		15	"scratch"
		16	]
13	}	17	}

internal/server/api/testdata/create-volume-claim-request.golden.json

@@ @@ -0,0 +1,4 @@ @@
+{
+  "name": "project-data",
+  "size_gb": 50
+}

internal/server/api/testdata/volume-claim.golden.json

@@ @@ -0,0 +1,10 @@ @@
+{
+  "id": "c-3456",
+  "name": "project-data",
+  "size_gb": 50,
+  "status": "pending",
+  "host_id": "",
+  "vm_id": "",
+  "present": null,
+  "created_at": "2026-07-27T12:05:00Z"
+}

internal/server/api/types/types.go

@@ @@ -276,6 +276,34 @@ type CreateVMRequest struct { @@
+	// no silent fallback, a VM that asked for the LAN either gets it or is
+	// never created.
+	Network string `json:"network"`
+	// VolumeClaims names this tenant's claims (ids or names) to attach after
+	// the root and seed disks, in this order: the first is /dev/vdc. A
+	// pending claim is bound to this VM's host; a bound one pins the VM there.
+	VolumeClaims []string `json:"volume_claims"`
+}
+// CreateVolumeClaimRequest is the POST /api/v1/volume-claims body.
+type CreateVolumeClaimRequest struct {
+	Name   string `json:"name"`
+	SizeGB int64  `json:"size_gb"`
+}
+// VolumeClaim is durable storage a tenant holds. Status is "pending" until
+// the first VM naming it is created, then "bound" to that VM's host for good.
+type VolumeClaim struct {
+	ID     string `json:"id"`
+	Name   string `json:"name"`
+	SizeGB int64  `json:"size_gb"`
+	Status string `json:"status"`
+	HostID string `json:"host_id"` // "" until bound
+	VMID   string `json:"vm_id"`   // "" when no VM holds it
+	// Present is what the claim's host last said about the file behind it, and
+	// null until it has said anything — a pending claim (no file to look for),
+	// a host that is not reporting, or a report that predates the volume. Null
+	// is not false: "nobody has looked" and "it is gone" are different answers
+	// and only one of them is alarming.
+	Present   *bool     `json:"present"`
+	CreatedAt time.Time `json:"created_at"`
+// Default VM sizes the control plane applies when a create request leaves a

internal/server/api/volumes.go

@@ @@ -0,0 +1,229 @@ @@
+package api
+import (
+	"database/sql"
+	"errors"
+	"net/http"
+	"regexp"
+	"strconv"
+	"github.com/a73x/eitri/internal/server/api/types"
+	"github.com/a73x/eitri/internal/server/registry"
+	"github.com/a73x/eitri/internal/server/store"
+)
+// claimName mirrors the VM name rule: DNS-label shaped, so a claim can be
+// named on the command line without quoting.
+var claimName = regexp.MustCompile(`^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$`)
+// maxClaimGB bounds a single claim; the host's disk is the real cap and
+// capacity admission judges it.
+const maxClaimGB = 4096
+func claimToWire(c store.VolumeClaim, present *bool) types.VolumeClaim {
+	status := "pending"
+	if c.BoundVolumeID != "" {
+		status = "bound"
+	}
+	return types.VolumeClaim{ID: c.ID, Name: c.Name, SizeGB: c.SizeGB, Status: status,
+		HostID: c.HostID, VMID: c.VMID, Present: present, CreatedAt: c.CreatedAt}
+}
+// presence answers "is the file there" with what the claim's host is saying
+// NOW, and nil whenever nothing is being said: an unbound claim (no file to
+// look for), a host that is not reporting, or a report that does not name the
+// volume. online is the caller's read of that host's registry entry.
+//
+// A host that has gone quiet says NOTHING rather than saying no — the same
+// reading every sibling admission takes of silence (see refuseBelowFloor).
+// Its last report is minutes old and the file it described may have been
+// reclaimed since; answering false there would raise an alarm about a disk
+// nobody has actually looked at.
+func presence(hs registry.HostState, online bool, c store.VolumeClaim) *bool {
+	if c.BoundVolumeID == "" || !online {
+		return nil
+	}
+	for _, v := range hs.Report.Volumes {
+		if v.VolumeID == c.BoundVolumeID {
+			p := v.Present
+			return &p
+		}
+	}
+	return nil
+}
+// presenceOf is presence for one claim read on its own, taking the registry
+// hit the list handler hoists out of its loop.
+func (a *API) presenceOf(c store.VolumeClaim) *bool {
+	if c.BoundVolumeID == "" {
+		return nil
+	}
+	hs, ok := a.reg.Get(c.HostID)
+	return presence(hs, ok && hs.Online, c)
+}
+func (a *API) handleCreateVolumeClaim(w http.ResponseWriter, r *http.Request) {
+	var req types.CreateVolumeClaimRequest
+	if !decodeJSON(w, r, &req) {
+		return
+	}
+	if !claimName.MatchString(req.Name) {
+		http.Error(w, "name must be 1-63 chars of [a-z0-9-], starting and ending alphanumeric", http.StatusBadRequest)
+		return
+	}
+	if req.SizeGB < 1 || req.SizeGB > maxClaimGB {
+		http.Error(w, "size_gb must be in [1, "+strconv.Itoa(maxClaimGB)+"]", http.StatusBadRequest)
+		return
+	}
+	tenant := principalFromContext(r).Tenant
+	c, err := a.st.CreateVolumeClaim(tenant, req.Name, req.SizeGB)
+	if err != nil {
+		if errors.Is(err, store.ErrClaimNameTaken) {
+			http.Error(w, "name already in use", http.StatusConflict)
+			return
+		}
+		http.Error(w, "internal error", http.StatusInternalServerError)
+		return
+	}
+	a.audit(tenant, "volume_claim.create", map[string]string{"claim_id": c.ID, "name": c.Name,
+		"size_gb": strconv.FormatInt(c.SizeGB, 10)})
+	a.notif.notify()
+	// A fresh claim is Pending: it names no host, so there is nothing that
+	// could have reported on it yet.
+	writeJSON(w, http.StatusCreated, claimToWire(c, nil))
+}
+func (a *API) handleListVolumeClaims(w http.ResponseWriter, r *http.Request) {
+	list, err := a.st.ListVolumeClaims(principalFromContext(r).Tenant)
+	if err != nil {
+		http.Error(w, "internal error", http.StatusInternalServerError)
+		return
+	}
+	// One registry read per HOST, not per claim. reg.Get deep-clones the
+	// host's whole report, and claims cluster on hosts by construction — every
+	// claim a VM was given is bound to that VM's host — so the per-claim read
+	// this replaces cloned the same report once for each of them.
+	type liveHost struct {
+		state  registry.HostState
+		online bool
+	}
+	hosts := make(map[string]liveHost)
+	out := make([]types.VolumeClaim, 0, len(list))
+	for _, c := range list {
+		if c.BoundVolumeID == "" {
+			out = append(out, claimToWire(c, nil))
+			continue
+		}
+		h, seen := hosts[c.HostID]
+		if !seen {
+			hs, ok := a.reg.Get(c.HostID)
+			h = liveHost{state: hs, online: ok && hs.Online}
+			hosts[c.HostID] = h
+		}
+		out = append(out, claimToWire(c, presence(h.state, h.online, c)))
+	}
+	writeJSON(w, http.StatusOK, out)
+}
+// ownedClaim reads one claim the caller may act on. A foreign claim answers
+// exactly like a missing one — existence is not leaked across tenants.
+func (a *API) ownedClaim(w http.ResponseWriter, r *http.Request) (store.VolumeClaim, bool) {
+	c, err := a.st.GetVolumeClaim(r.PathValue("id"))
+	switch {
+	case errors.Is(err, sql.ErrNoRows), err == nil && !mayActAs(principalFromContext(r), c.Tenant):
+		http.Error(w, "not found", http.StatusNotFound)
+		return store.VolumeClaim{}, false
+	case err != nil:
+		http.Error(w, "internal error", http.StatusInternalServerError)
+		return store.VolumeClaim{}, false
+	}
+	return c, true
+}
+func (a *API) handleGetVolumeClaim(w http.ResponseWriter, r *http.Request) {
+	c, ok := a.ownedClaim(w, r)
+	if !ok {
+		return
+	}
+	writeJSON(w, http.StatusOK, claimToWire(c, a.presenceOf(c)))
+}
+func (a *API) handleDeleteVolumeClaim(w http.ResponseWriter, r *http.Request) {
+	c, ok := a.ownedClaim(w, r)
+	if !ok {
+		return
+	}
+	if err := a.st.TombstoneVolumeClaim(c.ID); err != nil {
+		var attached *store.ClaimAttachedError
+		switch {
+		case errors.As(err, &attached):
+			http.Error(w, "volume claim is attached to vm "+attached.VMID+"; delete that VM first", http.StatusConflict)
+		case errors.Is(err, sql.ErrNoRows):
+			http.Error(w, "not found", http.StatusNotFound)
+		default:
+			http.Error(w, "internal error", http.StatusInternalServerError)
+		}
+		return
+	}
+	a.audit(c.Tenant, "volume_claim.delete", map[string]string{"claim_id": c.ID, "name": c.Name, "volume_id": c.BoundVolumeID})
+	// A bound claim's host has to be told; an unbound one has no host to tell.
+	if c.HostID != "" {
+		a.hub.Poke(c.HostID)
+	}
+	a.notif.notify()
+	w.WriteHeader(http.StatusNoContent)
+}
+// resolveClaims turns a create's claim ids-or-names into ids, in order, and
+// sums the sizes of the PENDING ones — the disk this create newly commits;
+// a bound claim is already in the host's held total (see CommittedOnHost).
+//
+// It refuses a reference that names nothing, and one claim named twice. The
+// second refusal is this layer's job rather than the store's: the store would
+// see two attach attempts and answer "already attached to <this vm>", naming
+// the VM being created as though a race had happened. A caller who wrote a
+// name twice deserves to read that they wrote it twice.
+//
+// ID BEFORE NAME, in two maps rather than one. Nothing stops a tenant naming
+// one claim after another's 32-hex id, and a single keyspace would let that
+// name shadow the id it copies — silently attaching the wrong disk, with the
+// later-created claim winning because it overwrites. An id is the unambiguous
+// handle, so an id that resolves wins outright and no name can displace it.
+//
+// msg/code is "" / 0 when the create may proceed.
+func (a *API) resolveClaims(tenant string, refs []string) (ids []string, pendingGB int64, msg string, code int, err error) {
+	if len(refs) == 0 {
+		return nil, 0, "", 0, nil
+	}
+	list, err := a.st.ListVolumeClaims(tenant)
+	if err != nil {
+		return nil, 0, "", 0, err
+	}
+	byID := make(map[string]store.VolumeClaim, len(list))
+	byName := make(map[string]store.VolumeClaim, len(list))
+	for _, c := range list {
+		byID[c.ID] = c
+		byName[c.Name] = c
+	}
+	seen := make(map[string]bool, len(refs))
+	for _, ref := range refs {
+		c, ok := byID[ref]
+		if !ok {
+			c, ok = byName[ref]
+		}
+		if !ok {
+			return nil, 0, "unknown volume claim " + ref, http.StatusNotFound, nil
+		}
+		if seen[c.ID] {
+			// By NAME, whichever way it was referenced: naming a claim once by
+			// id and once by name is the way this mistake actually happens.
+			return nil, 0, "volume claim " + c.Name + " named twice", http.StatusBadRequest, nil
+		}
+		seen[c.ID] = true
+		ids = append(ids, c.ID)
+		if c.BoundVolumeID == "" {
+			pendingGB += c.SizeGB
+		}
+	}
+	return ids, pendingGB, "", 0, nil
+}

internal/server/api/volumes_test.go

@@ @@ -0,0 +1,324 @@ @@
+package api
+import (
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"sort"
+	"sync"
+	"testing"
+	"github.com/stretchr/testify/require"
+	"github.com/a73x/eitri/internal/server/registry"
+	"github.com/a73x/eitri/internal/server/release"
+	"github.com/a73x/eitri/internal/server/store"
+)
+func claimPost(t *testing.T, ts *httptest.Server, name string, size int64) map[string]any {
+	t.Helper()
+	resp := do(t, "POST", ts.URL+"/api/v1/volume-claims", testPAT, map[string]any{"name": name, "size_gb": size})
+	require.Equal(t, 201, resp.StatusCode)
+	var out map[string]any
+	require.NoError(t, json.NewDecoder(resp.Body).Decode(&out))
+	return out
+}
+func getClaim(t *testing.T, ts *httptest.Server, id string) (int, map[string]any) {
+	t.Helper()
+	resp := do(t, "GET", ts.URL+"/api/v1/volume-claims/"+id, testPAT, nil)
+	var out map[string]any
+	_ = json.NewDecoder(resp.Body).Decode(&out)
+	return resp.StatusCode, out
+}
+// floorVolumes makes the placeholder-free floor the tests drive against: the
+// shipped Since is a tag no test host reports, so every test that wants a
+// host ABOVE the floor names its own.
+func floorVolumes(t *testing.T, since string) {
+	t.Helper()
+	prev := volumesFeature
+	volumesFeature = release.Feature{Name: "volumes", Since: since}
+	t.Cleanup(func() { volumesFeature = prev })
+}
+// onlineAt is the pair of facts that make a host judgeable: the version its
+// agent named in the Hello, and a report, which is what makes it Online.
+func onlineAt(reg *registry.Registry, hostID, v string) {
+	reg.SetAgentVersion(hostID, v)
+	reg.UpdateReport(hostID, registry.Report{})
+}
+func TestVolumeClaimCreateListGetDelete(t *testing.T) {
+	ts, _, _, _, _ := newServer(t)
+	c := claimPost(t, ts, "data", 5)
+	require.Equal(t, "pending", c["status"])
+	require.Nil(t, c["present"], "nothing has reported on a claim no host holds")
+	resp := do(t, "GET", ts.URL+"/api/v1/volume-claims", testPAT, nil)
+	require.Equal(t, 200, resp.StatusCode)
+	require.Len(t, decodeJSONKeys(t, resp), 1)
+	code, _ := getClaim(t, ts, c["id"].(string))
+	require.Equal(t, 200, code)
+	require.Equal(t, 204, do(t, "DELETE", ts.URL+"/api/v1/volume-claims/"+c["id"].(string), testPAT, nil).StatusCode)
+	code, _ = getClaim(t, ts, c["id"].(string))
+	require.Equal(t, 404, code)
+}
+func TestVolumeClaimValidation(t *testing.T) {
+	ts, _, _, _, _ := newServer(t)
+	for _, body := range []map[string]any{
+		{"name": "", "size_gb": 1}, {"name": "x", "size_gb": 0}, {"name": "x", "size_gb": -1},
+		{"name": "bad name", "size_gb": 1}, {"name": "x", "size_gb": 4097},
+	} {
+		require.Equal(t, 400, do(t, "POST", ts.URL+"/api/v1/volume-claims", testPAT, body).StatusCode, "%v", body)
+	}
+	claimPost(t, ts, "dup", 1)
+	require.Equal(t, 409, do(t, "POST", ts.URL+"/api/v1/volume-claims", testPAT, map[string]any{"name": "dup", "size_gb": 1}).StatusCode)
+}
+func TestVolumeClaimsAreTenantScoped(t *testing.T) {
+	ts, st, _, _, _ := newServer(t)
+	c := claimPost(t, ts, "data", 5)
+	other := patForOtherTenant(t, st)
+	id := c["id"].(string)
+	require.Equal(t, 404, do(t, "GET", ts.URL+"/api/v1/volume-claims/"+id, other, nil).StatusCode)
+	require.Equal(t, 404, do(t, "DELETE", ts.URL+"/api/v1/volume-claims/"+id, other, nil).StatusCode)
+	require.Len(t, decodeJSONKeys(t, do(t, "GET", ts.URL+"/api/v1/volume-claims", other, nil)), 0)
+}
+// A claim belonging to somebody else is not a claim this tenant can mount, on
+// its own host or anywhere. resolveClaims reads the CALLER's claims only, so a
+// foreign id resolves to nothing and answers exactly like a typo — existence
+// is not leaked across tenants here any more than on the claim routes.
+func TestCreateVMCannotNameAnotherTenantsClaim(t *testing.T) {
+	ts, st, _, reg, _ := newServer(t)
+	floorVolumes(t, "v0.0.7")
+	hostID := enroll(t, ts)["host_id"] // the CALLER's own host
+	onlineAt(reg, hostID, "v0.0.7")
+	other := patForOtherTenant(t, st)
+	resp := do(t, "POST", ts.URL+"/api/v1/volume-claims", other, map[string]any{"name": "theirs", "size_gb": 5})
+	require.Equal(t, 201, resp.StatusCode)
+	var theirs map[string]any
+	require.NoError(t, json.NewDecoder(resp.Body).Decode(&theirs))
+	theirID := theirs["id"].(string)
+	resp = do(t, "POST", ts.URL+"/api/v1/vms", testPAT, map[string]any{
+		"host_id": hostID, "name": "thief", "volume_claims": []string{theirID}})
+	require.Equal(t, 404, resp.StatusCode)
+	body, _ := io.ReadAll(resp.Body)
+	require.Contains(t, string(body), "unknown volume claim")
+	// By name too, and the name is one this tenant does not have either.
+	resp = do(t, "POST", ts.URL+"/api/v1/vms", testPAT, map[string]any{
+		"host_id": hostID, "name": "thief", "volume_claims": []string{"theirs"}})
+	require.Equal(t, 404, resp.StatusCode)
+}
+// A claim NAMED after another claim's id must not shadow it. Names are the
+// tenant's to choose and ids are 32 lowercase hex — a legal name — so a single
+// keyspace would let the later claim win the lookup and quietly attach the
+// wrong disk. The id is the unambiguous handle and wins.
+func TestCreateVMResolvesAClaimIDBeforeAName(t *testing.T) {
+	ts, _, _, reg, _ := newServer(t)
+	floorVolumes(t, "v0.0.7")
+	hostID := enroll(t, ts)["host_id"]
+	onlineAt(reg, hostID, "v0.0.7")
+	x := claimPost(t, ts, "alpha", 1)
+	xID := x["id"].(string)
+	// Created AFTER x, so in a single map its name would overwrite x's id.
+	y := claimPost(t, ts, xID, 2)
+	yID := y["id"].(string)
+	require.Equal(t, 201, do(t, "POST", ts.URL+"/api/v1/vms", testPAT, map[string]any{
+		"host_id": hostID, "name": "holder", "volume_claims": []string{xID}}).StatusCode)
+	_, gotX := getClaim(t, ts, xID)
+	require.Equal(t, "bound", gotX["status"], "the claim whose ID was named is the one attached")
+	require.NotEmpty(t, gotX["vm_id"])
+	_, gotY := getClaim(t, ts, yID)
+	require.Equal(t, "pending", gotY["status"], "the claim that merely borrowed that id as its name is untouched")
+	require.Equal(t, "", gotY["vm_id"])
+}
+// presence is the one field on a claim the control plane cannot decide for
+// itself, so every way of knowing nothing has to read as null rather than as
+// "the file is gone".
+func TestPresenceSaysNothingUntilAHostDoes(t *testing.T) {
+	bound := store.VolumeClaim{ID: "c-1", HostID: "h-1", BoundVolumeID: "vol-1"}
+	reporting := registry.HostState{Report: registry.Report{
+		Volumes: []registry.VolumeStatus{{VolumeID: "vol-1", Present: true, SizeGB: 5}}}}
+	gone := registry.HostState{Report: registry.Report{
+		Volumes: []registry.VolumeStatus{{VolumeID: "vol-1", Present: false, SizeGB: 5}}}}
+	t.Run("unbound claim", func(t *testing.T) {
+		require.Nil(t, presence(reporting, true, store.VolumeClaim{ID: "c-2"}))
+	})
+	t.Run("host not reporting", func(t *testing.T) {
+		require.Nil(t, presence(reporting, false, bound),
+			"a host that has gone quiet says nothing; its last report may be minutes stale")
+	})
+	t.Run("report does not name the volume", func(t *testing.T) {
+		require.Nil(t, presence(registry.HostState{}, true, bound))
+	})
+	t.Run("host says it is there", func(t *testing.T) {
+		got := presence(reporting, true, bound)
+		require.NotNil(t, got)
+		require.True(t, *got)
+	})
+	t.Run("host says it is gone", func(t *testing.T) {
+		got := presence(gone, true, bound)
+		require.NotNil(t, got)
+		require.False(t, *got, "false is a real answer and must survive; only silence is null")
+	})
+}
+// Admission steps 1, 2, 3 and 6 of the spec, plus delete-while-attached.
+func TestCreateVMWithVolumeClaims(t *testing.T) {
+	ts, st, _, reg, _ := newServer(t)
+	floorVolumes(t, "v0.0.7")
+	host := enroll(t, ts)
+	hostID := host["host_id"]
+	onlineAt(reg, hostID, "v0.0.7")
+	c := claimPost(t, ts, "data", 5)
+	cid := c["id"].(string)
+	create := func(host, name string, claims ...string) *http.Response {
+		return do(t, "POST", ts.URL+"/api/v1/vms", testPAT, map[string]any{"host_id": host, "name": name, "volume_claims": claims})
+	}
+	// 1. unknown claim → 404 naming it
+	resp := create(hostID, "a", "nope")
+	require.Equal(t, 404, resp.StatusCode)
+	body, _ := io.ReadAll(resp.Body)
+	require.Contains(t, string(body), "nope")
+	// happy path binds; a claim may be named by name as well as id
+	require.Equal(t, 201, create(hostID, "a", "data").StatusCode)
+	_, got := getClaim(t, ts, cid)
+	require.Equal(t, "bound", got["status"])
+	require.Equal(t, hostID, got["host_id"])
+	vmID := got["vm_id"].(string)
+	require.NotEmpty(t, vmID)
+	// 2. attached → 409 naming the holder
+	resp = create(hostID, "b", cid)
+	require.Equal(t, 409, resp.StatusCode)
+	body, _ = io.ReadAll(resp.Body)
+	require.Contains(t, string(body), vmID)
+	// delete while attached → 409
+	require.Equal(t, 409, do(t, "DELETE", ts.URL+"/api/v1/volume-claims/"+cid, testPAT, nil).StatusCode)
+	// 3. pinned to another host → 409 naming the host
+	require.Equal(t, 204, do(t, "DELETE", ts.URL+"/api/v1/vms/"+vmID, testPAT, nil).StatusCode)
+	require.NoError(t, st.HardDeleteVM(vmID, hostID))
+	h2 := enroll(t, ts)["host_id"]
+	onlineAt(reg, h2, "v0.0.7")
+	resp = create(h2, "c", cid)
+	require.Equal(t, 409, resp.StatusCode)
+	body, _ = io.ReadAll(resp.Body)
+	require.Contains(t, string(body), hostID)
+	// ... and on the pinned host it re-attaches.
+	require.Equal(t, 201, create(hostID, "c", cid).StatusCode)
+}
+// One claim named twice is refused before the store sees it: the store's own
+// refusal would say the claim is attached to the VM being created, which reads
+// as a race that never happened.
+func TestCreateVMRefusesAClaimNamedTwice(t *testing.T) {
+	ts, _, _, reg, _ := newServer(t)
+	floorVolumes(t, "v0.0.7")
+	hostID := enroll(t, ts)["host_id"]
+	onlineAt(reg, hostID, "v0.0.7")
+	c := claimPost(t, ts, "data", 5)
+	resp := do(t, "POST", ts.URL+"/api/v1/vms", testPAT, map[string]any{
+		"host_id": hostID, "name": "double", "volume_claims": []string{"data", c["id"].(string)}})
+	require.Equal(t, 400, resp.StatusCode)
+	body, _ := io.ReadAll(resp.Body)
+	require.Contains(t, string(body), "volume claim data named twice",
+		"the refusal names the claim, not the VM the store would have blamed")
+	// And nothing was bound on the way to the refusal.
+	_, got := getClaim(t, ts, c["id"].(string))
+	require.Equal(t, "pending", got["status"])
+}
+// 4. The floor judges a silent host for a volume-bearing create only.
+func TestCreateVMWithVolumeClaimsFloorsTheAgent(t *testing.T) {
+	ts, _, _, reg, _ := newServer(t)
+	floorVolumes(t, "v0.0.7")
+	hostID := enroll(t, ts)["host_id"]
+	claimPost(t, ts, "data", 5)
+	create := func(name string, claims ...string) *http.Response {
+		return do(t, "POST", ts.URL+"/api/v1/vms", testPAT, map[string]any{"host_id": hostID, "name": name, "volume_claims": claims})
+	}
+	resp := create("a", "data")
+	require.Equal(t, 409, resp.StatusCode, "a silent host is refused a volume-bearing create")
+	body, _ := io.ReadAll(resp.Body)
+	require.Contains(t, string(body), "volumes")
+	require.Contains(t, string(body), "upgrade-agent")
+	require.Equal(t, 201, create("plain").StatusCode, "a create with no claims is not floored")
+	onlineAt(reg, hostID, "v0.0.6")
+	require.Equal(t, 409, create("b", "data").StatusCode, "a pre-volumes agent is refused")
+	onlineAt(reg, hostID, "v0.0.7")
+	require.Equal(t, 201, create("b", "data").StatusCode)
+}
+// Two creates race for one pending claim: exactly one wins.
+func TestTwoCreatesRaceForOneClaim(t *testing.T) {
+	ts, _, _, reg, _ := newServer(t)
+	floorVolumes(t, "v0.0.7")
+	hostID := enroll(t, ts)["host_id"]
+	onlineAt(reg, hostID, "v0.0.7")
+	claimPost(t, ts, "data", 1)
+	codes := make(chan int, 2)
+	var wg sync.WaitGroup
+	for _, name := range []string{"r1", "r2"} {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			codes <- do(t, "POST", ts.URL+"/api/v1/vms", testPAT, map[string]any{"host_id": hostID, "name": name, "volume_claims": []string{"data"}}).StatusCode
+		}()
+	}
+	wg.Wait()
+	close(codes)
+	var got []int
+	for c := range codes {
+		got = append(got, c)
+	}
+	sort.Ints(got)
+	require.Equal(t, []int{201, 409}, got)
+}
+// A claim its host has reported on carries that host's answer: present is the
+// one field on a claim nothing in the control plane can decide for itself.
+func TestVolumeClaimReportsWhatTheHostFound(t *testing.T) {
+	ts, st, _, reg, _ := newServer(t)
+	floorVolumes(t, "v0.0.7")
+	hostID := enroll(t, ts)["host_id"]
+	onlineAt(reg, hostID, "v0.0.7")
+	c := claimPost(t, ts, "data", 5)
+	cid := c["id"].(string)
+	require.Equal(t, 201, do(t, "POST", ts.URL+"/api/v1/vms", testPAT, map[string]any{
+		"host_id": hostID, "name": "holder", "volume_claims": []string{"data"}}).StatusCode)
+	// Bound, but the host has not named the volume yet: null, not false.
+	_, got := getClaim(t, ts, cid)
+	require.Nil(t, got["present"])
+	vols, err := st.ListVolumesForHost(hostID)
+	require.NoError(t, err)
+	require.Len(t, vols, 1)
+	reg.SetAgentVersion(hostID, "v0.0.7")
+	reg.UpdateReport(hostID, registry.Report{
+		Volumes: []registry.VolumeStatus{{VolumeID: vols[0].ID, Present: true, SizeGB: 5}}})
+	_, got = getClaim(t, ts, cid)
+	require.Equal(t, true, got["present"])
+}

internal/server/api/wire_golden_test.go

Old		New
@@ -183,6 +183,7 @@ func TestWireGolden(t *testing.T) {
183	MemMB: 4096,	183	MemMB: 4096,
184	DiskGB: 20,	184	DiskGB: 20,
185	Network: "lan",	185	Network: "lan",
		186	VolumeClaims: []string{"project-data", "scratch"},
186	})	187	})
187		188
188	goldenCheck(t, "patch-vm-request", types.PatchVMRequest{	189	goldenCheck(t, "patch-vm-request", types.PatchVMRequest{
@@ -305,4 +306,22 @@ func TestWireGolden(t *testing.T) {
305	Sessions: &types.ExposureSessions{Active: 7, Refused: 12, Dropped: 3},	306	Sessions: &types.ExposureSessions{Active: 7, Refused: 12, Dropped: 3},
306	CreatedAt: base.Add(4 * time.Minute),	307	CreatedAt: base.Add(4 * time.Minute),
307	}})	308	}})
		309
		310	goldenCheck(t, "create-volume-claim-request", types.CreateVolumeClaimRequest{
		311	Name: "project-data",
		312	SizeGB: 50,
		313	})
		314
		315	// The PENDING exemplar, deliberately: it is the shape with the most null in
		316	// it, and each null is load-bearing. host_id and vm_id are "" because
		317	// nothing has placed or attached it, and present is null because no host
		318	// has been asked about a volume that does not exist yet — null is not
		319	// false, and a client must not render "missing" for "unreported".
		320	goldenCheck(t, "volume-claim", types.VolumeClaim{
		321	ID: "c-3456",
		322	Name: "project-data",
		323	SizeGB: 50,
		324	Status: "pending",
		325	CreatedAt: base.Add(5 * time.Minute),
		326	})
308	}	327	}

internal/server/registry/registry.go

Old		New
@@ -69,10 +69,22 @@ type ExposureStatus struct {
69	// See ExposureSessions in the proto for why the two totals are not gauges.	69	// See ExposureSessions in the proto for why the two totals are not gauges.
70	type ExposureSessions struct{ Active, Refused, Dropped int64 }	70	type ExposureSessions struct{ Active, Refused, Dropped int64 }
71		71
		72	// VolumeStatus is what a host found on disk for one volume id: whether the
		73	// file is there, and how big it is. Never persisted — like ExposureStatus it
		74	// lives only while the host is connected. It is also the only evidence a
		75	// tombstoned volume's row may be reaped on, so Present=false is a statement
		76	// the agent makes, never an absence the server infers.
		77	type VolumeStatus struct {
		78	VolumeID string
		79	Present bool
		80	SizeGB int64
		81	}
		82
72	type Report struct {	83	type Report struct {
73	VMs []VMStatus	84	VMs []VMStatus
74	Quarantined []QuarantinedVM	85	Quarantined []QuarantinedVM
75	Exposures []ExposureStatus	86	Exposures []ExposureStatus
		87	Volumes []VolumeStatus
76	Capacity Capacity	88	Capacity Capacity
77	Metrics Metrics	89	Metrics Metrics
78	FenceViolation bool	90	FenceViolation bool
@@ -187,6 +199,7 @@ func (r *Registry) Get(hostID string) (HostState, bool) {
187	// Deep-copy slices so callers cannot corrupt registry state.	199	// Deep-copy slices so callers cannot corrupt registry state.
188	st.Report.VMs = slices.Clone(st.Report.VMs)	200	st.Report.VMs = slices.Clone(st.Report.VMs)
189	st.Report.Exposures = slices.Clone(st.Report.Exposures)	201	st.Report.Exposures = slices.Clone(st.Report.Exposures)
		202	st.Report.Volumes = slices.Clone(st.Report.Volumes)
190	quarantined := slices.Clone(st.Report.Quarantined)	203	quarantined := slices.Clone(st.Report.Quarantined)
191	for i := range quarantined {	204	for i := range quarantined {
192	quarantined[i].VMSpecJSON = bytes.Clone(quarantined[i].VMSpecJSON)	205	quarantined[i].VMSpecJSON = bytes.Clone(quarantined[i].VMSpecJSON)

internal/server/store/exposures_test.go

Old		New
@@ -304,9 +304,9 @@ func TestForceRemovingAHostDestroysItsExposures(t *testing.T) {
304	e, err := s.CreateExposure(vm.ID, 8080, 0, "tcp")	304	e, err := s.CreateExposure(vm.ID, 8080, 0, "tcp")
305	require.NoError(t, err)	305	require.NoError(t, err)
306		306
307	purged, err := s.ForceRemoveHost(h.ID)	307	gone, err := s.ForceRemoveHost(h.ID)
308	require.NoError(t, err)	308	require.NoError(t, err)
309	assert.Equal(t, 1, purged)	309	assert.Equal(t, 1, gone.VMsPurged)
310		310
311	_, err = s.GetExposure(e.ID)	311	_, err = s.GetExposure(e.ID)
312	assert.ErrorIs(t, err, sql.ErrNoRows)	312	assert.ErrorIs(t, err, sql.ErrNoRows)

internal/server/store/store.go

Old		New
@@ -31,6 +31,13 @@ var ErrHostNotFound = errors.New("host not found")
31		31
32	var ErrHostNotEnrolled = errors.New("host not accepting new VMs")	32	var ErrHostNotEnrolled = errors.New("host not accepting new VMs")
33		33
		34	// ErrHostHoldsVolumes is returned by RemoveHost while volume rows remain on the
		35	// host. It is a distinct error because it is a STUCK decommission, not a slow
		36	// one: VMs drain themselves, volumes do not — a volume outlives every guest by
		37	// design, so nothing releases it until its claim is deleted. The graceful sweep
		38	// would otherwise retry forever in silence. Wrapped with the count.
		39	var ErrHostHoldsVolumes = errors.New("host still holds volumes")
		40
34	// SystemTenant is the audit scope for events with no resolvable tenant — a	41	// SystemTenant is the audit scope for events with no resolvable tenant — a
35	// denied enroll attempt, a reap ack racing its host's deletion. It is NOT a	42	// denied enroll attempt, a reap ack racing its host's deletion. It is NOT a
36	// tenant: no tenants row exists for it, no principal can ever hold it (JIT	43	// tenant: no tenants row exists for it, no principal can ever hold it (JIT
@@ -114,6 +121,14 @@ type VM struct {
114	// makes nil unambiguously "unrecorded" rather than "trusts nothing", which	121	// makes nil unambiguously "unrecorded" rather than "trusts nothing", which
115	// is the distinction every reader of this field depends on.	122	// is the distinction every reader of this field depends on.
116	TrustedCAs []TrustedCA	123	TrustedCAs []TrustedCA
		124	// VolumeClaimIDs is INPUT TO CreateVM ONLY: the claims this VM asks to
		125	// mount, already resolved from names to ids by the API. No read ever fills
		126	// it — the attachment table is the record, and VolumeIDs is how it reads
		127	// back.
		128	VolumeClaimIDs []string
		129	// VolumeIDs is the read side: the volumes attached to this VM, in the order
		130	// it named their claims, which is the order the guest sees the devices in.
		131	VolumeIDs []string
117	// The address the gate and every exposure target is AssignedIP.	132	// The address the gate and every exposure target is AssignedIP.
118	CreatedAt time.Time	133	CreatedAt time.Time
119	DeletedAt *time.Time	134	DeletedAt *time.Time
@@ -272,6 +287,35 @@ CREATE TABLE IF NOT EXISTS exposures (
272	);	287	);
273	CREATE UNIQUE INDEX IF NOT EXISTS exposures_host_port_proto ON exposures(host_id, host_port, protocol);	288	CREATE UNIQUE INDEX IF NOT EXISTS exposures_host_port_proto ON exposures(host_id, host_port, protocol);
274		289
		290	-- Volumes. A claim is the tenant's request for storage; a volume is the
		291	-- fleet's placement of bytes on one host, created by the first VM that names
		292	-- the claim. "Unbound" is the absence of a volumes row, never a NULL.
		293	CREATE TABLE IF NOT EXISTS volume_claims (
		294	id TEXT PRIMARY KEY,
		295	tenant TEXT NOT NULL REFERENCES tenants(id),
		296	name TEXT NOT NULL,
		297	size_gb INTEGER NOT NULL,
		298	bound_volume_id TEXT,
		299	created_at DATETIME NOT NULL,
		300	deleted_at DATETIME
		301	);
		302	CREATE UNIQUE INDEX IF NOT EXISTS volume_claims_tenant_name ON volume_claims(tenant, name) WHERE deleted_at IS NULL;
		303	CREATE TABLE IF NOT EXISTS volumes (
		304	id TEXT PRIMARY KEY,
		305	host_id TEXT NOT NULL REFERENCES hosts(id),
		306	claim_id TEXT NOT NULL REFERENCES volume_claims(id),
		307	size_gb INTEGER NOT NULL,
		308	created_at DATETIME NOT NULL,
		309	deleted_at DATETIME
		310	);
		311	-- The database refuses a double attach; nothing above it has to remember to.
		312	-- Hard-deleting a reaped VM takes its attachment with it, like exposures.
		313	CREATE TABLE IF NOT EXISTS volume_attachments (
		314	claim_id TEXT NOT NULL REFERENCES volume_claims(id),
		315	vm_id TEXT NOT NULL REFERENCES vms(id) ON DELETE CASCADE
		316	);
		317	CREATE UNIQUE INDEX IF NOT EXISTS volume_attachments_claim ON volume_attachments(claim_id);
		318
275	-- Console sessions. Server-side so revocation works and restarts keep	319	-- Console sessions. Server-side so revocation works and restarts keep
276	-- users signed in. id holds the SHA-256 of the session value, never the value	320	-- users signed in. id holds the SHA-256 of the session value, never the value
277	-- itself (see CreateSession); expiry enforced on read.	321	-- itself (see CreateSession); expiry enforced on read.
@@ -903,6 +947,14 @@ func (s *Store) CreateVM(vm VM) error {
903	return fmt.Errorf("insert vm: %w", err)	947	return fmt.Errorf("insert vm: %w", err)
904	}	948	}
905		949
		950	// Claims bind HERE, in the same tx that places the VM: a claim the fleet
		951	// cannot honour (another tenant's, already held, or already living on
		952	// another host) takes the whole create down with it, so a refused VM
		953	// leaves no row and no half-bound claim behind.
		954	if err := bindClaims(tx, vm); err != nil {
		955	return err
		956	}
		957
906	if err := bumpEpoch(tx); err != nil {	958	if err := bumpEpoch(tx); err != nil {
907	return fmt.Errorf("bump epoch: %w", err)	959	return fmt.Errorf("bump epoch: %w", err)
908	}	960	}
@@ -1076,6 +1128,11 @@ func (c Commitment) Held() Alloc {
1076	// only, so during a teardown Held() reads higher. Erring that way is the safe	1128	// only, so during a teardown Held() reads higher. Erring that way is the safe
1077	// direction for admission: refusing a VM for a bed that is being stripped costs	1129	// direction for admission: refusing a VM for a bed that is being stripped costs
1078	// a retry, admitting one into it costs a failed VM.	1130	// a retry, admitting one into it costs a failed VM.
		1131	//
		1132	// Live volumes on the host count into Live.DiskGB too. A bound volume is disk
		1133	// the host has committed WITH OR WITHOUT A VM: the bytes outlive every guest
		1134	// that mounts them, which is the point of a volume, so the space stays spoken
		1135	// for between one VM's destroy and the next VM's create.
1079	func (s *Store) CommittedOnHost(hostID string) (Commitment, error) {	1136	func (s *Store) CommittedOnHost(hostID string) (Commitment, error) {
1080	var c Commitment	1137	var c Commitment
1081	err := s.db.QueryRow(`	1138	err := s.db.QueryRow(`
@@ -1090,7 +1147,21 @@ func (s *Store) CommittedOnHost(hostID string) (Commitment, error) {
1090	&c.Live.VCPUs, &c.Live.MemMB, &c.Live.DiskGB,	1147	&c.Live.VCPUs, &c.Live.MemMB, &c.Live.DiskGB,
1091	&c.Pending.VCPUs, &c.Pending.MemMB, &c.Pending.DiskGB,	1148	&c.Pending.VCPUs, &c.Pending.MemMB, &c.Pending.DiskGB,
1092	&c.PendingVMs)	1149	&c.PendingVMs)
1093	return c, err	1150	if err != nil {
		1151	return c, err
		1152	}
		1153	// Only live volumes. A tombstoned one is a reclaim in flight and its file
		1154	// may still be on the disk, exactly like a tombstoned VM's — the honest
		1155	// place for it is Pending, and it is not counted at all until something
		1156	// asks for that distinction.
		1157	var vol int64
		1158	if err := s.db.QueryRow(
		1159	`SELECT COALESCE(SUM(size_gb),0) FROM volumes WHERE host_id=? AND deleted_at IS NULL`, hostID,
		1160	).Scan(&vol); err != nil {
		1161	return c, err
		1162	}
		1163	c.Live.DiskGB += vol
		1164	return c, nil
1094	}	1165	}
1095		1166
1096	// AuditEntry is one row of the append-only audit trail.	1167	// AuditEntry is one row of the append-only audit trail.
@@ -1320,9 +1391,16 @@ func (s *Store) TenantForUserCA(pubkey string) (tenant string, ok bool, err erro
1320	}	1391	}
1321		1392
1322	// RemoveHost finalizes decommission: it deletes the host row, refusing	1393	// RemoveHost finalizes decommission: it deletes the host row, refusing
1323	// (in-transaction) while any VM rows remain for the host (not yet reaped).	1394	// (in-transaction) while any VM rows remain for the host (not yet reaped), and
		1395	// while any volume rows do — live or tombstoned, because either way the bytes
		1396	// are still on that disk.
1324	// Nothing is recycled — next_cidr_index only moves forward, so a removed host's	1397	// Nothing is recycled — next_cidr_index only moves forward, so a removed host's
1325	// subnet is retired with it rather than handed to the next host to enroll.	1398	// subnet is retired with it rather than handed to the next host to enroll.
		1399	//
		1400	// A held volume is a different refusal from an undrained VM and returns
		1401	// ErrHostHoldsVolumes to say so. A VM drains on its own; a volume never does,
		1402	// because outliving its guests is the whole point of one. The operator has to
		1403	// delete the claims, let the agent reclaim them, and the sweep then succeeds.
1326	func (s *Store) RemoveHost(id string) error {	1404	func (s *Store) RemoveHost(id string) error {
1327	tx, err := s.db.Begin()	1405	tx, err := s.db.Begin()
1328	if err != nil {	1406	if err != nil {
@@ -1330,13 +1408,18 @@ func (s *Store) RemoveHost(id string) error {
1330	}	1408	}
1331	defer tx.Rollback()	1409	defer tx.Rollback()
1332		1410
1333	var n int	1411	var n, vols int
1334	if err := tx.QueryRow(`SELECT COUNT(*) FROM vms WHERE host_id=?`, id).Scan(&n); err != nil {	1412	if err := tx.QueryRow(`SELECT
		1413	(SELECT COUNT(*) FROM vms WHERE host_id=?),
		1414	(SELECT COUNT(*) FROM volumes WHERE host_id=?)`, id, id).Scan(&n, &vols); err != nil {
1335	return fmt.Errorf("count host vms: %w", err)	1415	return fmt.Errorf("count host vms: %w", err)
1336	}	1416	}
1337	if n > 0 {	1417	if n > 0 {
1338	return fmt.Errorf("host %s still has %d VM(s); not drained", id, n)	1418	return fmt.Errorf("host %s still has %d VM(s); not drained", id, n)
1339	}	1419	}
		1420	if vols > 0 {
		1421	return fmt.Errorf("host %s still has %d volume(s); delete their claims first: %w", id, vols, ErrHostHoldsVolumes)
		1422	}
1340		1423
1341	res, err := tx.Exec(`DELETE FROM hosts WHERE id=?`, id)	1424	res, err := tx.Exec(`DELETE FROM hosts WHERE id=?`, id)
1342	if err != nil {	1425	if err != nil {
@@ -1351,16 +1434,43 @@ func (s *Store) RemoveHost(id string) error {
1351	return tx.Commit()	1434	return tx.Commit()
1352	}	1435	}
1353		1436
		1437	// ForceRemoval is the tally a forced host removal returns: what it destroyed
		1438	// and what it handed back. The one path that loses data on purpose has to be
		1439	// able to say what it lost, so the audit row can say it too — "the host is
		1440	// gone" is not an answer to "which volumes went with it".
		1441	//
		1442	// VolumeIDs names the destroyed volumes, in the order the store lists them
		1443	// (by id), so the same removal reports the same way twice. VMsPurged counts VM
		1444	// rows, live and tombstoned alike. ClaimsUnbound counts only the claims that
		1445	// LIVE ON as Pending: a claim already tombstoned here is unbound and then
		1446	// deleted outright, and nothing is waiting for it afterwards.
		1447	type ForceRemoval struct {
		1448	VMsPurged int
		1449	VolumesDestroyed int
		1450	ClaimsUnbound int
		1451	VolumeIDs []string
		1452	}
		1453
1354	// ForceRemoveHost finalizes a host whose agent will never drain it (dead	1454	// ForceRemoveHost finalizes a host whose agent will never drain it (dead
1355	// hardware): it purges every VM row for the host and deletes the host row, both	1455	// hardware): it purges every VM row for the host and deletes the host row, both
1356	// in one transaction. Like RemoveHost it recycles nothing. Unlike the	1456	// in one transaction. Like RemoveHost it recycles nothing. Unlike the
1357	// graceful path it does NOT wait for the agent to ack destroys, so it must only	1457	// graceful path it does NOT wait for the agent to ack destroys, so it must only
1358	// be used when the host is known gone; any VMs still physically running are	1458	// be used when the host is known gone; any VMs still physically running are
1359	// orphaned with the hardware. Returns the number of VM rows purged.	1459	// orphaned with the hardware. Returns the tally of what went.
1360	func (s *Store) ForceRemoveHost(id string) (int, error) {	1460	//
		1461	// THIS IS THE ONE PATH THAT LOSES DATA ON PURPOSE. The host's volumes go with
		1462	// the host, because that is where the bytes were: a live claim placed here
		1463	// returns to Pending and the next VM that names it places it somewhere else,
		1464	// with nothing of the old contents. A claim already tombstoned here is deleted
		1465	// outright — the reclaim it was waiting for can never be acked by a host that
		1466	// is gone, so keeping the row would only leave an unreapable tombstone behind.
		1467	// The operator asking for force has already decided the hardware is lost;
		1468	// refusing to admit its disks went with it would help nobody.
		1469	func (s *Store) ForceRemoveHost(id string) (ForceRemoval, error) {
		1470	var out ForceRemoval
1361	tx, err := s.db.Begin()	1471	tx, err := s.db.Begin()
1362	if err != nil {	1472	if err != nil {
1363	return 0, err	1473	return out, err
1364	}	1474	}
1365	defer tx.Rollback()	1475	defer tx.Rollback()
1366		1476
@@ -1369,27 +1479,70 @@ func (s *Store) ForceRemoveHost(id string) (int, error) {
1369	var exists int	1479	var exists int
1370	switch err := tx.QueryRow(`SELECT 1 FROM hosts WHERE id=?`, id).Scan(&exists); {	1480	switch err := tx.QueryRow(`SELECT 1 FROM hosts WHERE id=?`, id).Scan(&exists); {
1371	case errors.Is(err, sql.ErrNoRows):	1481	case errors.Is(err, sql.ErrNoRows):
1372	return 0, sql.ErrNoRows	1482	return out, sql.ErrNoRows
1373	case err != nil:	1483	case err != nil:
1374	return 0, fmt.Errorf("lookup host: %w", err)	1484	return out, fmt.Errorf("lookup host: %w", err)
		1485	}
		1486
		1487	// The volumes whose bytes lived on this host and the claims they were
		1488	// placed for, read BEFORE anything is deleted and drained to completion
		1489	// before the first write — the store runs one connection, so an open
		1490	// result set blocks every statement below.
		1491	volumeIDs, claimIDs, err := volumesOnHost(tx, id)
		1492	if err != nil {
		1493	return out, err
		1494	}
		1495
		1496	// Unbind first, while the volumes rows this matches still exist.
		1497	res, err := tx.Exec(`UPDATE volume_claims SET bound_volume_id=NULL
		1498	WHERE bound_volume_id IN (SELECT id FROM volumes WHERE host_id=?)`, id)
		1499	if err != nil {
		1500	return out, fmt.Errorf("unbind host claims: %w", err)
1375	}	1501	}
		1502	unbound, _ := res.RowsAffected()
1376		1503
1377	res, err := tx.Exec(`DELETE FROM vms WHERE host_id=?`, id)	1504	// VMs before volumes: deleting a VM cascades its attachments away, and a
		1505	// volume_claims row cannot go while an attachment still references it.
		1506	res, err = tx.Exec(`DELETE FROM vms WHERE host_id=?`, id)
1378	if err != nil {	1507	if err != nil {
1379	return 0, fmt.Errorf("purge host vms: %w", err)	1508	return out, fmt.Errorf("purge host vms: %w", err)
1380	}	1509	}
1381	purged, _ := res.RowsAffected()	1510	purged, _ := res.RowsAffected()
1382		1511
		1512	res, err = tx.Exec(`DELETE FROM volumes WHERE host_id=?`, id)
		1513	if err != nil {
		1514	return out, fmt.Errorf("purge host volumes: %w", err)
		1515	}
		1516	destroyed, _ := res.RowsAffected()
		1517
		1518	// A claim already tombstoned here has nothing left to reclaim: its host is
		1519	// gone and no ack is coming. Live claims survive as Pending, so a claim
		1520	// deleted here is one the unbind count above must not claim to have freed.
		1521	var tombstoned int64
		1522	for _, claimID := range claimIDs {
		1523	res, err := tx.Exec(`DELETE FROM volume_claims WHERE id=? AND deleted_at IS NOT NULL`, claimID)
		1524	if err != nil {
		1525	return out, fmt.Errorf("purge tombstoned claim: %w", err)
		1526	}
		1527	n, _ := res.RowsAffected()
		1528	tombstoned += n
		1529	}
		1530
1383	if _, err := tx.Exec(`DELETE FROM hosts WHERE id=?`, id); err != nil {	1531	if _, err := tx.Exec(`DELETE FROM hosts WHERE id=?`, id); err != nil {
1384	return 0, fmt.Errorf("delete host: %w", err)	1532	return out, fmt.Errorf("delete host: %w", err)
1385	}	1533	}
1386	if err := bumpEpoch(tx); err != nil {	1534	if err := bumpEpoch(tx); err != nil {
1387	return 0, fmt.Errorf("bump epoch: %w", err)	1535	return out, fmt.Errorf("bump epoch: %w", err)
1388	}	1536	}
1389	if err := tx.Commit(); err != nil {	1537	if err := tx.Commit(); err != nil {
1390	return 0, err	1538	return out, err
1391	}	1539	}
1392	return int(purged), nil	1540	return ForceRemoval{
		1541	VMsPurged: int(purged),
		1542	VolumesDestroyed: int(destroyed),
		1543	ClaimsUnbound: int(unbound - tombstoned),
		1544	VolumeIDs: volumeIDs,
		1545	}, nil
1393	}	1546	}
1394		1547
1395	// usableAddress reports whether ip is an address something could actually be	1548	// usableAddress reports whether ip is an address something could actually be
@@ -1569,6 +1722,27 @@ func scanVM(rows *sql.Rows) (VM, error) {
1569	// and SpecForHost all share this — they differ only in WHERE clause,	1722	// and SpecForHost all share this — they differ only in WHERE clause,
1570	// row-count expectations, and whether q is sql.DB or an in-flight sql.Tx.	1723	// row-count expectations, and whether q is sql.DB or an in-flight sql.Tx.
1571	func queryVMs(q querier, where string, args ...any) ([]VM, error) {	1724	func queryVMs(q querier, where string, args ...any) ([]VM, error) {
		1725	// Scan every row and CLOSE before reading volumes: the store runs one
		1726	// connection, so a second query issued while this result set is open would
		1727	// deadlock. scanVMRows is a separate function so the close is a scope
		1728	// boundary the compiler enforces rather than a rule to remember.
		1729	vms, err := scanVMRows(q, where, args...)
		1730	if err != nil {
		1731	return nil, err
		1732	}
		1733	// A VM's volumes are a list, so they live in their own table and are
		1734	// filled in here rather than being a column of the row above.
		1735	for i := range vms {
		1736	ids, err := volumeIDsForVM(q, vms[i].ID)
		1737	if err != nil {
		1738	return nil, fmt.Errorf("read vm volumes: %w", err)
		1739	}
		1740	vms[i].VolumeIDs = ids
		1741	}
		1742	return vms, nil
		1743	}
		1744
		1745	func scanVMRows(q querier, where string, args ...any) ([]VM, error) {
1572	rows, err := q.Query(`SELECT `+vmColumns+` FROM vms`+where, args...)	1746	rows, err := q.Query(`SELECT `+vmColumns+` FROM vms`+where, args...)
1573	if err != nil {	1747	if err != nil {
1574	return nil, err	1748	return nil, err

internal/server/store/store_test.go

Old		New
@@ -843,9 +843,12 @@ func TestForceRemoveHostPurgesVMs(t *testing.T) {
843	// filter, so a tombstoned row is purged — and counted — like a live one).	843	// filter, so a tombstoned row is purged — and counted — like a live one).
844	require.NoError(t, s.TombstoneVM(vmB.ID))	844	require.NoError(t, s.TombstoneVM(vmB.ID))
845		845
846	purged, err := s.ForceRemoveHost(h.ID)	846	gone, err := s.ForceRemoveHost(h.ID)
847	require.NoError(t, err)	847	require.NoError(t, err)
848	assert.Equal(t, 2, purged, "both VM rows — live and tombstoned — must be purged")	848	assert.Equal(t, 2, gone.VMsPurged, "both VM rows — live and tombstoned — must be purged")
		849	assert.Zero(t, gone.VolumesDestroyed, "these VMs held no volumes")
		850	assert.Zero(t, gone.ClaimsUnbound)
		851	assert.Empty(t, gone.VolumeIDs)
849		852
850	_, err = s.GetHost(h.ID)	853	_, err = s.GetHost(h.ID)
851	assert.Error(t, err, "host row must be gone")	854	assert.Error(t, err, "host row must be gone")

internal/server/store/volumes.go

@@ @@ -0,0 +1,315 @@ @@
+package store
+import (
+	"database/sql"
+	"errors"
+	"fmt"
+	"time"
+	"github.com/a73x/eitri/internal/random"
+	sqlite "modernc.org/sqlite"
+)
+// VolumeClaim is a tenant's request for durable storage. HostID and VMID are
+// joined facts: where the bytes were placed, and which VM holds them now.
+type VolumeClaim struct {
+	ID, Tenant, Name string
+	SizeGB           int64
+	BoundVolumeID    string
+	HostID           string
+	VMID             string
+	CreatedAt        time.Time
+	DeletedAt        *time.Time
+}
+// Volume is the fleet's placement of one claim's bytes on one host.
+type Volume struct {
+	ID, HostID, ClaimID string
+	SizeGB              int64
+	CreatedAt           time.Time
+	DeletedAt           *time.Time
+}
+var (
+	ErrClaimNotFound  = errors.New("volume claim not found")
+	ErrClaimNameTaken = errors.New("volume claim name already in use")
+	ErrClaimAttached  = errors.New("volume claim is attached to a vm")
+	ErrClaimPinned    = errors.New("volume claim is bound to another host")
+)
+// ClaimAttachedError names the VM that holds the claim.
+type ClaimAttachedError struct{ ClaimID, VMID string }
+func (e *ClaimAttachedError) Error() string {
+	return fmt.Sprintf("volume claim %s is attached to vm %s", e.ClaimID, e.VMID)
+}
+func (e *ClaimAttachedError) Is(target error) bool { return target == ErrClaimAttached }
+// ClaimPinnedError names the host the claim's data lives on.
+type ClaimPinnedError struct{ ClaimID, HostID string }
+func (e *ClaimPinnedError) Error() string {
+	return fmt.Sprintf("volume claim %s is bound to host %s", e.ClaimID, e.HostID)
+}
+func (e *ClaimPinnedError) Is(target error) bool { return target == ErrClaimPinned }
+// claimColumns / claimFrom are one positional contract with scanClaim.
+const claimColumns = `c.id, c.tenant, c.name, c.size_gb, COALESCE(c.bound_volume_id,''),
+	COALESCE(v.host_id,''), COALESCE(a.vm_id,''), c.created_at, c.deleted_at`
+const claimFrom = ` FROM volume_claims c
+	LEFT JOIN volumes v ON v.id = c.bound_volume_id
+	LEFT JOIN volume_attachments a ON a.claim_id = c.id`
+// scanner is satisfied by *sql.Row and *sql.Rows alike, so the single-row read
+// and the list share one scan.
+type scanner interface{ Scan(dest ...any) error }
+func scanClaim(row scanner) (VolumeClaim, error) {
+	var c VolumeClaim
+	var createdAt string
+	var deletedAt sql.NullString
+	if err := row.Scan(&c.ID, &c.Tenant, &c.Name, &c.SizeGB, &c.BoundVolumeID, &c.HostID, &c.VMID, &createdAt, &deletedAt); err != nil {
+		return VolumeClaim{}, err
+	}
+	c.CreatedAt, _ = time.Parse(time.RFC3339, createdAt)
+	if deletedAt.Valid {
+		t, _ := time.Parse(time.RFC3339, deletedAt.String)
+		c.DeletedAt = &t
+	}
+	return c, nil
+}
+// CreateVolumeClaim records a request for storage. It places nothing: a claim
+// is Pending until the first VM that names it decides which host it lands on.
+func (s *Store) CreateVolumeClaim(tenant, name string, sizeGB int64) (VolumeClaim, error) {
+	c := VolumeClaim{ID: random.Hex(16), Tenant: tenant, Name: name, SizeGB: sizeGB, CreatedAt: time.Now().UTC()}
+	_, err := s.db.Exec(`INSERT INTO volume_claims(id, tenant, name, size_gb, created_at) VALUES (?,?,?,?,?)`,
+		c.ID, c.Tenant, c.Name, c.SizeGB, c.CreatedAt.Format(time.RFC3339))
+	if err != nil {
+		// SQLITE_CONSTRAINT_UNIQUE (2067): volume_claims_tenant_name.
+		if serr, ok := errors.AsType[*sqlite.Error](err); ok && serr.Code() == 2067 {
+			return VolumeClaim{}, ErrClaimNameTaken
+		}
+		return VolumeClaim{}, fmt.Errorf("insert volume claim: %w", err)
+	}
+	return c, nil
+}
+// ListVolumeClaims returns tenant's live claims, oldest first.
+func (s *Store) ListVolumeClaims(tenant string) ([]VolumeClaim, error) {
+	rows, err := s.db.Query(`SELECT `+claimColumns+claimFrom+` WHERE c.tenant=? AND c.deleted_at IS NULL ORDER BY c.created_at, c.id`, tenant)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var out []VolumeClaim
+	for rows.Next() {
+		c, err := scanClaim(rows)
+		if err != nil {
+			return nil, err
+		}
+		out = append(out, c)
+	}
+	return out, rows.Err()
+}
+// GetVolumeClaim reads one live claim; a tombstoned one reads as sql.ErrNoRows.
+func (s *Store) GetVolumeClaim(id string) (VolumeClaim, error) {
+	return scanClaim(s.db.QueryRow(`SELECT `+claimColumns+claimFrom+` WHERE c.id=? AND c.deleted_at IS NULL`, id))
+}
+// GetVolumeClaimAny reads one claim whether live or tombstoned. It exists for
+// the reap, which asks after a claim it is about to delete precisely because
+// that claim is already tombstoned: GetVolumeClaim would answer ErrNoRows and
+// the terminal audit row would lose its tenant.
+func (s *Store) GetVolumeClaimAny(id string) (VolumeClaim, error) {
+	return scanClaim(s.db.QueryRow(`SELECT `+claimColumns+claimFrom+` WHERE c.id=?`, id))
+}
+// TombstoneVolumeClaim marks a claim and its bound volume (if any) for
+// reclaim. Refused while a VM holds the claim: the guest may be writing.
+func (s *Store) TombstoneVolumeClaim(id string) error {
+	tx, err := s.db.Begin()
+	if err != nil {
+		return err
+	}
+	defer tx.Rollback()
+	var vmID, bound sql.NullString
+	err = tx.QueryRow(`SELECT a.vm_id, c.bound_volume_id FROM volume_claims c
+		LEFT JOIN volume_attachments a ON a.claim_id=c.id WHERE c.id=? AND c.deleted_at IS NULL`, id).Scan(&vmID, &bound)
+	if err != nil {
+		return err // sql.ErrNoRows passes through
+	}
+	if vmID.Valid {
+		return &ClaimAttachedError{ClaimID: id, VMID: vmID.String}
+	}
+	now := time.Now().UTC().Format(time.RFC3339)
+	if _, err := tx.Exec(`UPDATE volume_claims SET deleted_at=? WHERE id=?`, now, id); err != nil {
+		return err
+	}
+	if bound.Valid {
+		if _, err := tx.Exec(`UPDATE volumes SET deleted_at=? WHERE id=? AND deleted_at IS NULL`, now, bound.String); err != nil {
+			return err
+		}
+		// The host must see the tombstone; an unbound claim has no host.
+		if err := bumpEpoch(tx); err != nil {
+			return err
+		}
+	}
+	return tx.Commit()
+}
+// HardDeleteVolume removes a tombstoned volume row once its host has reported
+// the file gone, and the tombstoned claim that owned it. sql.ErrNoRows when the
+// volume is absent, or when EITHER the volume or its claim is still live.
+//
+// The claim's tombstone is part of the guard, not an assumption: reaping the
+// bytes of a claim somebody still holds would leave that claim Pending and
+// re-bindable, so the next VM naming it would silently get an empty disk where
+// the tenant expects their data. Deleting both rows together, or neither, is
+// the only pair of outcomes that cannot lie.
+func (s *Store) HardDeleteVolume(id string) error {
+	tx, err := s.db.Begin()
+	if err != nil {
+		return err
+	}
+	defer tx.Rollback()
+	var claimID string
+	if err := tx.QueryRow(`SELECT v.claim_id FROM volumes v
+		JOIN volume_claims c ON c.id = v.claim_id
+		WHERE v.id=? AND v.deleted_at IS NOT NULL AND c.deleted_at IS NOT NULL`, id).Scan(&claimID); err != nil {
+		return err
+	}
+	// Volume first: volumes.claim_id references the row deleted below.
+	if _, err := tx.Exec(`DELETE FROM volumes WHERE id=?`, id); err != nil {
+		return err
+	}
+	if _, err := tx.Exec(`DELETE FROM volume_claims WHERE id=?`, claimID); err != nil {
+		return err
+	}
+	if err := bumpEpoch(tx); err != nil {
+		return err
+	}
+	return tx.Commit()
+}
+// volumesOnHost lists the volumes living on one host and the claims they are
+// placed for, in id order so a caller reporting them reports them the same way
+// twice. It fully drains its result set before returning, so the caller may
+// write inside the same transaction — the store's single connection allows
+// nothing else.
+func volumesOnHost(q querier, hostID string) (volumeIDs, claimIDs []string, err error) {
+	rows, err := q.Query(`SELECT id, claim_id FROM volumes WHERE host_id=? ORDER BY id`, hostID)
+	if err != nil {
+		return nil, nil, fmt.Errorf("list host volumes: %w", err)
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var volID, claimID string
+		if err := rows.Scan(&volID, &claimID); err != nil {
+			return nil, nil, fmt.Errorf("scan host volume: %w", err)
+		}
+		volumeIDs = append(volumeIDs, volID)
+		claimIDs = append(claimIDs, claimID)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, nil, err
+	}
+	return volumeIDs, claimIDs, nil
+}
+// ListVolumesForHost is the host's full volume set, tombstoned rows included:
+// the snapshot carries both so the agent can reclaim.
+func (s *Store) ListVolumesForHost(hostID string) ([]Volume, error) {
+	rows, err := s.db.Query(`SELECT id, host_id, claim_id, size_gb, created_at, deleted_at
+		FROM volumes WHERE host_id=? ORDER BY created_at, id`, hostID)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var out []Volume
+	for rows.Next() {
+		var v Volume
+		var createdAt string
+		var deletedAt sql.NullString
+		if err := rows.Scan(&v.ID, &v.HostID, &v.ClaimID, &v.SizeGB, &createdAt, &deletedAt); err != nil {
+			return nil, err
+		}
+		v.CreatedAt, _ = time.Parse(time.RFC3339, createdAt)
+		if deletedAt.Valid {
+			t, _ := time.Parse(time.RFC3339, deletedAt.String)
+			v.DeletedAt = &t
+		}
+		out = append(out, v)
+	}
+	return out, rows.Err()
+}
+// bindClaims runs inside CreateVM's transaction: every claim must be the VM's
+// tenant's and live, unattached, and either Pending or already on this host.
+// A Pending claim is bound HERE — a volumes row on vm.HostID — so binding is
+// atomic with placement and never a background job.
+func bindClaims(tx *sql.Tx, vm VM) error {
+	now := time.Now().UTC().Format(time.RFC3339)
+	for _, claimID := range vm.VolumeClaimIDs {
+		var sizeGB int64
+		var bound, boundHost, holder sql.NullString
+		err := tx.QueryRow(`SELECT c.size_gb, c.bound_volume_id, v.host_id, a.vm_id FROM volume_claims c
+			LEFT JOIN volumes v ON v.id=c.bound_volume_id
+			LEFT JOIN volume_attachments a ON a.claim_id=c.id
+			WHERE c.id=? AND c.tenant=? AND c.deleted_at IS NULL`, claimID, vm.Tenant).
+			Scan(&sizeGB, &bound, &boundHost, &holder)
+		switch {
+		case errors.Is(err, sql.ErrNoRows):
+			return ErrClaimNotFound
+		case err != nil:
+			return fmt.Errorf("lookup claim: %w", err)
+		case holder.Valid:
+			return &ClaimAttachedError{ClaimID: claimID, VMID: holder.String}
+		case bound.Valid && boundHost.String != vm.HostID:
+			return &ClaimPinnedError{ClaimID: claimID, HostID: boundHost.String}
+		}
+		if !bound.Valid {
+			volID := random.Hex(16)
+			if _, err := tx.Exec(`INSERT INTO volumes(id, host_id, claim_id, size_gb, created_at) VALUES (?,?,?,?,?)`,
+				volID, vm.HostID, claimID, sizeGB, now); err != nil {
+				return fmt.Errorf("place volume: %w", err)
+			}
+			if _, err := tx.Exec(`UPDATE volume_claims SET bound_volume_id=? WHERE id=?`, volID, claimID); err != nil {
+				return fmt.Errorf("bind claim: %w", err)
+			}
+		}
+		if _, err := tx.Exec(`INSERT INTO volume_attachments(claim_id, vm_id) VALUES (?,?)`, claimID, vm.ID); err != nil {
+			// The unique index is the race's referee: the other create won.
+			// The store runs one connection, so this is belt-and-braces.
+			if serr, ok := errors.AsType[*sqlite.Error](err); ok && serr.Code() == 2067 {
+				return &ClaimAttachedError{ClaimID: claimID, VMID: "another vm"}
+			}
+			return fmt.Errorf("attach claim: %w", err)
+		}
+	}
+	return nil
+}
+// volumeIDsForVM is the attachment order, which is the order the VM named its
+// claims (rowid of the attachment row).
+func volumeIDsForVM(q querier, vmID string) ([]string, error) {
+	rows, err := q.Query(`SELECT c.bound_volume_id FROM volume_attachments a
+		JOIN volume_claims c ON c.id=a.claim_id WHERE a.vm_id=? ORDER BY a.rowid`, vmID)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var out []string
+	for rows.Next() {
+		var id sql.NullString
+		if err := rows.Scan(&id); err != nil {
+			return nil, err
+		}
+		if id.Valid {
+			out = append(out, id.String)
+		}
+	}
+	return out, rows.Err()
+}

internal/server/store/volumes_test.go

@@ @@ -0,0 +1,383 @@ @@
+package store
+import (
+	"database/sql"
+	"regexp"
+	"sort"
+	"testing"
+	"github.com/stretchr/testify/require"
+)
+// enrollSecondHost gives the fleet a host that is not enrollHost's, so a claim
+// bound on one host can be asked for from the other.
+func enrollSecondHost(t *testing.T, s *Store) Host {
+	t.Helper()
+	tok, err := s.CreateEnrollmentToken(testTenant)
+	require.NoError(t, err)
+	h, err := s.RedeemEnrollmentToken(tok, EnrollFacts{Name: "host-b", OS: "linux", Arch: "amd64", Provisioner: "cloudhv", Remote: ""})
+	require.NoError(t, err)
+	return h
+}
+func testVM(id, host string, claims ...string) VM {
+	return VM{ID: id, HostID: host, Name: id, ImageURL: "u", ImageSHA256: "s",
+		VCPUs: 1, MemMB: 1, DiskGB: 1, PowerState: "running", VolumeClaimIDs: claims}
+}
+func TestVolumeClaimLifecycle(t *testing.T) {
+	st := newStore(t)
+	hostID := enrollHost(t, st).ID
+	c, err := st.CreateVolumeClaim(testTenant, "data", 5)
+	require.NoError(t, err)
+	require.Equal(t, "", c.BoundVolumeID, "a fresh claim is Pending")
+	_, err = st.CreateVolumeClaim(testTenant, "data", 5)
+	require.ErrorIs(t, err, ErrClaimNameTaken)
+	// First VM naming the claim binds it on its host.
+	require.NoError(t, st.CreateVM(testVM("vm1", hostID, c.ID)))
+	got, err := st.GetVolumeClaim(c.ID)
+	require.NoError(t, err)
+	require.NotEmpty(t, got.BoundVolumeID)
+	require.Equal(t, hostID, got.HostID)
+	require.Equal(t, "vm1", got.VMID)
+	vm, err := st.GetVM("vm1")
+	require.NoError(t, err)
+	require.Equal(t, []string{got.BoundVolumeID}, vm.VolumeIDs)
+	// Second VM: attached → refused naming the holder; nothing of it remains.
+	err = st.CreateVM(testVM("vm2", hostID, c.ID))
+	var attached *ClaimAttachedError
+	require.ErrorAs(t, err, &attached)
+	require.Equal(t, "vm1", attached.VMID)
+	require.ErrorIs(t, err, ErrClaimAttached)
+	_, err = st.GetVM("vm2")
+	require.ErrorIs(t, err, sql.ErrNoRows, "the refused create rolled back entirely")
+	// Delete while attached → refused.
+	require.ErrorIs(t, st.TombstoneVolumeClaim(c.ID), ErrClaimAttached)
+	// Destroy the VM: attachment gone, volume and claim untouched.
+	require.NoError(t, st.TombstoneVM("vm1"))
+	require.NoError(t, st.HardDeleteVM("vm1", hostID))
+	got, err = st.GetVolumeClaim(c.ID)
+	require.NoError(t, err)
+	require.Equal(t, "", got.VMID)
+	require.Equal(t, hostID, got.HostID, "the data still lives where it was placed")
+	// A VM on another host cannot take a bound claim.
+	other := enrollSecondHost(t, st).ID
+	err = st.CreateVM(testVM("vm3", other, c.ID))
+	var pinned *ClaimPinnedError
+	require.ErrorAs(t, err, &pinned)
+	require.Equal(t, hostID, pinned.HostID)
+	require.ErrorIs(t, err, ErrClaimPinned)
+	// Delete cascades claim → volume as tombstones; the host still sees it.
+	require.NoError(t, st.TombstoneVolumeClaim(c.ID))
+	vols, err := st.ListVolumesForHost(hostID)
+	require.NoError(t, err)
+	require.Len(t, vols, 1)
+	require.NotNil(t, vols[0].DeletedAt)
+	_, err = st.GetVolumeClaim(c.ID)
+	require.ErrorIs(t, err, sql.ErrNoRows, "a tombstoned claim reads as gone")
+	// Reap after the host reports it gone: both rows go.
+	volID := vols[0].ID
+	require.NoError(t, st.HardDeleteVolume(volID))
+	vols, err = st.ListVolumesForHost(hostID)
+	require.NoError(t, err)
+	require.Empty(t, vols)
+	require.ErrorIs(t, st.HardDeleteVolume(volID), sql.ErrNoRows)
+}
+// The reap needs the tenant of a claim that is already tombstoned, which is
+// exactly the claim GetVolumeClaim refuses to answer for.
+func TestGetVolumeClaimAnyReadsATombstonedClaim(t *testing.T) {
+	st := newStore(t)
+	c, err := st.CreateVolumeClaim(testTenant, "data", 5)
+	require.NoError(t, err)
+	require.NoError(t, st.TombstoneVolumeClaim(c.ID))
+	_, err = st.GetVolumeClaim(c.ID)
+	require.ErrorIs(t, err, sql.ErrNoRows)
+	got, err := st.GetVolumeClaimAny(c.ID)
+	require.NoError(t, err)
+	require.Equal(t, testTenant, got.Tenant)
+	require.NotNil(t, got.DeletedAt)
+	_, err = st.GetVolumeClaimAny("nope")
+	require.ErrorIs(t, err, sql.ErrNoRows)
+}
+func TestCreateVMRefusesForeignOrMissingClaim(t *testing.T) {
+	st := newStore(t)
+	hostID := enrollHost(t, st).ID
+	other, err := st.CreateTenantForIdentity("https://test-issuer", "other-subject", "other@test.local")
+	require.NoError(t, err)
+	c, err := st.CreateVolumeClaim(other.ID, "data", 5)
+	require.NoError(t, err)
+	require.ErrorIs(t, st.CreateVM(testVM("vm1", hostID, c.ID)), ErrClaimNotFound,
+		"another tenant's claim is indistinguishable from none")
+	require.ErrorIs(t, st.CreateVM(testVM("vm1", hostID, "nope")), ErrClaimNotFound)
+}
+func TestCommittedOnHostCountsBoundVolumes(t *testing.T) {
+	st := newStore(t)
+	hostID := enrollHost(t, st).ID
+	c, err := st.CreateVolumeClaim(testTenant, "data", 7)
+	require.NoError(t, err)
+	vm := testVM("vm1", hostID, c.ID)
+	vm.DiskGB = 3
+	require.NoError(t, st.CreateVM(vm))
+	held, err := st.CommittedOnHost(hostID)
+	require.NoError(t, err)
+	require.EqualValues(t, 10, held.Live.DiskGB, "a bound volume is disk the host has committed, VM or no VM")
+	require.NoError(t, st.TombstoneVM("vm1"))
+	require.NoError(t, st.HardDeleteVM("vm1", hostID))
+	held, err = st.CommittedOnHost(hostID)
+	require.NoError(t, err)
+	require.EqualValues(t, 7, held.Live.DiskGB)
+}
+// Attachment order is the order the VM named its claims; the guest's device
+// letters follow it.
+func TestVMVolumeIDsKeepAttachmentOrder(t *testing.T) {
+	st := newStore(t)
+	hostID := enrollHost(t, st).ID
+	b, err := st.CreateVolumeClaim(testTenant, "b", 1)
+	require.NoError(t, err)
+	a, err := st.CreateVolumeClaim(testTenant, "a", 1)
+	require.NoError(t, err)
+	require.NoError(t, st.CreateVM(testVM("vm1", hostID, b.ID, a.ID)))
+	vm, err := st.GetVM("vm1")
+	require.NoError(t, err)
+	cb, err := st.GetVolumeClaim(b.ID)
+	require.NoError(t, err)
+	ca, err := st.GetVolumeClaim(a.ID)
+	require.NoError(t, err)
+	require.Equal(t, []string{cb.BoundVolumeID, ca.BoundVolumeID}, vm.VolumeIDs)
+}
+func TestListVolumeClaimsIsTenantScoped(t *testing.T) {
+	st := newStore(t)
+	hostID := enrollHost(t, st).ID
+	c, err := st.CreateVolumeClaim(testTenant, "data", 5)
+	require.NoError(t, err)
+	other, err := st.CreateTenantForIdentity("https://test-issuer", "other-subject", "other@test.local")
+	require.NoError(t, err)
+	_, err = st.CreateVolumeClaim(other.ID, "data", 5)
+	require.NoError(t, err, "the same name in another tenant is a different claim")
+	claims, err := st.ListVolumeClaims(testTenant)
+	require.NoError(t, err)
+	require.Len(t, claims, 1)
+	require.Equal(t, c.ID, claims[0].ID)
+	require.Equal(t, "", claims[0].HostID, "an unbound claim names no host")
+	require.NoError(t, st.CreateVM(testVM("vm1", hostID, c.ID)))
+	claims, err = st.ListVolumeClaims(testTenant)
+	require.NoError(t, err)
+	require.Len(t, claims, 1)
+	require.Equal(t, hostID, claims[0].HostID)
+	require.Equal(t, "vm1", claims[0].VMID)
+	// A tombstoned claim leaves the tenant's list.
+	require.NoError(t, st.TombstoneVM("vm1"))
+	require.NoError(t, st.HardDeleteVM("vm1", hostID))
+	require.NoError(t, st.TombstoneVolumeClaim(c.ID))
+	claims, err = st.ListVolumeClaims(testTenant)
+	require.NoError(t, err)
+	require.Empty(t, claims)
+}
+// A Pending claim has no bytes anywhere, so deleting it is bookkeeping only.
+func TestTombstoneVolumeClaimUnbound(t *testing.T) {
+	st := newStore(t)
+	c, err := st.CreateVolumeClaim(testTenant, "data", 5)
+	require.NoError(t, err)
+	before, err := st.Epoch()
+	require.NoError(t, err)
+	require.NoError(t, st.TombstoneVolumeClaim(c.ID))
+	after, err := st.Epoch()
+	require.NoError(t, err)
+	require.Equal(t, before, after, "an unbound claim is on no host, so no agent has to re-snapshot")
+	require.ErrorIs(t, st.TombstoneVolumeClaim(c.ID), sql.ErrNoRows)
+}
+// countRows is the only way a test can prove a row is DELETED rather than
+// tombstoned: every read above deliberately hides tombstones.
+func countRows(t *testing.T, s *Store, query string, args ...any) int {
+	t.Helper()
+	var n int
+	require.NoError(t, s.db.QueryRow(query, args...).Scan(&n))
+	return n
+}
+// A host holding volumes is not a decommission that needs more time — nothing
+// releases a volume but its claim's deletion, so the graceful path refuses and
+// says which volumes are in the way.
+func TestRemoveHostRefusesWhileVolumesRemain(t *testing.T) {
+	st := newStore(t)
+	hostID := enrollHost(t, st).ID
+	c, err := st.CreateVolumeClaim(testTenant, "data", 5)
+	require.NoError(t, err)
+	require.NoError(t, st.CreateVM(testVM("vm1", hostID, c.ID)))
+	require.NoError(t, st.TombstoneVM("vm1"))
+	require.NoError(t, st.HardDeleteVM("vm1", hostID))
+	// Live volume, no VMs left: drained of guests, still holding bytes.
+	err = st.RemoveHost(hostID)
+	require.ErrorIs(t, err, ErrHostHoldsVolumes)
+	require.Contains(t, err.Error(), "1 volume(s)")
+	// Tombstoning the claim does not release the host either: the file is on
+	// that disk until the agent acks the reclaim.
+	require.NoError(t, st.TombstoneVolumeClaim(c.ID))
+	require.ErrorIs(t, st.RemoveHost(hostID), ErrHostHoldsVolumes)
+	// Reaped: the host is finally removable.
+	vols, err := st.ListVolumesForHost(hostID)
+	require.NoError(t, err)
+	require.Len(t, vols, 1)
+	require.NoError(t, st.HardDeleteVolume(vols[0].ID))
+	require.NoError(t, st.RemoveHost(hostID))
+}
+// The ids this store mints are a WIRE CONTRACT, not a local detail: the agent's
+// validVolumeID (internal/agent/reconcile/volumes.go) refuses any volume id that
+// is not exactly 32 lowercase hex characters, because the id is joined into a
+// path the agent hands to os.RemoveAll. A shorter id, an uppercase one, or a
+// uuid with dashes would be refused on every host in the fleet and the volume
+// would never materialize — so the shape is pinned on the minting side too.
+func TestMintedIDsAreThirtyTwoLowercaseHex(t *testing.T) {
+	hex32 := regexp.MustCompile(`^[0-9a-f]{32}$`)
+	st := newStore(t)
+	hostID := enrollHost(t, st).ID
+	c, err := st.CreateVolumeClaim(testTenant, "data", 5)
+	require.NoError(t, err)
+	require.Regexp(t, hex32, c.ID, "claim id shape the agent's validVolumeID demands")
+	// bindClaims mints the volume id when the first VM places the claim.
+	require.NoError(t, st.CreateVM(testVM("vm1", hostID, c.ID)))
+	vols, err := st.ListVolumesForHost(hostID)
+	require.NoError(t, err)
+	require.Len(t, vols, 1)
+	require.Regexp(t, hex32, vols[0].ID, "volume id shape the agent's validVolumeID demands")
+	require.Equal(t, c.ID, vols[0].ClaimID)
+}
+// Force is for hardware that is gone, and it takes the disks with it.
+func TestForceRemoveHostLosesItsVolumes(t *testing.T) {
+	st := newStore(t)
+	hostID := enrollHost(t, st).ID
+	live, err := st.CreateVolumeClaim(testTenant, "live", 5)
+	require.NoError(t, err)
+	doomed, err := st.CreateVolumeClaim(testTenant, "doomed", 5)
+	require.NoError(t, err)
+	require.NoError(t, st.CreateVM(testVM("vm1", hostID, live.ID, doomed.ID)))
+	// The doomed claim is already deleted and waiting on a reclaim ack that
+	// will never come.
+	require.NoError(t, st.TombstoneVM("vm1"))
+	require.NoError(t, st.HardDeleteVM("vm1", hostID))
+	require.NoError(t, st.TombstoneVolumeClaim(doomed.ID))
+	// Read the doomed volumes' ids while they still exist: after the force they
+	// are nowhere but the tally, which is the whole point of reporting them.
+	before, err := st.ListVolumesForHost(hostID)
+	require.NoError(t, err)
+	require.Len(t, before, 2)
+	wantIDs := []string{before[0].ID, before[1].ID}
+	sort.Strings(wantIDs)
+	gone, err := st.ForceRemoveHost(hostID)
+	require.NoError(t, err)
+	require.Equal(t, 0, gone.VMsPurged, "the VM was already reaped; the volumes were not")
+	// The tally is the receipt for the loss: both volumes, named, and only the
+	// LIVE claim counted as handed back — the tombstoned one was deleted, not
+	// returned to Pending.
+	require.Equal(t, 2, gone.VolumesDestroyed)
+	require.Equal(t, wantIDs, gone.VolumeIDs, "the destroyed volumes, in id order")
+	require.Equal(t, 1, gone.ClaimsUnbound, "the tombstoned claim went; only the live one is Pending again")
+	// The live claim survives as Pending: the request stands, the bytes do not.
+	got, err := st.GetVolumeClaim(live.ID)
+	require.NoError(t, err)
+	require.Equal(t, "", got.BoundVolumeID, "a claim whose host died is Pending again")
+	require.Equal(t, "", got.HostID)
+	// The tombstoned one is gone outright — no host is left to ack its reclaim.
+	require.Zero(t, countRows(t, st, `SELECT COUNT(*) FROM volume_claims WHERE id=?`, doomed.ID))
+	require.Zero(t, countRows(t, st, `SELECT COUNT(*) FROM volumes WHERE host_id=?`, hostID))
+	// And the freed claim can be placed again, elsewhere.
+	other := enrollSecondHost(t, st).ID
+	require.NoError(t, st.CreateVM(testVM("vm2", other, live.ID)))
+	got, err = st.GetVolumeClaim(live.ID)
+	require.NoError(t, err)
+	require.Equal(t, other, got.HostID)
+}
+// Force must also work while the VMs are still there — the usual dead-hardware
+// case, where nothing drained at all.
+func TestForceRemoveHostPurgesVMsHoldingVolumes(t *testing.T) {
+	st := newStore(t)
+	hostID := enrollHost(t, st).ID
+	c, err := st.CreateVolumeClaim(testTenant, "data", 5)
+	require.NoError(t, err)
+	require.NoError(t, st.CreateVM(testVM("vm1", hostID, c.ID)))
+	vols, err := st.ListVolumesForHost(hostID)
+	require.NoError(t, err)
+	require.Len(t, vols, 1)
+	gone, err := st.ForceRemoveHost(hostID)
+	require.NoError(t, err)
+	require.Equal(t, 1, gone.VMsPurged)
+	require.Equal(t, 1, gone.VolumesDestroyed)
+	require.Equal(t, 1, gone.ClaimsUnbound, "the claim is Pending again, on no host")
+	require.Equal(t, []string{vols[0].ID}, gone.VolumeIDs)
+	got, err := st.GetVolumeClaim(c.ID)
+	require.NoError(t, err)
+	require.Equal(t, "", got.BoundVolumeID)
+	require.Equal(t, "", got.VMID, "the attachment went with the VM")
+	require.Zero(t, countRows(t, st, `SELECT COUNT(*) FROM volumes WHERE host_id=?`, hostID))
+}
+// A refusal has to name what is in the way — the API turns these into the 409
+// the caller reads.
+func TestClaimErrorsNameWhatIsInTheWay(t *testing.T) {
+	require.Equal(t, "volume claim c1 is attached to vm vm1",
+		(&ClaimAttachedError{ClaimID: "c1", VMID: "vm1"}).Error())
+	require.Equal(t, "volume claim c1 is bound to host h1",
+		(&ClaimPinnedError{ClaimID: "c1", HostID: "h1"}).Error())
+}
+// One bad claim takes the whole create down, including the claims already
+// bound earlier in the same list: a VM either gets all its volumes or none.
+func TestCreateVMBindsAllClaimsOrNone(t *testing.T) {
+	st := newStore(t)
+	hostID := enrollHost(t, st).ID
+	good, err := st.CreateVolumeClaim(testTenant, "good", 5)
+	require.NoError(t, err)
+	require.ErrorIs(t, st.CreateVM(testVM("vm1", hostID, good.ID, "nope")), ErrClaimNotFound)
+	got, err := st.GetVolumeClaim(good.ID)
+	require.NoError(t, err)
+	require.Equal(t, "", got.BoundVolumeID, "the first claim's binding rolled back with the create")
+	vols, err := st.ListVolumesForHost(hostID)
+	require.NoError(t, err)
+	require.Empty(t, vols, "no volume was placed on the host")
+}
+func TestHardDeleteVolumeRefusesLiveVolume(t *testing.T) {
+	st := newStore(t)
+	hostID := enrollHost(t, st).ID
+	c, err := st.CreateVolumeClaim(testTenant, "data", 5)
+	require.NoError(t, err)
+	require.NoError(t, st.CreateVM(testVM("vm1", hostID, c.ID)))
+	got, err := st.GetVolumeClaim(c.ID)
+	require.NoError(t, err)
+	require.ErrorIs(t, st.HardDeleteVolume(got.BoundVolumeID), sql.ErrNoRows,
+		"a live volume is not reapable; the host still has the file")
+}

internal/server/syncsvc/syncsvc.go

@@ @@ -15,6 +15,7 @@ import ( @@
+	"github.com/a73x/eitri/internal/server/hosttoken"
+	"github.com/a73x/eitri/internal/server/hub"
+	"github.com/a73x/eitri/internal/server/registry"
+	"github.com/a73x/eitri/internal/server/release"
+	"github.com/a73x/eitri/internal/server/store"
+	"github.com/a73x/eitri/internal/transport"
+	"github.com/quic-go/quic-go"
+			// advertises the name, so the agent is only being told what it
+			// said it could serve.
+			Network: v.Network,
+			// The volumes this guest attaches, in the order it named their
+			// claims. The agent materialises each file before boot; an agent
+			// too old to read this field is kept away from such a VM by the
+			// snapshot floor below.
+			VolumeIds: v.VolumeIDs,
+			// With a CA in hand the fleet issues host certificates, so a guest
+			// must present one. The host generates the key, reports the public
+			// half, and holds the guest at the gate until the certificate for
+			Protocol: e.Protocol,
+		})
+	// Tombstoned rows ride along with the live ones: reclaiming a file is work
+	// the agent can only do while it is still told the file exists.
+	vols, err := s.st.ListVolumesForHost(hostID)
+	if err != nil {
+		return nil, fmt.Errorf("list host volumes: %w", err)
+	}
+	for _, v := range vols {
+		snap.Volumes = append(snap.Volumes, &pb.VolumeSpec{
+			VolumeId: v.ID, SizeGb: v.SizeGB, Tombstoned: v.DeletedAt != nil,
+		})
+	}
+	// The floor is raised only by a snapshot that uses the feature: a host
+	// with no volumes keeps serving a pre-volumes agent.
+	if len(snap.Volumes) > 0 {
+		snap.MinAgentVersion = release.Volumes.Since
+	}
+	return snap, nil
+	r.VMs = toRegistryVMs(rep.GetVms())
+	r.Quarantined = toRegistryQuarantined(rep.GetQuarantined())
+	r.Exposures = toRegistryExposures(rep.GetExposures())
+	r.Volumes = toRegistryVolumes(rep.GetVolumes())
+	r.Capacity = toRegistryCapacity(rep.GetCapacity())
+	r.Metrics = toRegistryMetrics(rep.GetMetrics())
+	if anyDeleted {
+		s.hub.Poke(hostID)
+	// A fenced report is the agent saying it refused this snapshot, so nothing
+	// in it is an answer to what the snapshot asked for. It must not drive a
+	// delete: the reap reads an omitted volume as gone, and a report from a
+	// host that acted on nothing would reap every tombstoned row on it.
+	if !rep.GetFenceViolation() {
+		s.reapVolumes(hostID, rep.GetVolumes())
+	}
+}
+// reapVolumes hard-deletes a tombstoned volume once its host reports the file
+// gone: present=false, or absent from a report by an agent that reports
+// volumes at all. A live volume is never touched here whatever the report
+// says — the row is the truth the agent converges toward, and a file it has
+// not made yet is a converge still owed, not a row to delete — and a
+// pre-volumes agent's silence proves nothing, so it reaps nothing. A fenced
+// report reaches none of this: see the caller.
+func (s *Service) reapVolumes(hostID string, reported []*pb.VolumeStatus) {
+	hs, ok := s.reg.Get(hostID)
+	if !ok || !release.Volumes.SupportedBy(hs.AgentVersion) {
+		return
+	}
+	vols, err := s.st.ListVolumesForHost(hostID)
+	if err != nil {
+		slog.Warn("list host volumes", "host", hostID, "err", err)
+		return
+	}
+	present := map[string]bool{}
+	for _, v := range reported {
+		present[v.GetVolumeId()] = v.GetPresent()
+	}
+	reaped := false
+	for _, v := range vols {
+		if v.DeletedAt == nil || present[v.ID] {
+			continue
+		}
+		// The claim is tombstoned by now, so read it whatever its state: the
+		// terminal audit row belongs in the tenant's timeline, and the system
+		// scope is only for the race where the row has already gone.
+		tenant := store.SystemTenant
+		if c, err := s.st.GetVolumeClaimAny(v.ClaimID); err == nil {
+			tenant = c.Tenant
+		}
+		// Refused unless the claim is tombstoned too — the store's guard, not
+		// this loop's assumption. See HardDeleteVolume.
+		if err := s.st.HardDeleteVolume(v.ID); err != nil {
+			slog.Warn("reap volume", "volume", v.ID, "host", hostID, "err", err)
+			continue
+		}
+		reaped = true
+		detail, _ := json.Marshal(map[string]string{"volume_id": v.ID, "host_id": hostID, "claim_id": v.ClaimID})
+		if err := s.st.AppendAudit(tenant, "volume.reap", string(detail)); err != nil {
+			slog.Warn("audit volume.reap failed", "volume", v.ID, "host", hostID, "err", err)
+		}
+	}
+	// The agent still holds a snapshot listing the tombstone and would re-report
+	// it every tick; poke it once so the next snapshot is free of it.
+	if reaped {
+		s.hub.Poke(hostID)
+	}
+}
+// toRegistryVolumes maps reported volume state to registry rows. Returns nil
+// (not an empty slice) for empty input, matching append-into-nil behavior.
+func toRegistryVolumes(in []*pb.VolumeStatus) []registry.VolumeStatus {
+	if len(in) == 0 {
+		return nil
+	}
+	out := make([]registry.VolumeStatus, 0, len(in))
+	for _, v := range in {
+		out = append(out, registry.VolumeStatus{VolumeID: v.GetVolumeId(), Present: v.GetPresent(), SizeGB: v.GetSizeGb()})
+	}
+	return out
+// toRegistryVMs maps reported VMStatus rows to registry rows. Returns nil (not

internal/server/syncsvc/volumes_test.go

@@ @@ -0,0 +1,164 @@ @@
+package syncsvc
+import (
+	"testing"
+	"github.com/a73x/eitri/internal/pb"
+	"github.com/a73x/eitri/internal/server/registry"
+	"github.com/a73x/eitri/internal/server/release"
+	"github.com/a73x/eitri/internal/server/store"
+	"github.com/stretchr/testify/require"
+)
+// claimedVM gives the host one VM holding one 4GiB claim, and returns the
+// claim as the store now reads it — bound to a volume, held by the VM.
+func claimedVM(t *testing.T, st *store.Store, hostID, vmID string) store.VolumeClaim {
+	t.Helper()
+	c, err := st.CreateVolumeClaim(testTenant, vmID+"-data", 4)
+	require.NoError(t, err)
+	require.NoError(t, st.CreateVM(store.VM{ID: vmID, HostID: hostID, Name: vmID, ImageURL: "u", ImageSHA256: "s",
+		VCPUs: 1, MemMB: 1, DiskGB: 1, PowerState: "running", VolumeClaimIDs: []string{c.ID}}))
+	c, err = st.GetVolumeClaim(c.ID)
+	require.NoError(t, err)
+	return c
+}
+func TestSnapshotCarriesVolumesAndFloor(t *testing.T) {
+	f := setup(t)
+	c := claimedVM(t, f.st, f.host.ID, "vm1")
+	snap, err := f.svc.buildSnapshot(f.host.ID)
+	require.NoError(t, err)
+	require.Len(t, snap.Volumes, 1)
+	require.Equal(t, c.BoundVolumeID, snap.Volumes[0].VolumeId)
+	require.EqualValues(t, 4, snap.Volumes[0].SizeGb)
+	require.False(t, snap.Volumes[0].Tombstoned)
+	require.Equal(t, []string{c.BoundVolumeID}, snap.Vms[0].VolumeIds)
+	require.Equal(t, release.Volumes.Since, snap.MinAgentVersion)
+}
+func TestSnapshotWithoutVolumesHasNoFloor(t *testing.T) {
+	f := setup(t)
+	snap, err := f.svc.buildSnapshot(f.host.ID)
+	require.NoError(t, err)
+	require.Equal(t, "", snap.MinAgentVersion, "a pre-volumes agent must keep taking ordinary snapshots")
+}
+// tombstoneFreeClaim gets the claim to the only state a reap may act on: the
+// VM that held it destroyed, the claim itself tombstoned.
+func tombstoneFreeClaim(t *testing.T, st *store.Store, c store.VolumeClaim) {
+	t.Helper()
+	require.NoError(t, st.TombstoneVM(c.VMID))
+	require.NoError(t, st.HardDeleteVM(c.VMID, c.HostID))
+	require.NoError(t, st.TombstoneVolumeClaim(c.ID))
+}
+func TestTombstonedVolumeIsReapedWhenHostReportsItGone(t *testing.T) {
+	f := setup(t)
+	f.reg.SetAgentVersion(f.host.ID, release.Volumes.Since)
+	c := claimedVM(t, f.st, f.host.ID, "vm1")
+	tombstoneFreeClaim(t, f.st, c)
+	f.svc.applyReport(f.host.ID, &pb.Report{Volumes: []*pb.VolumeStatus{{VolumeId: c.BoundVolumeID, Present: true, SizeGb: 4}}})
+	vols, err := f.st.ListVolumesForHost(f.host.ID)
+	require.NoError(t, err)
+	require.Len(t, vols, 1, "the host still has the file")
+	f.svc.applyReport(f.host.ID, &pb.Report{Volumes: []*pb.VolumeStatus{{VolumeId: c.BoundVolumeID, Present: false}}})
+	vols, err = f.st.ListVolumesForHost(f.host.ID)
+	require.NoError(t, err)
+	require.Empty(t, vols, "reported gone: row reaped")
+	// The claim goes with it, so its name is free again.
+	_, err = f.st.GetVolumeClaim(c.ID)
+	require.Error(t, err)
+}
+// A live volume the host does not list is NOT reaped: the row is the truth
+// the agent converges toward, and absence on one tick is a file not yet made.
+func TestLiveVolumeSurvivesAnEmptyReport(t *testing.T) {
+	f := setup(t)
+	f.reg.SetAgentVersion(f.host.ID, release.Volumes.Since)
+	claimedVM(t, f.st, f.host.ID, "vm1")
+	f.svc.applyReport(f.host.ID, &pb.Report{})
+	vols, err := f.st.ListVolumesForHost(f.host.ID)
+	require.NoError(t, err)
+	require.Len(t, vols, 1)
+}
+// Even told outright the file is gone, a live volume stays: the tenant still
+// holds the claim, and the agent's next converge is what makes the file again.
+func TestLiveVolumeSurvivesAPresentFalseReport(t *testing.T) {
+	f := setup(t)
+	f.reg.SetAgentVersion(f.host.ID, release.Volumes.Since)
+	c := claimedVM(t, f.st, f.host.ID, "vm1")
+	f.svc.applyReport(f.host.ID, &pb.Report{Volumes: []*pb.VolumeStatus{{VolumeId: c.BoundVolumeID, Present: false}}})
+	vols, err := f.st.ListVolumesForHost(f.host.ID)
+	require.NoError(t, err)
+	require.Len(t, vols, 1)
+}
+// A pre-volumes agent never reports volumes, so its silence proves nothing
+// about the file; the tombstoned row waits for an agent that can answer.
+func TestTombstonedVolumeIsNotReapedOnAPreVolumesAgentReport(t *testing.T) {
+	f := setup(t)
+	f.reg.SetAgentVersion(f.host.ID, "v0.0.6")
+	c := claimedVM(t, f.st, f.host.ID, "vm1")
+	tombstoneFreeClaim(t, f.st, c)
+	f.svc.applyReport(f.host.ID, &pb.Report{})
+	vols, err := f.st.ListVolumesForHost(f.host.ID)
+	require.NoError(t, err)
+	require.Len(t, vols, 1)
+}
+// A fenced report is the agent refusing a stale snapshot: it acted on nothing,
+// so nothing in it answers what the snapshot asked for. Reaping off it would
+// delete every tombstoned row on the host for a report that never looked.
+func TestTombstonedVolumeIsNotReapedOnAFencedReport(t *testing.T) {
+	f := setup(t)
+	f.reg.SetAgentVersion(f.host.ID, release.Volumes.Since)
+	c := claimedVM(t, f.st, f.host.ID, "vm1")
+	tombstoneFreeClaim(t, f.st, c)
+	f.svc.applyReport(f.host.ID, &pb.Report{FenceViolation: true})
+	vols, err := f.st.ListVolumesForHost(f.host.ID)
+	require.NoError(t, err)
+	require.Len(t, vols, 1, "an omission in a fenced report is not a report that the file is gone")
+	// The same omission from an accepted report still reaps: the guard is the
+	// fence, not a new reluctance.
+	f.svc.applyReport(f.host.ID, &pb.Report{})
+	vols, err = f.st.ListVolumesForHost(f.host.ID)
+	require.NoError(t, err)
+	require.Empty(t, vols)
+}
+// The reap is a tenant event, not a system one: the claim is gone by the time
+// it lands, so the tenant is read from the tombstoned row.
+func TestReapIsAuditedUnderTheClaimsTenant(t *testing.T) {
+	f := setup(t)
+	f.reg.SetAgentVersion(f.host.ID, release.Volumes.Since)
+	c := claimedVM(t, f.st, f.host.ID, "vm1")
+	tombstoneFreeClaim(t, f.st, c)
+	f.svc.applyReport(f.host.ID, &pb.Report{Volumes: []*pb.VolumeStatus{{VolumeId: c.BoundVolumeID, Present: false}}})
+	events, err := f.st.ListAudit(testTenant, 50)
+	require.NoError(t, err)
+	var found bool
+	for _, e := range events {
+		if e.Action == "volume.reap" {
+			found = true
+			require.Contains(t, e.Detail, c.BoundVolumeID)
+		}
+	}
+	require.True(t, found, "volume.reap audited under %s", testTenant)
+}
+// The report reaches the registry, so the console can say what the host found.
+func TestReportedVolumesReachTheRegistry(t *testing.T) {
+	f := setup(t)
+	c := claimedVM(t, f.st, f.host.ID, "vm1")
+	f.svc.applyReport(f.host.ID, &pb.Report{Volumes: []*pb.VolumeStatus{{VolumeId: c.BoundVolumeID, Present: true, SizeGb: 4}}})
+	hs, ok := f.reg.Get(f.host.ID)
+	require.True(t, ok)
+	require.Equal(t, []registry.VolumeStatus{{VolumeID: c.BoundVolumeID, Present: true, SizeGB: 4}}, hs.Volumes)
+}

internal/site/site.go

Old		New
@@ -17,6 +17,7 @@ var pages = []string{
17	"joining",	17	"joining",
18	"connecting",	18	"connecting",
19	"networking",	19	"networking",
		20	"volumes",
20	"self-hosting",	21	"self-hosting",
21	"byo-idp",	22	"byo-idp",
22	"mcp",	23	"mcp",

internal/smoke/gatecheck.go

+	return &gateHooks{
+		register: func(ctx context.Context) error { return auth.Register(ctx) },
+		exec:     func(ctx context.Context, vmName string) error { return gateExec(ctx, cfg, auth, vmName, now, sleep) },
+		run: func(ctx context.Context, vmName, cmd string) (string, error) {
+			return gateRun(ctx, cfg, auth, vmName, cmd, now, sleep)
+		},
+	return nil
+// gateRun runs one command inside a guest through the SSH-CA gate and returns
+// what it printed. Like gateExec it retries the DIAL over the guest's pre-sshd
+// boot window, since a leg that reaches a fresh guest is racing sshd's start.
+// The command itself is not retried: what the callers run is not idempotent —
+// a second mount of an already-mounted volume fails on its own — so a command
+// that ran and failed is the answer, not a reason to try again.
+func gateRun(ctx context.Context, cfg Config, auth gateclient.Credentials, vmName, cmd string, now func() time.Time, sleep func(time.Duration)) (string, error) {
+	var out string
+	var lastErr error
+	err := pollLoop(ctx, now, sleep, 120*time.Second, 5*time.Second, func() (bool, error) {
+		client, dialErr := gateclient.Dial(ctx, gateclient.DialConfig{
+			Gate: cfg.SmokeGate,
+			Auth: auth,
+		}, vmName)
+		if dialErr != nil {
+			lastErr = dialErr
+			return false, nil
+		}
+		defer client.Close()
+		got, runErr := runGuestCommand(client, cmd)
+		if runErr != nil {
+			return false, runErr
+		}
+		out = got
+		return true, nil
+	})
+	if errors.Is(err, errPollTimeout) {
+		return "", fmt.Errorf("could not reach guest %q through the gate within 120s: %w", vmName, lastErr)
+	}
+	return out, err
+}
+// runGuestCommand runs cmd in a new session on client and returns its stdout.
+func runGuestCommand(client *ssh.Client, cmd string) (string, error) {
+	session, err := client.NewSession()

internal/smoke/scenario.go

Old		New
@@ -55,6 +55,8 @@ type vmAPI interface {
55	PatchVM(ctx context.Context, id, powerState string) error	55	PatchVM(ctx context.Context, id, powerState string) error
56	CreateExposure(ctx context.Context, vmID string, guestPort, hostPort int64, protocol string) (client.Exposure, error)	56	CreateExposure(ctx context.Context, vmID string, guestPort, hostPort int64, protocol string) (client.Exposure, error)
57	DeleteExposure(ctx context.Context, id string) error	57	DeleteExposure(ctx context.Context, id string) error
		58	CreateVolumeClaim(ctx context.Context, name string, sizeGB int64) (client.VolumeClaim, error)
		59	DeleteVolumeClaim(ctx context.Context, id string) error
58	}	60	}
59		61
60	// getVM finds the VM with the given id in the current listing. The bool	62	// getVM finds the VM with the given id in the current listing. The bool
@@ -108,6 +110,10 @@ func proveHostRelease(h client.Host, want string) error {
108	type gateHooks struct {	110	type gateHooks struct {
109	register func(ctx context.Context) error // upload the smoke user CA to the tenant (before create)	111	register func(ctx context.Context) error // upload the smoke user CA to the tenant (before create)
110	exec func(ctx context.Context, vmName string) error // reach the guest through the gate (after boot)	112	exec func(ctx context.Context, vmName string) error // reach the guest through the gate (after boot)
		113	// run reaches a guest the same way exec does and hands back what the
		114	// command printed. It is how a leg proves something INSIDE a guest — the
		115	// volume leg's marker — rather than proving the gate itself.
		116	run func(ctx context.Context, vmName, cmd string) (string, error)
111	}	117	}
112		118
113	// mcpLeg is the remote-MCP proof: one full cycle driven entirely through the	119	// mcpLeg is the remote-MCP proof: one full cycle driven entirely through the
@@ -299,8 +305,15 @@ func runScenario(ctx context.Context, vmName string, c vmAPI, dialConsole consol
299	}	305	}
300	return "", err	306	return "", err
301	}	307	}
302	if err := proveBoot(ctx, reboot, now, sleep, "after power cycle"); err != nil {	308	rebootErr := proveBoot(ctx, reboot, now, sleep, "after power cycle")
303	return "", err	309	// The tail's last use, so it is detached HERE rather than left to the
		310	// deferred close: the volume leg below runs for minutes after this VM is
		311	// reaped, and a tail still open would spend them re-dialing a console that
		312	// no longer has a VM behind it. close is idempotent, so the defer above
		313	// stays as the backstop for the early returns between attach and here.
		314	reboot.close()
		315	if rebootErr != nil {
		316	return "", rebootErr
304	}	317	}
305	if gate != nil {	318	if gate != nil {
306	if err := gate.exec(ctx, vmName); err != nil {	319	if err := gate.exec(ctx, vmName); err != nil {
@@ -330,6 +343,23 @@ func runScenario(ctx context.Context, vmName string, c vmAPI, dialConsole consol
330	return "", err	343	return "", err
331	}	344	}
332		345
		346	// Durable storage: a claim that outlives the VM it was attached to. It runs
		347	// on its own pair of VMs — the proof needs one guest to die while another
		348	// picks the volume up — and only where the gate is configured, because the
		349	// marker is written and read INSIDE the guests.
		350	//
		351	// It runs here, after this scenario's own VM is reaped, so the smoke never
		352	// asks the host for more than two guests at once. Its VMs are serial by
		353	// nature (the second cannot exist until the first is destroyed), so the
		354	// host sees one smoke VM at a time from this point on.
		355	volumeOK := false
		356	if gate != nil {
		357	if err := proveVolume(ctx, c, hostID, vmName, gate, readPubKey, now, sleep); err != nil {
		358	return "", err
		359	}
		360	volumeOK = true
		361	}
		362
333	msg := fmt.Sprintf("SMOKE COMPLETE — booted under UEFI, cold_start=%ds, reboot: ok, reaped OK", int64(coldStart.Seconds()))	363	msg := fmt.Sprintf("SMOKE COMPLETE — booted under UEFI, cold_start=%ds, reboot: ok, reaped OK", int64(coldStart.Seconds()))
334	if gateOK {	364	if gateOK {
335	// One clause per thing the gate leg proved: reaching the guest with a	365	// One clause per thing the gate leg proved: reaching the guest with a
@@ -340,6 +370,9 @@ func runScenario(ctx context.Context, vmName string, c vmAPI, dialConsole consol
340	if exposureOK {	370	if exposureOK {
341	msg += ", exposed port: ok"	371	msg += ", exposed port: ok"
342	}	372	}
		373	if volumeOK {
		374	msg += ", volume outlived its VM: ok"
		375	}
343	if mcpOK {	376	if mcpOK {
344	msg += ", remote MCP: ok, published UDP port: ok"	377	msg += ", remote MCP: ok, published UDP port: ok"
345	}	378	}

internal/smoke/scenario_test.go

Old		New
@@ -3,6 +3,7 @@ package smoke
3	import (	3	import (
4	"context"	4	"context"
5	"errors"	5	"errors"
		6	"fmt"
6	"io"	7	"io"
7	"strings"	8	"strings"
8	"testing"	9	"testing"
@@ -62,16 +63,19 @@ func (c *fakeClock) sleep(d time.Duration) {
62	}	63	}
63		64
64	// testAPI implements vmAPI by delegating to per-test closures. patchVMFunc,	65	// testAPI implements vmAPI by delegating to per-test closures. patchVMFunc,
65	// createExposureFunc, and deleteExposureFunc default to a benign answer when	66	// createExposureFunc, deleteExposureFunc, createVolumeClaimFunc, and
66	// nil, so a test whose subject is elsewhere need not supply them.	67	// deleteVolumeClaimFunc default to a benign answer when nil, so a test whose
		68	// subject is elsewhere need not supply them.
67	type testAPI struct {	69	type testAPI struct {
68	listHostsFunc func(ctx context.Context) ([]client.Host, error)	70	listHostsFunc func(ctx context.Context) ([]client.Host, error)
69	createVMFunc func(ctx context.Context, req client.CreateVMRequest) (client.CreateVMResponse, error)	71	createVMFunc func(ctx context.Context, req client.CreateVMRequest) (client.CreateVMResponse, error)
70	listVMsFunc func(ctx context.Context) ([]client.VM, error)	72	listVMsFunc func(ctx context.Context) ([]client.VM, error)
71	deleteVMFunc func(ctx context.Context, id string) error	73	deleteVMFunc func(ctx context.Context, id string) error
72	patchVMFunc func(ctx context.Context, id, powerState string) error	74	patchVMFunc func(ctx context.Context, id, powerState string) error
73	createExposureFunc func(ctx context.Context, vmID string, guestPort, hostPort int64, protocol string) (client.Exposure, error)	75	createExposureFunc func(ctx context.Context, vmID string, guestPort, hostPort int64, protocol string) (client.Exposure, error)
74	deleteExposureFunc func(ctx context.Context, id string) error	76	deleteExposureFunc func(ctx context.Context, id string) error
		77	createVolumeClaimFunc func(ctx context.Context, name string, sizeGB int64) (client.VolumeClaim, error)
		78	deleteVolumeClaimFunc func(ctx context.Context, id string) error
75	}	79	}
76		80
77	func (a *testAPI) ListHosts(ctx context.Context) ([]client.Host, error) {	81	func (a *testAPI) ListHosts(ctx context.Context) ([]client.Host, error) {
@@ -104,6 +108,20 @@ func (a *testAPI) DeleteExposure(ctx context.Context, id string) error {
104	return a.deleteExposureFunc(ctx, id)	108	return a.deleteExposureFunc(ctx, id)
105	}	109	}
106		110
		111	func (a *testAPI) CreateVolumeClaim(ctx context.Context, name string, sizeGB int64) (client.VolumeClaim, error) {
		112	if a.createVolumeClaimFunc == nil {
		113	return client.VolumeClaim{ID: "claim-fake", Name: name, SizeGB: sizeGB, Status: "pending"}, nil
		114	}
		115	return a.createVolumeClaimFunc(ctx, name, sizeGB)
		116	}
		117
		118	func (a *testAPI) DeleteVolumeClaim(ctx context.Context, id string) error {
		119	if a.deleteVolumeClaimFunc == nil {
		120	return nil
		121	}
		122	return a.deleteVolumeClaimFunc(ctx, id)
		123	}
		124
107	func noopReadPubKey() string { return "ssh-ed25519 AAAAfake test@smoke" }	125	func noopReadPubKey() string { return "ssh-ed25519 AAAAfake test@smoke" }
108		126
109	// bootedGuest is what a guest that reached userspace has on its console.	127	// bootedGuest is what a guest that reached userspace has on its console.
@@ -413,42 +431,63 @@ func TestRunScenarioNoHosts(t *testing.T) {
413	// --- runScenario: gate hooks -----------------------------------------------	431	// --- runScenario: gate hooks -----------------------------------------------
414		432
415	// happyPathAPI returns a testAPI that succeeds all the way through reap,	433	// happyPathAPI returns a testAPI that succeeds all the way through reap,
416	// tracking call order in calls (a shared slice each hook also appends to).	434	// tracking call order in calls (a shared slice each hook also appends to). It
		435	// keeps a registry rather than one hard-coded VM, because the legs that run
		436	// under a gate (the volume leg) create VMs of their own alongside the
		437	// scenario's: each one boots after a couple of polls and vanishes when deleted.
417	func happyPathAPI(t testing.T, calls []string) *testAPI {	438	func happyPathAPI(t testing.T, calls []string) *testAPI {
418	t.Helper()	439	t.Helper()
419	listVMsCalls := 0	440	type fakeVM struct {
420	power := "running"	441	polls int
421	deleted := false	442	power string
		443	}
		444	vms := map[string]*fakeVM{}
		445	created := 0
422	return &testAPI{	446	return &testAPI{
423	listHostsFunc: func(ctx context.Context) ([]client.Host, error) {	447	listHostsFunc: func(ctx context.Context) ([]client.Host, error) {
424	return []client.Host{{ID: "host-1", UplinkAddr: "192.168.0.190"}}, nil	448	return []client.Host{{ID: "host-1", UplinkAddr: "192.168.0.190"}}, nil
425	},	449	},
426	createVMFunc: func(ctx context.Context, req client.CreateVMRequest) (client.CreateVMResponse, error) {	450	createVMFunc: func(ctx context.Context, req client.CreateVMRequest) (client.CreateVMResponse, error) {
427	calls = append(calls, "createVM")	451	calls = append(calls, "createVM")
428	return client.CreateVMResponse{ID: "vm-1"}, nil	452	created++
		453	id := fmt.Sprintf("vm-%d", created)
		454	vms[id] = &fakeVM{power: "running"}
		455	return client.CreateVMResponse{ID: id}, nil
429	},	456	},
430	listVMsFunc: func(ctx context.Context) ([]client.VM, error) {	457	listVMsFunc: func(ctx context.Context) ([]client.VM, error) {
431	listVMsCalls++	458	var out []client.VM
432	switch {	459	for id, vm := range vms {
433	case deleted:	460	vm.polls++
434	return nil, nil	461	if vm.polls < 3 {
435	case listVMsCalls < 3:	462	out = append(out, client.VM{ID: id, Phase: "booting"})
436	return []client.VM{{ID: "vm-1", Phase: "booting"}}, nil	463	continue
437	default:	464	}
438	return []client.VM{{ID: "vm-1", Phase: "ready", AssignedIP: "10.0.0.9", ActualPower: power}}, nil	465	out = append(out, client.VM{ID: id, Phase: "ready", AssignedIP: "10.0.0.9", ActualPower: vm.power})
439	}	466	}
		467	return out, nil
440	},	468	},
441	patchVMFunc: func(ctx context.Context, id, powerState string) error {	469	patchVMFunc: func(ctx context.Context, id, powerState string) error {
442	power = powerState	470	if vm, ok := vms[id]; ok {
		471	vm.power = powerState
		472	}
443	return nil	473	return nil
444	},	474	},
445	deleteVMFunc: func(ctx context.Context, id string) error {	475	deleteVMFunc: func(ctx context.Context, id string) error {
446	deleted = true	476	delete(vms, id)
447	return nil	477	return nil
448	},	478	},
449	}	479	}
450	}	480	}
451		481
		482	// okGateRun answers the volume leg the way a working fleet does: the write
		483	// command prints nothing, and the read hands back the marker that was written.
		484	func okGateRun(ctx context.Context, vmName, cmd string) (string, error) {
		485	if strings.Contains(cmd, "cat /mnt/marker") {
		486	return volumeMarker + "\n", nil
		487	}
		488	return "", nil
		489	}
		490
452	func TestRunScenarioGateRegistersBeforeCreateAndExecsAfterBoot(t *testing.T) {	491	func TestRunScenarioGateRegistersBeforeCreateAndExecsAfterBoot(t *testing.T) {
453	var calls []string	492	var calls []string
454	api := happyPathAPI(t, &calls)	493	api := happyPathAPI(t, &calls)
@@ -464,6 +503,7 @@ func TestRunScenarioGateRegistersBeforeCreateAndExecsAfterBoot(t *testing.T) {
464	calls = append(calls, "exec")	503	calls = append(calls, "exec")
465	return nil	504	return nil
466	},	505	},
		506	run: okGateRun,
467	}	507	}
468		508
469	clock := &fakeClock{t: time.Unix(0, 0)}	509	clock := &fakeClock{t: time.Unix(0, 0)}
@@ -480,10 +520,16 @@ func TestRunScenarioGateRegistersBeforeCreateAndExecsAfterBoot(t *testing.T) {
480	if !strings.Contains(msg, "BYO cloud-init merge: ok") {	520	if !strings.Contains(msg, "BYO cloud-init merge: ok") {
481	t.Errorf("message = %q, want it to mention BYO cloud-init merge: ok", msg)	521	t.Errorf("message = %q, want it to mention BYO cloud-init merge: ok", msg)
482	}	522	}
		523	if !strings.Contains(msg, "volume outlived its VM: ok") {
		524	t.Errorf("message = %q, want it to mention the volume leg", msg)
		525	}
483		526
484	// exec appears twice: once after the first boot proof, once after the	527	// exec appears twice: once after the first boot proof, once after the
485	// power-cycle proof — SSH through the gate must survive a reboot too.	528	// power-cycle proof — SSH through the gate must survive a reboot too. The
486	want := []string{"register", "createVM", "exec", "exec"}	529	// two creates at the end are the volume leg's own pair of VMs: it only
		530	// runs where the gate can reach inside a guest, and it waits until this
		531	// scenario's VM is reaped so the host is never asked for a third.
		532	want := []string{"register", "createVM", "exec", "exec", "createVM", "createVM"}
487	if len(calls) != len(want) {	533	if len(calls) != len(want) {
488	t.Fatalf("call order = %v, want %v", calls, want)	534	t.Fatalf("call order = %v, want %v", calls, want)
489	}	535	}
@@ -502,9 +548,10 @@ func TestRunScenarioCreatesItsVMWithATenantCloudInit(t *testing.T) {
502	var calls []string	548	var calls []string
503	api := happyPathAPI(t, &calls)	549	api := happyPathAPI(t, &calls)
504	var got client.CreateVMRequest	550	var got client.CreateVMRequest
		551	create := api.createVMFunc
505	api.createVMFunc = func(ctx context.Context, req client.CreateVMRequest) (client.CreateVMResponse, error) {	552	api.createVMFunc = func(ctx context.Context, req client.CreateVMRequest) (client.CreateVMResponse, error) {
506	got = req	553	got = req
507	return client.CreateVMResponse{ID: "vm-1"}, nil	554	return create(ctx, req)
508	}	555	}
509		556
510	clock := &fakeClock{t: time.Unix(0, 0)}	557	clock := &fakeClock{t: time.Unix(0, 0)}

internal/smoke/volume.go

@@ @@ -0,0 +1,214 @@ @@
+package smoke
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"strings"
+	"time"
+	"github.com/a73x/eitri/internal/server/api/client"
+)
+// volumeMarker is what the first VM writes to the volume and the second one
+// must read back. Nothing about it is special except that an empty disk cannot
+// produce it: a freshly created volume answers with nothing, a recycled one
+// with somebody else's bytes, and both are failures.
+const volumeMarker = "eitri-smoke"
+// volumeClaimSizeGB is the smallest claim worth making. The leg proves the
+// data survives its VM, not that any particular size can be allocated.
+const volumeClaimSizeGB = 1
+// volumeDevice is where the first attached claim lands in the guest — after
+// the root and seed disks, so the first (and here only) claim is /dev/vdc.
+const volumeDevice = "/dev/vdc"
+// writeMarkerCmd formats the volume and leaves the marker on it, unmounting
+// before it returns so every byte is on the disk and not in the guest's page
+// cache when the VM that wrote it is destroyed.
+const writeMarkerCmd = "sudo mkfs.ext4 -q -F " + volumeDevice +
+	" && sudo mount " + volumeDevice + " /mnt" +
+	" && echo " + volumeMarker + " | sudo tee /mnt/marker >/dev/null" +
+	" && sudo umount /mnt"
+// readMarkerCmd mounts the same volume in a VM that never wrote to it and
+// reads back what the previous one left.
+const readMarkerCmd = "sudo mount " + volumeDevice + " /mnt && cat /mnt/marker"
+// proveVolume proves durable storage outlives the VM it was attached to: claim
+// a volume, attach it to a VM that writes a marker on it, destroy that VM
+// entirely, attach the same claim to a second VM, and read the marker back. A
+// volume that answers with anything else — an empty disk, somebody else's data
+// — fails the leg, because a claim that cannot carry a byte past the guest
+// that wrote it is not durable storage.
+//
+// The guest commands go through the SSH-CA gate, so the leg only runs where the
+// gate is configured. Whatever the verdict, both VMs and the claim are gone
+// before it returns: this runs against a live fleet.
+func proveVolume(ctx context.Context, c vmAPI, hostID, vmName string, gate *gateHooks, readPubKey func() string, now func() time.Time, sleep func(time.Duration)) error {
+	claim, err := c.CreateVolumeClaim(ctx, vmName+"-vol", volumeClaimSizeGB)
+	if err != nil {
+		return fmt.Errorf("create volume claim: %w", err)
+	}
+	// VMs this leg still has to clean up. A VM holds its claim until the agent
+	// hard-destroys it — minutes after the delete is accepted, once the
+	// tombstone grace runs out — so every path here waits that reap out before
+	// it tries to delete the claim. A cleanup that gave up sooner would leave
+	// the claim, and the storage behind it, on the fleet on every run.
+	var holders []volumeVM
+	defer func() {
+		// Cleanup outlives a cancelled ctx: leaving a VM and a paid-for volume
+		// on a live plane is worse than the failure that got us here. A cleanup
+		// that itself fails says so on stderr — the leg's own verdict, above,
+		// stands either way.
+		out := context.WithoutCancel(ctx)
+		for _, h := range holders {
+			if _, err := deleteVMAndWaitReaped(out, c, h.id, h.name, claim.ID, now, sleep); err != nil {
+				fmt.Fprintf(os.Stderr, "eitri-smoke: volume cleanup: %v\n", err)
+			}
+		}
+		if err := deleteClaimWhenReleased(out, c, claim.ID, now, sleep); err != nil {
+			fmt.Fprintf(os.Stderr, "eitri-smoke: volume claim %s left behind: %v\n", claim.ID, err)
+		}
+	}()
+	writerName := vmName + "-v1"
+	writerID, err := createVolumeVM(ctx, c, hostID, writerName, claim.ID, readPubKey, now, sleep)
+	if writerID != "" {
+		holders = append(holders, volumeVM{id: writerID, name: writerName})
+	}
+	if err != nil {
+		return err
+	}
+	if _, err := gate.run(ctx, writerName, writeMarkerCmd); err != nil {
+		return fmt.Errorf("FAIL: could not write the marker to the volume on %s: %w", writerName, err)
+	}
+	// Destroy the writer outright — not a reboot, not a detach. The claim is
+	// only proven durable if it survives the guest, the disk image, and the
+	// record of the VM that made it.
+	deleted, err := deleteVMAndWaitReaped(ctx, c, writerID, writerName, claim.ID, now, sleep)
+	if deleted {
+		holders = drop(holders, writerID)
+	}
+	if err != nil {
+		return err
+	}
+	readerName := vmName + "-v2"
+	readerID, err := createVolumeVM(ctx, c, hostID, readerName, claim.ID, readPubKey, now, sleep)
+	if readerID != "" {
+		holders = append(holders, volumeVM{id: readerID, name: readerName})
+	}
+	if err != nil {
+		return err
+	}
+	out, err := gate.run(ctx, readerName, readMarkerCmd)
+	if err != nil {
+		return fmt.Errorf("FAIL: could not read the marker back from the volume on %s: %w", readerName, err)
+	}
+	if got := strings.TrimSpace(out); got != volumeMarker {
+		return fmt.Errorf("FAIL: the volume reattached to %s answered %q, want %q — the data did not survive the VM that wrote it",
+			readerName, truncateBanner(got), volumeMarker)
+	}
+	// The reader goes here rather than in the cleanup above only so a failure
+	// deleting it is the leg's verdict rather than a line on stderr; either way
+	// the claim cannot be deleted until this reap finishes.
+	deleted, err = deleteVMAndWaitReaped(ctx, c, readerID, readerName, claim.ID, now, sleep)
+	if deleted {
+		holders = drop(holders, readerID)
+	}
+	return err
+}
+// volumeVM is one of the leg's VMs: the id to act on, the name to report.
+type volumeVM struct{ id, name string }
+// createVolumeVM creates one VM with the claim attached and waits for it to be
+// ready. It returns the VM's id even when the wait fails, so the caller can
+// still clean up a VM the fleet did create.
+func createVolumeVM(ctx context.Context, c vmAPI, hostID, name, claimID string, readPubKey func() string, now func() time.Time, sleep func(time.Duration)) (string, error) {
+	created, err := c.CreateVM(ctx, client.CreateVMRequest{
+		HostID:           hostID,
+		Name:             name,
+		SSHAuthorizedKey: readPubKey(),
+		VolumeClaims:     []string{claimID},
+	})
+	if err != nil {
+		return "", fmt.Errorf("create vm %s with claim %s: %w", name, claimID, err)
+	}
+	var lastPhase string
+	err = pollLoop(ctx, now, sleep, 600*time.Second, 5*time.Second, func() (bool, error) {
+		vm, _, err := getVM(ctx, c, created.ID)
+		if err != nil {
+			return false, fmt.Errorf("poll vm ready: %w", err)
+		}
+		lastPhase = vm.Phase
+		return vm.Phase == "ready" && vm.AssignedIP != "", nil
+	})
+	if errors.Is(err, errPollTimeout) {
+		return created.ID, fmt.Errorf("FAIL: VM %s (with a volume attached) not ready within 600s (phase=%s)", name, lastPhase)
+	}
+	return created.ID, err
+}
+// deleteVMAndWaitReaped deletes a VM and waits until the fleet stops listing
+// it — the moment the claim it holds becomes attachable again. The window
+// matches the scenario's reap poll: it must outlast the agent's tombstone
+// grace, which a fielded plane runs at five minutes.
+//
+// deleted reports whether the DELETE itself was accepted, and it is true even
+// when the reap then times out, so a caller never re-issues a delete that
+// worked and never blames one that did.
+func deleteVMAndWaitReaped(ctx context.Context, c vmAPI, vmID, name, claimID string, now func() time.Time, sleep func(time.Duration)) (deleted bool, err error) {
+	if err := c.DeleteVM(ctx, vmID); err != nil {
+		return false, fmt.Errorf("delete vm %s: %w", name, err)
+	}
+	err = pollLoop(ctx, now, sleep, 7*time.Minute, 5*time.Second, func() (bool, error) {
+		_, present, err := getVM(ctx, c, vmID)
+		if err != nil {
+			return false, fmt.Errorf("poll vm reaped: %w", err)
+		}
+		return !present, nil
+	})
+	if errors.Is(err, errPollTimeout) {
+		return true, fmt.Errorf("FAIL: VM %s (%s) was deleted but not reaped within 7m; it still holds claim %s",
+			name, vmID, claimID)
+	}
+	return true, err
+}
+// deleteClaimWhenReleased deletes a claim once nothing holds it. Its callers
+// have already waited for the reap of every VM that did, so this is a short
+// retry over the tick between a row disappearing and the plane agreeing it has
+// — NOT the wait for the reap itself, which no 30-second poll could outlast.
+func deleteClaimWhenReleased(ctx context.Context, c vmAPI, claimID string, now func() time.Time, sleep func(time.Duration)) error {
+	var lastErr error
+	err := pollLoop(ctx, now, sleep, 30*time.Second, 3*time.Second, func() (bool, error) {
+		if derr := c.DeleteVolumeClaim(ctx, claimID); derr != nil {
+			lastErr = derr
+			return false, nil
+		}
+		return true, nil
+	})
+	if errors.Is(err, errPollTimeout) {
+		return fmt.Errorf("still refused 30s after its VMs were reaped: %w", lastErr)
+	}
+	return err
+}
+// drop removes the VM with this id, keeping the rest in order.
+func drop(vms []volumeVM, id string) []volumeVM {
+	kept := vms[:0]
+	for _, vm := range vms {
+		if vm.id != id {
+			kept = append(kept, vm)
+		}
+	}
+	return kept
+}

internal/smoke/volume_test.go

@@ @@ -0,0 +1,325 @@ @@
+package smoke
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+	"github.com/a73x/eitri/internal/server/api/client"
+)
+// volumeFleet is a small stateful fake fleet for the volume leg. It models the
+// one property the leg has to get right: a deleted VM is NOT gone. It keeps
+// its row — and with it the claim it holds — for reapAfter, the agent's
+// tombstone grace, and the claim is refused for every second of that. A fake
+// that reaped instantly would call a leg that leaks storage on a live fleet
+// green.
+type volumeFleet struct {
+	t       *testing.T
+	clock   *fakeClock
+	events  []string                 // what the leg asked for, in order
+	created []client.CreateVMRequest // every VM create, in order
+	live    map[string]string        // live vm id -> name
+	dying   map[string]dyingVM       // deleted, still listed until reaped
+	next    int
+	claimID string
+	// refusals is how many delete-claim attempts are refused after the last VM
+	// row is gone — the tick between the reap and the plane agreeing to it.
+	refusals     int
+	claimDeletes int
+	// reapAfter is how long a deleted VM keeps its row. A fielded plane runs
+	// the agent's five-minute tombstone grace, so the fake does too.
+	reapAfter time.Duration
+}
+// dyingVM is a deleted VM still in its tombstone grace.
+type dyingVM struct {
+	name      string
+	destroyAt time.Time
+}
+func newVolumeFleet(t *testing.T, clock *fakeClock) *volumeFleet {
+	t.Helper()
+	return &volumeFleet{
+		t:         t,
+		clock:     clock,
+		live:      map[string]string{},
+		dying:     map[string]dyingVM{},
+		claimID:   "claim-1",
+		reapAfter: 5 * time.Minute,
+	}
+}
+// reap drops the rows whose tombstone grace has run out. It is what every
+// listing and every claim delete consults, so "is it gone yet" has exactly one
+// answer in this fake.
+func (f *volumeFleet) reap() {
+	for id, vm := range f.dying {
+		if !f.clock.now().Before(vm.destroyAt) {
+			delete(f.dying, id)
+		}
+	}
+}
+func (f *volumeFleet) api() *testAPI {
+	return &testAPI{
+		createVolumeClaimFunc: func(ctx context.Context, name string, sizeGB int64) (client.VolumeClaim, error) {
+			f.events = append(f.events, "claim:"+name)
+			return client.VolumeClaim{ID: f.claimID, Name: name, SizeGB: sizeGB}, nil
+		},
+		deleteVolumeClaimFunc: func(ctx context.Context, id string) error {
+			if id != f.claimID {
+				f.t.Errorf("DeleteVolumeClaim id = %q, want %q", id, f.claimID)
+			}
+			f.claimDeletes++
+			f.reap()
+			// A VM in its tombstone grace still holds the claim: the row is
+			// there, the file behind it is still attached to that VM.
+			if len(f.live)+len(f.dying) > 0 {
+				return fmt.Errorf("409: claim %s is attached to a VM", id)
+			}
+			if f.claimDeletes <= f.refusals {
+				return fmt.Errorf("409: claim %s is still attached", id)
+			}
+			f.events = append(f.events, "claim-delete")
+			return nil
+		},
+		createVMFunc: func(ctx context.Context, req client.CreateVMRequest) (client.CreateVMResponse, error) {
+			f.next++
+			id := fmt.Sprintf("vm-%d", f.next)
+			f.created = append(f.created, req)
+			f.live[id] = req.Name
+			f.events = append(f.events, "vm:"+req.Name)
+			return client.CreateVMResponse{ID: id}, nil
+		},
+		listVMsFunc: func(ctx context.Context) ([]client.VM, error) {
+			f.reap()
+			var out []client.VM
+			for id, name := range f.live {
+				out = append(out, client.VM{ID: id, Name: name, Phase: "ready", AssignedIP: "10.0.0.9"})
+			}
+			for id, vm := range f.dying {
+				out = append(out, client.VM{ID: id, Name: vm.name, Phase: "ready", AssignedIP: "10.0.0.9", Deleted: true})
+			}
+			return out, nil
+		},
+		deleteVMFunc: func(ctx context.Context, id string) error {
+			name, ok := f.live[id]
+			if !ok {
+				// A delete re-issued against a row that is already deleted is
+				// recorded before it is refused, so a test can see the wasted
+				// call and the 404 it earns.
+				if vm, dying := f.dying[id]; dying {
+					f.events = append(f.events, "delete:"+vm.name)
+					return fmt.Errorf("404: VM %s is already deleted", id)
+				}
+				return fmt.Errorf("404: no VM %s", id)
+			}
+			f.events = append(f.events, "delete:"+name)
+			delete(f.live, id)
+			f.dying[id] = dyingVM{name: name, destroyAt: f.clock.now().Add(f.reapAfter)}
+			return nil
+		},
+	}
+}
+// markerGate answers the way a working guest does: the write command says
+// nothing, and the read command hands the marker back.
+func markerGate(record *[]string, marker string, writeErr error) *gateHooks {
+	return &gateHooks{
+		run: func(ctx context.Context, vmName, cmd string) (string, error) {
+			*record = append(*record, vmName)
+			if strings.Contains(cmd, "mkfs") {
+				return "", writeErr
+			}
+			return marker + "\n", nil
+		},
+	}
+}
+func TestProveVolumeCarriesAMarkerPastTheVMThatWroteIt(t *testing.T) {
+	clock := &fakeClock{t: time.Unix(0, 0)}
+	fleet := newVolumeFleet(t, clock)
+	var ran []string
+	gate := markerGate(&ran, volumeMarker, nil)
+	if err := proveVolume(context.Background(), fleet.api(), "host-1", "smoke-test", gate, noopReadPubKey, clock.now, clock.sleep); err != nil {
+		t.Fatalf("proveVolume: %v", err)
+	}
+	// The claim is storage the VMs attach to, so it exists before either of
+	// them: a claim made after the fact could not be the disk they booted with.
+	want := []string{
+		"claim:smoke-test-vol",
+		"vm:smoke-test-v1",
+		"delete:smoke-test-v1",
+		"vm:smoke-test-v2",
+		"delete:smoke-test-v2",
+		"claim-delete",
+	}
+	if strings.Join(fleet.events, ",") != strings.Join(want, ",") {
+		t.Fatalf("fleet calls = %v, want %v", fleet.events, want)
+	}
+	if len(fleet.created) != 2 {
+		t.Fatalf("VM creates = %d, want 2", len(fleet.created))
+	}
+	for i, req := range fleet.created {
+		if len(req.VolumeClaims) != 1 || req.VolumeClaims[0] != "claim-1" {
+			t.Errorf("VM %d VolumeClaims = %v, want [claim-1] — both VMs must name the same claim", i+1, req.VolumeClaims)
+		}
+		if req.HostID != "host-1" {
+			t.Errorf("VM %d HostID = %q, want host-1 — the claim is bound to one host", i+1, req.HostID)
+		}
+		if req.SSHAuthorizedKey != noopReadPubKey() {
+			t.Errorf("VM %d SSHAuthorizedKey = %q, want the local key", i+1, req.SSHAuthorizedKey)
+		}
+	}
+	if len(ran) != 2 || ran[0] != "smoke-test-v1" || ran[1] != "smoke-test-v2" {
+		t.Errorf("guest commands ran on %v, want [smoke-test-v1 smoke-test-v2]", ran)
+	}
+}
+// TestProveVolumeFailsWhenTheMarkerComesBackWrong is the leg's whole point: a
+// volume that reattaches but answers with something else did not carry the
+// data across, and the failure quotes what it did answer with.
+func TestProveVolumeFailsWhenTheMarkerComesBackWrong(t *testing.T) {
+	clock := &fakeClock{t: time.Unix(0, 0)}
+	fleet := newVolumeFleet(t, clock)
+	var ran []string
+	gate := markerGate(&ran, "a-fresh-empty-disk", nil)
+	err := proveVolume(context.Background(), fleet.api(), "host-1", "smoke-test", gate, noopReadPubKey, clock.now, clock.sleep)
+	if err == nil {
+		t.Fatal("proveVolume: want an error when the marker came back wrong, got nil")
+	}
+	if !strings.Contains(err.Error(), "a-fresh-empty-disk") {
+		t.Errorf("error = %q, want it to quote what the volume answered with", err.Error())
+	}
+	if !strings.Contains(err.Error(), volumeMarker) {
+		t.Errorf("error = %q, want it to name the marker it wanted", err.Error())
+	}
+	// The failing VM and the storage behind it are still cleaned up.
+	if len(fleet.live) != 0 {
+		t.Errorf("live VMs = %v, want none — the leg deletes its VMs on the failure path too", fleet.live)
+	}
+	if fleet.claimDeletes == 0 {
+		t.Error("the claim was never deleted; a failed leg must not leave storage behind")
+	}
+}
+// TestProveVolumeCleansUpWhenTheGuestCommandFails covers the earlier failure
+// point: the first VM is up but cannot write, so that VM and the claim both go.
+func TestProveVolumeCleansUpWhenTheGuestCommandFails(t *testing.T) {
+	clock := &fakeClock{t: time.Unix(0, 0)}
+	fleet := newVolumeFleet(t, clock)
+	var ran []string
+	gate := markerGate(&ran, volumeMarker, errors.New("mkfs: no such device"))
+	err := proveVolume(context.Background(), fleet.api(), "host-1", "smoke-test", gate, noopReadPubKey, clock.now, clock.sleep)
+	if err == nil {
+		t.Fatal("proveVolume: want an error when the guest could not write the marker, got nil")
+	}
+	if !strings.Contains(err.Error(), "no such device") {
+		t.Errorf("error = %q, want it to carry the guest's own complaint", err.Error())
+	}
+	if len(fleet.created) != 1 {
+		t.Errorf("VM creates = %d, want 1 — the second VM proves nothing once the first never wrote", len(fleet.created))
+	}
+	want := []string{"claim:smoke-test-vol", "vm:smoke-test-v1", "delete:smoke-test-v1", "claim-delete"}
+	if strings.Join(fleet.events, ",") != strings.Join(want, ",") {
+		t.Errorf("fleet calls = %v, want %v", fleet.events, want)
+	}
+}
+// TestProveVolumeFreesTheClaimAfterTheTombstoneGrace is the leak guard. A VM
+// holds its claim until the agent hard-destroys it, minutes after the delete
+// is accepted, so a leg that deletes the claim on the way out without waiting
+// that grace out leaves a claim and the storage behind it on a live fleet —
+// every single run. The proof that it waited is the clock: two full graces of
+// virtual time passed, one per VM.
+func TestProveVolumeFreesTheClaimAfterTheTombstoneGrace(t *testing.T) {
+	clock := &fakeClock{t: time.Unix(0, 0)}
+	fleet := newVolumeFleet(t, clock)
+	start := clock.now()
+	var ran []string
+	gate := markerGate(&ran, volumeMarker, nil)
+	if err := proveVolume(context.Background(), fleet.api(), "host-1", "smoke-test", gate, noopReadPubKey, clock.now, clock.sleep); err != nil {
+		t.Fatalf("proveVolume: %v", err)
+	}
+	if fleet.events[len(fleet.events)-1] != "claim-delete" {
+		t.Fatalf("fleet calls = %v, want the claim actually deleted at the end", fleet.events)
+	}
+	if len(fleet.live) != 0 || len(fleet.dying) != 0 {
+		t.Errorf("rows left = %v / %v, want none", fleet.live, fleet.dying)
+	}
+	if waited := clock.now().Sub(start); waited < 2*fleet.reapAfter {
+		t.Errorf("leg waited %s, want at least %s — one tombstone grace per VM before the claim can go",
+			waited, 2*fleet.reapAfter)
+	}
+}
+// TestProveVolumeRetriesTheClaimDeleteAfterTheReap: the plane can still refuse
+// for a tick after the row is gone, so the delete is retried rather than
+// reported as storage the leg never actually left behind.
+func TestProveVolumeRetriesTheClaimDeleteAfterTheReap(t *testing.T) {
+	clock := &fakeClock{t: time.Unix(0, 0)}
+	fleet := newVolumeFleet(t, clock)
+	fleet.refusals = 2
+	var ran []string
+	gate := markerGate(&ran, volumeMarker, nil)
+	if err := proveVolume(context.Background(), fleet.api(), "host-1", "smoke-test", gate, noopReadPubKey, clock.now, clock.sleep); err != nil {
+		t.Fatalf("proveVolume: %v", err)
+	}
+	if fleet.claimDeletes != 3 {
+		t.Errorf("DeleteVolumeClaim attempts = %d, want 3 — two refusals then the delete that lands", fleet.claimDeletes)
+	}
+	if fleet.events[len(fleet.events)-1] != "claim-delete" {
+		t.Errorf("fleet calls = %v, want the claim deleted last", fleet.events)
+	}
+}
+// TestProveVolumeFailsWhenTheFirstVMIsNeverReaped: the second VM cannot attach
+// a claim the first one still holds, so a VM that will not go away fails the
+// leg here rather than as a mystery 409 on the next create. The delete itself
+// was accepted — the fleet simply never finished it — so the message says
+// that, and the cleanup does not re-issue a delete that already worked.
+func TestProveVolumeFailsWhenTheFirstVMIsNeverReaped(t *testing.T) {
+	clock := &fakeClock{t: time.Unix(0, 0)}
+	fleet := newVolumeFleet(t, clock)
+	fleet.reapAfter = 24 * time.Hour // accepted, never finished
+	var ran []string
+	gate := markerGate(&ran, volumeMarker, nil)
+	err := proveVolume(context.Background(), fleet.api(), "host-1", "smoke-test", gate, noopReadPubKey, clock.now, clock.sleep)
+	if err == nil {
+		t.Fatal("proveVolume: want an error when the first VM was never reaped, got nil")
+	}
+	if !strings.Contains(err.Error(), "smoke-test-v1") {
+		t.Errorf("error = %q, want it to name the VM that would not go away", err.Error())
+	}
+	if !strings.Contains(err.Error(), "deleted but not reaped") {
+		t.Errorf("error = %q, want it to say the delete was accepted and the reap never finished", err.Error())
+	}
+	if !strings.Contains(err.Error(), "claim-1") {
+		t.Errorf("error = %q, want it to name the claim the VM is still holding", err.Error())
+	}
+	if len(fleet.created) != 1 {
+		t.Errorf("VM creates = %d, want 1 — the second VM is never created", len(fleet.created))
+	}
+	// One delete, not two: the cleanup must not re-issue a delete that was
+	// accepted, or the 404 it earns would blame a delete that worked.
+	deletes := 0
+	for _, e := range fleet.events {
+		if strings.HasPrefix(e, "delete:") {
+			deletes++
+		}
+	}
+	if deletes != 1 {
+		t.Errorf("DeleteVM calls = %d, want 1 — the accepted delete is not re-issued on the way out", deletes)
+	}
+}

internal/transport/fieldnumbers_test.go

Old		New
@@ -108,6 +108,7 @@ var wireSchema = map[string]map[string]protoreflect.FieldNumber{
108	"guest_cidr": 8,	108	"guest_cidr": 8,
109	"exposures": 9,	109	"exposures": 9,
110	"host_uplink_addr": 10,	110	"host_uplink_addr": 10,
		111	"volumes": 11,
111	},	112	},
112	"VMSpec": {	113	"VMSpec": {
113	"vm_id": 1,	114	"vm_id": 1,
@@ -126,12 +127,14 @@ var wireSchema = map[string]map[string]protoreflect.FieldNumber{
126	"ssh_user_ca_authorized_keys": 18,	127	"ssh_user_ca_authorized_keys": 18,
127	"host_cert_required": 19,	128	"host_cert_required": 19,
128	"network": 20,	129	"network": 20,
		130	"volume_ids": 21,
129	},	131	},
130	"Snapshot": {	132	"Snapshot": {
131	"epoch": 1,	133	"epoch": 1,
132	"vms": 2,	134	"vms": 2,
133	"agent_upgrade": 3,	135	"agent_upgrade": 3,
134	"exposures": 4,	136	"exposures": 4,
		137	"volumes": 5,
135	"min_agent_version": 6,	138	"min_agent_version": 6,
136	},	139	},
137	"AgentUpgrade": {	140	"AgentUpgrade": {
@@ -172,6 +175,16 @@ var wireSchema = map[string]map[string]protoreflect.FieldNumber{
172	"refused": 2,	175	"refused": 2,
173	"dropped": 3,	176	"dropped": 3,
174	},	177	},
		178	"VolumeSpec": {
		179	"volume_id": 1,
		180	"size_gb": 2,
		181	"tombstoned": 3,
		182	},
		183	"VolumeStatus": {
		184	"volume_id": 1,
		185	"present": 2,
		186	"size_gb": 3,
		187	},
175	}	188	}
176		189
177	// TestWireFieldNumbersAreLocked walks every message in the compiled wire schema	190	// TestWireFieldNumbersAreLocked walks every message in the compiled wire schema

proto/eitri/v1/sync.proto

Old		New
@@ -142,6 +142,7 @@ message Report {
142	// report rather than Hello for the same reason guest_cidr is: it can change	142	// report rather than Hello for the same reason guest_cidr is: it can change
143	// while an agent stays connected.	143	// while an agent stays connected.
144	string host_uplink_addr = 10;	144	string host_uplink_addr = 10;
		145	repeated VolumeStatus volumes = 11;
145	}	146	}
146		147
147	// VMSpec is the half of a VM the control plane owns. See VMStatus.	148	// VMSpec is the half of a VM the control plane owns. See VMStatus.
@@ -192,6 +193,10 @@ message VMSpec {
192	// it (see Hello.host_networks); an agent that has the name but no longer	193	// it (see Hello.host_networks); an agent that has the name but no longer
193	// the configuration fails the VM legibly rather than silently NAT-ing it.	194	// the configuration fails the VM legibly rather than silently NAT-ing it.
194	string network = 20;	195	string network = 20;
		196	// volume_ids are the bound volumes to attach after root and seed, in this
		197	// order — the first is /dev/vdc. Every id names a VolumeSpec in the same
		198	// snapshot.
		199	repeated string volume_ids = 21;
195	}	200	}
196		201
197	// Snapshot is the FULL spec for one host; the agent converges toward it.	202	// Snapshot is the FULL spec for one host; the agent converges toward it.
@@ -200,7 +205,7 @@ message Snapshot {
200	repeated VMSpec vms = 2; // FULL set for this host, including tombstoned	205	repeated VMSpec vms = 2; // FULL set for this host, including tombstoned
201	AgentUpgrade agent_upgrade = 3; // optional operator-initiated agent self-upgrade	206	AgentUpgrade agent_upgrade = 3; // optional operator-initiated agent self-upgrade
202	repeated ExposureSpec exposures = 4; // FULL set for this host	207	repeated ExposureSpec exposures = 4; // FULL set for this host
203	// 5 is taken by volumes in the next change.	208	repeated VolumeSpec volumes = 5; // FULL set for this host
204	// min_agent_version is the lowest agent release that understands every	209	// min_agent_version is the lowest agent release that understands every
205	// field in this snapshot. An agent below it fails every VM here with a	210	// field in this snapshot. An agent below it fails every VM here with a
206	// legible reason rather than materialising a spec it only half-reads.	211	// legible reason rather than materialising a spec it only half-reads.
@@ -301,3 +306,21 @@ message ExposureSessions {
301	// — as opposed to ones the cap refused.	306	// — as opposed to ones the cap refused.
302	int64 dropped = 3;	307	int64 dropped = 3;
303	}	308	}
		309
		310	// VolumeSpec is one volume the control plane has placed on this host. The
		311	// agent keeps a sparse raw file for it at volumes/<volume_id>/disk.raw, beside
		312	// vms/ and never inside a VM's directory: a VM's directory dies with the VM,
		313	// a volume's does not. Snapshot.volumes is the FULL set for the host.
		314	message VolumeSpec {
		315	string volume_id = 1;
		316	int64 size_gb = 2;
		317	bool tombstoned = 3; // delete the file after grace; set only once no VM references it
		318	}
		319
		320	// VolumeStatus is what the host found on its disk for one volume id —
		321	// including ids the snapshot did not name, which are reported and kept.
		322	message VolumeStatus {
		323	string volume_id = 1;
		324	bool present = 2;
		325	int64 size_gb = 3; // as found, not as asked: a guest filesystem sits on it, so it is never resized
		326	}

scripts/coverage.sh

Old		New
@@ -83,7 +83,7 @@ declare -A FLOOR=(
83	[internal/transport]=83	83	[internal/transport]=83
84	[internal/shape]=92	84	[internal/shape]=92
85	[internal/site]=87	85	[internal/site]=87
86	[internal/smoke]=57	86	[internal/smoke]=58
87	[internal/cli]=72	87	[internal/cli]=72
88	[internal/mcpserver]=77	88	[internal/mcpserver]=77
89	[internal/oidcprovider]=81	89	[internal/oidcprovider]=81

site/docs.md

Old		New
@@ -11,6 +11,8 @@
11		11
12	- [networking](networking.md)—publishing a port a guest serves, or giving a VM	12	- [networking](networking.md)—publishing a port a guest serves, or giving a VM
13	an address on your own LAN	13	an address on your own LAN
		14	- [volumes](volumes.md)—durable disks a VM attaches at create and keeps after
		15	it dies
14	- [mcp](mcp.md)—let an AI agent create and drive VMs	16	- [mcp](mcp.md)—let an AI agent create and drive VMs
15		17
16	Run a fleet	18	Run a fleet

web/src/lib/api-types.ts

@@ @@ -1480,6 +1480,160 @@ export interface paths { @@
+        patch?: never;
+        trace?: never;
+    };
+    "/api/v1/volume-claims": {
+        parameters: {
+            query?: never;
+            header?: never;
+            path?: never;
+            cookie?: never;
+        };
+        /** List the tenant's claims: where each is bound, which VM holds it, and whether its host has the file. */
+        get: {
+            parameters: {
+                query?: never;
+                header?: never;
+                path?: never;
+                cookie?: never;
+            };
+            requestBody?: never;
+            responses: {
+                /** @description success */
+: {
+                    headers: {
+                        [name: string]: unknown;
+                    };
+                    content: {
+                        "application/json": components["schemas"]["VolumeClaim"][];
+                    };
+                };
+                /** @description error (plain text) */
+                default: {
+                    headers: {
+                        [name: string]: unknown;
+                    };
+                    content: {
+                        "text/plain": string;
+                    };
+                };
+            };
+        };
+        put?: never;
+        /** Claim durable storage. Pending until the first VM naming it is created; that VM's host then holds the bytes, and every later VM using the claim is placed there. */
+        post: {
+            parameters: {
+                query?: never;
+                header?: never;
+                path?: never;
+                cookie?: never;
+            };
+            requestBody: {
+                content: {
+                    "application/json": components["schemas"]["CreateVolumeClaimRequest"];
+                };
+            };
+            responses: {
+                /** @description success */
+: {
+                    headers: {
+                        [name: string]: unknown;
+                    };
+                    content: {
+                        "application/json": components["schemas"]["VolumeClaim"];
+                    };
+                };
+                /** @description error (plain text) */
+                default: {
+                    headers: {
+                        [name: string]: unknown;
+                    };
+                    content: {
+                        "text/plain": string;
+                    };
+                };
+            };
+        };
+        delete?: never;
+        options?: never;
+        head?: never;
+        patch?: never;
+        trace?: never;
+    };
+    "/api/v1/volume-claims/{id}": {
+        parameters: {
+            query?: never;
+            header?: never;
+            path?: never;
+            cookie?: never;
+        };
+        /** One claim. */
+        get: {
+            parameters: {
+                query?: never;
+                header?: never;
+                path: {
+                    id: string;
+                };
+                cookie?: never;
+            };
+            requestBody?: never;
+            responses: {
+                /** @description success */
+: {
+                    headers: {
+                        [name: string]: unknown;
+                    };
+                    content: {
+                        "application/json": components["schemas"]["VolumeClaim"];
+                    };
+                };
+                /** @description error (plain text) */
+                default: {
+                    headers: {
+                        [name: string]: unknown;
+                    };
+                    content: {
+                        "text/plain": string;
+                    };
+                };
+            };
+        };
+        put?: never;
+        post?: never;
+        /** Delete a claim and the data behind it. Refused (409) while a VM holds it. */
+        delete: {
+            parameters: {
+                query?: never;
+                header?: never;
+                path: {
+                    id: string;
+                };
+                cookie?: never;
+            };
+            requestBody?: never;
+            responses: {
+                /** @description success */
+: {
+                    headers: {
+                        [name: string]: unknown;
+                    };
+                    content?: never;
+                };
+                /** @description error (plain text) */
+                default: {
+                    headers: {
+                        [name: string]: unknown;
+                    };
+                    content: {
+                        "text/plain": string;
+                    };
+                };
+            };
+        };
+        options?: never;
+        head?: never;
+        patch?: never;
+        trace?: never;
+    };
+export type webhooks = Record<string, never>;
+export interface components {
@@ @@ -1530,11 +1684,16 @@ export interface components { @@
+            power_state?: string;
+            ssh_authorized_key?: string;
+            vcpus?: number;
+            volume_claims?: string[];
+        };
+        CreateVMResponse: {
+            id: string;
+            name: string;
+        };
+        CreateVolumeClaimRequest: {
+            name?: string;
+            size_gb?: number;
+        };
+        Delegation: {
+            ca_fingerprint: string;
+            expires_at: string;
@@ @@ -1711,6 +1870,17 @@ export interface components { @@
+            trusted_cas?: components["schemas"]["TrustedCA"][] | null;
+            vcpus: number;
+        };
+        VolumeClaim: {
+            /** Format: date-time */
+            created_at: string;
+            host_id: string;
+            id: string;
+            name: string;
+            present?: boolean | null;
+            size_gb: number;
+            status: string;
+            vm_id: string;
+        };
+    };
+    responses: never;
+    parameters: never;