a73x

ee0c9258

feat(fleet): sweep VMs no agent will reap

a73x   2026-07-29 19:39

Commit message
feat(fleet): sweep VMs no agent will reap

A VM deleted while its host has no live agent — the host is gone, or
crashed — has no agent to send the destroy ack that hard-deletes the
row. The server backstops that ack: a periodic sweep hard-deletes any
VM tombstoned longer than a fixed grace whose host is not currently
online, reusing the existing StartBackground loop. An online host is
left to its agent's normal quarantine→destroy→ack chain. The grace
exceeds the agent's own TombstoneGrace so a live agent always reaps
first, and is far longer than any routine restart or redeploy so a
transient outage never forces a delete. If a swept host later returns,
the VM is absent from desired state and the agent's vanish-reap tears
down any lingering guest.
internal/server/api/abandoned_vm_test.go
Old New
@@ -0,0 +1,88 @@
1 package api
2
3 import (
4 "database/sql"
5 "errors"
6 "net/http"
7 "net/http/httptest"
8 "testing"
9 "time"
10
11 "github.com/a73x/eitri/internal/server/registry"
12 "github.com/a73x/eitri/internal/server/store"
13 "github.com/stretchr/testify/assert"
14 "github.com/stretchr/testify/require"
15 )
16
17 // tombstoneOneVM enrolls a host, creates a VM on it, and DELETEs (tombstones)
18 // it, returning the vm id. The host never reports, so its agent is offline in
19 // the registry — the ack that would normally hard-delete the row never comes.
20 func tombstoneOneVM(t *testing.T, ts *httptest.Server, st *store.Store) string {
21 t.Helper()
22 out := enroll(t, ts)
23 hostID := out["host_id"]
24 resp := do(t, "POST", ts.URL+"/api/v1/vms", testPAT, map[string]any{"host_id": hostID, "name": "vm-a"})
25 require.Equal(t, http.StatusCreated, resp.StatusCode)
26
27 vms, err := st.ListVMs()
28 require.NoError(t, err)
29 require.Len(t, vms, 1)
30 id := vms[0].ID
31
32 resp = do(t, "DELETE", ts.URL+"/api/v1/vms/"+id, testPAT, nil)
33 require.Equal(t, http.StatusNoContent, resp.StatusCode)
34
35 got, err := st.GetVM(id)
36 require.NoError(t, err, "tombstoned row must still exist (no agent acked it)")
37 require.NotNil(t, got.DeletedAt, "row is tombstoned")
38 return id
39 }
40
41 // TestAbandonedTombstoneOnOfflineHostIsReaped pins the failed-VM-delete fix: a
42 // VM deleted while its host's agent is offline (never acks the destroy) would
43 // otherwise linger forever. Past the grace, the server-side sweep hard-deletes
44 // it so the console row clears.
45 func TestAbandonedTombstoneOnOfflineHostIsReaped(t *testing.T) {
46 ts, a, st := apiServer(t)
47 id := tombstoneOneVM(t, ts, st)
48
49 // Well past the abandoned-reap grace, host still offline (never reported).
50 assert.True(t, a.sweepAbandonedVMs(time.Now().Add(2*abandonedVMReapGrace)),
51 "sweep should reap a long-tombstoned VM on an offline host")
52
53 _, err := st.GetVM(id)
54 assert.True(t, errors.Is(err, sql.ErrNoRows), "the zombie row must be gone after the sweep")
55 }
56
57 // TestFreshTombstoneIsNotReaped guards the grace: a just-deleted VM must be
58 // left for the agent to ack normally — a transient host outage (agent restart,
59 // redeploy) must not trip an immediate server-side force-delete.
60 func TestFreshTombstoneIsNotReaped(t *testing.T) {
61 ts, a, st := apiServer(t)
62 id := tombstoneOneVM(t, ts, st)
63
64 assert.False(t, a.sweepAbandonedVMs(time.Now()),
65 "a VM tombstoned within the grace must not be force-deleted")
66
67 _, err := st.GetVM(id)
68 assert.NoError(t, err, "row must survive a within-grace sweep")
69 }
70
71 // TestOnlineHostTombstoneIsNotReaped guards the offline gate: while the host's
72 // agent is live it will ack the destroy through the normal quarantine→destroy
73 // path, so the server must not race it — even for an old tombstone.
74 func TestOnlineHostTombstoneIsNotReaped(t *testing.T) {
75 ts, a, st := apiServer(t)
76 id := tombstoneOneVM(t, ts, st)
77
78 // Make the host online: a fresh report sets LastSeen to now.
79 vms, err := st.ListVMs()
80 require.NoError(t, err)
81 a.reg.UpdateReport(vms[0].HostID, registry.Report{})
82
83 assert.False(t, a.sweepAbandonedVMs(time.Now().Add(2*abandonedVMReapGrace)),
84 "a live agent owns the reap; the server must not force-delete")
85
86 _, err = st.GetVM(id)
87 assert.NoError(t, err, "row must survive while the host is online")
88 }
internal/server/api/api.go
Old New
@@ -151,8 +151,10 @@ func (a *API) AuthHandler() http.Handler {
151 return mux 151 return mux
152 } 152 }
153 153
154 // StartBackground launches the decommission sweeper, which finalizes drained 154 // StartBackground launches the periodic server-side sweeps: finalizing drained
155 // decommissioning hosts. It returns when ctx is cancelled. 155 // decommissioning hosts, and reaping tombstoned VMs whose host's agent never
156 // acked the destroy (abandoned on an offline host). It returns when ctx is
157 // cancelled.
156 func (a *API) StartBackground(ctx context.Context) { 158 func (a *API) StartBackground(ctx context.Context) {
157 t := time.NewTicker(2 * time.Second) 159 t := time.NewTicker(2 * time.Second)
158 defer t.Stop() 160 defer t.Stop()
@@ -161,13 +163,65 @@ func (a *API) StartBackground(ctx context.Context) {
161 case <-ctx.Done(): 163 case <-ctx.Done():
162 return 164 return
163 case <-t.C: 165 case <-t.C:
164 if a.sweepDecommissioned() { 166 changed := a.sweepDecommissioned()
167 if a.sweepAbandonedVMs(time.Now()) {
168 changed = true
169 }
170 if changed {
165 a.notif.notify() 171 a.notif.notify()
166 } 172 }
167 } 173 }
168 } 174 }
169 } 175 }
170 176
177 // abandonedVMReapGrace is how long a tombstoned VM may sit unacked on an offline
178 // host before the server hard-deletes it itself. The normal delete path is the
179 // agent's quarantine→destroy→ack chain; this is the backstop for when no agent
180 // is there to run it (the host is gone, or crashed the way a failed-VM host
181 // tends to). The bound is deliberately generous: it exceeds the agent's own
182 // TombstoneGrace so a live agent always reaps first, and it is far longer than
183 // any routine agent restart or redeploy so a transient outage never trips a
184 // server-side force-delete. Correctness survives a host that returns AFTER a
185 // reap: the VM is then absent from desired state, and the agent's vanish-reap
186 // tears down any lingering guest. No config knob — a fixed policy bound.
187 const abandonedVMReapGrace = 15 * time.Minute
188
189 // sweepAbandonedVMs hard-deletes every VM tombstoned longer than
190 // abandonedVMReapGrace whose host is NOT currently online — the reap the agent
191 // would have acked, done server-side because no agent is connected to do it.
192 // An online host is left alone: its live agent owns the reap and will ack
193 // through the normal path, and racing it risks flipping the VM onto the longer
194 // vanished-grace clock. Returns true if it removed at least one row. now is
195 // threaded in so the age check and the loop share one clock.
196 func (a *API) sweepAbandonedVMs(now time.Time) bool {
197 vms, err := a.st.ListVMs()
198 if err != nil {
199 return false
200 }
201 reaped := false
202 for _, vm := range vms {
203 if vm.DeletedAt == nil {
204 continue // live VM — not a delete in progress
205 }
206 if now.Sub(*vm.DeletedAt) < abandonedVMReapGrace {
207 continue // within grace — leave it for the agent to ack
208 }
209 if st, ok := a.reg.Get(vm.HostID); ok && st.Online {
210 continue // agent is live; it owns the reap
211 }
212 if err := a.st.HardDeleteVM(vm.ID); err != nil {
213 slog.Warn("sweep abandoned VM failed", "vm", vm.ID, "host", vm.HostID, "err", err)
214 continue
215 }
216 reaped = true
217 a.audit(vm.Tenant, "vm.reap", map[string]string{
218 "vm_id": vm.ID, "host_id": vm.HostID,
219 "reason": "tombstoned VM reaped server-side: host offline past grace",
220 })
221 }
222 return reaped
223 }
224
171 // sweepDecommissioned finalizes any decommissioning host with no VM rows left 225 // sweepDecommissioned finalizes any decommissioning host with no VM rows left
172 // (fully drained). Returns true if it removed at least one host. 226 // (fully drained). Returns true if it removed at least one host.
173 func (a *API) sweepDecommissioned() bool { 227 func (a *API) sweepDecommissioned() bool {