internal/server/api/abandoned_vm_test.go

Ref: main v0.0.5   Size: 3.9 KiB   History

package api

import (
	"database/sql"
	"errors"
	"net/http"
	"net/http/httptest"
	"testing"
	"time"

	"github.com/a73x/eitri/internal/server/registry"
	"github.com/a73x/eitri/internal/server/store"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

// tombstoneOneVM enrolls a host, creates a VM on it, and DELETEs (tombstones)
// it, returning the vm id. The host never reports, so its agent is offline in
// the registry — the ack that would normally hard-delete the row never comes.
func tombstoneOneVM(t *testing.T, ts *httptest.Server, st *store.Store) string {
	t.Helper()
	out := enroll(t, ts)
	hostID := out["host_id"]
	resp := do(t, "POST", ts.URL+"/api/v1/vms", testPAT, map[string]any{"host_id": hostID, "name": "vm-a"})
	require.Equal(t, http.StatusCreated, resp.StatusCode)

	vms, err := st.ListVMs()
	require.NoError(t, err)
	require.Len(t, vms, 1)
	id := vms[0].ID

	resp = do(t, "DELETE", ts.URL+"/api/v1/vms/"+id, testPAT, nil)
	require.Equal(t, http.StatusNoContent, resp.StatusCode)

	got, err := st.GetVM(id)
	require.NoError(t, err, "tombstoned row must still exist (no agent acked it)")
	require.NotNil(t, got.DeletedAt, "row is tombstoned")
	return id
}

// TestAbandonedTombstoneOnOfflineHostIsReaped pins the failed-VM-delete fix: a
// VM deleted while its host's agent is offline (never acks the destroy) would
// otherwise linger forever. Past the grace, the server-side sweep hard-deletes
// it so the console row clears.
func TestAbandonedTombstoneOnOfflineHostIsReaped(t *testing.T) {
	ts, st, _, _, a := newServer(t)
	id := tombstoneOneVM(t, ts, st)

	// Sixteen minutes on, host still offline (never reported). The clock offsets
	// here are literals, not multiples of abandonedVMReapGrace: a sweep driven
	// by the constant it is testing reaps on schedule at any grace, including
	// the millisecond that would force-delete a VM whose agent is mid-restart.
	assert.True(t, a.sweepAbandonedVMs(time.Now().Add(16*time.Minute)),
		"a VM tombstoned sixteen minutes ago on an offline host must be reaped: past the grace nobody is coming to ack the "+
			"destroy, and the row sits in the operator's console forever")

	_, err := st.GetVM(id)
	assert.True(t, errors.Is(err, sql.ErrNoRows), "the zombie row must be gone after the sweep")
}

// TestFreshTombstoneIsNotReaped guards the grace: a just-deleted VM must be
// left for the agent to ack normally — a transient host outage (agent restart,
// redeploy) must not trip an immediate server-side force-delete.
func TestFreshTombstoneIsNotReaped(t *testing.T) {
	ts, st, _, _, a := newServer(t)
	id := tombstoneOneVM(t, ts, st)

	assert.False(t, a.sweepAbandonedVMs(time.Now()),
		"a VM tombstoned within the grace must not be force-deleted")
	assert.False(t, a.sweepAbandonedVMs(time.Now().Add(14*time.Minute)),
		"and the grace must still be running fourteen minutes in: it is the window an agent has to come back and ack its "+
			"own destroy — an agent restart or a host redeploy takes minutes, and a grace shorter than that turns every "+
			"transient outage into a server-side force-delete of a VM the agent was about to tear down properly")

	_, err := st.GetVM(id)
	assert.NoError(t, err, "row must survive a within-grace sweep")
}

// TestOnlineHostTombstoneIsNotReaped guards the offline gate: while the host's
// agent is live it will ack the destroy through the normal quarantine→destroy
// path, so the server must not race it — even for an old tombstone.
func TestOnlineHostTombstoneIsNotReaped(t *testing.T) {
	ts, st, _, _, a := newServer(t)
	id := tombstoneOneVM(t, ts, st)

	// Make the host online: a fresh report sets LastSeen to now.
	vms, err := st.ListVMs()
	require.NoError(t, err)
	a.reg.UpdateReport(vms[0].HostID, registry.Report{})

	assert.False(t, a.sweepAbandonedVMs(time.Now().Add(16*time.Minute)),
		"a live agent owns the reap; the server must not force-delete")

	_, err = st.GetVM(id)
	assert.NoError(t, err, "row must survive while the host is online")
}