4dc88c5f

fix(store): the persistent column stays as rollback ballast, and legacy trust freezes at upgrade

a73x 2026-08-12 14:45

Commit message

fix(store): the persistent column stays as rollback ballast, and legacy trust freezes at upgrade

Two migrations, both about a release that has to be survivable.

The retired persistent column is kept one more release instead of dropped.
v0.0.5's Open runs `UPDATE vms SET persistent = 1` before it will serve
anything, so a database v0.0.6 had dropped the column from could not be rolled
back to v0.0.5 — a failed release whose recovery is rollback would have left
the plane down in both directions. The column now rides in the ensureColumn
list, so a database this release CREATED carries it too and a fresh plane is as
rollable as an upgraded one. Nothing reads or writes it. The real drop lands the
release after this one, alongside reserving proto field 9: the same discipline,
retire the compatibility only once no v0.0.5 can exist.

A VM whose row recorded no trusted CA set was being served the tenant's LIVE set
on every push, forever — so "a guest trusts the set it was created against" was
false for every VM that predates the column, and the console could never call
such a guest's trust stale, having nothing to compare against. Open now freezes
the tenant's current set onto those rows, in the JSON the create path writes.
The upgrade is the last moment that trust can be written down, and the set they
get is the one they were already being served. A tenant with no registered CA
freezes an empty set, which is the true statement about that guest. The
snapshot's live fallback survives as a safety net for a row created by a
pre-upgrade server still running alongside this one, and says in comment that it
is now vestigial and must not become the way legacy VMs are served again.

The backfill runs on every Open rather than once ever, and is idempotent by
construction: a row that has a set is never touched. Running it every time is
what a rollback needs — v0.0.5 writes no trusted_cas at all, so a plane that
rolls back and forward again mints fresh unrecorded rows, and a one-shot marker
would strand exactly those on the live set forever.

internal/server/store/store.go

Old		New
@@ -355,6 +355,22 @@ func Open(path, cidrPool string) (*Store, error) {
355	// guest trusts no CA" versus "we did not write down which CAs it	355	// guest trusts no CA" versus "we did not write down which CAs it
356	// trusts". Rows that predate this column get NULL and say so.	356	// trusts". Rows that predate this column get NULL and say so.
357	{"vms", "trusted_cas", "TEXT"},	357	{"vms", "trusted_cas", "TEXT"},
		358	// Rollback ballast, and nothing else: no code in this release reads or
		359	// writes this column, and no VM's behaviour depends on it. It is here
		360	// because v0.0.5's Open runs `UPDATE vms SET persistent = 1` before it
		361	// will serve anything, so a database this binary has touched must still
		362	// carry the column or the previous release cannot start on it — and a
		363	// release whose recovery plan is "roll back" would leave the plane down
		364	// in both directions. Carried here rather than in the CREATE TABLE above
		365	// so the guarantee covers a database this release CREATED as well as one
		366	// it upgraded; a fresh plane has to be rollable too. NOT NULL DEFAULT 1
		367	// because nothing writes it and every VM is persistent, so v0.0.5 reads
		368	// back exactly the value it would have written itself.
		369	//
		370	// The real drop lands the release AFTER this one, alongside reserving
		371	// proto field 9 — the same discipline for the same reason: retire the
		372	// compatibility only once no v0.0.5 can still be out there.
		373	{"vms", "persistent", "INTEGER NOT NULL DEFAULT 1"},
358	// The address a host presents on the network it reaches the fleet	374	// The address a host presents on the network it reaches the fleet
359	// over. Reported every tick like the guest subnet, and stored for the	375	// over. Reported every tick like the guest subnet, and stored for the
360	// same reason: the console renders `host:port` for every exposure,	376	// same reason: the console renders `host:port` for every exposure,
@@ -393,17 +409,6 @@ func Open(path, cidrPool string) (*Store, error) {
393	return nil, err	409	return nil, err
394	}	410	}
395		411
396	// Persistence stopped being a per-VM answer a release ago: a guest lost to a
397	// host reboot or a dead hypervisor is booted again, every time, and the
398	// previous release backfilled the rows that said otherwise. What is left is a
399	// column every row agrees on, which is not a fact about a VM. The snapshot
400	// still tells agents the policy — see syncsvc — it just no longer reads it
401	// back out of a table to do so.
402	if err := dropColumn(db, "vms", "persistent"); err != nil {
403	db.Close()
404	return nil, err
405	}
406
407	// One identity binds at most one tenant (per issuer). Partial index so	412	// One identity binds at most one tenant (per issuer). Partial index so
408	// unbound rows (empty issuer+subject) don't collide.	413	// unbound rows (empty issuer+subject) don't collide.
409	if _, err := db.Exec(`CREATE UNIQUE INDEX IF NOT EXISTS tenants_identity	414	if _, err := db.Exec(`CREATE UNIQUE INDEX IF NOT EXISTS tenants_identity
@@ -418,7 +423,17 @@ func Open(path, cidrPool string) (*Store, error) {
418	return nil, fmt.Errorf("seed cidr_pool: %w", err)	423	return nil, fmt.Errorf("seed cidr_pool: %w", err)
419	}	424	}
420		425
421	return &Store{db: db, dbDir: filepath.Dir(path)}, nil	426	s := &Store{db: db, dbDir: filepath.Dir(path)}
		427
		428	// The last migration, and the only one that writes rows rather than shape:
		429	// a VM that has no recorded CA set is given one now, so that a set frozen at
		430	// create is the only answer anything serves. See backfillTrustedCAs.
		431	if err := s.backfillTrustedCAs(); err != nil {
		432	db.Close()
		433	return nil, err
		434	}
		435
		436	return s, nil
422	}	437	}
423		438
424	func (s *Store) Close() error { return s.db.Close() }	439	func (s *Store) Close() error { return s.db.Close() }

internal/server/store/store_test.go

Old		New
@@ -198,15 +198,18 @@ func TestOpenDropsTheEscrowedHostKeyColumn(t *testing.T) {
198	assert.Equal(t, "cert-line", vm.SSHHostCert)	198	assert.Equal(t, "cert-line", vm.SSHHostCert)
199	}	199	}
200		200
201	// TestOpenDropsTheRetiredPersistentColumn is the upgrade for a fleet whose	201	// TestTheRetiredPersistentColumnStaysForRollback pins the compatibility this
202	// database still carries the restart policy as a column. Dropping a column	202	// release keeps rather than the drop it would rather do. Nothing here reads or
203	// rewrites the table every row of which must come back out intact, so this	203	// writes vms.persistent, but v0.0.5's Open runs `UPDATE vms SET persistent = 1`
204	// pins the rows rather than the schema: the VMs an operator has — live and	204	// before it will serve anything: a database this binary has opened must still
205	// tombstoned — survive the migration with their fields, and a second Open	205	// have the column, or rolling back a bad release leaves the plane down in both
206	// finds nothing left to drop.	206	// directions. Both databases must satisfy that — the one upgraded from v0.0.5
207	func TestOpenDropsTheRetiredPersistentColumn(t *testing.T) {	207	// and the one this release created from nothing — so the test asserts the same
208	path := t.TempDir() + "/eitri.db"	208	// thing twice, once about each, by running v0.0.5's statement verbatim.
209	s, err := Open(path, "10.77.0.0/16")	209	func TestTheRetiredPersistentColumnStaysForRollback(t *testing.T) {
		210	// (a) upgraded: a v0.0.5-shaped database, column and all.
		211	upgraded := t.TempDir() + "/eitri.db"
		212	s, err := Open(upgraded, "10.77.0.0/16")
210	require.NoError(t, err)	213	require.NoError(t, err)
211	_, err = s.CreateTenantForIdentity("https://test-issuer", "test-subject", testTenant+"@test.local")	214	_, err = s.CreateTenantForIdentity("https://test-issuer", "test-subject", testTenant+"@test.local")
212	require.NoError(t, err)	215	require.NoError(t, err)
@@ -218,19 +221,26 @@ func TestOpenDropsTheRetiredPersistentColumn(t *testing.T) {
218	}))	221	}))
219	}	222	}
220	require.NoError(t, s.TombstoneVM("deleted"))	223	require.NoError(t, s.TombstoneVM("deleted"))
221	// The shape the previous release left behind: the column, on every row.	224	// A value only the older release could have written. Nothing here sets the
222	_, err = s.db.Exec(`ALTER TABLE vms ADD COLUMN persistent INTEGER NOT NULL DEFAULT 1`)	225	// column, so if it survives the reopen the column was left alone rather than
		226	// dropped and put back — and the row v0.0.5 has to fix is still there to fix.
		227	_, err = s.db.Exec(`UPDATE vms SET persistent = 0 WHERE id='live'`)
223	require.NoError(t, err)	228	require.NoError(t, err)
224	require.NoError(t, s.Close())	229	require.NoError(t, s.Close())
225		230
226	up, err := Open(path, "10.77.0.0/16")	231	up, err := Open(upgraded, "10.77.0.0/16")
227	require.NoError(t, err)	232	require.NoError(t, err)
228	t.Cleanup(func() { up.Close() })	233	t.Cleanup(func() { up.Close() })
229		234
230	var n int	235	var kept int
231	require.NoError(t, up.db.QueryRow(	236	require.NoError(t, up.db.QueryRow(`SELECT persistent FROM vms WHERE id='live'`).Scan(&kept))
232	`SELECT count(*) FROM pragma_table_info('vms') WHERE name = 'persistent'`).Scan(&n))	237	assert.Equal(t, 0, kept, "the column and its rows must survive an upgrade untouched")
233	assert.Zero(t, n, "the column must be gone")	238
		239	// v0.0.5's first write, run against the database v0.0.6 handed back.
		240	_, err = up.db.Exec(`UPDATE vms SET persistent = 1 WHERE persistent = 0`)
		241	require.NoError(t, err, "v0.0.5 must be able to start on a database v0.0.6 has opened")
		242	require.NoError(t, up.db.QueryRow(`SELECT persistent FROM vms WHERE id='live'`).Scan(&kept))
		243	assert.Equal(t, 1, kept, "v0.0.5's backfill must reach the row it targets")
234		244
235	for _, id := range []string{"live", "deleted"} {	245	for _, id := range []string{"live", "deleted"} {
236	vm, err := up.GetVM(id)	246	vm, err := up.GetVM(id)
@@ -242,13 +252,23 @@ func TestOpenDropsTheRetiredPersistentColumn(t *testing.T) {
242	require.NoError(t, err)	252	require.NoError(t, err)
243	assert.NotNil(t, deleted.DeletedAt, "a tombstoned VM stays tombstoned")	253	assert.NotNil(t, deleted.DeletedAt, "a tombstoned VM stays tombstoned")
244		254
245	// Idempotent: a database already migrated has nothing to drop.	255	// (b) fresh: a database this release created. The column is not in the
246	require.NoError(t, up.Close())	256	// CREATE TABLE, so only the migration can have put it there — and a plane
247	again, err := Open(path, "10.77.0.0/16")	257	// installed at v0.0.6 has to be as rollable as one upgraded to it.
248	require.NoError(t, err)	258	fresh := newStore(t)
249	t.Cleanup(func() { again.Close() })	259	hf := enrollHost(t, fresh)
250	_, err = again.GetVM("live")	260	require.NoError(t, fresh.CreateVM(VM{
251	require.NoError(t, err)	261	ID: "new", HostID: hf.ID, Name: "new", ImageURL: "u", ImageSHA256: "abc",
		262	VCPUs: 1, MemMB: 512, DiskGB: 5, PowerState: "running",
		263	}))
		264	_, err = fresh.db.Exec(`UPDATE vms SET persistent = 1 WHERE persistent = 0`)
		265	require.NoError(t, err, "a database created by this release must roll back too")
		266
		267	// A row this release wrote left the column alone, so v0.0.5 reads back the
		268	// value it would have written itself: every VM is persistent.
		269	var persistent int
		270	require.NoError(t, fresh.db.QueryRow(`SELECT persistent FROM vms WHERE id='new'`).Scan(&persistent))
		271	assert.Equal(t, 1, persistent, "a VM created here must read as persistent to v0.0.5")
252	}	272	}
253		273
254	func TestOpenReplacesTheProtocolBlindHostPortIndex(t *testing.T) {	274	func TestOpenReplacesTheProtocolBlindHostPortIndex(t *testing.T) {

internal/server/store/trustedcas.go

@@ @@ -0,0 +1,121 @@ @@
+package store
+import (
+	"encoding/json"
+	"fmt"
+	"log/slog"
+	"golang.org/x/crypto/ssh"
+)
+// backfillTrustedCAs freezes a CA set onto every VM row that has none, and is
+// the upgrade half of "a guest trusts the set it was created against".
+//
+// A row written before vms.trusted_cas existed records nothing, and the
+// snapshot answers that by serving the tenant's LIVE set instead (see
+// syncsvc.buildSnapshot). Left alone that is permanent: such a VM would keep
+// receiving whatever the tenant has registered today for the rest of its life,
+// the console could never mark it stale because there is no recorded set to
+// compare against, and "frozen at create" would stay false for every guest that
+// predates the column. One fact would have two authorities.
+//
+// Freezing here, at the upgrade, is the closest approximation of freeze-at-
+// create those rows can still be given: the set they get is the one they were
+// already being served, and from now on it is written down. A tenant with no
+// registered CA gives its VMs an EMPTY set rather than leaving them unrecorded —
+// "this guest trusts nothing" is a true statement about a guest whose tenant has
+// registered nothing, and it is what the live fallback was serving anyway. It is
+// also rare: create refuses a tenant with no CA, so only a VM whose tenant
+// removed its last CA afterwards can be in that position.
+//
+// This runs on every Open rather than once ever, and is idempotent by
+// construction (a row with a set is never touched). Running it every time is
+// deliberate: v0.0.5 writes no trusted_cas at all, so a plane that rolls back
+// and forward again mints fresh unrecorded rows, and a one-shot marker would
+// leave exactly those rows unfrozen forever — the bug this fixes.
+func (s *Store) backfillTrustedCAs() error {
+	unrecorded, err := s.unrecordedVMs()
+	if err != nil {
+		return err
+	}
+	if len(unrecorded) == 0 {
+		return nil
+	}
+	// Tombstoned rows are included on purpose: a tombstone can be restored, and
+	// a restored VM must not come back with a moving CA set.
+	frozen := map[string]string{} // tenant -> JSON, one read per tenant
+	for _, r := range unrecorded {
+		enc, ok := frozen[r.tenant]
+		if !ok {
+			cas, err := s.ListTenantUserCAs(r.tenant)
+			if err != nil {
+				return fmt.Errorf("list user cas for %s: %w", r.tenant, err)
+			}
+			b, err := json.Marshal(freezeCAs(cas))
+			if err != nil {
+				return fmt.Errorf("marshal frozen cas for %s: %w", r.tenant, err)
+			}
+			enc = string(b)
+			frozen[r.tenant] = enc
+		}
+		// IS NULL again in the predicate so a row that acquired a set between
+		// the scan and here keeps the one it was given.
+		if _, err := s.db.Exec(
+			`UPDATE vms SET trusted_cas=? WHERE id=? AND trusted_cas IS NULL`, enc, r.id); err != nil {
+			return fmt.Errorf("freeze trusted cas on %s: %w", r.id, err)
+		}
+	}
+	slog.Info("froze the trusted CA set of VMs that had no record", "vms", len(unrecorded))
+	return nil
+}
+// unrecordedVM is one VM row carrying no trusted_cas, and its tenant — the only
+// two things the backfill needs to decide what to write.
+type unrecordedVM struct{ id, tenant string }
+// unrecordedVMs reads the rows the backfill has work to do on, whole, before it
+// writes anything: the write is an UPDATE on the same table this reads.
+func (s *Store) unrecordedVMs() ([]unrecordedVM, error) {
+	rows, err := s.db.Query(`SELECT id, tenant FROM vms WHERE trusted_cas IS NULL`)
+	if err != nil {
+		return nil, fmt.Errorf("scan for unrecorded trusted cas: %w", err)
+	}
+	defer rows.Close()
+	var out []unrecordedVM
+	for rows.Next() {
+		var v unrecordedVM
+		if err := rows.Scan(&v.id, &v.tenant); err != nil {
+			return nil, fmt.Errorf("scan unrecorded vm: %w", err)
+		}
+		out = append(out, v)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, fmt.Errorf("scan for unrecorded trusted cas: %w", err)
+	}
+	return out, nil
+}
+// freezeCAs turns a tenant's registered CAs into the record that goes onto a VM
+// row: the label, the SHA256 fingerprint OpenSSH would print, and the
+// authorized_keys line itself — the line because the row has to SERVE this trust
+// to an agent and a fingerprint is one-way. A key line that will not parse still
+// contributes its line with an empty fingerprint, since the line is what reaches
+// the guest and dropping it would quietly narrow the tenant's trust.
+//
+// This is the same record api.freezeTrustedCAs writes at create, deliberately
+// producing an identical shape — the JSON is the tag set on TrustedCA, which
+// both share. It is a second copy because the API package imports this one and
+// not the other way around; collapsing the two means moving the create-path
+// helper down here, which is a change to the API layer.
+func freezeCAs(cas []TenantUserCA) []TrustedCA {
+	out := make([]TrustedCA, 0, len(cas))
+	for _, c := range cas {
+		fp := ""
+		if pub, _, _, _, err := ssh.ParseAuthorizedKey([]byte(c.Pubkey)); err == nil {
+			fp = ssh.FingerprintSHA256(pub)
+		}
+		out = append(out, TrustedCA{Label: c.Label, Fingerprint: fp, AuthorizedKey: c.Pubkey})
+	}
+	return out
+}

internal/server/store/trustedcas_test.go

Old		New
@@ -1,11 +1,15 @@
1	package store	1	package store
2		2
3	import (	3	import (
		4	"crypto/ed25519"
		5	"crypto/rand"
4	"path/filepath"	6	"path/filepath"
		7	"strings"
5	"testing"	8	"testing"
6		9
7	"github.com/stretchr/testify/assert"	10	"github.com/stretchr/testify/assert"
8	"github.com/stretchr/testify/require"	11	"github.com/stretchr/testify/require"
		12	"golang.org/x/crypto/ssh"
9	)	13	)
10		14
11	// TestTrustedCAsRoundtrip pins that a VM's frozen CA set survives the store	15	// TestTrustedCAsRoundtrip pins that a VM's frozen CA set survives the store
@@ -43,36 +47,176 @@ func TestTrustedCAsUnrecordedIsNilNotEmpty(t *testing.T) {
43	assert.Nil(t, got.TrustedCAs)	47	assert.Nil(t, got.TrustedCAs)
44	}	48	}
45		49
46	// TestTrustedCAsMigrationLeavesAPreFeatureRowUnrecorded runs a row through the	50	// legacyDB builds the database a pre-freeze server leaves behind: one VM whose
47	// actual migration: a database whose vms table has no trusted_cas column at	51	// trusted_cas is NULL — nothing written down about what it trusts — plus
48	// all, reopened so ensureColumn adds it. The row that was already there must	52	// whatever the tenant has registered by then, which register may add. Reopening
49	// come back unrecorded rather than as an empty set — it is a VM whose guest was	53	// the returned path runs the real migration.
50	// seeded long ago and whose trust nobody wrote down, and inventing a set for it	54	func legacyDB(t testing.T, register func(Store)) string {
51	// would be a fabricated fact on a page whose entire point is not to have any.	55	t.Helper()
52	func TestTrustedCAsMigrationLeavesAPreFeatureRowUnrecorded(t *testing.T) {
53	path := filepath.Join(t.TempDir(), "eitri.db")	56	path := filepath.Join(t.TempDir(), "eitri.db")
54	s, err := Open(path, "10.77.0.0/16")	57	s, err := Open(path, "10.77.0.0/16")
55	require.NoError(t, err)	58	require.NoError(t, err)
56	_, err = s.CreateTenantForIdentity("https://test-issuer", "test-subject", testTenant+"@test.local")	59	_, err = s.CreateTenantForIdentity("https://test-issuer", "test-subject", testTenant+"@test.local")
57	require.NoError(t, err)	60	require.NoError(t, err)
58	h := enrollHost(t, s)	61	h := enrollHost(t, s)
		62	if register != nil {
		63	register(s)
		64	}
		65	// No TrustedCAs: CreateVM writes NULL, which is exactly the row a server
		66	// from before the column existed wrote.
59	require.NoError(t, s.CreateVM(VM{ID: "vm1", HostID: h.ID, Name: "web-1", ImageURL: "u",	67	require.NoError(t, s.CreateVM(VM{ID: "vm1", HostID: h.ID, Name: "web-1", ImageURL: "u",
60	ImageSHA256: "s", VCPUs: 1, MemMB: 512, DiskGB: 5, PowerState: "running",	68	ImageSHA256: "s", VCPUs: 1, MemMB: 512, DiskGB: 5, PowerState: "running"}))
61	TrustedCAs: []TrustedCA{{Label: "laptop", AuthorizedKey: "ssh-ed25519 AAAALAPTOP"}}}))	69	require.NoError(t, s.Close())
		70	return path
		71	}
62		72
63	// Rewind the schema to before the column existed, taking the recorded set	73	func reopen(t testing.T, path string) Store {
64	// with it — which is precisely the state of every row in a database that	74	t.Helper()
65	// predates this feature.	75	s, err := Open(path, "10.77.0.0/16")
66	_, err = s.db.Exec(`ALTER TABLE vms DROP COLUMN trusted_cas`)
67	require.NoError(t, err)	76	require.NoError(t, err)
68	require.NoError(t, s.Close())	77	t.Cleanup(func() { s.Close() })
		78	return s
		79	}
69		80
70	s2, err := Open(path, "10.77.0.0/16")	81	// testCA mints a real ed25519 CA line, because the record the backfill writes
		82	// carries a fingerprint derived from the key — an invented string cannot pin it.
		83	func testCA(t *testing.T) (line, fingerprint string) {
		84	t.Helper()
		85	_, priv, err := ed25519.GenerateKey(rand.Reader)
71	require.NoError(t, err)	86	require.NoError(t, err)
72	t.Cleanup(func() { s2.Close() })	87	signer, err := ssh.NewSignerFromKey(priv)
		88	require.NoError(t, err)
		89	pub := signer.PublicKey()
		90	return strings.TrimSpace(string(ssh.MarshalAuthorizedKey(pub))), ssh.FingerprintSHA256(pub)
		91	}
73		92
74	got, err := s2.GetVM("vm1")	93	// TestOpenFreezesAnUnrecordedRowOntoTheTenantsCurrentSet is the upgrade half of
		94	// the freeze. A row that recorded nothing was served the tenant's LIVE set on
		95	// every push, forever — so "a guest trusts the set it was created against" was
		96	// simply false for it, and the console could never call its trust stale, having
		97	// nothing to compare against. The upgrade is the last moment that VM's trust can
		98	// still be written down, so it is written down here: the set it was already
		99	// being served, now a record.
		100	func TestOpenFreezesAnUnrecordedRowOntoTheTenantsCurrentSet(t *testing.T) {
		101	caLine, caFP := testCA(t)
		102	path := legacyDB(t, func(s *Store) {
		103	require.NoError(t, s.AddTenantUserCA(testTenant, caLine, "tenant", "laptop", "admin"))
		104	})
		105
		106	s := reopen(t, path)
		107	got, err := s.GetVM("vm1")
75	require.NoError(t, err)	108	require.NoError(t, err)
76	assert.Nil(t, got.TrustedCAs, "a row that predates the column has no record, not an empty one")	109	assert.Equal(t, []TrustedCA{{Label: "laptop", Fingerprint: caFP, AuthorizedKey: caLine}},
		110	got.TrustedCAs, "the row must record the set it was being served")
77	assert.Equal(t, "web-1", got.Name, "the rest of the row survives the migration")	111	assert.Equal(t, "web-1", got.Name, "the rest of the row survives the migration")
		112
		113	// The column itself, not just the round-trip: the backfill writes the JSON
		114	// the create path writes, so no reader can tell the two apart.
		115	var raw string
		116	require.NoError(t, s.db.QueryRow(`SELECT trusted_cas FROM vms WHERE id='vm1'`).Scan(&raw))
		117	assert.JSONEq(t,
		118	`[{"label":"laptop","fingerprint":"`+caFP+`","authorized_key":"`+caLine+`"}]`, raw)
		119	}
		120
		121	// TestOpenFreezesARowThatPredatesTheColumn runs the same freeze through the
		122	// other legacy shape: a database whose vms table has no trusted_cas column at
		123	// all, so ensureColumn adds it back as NULL first and the backfill fills it in
		124	// the same Open.
		125	func TestOpenFreezesARowThatPredatesTheColumn(t *testing.T) {
		126	caLine, caFP := testCA(t)
		127	path := legacyDB(t, func(s *Store) {
		128	require.NoError(t, s.AddTenantUserCA(testTenant, caLine, "tenant", "laptop", "admin"))
		129	})
		130	pre := reopen(t, path)
		131	_, err := pre.db.Exec(`ALTER TABLE vms DROP COLUMN trusted_cas`)
		132	require.NoError(t, err)
		133	require.NoError(t, pre.Close())
		134
		135	got, err := reopen(t, path).GetVM("vm1")
		136	require.NoError(t, err)
		137	assert.Equal(t, []TrustedCA{{Label: "laptop", Fingerprint: caFP, AuthorizedKey: caLine}},
		138	got.TrustedCAs)
		139	}
		140
		141	// TestOpenFreezesAnEmptySetForATenantWithNoCAs pins the honest answer for the
		142	// rare tenant that has registered nothing: an empty set — "this guest trusts no
		143	// CA" — and not NULL. It is what the live fallback was already serving such a
		144	// VM, and create refuses a tenant with no CA, so only a tenant that removed its
		145	// last CA afterwards can be here.
		146	func TestOpenFreezesAnEmptySetForATenantWithNoCAs(t *testing.T) {
		147	s := reopen(t, legacyDB(t, nil))
		148
		149	got, err := s.GetVM("vm1")
		150	require.NoError(t, err)
		151	assert.NotNil(t, got.TrustedCAs, "an unrecorded row must not stay unrecorded")
		152	assert.Empty(t, got.TrustedCAs, "a tenant with no CAs freezes an empty set")
		153
		154	var raw string
		155	require.NoError(t, s.db.QueryRow(`SELECT trusted_cas FROM vms WHERE id='vm1'`).Scan(&raw))
		156	assert.Equal(t, "[]", raw)
		157	}
		158
		159	// TestOpenLeavesAnAlreadyFrozenRowAlone is what keeps the migration from
		160	// becoming the live-set read it replaces: a row that has a set never gets a
		161	// newer one, however often the server restarts and whatever the tenant has
		162	// registered since.
		163	func TestOpenLeavesAnAlreadyFrozenRowAlone(t *testing.T) {
		164	first, fp := testCA(t)
		165	path := legacyDB(t, func(s *Store) {
		166	require.NoError(t, s.AddTenantUserCA(testTenant, first, "tenant", "laptop", "admin"))
		167	})
		168	require.NoError(t, reopen(t, path).Close()) // freezes
		169
		170	// A second CA arrives, and the server restarts again.
		171	again := reopen(t, path)
		172	second, _ := testCA(t)
		173	require.NoError(t, again.AddTenantUserCA(testTenant, second, "tenant", "ci", "admin"))
		174	require.NoError(t, again.Close())
		175
		176	got, err := reopen(t, path).GetVM("vm1")
		177	require.NoError(t, err)
		178	assert.Equal(t, []TrustedCA{{Label: "laptop", Fingerprint: fp, AuthorizedKey: first}},
		179	got.TrustedCAs, "a frozen set must not be refreshed by a later Open")
		180	}
		181
		182	// TestOpenFreezesARowLeftByARolledBackServer is why the backfill runs on every
		183	// Open rather than once ever. The previous release writes no trusted_cas at all,
		184	// so a plane that rolls back and forward again mints fresh unrecorded rows; a
		185	// one-shot marker would leave exactly those on the live set forever, which is
		186	// the bug this migration closes.
		187	func TestOpenFreezesARowLeftByARolledBackServer(t *testing.T) {
		188	caLine, caFP := testCA(t)
		189	path := legacyDB(t, func(s *Store) {
		190	require.NoError(t, s.AddTenantUserCA(testTenant, caLine, "tenant", "laptop", "admin"))
		191	})
		192	s := reopen(t, path)
		193	// What the older server's create leaves behind, after this one has run.
		194	_, err := s.db.Exec(`UPDATE vms SET trusted_cas=NULL WHERE id='vm1'`)
		195	require.NoError(t, err)
		196	require.NoError(t, s.Close())
		197
		198	got, err := reopen(t, path).GetVM("vm1")
		199	require.NoError(t, err)
		200	assert.Equal(t, []TrustedCA{{Label: "laptop", Fingerprint: caFP, AuthorizedKey: caLine}},
		201	got.TrustedCAs, "a row minted by a rolled-back server must still be frozen")
		202	}
		203
		204	// TestOpenFreezesEachTenantsOwnSet: the backfill reads and caches per tenant,
		205	// which is where a fleet-wide read would hand one tenant's CA to another
		206	// tenant's guest — the failure the create-side refusal is tested against for
		207	// the same reason.
		208	func TestOpenFreezesEachTenantsOwnSet(t *testing.T) {
		209	mine, mineFP := testCA(t)
		210	theirs, _ := testCA(t)
		211	path := legacyDB(t, func(s *Store) {
		212	other, err := s.CreateTenantForIdentity("https://test-issuer", "other-subject", "other@test.local")
		213	require.NoError(t, err)
		214	require.NoError(t, s.AddTenantUserCA(testTenant, mine, "tenant", "laptop", "admin"))
		215	require.NoError(t, s.AddTenantUserCA(other.ID, theirs, "tenant", "someone-elses", "admin"))
		216	})
		217
		218	got, err := reopen(t, path).GetVM("vm1")
		219	require.NoError(t, err)
		220	assert.Equal(t, []TrustedCA{{Label: "laptop", Fingerprint: mineFP, AuthorizedKey: mine}},
		221	got.TrustedCAs, "another tenant's CA must not be frozen onto this tenant's guest")
78	}	222	}

internal/server/syncsvc/syncsvc.go

Old		New
@@ -339,15 +339,20 @@ func (s Service) buildSnapshot(hostID string) (pb.DesiredStateSnapshot, error)
339	cas = append(cas, c.AuthorizedKey)	339	cas = append(cas, c.AuthorizedKey)
340	}	340	}
341		341
342	// A row that predates the column recorded nothing, and there is no set	342	// A row that recorded nothing has no set to serve — so fall back to the
343	// to serve — so fall back to the tenant's live set, which is exactly	343	// tenant's live set, which is exactly what this VM would have been sent
344	// what this VM would have been sent before the freeze existed. Such a	344	// before the freeze existed.
345	// guest has almost certainly been seeded for months and its trust is	345	//
346	// long since fixed in its own filesystem, so the fallback is moot for	346	// This is now vestigial: store.Open freezes a set onto every unrecorded
347	// all but one case: a pre-upgrade VM still mid-create when the server	347	// row it finds, so by the time a snapshot is built there are none left.
348	// rolled. For that one the fallback preserves the old behaviour and it	348	// It survives as a safety net for the one row the backfill cannot have
349	// gets a working guest, where an empty set would seed a guest that	349	// seen — a VM created by a pre-upgrade server still running alongside
350	// trusts no CA at all and is unreachable for good.	350	// this one, mid-rollout. For that one the fallback preserves the old
		351	// behaviour and the guest works, where an empty set would seed a guest
		352	// that trusts no CA at all and is unreachable for good. It must NOT
		353	// become the way legacy VMs are served again: a live set here is a
		354	// second authority for a fact the row is supposed to own, and a guest
		355	// fed by it can never be shown as trusting a stale CA.
351	if v.TrustedCAs == nil {	356	if v.TrustedCAs == nil {
352	legacy, ok := caCache[v.Tenant]	357	legacy, ok := caCache[v.Tenant]
353	if !ok {	358	if !ok {

internal/server/syncsvc/trustedcas_test.go

@@ @@ -2,7 +2,10 @@ package syncsvc @@
+import (
+	"testing"
+	"time"
+	"github.com/a73x/eitri/internal/server/hub"
+	"github.com/a73x/eitri/internal/server/registry"
+	"github.com/a73x/eitri/internal/server/store"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	assert.Equal(t, []string{caLaptop}, snapCAs(t, f, "vm1"),
+		"another tenant's CA must not be served to this tenant's guest")
+// TestSnapshotServesTheSetFrozenAtUpgrade is the legacy VM's version of the
+// same promise, end to end. A row written before the freeze existed has its set
+// recorded by the store's backfill when the server upgrades; from then on the
+// snapshot serves THAT set, and a CA the tenant registers afterwards does not
+// reach the guest — where before the upgrade it would have, on every push, for
+// the rest of that VM's life.
+func TestSnapshotServesTheSetFrozenAtUpgrade(t *testing.T) {
+	path := t.TempDir() + "/db"
+	// The database the old server leaves: one VM, no recorded set, one CA.
+	old, err := store.Open(path, "10.77.0.0/16")
+	require.NoError(t, err)
+	seedTestTenant(t, old)
+	tok, _ := old.CreateEnrollmentToken(testTenant)
+	host, err := old.RedeemEnrollmentToken(tok, store.EnrollFacts{Name: "h", OS: "linux",
+		Arch: "amd64", Provisioner: "cloudhv", Remote: ""})
+	require.NoError(t, err)
+	require.NoError(t, old.AddTenantUserCA(testTenant, caLaptop, "tenant", "laptop", "admin"))
+	require.NoError(t, old.CreateVM(store.VM{ID: "vm1", HostID: host.ID, Name: "web-1",
+		ImageURL: "u", ImageSHA256: "s", VCPUs: 1, MemMB: 512, DiskGB: 5, PowerState: "running"}))
+	require.NoError(t, old.Close())
+	// The upgrade: Open freezes the set that row was being served.
+	st, err := store.Open(path, "10.77.0.0/16")
+	require.NoError(t, err)
+	t.Cleanup(func() { st.Close() })
+	svc := newWithWriteTimeout(st, registry.New(time.Now), hub.New(), []byte("s3cret"), 0, 0)
+	f := &fixture{st: st, host: host, svc: svc}
+	require.NoError(t, st.AddTenantUserCA(testTenant, caCI, "tenant", "ci", "admin"))
+	assert.Equal(t, []string{caLaptop}, snapCAs(t, f, "vm1"),
+		"a CA registered after the upgrade must not reach a guest the upgrade froze")
+}