471ce90d
feat(agent): a lost guest reports what its hypervisor said
a73x 2026-08-06 09:12
Commit message
docs/shape.html
| Old | New | ||
|---|---|---|---|
| @@ -146,6 +146,7 @@ | |||
| 146 | "imports": [ | 146 | "imports": [ |
| 147 | "internal/agent/exec", | 147 | "internal/agent/exec", |
| 148 | "internal/agent/hostinfo", | 148 | "internal/agent/hostinfo", |
| 149 | "internal/agent/hyperlog", | ||
| 149 | "internal/agent/pidfile", | 150 | "internal/agent/pidfile", |
| 150 | "internal/agent/state" | 151 | "internal/agent/state" |
| 151 | ] | 152 | ] |
| @@ -180,6 +181,12 @@ | |||
| 180 | ] | 181 | ] |
| 181 | }, | 182 | }, |
| 182 | { | 183 | { |
| 184 | "importPath": "internal/agent/hyperlog", | ||
| 185 | "plane": "data", | ||
| 186 | "synopsis": "Package hyperlog reads back the last thing a hypervisor said before it stopped running.", | ||
| 187 | "imports": [] | ||
| 188 | }, | ||
| 189 | { | ||
| 183 | "importPath": "internal/agent/imagecache", | 190 | "importPath": "internal/agent/imagecache", |
| 184 | "plane": "data", | 191 | "plane": "data", |
| 185 | "synopsis": "Package imagecache downloads and verifies content-addressed base images (decoded to raw in-process, LRU-evicted beyond MaxBytes).", | 192 | "synopsis": "Package imagecache downloads and verifies content-addressed base images (decoded to raw in-process, LRU-evicted beyond MaxBytes).", |
| @@ -283,6 +290,7 @@ | |||
| 283 | "imports": [ | 290 | "imports": [ |
| 284 | "internal/agent/exec", | 291 | "internal/agent/exec", |
| 285 | "internal/agent/hostinfo", | 292 | "internal/agent/hostinfo", |
| 293 | "internal/agent/hyperlog", | ||
| 286 | "internal/agent/pidfile", | 294 | "internal/agent/pidfile", |
| 287 | "internal/agent/state" | 295 | "internal/agent/state" |
| 288 | ] | 296 | ] |
docs/shape.json
| Old | New | ||
|---|---|---|---|
| @@ -95,6 +95,7 @@ | |||
| 95 | "imports": [ | 95 | "imports": [ |
| 96 | "internal/agent/exec", | 96 | "internal/agent/exec", |
| 97 | "internal/agent/hostinfo", | 97 | "internal/agent/hostinfo", |
| 98 | "internal/agent/hyperlog", | ||
| 98 | "internal/agent/pidfile", | 99 | "internal/agent/pidfile", |
| 99 | "internal/agent/state" | 100 | "internal/agent/state" |
| 100 | ] | 101 | ] |
| @@ -129,6 +130,12 @@ | |||
| 129 | ] | 130 | ] |
| 130 | }, | 131 | }, |
| 131 | { | 132 | { |
| 133 | "importPath": "internal/agent/hyperlog", | ||
| 134 | "plane": "data", | ||
| 135 | "synopsis": "Package hyperlog reads back the last thing a hypervisor said before it stopped running.", | ||
| 136 | "imports": [] | ||
| 137 | }, | ||
| 138 | { | ||
| 132 | "importPath": "internal/agent/imagecache", | 139 | "importPath": "internal/agent/imagecache", |
| 133 | "plane": "data", | 140 | "plane": "data", |
| 134 | "synopsis": "Package imagecache downloads and verifies content-addressed base images (decoded to raw in-process, LRU-evicted beyond MaxBytes).", | 141 | "synopsis": "Package imagecache downloads and verifies content-addressed base images (decoded to raw in-process, LRU-evicted beyond MaxBytes).", |
| @@ -232,6 +239,7 @@ | |||
| 232 | "imports": [ | 239 | "imports": [ |
| 233 | "internal/agent/exec", | 240 | "internal/agent/exec", |
| 234 | "internal/agent/hostinfo", | 241 | "internal/agent/hostinfo", |
| 242 | "internal/agent/hyperlog", | ||
| 235 | "internal/agent/pidfile", | 243 | "internal/agent/pidfile", |
| 236 | "internal/agent/state" | 244 | "internal/agent/state" |
| 237 | ] | 245 | ] |
internal/agent/cloudhv/cloudhv.go
| Old | New | ||
|---|---|---|---|
| @@ -18,6 +18,7 @@ import ( | |||
| 18 | 18 | ||
| 19 | agentexec "github.com/a73x/eitri/internal/agent/exec" | 19 | agentexec "github.com/a73x/eitri/internal/agent/exec" |
| 20 | "github.com/a73x/eitri/internal/agent/hostinfo" | 20 | "github.com/a73x/eitri/internal/agent/hostinfo" |
| 21 | "github.com/a73x/eitri/internal/agent/hyperlog" | ||
| 21 | "github.com/a73x/eitri/internal/agent/pidfile" | 22 | "github.com/a73x/eitri/internal/agent/pidfile" |
| 22 | "github.com/a73x/eitri/internal/agent/state" | 23 | "github.com/a73x/eitri/internal/agent/state" |
| 23 | ) | 24 | ) |
| @@ -269,8 +270,7 @@ func (p *Provisioner) Boot(ctx context.Context, vmID string, spec state.VMSpec) | |||
| 269 | // Guest console output (serial) goes to a unix socket (--serial socket=…) | 270 | // Guest console output (serial) goes to a unix socket (--serial socket=…) |
| 270 | // that the serialpump drains into serial.log. | 271 | // that the serialpump drains into serial.log. |
| 271 | // CH's own diagnostic output (startup errors, API logs) goes to ch.log. | 272 | // CH's own diagnostic output (startup errors, API logs) goes to ch.log. |
| 272 | chLogPath := filepath.Join(p.st.VMDir(vmID), "ch.log") | 273 | chLog, err := os.OpenFile(p.logPath(vmID), os.O_CREATE|os.O_APPEND|os.O_WRONLY, chLogMode) |
| 273 | chLog, err := os.OpenFile(chLogPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, chLogMode) | ||
| 274 | if err != nil { | 274 | if err != nil { |
| 275 | return fmt.Errorf("open ch.log %s: %w", vmID, err) | 275 | return fmt.Errorf("open ch.log %s: %w", vmID, err) |
| 276 | } | 276 | } |
| @@ -340,6 +340,17 @@ func (p *Provisioner) ownedPID(vmID string) int { | |||
| 340 | // agent wrote cannot have been recycled while its process is alive. reconcile's | 340 | // agent wrote cannot have been recycled while its process is alive. reconcile's |
| 341 | // own boot-ID check still decides what a lost VM means — this only decides | 341 | // own boot-ID check still decides what a lost VM means — this only decides |
| 342 | // which process, if any, is ours to ask about. | 342 | // which process, if any, is ours to ask about. |
| 343 | // logPath is where cloud-hypervisor's own diagnostic output is kept (startup | ||
| 344 | // errors, API logs) — distinct from serial.log, which carries the GUEST's | ||
| 345 | // console. | ||
| 346 | func (p *Provisioner) logPath(vmID string) string { | ||
| 347 | return filepath.Join(p.st.VMDir(vmID), "ch.log") | ||
| 348 | } | ||
| 349 | |||
| 350 | // FailureReason quotes the last thing cloud-hypervisor said. See | ||
| 351 | // reconcile.Provisioner. | ||
| 352 | func (p *Provisioner) FailureReason(vmID string) string { return hyperlog.Reason(p.logPath(vmID)) } | ||
| 353 | |||
| 343 | func (p *Provisioner) Running(vmID string) bool { | 354 | func (p *Provisioner) Running(vmID string) bool { |
| 344 | pid := p.ownedPID(vmID) | 355 | pid := p.ownedPID(vmID) |
| 345 | if pid == 0 { | 356 | if pid == 0 { |
internal/agent/cloudhv/cloudhv_test.go
| Old | New | ||
|---|---|---|---|
| @@ -666,3 +666,20 @@ func TestAPidfileWithoutABootIDIsStillOurs(t *testing.T) { | |||
| 666 | 666 | ||
| 667 | assert.True(t, p.Running(vmID), "an agent upgrade must not orphan the guests it inherits") | 667 | assert.True(t, p.Running(vmID), "an agent upgrade must not orphan the guests it inherits") |
| 668 | } | 668 | } |
| 669 | |||
| 670 | // TestFailureReasonQuotesCHLog pins the wiring, not the tail logic (that is | ||
| 671 | // hyperlog's own test): FailureReason must read the SAME file Boot redirects | ||
| 672 | // cloud-hypervisor's stdout and stderr to. A backend quoting the wrong file | ||
| 673 | // reports nothing forever and looks exactly like a backend with nothing to say. | ||
| 674 | func TestFailureReasonQuotesCHLog(t *testing.T) { | ||
| 675 | st, err := state.Open(t.TempDir()) | ||
| 676 | require.NoError(t, err) | ||
| 677 | p := New(st, "ch", "fw", nil, newFakeNet()) | ||
| 678 | |||
| 679 | require.Empty(t, p.FailureReason("vm1"), "no log yet means nothing to add") | ||
| 680 | |||
| 681 | require.NoError(t, os.MkdirAll(st.VMDir("vm1"), 0o700)) | ||
| 682 | require.NoError(t, os.WriteFile(filepath.Join(st.VMDir("vm1"), "ch.log"), | ||
| 683 | []byte("cloud-hypervisor booting\nError: VmBoot(DeviceManager(Kernel))\n"), 0o600)) | ||
| 684 | assert.Equal(t, "Error: VmBoot(DeviceManager(Kernel))", p.FailureReason("vm1")) | ||
| 685 | } | ||
internal/agent/hyperlog/hyperlog.go
| Old | New | ||
|---|---|---|---|
| @@ -0,0 +1,117 @@ | |||
| 1 | // Package hyperlog reads back the last thing a hypervisor said before it | ||
| 2 | // stopped running. | ||
| 3 | // | ||
| 4 | // Every backend already redirects its hypervisor's stdout and stderr to a file | ||
| 5 | // beside the VM's disk — cloud-hypervisor to ch.log, vfkit to vfkit.log — but | ||
| 6 | // nothing ever read them, so a guest whose hypervisor exited on startup was | ||
| 7 | // reported as lost and nothing more. "ephemeral VM lost" is a true statement | ||
| 8 | // about the process table and says nothing about the cause; the cause was | ||
| 9 | // sitting on disk the whole time. | ||
| 10 | // | ||
| 11 | // This is deliberately the hypervisor's own words rather than an interpretation | ||
| 12 | // of them. eitri cannot enumerate what cloud-hypervisor or vfkit might fail at, | ||
| 13 | // and a guess dressed up as a diagnosis is worse than a quote. | ||
| 14 | package hyperlog | ||
| 15 | |||
| 16 | import ( | ||
| 17 | "bytes" | ||
| 18 | "io" | ||
| 19 | "os" | ||
| 20 | "strings" | ||
| 21 | "unicode" | ||
| 22 | ) | ||
| 23 | |||
| 24 | const ( | ||
| 25 | // tailBytes bounds the read. A hypervisor's fatal message is its last line, | ||
| 26 | // so only the end of the file is interesting, and a log that has been | ||
| 27 | // running for weeks must not be pulled into memory to find it. | ||
| 28 | tailBytes = 8 << 10 | ||
| 29 | // maxLen bounds the returned string. It lands in a VM's last_error, which | ||
| 30 | // is a database column rendered in the console and the CLI, so a runaway | ||
| 31 | // line has to be cut somewhere it stays readable. | ||
| 32 | maxLen = 240 | ||
| 33 | ) | ||
| 34 | |||
| 35 | // Reason returns the last meaningful line the hypervisor wrote to path, or "" | ||
| 36 | // when there is nothing to report — no file, an empty one, or output that is | ||
| 37 | // all whitespace. Empty means "nothing to add", never "nothing went wrong": | ||
| 38 | // callers append it to their own message only when it is non-empty. | ||
| 39 | func Reason(path string) string { | ||
| 40 | f, err := os.Open(path) | ||
| 41 | if err != nil { | ||
| 42 | return "" | ||
| 43 | } | ||
| 44 | defer f.Close() | ||
| 45 | |||
| 46 | // Seek to the last tailBytes. A file shorter than that is read whole; the | ||
| 47 | // first line of the window may be a fragment, which is why only complete | ||
| 48 | // trailing lines are considered below. | ||
| 49 | size, err := f.Seek(0, io.SeekEnd) | ||
| 50 | if err != nil { | ||
| 51 | return "" | ||
| 52 | } | ||
| 53 | start := max(size-tailBytes, 0) | ||
| 54 | if _, err := f.Seek(start, io.SeekStart); err != nil { | ||
| 55 | return "" | ||
| 56 | } | ||
| 57 | buf, err := io.ReadAll(f) | ||
| 58 | if err != nil { | ||
| 59 | return "" | ||
| 60 | } | ||
| 61 | // Drop a leading partial line when the window did not start at the file's | ||
| 62 | // beginning, so a truncated fragment is never reported as the reason. | ||
| 63 | if start > 0 { | ||
| 64 | if i := bytes.IndexByte(buf, '\n'); i >= 0 { | ||
| 65 | buf = buf[i+1:] | ||
| 66 | } else { | ||
| 67 | return "" // one enormous line, no complete one to quote | ||
| 68 | } | ||
| 69 | } | ||
| 70 | |||
| 71 | lines := strings.Split(string(buf), "\n") | ||
| 72 | for i := len(lines) - 1; i >= 0; i-- { | ||
| 73 | if s := clean(lines[i]); s != "" { | ||
| 74 | return s | ||
| 75 | } | ||
| 76 | } | ||
| 77 | return "" | ||
| 78 | } | ||
| 79 | |||
| 80 | // clean reduces one log line to something safe to store and show: printable | ||
| 81 | // characters only, collapsed whitespace, bounded length. Hypervisors colour | ||
| 82 | // their output and draw progress with control characters, none of which belong | ||
| 83 | // in a database column. | ||
| 84 | func clean(line string) string { | ||
| 85 | var b strings.Builder | ||
| 86 | b.Grow(len(line)) | ||
| 87 | prevSpace := true // leading whitespace is dropped | ||
| 88 | for _, r := range line { | ||
| 89 | switch { | ||
| 90 | case r == 0x1b: // start of an ANSI escape; drop the rest of the line | ||
| 91 | return strings.TrimRight(b.String(), " ") | ||
| 92 | case unicode.IsSpace(r): | ||
| 93 | if !prevSpace { | ||
| 94 | b.WriteRune(' ') | ||
| 95 | prevSpace = true | ||
| 96 | } | ||
| 97 | case unicode.IsPrint(r): | ||
| 98 | b.WriteRune(r) | ||
| 99 | prevSpace = false | ||
| 100 | } | ||
| 101 | // Anything else (other control characters) is dropped. | ||
| 102 | } | ||
| 103 | out := strings.TrimRight(b.String(), " ") | ||
| 104 | if len(out) > maxLen { | ||
| 105 | // Cut on a rune boundary so the result stays valid UTF-8. | ||
| 106 | cut := maxLen | ||
| 107 | for cut > 0 && !utf8Start(out[cut]) { | ||
| 108 | cut-- | ||
| 109 | } | ||
| 110 | out = strings.TrimRight(out[:cut], " ") + "…" | ||
| 111 | } | ||
| 112 | return out | ||
| 113 | } | ||
| 114 | |||
| 115 | // utf8Start reports whether b begins a UTF-8 rune (i.e. is not a continuation | ||
| 116 | // byte). | ||
| 117 | func utf8Start(b byte) bool { return b&0xC0 != 0x80 } | ||
internal/agent/hyperlog/hyperlog_test.go
| Old | New | ||
|---|---|---|---|
| @@ -0,0 +1,84 @@ | |||
| 1 | package hyperlog | ||
| 2 | |||
| 3 | import ( | ||
| 4 | "os" | ||
| 5 | "path/filepath" | ||
| 6 | "strings" | ||
| 7 | "testing" | ||
| 8 | ) | ||
| 9 | |||
| 10 | // write puts body in a temp file and returns its path. | ||
| 11 | func write(t *testing.T, body string) string { | ||
| 12 | t.Helper() | ||
| 13 | p := filepath.Join(t.TempDir(), "hv.log") | ||
| 14 | if err := os.WriteFile(p, []byte(body), 0o600); err != nil { | ||
| 15 | t.Fatal(err) | ||
| 16 | } | ||
| 17 | return p | ||
| 18 | } | ||
| 19 | |||
| 20 | func TestReason(t *testing.T) { | ||
| 21 | for _, tc := range []struct { | ||
| 22 | name, body, want string | ||
| 23 | }{ | ||
| 24 | {"last line wins", "starting\nvcpu 0 ready\nError: VmBoot(DeviceManager)\n", | ||
| 25 | "Error: VmBoot(DeviceManager)"}, | ||
| 26 | {"trailing blank lines skipped", "Error: no bootable device\n\n\n \n", | ||
| 27 | "Error: no bootable device"}, | ||
| 28 | {"no trailing newline", "line one\nfatal: bad kernel", "fatal: bad kernel"}, | ||
| 29 | {"single line", "vfkit: failed to start", "vfkit: failed to start"}, | ||
| 30 | {"empty file", "", ""}, | ||
| 31 | {"whitespace only", "\n\n \t\n", ""}, | ||
| 32 | {"collapses whitespace", "Error: too many\tspaces\n", "Error: too many spaces"}, | ||
| 33 | {"strips control characters", "Error: bad\x00 image\x07\n", "Error: bad image"}, | ||
| 34 | {"truncates at an ANSI escape", "\x1b[31mError: red\x1b[0m\n", ""}, | ||
| 35 | } { | ||
| 36 | t.Run(tc.name, func(t *testing.T) { | ||
| 37 | if got := Reason(write(t, tc.body)); got != tc.want { | ||
| 38 | t.Errorf("Reason() = %q, want %q", got, tc.want) | ||
| 39 | } | ||
| 40 | }) | ||
| 41 | } | ||
| 42 | } | ||
| 43 | |||
| 44 | // TestReasonMissingFile pins that an absent log is "nothing to add" rather than | ||
| 45 | // an error the caller has to handle: a VM that never started writes no log, and | ||
| 46 | // that is not itself a fault worth reporting. | ||
| 47 | func TestReasonMissingFile(t *testing.T) { | ||
| 48 | if got := Reason(filepath.Join(t.TempDir(), "absent.log")); got != "" { | ||
| 49 | t.Errorf("Reason() on a missing file = %q, want empty", got) | ||
| 50 | } | ||
| 51 | if got := Reason(t.TempDir()); got != "" { | ||
| 52 | t.Errorf("Reason() on a directory = %q, want empty", got) | ||
| 53 | } | ||
| 54 | } | ||
| 55 | |||
| 56 | // TestReasonReadsOnlyTheTail pins that a long-lived log is not pulled into | ||
| 57 | // memory whole, and that the partial first line of the read window is never | ||
| 58 | // mistaken for the reason. | ||
| 59 | func TestReasonReadsOnlyTheTail(t *testing.T) { | ||
| 60 | body := strings.Repeat("chatter chatter chatter\n", 4000) + "Error: the last word\n" | ||
| 61 | if got := Reason(write(t, body)); got != "Error: the last word" { | ||
| 62 | t.Errorf("Reason() = %q, want the final line", got) | ||
| 63 | } | ||
| 64 | // A window that lands mid-line must not quote the fragment: a file that is | ||
| 65 | // one enormous line has no complete trailing line to report. | ||
| 66 | if got := Reason(write(t, strings.Repeat("x", 40<<10))); got != "" { | ||
| 67 | t.Errorf("Reason() on one huge line = %q, want empty", got) | ||
| 68 | } | ||
| 69 | } | ||
| 70 | |||
| 71 | // TestReasonBoundsLength pins the cap: last_error is a database column shown in | ||
| 72 | // the console, so a runaway line is cut rather than stored whole. | ||
| 73 | func TestReasonBoundsLength(t *testing.T) { | ||
| 74 | got := Reason(write(t, "Error: "+strings.Repeat("verbose ", 200)+"\n")) | ||
| 75 | if len(got) > maxLen+len("…") { | ||
| 76 | t.Errorf("Reason() len = %d, want <= %d", len(got), maxLen+len("…")) | ||
| 77 | } | ||
| 78 | if !strings.HasSuffix(got, "…") { | ||
| 79 | t.Errorf("a truncated reason must say so, got %q", got) | ||
| 80 | } | ||
| 81 | if !strings.HasPrefix(got, "Error: verbose") { | ||
| 82 | t.Errorf("truncation must keep the START of the line, got %q", got) | ||
| 83 | } | ||
| 84 | } | ||
internal/agent/reconcile/reconcile.go
| Old | New | ||
|---|---|---|---|
| @@ -92,6 +92,16 @@ type Provisioner interface { | |||
| 92 | 92 | ||
| 93 | Running(vmID string) bool | 93 | Running(vmID string) bool |
| 94 | 94 | ||
| 95 | // FailureReason returns what the hypervisor said before it stopped running, | ||
| 96 | // or "" when the backend has nothing to add. It is asked only once a VM has | ||
| 97 | // been found lost, to give that report a cause: the process table can say a | ||
| 98 | // guest is gone but never why, and every backend already keeps its | ||
| 99 | // hypervisor's own output on disk. | ||
| 100 | // | ||
| 101 | // Empty means "nothing to add", NOT "nothing went wrong" — a guest killed | ||
| 102 | // by a host reboot leaves no complaint behind and is still lost. | ||
| 103 | FailureReason(vmID string) string | ||
| 104 | |||
| 95 | // Address returns the VM's current guest address, or "" when the backend | 105 | // Address returns the VM's current guest address, or "" when the backend |
| 96 | // does not know one (never booted, or gone). It is POLLED rather than | 106 | // does not know one (never booted, or gone). It is POLLED rather than |
| 97 | // returned by Boot: where the host OS's own DHCP server hands out the | 107 | // returned by Boot: where the host OS's own DHCP server hands out the |
| @@ -808,8 +818,15 @@ func (e *Engine) converge(ctx context.Context, d *pb.VMDesired, rec state.Record | |||
| 808 | 818 | ||
| 809 | if lost { | 819 | if lost { |
| 810 | if !d.Persistent { | 820 | if !d.Persistent { |
| 811 | // Ephemeral lost VMs are reported failed and NEVER restarted. | 821 | // Ephemeral lost VMs are reported failed and NEVER restarted. This |
| 822 | // is the only report the operator gets, so it carries whatever the | ||
| 823 | // hypervisor said on its way out — a guest handed an image its host | ||
| 824 | // cannot execute dies here, and "ephemeral VM lost" alone names | ||
| 825 | // neither the image nor the architecture. | ||
| 812 | errMsg := "ephemeral VM lost" | 826 | errMsg := "ephemeral VM lost" |
| 827 | if why := e.Prov.FailureReason(d.VmId); why != "" { | ||
| 828 | errMsg += ": " + why | ||
| 829 | } | ||
| 813 | rec.LastError = errMsg | 830 | rec.LastError = errMsg |
| 814 | _ = e.St.SaveVM(rec) | 831 | _ = e.St.SaveVM(rec) |
| 815 | res.report(d.VmId, rec.IP, "stopped", "failed", errMsg) | 832 | res.report(d.VmId, rec.IP, "stopped", "failed", errMsg) |
internal/agent/reconcile/reconcile_test.go
| Old | New | ||
|---|---|---|---|
| @@ -29,13 +29,13 @@ import ( | |||
| 29 | // addressing is only observable through the seam, so the fake has to own it for | 29 | // addressing is only observable through the seam, so the fake has to own it for |
| 30 | // these tests to mean anything. | 30 | // these tests to mean anything. |
| 31 | type fakeProv struct { | 31 | type fakeProv struct { |
| 32 | mu sync.Mutex | 32 | mu sync.Mutex |
| 33 | running map[string]bool | 33 | running map[string]bool |
| 34 | prepCalls int // total PrepareRootDisk invocations, including failed ones | 34 | prepCalls int // total PrepareRootDisk invocations, including failed ones |
| 35 | prepared []string | 35 | prepared []string |
| 36 | booted []string | 36 | booted []string |
| 37 | shutdown []string | 37 | shutdown []string |
| 38 | destroyed []string | 38 | destroyed []string |
| 39 | prepErr error | 39 | prepErr error |
| 40 | bootErr error // one-shot: consumed and cleared on first Boot call | 40 | bootErr error // one-shot: consumed and cleared on first Boot call |
| 41 | destroyErr error // sticky: every Destroy fails until it is cleared | 41 | destroyErr error // sticky: every Destroy fails until it is cleared |
| @@ -44,6 +44,10 @@ type fakeProv struct { | |||
| 44 | cidr string | 44 | cidr string |
| 45 | addrs map[string]string // vmID -> ip (sticky, mirrors the DHCP table) | 45 | addrs map[string]string // vmID -> ip (sticky, mirrors the DHCP table) |
| 46 | 46 | ||
| 47 | // failReason is what the fake's "hypervisor" left behind; empty is a | ||
| 48 | // backend with nothing to add, which is the common case. | ||
| 49 | failReason string | ||
| 50 | |||
| 47 | // lateAddress switches the fake to the shape a host-run DHCP server has: | 51 | // lateAddress switches the fake to the shape a host-run DHCP server has: |
| 48 | // Boot attaches the NIC but assigns nothing, and the address exists only | 52 | // Boot attaches the NIC but assigns nothing, and the address exists only |
| 49 | // once the guest has booted and asked for one (answerAddress). | 53 | // once the guest has booted and asked for one (answerAddress). |
| @@ -163,6 +167,12 @@ func (f *fakeProv) Running(id string) bool { | |||
| 163 | return f.running[id] | 167 | return f.running[id] |
| 164 | } | 168 | } |
| 165 | 169 | ||
| 170 | func (f *fakeProv) FailureReason(string) string { | ||
| 171 | f.mu.Lock() | ||
| 172 | defer f.mu.Unlock() | ||
| 173 | return f.failReason | ||
| 174 | } | ||
| 175 | |||
| 166 | type fixture struct { | 176 | type fixture struct { |
| 167 | eng *Engine | 177 | eng *Engine |
| 168 | prov *fakeProv | 178 | prov *fakeProv |
| @@ -389,6 +399,36 @@ func TestUserStopIsStoppedNotLost(t *testing.T) { | |||
| 389 | assert.NotEqual(t, "failed", av.Phase, "recorded stop request: stopped != lost") | 399 | assert.NotEqual(t, "failed", av.Phase, "recorded stop request: stopped != lost") |
| 390 | } | 400 | } |
| 391 | 401 | ||
| 402 | // TestEphemeralLostCarriesTheHypervisorsReason pins that the one report an | ||
| 403 | // ephemeral VM ever produces says WHY. "ephemeral VM lost" is a true statement | ||
| 404 | // about the process table and a useless one to debug from: a guest handed an | ||
| 405 | // image its host cannot execute dies exactly here, and the hypervisor's own | ||
| 406 | // complaint is the only thing that names the cause. | ||
| 407 | func TestEphemeralLostCarriesTheHypervisorsReason(t *testing.T) { | ||
| 408 | f := setup(t) | ||
| 409 | f.step(snap(1, vm("vm1"))) | ||
| 410 | f.prov.running["vm1"] = false | ||
| 411 | f.prov.failReason = "Error: VmBoot(NoBootableDevice)" | ||
| 412 | |||
| 413 | av := findVM(f.step(snap(1, vm("vm1"))), "vm1") | ||
| 414 | assert.Equal(t, "failed", av.Phase) | ||
| 415 | assert.Equal(t, "ephemeral VM lost: Error: VmBoot(NoBootableDevice)", av.LastError) | ||
| 416 | } | ||
| 417 | |||
| 418 | // TestEphemeralLostWithoutAReasonStaysBare pins the empty case: a backend with | ||
| 419 | // nothing to add must not produce a dangling separator. Empty means "nothing to | ||
| 420 | // add", not "nothing went wrong" — a guest killed by a host reboot leaves no | ||
| 421 | // complaint behind and is still lost. | ||
| 422 | func TestEphemeralLostWithoutAReasonStaysBare(t *testing.T) { | ||
| 423 | f := setup(t) | ||
| 424 | f.step(snap(1, vm("vm1"))) | ||
| 425 | f.prov.running["vm1"] = false | ||
| 426 | f.prov.failReason = "" | ||
| 427 | |||
| 428 | av := findVM(f.step(snap(1, vm("vm1"))), "vm1") | ||
| 429 | assert.Equal(t, "ephemeral VM lost", av.LastError) | ||
| 430 | } | ||
| 431 | |||
| 392 | func TestEphemeralLostOnHostRebootNeverRestarts(t *testing.T) { | 432 | func TestEphemeralLostOnHostRebootNeverRestarts(t *testing.T) { |
| 393 | f := setup(t) | 433 | f := setup(t) |
| 394 | f.step(snap(1, vm("vm1"))) | 434 | f.step(snap(1, vm("vm1"))) |
internal/agent/syncclient/client_test.go
| Old | New | ||
|---|---|---|---|
| @@ -37,6 +37,7 @@ func (noopProv) Shutdown(context.Context, string) error { r | |||
| 37 | func (noopProv) Destroy(context.Context, string) error { return nil } | 37 | func (noopProv) Destroy(context.Context, string) error { return nil } |
| 38 | func (noopProv) Running(string) bool { return false } | 38 | func (noopProv) Running(string) bool { return false } |
| 39 | func (noopProv) Address(string) string { return "10.77.1.2" } | 39 | func (noopProv) Address(string) string { return "10.77.1.2" } |
| 40 | func (noopProv) FailureReason(string) string { return "" } | ||
| 40 | 41 | ||
| 41 | // testQUICIdle is the deliberately-short idle timeout the test listeners use so | 42 | // testQUICIdle is the deliberately-short idle timeout the test listeners use so |
| 42 | // reconnect/drop cases don't wait out the production SyncMaxIdleTimeout. | 43 | // reconnect/drop cases don't wait out the production SyncMaxIdleTimeout. |
internal/agent/vfkit/vfkit.go
| Old | New | ||
|---|---|---|---|
| @@ -32,6 +32,7 @@ import ( | |||
| 32 | 32 | ||
| 33 | agentexec "github.com/a73x/eitri/internal/agent/exec" | 33 | agentexec "github.com/a73x/eitri/internal/agent/exec" |
| 34 | "github.com/a73x/eitri/internal/agent/hostinfo" | 34 | "github.com/a73x/eitri/internal/agent/hostinfo" |
| 35 | "github.com/a73x/eitri/internal/agent/hyperlog" | ||
| 35 | "github.com/a73x/eitri/internal/agent/pidfile" | 36 | "github.com/a73x/eitri/internal/agent/pidfile" |
| 36 | "github.com/a73x/eitri/internal/agent/state" | 37 | "github.com/a73x/eitri/internal/agent/state" |
| 37 | ) | 38 | ) |
| @@ -176,6 +177,10 @@ func (p *Provisioner) sockPath(vmID string) string { return p.vmFile(vmID, "vfki | |||
| 176 | func (p *Provisioner) varStore(vmID string) string { return p.vmFile(vmID, "efi-vars.fd") } | 177 | func (p *Provisioner) varStore(vmID string) string { return p.vmFile(vmID, "efi-vars.fd") } |
| 177 | func (p *Provisioner) logPath(vmID string) string { return p.vmFile(vmID, "vfkit.log") } | 178 | func (p *Provisioner) logPath(vmID string) string { return p.vmFile(vmID, "vfkit.log") } |
| 178 | 179 | ||
| 180 | // FailureReason quotes the last thing vfkit said. See | ||
| 181 | // reconcile.Provisioner. | ||
| 182 | func (p *Provisioner) FailureReason(vmID string) string { return hyperlog.Reason(p.logPath(vmID)) } | ||
| 183 | |||
| 179 | // SocketPath is the VM's vfkit REST socket, exported so the composition root | 184 | // SocketPath is the VM's vfkit REST socket, exported so the composition root |
| 180 | // can hand ConsoleSource the same path Boot writes. | 185 | // can hand ConsoleSource the same path Boot writes. |
| 181 | func (p *Provisioner) SocketPath(vmID string) string { return p.sockPath(vmID) } | 186 | func (p *Provisioner) SocketPath(vmID string) string { return p.sockPath(vmID) } |
internal/agent/vfkit/vfkit_test.go
| Old | New | ||
|---|---|---|---|
| @@ -653,3 +653,17 @@ func TestShutdownDoesNotLeakAConnectionPerCall(t *testing.T) { | |||
| 653 | } | 653 | } |
| 654 | }) | 654 | }) |
| 655 | } | 655 | } |
| 656 | |||
| 657 | // TestFailureReasonQuotesVfkitLog pins the wiring, not the tail logic (that is | ||
| 658 | // hyperlog's own test): FailureReason must read the SAME file Boot redirects | ||
| 659 | // vfkit's stdout and stderr to. | ||
| 660 | func TestFailureReasonQuotesVfkitLog(t *testing.T) { | ||
| 661 | p := newTestProv(t, nil) | ||
| 662 | |||
| 663 | require.Empty(t, p.FailureReason("vm1"), "no log yet means nothing to add") | ||
| 664 | |||
| 665 | require.NoError(t, os.MkdirAll(p.st.VMDir("vm1"), 0o700)) | ||
| 666 | require.NoError(t, os.WriteFile(p.logPath("vm1"), | ||
| 667 | []byte("vfkit starting\nvirtual machine failed to start: unsupported guest\n"), 0o600)) | ||
| 668 | assert.Equal(t, "virtual machine failed to start: unsupported guest", p.FailureReason("vm1")) | ||
| 669 | } | ||
scripts/coverage.sh
| Old | New | ||
|---|---|---|---|
| @@ -24,6 +24,7 @@ declare -A FLOOR=( | |||
| 24 | [internal/agent/imagecache]=72 | 24 | [internal/agent/imagecache]=72 |
| 25 | [internal/agent/netenv]=76 | 25 | [internal/agent/netenv]=76 |
| 26 | [internal/agent/cloudhv]=80 | 26 | [internal/agent/cloudhv]=80 |
| 27 | [internal/agent/hyperlog]=90 | ||
| 27 | [internal/agent/pidfile]=95 | 28 | [internal/agent/pidfile]=95 |
| 28 | [internal/agent/vfkit]=85 | 29 | [internal/agent/vfkit]=85 |
| 29 | [internal/agent/syncclient]=74 | 30 | [internal/agent/syncclient]=74 |