0326f691
build: mutation testing is one make target away
a73x 2026-08-22 16:01
Commit message
Makefile
| Old | New | ||
|---|---|---|---|
| @@ -14,11 +14,18 @@ GOVULN := $(shell go env GOPATH)/bin/govulncheck | |||
| 14 | # Style/complexity linters run informationally (exit 0); promote into | 14 | # Style/complexity linters run informationally (exit 0); promote into |
| 15 | # .golangci.yml's enable list once a linter's baseline is clean. | 15 | # .golangci.yml's enable list once a linter's baseline is clean. |
| 16 | LINT_WARN := errcheck,revive,gocyclo,funlen,gocritic,misspell,unconvert,nakedret | 16 | LINT_WARN := errcheck,revive,gocyclo,funlen,gocritic,misspell,unconvert,nakedret |
| 17 | # Pinned mutation tester (github.com/go-gremlins/gremlins). Bump deliberately. | ||
| 18 | GREMLINS_VERSION := v0.6.0 | ||
| 19 | GREMLINS := $(shell go env GOPATH)/bin/gremlins | ||
| 20 | GREMLINS_WORKERS ?= 2 | ||
| 21 | GREMLINS_TIMEOUT ?= 10 | ||
| 22 | MUTATE_BASE ?= origin/main | ||
| 17 | 23 | ||
| 18 | .PHONY: build build-go build-darwin web test vet proto clean \ | 24 | .PHONY: build build-go build-darwin web test vet proto clean \ |
| 19 | lint lint-extra arch cover fmt fmt-check tidy-check proto-check shape shape-check api api-check \ | 25 | lint lint-extra arch cover fmt fmt-check tidy-check proto-check shape shape-check api api-check \ |
| 20 | site site-check ci deadcode web-test web-check vuln vuln-tool scan-image \ | 26 | site site-check ci deadcode web-test web-check vuln vuln-tool scan-image \ |
| 21 | deploy release ship hooks site-image server-image backup-image | 27 | deploy release ship hooks site-image server-image backup-image \ |
| 28 | mutate mutate-diff mutate-report gremlins-tool | ||
| 22 | 29 | ||
| 23 | # Enable the repo's client-side merge gate: point git at .githooks, whose | 30 | # Enable the repo's client-side merge gate: point git at .githooks, whose |
| 24 | # pre-push hook runs `make ci` before any push that updates main. Run once per | 31 | # pre-push hook runs `make ci` before any push that updates main. Run once per |
| @@ -130,6 +137,75 @@ vuln: vuln-tool | |||
| 130 | scan-image: | 137 | scan-image: |
| 131 | ./scripts/scan-image.sh $(IMAGE) | 138 | ./scripts/scan-image.sh $(IMAGE) |
| 132 | 139 | ||
| 140 | .PHONY: gremlins-tool | ||
| 141 | gremlins-tool: | ||
| 142 | @go version -m $(GREMLINS) 2>/dev/null | \ | ||
| 143 | grep -qE "^\s+mod\s+github.com/go-gremlins/gremlins\s+$(GREMLINS_VERSION)\s" || \ | ||
| 144 | go install github.com/go-gremlins/gremlins/cmd/gremlins@$(GREMLINS_VERSION) | ||
| 145 | |||
| 146 | # Mutation testing: rewrite one operator at a time and see whether the suite | ||
| 147 | # notices. It answers what coverage cannot — a line can be executed by every | ||
| 148 | # test and still be asserted by none. | ||
| 149 | # | ||
| 150 | # Deliberately NOT part of `ci`: gremlins runs the package's whole test suite | ||
| 151 | # once per mutant, so cost scales with suite runtime x mutation count. This is a | ||
| 152 | # tool you point at a package while hardening it, not a gate on every push. | ||
| 153 | # | ||
| 154 | # PKG is required. A default of ./... would be the version of this target that | ||
| 155 | # gets run once and never again. | ||
| 156 | # | ||
| 157 | # GREMLINS_WORKERS is a memory bound, not a speed knob. Gremlins copies the | ||
| 158 | # ENTIRE module root — gitignored artifacts included, ~1.8G here because of | ||
| 159 | # dist/ and bin/ — once per worker into TMPDIR. On this box TMPDIR is a 31G | ||
| 160 | # tmpfs and the default worker count is NumCPU: the first run filled RAM and | ||
| 161 | # wedged the machine. Two workers is ~4G, and tmpfs is what keeps the copies | ||
| 162 | # fast. Do NOT redirect TMPDIR to a spinning-disk path to "save memory": cold | ||
| 163 | # compiles in each fresh workdir then outrun the mutant timeout, and timed-out | ||
| 164 | # mutants are EXCLUDED from the score — internal/server/store read 18 killed / | ||
| 165 | # 207 timed out / "100% efficacy" on ext4, against 199 killed / 12 timed out / | ||
| 166 | # 93.43% on tmpfs. A slow disk here does not fail: it flatters. | ||
| 167 | # | ||
| 168 | # GREMLINS_TIMEOUT is load-bearing too. Gremlins derives each mutant's timeout | ||
| 169 | # from the UNMUTATED suite's elapsed time, which for a small package is under a | ||
| 170 | # second — shorter than the compile the mutant needs in its own workdir. | ||
| 171 | # | ||
| 172 | # make mutate PKG=./internal/cloudinit | ||
| 173 | mutate: gremlins-tool | ||
| 174 | @test -n "$(PKG)" || { echo "usage: make mutate PKG=./internal/<pkg>"; exit 1; } | ||
| 175 | $(GREMLINS) unleash \ | ||
| 176 | --workers $(GREMLINS_WORKERS) \ | ||
| 177 | --timeout-coefficient $(GREMLINS_TIMEOUT) \ | ||
| 178 | $(PKG) | ||
| 179 | |||
| 180 | # Mutate only the lines this branch changed against MUTATE_BASE. Cheap enough to | ||
| 181 | # run per branch: the mutant count is the size of the diff, not of the tree. | ||
| 182 | # | ||
| 183 | # The trailing "." is not a default, it is a requirement. Gremlins keys its | ||
| 184 | # diff (git diff --merge-base) by repo-relative path, but names parsed files | ||
| 185 | # relative to the directory it was pointed at — so `-D main ./internal/foo` | ||
| 186 | # matches nothing and reports every mutant SKIPPED, which reads exactly like a | ||
| 187 | # clean run. Only the module root makes the two path forms agree. | ||
| 188 | mutate-diff: gremlins-tool | ||
| 189 | $(GREMLINS) unleash \ | ||
| 190 | --workers $(GREMLINS_WORKERS) \ | ||
| 191 | --timeout-coefficient $(GREMLINS_TIMEOUT) \ | ||
| 192 | -D $(MUTATE_BASE) . | ||
| 193 | |||
| 194 | # Warn tier, the shape lint-extra uses: run over the diff, report, never fail. | ||
| 195 | # | ||
| 196 | # The base is origin/main, not main, so this covers what the push is about to | ||
| 197 | # publish rather than what the branch has already merged locally — on main | ||
| 198 | # itself, a diff against main is empty and the check would silently pass. | ||
| 199 | # | ||
| 200 | # No threshold yet. Setting one needs a few branches' worth of observed scores | ||
| 201 | # first: the tree has equivalent mutants that CANNOT be killed (internal/ | ||
| 202 | # cloudinit's `i+1 < len(root.Content)` is the same loop for every even-length | ||
| 203 | # YAML mapping either way), so a 100% floor would fail honest work. | ||
| 204 | mutate-report: gremlins-tool | ||
| 205 | -@$(GREMLINS) unleash --workers $(GREMLINS_WORKERS) \ | ||
| 206 | --timeout-coefficient $(GREMLINS_TIMEOUT) \ | ||
| 207 | -D $(MUTATE_BASE) . 2>&1 | tail -6 || true | ||
| 208 | |||
| 133 | # Per-package coverage ratchet (see scripts/coverage.sh). | 209 | # Per-package coverage ratchet (see scripts/coverage.sh). |
| 134 | cover: | 210 | cover: |
| 135 | ./scripts/coverage.sh | 211 | ./scripts/coverage.sh |
| @@ -305,7 +381,7 @@ deadcode: | |||
| 305 | # The merge gate. Mirrors the required checks in CI. `test` is the authoritative | 381 | # The merge gate. Mirrors the required checks in CI. `test` is the authoritative |
| 306 | # race-detector run; `cover` re-runs without -race to enforce the ratchet; `arch` | 382 | # race-detector run; `cover` re-runs without -race to enforce the ratchet; `arch` |
| 307 | # re-runs the fitness tests with -count=1 (the race run may serve them cached). | 383 | # re-runs the fitness tests with -count=1 (the race run may serve them cached). |
| 308 | ci: vet build-go build-darwin arch lint fmt-check test cover tidy-check proto-check api-check shape-check deadcode site-check web-test web-check vuln | 384 | ci: vet build-go build-darwin arch lint fmt-check test cover tidy-check proto-check api-check shape-check deadcode site-check web-test web-check vuln mutate-report |
| 309 | 385 | ||
| 310 | # Compile every Go package (no Node/web build needed — the embed dir ships a | 386 | # Compile every Go package (no Node/web build needed — the embed dir ships a |
| 311 | # placeholder, so the server builds and serves a "UI not built" notice). | 387 | # placeholder, so the server builds and serves a "UI not built" notice). |