9e9d75b6

Add bench-baseline and bench-check

a73x 2026-04-17 16:31

Commit message

Add bench-baseline and bench-check

Baseline stores workload_sha + zig_version + waystty_sha + per-section
stats. Check compares current p99 against baseline, flagging sections
exceeding 20% p99 growth. Threshold overridable via
WAYSTTY_BENCH_REGRESSION_PCT.

main.zig now writes BaselineRecord JSON to WAYSTTY_BENCH_JSON path
alongside the existing human-readable table.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

build.zig

@@ @@ -370,4 +370,29 @@ pub fn build(b: *std.Build) void { @@
+    const test_render_run = b.addRunArtifact(test_render_exe);
+    test_render_run.step.dependOn(b.getInstallStep());
+    test_render_step.dependOn(&test_render_run.step);
+    // bench-baseline / bench-check — frame-timing regression guard
+    const bench_baseline_mod = b.createModule(.{
+        .root_source_file = b.path("src/tools/bench_baseline.zig"),
+        .target = target,
+        .optimize = optimize,
+    });
+    bench_baseline_mod.addImport("bench_stats", bench_stats_mod);
+    const bench_baseline_exe = b.addExecutable(.{
+        .name = "bench-baseline",
+        .root_module = bench_baseline_mod,
+    });
+    b.installArtifact(bench_baseline_exe);
+    const bench_baseline_step = b.step("bench-baseline", "Save current frame-timing profile to tests/bench/baseline.json");
+    const bench_baseline_run = b.addRunArtifact(bench_baseline_exe);
+    bench_baseline_run.addArg("save");
+    bench_baseline_run.step.dependOn(b.getInstallStep());
+    bench_baseline_step.dependOn(&bench_baseline_run.step);
+    const bench_check_step = b.step("bench-check", "Compare current frame timings against baseline");
+    const bench_check_run = b.addRunArtifact(bench_baseline_exe);
+    bench_check_run.addArg("check");
+    bench_check_run.step.dependOn(b.getInstallStep());
+    bench_check_step.dependOn(&bench_check_run.step);

src/main.zig

+    window.setTitle(title);
+/// If `WAYSTTY_BENCH_JSON` is set, write a BaselineRecord JSON to that path.
+/// Machine-readable companion to `printFrameStats`.
+fn writeBenchJson(alloc: std.mem.Allocator, stats: FrameTimingStats, workload: ?[:0]const u8) !void {
+    const path = std.posix.getenv("WAYSTTY_BENCH_JSON") orelse return;
+    // sha256 of the bench workload string (empty string if no workload)
+    var digest: [32]u8 = undefined;
+    std.crypto.hash.sha2.Sha256.hash(workload orelse "", &digest, .{});
+    var sha_hex: [64]u8 = undefined;
+    const hex_lut = "0123456789abcdef";
+    for (digest, 0..) |b, i| {
+        sha_hex[i * 2] = hex_lut[b >> 4];
+        sha_hex[i * 2 + 1] = hex_lut[b & 0x0f];
+    }
+    // git HEAD (falls back to "unknown")
+    const git_head = blk: {
+        const r = std.process.Child.run(.{
+            .allocator = alloc,
+            .argv = &.{ "git", "rev-parse", "HEAD" },
+        }) catch {
+            break :blk try alloc.dupe(u8, "unknown");
+        };
+        defer alloc.free(r.stdout);
+        defer alloc.free(r.stderr);
+        if (r.term != .Exited or r.term.Exited != 0) {
+            break :blk try alloc.dupe(u8, "unknown");
+        }
+        const trimmed = std.mem.trim(u8, r.stdout, "\n \t");
+        break :blk try alloc.dupe(u8, trimmed);
+    };
+    defer alloc.free(git_head);
+    const rec = bench_stats.BaselineRecord{
+        .workload_sha = &sha_hex,
+        .zig_version = @import("builtin").zig_version_string,
+        .waystty_sha = git_head,
+        .frame_count = stats.frame_count,
+        .sections = .{
+            .snapshot = stats.snapshot,
+            .row_rebuild = stats.row_rebuild,
+            .atlas_upload = stats.atlas_upload,
+            .instance_upload = stats.instance_upload,
+            .gpu_submit = stats.gpu_submit,
+        },
+    };
+    const json_bytes = try bench_stats.writeBaselineJson(alloc, rec);
+    defer alloc.free(json_bytes);
+    const out = try std.fs.cwd().createFile(path, .{});
+    defer out.close();
+    try out.writeAll(json_bytes);
+}
+pub fn main() !void {
+    var gpa: std.heap.DebugAllocator(.{}) = .init;
+    defer _ = gpa.deinit();
@@ @@ -645,7 +700,11 @@ fn runTerminal(alloc: std.mem.Allocator) !void { @@
+    // Dump timing stats on exit
+    const final_stats = computeFrameStats(&frame_ring);
+    printFrameStats(final_stats);
+    writeBenchJson(alloc, final_stats, bench_script) catch |err| {
+        std.log.warn("bench_json write failed: {s}", .{@errorName(err)});
+    };
+    _ = try ctx.vkd.deviceWaitIdle(ctx.device);

src/tools/bench_baseline.zig

@@ @@ -0,0 +1,121 @@ @@
+const std = @import("std");
+const bench_stats = @import("bench_stats");
+pub fn main() !void {
+    var gpa: std.heap.DebugAllocator(.{}) = .init;
+    defer _ = gpa.deinit();
+    const alloc = gpa.allocator();
+    const args = try std.process.argsAlloc(alloc);
+    defer std.process.argsFree(alloc, args);
+    const Mode = enum { save, check };
+    const mode: Mode = if (args.len >= 2 and std.mem.eql(u8, args[1], "save"))
+        .save
+    else
+        .check;
+    const baseline_path = "tests/bench/baseline.json";
+    const tmp_json = "/tmp/waystty-bench-current.json";
+    try std.fs.cwd().makePath("tests/bench");
+    // Run waystty with WAYSTTY_BENCH=1 WAYSTTY_BENCH_JSON=<tmp>
+    var env = try std.process.getEnvMap(alloc);
+    defer env.deinit();
+    try env.put("WAYSTTY_BENCH", "1");
+    try env.put("WAYSTTY_BENCH_JSON", tmp_json);
+    const child = try std.process.Child.run(.{
+        .allocator = alloc,
+        .argv = &.{"zig-out/bin/waystty"},
+        .env_map = &env,
+    });
+    defer alloc.free(child.stdout);
+    defer alloc.free(child.stderr);
+    if (child.term != .Exited or child.term.Exited != 0) {
+        std.debug.print("bench: waystty exited abnormally: {any}\n  stderr: {s}\n", .{ child.term, child.stderr });
+        std.process.exit(2);
+    }
+    const current_bytes = std.fs.cwd().readFileAlloc(alloc, tmp_json, 16 * 1024) catch |err| {
+        std.debug.print("bench: no JSON output at {s}: {s}\n", .{ tmp_json, @errorName(err) });
+        std.process.exit(2);
+    };
+    defer alloc.free(current_bytes);
+    const current = try bench_stats.readBaselineJson(alloc, current_bytes);
+    defer {
+        alloc.free(current.workload_sha);
+        alloc.free(current.zig_version);
+        alloc.free(current.waystty_sha);
+    }
+    if (mode == .save) {
+        const json_out = try bench_stats.writeBaselineJson(alloc, current);
+        defer alloc.free(json_out);
+        const out = try std.fs.cwd().createFile(baseline_path, .{});
+        defer out.close();
+        try out.writeAll(json_out);
+        std.debug.print("bench: wrote {s} (frame_count={d})\n", .{ baseline_path, current.frame_count });
+        return;
+    }
+    // check mode — compare current to baseline.
+    const baseline_bytes = std.fs.cwd().readFileAlloc(alloc, baseline_path, 16 * 1024) catch |err| {
+        std.debug.print("bench: no baseline at {s}: {s}\n  run: zig build bench-baseline\n", .{ baseline_path, @errorName(err) });
+        std.process.exit(2);
+    };
+    defer alloc.free(baseline_bytes);
+    const baseline = try bench_stats.readBaselineJson(alloc, baseline_bytes);
+    defer {
+        alloc.free(baseline.workload_sha);
+        alloc.free(baseline.zig_version);
+        alloc.free(baseline.waystty_sha);
+    }
+    if (!std.mem.eql(u8, baseline.workload_sha, current.workload_sha)) {
+        std.debug.print("WARN: bench script changed since baseline; consider regenerating via `zig build bench-baseline`\n", .{});
+    }
+    const pct_threshold: f64 = blk: {
+        const v = std.posix.getenv("WAYSTTY_BENCH_REGRESSION_PCT") orelse break :blk 20.0;
+        break :blk std.fmt.parseFloat(f64, v) catch 20.0;
+    };
+    var regressed = false;
+    const SectionName = struct {
+        name: []const u8,
+        base_p99: u32,
+        cur_p99: u32,
+    };
+    const sections = [_]SectionName{
+        .{ .name = "snapshot", .base_p99 = baseline.sections.snapshot.p99, .cur_p99 = current.sections.snapshot.p99 },
+        .{ .name = "row_rebuild", .base_p99 = baseline.sections.row_rebuild.p99, .cur_p99 = current.sections.row_rebuild.p99 },
+        .{ .name = "atlas_upload", .base_p99 = baseline.sections.atlas_upload.p99, .cur_p99 = current.sections.atlas_upload.p99 },
+        .{ .name = "instance_upload", .base_p99 = baseline.sections.instance_upload.p99, .cur_p99 = current.sections.instance_upload.p99 },
+        .{ .name = "gpu_submit", .base_p99 = baseline.sections.gpu_submit.p99, .cur_p99 = current.sections.gpu_submit.p99 },
+    };
+    std.debug.print("bench: threshold {d:.1}% p99 growth\n", .{pct_threshold});
+    for (sections) |s| {
+        const delta_pct: f64 = if (s.base_p99 == 0)
+.0
+        else
+            ((@as(f64, @floatFromInt(s.cur_p99)) - @as(f64, @floatFromInt(s.base_p99))) / @as(f64, @floatFromInt(s.base_p99))) * 100.0;
+        const status = if (delta_pct > pct_threshold) "REGRESSION" else "OK";
+        if (delta_pct > pct_threshold) regressed = true;
+        const sign: []const u8 = if (delta_pct >= 0) "+" else "-";
+        const abs_delta: f64 = if (delta_pct >= 0) delta_pct else -delta_pct;
+        std.debug.print(
+            "bench: {s:<16} p99 {d:>5}us (baseline {d:>5}us)  {s}{d:>5.1}%  {s}\n",
+            .{ s.name, s.cur_p99, s.base_p99, sign, abs_delta, status },
+        );
+    }
+    if (regressed) std.process.exit(1);
+}

tests/bench/baseline.json

@@ @@ -0,0 +1,38 @@ @@
+{
+  "workload_sha": "066a95eee2d2f6195c0eb997e7e18a63f75b48010b538dd287056f948cc65005",
+  "zig_version": "0.15.2",
+  "waystty_sha": "828c61f589b2bac3785e37531a094d6abb1f40ad",
+  "frame_count": 5,
+  "sections": {
+    "snapshot": {
+      "min": 10,
+      "avg": 201,
+      "p99": 214,
+      "max": 725
+    },
+    "row_rebuild": {
+      "min": 115,
+      "avg": 2075,
+      "p99": 2166,
+      "max": 5447
+    },
+    "atlas_upload": {
+      "min": 0,
+      "avg": 10,
+      "p99": 0,
+      "max": 50
+    },
+    "instance_upload": {
+      "min": 4,
+      "avg": 37,
+      "p99": 21,
+      "max": 144
+    },
+    "gpu_submit": {
+      "min": 37,
+      "avg": 58,
+      "p99": 66,
+      "max": 79
+    }
+  }
+}
+\ No newline at end of file