9e9d75b6

Add bench-baseline and bench-check

a73x   2026-04-17 16:31

Baseline stores workload_sha + zig_version + waystty_sha + per-section
stats. Check compares current p99 against baseline, flagging sections
exceeding 20% p99 growth. Threshold overridable via
WAYSTTY_BENCH_REGRESSION_PCT.

main.zig now writes BaselineRecord JSON to WAYSTTY_BENCH_JSON path
alongside the existing human-readable table.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

---

diff --git a/build.zig b/build.zig
index 42f3089..8afb3bf 100644
--- a/build.zig
+++ b/build.zig
@@ -370,4 +370,29 @@ pub fn build(b: *std.Build) void {
    const test_render_run = b.addRunArtifact(test_render_exe);
    test_render_run.step.dependOn(b.getInstallStep());
    test_render_step.dependOn(&test_render_run.step);

    // bench-baseline / bench-check — frame-timing regression guard
    const bench_baseline_mod = b.createModule(.{
        .root_source_file = b.path("src/tools/bench_baseline.zig"),
        .target = target,
        .optimize = optimize,
    });
    bench_baseline_mod.addImport("bench_stats", bench_stats_mod);
    const bench_baseline_exe = b.addExecutable(.{
        .name = "bench-baseline",
        .root_module = bench_baseline_mod,
    });
    b.installArtifact(bench_baseline_exe);

    const bench_baseline_step = b.step("bench-baseline", "Save current frame-timing profile to tests/bench/baseline.json");
    const bench_baseline_run = b.addRunArtifact(bench_baseline_exe);
    bench_baseline_run.addArg("save");
    bench_baseline_run.step.dependOn(b.getInstallStep());
    bench_baseline_step.dependOn(&bench_baseline_run.step);

    const bench_check_step = b.step("bench-check", "Compare current frame timings against baseline");
    const bench_check_run = b.addRunArtifact(bench_baseline_exe);
    bench_check_run.addArg("check");
    bench_check_run.step.dependOn(b.getInstallStep());
    bench_check_step.dependOn(&bench_check_run.step);
}
diff --git a/src/main.zig b/src/main.zig
index 868fca4..5181ede 100644
--- a/src/main.zig
+++ b/src/main.zig
@@ -70,6 +70,61 @@ fn updateWindowTitle(_: *vt.Terminal, ctx: ?*anyopaque, title: ?[:0]const u8) vo
    window.setTitle(title);
}

/// If `WAYSTTY_BENCH_JSON` is set, write a BaselineRecord JSON to that path.
/// Machine-readable companion to `printFrameStats`.
fn writeBenchJson(alloc: std.mem.Allocator, stats: FrameTimingStats, workload: ?[:0]const u8) !void {
    const path = std.posix.getenv("WAYSTTY_BENCH_JSON") orelse return;

    // sha256 of the bench workload string (empty string if no workload)
    var digest: [32]u8 = undefined;
    std.crypto.hash.sha2.Sha256.hash(workload orelse "", &digest, .{});
    var sha_hex: [64]u8 = undefined;
    const hex_lut = "0123456789abcdef";
    for (digest, 0..) |b, i| {
        sha_hex[i * 2] = hex_lut[b >> 4];
        sha_hex[i * 2 + 1] = hex_lut[b & 0x0f];
    }

    // git HEAD (falls back to "unknown")
    const git_head = blk: {
        const r = std.process.Child.run(.{
            .allocator = alloc,
            .argv = &.{ "git", "rev-parse", "HEAD" },
        }) catch {
            break :blk try alloc.dupe(u8, "unknown");
        };
        defer alloc.free(r.stdout);
        defer alloc.free(r.stderr);
        if (r.term != .Exited or r.term.Exited != 0) {
            break :blk try alloc.dupe(u8, "unknown");
        }
        const trimmed = std.mem.trim(u8, r.stdout, "\n \t");
        break :blk try alloc.dupe(u8, trimmed);
    };
    defer alloc.free(git_head);

    const rec = bench_stats.BaselineRecord{
        .workload_sha = &sha_hex,
        .zig_version = @import("builtin").zig_version_string,
        .waystty_sha = git_head,
        .frame_count = stats.frame_count,
        .sections = .{
            .snapshot = stats.snapshot,
            .row_rebuild = stats.row_rebuild,
            .atlas_upload = stats.atlas_upload,
            .instance_upload = stats.instance_upload,
            .gpu_submit = stats.gpu_submit,
        },
    };

    const json_bytes = try bench_stats.writeBaselineJson(alloc, rec);
    defer alloc.free(json_bytes);

    const out = try std.fs.cwd().createFile(path, .{});
    defer out.close();
    try out.writeAll(json_bytes);
}

pub fn main() !void {
    var gpa: std.heap.DebugAllocator(.{}) = .init;
    defer _ = gpa.deinit();
@@ -645,7 +700,11 @@ fn runTerminal(alloc: std.mem.Allocator) !void {
    }

    // Dump timing stats on exit
    printFrameStats(computeFrameStats(&frame_ring));
    const final_stats = computeFrameStats(&frame_ring);
    printFrameStats(final_stats);
    writeBenchJson(alloc, final_stats, bench_script) catch |err| {
        std.log.warn("bench_json write failed: {s}", .{@errorName(err)});
    };

    _ = try ctx.vkd.deviceWaitIdle(ctx.device);
}
diff --git a/src/tools/bench_baseline.zig b/src/tools/bench_baseline.zig
new file mode 100644
index 0000000..3477c10
--- /dev/null
+++ b/src/tools/bench_baseline.zig
@@ -0,0 +1,121 @@
const std = @import("std");
const bench_stats = @import("bench_stats");

pub fn main() !void {
    var gpa: std.heap.DebugAllocator(.{}) = .init;
    defer _ = gpa.deinit();
    const alloc = gpa.allocator();

    const args = try std.process.argsAlloc(alloc);
    defer std.process.argsFree(alloc, args);

    const Mode = enum { save, check };
    const mode: Mode = if (args.len >= 2 and std.mem.eql(u8, args[1], "save"))
        .save
    else
        .check;

    const baseline_path = "tests/bench/baseline.json";
    const tmp_json = "/tmp/waystty-bench-current.json";

    try std.fs.cwd().makePath("tests/bench");

    // Run waystty with WAYSTTY_BENCH=1 WAYSTTY_BENCH_JSON=<tmp>
    var env = try std.process.getEnvMap(alloc);
    defer env.deinit();
    try env.put("WAYSTTY_BENCH", "1");
    try env.put("WAYSTTY_BENCH_JSON", tmp_json);

    const child = try std.process.Child.run(.{
        .allocator = alloc,
        .argv = &.{"zig-out/bin/waystty"},
        .env_map = &env,
    });
    defer alloc.free(child.stdout);
    defer alloc.free(child.stderr);

    if (child.term != .Exited or child.term.Exited != 0) {
        std.debug.print("bench: waystty exited abnormally: {any}\n  stderr: {s}\n", .{ child.term, child.stderr });
        std.process.exit(2);
    }

    const current_bytes = std.fs.cwd().readFileAlloc(alloc, tmp_json, 16 * 1024) catch |err| {
        std.debug.print("bench: no JSON output at {s}: {s}\n", .{ tmp_json, @errorName(err) });
        std.process.exit(2);
    };
    defer alloc.free(current_bytes);

    const current = try bench_stats.readBaselineJson(alloc, current_bytes);
    defer {
        alloc.free(current.workload_sha);
        alloc.free(current.zig_version);
        alloc.free(current.waystty_sha);
    }

    if (mode == .save) {
        const json_out = try bench_stats.writeBaselineJson(alloc, current);
        defer alloc.free(json_out);
        const out = try std.fs.cwd().createFile(baseline_path, .{});
        defer out.close();
        try out.writeAll(json_out);
        std.debug.print("bench: wrote {s} (frame_count={d})\n", .{ baseline_path, current.frame_count });
        return;
    }

    // check mode — compare current to baseline.
    const baseline_bytes = std.fs.cwd().readFileAlloc(alloc, baseline_path, 16 * 1024) catch |err| {
        std.debug.print("bench: no baseline at {s}: {s}\n  run: zig build bench-baseline\n", .{ baseline_path, @errorName(err) });
        std.process.exit(2);
    };
    defer alloc.free(baseline_bytes);

    const baseline = try bench_stats.readBaselineJson(alloc, baseline_bytes);
    defer {
        alloc.free(baseline.workload_sha);
        alloc.free(baseline.zig_version);
        alloc.free(baseline.waystty_sha);
    }

    if (!std.mem.eql(u8, baseline.workload_sha, current.workload_sha)) {
        std.debug.print("WARN: bench script changed since baseline; consider regenerating via `zig build bench-baseline`\n", .{});
    }

    const pct_threshold: f64 = blk: {
        const v = std.posix.getenv("WAYSTTY_BENCH_REGRESSION_PCT") orelse break :blk 20.0;
        break :blk std.fmt.parseFloat(f64, v) catch 20.0;
    };

    var regressed = false;

    const SectionName = struct {
        name: []const u8,
        base_p99: u32,
        cur_p99: u32,
    };

    const sections = [_]SectionName{
        .{ .name = "snapshot", .base_p99 = baseline.sections.snapshot.p99, .cur_p99 = current.sections.snapshot.p99 },
        .{ .name = "row_rebuild", .base_p99 = baseline.sections.row_rebuild.p99, .cur_p99 = current.sections.row_rebuild.p99 },
        .{ .name = "atlas_upload", .base_p99 = baseline.sections.atlas_upload.p99, .cur_p99 = current.sections.atlas_upload.p99 },
        .{ .name = "instance_upload", .base_p99 = baseline.sections.instance_upload.p99, .cur_p99 = current.sections.instance_upload.p99 },
        .{ .name = "gpu_submit", .base_p99 = baseline.sections.gpu_submit.p99, .cur_p99 = current.sections.gpu_submit.p99 },
    };

    std.debug.print("bench: threshold {d:.1}% p99 growth\n", .{pct_threshold});
    for (sections) |s| {
        const delta_pct: f64 = if (s.base_p99 == 0)
            0.0
        else
            ((@as(f64, @floatFromInt(s.cur_p99)) - @as(f64, @floatFromInt(s.base_p99))) / @as(f64, @floatFromInt(s.base_p99))) * 100.0;
        const status = if (delta_pct > pct_threshold) "REGRESSION" else "OK";
        if (delta_pct > pct_threshold) regressed = true;
        const sign: []const u8 = if (delta_pct >= 0) "+" else "-";
        const abs_delta: f64 = if (delta_pct >= 0) delta_pct else -delta_pct;
        std.debug.print(
            "bench: {s:<16} p99 {d:>5}us (baseline {d:>5}us)  {s}{d:>5.1}%  {s}\n",
            .{ s.name, s.cur_p99, s.base_p99, sign, abs_delta, status },
        );
    }

    if (regressed) std.process.exit(1);
}
diff --git a/tests/bench/baseline.json b/tests/bench/baseline.json
new file mode 100644
index 0000000..e8b01a6
--- /dev/null
+++ b/tests/bench/baseline.json
@@ -0,0 +1,38 @@
{
  "workload_sha": "066a95eee2d2f6195c0eb997e7e18a63f75b48010b538dd287056f948cc65005",
  "zig_version": "0.15.2",
  "waystty_sha": "828c61f589b2bac3785e37531a094d6abb1f40ad",
  "frame_count": 5,
  "sections": {
    "snapshot": {
      "min": 10,
      "avg": 201,
      "p99": 214,
      "max": 725
    },
    "row_rebuild": {
      "min": 115,
      "avg": 2075,
      "p99": 2166,
      "max": 5447
    },
    "atlas_upload": {
      "min": 0,
      "avg": 10,
      "p99": 0,
      "max": 50
    },
    "instance_upload": {
      "min": 4,
      "avg": 37,
      "p99": 21,
      "max": 144
    },
    "gpu_submit": {
      "min": 37,
      "avg": 58,
      "p99": 66,
      "max": 79
    }
  }
}
\ No newline at end of file