src/tools/bench_baseline.zig

Ref: main   Size: 4.7 KiB

const std = @import("std");
const bench_stats = @import("bench_stats");

pub fn main() !void {
    var gpa: std.heap.DebugAllocator(.{}) = .init;
    defer _ = gpa.deinit();
    const alloc = gpa.allocator();

    const args = try std.process.argsAlloc(alloc);
    defer std.process.argsFree(alloc, args);

    const Mode = enum { save, check };
    const mode: Mode = if (args.len >= 2 and std.mem.eql(u8, args[1], "save"))
        .save
    else
        .check;

    const baseline_path = "tests/bench/baseline.json";
    const tmp_json = "/tmp/waystty-bench-current.json";

    try std.fs.cwd().makePath("tests/bench");

    // Run waystty with WAYSTTY_BENCH=1 WAYSTTY_BENCH_JSON=<tmp>
    var env = try std.process.getEnvMap(alloc);
    defer env.deinit();
    try env.put("WAYSTTY_BENCH", "1");
    try env.put("WAYSTTY_BENCH_JSON", tmp_json);

    const child = try std.process.Child.run(.{
        .allocator = alloc,
        .argv = &.{"zig-out/bin/waystty"},
        .env_map = &env,
    });
    defer alloc.free(child.stdout);
    defer alloc.free(child.stderr);

    if (child.term != .Exited or child.term.Exited != 0) {
        std.debug.print("bench: waystty exited abnormally: {any}\n  stderr: {s}\n", .{ child.term, child.stderr });
        std.process.exit(2);
    }

    const current_bytes = std.fs.cwd().readFileAlloc(alloc, tmp_json, 16 * 1024) catch |err| {
        std.debug.print("bench: no JSON output at {s}: {s}\n", .{ tmp_json, @errorName(err) });
        std.process.exit(2);
    };
    defer alloc.free(current_bytes);

    const current = try bench_stats.readBaselineJson(alloc, current_bytes);
    defer {
        alloc.free(current.workload_sha);
        alloc.free(current.zig_version);
        alloc.free(current.waystty_sha);
    }

    if (mode == .save) {
        const json_out = try bench_stats.writeBaselineJson(alloc, current);
        defer alloc.free(json_out);
        const out = try std.fs.cwd().createFile(baseline_path, .{});
        defer out.close();
        try out.writeAll(json_out);
        std.debug.print("bench: wrote {s} (frame_count={d})\n", .{ baseline_path, current.frame_count });
        return;
    }

    // check mode — compare current to baseline.
    const baseline_bytes = std.fs.cwd().readFileAlloc(alloc, baseline_path, 16 * 1024) catch |err| {
        std.debug.print("bench: no baseline at {s}: {s}\n  run: zig build bench-baseline\n", .{ baseline_path, @errorName(err) });
        std.process.exit(2);
    };
    defer alloc.free(baseline_bytes);

    const baseline = try bench_stats.readBaselineJson(alloc, baseline_bytes);
    defer {
        alloc.free(baseline.workload_sha);
        alloc.free(baseline.zig_version);
        alloc.free(baseline.waystty_sha);
    }

    if (!std.mem.eql(u8, baseline.workload_sha, current.workload_sha)) {
        std.debug.print("WARN: bench script changed since baseline; consider regenerating via `zig build bench-baseline`\n", .{});
    }

    const pct_threshold: f64 = blk: {
        const v = std.posix.getenv("WAYSTTY_BENCH_REGRESSION_PCT") orelse break :blk 20.0;
        break :blk std.fmt.parseFloat(f64, v) catch 20.0;
    };

    var regressed = false;

    const SectionName = struct {
        name: []const u8,
        base_p99: u32,
        cur_p99: u32,
    };

    const sections = [_]SectionName{
        .{ .name = "snapshot", .base_p99 = baseline.sections.snapshot.p99, .cur_p99 = current.sections.snapshot.p99 },
        .{ .name = "row_rebuild", .base_p99 = baseline.sections.row_rebuild.p99, .cur_p99 = current.sections.row_rebuild.p99 },
        .{ .name = "atlas_upload", .base_p99 = baseline.sections.atlas_upload.p99, .cur_p99 = current.sections.atlas_upload.p99 },
        .{ .name = "instance_upload", .base_p99 = baseline.sections.instance_upload.p99, .cur_p99 = current.sections.instance_upload.p99 },
        .{ .name = "gpu_submit", .base_p99 = baseline.sections.gpu_submit.p99, .cur_p99 = current.sections.gpu_submit.p99 },
    };

    std.debug.print("bench: threshold {d:.1}% p99 growth\n", .{pct_threshold});
    for (sections) |s| {
        const delta_pct: f64 = if (s.base_p99 == 0)
            0.0
        else
            ((@as(f64, @floatFromInt(s.cur_p99)) - @as(f64, @floatFromInt(s.base_p99))) / @as(f64, @floatFromInt(s.base_p99))) * 100.0;
        const status = if (delta_pct > pct_threshold) "REGRESSION" else "OK";
        if (delta_pct > pct_threshold) regressed = true;
        const sign: []const u8 = if (delta_pct >= 0) "+" else "-";
        const abs_delta: f64 = if (delta_pct >= 0) delta_pct else -delta_pct;
        std.debug.print(
            "bench: {s:<16} p99 {d:>5}us (baseline {d:>5}us)  {s}{d:>5.1}%  {s}\n",
            .{ s.name, s.cur_p99, s.base_p99, sign, abs_delta, status },
        );
    }

    if (regressed) std.process.exit(1);
}