632d1166

feat: the daemon encodes a grid row as cells, and the size is measured

a73x 2026-09-04 17:00

Commit message

feat: the daemon encodes a grid row as cells, and the size is measured

encodeViewportRow and encodeScrollback read ghostty's page cells straight
into CellRow bytes: style, wide state, and the cell's text including its
grapheme continuations. The row ends at the last cell that is not a default
blank, and a cell carrying only a background counts as content — a coloured
EL would otherwise vanish. encodeScrollback clamps the way dumpScrollback
does and returns the rows dense with the window it actually encoded.

A space is sent as a space rather than as the empty text a blank cell
carries. Both paint the same, but only a one-byte narrow cell qualifies for
the run's ascii form, so one empty cell drops the whole run out of it and
every other cell then pays a head byte: blanking interior spaces cost 36%
of a prose screen.

The measurement the design's gate reads is in decisions.md. The gate is NOT
met — prose 1.32 against its 1.5, but vim 1.45 and htop 1.56 against their
1.2, because a run header is 16 absolute bytes where SGR spends four to
seven on a delta, and a highlighted source line changes style ten times.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CsWfuJFQbTfGtKZLS5qw4q

docs/decisions.md

+matching sha alone. That rule was established at v0.0.1-16 (2026-08-29),
+where a publish was refused against an identical binary built 76 seconds
+earlier and the download-and-cmp is what established it was identical.
+## 2026-09-04 — cells on the wire: the size measurement
+The design's go/no-go gate is what a cell row costs against the VT row it
+replaces. `engine.zig`'s test "cells: wire size vs VT rows" measures it on
+four 80x24 screens fed real output shapes, comparing `dumpVtRow` against
+`encodeViewportRow` over the whole viewport:
+```
+cells-measure prose: vt=988 cells=1308 ratio=1.32
+cells-measure vim: vt=3132 cells=4533 ratio=1.45
+cells-measure htop: vt=4167 cells=6488 ratio=1.56
+cells-measure shell: vt=128 cells=127 ratio=0.99
+```
+**Gate verdict: NOT met.** The gate asked for prose <= 1.5 and vim/htop
+<= 1.2. Prose passes at 1.32 and the shell screen is free at 0.99, but the
+two style-dense screens are 1.45 and 1.56, well over their 1.2. The cost is
+the run header: 16 bytes of style per run, and a syntax-highlighted source
+line or an htop row changes style eight to ten times, so a row pays 130-160
+bytes of header for 70 bytes of text. VT spends four to seven bytes on the
+same style change because SGR is a delta against the previous style and a
+run header is absolute. Whether that is worth paying for is the design's
+call, not the encoder's — the alternative the spec names is VT frames for
+terminal clients and cells for native and browser ones.
+The first measurement of the same encoder read 2.08 / 1.67 / 1.80 / 1.17.
+The difference was one line: the encoder sent a space cell as the EMPTY text
+a blank cell carries, on the reasoning that both paint the same and the empty
+form is shorter. It is shorter for one cell and much longer for the row.
+Only a one-byte narrow cell qualifies for the run's ascii form, in which a
+cell is its single byte and nothing else, so one empty cell drops the whole
+run out of that form and every remaining cell then pays a head byte as well
+as its text. Prose is a third spaces, so blanking them cost 36% of the
+screen. Sending the space as a space is also the better answer for copy:
+an interior space is a space. Trailing blanks were never the question —
+the row already ends at the last cell that is not one.

src/engine/engine.zig

@@ @@ -3,6 +3,7 @@ @@
+// folder rule 4 exemption: ghostty-vt's grid is fed and read in VT bytes — this module IS the VT, and its selection formatter emits SGR.
+const std = @import("std");
+const vt = @import("ghostty-vt");
+const proto = @import("protocol.zig");
+/// MuxHandler's own `vt` method shadows the `vt` import inside its body,
+/// so the dep types it names are spelled through these aliases.
@@ @@ -557,6 +558,132 @@ pub const Engine = struct { @@
+        return self.formatSelection(alloc, "\x1b[0m", sel);
+    pub const EncodedRows = struct { first: u32, count: u16, bytes: []u8 };
+    fn packColor(col: anytype) u32 {
+        return switch (col) {
+            .none => proto.color_none,
+            .palette => |p| proto.colorPalette(p),
+            .rgb => |c| proto.colorRgb(c.r, c.g, c.b),
+        };
+    }
+    fn packStyle(style: vt.Style) proto.CellStyle {
+        return .{
+            .fg = packColor(style.fg_color),
+            .bg = packColor(style.bg_color),
+            .ul = packColor(style.underline_color),
+            .flags = @bitCast(style.flags),
+        };
+    }
+    /// One row of the active screen at `pt` as a CellRow. The last cell sent
+    /// is the last one that is not a default blank; a bare-background cell
+    /// counts as content, or a coloured EL would vanish.
+    fn encodeRowAt(self: *Engine, alloc: std.mem.Allocator, pt: vt.point.Point) ![]u8 {
+        const screen = self.term.screens.active;
+        var list: std.ArrayList(u8) = .empty;
+        errdefer list.deinit(alloc);
+        var w = try proto.CellRowWriter.begin(&list, alloc);
+        errdefer w.deinit();
+        const pin = screen.pages.pin(pt) orelse {
+            w.finish();
+            return list.toOwnedSlice(alloc);
+        };
+        const page = &pin.node.data;
+        const rac = pin.rowAndCell();
+        const cells = page.getCells(rac.row);
+        // A page is built at the terminal's width, but the min keeps a
+        // mismatch a short row rather than a read past the row's cells.
+        const cols: usize = @min(@as(usize, self.term.cols), cells.len);
+        // Find the last cell worth sending.
+        var last: usize = 0;
+        var any = false;
+        for (cells[0..cols], 0..) |c, x| {
+            const blank = c.style_id == 0 and c.wide == .narrow and switch (c.content_tag) {
+                .codepoint => c.content.codepoint == 0 or c.content.codepoint == ' ',
+                .codepoint_grapheme => false,
+                .bg_color_palette, .bg_color_rgb => false,
+            };
+            if (!blank) {
+                last = x;
+                any = true;
+            }
+        }
+        if (!any) {
+            w.finish();
+            return list.toOwnedSlice(alloc);
+        }
+        var text: [proto.cell_text_max]u8 = undefined;
+        for (cells[0 .. last + 1]) |*c| {
+            var style: proto.CellStyle = if (c.style_id == 0) .{} else packStyle(page.styles.get(page.memory, c.style_id).*);
+            var len: usize = 0;
+            switch (c.content_tag) {
+                .codepoint, .codepoint_grapheme => {
+                    if (c.content.codepoint != 0) {
+                        len += std.unicode.utf8Encode(@intCast(c.content.codepoint), text[len..]) catch 0;
+                    }
+                    if (c.hasGrapheme()) {
+                        if (page.lookupGrapheme(c)) |cps| {
+                            for (cps) |cp| {
+                                var buf: [4]u8 = undefined;
+                                const n = std.unicode.utf8Encode(@intCast(cp), &buf) catch continue;
+                                if (len + n > proto.cell_text_max) break;
+                                @memcpy(text[len .. len + n], buf[0..n]);
+                                len += n;
+                            }
+                        }
+                    }
+                },
+                .bg_color_palette => style.bg = proto.colorPalette(c.content.color_palette),
+                .bg_color_rgb => style.bg = proto.colorRgb(c.content.color_rgb.r, c.content.color_rgb.g, c.content.color_rgb.b),
+            }
+            const wide: proto.Wide = switch (c.wide) {
+                .narrow => .narrow,
+                .wide => .wide,
+                .spacer_tail => .spacer_tail,
+                .spacer_head => .spacer_head,
+            };
+            // A space is sent AS a space, not as the empty text a blank cell
+            // carries. Both paint the same, but only a one-byte narrow cell
+            // qualifies for the run's ascii form, and one empty cell drops the
+            // whole run out of it — every remaining cell then pays a head byte.
+            // Blanking interior spaces cost 36% on a prose screen for nothing
+            // (measured 2026-09-04, decisions.md). Trailing blanks are already
+            // gone: the row ends at the last cell that is not one.
+            const t: []const u8 = text[0..len];
+            try w.cell(style, wide, t);
+        }
+        w.finish();
+        return list.toOwnedSlice(alloc);
+    }
+    pub fn encodeViewportRow(self: *Engine, alloc: std.mem.Allocator, y: u16) ![]u8 {
+        std.debug.assert(y < self.term.rows);
+        return self.encodeRowAt(alloc, .{ .viewport = .{ .x = 0, .y = y } });
+    }
+    /// Screen-space rows [start, start+count) on the active screen, clamped
+    /// to what exists, dense. Row 0 is the oldest retained history row.
+    pub fn encodeScrollback(self: *Engine, alloc: std.mem.Allocator, start: u32, count: u16) !EncodedRows {
+        const total: u32 = self.historyRows() + self.term.rows;
+        const first = @min(start, total -| 1);
+        const last = @min(first + count -| 1, total -| 1);
+        var list: std.ArrayList(u8) = .empty;
+        errdefer list.deinit(alloc);
+        var y = first;
+        var n: u16 = 0;
+        while (y <= last and total > 0) : (y += 1) {
+            const row = try self.encodeRowAt(alloc, .{ .screen = .{ .x = 0, .y = y } });
+            defer alloc.free(row);
+            try list.appendSlice(alloc, row);
+            n += 1;
+        }
+        return .{ .first = first, .count = n, .bytes = try list.toOwnedSlice(alloc) };
+    }
+    /// Screen-space rows, zero being the oldest retained. Formatting writes
+    /// into a fixed-size allocation: hostile coordinates cannot blow it up.
+    pub fn extractSelection(
+    try e.resize(90, 28);
+    try std.testing.expectEqualStrings("", e.ptyOutput());
+fn decodeAll(alloc: std.mem.Allocator, bytes: []const u8) ![]proto.DecodedCell {
+    var out: std.ArrayList(proto.DecodedCell) = .empty;
+    var r = try proto.CellRowReader.init(bytes);
+    while (try r.next()) |c| try out.append(alloc, c);
+    return out.toOwnedSlice(alloc);
+}
+test "encodeViewportRow: ascii, a wide glyph with its spacer, a grapheme, and trailing blanks dropped" {
+    const alloc = std.testing.allocator;
+    var e = try Engine.init(alloc, .{ .cols = 12, .rows = 2 });
+    defer e.deinit();
+    e.feed("ab漢e\u{301}");
+    const row = try e.encodeViewportRow(alloc, 0);
+    defer alloc.free(row);
+    const cells = try decodeAll(alloc, row);
+    defer alloc.free(cells);
+    // a b 漢 (spacer) é — five cells; the seven blanks after are not sent.
+    try std.testing.expectEqual(@as(usize, 5), cells.len);
+    try std.testing.expectEqualStrings("a", cells[0].text);
+    try std.testing.expectEqual(proto.Wide.wide, cells[2].wide);
+    try std.testing.expectEqualStrings("漢", cells[2].text);
+    try std.testing.expectEqual(proto.Wide.spacer_tail, cells[3].wide);
+    try std.testing.expectEqualStrings("e\u{301}", cells[4].text);
+}
+test "encodeViewportRow: styles pack as the wire says, and a bg-only cell is not blank" {
+    const alloc = std.testing.allocator;
+    var e = try Engine.init(alloc, .{ .cols = 8, .rows = 1 });
+    defer e.deinit();
+    // bold red on palette-4 blue 'x', then EL with the blue background held:
+    // the cells after x carry a background and no glyph.
+    e.feed("\x1b[1;31;44mx\x1b[0m\x1b[44m\x1b[K");
+    const row = try e.encodeViewportRow(alloc, 0);
+    defer alloc.free(row);
+    const cells = try decodeAll(alloc, row);
+    defer alloc.free(cells);
+    try std.testing.expectEqual(@as(usize, 8), cells.len);
+    try std.testing.expectEqual(proto.colorPalette(1), cells[0].style.fg);
+    try std.testing.expectEqual(proto.colorPalette(4), cells[0].style.bg);
+    try std.testing.expectEqual(@as(u16, 1), cells[0].style.flags & 1);
+    try std.testing.expectEqualStrings("", cells[7].text);
+    try std.testing.expectEqual(proto.colorPalette(4), cells[7].style.bg);
+}
+test "encodeViewportRow: an interior space keeps the row in one ascii run" {
+    const alloc = std.testing.allocator;
+    var e = try Engine.init(alloc, .{ .cols = 8, .rows = 1 });
+    defer e.deinit();
+    e.feed("a b");
+    const row = try e.encodeViewportRow(alloc, 0);
+    defer alloc.free(row);
+    // Sending the space as empty text would drop this run out of the ascii
+    // form and cost a head byte on every cell of it — 36% of a prose screen.
+    try std.testing.expectEqualSlices(u8, &[_]u8{
+, 0, // ncells
+,   0,   0,   0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // run header, ascii
+        'a', ' ', 'b',
+    }, row);
+}
+test "encodeViewportRow: an empty row is ncells 0 and nothing else" {
+    const alloc = std.testing.allocator;
+    var e = try Engine.init(alloc, .{ .cols = 8, .rows = 1 });
+    defer e.deinit();
+    const row = try e.encodeViewportRow(alloc, 0);
+    defer alloc.free(row);
+    try std.testing.expectEqualSlices(u8, &[_]u8{ 0, 0 }, row);
+}
+test "encodeScrollback: clamps like dumpScrollback and returns dense rows" {
+    const alloc = std.testing.allocator;
+    var e = try Engine.init(alloc, .{ .cols = 8, .rows = 2, .max_scrollback = 10 });
+    defer e.deinit();
+    e.feed("one\r\ntwo\r\nthree\r\nfour");
+    // history: one two; viewport: three four. Ask for 3 rows from 1: two three four.
+    const got = try e.encodeScrollback(alloc, 1, 3);
+    defer alloc.free(got.bytes);
+    try std.testing.expectEqual(@as(u32, 1), got.first);
+    try std.testing.expectEqual(@as(u16, 3), got.count);
+    var r = try proto.CellRowReader.init(got.bytes);
+    try std.testing.expectEqualStrings("t", (try r.next()).?.text);
+    // Past the end clamps to what exists.
+    const tail = try e.encodeScrollback(alloc, 100, 5);
+    defer alloc.free(tail.bytes);
+    try std.testing.expectEqual(@as(u32, 3), tail.first);
+    try std.testing.expectEqual(@as(u16, 1), tail.count);
+}
+fn vtBytes(alloc: std.mem.Allocator, e: *Engine) !usize {
+    var n: usize = 0;
+    var y: u16 = 0;
+    while (y < e.term.rows) : (y += 1) {
+        const b = try e.dumpVtRow(alloc, y);
+        defer alloc.free(b);
+        n += b.len;
+    }
+    return n;
+}
+fn cellBytes(alloc: std.mem.Allocator, e: *Engine) !usize {
+    var n: usize = 0;
+    var y: u16 = 0;
+    while (y < e.term.rows) : (y += 1) {
+        const b = try e.encodeViewportRow(alloc, y);
+        defer alloc.free(b);
+        n += b.len;
+    }
+    return n;
+}
+test "cells: wire size vs VT rows (measurement; the spec's gate reads this)" {
+    const alloc = std.testing.allocator;
+    const Screen = struct { name: []const u8, feed: []const u8 };
+    const prose = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor in\r\n" ** 24;
+    const vim = ("\x1b[33m 12 \x1b[0m\x1b[34mfn\x1b[0m main() \x1b[35m{\x1b[0m \x1b[32m// a comment that runs on\x1b[0m \x1b[31mreturn\x1b[0m 0;\r\n") ** 24;
+    const htop = ("\x1b[42m 1 \x1b[0m\x1b[7m[||||||||    12.3%]\x1b[0m \x1b[36m1234\x1b[0m \x1b[33mroot\x1b[0m \x1b[1m20\x1b[0m   0 \x1b[32m 12.0\x1b[0m \x1b[31m 0.4\x1b[0m /usr/bin/thing --flag\r\n") ** 24;
+    const shell = "$ ls\r\nbuild.zig  docs  src  test\r\n$ \r\n";
+    const screens = [_]Screen{
+        .{ .name = "prose", .feed = prose },
+        .{ .name = "vim", .feed = vim },
+        .{ .name = "htop", .feed = htop },
+        .{ .name = "shell", .feed = shell },
+    };
+    for (screens) |s| {
+        var e = try Engine.init(alloc, .{ .cols = 80, .rows = 24 });
+        defer e.deinit();
+        e.feed(s.feed);
+        const v = try vtBytes(alloc, e);
+        const c = try cellBytes(alloc, e);
+        std.debug.print("\ncells-measure {s}: vt={d} cells={d} ratio={d:.2}\n", .{ s.name, v, c, @as(f64, @floatFromInt(c)) / @as(f64, @floatFromInt(v)) });
+    }
+}