llm
Download
386 lines 16 KiBPythonModified 2026-08-31 raw
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386
#!/usr/bin/env python3
"""Render benchmark results (results.json) as comparison graphs.

One horizontal-bar panel per benchmark, languages sorted fastest-first.
Tomo is drawn in an accent color; every other language is neutral gray, so
the graph reads as "Tomo vs. the field" and is colorblind-safe by
construction (one accent vs. gray). Each bar is directly labeled with its
wall-clock time and slowdown relative to the fastest language.

Where a few entries are far slower than the rest of the field (Python is 300x
the leader on spectral-norm), the panel's x-axis is truncated to the rest of
the field: an ordinary axis, just a shorter one. The entries that don't fit run
off the end of it, marked with an arrowhead and labeled with their real time.
See truncate_at() for where.

Writes a combined overview (<out>.svg/.png, all benchmarks stacked) plus one
standalone graph per benchmark (<out>-<benchmark>.svg/.png).

Usage: python3 plot.py [results.json] [-o out_basename]
"""
import json
import os
import sys

import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
from matplotlib.patches import Patch

# Hatching marks the strippable part of a size bar (see size_panel). The 1.0
# default is faint enough to read as texture rather than as a distinct region,
# which is the whole point of drawing it.
plt.rcParams["hatch.linewidth"] = 2.0

HERE = os.path.dirname(os.path.abspath(__file__))

# "Tomo vs the field": one accent + one neutral. Ink tokens for all text.
ACCENT = "#e4572e"   # Tomo
NEUTRAL = "#b8bcc2"  # every other language
INVALID = "#d64545"  # output-mismatch marker
INK = "#202124"
MUTED = "#5f6368"
GRID = "#e6e8eb"


# A few very slow entries squash everyone else: on spectral-norm Python takes
# 182s against a 0.61s leader, so every other bar is a sliver. The panel then
# truncates its x-axis to the rest of the field, an ordinary axis with an
# ordinary linear scale and ordinary ticks, just a shorter one, and the entries
# that do not fit run off the end of it, marked with an arrowhead and labeled
# with their real time inside the bar. Where to truncate is derived from the
# data rather than fixed at some number of seconds: sort the times and truncate
# above the *lowest* neighbor-to-neighbor jump that is big enough to matter and
# leaves only a few entries above it, taking the lowest such jump rather than
# the biggest, because it is the one that wins back the most axis for the pack
# (mandelbrot's biggest jump is Lua->Python at the very top, where truncating
# there would leave the field just as squashed).
BREAK_JUMP = 2.5        # a jump this large can host the truncation...
BREAK_MAX_SHARE = 0.25  # ...if at most this fraction of the field is above it
BREAK_MIN_SQUASH = 0.4  # ...and the rest is squashed into this fraction of the
                        #    axis, i.e. there is real room to be won back

# Only a handful of languages produce a standalone static binary at all, and it
# thins out further per benchmark. Below this many entries a panel is a list,
# not a comparison, so the size chart leaves it out (and says which).
MIN_SIZE_LANGS = 5


def load(path):
    with open(path) as f:
        return json.load(f)


def truncate_at(times):
    """The slowest time the panel still shows, or None to show them all.

    `times` must be sorted ascending. Everything above the returned value runs
    off the end of the axis.
    """
    slowest = times[-1] if times else 0.0
    if len(times) < 4 or slowest <= 0:
        return None
    for i in range(max(1, len(times) - int(len(times) * BREAK_MAX_SHARE)), len(times)):
        jump = times[i] / times[i - 1] if times[i - 1] > 0 else 0.0
        if jump < BREAK_JUMP:
            continue
        # ...but only if the field really is being squashed; if it already
        # fills a decent share of the axis there is nothing to win back.
        if times[i - 1] / slowest <= BREAK_MIN_SQUASH:
            return times[i - 1]
    return None


def draw_offchart(ax, xlim, yi, height, color):
    """Arrowhead at the axis edge: this bar keeps going past the chart."""
    h = height / 2
    tip = xlim * 1.022
    ax.fill([xlim, tip, xlim], [yi + h, yi, yi - h], color=color, zorder=4,
            linewidth=0, clip_on=False)


def panel(ax, bname, block):
    rows = []
    for lang, r in block["results"].items():
        rows.append((lang, r["label"], r["seconds"], r.get("valid", True)))
    rows.sort(key=lambda t: t[2])  # fastest first
    fastest = min((s for *_, s, _ in [(*r,) for r in rows]), default=1.0)
    fastest = rows[0][2] if rows else 1.0

    labels = [lbl for _, lbl, _, _ in rows]
    times = [s for *_, s, _ in rows]
    y = range(len(rows))

    colors = []
    for lang, _, _, valid in rows:
        if not valid:
            colors.append(INVALID)
        elif lang == "tomo":
            colors.append(ACCENT)
        else:
            colors.append(NEUTRAL)

    # A few runaway entries would squash the rest of the field into slivers, so
    # the axis stops at the rest of the field (see truncate_at) and they run off
    # the end of it. The axis is an ordinary one either way, with the same
    # linear scale and the same ticks, so bar lengths mean the same thing
    # throughout.
    shown = truncate_at(times)
    xmax = shown if shown is not None else (max(times) if times else 1.0)
    # The usual headroom past the longest bar is where its label goes; an
    # off-chart bar fills it and carries its label inside itself instead.
    xlim = xmax * 1.18
    widths = [min(s, xlim) for s in times]

    # Draw top (fastest) at the top of the panel.
    ax.barh(y, widths, color=colors, height=0.68, zorder=3)
    ax.invert_yaxis()
    ax.set_yticks(list(y))
    ax.set_yticklabels(labels, fontsize=10,
                       color=INK)
    # Emphasize the Tomo tick label.
    for tick, (lang, *_ ) in zip(ax.get_yticklabels(), rows):
        if lang == "tomo":
            tick.set_color(ACCENT)
            tick.set_fontweight("bold")

    ax.set_xlim(0, xlim)
    for spine in ("top", "right", "left"):
        ax.spines[spine].set_visible(False)
    ax.spines["bottom"].set_color(GRID)
    ax.tick_params(axis="x", colors=MUTED, labelsize=9, length=0)
    ax.tick_params(axis="y", length=0)
    ax.xaxis.grid(True, color=GRID, linewidth=1, zorder=0)
    ax.set_axisbelow(True)
    for yi, (lang, _, s, valid), w, color in zip(y, rows, widths, colors):
        rel = s / fastest if fastest else 1.0
        txt = f"{s:.2f}s" + (f"  ({rel:.1f}×)" if rel >= 1.005 else "  (fastest)")
        if not valid:
            txt += "  ✗ output"
        if w >= xlim:  # runs off the chart
            draw_offchart(ax, xlim, yi, 0.68, color)
            # The arrowhead already says "keeps going"; the label just needs
            # to be inside the bar, since there is no room past it.
            ax.text(w - xmax * 0.016, yi, txt, va="center", ha="right",
                    fontsize=9, color="white" if lang == "tomo" else INK,
                    fontweight="bold" if lang == "tomo" else "normal", zorder=5)
        else:
            ax.text(w + xmax * 0.012, yi, txt, va="center", ha="left",
                    fontsize=9, color=INK if lang == "tomo" else MUTED,
                    fontweight="bold" if lang == "tomo" else "normal", zorder=4)

    args = " ".join(block.get("args", []))
    ax.set_title(f"{bname}   (n = {args})", loc="left", fontsize=13,
                 color=INK, fontweight="bold", pad=8)
    ax.set_xlabel("wall-clock seconds (lower is faster)", fontsize=9, color=MUTED)


def human_bytes(n):
    """Compact size label: KB below 1 MB, else MB (binary units)."""
    if n < 1024 * 1024:
        return f"{n / 1024:.0f} KB"
    return f"{n / (1024 * 1024):.2f} MB"


def size_panel(ax, bname, block):
    rows = []
    for lang, r in block["results"].items():
        # Older sizes.json files carry no stripped measurement; treat the whole
        # binary as content rather than inventing a band.
        rows.append((lang, r["label"], r["bytes"],
                     r.get("stripped_bytes", r["bytes"])))
    rows.sort(key=lambda t: t[2])  # smallest first
    smallest = rows[0][2] if rows else 1

    labels = [lbl for _, lbl, _, _ in rows]
    vals = [b for _, _, b, _ in rows]
    kept = [k for *_, k in rows]
    y = range(len(rows))

    colors = [ACCENT if lang == "tomo" else NEUTRAL for lang, _, _, _ in rows]
    # Two segments per bar: what survives `strip` drawn solid, and what it
    # removes (debug info, symbol tables) hatched onto the end. The bar's full
    # length is still the size as built, which is what actually ships.
    ax.barh(y, kept, color=colors, height=0.68, zorder=3)
    # linewidth=0 on purpose: a stroked edge is centred on the bar's boundary,
    # so it would make this segment render a pixel or two taller than the solid
    # one beside it. The hatch lines still take their color from edgecolor.
    ax.barh(y, [v - k for v, k in zip(vals, kept)], left=kept, height=0.68,
            color="white", edgecolor=colors, hatch="/////", linewidth=0,
            zorder=3)
    ax.invert_yaxis()
    ax.set_yticks(list(y))
    ax.set_yticklabels(labels, fontsize=10, color=INK)
    for tick, (lang, *_) in zip(ax.get_yticklabels(), rows):
        if lang == "tomo":
            tick.set_color(ACCENT)
            tick.set_fontweight("bold")

    xmax = max(vals) if vals else 1
    ax.set_xlim(0, xmax * 1.34) # room for the longest "N KB (M×) · K KB code"
    for spine in ("top", "right", "left"):
        ax.spines[spine].set_visible(False)
    ax.spines["bottom"].set_color(GRID)
    ax.tick_params(axis="x", colors=MUTED, labelsize=9, length=0)
    ax.tick_params(axis="y", length=0)
    ax.xaxis.set_major_formatter(
        plt.FuncFormatter(lambda v, _p: f"{v / (1024 * 1024):.1f}"))
    ax.xaxis.grid(True, color=GRID, linewidth=1, zorder=0)
    ax.set_axisbelow(True)

    for yi, (lang, _, b, k) in zip(y, rows):
        # rows is sorted ascending, so only the first row is actually the
        # smallest; everything else carries its ratio. A second decimal when
        # the ratio would round to 1.0 keeps a genuinely larger binary from
        # reading as a tie.
        rel = b / smallest if smallest else 1.0
        ratio = f"{rel:.2f}×" if rel < 1.05 else f"{rel:.1f}×"
        txt = human_bytes(b) + ("  (smallest)" if yi == 0 else f"  ({ratio})")
        if b - k > 0.02 * b:
            txt += f"  ·  {human_bytes(k)} code"
        ax.text(b + xmax * 0.012, yi, txt, va="center", ha="left",
                fontsize=9, color=INK if lang == "tomo" else MUTED,
                fontweight="bold" if lang == "tomo" else "normal", zorder=4)

    ax.set_title(f"{bname}", loc="left", fontsize=13, color=INK,
                 fontweight="bold", pad=8)
    ax.set_xlabel("static binary size (MB, as built; smaller is leaner)",
                  fontsize=9, color=MUTED)


def main_sizes(path, out):
    data = load(path)
    benches, thin = [], []
    for b, blk in data.items():
        if not blk.get("results"):
            continue
        (benches if len(blk["results"]) >= MIN_SIZE_LANGS else thin).append((b, blk))
    if thin:
        dropped = ", ".join(f"{b} ({len(blk['results'])})" for b, blk in thin)
        print(f"skipping (fewer than {MIN_SIZE_LANGS} languages): {dropped}")
    if not benches:
        print("no benchmark has enough languages to chart")
        return
    n = len(benches)
    fig_h = 1.0 + 3.2 * n
    fig, axes = plt.subplots(n, 1, figsize=(9, fig_h), squeeze=False)
    fig.patch.set_facecolor("white")
    for ax, (bname, block) in zip(axes[:, 0], benches):
        ax.set_facecolor("white")
        size_panel(ax, bname, block)
    legend = [
        Patch(facecolor=ACCENT, label="Tomo"),
        Patch(facecolor=NEUTRAL, label="other languages"),
        Patch(facecolor="white", edgecolor=MUTED, hatch="////",
              label="removed by `strip`"),
    ]
    axes[0, 0].legend(handles=legend, loc="upper right", frameon=False,
                      fontsize=9, borderaxespad=0.6)
    # Title/footer bands are fixed heights (inches converted to figure
    # fractions), the same as main(): a fixed *fraction* opens a bigger and
    # bigger gap under the title as the figure grows with the panel count.
    top = 1.0 - 0.55 / fig_h
    bottom = 0.35 / fig_h
    fig.suptitle("Static binary size: Tomo vs. other compiled languages",
                 x=0.02, y=1.0 - 0.28 / fig_h, ha="left", va="top",
                 fontsize=14, fontweight="bold", color=INK)
    fig.text(0.02, 0.10 / fig_h,
             "statically linked · bars are the size as built; the hatched part "
             "is what `strip` removes · only languages that can produce a "
             "standalone static binary",
             ha="left", fontsize=8, color=MUTED)
    fig.tight_layout(rect=[0, bottom, 1, top])
    svg = os.path.join(HERE, out + ".svg")
    png = os.path.join(HERE, out + ".png")
    fig.savefig(svg)
    fig.savefig(png, dpi=140)
    print(f"wrote {svg}\nwrote {png}")
    plt.close(fig)


def main():
    argv = sys.argv[1:]
    out = "results"
    if "--sizes" in argv:
        argv.remove("--sizes")
        out = "sizes"
        if "-o" in argv:
            i = argv.index("-o")
            out = argv[i + 1]
            del argv[i:i + 2]
        path = argv[0] if argv else os.path.join(HERE, "sizes.json")
        main_sizes(path, out)
        return
    if "-o" in argv:
        i = argv.index("-o")
        out = argv[i + 1]
        del argv[i:i + 2]
    path = argv[0] if argv else os.path.join(HERE, "results.json")
    data = load(path)

    benches = list(data.items())
    n = len(benches)
    fig_h = 1.0 + 3.2 * n
    fig, axes = plt.subplots(n, 1, figsize=(9, fig_h),
                             squeeze=False)
    fig.patch.set_facecolor("white")
    for ax, (bname, block) in zip(axes[:, 0], benches):
        ax.set_facecolor("white")
        panel(ax, bname, block)

    # Legend sits in the empty upper-right of the first panel (the fastest
    # language's row leaves that corner clear), so it never fights the title.
    legend = [
        Patch(facecolor=ACCENT, label="Tomo"),
        Patch(facecolor=NEUTRAL, label="other languages"),
    ]
    axes[0, 0].legend(handles=legend, loc="upper right", frameon=False,
                      fontsize=9, borderaxespad=0.6)
    # Reserve title/footer bands as *fixed* heights (in inches, converted to
    # figure fractions) rather than fixed fractions, since otherwise a taller
    # figure (more benchmarks) opens a huge gap under the title.
    top = 1.0 - 0.55 / fig_h
    bottom = 0.35 / fig_h
    fig.suptitle("Tomo vs. other languages: Computer Language Benchmarks Game",
                 x=0.02, y=1.0 - 0.28 / fig_h, ha="left", va="top",
                 fontsize=14, fontweight="bold", color=INK)
    fig.text(0.02, 0.10 / fig_h,
             "best of 3 runs · same input · pinned to one core · outputs validated against the C reference",
             ha="left", fontsize=8, color=MUTED)
    fig.tight_layout(rect=[0, bottom, 1, top])

    svg = os.path.join(HERE, out + ".svg")
    png = os.path.join(HERE, out + ".png")
    fig.savefig(svg)
    fig.savefig(png, dpi=140)
    print(f"wrote {svg}\nwrote {png}")
    plt.close(fig)

    # One standalone graph per benchmark, so each can be linked/shared on
    # its own instead of only as a panel in the combined overview.
    for bname, block in benches:
        bfig, bax = plt.subplots(1, 1, figsize=(9, 3.4))
        bfig.patch.set_facecolor("white")
        bax.set_facecolor("white")
        panel(bax, bname, block)
        legend = [
            Patch(facecolor=ACCENT, label="Tomo"),
            Patch(facecolor=NEUTRAL, label="other languages"),
        ]
        bax.legend(handles=legend, loc="upper right", frameon=False,
                  fontsize=9, borderaxespad=0.6)
        bfig.text(0.02, 0.02,
                 "best of 3 runs · same input · pinned to one core · outputs validated against the C reference",
                 ha="left", fontsize=8, color=MUTED)
        bfig.tight_layout(rect=[0, 0.08, 1, 1])

        bsvg = os.path.join(HERE, f"{out}-{bname}.svg")
        bpng = os.path.join(HERE, f"{out}-{bname}.png")
        bfig.savefig(bsvg)
        bfig.savefig(bpng, dpi=140)
        print(f"wrote {bsvg}\nwrote {bpng}")
        plt.close(bfig)


if __name__ == "__main__":
    main()