papers/empirical-benchmark/emit.py

#!/usr/bin/env python3
"""Write results.tex from a bench-results.txt.

Every number this paper prints comes through here. A figure that is not in
the harness output cannot reach the page, and a figure that changes in the
output cannot be left stale on the page, because main.tex holds no literals.

    ./emit.py ../../bench/bench-results.txt

The previous revision of this paper published a column that had no line in
the output file at all. Generating the tables is what makes that impossible
rather than merely discouraged.
"""
import re
import statistics
import sys

ARMS = ["net/http", "fasthttp", "ZAP-HTTP", "native ZAP", "floor"]
ORDER = ["tiny", "small", "chat", "medium", "large"]
ROWEND = " " + "\\" * 2  # the LaTeX row terminator
LABEL = {
    "tiny": "tiny", "small": "small", "chat": "chat",
    "medium": "medium", "large": "large",
}

MEM = re.compile(
    r"\s+(net/http|fasthttp|ZAP-HTTP|native ZAP|floor)\s+(\d+) reqs in\s+\S+"
    r" \(\s*(\d+) req/s\) \|\s+([\d.]+) bytes/req \|\s+([\d.]+) allocs/req \|\s+(\d+) GCs")
CONC = re.compile(
    r"\s+(net/http|fasthttp|ZAP-HTTP|native ZAP)\s+\d+ reqs in\s+\S+ \(\s*(\d+) req/s\)")
WIRE = re.compile(r"(HTTP/1\.1|ZAP-HTTP|native ZAP) wire:\s+(\d+) bytes")
LOAD = re.compile(r"load averages:\s+([\d.]+)")


def num(x, dp=0):
    """LaTeX-friendly thousands separator."""
    return f"{x:,.{dp}f}".replace(",", r"\,")


# The harness marks the allocation section either way it has spelled it, and
# marks a rate section NOT MEASURED when the machine was too loaded to measure
# one. Both are findable; a run still in flight is not, and is refused.
ALLOC = re.compile(r"^=== allocation[^=\n]*===$", re.M)
WIREH = "=== wire bytes ==="
CONCH = re.compile(r"^=== concurrent throughput([^=\n]*)===$", re.M)
ENDH = "=== environment after ==="


def parse(path):
    txt = open(path).read()
    a = ALLOC.search(txt)
    for what, ok in [("an allocation section", a), (repr(WIREH), WIREH in txt),
                     (repr(ENDH), ENDH in txt)]:
        if not ok:
            sys.exit(f"emit: {path} is not a complete run -- no {what}. "
                     "A run still in flight has no numbers to publish; wait for "
                     "it to finish, or point emit.py at a finished one.")
    out = {"txt": txt, "alloc_header": a.group(0)}

    loads = LOAD.findall(txt)
    out["load"] = (float(loads[0]), float(loads[-1]))

    sec = txt.split(out["alloc_header"])[1].split(WIREH)[0]
    mem, bodies, cur = {}, {}, None
    for line in sec.splitlines():
        m = re.search(r"workload: (\w+) \(body=(\d+) B, headers=(\d+)\)", line)
        if m:
            cur = m.group(1)
            bodies[cur] = (int(m.group(2)), int(m.group(3)))
            continue
        m = MEM.match(line)
        if m and cur:
            mem.setdefault((cur, m.group(1)), []).append(
                (int(m.group(3)), float(m.group(4)), float(m.group(5)), int(m.group(6))))
    out["mem"], out["bodies"] = mem, bodies

    sec = txt.split("=== wire bytes ===")[1].split("--- PASS")[0]
    wire, cur = {}, None
    for line in sec.splitlines():
        m = re.search(r"workload: (\w+) \(", line)
        if m:
            cur = m.group(1); continue
        m = WIRE.search(line)
        if m and cur:
            wire[(cur, m.group(1))] = int(m.group(2))
    out["wire"] = wire

    m = CONCH.search(txt)
    out["conc_measured"] = bool(m) and "NOT MEASURED" not in m.group(1)
    if not out["conc_measured"]:
        out["conc"] = {}
        return out
    sec = txt.split(m.group(0))[1].split("=== per-op benchmarks")[0]
    conc, cur = {}, None
    for line in sec.splitlines():
        m = re.search(r"workload: (\w+) @ (\d+) workers", line)
        if m:
            cur = m.group(1); out["workers"] = int(m.group(2)); continue
        m = CONC.match(line)
        if m and cur:
            conc.setdefault((cur, m.group(1)), []).append(int(m.group(2)))
    out["conc"] = conc
    return out


def finish(lines):
    open("results.tex", "w").write("\n".join(lines) + "\n")
    print("results.tex written")


def main():
    src = sys.argv[1] if len(sys.argv) > 1 else "../../bench/bench-results.txt"
    d = parse(src)
    mem, wire, conc, bodies = d["mem"], d["wire"], d["conc"], d["bodies"]
    med = statistics.median
    sizes = [w for w in ORDER if w in bodies]

    L = []
    L.append("% GENERATED by emit.py from bench-results.txt -- do not hand-edit.")
    L.append("% Regenerate with:  ./emit.py ../../bench/bench-results.txt")
    L.append("%")
    L.append("% The run these numbers come from, copied from its own header so the")
    L.append("% fragment names its provenance even when the harness output is not")
    L.append("% beside it:")
    env = d["txt"].split("=== allocation + throughput")[0].strip().splitlines()
    for line in env:
        L.append("%   " + line)
    L.append("")

    def rows(name, body):
        L.append(r"\newcommand{\%s}{%%" % name)
        L.extend(body)
        L.append("}")
        L.append("")

    # --- bytes allocated per round trip -----------------------------------
    body = []
    for i, w in enumerate(sizes):
        B = bodies[w][0]
        size = f"{B} B" if B < 1024 else f"{B // 1024} KiB"
        cells = []
        for a in ARMS:
            v = med([x[1] for x in mem[(w, a)]])
            cells.append(num(v) if v >= 1000 else f"{v:.1f}")
        cells[2] = r"\textbf{%s}" % cells[2]
        body.append(f"  {LABEL[w]} ({size}) & " + " & ".join(cells) + ROWEND)
    rows("allocBytesRows", body)

    # --- allocations per round trip ---------------------------------------
    body = []
    for i, w in enumerate(sizes):
        cells = [f"{med([x[2] for x in mem[(w, a)]]):.1f}" for a in ARMS]
        body.append(f"  {LABEL[w]} & " + " & ".join(cells) + ROWEND)
    rows("allocCountRows", body)

    # --- GC cycles ---------------------------------------------------------
    body = []
    for i, w in enumerate(sizes):
        cells = [f"{med([x[3] for x in mem[(w, a)]]):.0f}" for a in ARMS]
        body.append(f"  {LABEL[w]} & " + " & ".join(cells) + ROWEND)
    rows("gcRows", body)

    # --- wire bytes --------------------------------------------------------
    body = []
    for i, w in enumerate(sizes):
        B = bodies[w][0]
        size = f"{B} B" if B < 1024 else f"{B // 1024} KiB"
        h = wire[(w, "HTTP/1.1")]
        cells = [num(h)]
        for a in ["ZAP-HTTP", "native ZAP"]:
            v = wire[(w, a)]
            pct = 100.0 * (v - h) / h
            cells.append(f"{num(v)} (${pct:+.1f}\\%$)")
        body.append(f"  {LABEL[w]} ({size}) & " + " & ".join(cells) + ROWEND)
    rows("wireRows", body)

    # --- concurrent throughput --------------------------------------------
    # A run that refused to measure rates prints no table and says so; the
    # section is switched off rather than filled with the last run's numbers.
    L.append(r"\newif\ifconcMeasured")
    L.append(r"\concMeasured" + ("true" if d["conc_measured"] else "false"))
    L.append("")
    cw = [w for w in ORDER if (w, "net/http") in conc]
    body = []
    for i, w in enumerate(cw):
        B = bodies[w][0]
        size = f"{B} B" if B < 1024 else f"{B // 1024} KiB"
        cells = [num(med(conc[(w, a)])) for a in ARMS[:4]]
        body.append(f"  {LABEL[w]} ({size}) & " + " & ".join(cells) + ROWEND)
    rows("concRows", body)

    # --- prose macros ------------------------------------------------------
    def macro(name, value):
        L.append(r"\newcommand{\%s}{%s}" % (name, value))

    macro("loadStart", f"{d['load'][0]:.1f}")
    macro("loadEnd", f"{d['load'][1]:.1f}")
    macro("loadLo", f"{min(d['load']):.0f}")
    macro("loadHi", f"{max(d['load']):.0f}")
    words = {2: "two", 3: "three", 4: "four", 5: "five"}[len(sizes)]
    macro("nWorkloads", words)
    macro("NWorkloads", words.capitalize())
    macro("nReps", str(len(mem[(sizes[0], "net/http")])))
    macro("nWorkers", str(d.get("workers", 32)))
    if conc:
        m = re.search(r"(\d+) reqs in", d["txt"].split(CONCH.search(d["txt"]).group(0))[1])
        macro("concReqs", num(int(m.group(1))))
    macro("memReqs", num(mem and int(re.search(r"(\d+) reqs in", d["txt"]).group(1))))

    # Paired ratios against net/http and fasthttp, over concurrent repetitions.
    # Only defined when the run actually timed something; main.tex guards every
    # use with \ifconcMeasured, so a rate-free run prints prose instead of a
    # number rather than borrowing one from an older run.
    if d["conc_measured"]:
        def bracket(w, arm, base):
            r = [x / y for x, y in zip(conc[(w, arm)], conc[(w, base)])]
            return min(r), max(r)

        lo = min(bracket(w, a, "net/http")[0] for w in cw for a in ["ZAP-HTTP", "native ZAP"])
        hi = max(bracket(w, a, "net/http")[1] for w in cw for a in ["ZAP-HTTP", "native ZAP"])
        macro("zapVsStdLo", f"{lo:.1f}")
        macro("zapVsStdHi", f"{hi:.1f}")
        for w in cw:
            for a, key in [("ZAP-HTTP", "Zap"), ("native ZAP", "Nat")]:
                r = [x / y for x, y in zip(conc[(w, a)], conc[(w, "fasthttp")])]
                macro(f"paired{key}{w.capitalize()}",
                      ", ".join(f"${x:.2f}$" for x in r))

    # the sequential spread the paper quotes, taken from the widest arm
    worst = max(((max(v[0] for v in vals) / max(min(v[0] for v in vals), 1), k)
                 for k, vals in mem.items()), key=lambda t: t[0])
    k = worst[1]
    macro("spreadArm", k[1].replace("/", r"/") if "/" not in k[1] else r"\texttt{%s}" % k[1])
    macro("spreadWorkload", LABEL[k[0]])
    macro("spreadValues", ", ".join(num(v[0]) for v in mem[k]))
    macro("spreadFactor", f"{worst[0]:.1f}")

    # per-arm bytes the prose quotes directly, so the prose cannot drift either
    KEY = {"net/http": "std", "fasthttp": "fast", "ZAP-HTTP": "zap",
           "native ZAP": "nat", "floor": "floor"}
    for w in sizes:
        for a in ARMS:
            v = med([x[1] for x in mem[(w, a)]])
            macro(KEY[a] + w.capitalize(), num(v) if v >= 1000 else f"{v:.1f}")
        macro("allocs" + w.capitalize(),
              f'{med([x[2] for x in mem[(w, "net/http")]]):.1f}')

    # wire-size percentages the prose quotes
    for a, key in [("ZAP-HTTP", "Zap"), ("native ZAP", "Nat")]:
        pcts = [100.0 * (wire[(w, a)] - wire[(w, "HTTP/1.1")]) / wire[(w, "HTTP/1.1")]
                for w in sizes]
        macro(f"wire{key}Lo", f"{min(pcts):+.1f}")
        macro(f"wire{key}Hi", f"{max(pcts):+.1f}")
        if "chat" in sizes:
            macro(f"wire{key}Chat", str(wire[("chat", a)] - wire[("chat", "HTTP/1.1")]))

    # native allocation: 2B + k, and the size-class case
    for w in sizes:
        B = bodies[w][0]
        b = med([x[1] for x in mem[(w, "native ZAP")]])
        macro(f"body{w.capitalize()}", num(B))
        macro(f"natK{w.capitalize()}", f"{b - 2 * B:.0f}")

    finish(L)


if __name__ == "__main__":
    main()