mirror of
https://github.com/hanzoai/laya.git
synced 2026-10-30 19:33:31 +00:00
* fix(router): score reordered-but-equal question schemas separately Grouping by dict equality can merge two requests whose questions are equal but keyed in a different insertion order; the group then scores the second caller with the first caller's option order. Options are positional in render_options, so that request's batched answers would not match its own single-request answer -- the same failure mode as serialising questions to JSON with sort_keys=True and reading them back. Group by an order-sensitive schema signature instead (json.dumps with sort_keys=False and default=str, matching render_criterion's tolerance), with a deterministic fake-agent regression test. * bench: consistency harness + benchmarks for the merged predict_batch Point the harness at the merged Router.predict_batch (#138): * tests/test_predict_batch.py -- tiny-checkpoint tests (CPU): agent-level batch==single, router-level mixed-language routing/order preservation, and the reordered-schema regression at the router layer * benchmarks/bench_predict_batch.py -- --router now speaks the requests-dict API and adds router_consistency(): predict_batch vs one predict per request on 50/50 mixed EN/Hindi, the merge bar on the workload that regroups --consistency also writes its JSON now * plot_results.py renders bench-rtx4070.png from the three result files RTX 4070, laya 0.3.11, torch 2.6.0+cu124: agent layer 1.5-1.8x (<=64 states, 4 questions), router layer up to 3.4x (32 requests, 93 -> 313 states/s); consistency 1800 agent answers + 600 router answers, 0 flips, max|dp| 2.2%.
98 lines
3.5 KiB
Python
98 lines
3.5 KiB
Python
#!/usr/bin/env python
|
|
"""Render benchmarks/bench-rtx4070.png from the three result JSONs.
|
|
|
|
python benchmarks/plot_results.py
|
|
|
|
Panels: (1) agent-level speedup per checkpoint, (2) router-layer mixed EN/Hindi
|
|
states/s, (3) consistency max|dp| per checkpoint with the 2e-2 bar.
|
|
"""
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
import matplotlib
|
|
|
|
matplotlib.use("Agg")
|
|
import matplotlib.pyplot as plt # noqa: E402
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
|
|
|
|
def load(name):
|
|
with open(os.path.join(HERE, name)) as f:
|
|
return json.load(f)
|
|
|
|
|
|
def main():
|
|
agent = load("results-rtx4070.json")
|
|
router = load("results-router-rtx4070.json")
|
|
consist = load("results-consistency-rtx4070.json")
|
|
|
|
fig, axes = plt.subplots(1, 3, figsize=(15, 4.2))
|
|
fig.suptitle("predict_batch on RTX 4070 — laya %s" % os.environ.get("LAYA_VER", "0.3.11"),
|
|
fontsize=13, fontweight="bold")
|
|
|
|
# panel 1: agent-level speedup
|
|
ax = axes[0]
|
|
for row in agent["bench"]:
|
|
by_states = {}
|
|
per_ckpt = {}
|
|
for row in agent["bench"]:
|
|
per_ckpt.setdefault(row["checkpoint"], []).append(row)
|
|
for ckpt, rows in per_ckpt.items():
|
|
rows.sort(key=lambda r: r["states"])
|
|
ax.plot([r["states"] for r in rows], [r["speedup"] for r in rows],
|
|
marker="o", label=ckpt)
|
|
ax.set_xscale("log", base=2)
|
|
ax.set_yscale("log", base=2)
|
|
ax.axhline(1.0, color="gray", lw=0.8, ls="--")
|
|
ax.set_xlabel("states per batch")
|
|
ax.set_ylabel("speedup (x)")
|
|
ax.set_title("Agent.predict_batch vs one-by-one\n(4 questions/state, median of 5)")
|
|
ax.legend(fontsize=8)
|
|
ax.grid(alpha=0.3)
|
|
|
|
# panel 2: router mixed-language throughput
|
|
ax = axes[1]
|
|
rows = sorted(router["router"], key=lambda r: r["states"])
|
|
xs = [r["states"] for r in rows]
|
|
ax.plot(xs, [r["looped_sps"] for r in rows], marker="o", label="one predict() per request")
|
|
ax.plot(xs, [r["batched_sps"] for r in rows], marker="o", label="Router.predict_batch")
|
|
for r in rows:
|
|
ax.annotate("%.1fx" % r["speedup"], (r["states"], r["batched_sps"]),
|
|
textcoords="offset points", xytext=(0, 7), fontsize=7, ha="center")
|
|
ax.set_xscale("log", base=2)
|
|
ax.set_xlabel("requests (50/50 EN/Hindi)")
|
|
ax.set_ylabel("states/s")
|
|
ax.set_title("Router layer: mixed EN/Hindi\n(regroups by checkpoint)")
|
|
ax.legend(fontsize=8)
|
|
ax.grid(alpha=0.3)
|
|
|
|
# panel 3: consistency
|
|
ax = axes[2]
|
|
rc = router.get("router_consistency", {})
|
|
entries = [(r["checkpoint"], r["max_delta"]) for r in consist["consistency"]]
|
|
if rc:
|
|
entries.append(("router\n(mixed EN/Hindi)", rc["max_delta"]))
|
|
names = [e[0] for e in entries]
|
|
deltas = [e[1] for e in entries]
|
|
bars = ax.bar(names, deltas, color=["#4c72b0", "#dd8452", "#55a868", "#c44e52"][:len(names)])
|
|
ax.axhline(0.02, color="red", ls="--", lw=1, label="2e-2 bar")
|
|
ax.set_ylabel("max |Δp| (label-attached)")
|
|
ax.set_title("batch vs single consistency\n(%d answers, 0 flips)" %
|
|
(sum(r["compared"] for r in consist["consistency"]) + rc.get("compared", 0)))
|
|
ax.legend(fontsize=8)
|
|
ax.grid(alpha=0.3, axis="y")
|
|
for bar, delta in zip(bars, deltas):
|
|
ax.text(bar.get_x() + bar.get_width() / 2, delta, "%.4f" % delta,
|
|
ha="center", va="bottom", fontsize=7)
|
|
|
|
plt.tight_layout(rect=[0, 0, 1, 0.94])
|
|
out = os.path.join(HERE, "bench-rtx4070.png")
|
|
plt.savefig(out, dpi=110)
|
|
print("wrote %s" % out)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|