mirror of
https://github.com/hanzoai/benchmarks.git
synced 2026-10-30 14:59:11 +00:00
zeekayandHanzo Dev
b9235d3ca0
Questions are the Laya 0.3.20 builders, unchanged (seed 13): ten application suites of 400, typed decisions (2,000 decisions), and MASSIVE at 51 languages x 100. Every backend is scored by the harness's own metric functions. Laya: typed decisions on hanzoai/kai-1@b50502c2, whose weights have the same SHA-256 as convaiinnovations/laya@55cf4c4e. 0.7660 accuracy, reproducing the published table exactly. Jev: every suite through OpenRouter (typesafe/jev-1.13 served as jev-1.13-20260917), 10,799 of 10,800 answered, $0.25. Laya's full-suite predictions follow when that run completes. Co-authored-by: Hanzo Dev <[email protected]>
32 lines
1.7 KiB
Python
32 lines
1.7 KiB
Python
# Reproduce the upstream typed-decisions benchmark (Part B of research/scripts/bench_local.py)
|
|
# on the Kai checkpoints at their pinned revisions, with the reference runtime on CPU in f32.
|
|
import importlib.util, json, os, sys, time
|
|
from huggingface_hub import hf_hub_download, snapshot_download
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
SRC = os.environ["LAYA_SRC"] # a checkout of github.com/NandhaKishorM/laya at 0.3.20
|
|
BUNDLE, REV = "hanzoai/kai-1", "b50502c28537df49a3621f6fa543f9e8521e8a9c"
|
|
|
|
data_root = os.path.join(HERE, "data")
|
|
p = hf_hub_download("LocalLLaMA/typed-decisions", "all/test-00000-of-00001.parquet",
|
|
repo_type="dataset", local_dir=os.path.join(data_root, "typed-decisions"))
|
|
print("dataset", p, flush=True)
|
|
local = snapshot_download(BUNDLE, revision=REV)
|
|
print("bundle", local, flush=True)
|
|
|
|
spec = importlib.util.spec_from_file_location("bl", os.path.join(SRC, "research/scripts/bench_local.py"))
|
|
bl = importlib.util.module_from_spec(spec); spec.loader.exec_module(bl)
|
|
import laya
|
|
bl.REPO = data_root
|
|
SUB = {"english": None, "multilingual": "multilingual", "typed-decisions": "typed-decisions"}
|
|
def load(name):
|
|
ag = laya.load(local, device="cpu", subfolder=SUB[name]); ag.model.eval(); return ag
|
|
bl.load = load
|
|
results = {"meta": {"bundle": BUNDLE, "revision": REV, "runtime": "laya " + getattr(laya, "__version__", "?"),
|
|
"device": "cpu", "dtype": "f32", "started": time.strftime("%Y-%m-%d %H:%M:%S")}}
|
|
bl.OUT = os.path.join(HERE, "typed_decisions_kai.json")
|
|
bl.run_part_b(results)
|
|
results["meta"]["finished"] = time.strftime("%Y-%m-%d %H:%M:%S")
|
|
json.dump(results, open(bl.OUT, "w"), indent=2)
|
|
print("wrote", bl.OUT, flush=True)
|