mirror of
https://github.com/hanzoai/benchmarks.git
synced 2026-10-30 14:59:11 +00:00
zeekay
9411e59870
Zach Landes's trolley-bench (github.com/zachlandes/trolley-bench@cde97d7, 5,240 setups) read on kai-1.3, kai-1.4 and kai15a once each on the CPU, Jev and GPT-6 Luna from the repo's recorded answers; the scorer reproduces the article's statistics. Kai fails both checks with a right answer (650 of 672 dominated choices against Jev's 69; 2,036 of 5,240 contradictions against 942), which the suite now says.
42 lines
1.9 KiB
Python
42 lines
1.9 KiB
Python
"""units.py CELLS OUT: the states `bench units` turns into train/fixtures/trolley-units.txt.gz.
|
|
|
|
python3 -I bench/trolley/units.py CELLS states.json.gz
|
|
bench units --states states.json.gz --out train/fixtures/trolley-units.txt.gz
|
|
|
|
CELLS is scenarios/v2/cells-v2.json at the pin (prereg.json). Every cell gives five states: its scene (prose or an
|
|
object, both orders being cells of their own), the request trolley-bench sends it ({state, questions} with all three
|
|
forms), and that request with each form alone (headline, criteria, inverse). `bench units` keeps each one's content units
|
|
(train::data::units: its leaves whitespace-squashed and joined, and the opening of each leaf of 200 characters or more),
|
|
so a record holding a scene, or any logged request of any form, under any key or spacing, is caught exactly; the stage
|
|
barrier's `trolley` set adds the near-duplicate and span checks over the scenes read at the pin.
|
|
"""
|
|
import gzip
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
PREREG = json.load(open(os.path.join(HERE, "prereg.json")))
|
|
|
|
|
|
def main():
|
|
path, out = sys.argv[1:3]
|
|
raw = open(path, "rb").read()
|
|
if hashlib.sha256(raw).hexdigest() != PREREG["source"]["cells_sha256"]:
|
|
sys.exit(f"{path}: not the pinned cells")
|
|
rows = []
|
|
for c in json.loads(raw)["cells"]:
|
|
q = c["questions"]
|
|
rows.append([c["state"], {}, {"id": c["id"], "what": "scene"}])
|
|
rows.append([{"state": c["state"], "questions": q}, {}, {"id": c["id"], "what": "request"}])
|
|
for form in ("pull", "pull_criteria", "stay"):
|
|
rows.append([{"state": c["state"], "questions": {form: q[form]}}, {}, {"id": c["id"], "what": form}])
|
|
with gzip.open(out, "wt") as f:
|
|
json.dump({"trolley": rows}, f)
|
|
print(f"{len(rows)} states from {len(rows) // 5} cells -> {out}", file=sys.stderr)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|