mirror of
https://github.com/hanzoai/benchmarks.git
synced 2026-10-30 14:59:11 +00:00
zeekay
a3c7859112
README.md, SUITE.md, capability/README.md and results/kai-1 compare Kai and Jev and name no other model; the harness's origin is the upstream builders, pinned in harness/three_way.py. merge.py's table.md shows Kai and Jev; its scores.json keeps every backend under the keys and paths hanzo-inc/kai's bench reads, so results/kai-1/scores.json is unchanged byte for byte. results/scores.json and table.md, merge.py's default output, are no longer committed. trained/ measured a baseline the documents no longer show and is removed. The jailbreak preregistration describes its base encoder instead of naming it. Bare python commands are python3, which is what a stock Linux has.
35 lines
1.9 KiB
Python
35 lines
1.9 KiB
Python
"""Typed decisions on the harness items with and without a same-wording training sibling.
|
|
|
|
python3 siblings.py <decision/> <contamination_report.json> name=preds.json.gz[:model] ...
|
|
|
|
|
|
A sibling: a training record the contamination barrier caught as a near duplicate of the item
|
|
(the build's contamination_report.json, `catches` with suite typed_decisions). A checkpoint
|
|
fine-tuned on typed-decisions' train split read those records; Kai's build dropped them.
|
|
"""
|
|
import gzip, json, sys
|
|
H, report = sys.argv[1], sys.argv[2]
|
|
q = json.load(gzip.open(f"{H}/results/questions.json.gz"))["typed_decisions"]
|
|
gold = {f"{i}/{k}": v["idx"] for i, it in enumerate(q) for k, v in it[0].items() if isinstance(v, dict) and "idx" in v}
|
|
catches = json.load(open(report))["catches"]
|
|
sib = {int(c["item"].split("/")[1]) for c in catches if c["source"] == "typed_decisions" and c["item"].startswith("typed_decisions/")}
|
|
print(f"items {len(q)}, with a sibling {len(sib)}, without {len(q) - len(sib)}")
|
|
def probs(path, model=None):
|
|
p = json.load(gzip.open(path))
|
|
if "models" in p: t = p["models"][model]["typed_decisions"]
|
|
else: t = p["suites"]["typed_decisions"]
|
|
return t[1] if isinstance(t, list) else t["p"]
|
|
def acc(pr, keep):
|
|
keys = [k for k in gold if int(k.split("/")[0]) in keep]
|
|
by = {}
|
|
for k in keys:
|
|
qn = k.split("/")[1]; typ = q[int(k.split("/")[0])][1][qn]["type"]
|
|
ok = max(range(len(pr[k])), key=lambda j: pr[k][j]) == gold[k]
|
|
by.setdefault(typ, []).append(ok)
|
|
allv = [x for v in by.values() for x in v]
|
|
return round(sum(allv) / len(allv), 3), {t: round(sum(v) / len(v), 3) for t, v in sorted(by.items())}
|
|
allitems = set(range(len(q))); clean = allitems - sib
|
|
for name, path, model in [(a.split("=")[0], a.split("=")[1].split(":")[0], (a.split("=")[1].split(":") + [None])[1]) for a in sys.argv[3:]]:
|
|
pr = probs(path, model)
|
|
print(f"{name:22s} all {acc(pr, allitems)} without siblings {acc(pr, clean)} with {acc(pr, sib)}")
|