mirror of
https://github.com/hanzoai/trolley-bench.git
synced 2026-10-30 15:00:49 +00:00
zachlandesandGitHub
462aaf404e
* Kept the original raw answers while making later runs reproducible. * Registered a variance-sized sample before the independent rerun. * Removed article drafts and private paths from the current tree.
45 lines
2.6 KiB
Python
45 lines
2.6 KiB
Python
"""Write models.yaml from results/meta/*.json and the run summaries: code commit, weights revision,
|
|
every weight file's SHA-256 (and Xet hash where the Hub stores the file in Xet), runtime and
|
|
precision, per model. Plain YAML, no dependency."""
|
|
import json
|
|
import pathlib
|
|
|
|
ROOT = pathlib.Path(__file__).resolve().parent.parent
|
|
|
|
|
|
def q(s):
|
|
return json.dumps(str(s))
|
|
|
|
|
|
out = ["# Generated by analysis/build_models_yaml.py from results/meta/ and results/raw/*.summary.json", "models:"]
|
|
out += [" - id: jev", " runtime: \"hosted, jev-1.13.0, through decision-gate 0.1.0 (@typesafe-ai/sdk 0.6.0)\"",
|
|
" passes: 3", " note: \"hosted, so only the model version and run date can be pinned\"",
|
|
" - id: gpt-6-luna", " runtime: \"hosted, gpt-6-luna, OpenAI Decisions API, Node fetch\"",
|
|
" passes: 3", " finished: 2026-10-08",
|
|
" note: \"No immutable revision is exposed; the model name and run date are the available pins\""]
|
|
metas = {p.stem: p for p in (ROOT / "results" / "meta").glob("*.json")}
|
|
metas.update({p.stem: p for p in (ROOT / "results" / "v2" / "meta").glob("*.json")})
|
|
for _, meta in sorted(metas.items()):
|
|
m = json.loads(meta.read_text())
|
|
# Reruns on other cell sets (nimble-9b-extra.json) pin the same code and weights as the v1 run
|
|
if meta.stem != m["model_id"]:
|
|
continue
|
|
summ = meta.parent.parent / "raw" / f"{m['model_id']}.summary.json"
|
|
s = json.loads(summ.read_text()) if summ.exists() else {}
|
|
receipt = ROOT / "results" / "v2" / "weights" / f"{m['model_id']}.json"
|
|
if receipt.exists():
|
|
m["weights"] = json.loads(receipt.read_text())["repos"]
|
|
m.pop("extra_files", None)
|
|
out += [f" - id: {m['model_id']}", f" code: {q(m['code']['repo'])}", f" commit: {m['code']['commit']}", f" runtime: {q(m['runtime'])}"]
|
|
if s:
|
|
out += [f" requests: {s.get('requests')}", f" answered: {s.get('answered')}", f" median_ms: {s.get('median_ms')}", f" finished: {s.get('finished')}"]
|
|
out.append(" weights:")
|
|
for w in m.get("weights", []):
|
|
out += [f" - repo: {w['repo']}", f" revision: {w['revision']}", f" bytes: {w['bytes']}", " files:"]
|
|
out += [f" {q(f)}: {{sha256: {v['sha256']}" + (f", xet_hash: {v.get('xet_hash', v.get('pin_is_xet_hash'))}" if v.get('xet_hash', v.get('pin_is_xet_hash')) else "") + "}"
|
|
for f, v in w["files"].items()]
|
|
for extra in m.get("extra_files", []):
|
|
out += [f" - file: {q(extra['file'])}", f" sha256: {extra['sha256']}", f" bytes: {extra['bytes']}"]
|
|
(ROOT / "models.yaml").write_text("\n".join(out) + "\n")
|
|
print("wrote models.yaml")
|