mirror of
https://github.com/hanzoai/trolley-bench.git
synced 2026-11-02 13:22:03 +00:00
Zachary Landes
8523fe418c
* Labelled weight hashes correctly: 18 files recorded as sha256 held the Hub's Xet hash, so the README's check failed on the right weights; every weights record now carries the true sha256 and, for Xet-stored files, xet_hash, checked against the Hub paths-info API * Made run/weights.py hash every file itself instead of trusting the cache blob name, and added --check to compare a rerun's meta with the committed one by sha256 * Put DECIDER_MODEL and HF_HUB_OFFLINE=1 into the decider-2b command, since its server reads the model from the environment, and had drive.py record the model-selecting variables as server_env * Shipped scenarios/cells-sample.json (324 stratified v1 cells) with its generator, documented --cells, and said plainly that --limit takes the first cells of one scenario and wording * No answer, count, statistic or page file changed; step 1 of the README still rebuilds every derived file byte for byte
133 lines
5.0 KiB
JSON
133 lines
5.0 KiB
JSON
{
|
|
"model_id": "clm-8b",
|
|
"requests": 1098,
|
|
"answered": 1098,
|
|
"failures": 0,
|
|
"warmup_ms": 1079,
|
|
"median_ms": 622.9,
|
|
"p95_ms": 995.7,
|
|
"model_s": 709.8,
|
|
"wall_s": 770,
|
|
"paused_for_load_s": 60.0,
|
|
"finished": "2026-09-29T02:35:53Z",
|
|
"meta": {
|
|
"model_id": "clm-8b",
|
|
"code": {
|
|
"repo": "https://github.com/Contrastive-LM/CLM",
|
|
"commit": "bb42c6c5bf914fd449bed2f6ca65be80602cb1f7"
|
|
},
|
|
"runtime": "CLM-8B: clm-serve (FastAPI, heads on CPU) over Qwen3-8B last-token embeddings from transformers on MPS in bfloat16 (run/clm_embedder.py, in place of the vLLM pooling server); native Noul, all questions in one call, each scored on its own",
|
|
"server_command": [
|
|
"sh",
|
|
"./run/clm_serve.sh",
|
|
"8015",
|
|
"8091",
|
|
"<scratch>/local-clm/hf/hub/models--Qwen--Qwen3-8B/snapshots/b968826d9c46dd6066d109eabc6255188de91218",
|
|
"<scratch>/local-clm/hf/hub/models--Contrastive-LM--CLM-v0.1-8B/snapshots/e939398d4556fcd9400c76fa8c5a513202f42b0a/CLM_v0.1-8B.pt",
|
|
"<scratch>/local-clm/code/CLM/.venv/bin/python"
|
|
],
|
|
"startup_s_including_download": 6,
|
|
"probe_answer": {
|
|
"model": "clm-latest",
|
|
"answers": {
|
|
"q": {
|
|
"type": "noul",
|
|
"noul": 0.9672293692609533
|
|
}
|
|
},
|
|
"usage": {
|
|
"billing_units": 1,
|
|
"input_tokens": 36,
|
|
"output_tokens": 0
|
|
}
|
|
},
|
|
"weights": [
|
|
{
|
|
"repo": "Contrastive-LM/CLM-v0.1-8B",
|
|
"revision": "e939398d4556fcd9400c76fa8c5a513202f42b0a",
|
|
"bytes": 75557149,
|
|
"files": {
|
|
"CLM_v0.1-8B.pt": {
|
|
"sha256": "b2b4a8c9c2d39263eff78a351eb909a342ce9b3bf21a3f07c1d1bf15f1c4eda5",
|
|
"xet_hash": "646e4318be04845898ccfd1ee08c6847bfe0a2654f6cdbfe86e5543f76e31b1d",
|
|
"bytes": 75557149
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"repo": "Qwen/Qwen3-8B",
|
|
"revision": "b968826d9c46dd6066d109eabc6255188de91218",
|
|
"bytes": 16397461266,
|
|
"files": {
|
|
".gitattributes": {
|
|
"sha256": "34448b82c17d60fec9b65b1f093c115ddbaadc04beb1b0140b6bfed2e012a930",
|
|
"bytes": 1570
|
|
},
|
|
"LICENSE": {
|
|
"sha256": "832dd9e00a68dd83b3c3fb9f5588dad7dcf337a0db50f7d9483f310cd292e92e",
|
|
"bytes": 11343
|
|
},
|
|
"README.md": {
|
|
"sha256": "0f36caaff9c2516411a7738db384606263ba653c1e63e61d72f511606164d5a6",
|
|
"bytes": 16660
|
|
},
|
|
"config.json": {
|
|
"sha256": "f7c4eadfbbf522470667b797a3c89be2524832d2d599797248dc304fff447c30",
|
|
"bytes": 728
|
|
},
|
|
"generation_config.json": {
|
|
"sha256": "2325da0f15bb848e018c5ae071b7943332e9f871d6b60e2ed22ca97d4cb993d2",
|
|
"bytes": 239
|
|
},
|
|
"merges.txt": {
|
|
"sha256": "8831e4f1a044471340f7c0a83d7bd71306a5b867e95fd870f74d0c5308a904d5",
|
|
"bytes": 1671853
|
|
},
|
|
"model-00001-of-00005.safetensors": {
|
|
"sha256": "31d6a825ae35f11fb85b195b4c42c146c051e446433125a215336abdf95cbf5f",
|
|
"xet_hash": "d5896122818183507065a14a89ab73d7c90784031154ad074896049c30f607d7",
|
|
"bytes": 3996250744
|
|
},
|
|
"model-00002-of-00005.safetensors": {
|
|
"sha256": "5991236cea6fe21f3d43cab0f0e84448734fbbe0789816202989f2ddc9d18282",
|
|
"xet_hash": "9a29f7c79ee4a1e5902d54bd8197bd28a4facd1f7b79c71c2321dee7a758e34b",
|
|
"bytes": 3993160032
|
|
},
|
|
"model-00003-of-00005.safetensors": {
|
|
"sha256": "c5185c4794be2d8a9784d5753c9922db38df478ce11f9ed0b415b7304d896836",
|
|
"xet_hash": "eb5ff94ab9f166bee2bde126be1685b655f9857949505ab9ef0f20b522e6f200",
|
|
"bytes": 3959604768
|
|
},
|
|
"model-00004-of-00005.safetensors": {
|
|
"sha256": "b5ee7de71fbf17db3d5704e0c8f2bc7d005ca9e1d7ca2aeb19827b0cfcaa917a",
|
|
"xet_hash": "03e5a8f00169473ee4be2b67cf8ce228bd13ea708b211424c7527ad2a062900c",
|
|
"bytes": 3187841392
|
|
},
|
|
"model-00005-of-00005.safetensors": {
|
|
"sha256": "20c2d6366ab85c90786ccdd829cd2b9e7d30ef3b2ebbb998280e7e4014b542ff",
|
|
"xet_hash": "00fc0074a30b3f49de8ac046752beeb14efe64c145ff83879c04888b6c1a51d8",
|
|
"bytes": 1244659840
|
|
},
|
|
"model.safetensors.index.json": {
|
|
"sha256": "f9fdbcb91c23971c13ec5d5f2573d2349e8f61f2f049371ec699281748fdb1bc",
|
|
"bytes": 32878
|
|
},
|
|
"tokenizer.json": {
|
|
"sha256": "aeb13307a71acd8fe81861d94ad54ab689df773318809eed3cbe794b4492dae4",
|
|
"xet_hash": "6aec39639a0a2d1ca966356b8c2b8426a484f80ff80731f44fa8482040713bdf",
|
|
"bytes": 11422654
|
|
},
|
|
"tokenizer_config.json": {
|
|
"sha256": "d5d09f07b48c3086c508b30d1c9114bd1189145b74e982a265350c923acd8101",
|
|
"bytes": 9732
|
|
},
|
|
"vocab.json": {
|
|
"sha256": "ca10d7e9fb3ed18575dd1e277a2579c16d108e32f27439684afa0e10b1440910",
|
|
"bytes": 2776833
|
|
}
|
|
}
|
|
}
|
|
]
|
|
}
|
|
}
|