mirror of
https://github.com/hanzoai/trolley-bench.git
synced 2026-10-30 15:00:49 +00:00
Zachary Landes
8523fe418c
* Labelled weight hashes correctly: 18 files recorded as sha256 held the Hub's Xet hash, so the README's check failed on the right weights; every weights record now carries the true sha256 and, for Xet-stored files, xet_hash, checked against the Hub paths-info API * Made run/weights.py hash every file itself instead of trusting the cache blob name, and added --check to compare a rerun's meta with the committed one by sha256 * Put DECIDER_MODEL and HF_HUB_OFFLINE=1 into the decider-2b command, since its server reads the model from the environment, and had drive.py record the model-selecting variables as server_env * Shipped scenarios/cells-sample.json (324 stratified v1 cells) with its generator, documented --cells, and said plainly that --limit takes the first cells of one scenario and wording * No answer, count, statistic or page file changed; step 1 of the README still rebuilds every derived file byte for byte
19 lines
1.2 KiB
JavaScript
19 lines
1.2 KiB
JavaScript
// Cut a stratified sample of cells-v1.json into scenarios/cells-sample.json for reduced reruns:
|
|
// every scenario, every wording, both orders, N in {0, 1, 2, 5, 10, 20} (switch also 50 and 100; literal_zero has only N = 0)
|
|
// The cells are copied unchanged, so rows from a sample run compare one for one with the committed v1 rows
|
|
import crypto from "node:crypto";
|
|
import fs from "node:fs";
|
|
import path from "node:path";
|
|
import { fileURLToPath } from "node:url";
|
|
|
|
const HERE = path.dirname(fileURLToPath(import.meta.url));
|
|
const NS = new Set([0, 1, 2, 5, 10, 20]);
|
|
const v1 = JSON.parse(fs.readFileSync(path.join(HERE, "cells-v1.json"), "utf8"));
|
|
const cells = v1.cells.filter((c) => NS.has(c.n) || (c.scenario === "switch" && (c.n === 50 || c.n === 100)));
|
|
const body = JSON.stringify({ prompt_version: v1.prompt_version, sample_of: "cells-v1.json", cells });
|
|
fs.writeFileSync(path.join(HERE, "cells-sample.json"), body + "\n");
|
|
const sha = crypto.createHash("sha256").update(body + "\n").digest("hex");
|
|
const counts = {};
|
|
for (const c of cells) counts[c.scenario] = (counts[c.scenario] ?? 0) + 1;
|
|
console.log(JSON.stringify({ prompt_version: v1.prompt_version, cells: cells.length, answers: cells.length * 3, sha256: sha, counts }));
|