benchmarks/decision/capability/results/invariance.json

{"meta":{"run 2026-09-27 09:27:04 +0000":{"host":"arm64-mac","machine":"aarch64","runtime":"bench (hanzo-ml)","load":[44.35400390625,36.0966796875,29.4619140625],"at":"2026-09-27 09:27:04 +0000","laya":{"runtime":"bench (hanzo-ml, kai::joint)","checkpoints":[{"id":"laya","repo":"hanzoai/kai-1","revision":"b50502c28537df49a3621f6fa543f9e8521e8a9c"},{"id":"laya-multilingual","repo":"hanzoai/kai-1-multilingual","revision":"3119843b5261e34903f93d03813467524c57982e"},{"id":"laya-agent","repo":"hanzoai/kai-1-agent","revision":"e3e416499b8a983d2aa0bcc684d5457ad6bfd6a1"}],"device":"metal","dtype":"f32"},"cases":{"perm":200,"orders":5,"swap":400,"alias":200}},"run 2026-09-27 09:52:25 +0000":{"host":"arm64-mac","machine":"aarch64","runtime":"bench (hanzo-ml)","load":[14.58251953125,18.81201171875,23.3603515625],"at":"2026-09-27 09:52:25 +0000","jev":{"model":"typesafe/jev-1.13","url":"https://openrouter.ai/api/alpha/decisions","price_per_input_token":4.2e-08},"cases":{"perm":200,"orders":5,"swap":400,"alias":200}},"run 2026-09-27 09:53:49 +0000":{"host":"arm64-mac","machine":"aarch64","runtime":"bench (hanzo-ml)","load":[15.64892578125,18.13671875,22.66650390625],"at":"2026-09-27 09:53:49 +0000","cases":{"perm":200,"orders":5,"swap":400,"alias":200}},"run 2026-09-27 14:42:48 +0000":{"host":"arm64-mac","machine":"aarch64","runtime":"bench (hanzo-ml)","load":[6.32958984375,8.21337890625,9.806640625],"at":"2026-09-27 14:42:48 +0000","cases":{"perm":200,"orders":5,"swap":400,"alias":200}},"run 2026-09-28 03:23:23 +0000":{"host":"arm64-host","machine":"aarch64","runtime":"bench (hanzo-ml)","load":[2.5615234375,2.5283203125,2.52880859375],"at":"2026-09-28 03:23:23 +0000","cases":{"perm":200,"orders":5,"swap":400,"alias":200}}},"keys":{"cap/invariance/jev/alias.code.acc":0.855,"cap/invariance/jev/alias.code.flip":0.035,"cap/invariance/jev/alias.snake.acc":0.845,"cap/invariance/jev/alias.snake.flip":0.01,"cap/invariance/jev/alias.text.acc":0.85,"cap/invariance/jev/perm.acc.max":0.87,"cap/invariance/jev/perm.acc.min":0.845,"cap/invariance/jev/perm.compared":200,"cap/invariance/jev/perm.flip":0.06,"cap/invariance/jev/perm.maxdp":0.47,"cap/invariance/jev/perm.tv":0.0332,"cap/invariance/jev/swap.acc":0.82,"cap/invariance/jev/swap.desc.acc":0.1925,"cap/invariance/jev/swap.desc.compared":400,"cap/invariance/jev/swap.desc.gap":0.5522,"cap/invariance/jev/swap.desc.maxgap":0.98,"cap/invariance/jev/swap.desc.mirror":0.0625,"cap/invariance/jev/swap.neg.acc":0.8325,"cap/invariance/jev/swap.neg.compared":400,"cap/invariance/jev/swap.neg.gap":0.0928,"cap/invariance/jev/swap.neg.maxgap":0.49,"cap/invariance/jev/swap.neg.mirror":0.8825,"cap/invariance/jev/unanswered":0,"cap/invariance/jev/usd":0.0666,"cap/invariance/laya/alias.code.acc":0.65,"cap/invariance/laya/alias.code.flip":0.205,"cap/invariance/laya/alias.snake.acc":0.61,"cap/invariance/laya/alias.snake.flip":0.21,"cap/invariance/laya/alias.text.acc":0.625,"cap/invariance/laya/perm.acc.max":0.65,"cap/invariance/laya/perm.acc.min":0.62,"cap/invariance/laya/perm.compared":200,"cap/invariance/laya/perm.flip":0.255,"cap/invariance/laya/perm.maxdp":0.996,"cap/invariance/laya/perm.tv":0.1445,"cap/invariance/laya/swap.acc":0.815,"cap/invariance/laya/swap.desc.acc":0.31,"cap/invariance/laya/swap.desc.compared":400,"cap/invariance/laya/swap.desc.gap":0.5103,"cap/invariance/laya/swap.desc.maxgap":1.0,"cap/invariance/laya/swap.desc.mirror":0.205,"cap/invariance/laya/swap.neg.acc":0.41,"cap/invariance/laya/swap.neg.compared":400,"cap/invariance/laya/swap.neg.gap":0.4644,"cap/invariance/laya/swap.neg.maxgap":1.0,"cap/invariance/laya/swap.neg.mirror":0.3,"cap/invariance/laya/unanswered":0},"detail":{"laya":{"perm_acc":{"perm0":0.625,"perm1":0.625,"perm2":0.63,"perm3":0.65,"perm4":0.62},"unanswered":0},"jev":{"perm_acc":{"perm0":0.845,"perm1":0.855,"perm2":0.87,"perm3":0.85,"perm4":0.85},"unanswered":0,"jev":{"alias.code":{"cost_usd":0.004756,"input_tokens":113227,"n_errors":0,"error_kinds":{}},"alias.snake":{"cost_usd":0.005036,"input_tokens":119904,"n_errors":0,"error_kinds":{}},"alias.text":{"cost_usd":0.004811,"input_tokens":114548,"n_errors":0,"error_kinds":{}},"perm0":{"cost_usd":0.004811,"input_tokens":114548,"n_errors":0,"error_kinds":{}},"perm1":{"cost_usd":0.004811,"input_tokens":114548,"n_errors":0,"error_kinds":{}},"perm2":{"cost_usd":0.004811,"input_tokens":114548,"n_errors":0,"error_kinds":{}},"perm3":{"cost_usd":0.004811,"input_tokens":114548,"n_errors":0,"error_kinds":{}},"perm4":{"cost_usd":0.004811,"input_tokens":114548,"n_errors":0,"error_kinds":{}},"swap.desc":{"cost_usd":0.009279,"input_tokens":220925,"n_errors":0,"error_kinds":{}},"swap.neg":{"cost_usd":0.009338,"input_tokens":222342,"n_errors":0,"error_kinds":{}},"swap0":{"cost_usd":0.009279,"input_tokens":220925,"n_errors":0,"error_kinds":{}}}}}}