benchmarks/decision/capability/results/cardinality.json at main - benchmarks - Hanzo Git

hanzoai/benchmarks

mirror of https://github.com/hanzoai/benchmarks.git synced 2026-10-30 14:59:11 +00:00

Files

T

zeekay 1b9ddf36dd decision/capability: kai-1 on the cardinality suite, read again in Python

cardinality.json carries kai-1's keys (weights 0834a74f, bench cap on
CUDA in F32, 2026-10-03) beside Laya's and Jev's, unchanged, and
results/cardinality/kai.* its answers and summaries at every size. Every
option scored: 4 options 96.75%, 16 93.50%, 77 90.75%, 150 90.75%,
against Jev's 90.50%, 85.25%, 84.25% and 94.00%.

cardinality.py builds the cases at 4, 16, 77 and 150 options as bench
cap does, asks Kai over POST /v1/decisions, and prints its accuracy beside
the recorded kai-1 and Jev. --check rebuilds the cases and reproduces
both records question by question. Jev is never called.

2026-10-15 10:43:22 -07:00

1 line

50 KiB

JSON

Raw Permalink Blame History

{"meta":{"run 2026-09-27 09:55:22 +0000":{"host":"arm64-mac","machine":"aarch64","runtime":"bench (hanzo-ml)","load":[15.0751953125,17.3525390625,21.90869140625],"at":"2026-09-27 09:55:22 +0000","jev":{"model":"typesafe/jev-1.13","url":"https://openrouter.ai/api/alpha/decisions","price_per_input_token":4.2e-08},"bi":["sentence-transformers/all-MiniLM-L6-v2","1110a243fdf4706b3f48f1d95db1a4f5529b4d41"],"cases":{"k4":{"n":400,"k":4},"k16":{"n":400,"k":16},"k77":{"n":400,"k":77},"k150":{"n":400,"k":150},"k1000":{"n":200,"k":1000},"k10000":{"n":200,"k":10000},"k100000":{"n":50,"k":100000}}},"run 2026-09-27 10:05:18 +0000":{"host":"arm64-mac","machine":"aarch64","runtime":"bench (hanzo-ml)","load":[12.3349609375,14.0439453125,17.96337890625],"at":"2026-09-27 10:05:18 +0000","jev":{"model":"typesafe/jev-1.13","url":"https://openrouter.ai/api/alpha/decisions","price_per_input_token":4.2e-08},"bi":["sentence-transformers/all-MiniLM-L6-v2","1110a243fdf4706b3f48f1d95db1a4f5529b4d41"],"cases":{"k4":{"n":400,"k":4},"k16":{"n":400,"k":16},"k77":{"n":400,"k":77},"k150":{"n":400,"k":150},"k1000":{"n":200,"k":1000},"k10000":{"n":200,"k":10000},"k100000":{"n":50,"k":100000}}},"run 2026-09-27 10:40:42 +0000":{"host":"arm64-mac","machine":"aarch64","runtime":"bench (hanzo-ml)","load":[33.05126953125,24.724609375,22.49462890625],"at":"2026-09-27 10:40:42 +0000","bi":["sentence-transformers/all-MiniLM-L6-v2","1110a243fdf4706b3f48f1d95db1a4f5529b4d41"],"cases":{"k4":{"n":400,"k":4},"k16":{"n":400,"k":16},"k77":{"n":400,"k":77},"k150":{"n":400,"k":150},"k1000":{"n":200,"k":1000},"k10000":{"n":200,"k":10000},"k100000":{"n":50,"k":100000}}},"run 2026-09-27 10:44:19 +0000":{"host":"arm64-mac","machine":"aarch64","runtime":"bench (hanzo-ml)","load":[24.7041015625,23.56787109375,22.46044921875],"at":"2026-09-27 10:44:19 +0000","laya":{"runtime":"bench (hanzo-ml, kai::joint)","checkpoints":[{"id":"laya","repo":"hanzoai/kai-1","revision":"b50502c28537df49a3621f6fa543f9e8521e8a9c"},{"id":"laya-multilingual","repo":"hanzoai/kai-1-multilingual","revision":"3119843b5261e34903f93d03813467524c57982e"},{"id":"laya-agent","repo":"hanzoai/kai-1-agent","revision":"e3e416499b8a983d2aa0bcc684d5457ad6bfd6a1"}],"device":"metal","dtype":"f32"},"bi":["sentence-transformers/all-MiniLM-L6-v2","1110a243fdf4706b3f48f1d95db1a4f5529b4d41"],"cases":{"k4":{"n":400,"k":4},"k16":{"n":400,"k":16},"k77":{"n":400,"k":77},"k150":{"n":400,"k":150},"k1000":{"n":200,"k":1000},"k10000":{"n":200,"k":10000},"k100000":{"n":50,"k":100000}}},"run 2026-09-27 10:47:57 +0000":{"host":"arm64-mac","machine":"aarch64","runtime":"bench (hanzo-ml)","load":[23.3974609375,24.2861328125,23.07666015625],"at":"2026-09-27 10:47:57 +0000","laya":{"runtime":"bench (hanzo-ml, kai::joint)","checkpoints":[{"id":"laya","repo":"hanzoai/kai-1","revision":"b50502c28537df49a3621f6fa543f9e8521e8a9c"},{"id":"laya-multilingual","repo":"hanzoai/kai-1-multilingual","revision":"3119843b5261e34903f93d03813467524c57982e"},{"id":"laya-agent","repo":"hanzoai/kai-1-agent","revision":"e3e416499b8a983d2aa0bcc684d5457ad6bfd6a1"}],"device":"metal","dtype":"f32"},"bi":["sentence-transformers/all-MiniLM-L6-v2","1110a243fdf4706b3f48f1d95db1a4f5529b4d41"],"cases":{"k4":{"n":400,"k":4},"k16":{"n":400,"k":16},"k77":{"n":400,"k":77},"k150":{"n":400,"k":150},"k1000":{"n":200,"k":1000},"k10000":{"n":200,"k":10000},"k100000":{"n":50,"k":100000}}},"run 2026-09-27 10:48:23 +0000":{"host":"arm64-mac","machine":"aarch64","runtime":"bench (hanzo-ml)","load":[20.61572265625,23.58251953125,22.8583984375],"at":"2026-09-27 10:48:23 +0000","bi":["sentence-transformers/all-MiniLM-L6-v2","1110a243fdf4706b3f48f1d95db1a4f5529b4d41"],"cases":{"k4":{"n":400,"k":4},"k16":{"n":400,"k":16},"k77":{"n":400,"k":77},"k150":{"n":400,"k":150},"k1000":{"n":200,"k":1000},"k10000":{"n":200,"k":10000},"k100000":{"n":50,"k":100000}}},"run 2026-09-27 11:14:43 +0000":{"host":"arm64-mac","machine":"aarch64","runtime":"bench (hanzo-ml)","load":[22.642578125,16.81787109375,17.13427734375],"at":"2026-09-27 11:14:43 +0000","bi":["sentence-transformers/all-MiniLM-L6-v2","1110a243fdf4706b3f48f1d95db1a4f5529b4d41"],"cases":{"k4":{"n":400,"k":4},"k16":{"n":400,"k":16},"k77":{"n":400,"k":77},"k150":{"n":400,"k":150},"k1000":{"n":200,"k":1000},"k10000":{"n":200,"k":10000},"k100000":{"n":50,"k":100000}}},"run 2026-09-27 14:38:23 +0000":{"host":"arm64-mac","machine":"aarch64","runtime":"bench (hanzo-ml)","load":[8.125,9.82958984375,10.75390625],"at":"2026-09-27 14:38:23 +0000","bi":["sentence-transformers/all-MiniLM-L6-v2","1110a243fdf4706b3f48f1d95db1a4f5529b4d41"],"cases":{"k4":{"n":400,"k":4},"k16":{"n":400,"k":16},"k77":{"n":400,"k":77},"k150":{"n":400,"k":150},"k1000":{"n":200,"k":1000},"k10000":{"n":200,"k":10000},"k100000":{"n":50,"k":100000}}},"run 2026-10-03 00:11:42 +0000":{"host":"arm64-host","machine":"aarch64","runtime":"bench (hanzo-ml)","load":[5.072265625,32.09814453125,32.53125],"at":"2026-10-03 00:11:42 +0000","kai":{"checkpoint":"kai-1","weights":"0834a74f2d140642a453373da09e5a128e3d2c8d9e8bb1dcaf86d7210a4ecdfc","device":"cuda","dtype":"f32","zeroed":[]},"bi":["sentence-transformers/all-MiniLM-L6-v2","1110a243fdf4706b3f48f1d95db1a4f5529b4d41"],"cases":{"k4":{"n":400,"k":4},"k16":{"n":400,"k":16},"k77":{"n":400,"k":77},"k150":{"n":400,"k":150},"k1000":{"n":200,"k":1000},"k10000":{"n":200,"k":10000}}},"run 2026-10-03 00:20:27 +0000":{"host":"arm64-host","machine":"aarch64","runtime":"bench (hanzo-ml)","load":[2.345703125,28.7958984375,35.15283203125],"at":"2026-10-03 00:20:27 +0000","kai":{"checkpoint":"kai-1","weights":"0834a74f2d140642a453373da09e5a128e3d2c8d9e8bb1dcaf86d7210a4ecdfc","device":"cuda","dtype":"f32","zeroed":[]},"bi":["sentence-transformers/all-MiniLM-L6-v2","1110a243fdf4706b3f48f1d95db1a4f5529b4d41"],"cases":{"k100000":{"n":50,"k":100000}}}},"keys":{"cap/cardinality/all/k1000.bi.casems":22.38,"cap/cardinality/all/k1000.bi.hit":0.655,"cap/cardinality/all/k1000.bi.indexs":0.835,"cap/cardinality/all/k1000.sl.casems":48.575,"cap/cardinality/all/k1000.sl.hit":0.04,"cap/cardinality/all/k1000.sl.indexs":5.095,"cap/cardinality/all/k10000.bi.casems":12.68,"cap/cardinality/all/k10000.bi.hit":0.405,"cap/cardinality/all/k10000.bi.indexs":3.722,"cap/cardinality/all/k10000.sl.casems":28.315,"cap/cardinality/all/k10000.sl.hit":0.015,"cap/cardinality/all/k10000.sl.indexs":30.554,"cap/cardinality/all/k100000.bi.casems":137.355,"cap/cardinality/all/k100000.bi.hit":0.26,"cap/cardinality/all/k100000.bi.indexs":27.155,"cap/cardinality/all/k100000.sl.casems":344.16,"cap/cardinality/all/k100000.sl.hit":0.02,"cap/cardinality/all/k100000.sl.indexs":392.47,"cap/cardinality/all/k150.bi.casems":21.11,"cap/cardinality/all/k150.bi.hit":0.8575,"cap/cardinality/all/k150.bi.indexs":0.059,"cap/cardinality/all/k150.sl.casems":44.19,"cap/cardinality/all/k150.sl.hit":0.2,"cap/cardinality/all/k150.sl.indexs":0.378,"cap/cardinality/all/k77.bi.casems":21.34,"cap/cardinality/all/k77.bi.hit":0.965,"cap/cardinality/all/k77.bi.indexs":0.132,"cap/cardinality/all/k77.sl.casems":44.395,"cap/cardinality/all/k77.sl.hit":0.08,"cap/cardinality/all/k77.sl.indexs":0.682,"cap/cardinality/jev/k1000.accall":0.0,"cap/cardinality/jev/k1000.bi.accall":0.64,"cap/cardinality/jev/k1000.bi.accuracy":0.64,"cap/cardinality/jev/k1000.bi.brier":0.5481,"cap/cardinality/jev/k1000.bi.ece":0.2198,"cap/cardinality/jev/k1000.bi.nll":9.5674,"cap/cardinality/jev/k1000.bi.p50ms":156.5148,"cap/cardinality/jev/k1000.bi.p95ms":242.7391,"cap/cardinality/jev/k1000.bi.recall1":0.64,"cap/cardinality/jev/k1000.bi.recall20":0.655,"cap/cardinality/jev/k1000.bi.recall5":0.655,"cap/cardinality/jev/k1000.bi.rejected":0.0,"cap/cardinality/jev/k1000.bi.unanswered":0,"cap/cardinality/jev/k1000.bi.usd":0.0071,"cap/cardinality/jev/k1000.rejected":1.0,"cap/cardinality/jev/k1000.sl.accall":0.035,"cap/cardinality/jev/k1000.sl.accuracy":0.035,"cap/cardinality/jev/k1000.sl.brier":1.4346,"cap/cardinality/jev/k1000.sl.ece":0.5853,"cap/cardinality/jev/k1000.sl.nll":26.5488,"cap/cardinality/jev/k1000.sl.p50ms":135.0096,"cap/cardinality/jev/k1000.sl.p95ms":186.7453,"cap/cardinality/jev/k1000.sl.recall1":0.035,"cap/cardinality/jev/k1000.sl.recall20":0.04,"cap/cardinality/jev/k1000.sl.recall5":0.04,"cap/cardinality/jev/k1000.sl.rejected":0.0,"cap/cardinality/jev/k1000.sl.unanswered":0,"cap/cardinality/jev/k1000.sl.usd":0.0068,"cap/cardinality/jev/k1000.unanswered":200,"cap/cardinality/jev/k1000.usd":0.0,"cap/cardinality/jev/k10000.accall":0.0,"cap/cardinality/jev/k10000.bi.accall":0.4,"cap/cardinality/jev/k10000.bi.accuracy":0.4,"cap/cardinality/jev/k10000.bi.brier":0.9115,"cap/cardinality/jev/k10000.bi.ece":0.3643,"cap/cardinality/jev/k10000.bi.nll":16.4607,"cap/cardinality/jev/k10000.bi.p50ms":148.3717,"cap/cardinality/jev/k10000.bi.p95ms":219.5043,"cap/cardinality/jev/k10000.bi.recall1":0.4,"cap/cardinality/jev/k10000.bi.recall20":0.405,"cap/cardinality/jev/k10000.bi.recall5":0.405,"cap/cardinality/jev/k10000.bi.rejected":0.0,"cap/cardinality/jev/k10000.bi.unanswered":0,"cap/cardinality/jev/k10000.bi.usd":0.007,"cap/cardinality/jev/k10000.rejected":1.0,"cap/cardinality/jev/k10000.sl.accall":0.015,"cap/cardinality/jev/k10000.sl.accuracy":0.015,"cap/cardinality/jev/k10000.sl.brier":1.4585,"cap/cardinality/jev/k10000.sl.ece":0.5989,"cap/cardinality/jev/k10000.sl.nll":27.2166,"cap/cardinality/jev/k10000.sl.p50ms":136.1146,"cap/cardinality/jev/k10000.sl.p95ms":219.06,"cap/cardinality/jev/k10000.sl.recall1":0.015,"cap/cardinality/jev/k10000.sl.recall20":0.015,"cap/cardinality/jev/k10000.sl.recall5":0.015,"cap/cardinality/jev/k10000.sl.rejected":0.0,"cap/cardinality/jev/k10000.sl.unanswered":0,"cap/cardinality/jev/k10000.sl.usd":0.0069,"cap/cardinality/jev/k10000.unanswered":200,"cap/cardinality/jev/k10000.usd":0.0,"cap/cardinality/jev/k100000.accall":0.0,"cap/cardinality/jev/k100000.bi.accall":0.26,"cap/cardinality/jev/k100000.bi.accuracy":0.26,"cap/cardinality/jev/k100000.bi.brier":1.0509,"cap/cardinality/jev/k100000.bi.ece":0.3938,"cap/cardinality/jev/k100000.bi.nll":20.4628,"cap/cardinality/jev/k100000.bi.p50ms":151.0528,"cap/cardinality/jev/k100000.bi.p95ms":266.1641,"cap/cardinality/jev/k100000.bi.recall1":0.26,"cap/cardinality/jev/k100000.bi.recall20":0.26,"cap/cardinality/jev/k100000.bi.recall5":0.26,"cap/cardinality/jev/k100000.bi.rejected":0.0,"cap/cardinality/jev/k100000.bi.unanswered":0,"cap/cardinality/jev/k100000.bi.usd":0.0017,"cap/cardinality/jev/k100000.rejected":1.0,"cap/cardinality/jev/k100000.sl.accall":0.02,"cap/cardinality/jev/k100000.sl.accuracy":0.02,"cap/cardinality/jev/k100000.sl.brier":1.4745,"cap/cardinality/jev/k100000.sl.ece":0.6256,"cap/cardinality/jev/k100000.sl.nll":27.0784,"cap/cardinality/jev/k100000.sl.p50ms":148.1115,"cap/cardinality/jev/k100000.sl.p95ms":235.1408,"cap/cardinality/jev/k100000.sl.recall1":0.02,"cap/cardinality/jev/k100000.sl.recall20":0.02,"cap/cardinality/jev/k100000.sl.recall5":0.02,"cap/cardinality/jev/k100000.sl.rejected":0.0,"cap/cardinality/jev/k100000.sl.unanswered":0,"cap/cardinality/jev/k100000.sl.usd":0.0017,"cap/cardinality/jev/k100000.unanswered":50,"cap/cardinality/jev/k100000.usd":0.0,"cap/cardinality/jev/k150.accall":0.94,"cap/cardinality/jev/k150.accuracy":0.94,"cap/cardinality/jev/k150.bi.accall":0.825,"cap/cardinality/jev/k150.bi.accuracy":0.825,"cap/cardinality/jev/k150.bi.brier":0.2934,"cap/cardinality/jev/k150.bi.ece":0.1044,"cap/cardinality/jev/k150.bi.nll":4.1037,"cap/cardinality/jev/k150.bi.p50ms":129.7024,"cap/cardinality/jev/k150.bi.p95ms":218.856,"cap/cardinality/jev/k150.bi.recall1":0.825,"cap/cardinality/jev/k150.bi.recall20":0.855,"cap/cardinality/jev/k150.bi.recall5":0.855,"cap/cardinality/jev/k150.bi.rejected":0.0,"cap/cardinality/jev/k150.bi.unanswered":0,"cap/cardinality/jev/k150.bi.usd":0.0097,"cap/cardinality/jev/k150.brier":0.1074,"cap/cardinality/jev/k150.ece":0.0317,"cap/cardinality/jev/k150.nll":0.4346,"cap/cardinality/jev/k150.p50ms":137.9604,"cap/cardinality/jev/k150.p95ms":194.0808,"cap/cardinality/jev/k150.recall1":0.9375,"cap/cardinality/jev/k150.recall20":0.99,"cap/cardinality/jev/k150.recall5":0.99,"cap/cardinality/jev/k150.rejected":0.0,"cap/cardinality/jev/k150.sl.accall":0.1925,"cap/cardinality/jev/k150.sl.accuracy":0.1925,"cap/cardinality/jev/k150.sl.brier":1.2677,"cap/cardinality/jev/k150.sl.ece":0.5504,"cap/cardinality/jev/k150.sl.nll":22.189,"cap/cardinality/jev/k150.sl.p50ms":136.4653,"cap/cardinality/jev/k150.sl.p95ms":212.9873,"cap/cardinality/jev/k150.sl.recall1":0.1925,"cap/cardinality/jev/k150.sl.recall20":0.1975,"cap/cardinality/jev/k150.sl.recall5":0.1975,"cap/cardinality/jev/k150.sl.rejected":0.0,"cap/cardinality/jev/k150.sl.unanswered":0,"cap/cardinality/jev/k150.sl.usd":0.0102,"cap/cardinality/jev/k150.unanswered":0,"cap/cardinality/jev/k150.usd":0.041,"cap/cardinality/jev/k16.accall":0.8525,"cap/cardinality/jev/k16.accuracy":0.8525,"cap/cardinality/jev/k16.brier":0.2212,"cap/cardinality/jev/k16.ece":0.067,"cap/cardinality/jev/k16.nll":1.1469,"cap/cardinality/jev/k16.p50ms":134.6807,"cap/cardinality/jev/k16.p95ms":208.5871,"cap/cardinality/jev/k16.recall1":0.8525,"cap/cardinality/jev/k16.recall20":1.0,"cap/cardinality/jev/k16.recall5":0.965,"cap/cardinality/jev/k16.rejected":0.0,"cap/cardinality/jev/k16.unanswered":0,"cap/cardinality/jev/k16.usd":0.0097,"cap/cardinality/jev/k4.accall":0.905,"cap/cardinality/jev/k4.accuracy":0.905,"cap/cardinality/jev/k4.brier":0.1402,"cap/cardinality/jev/k4.ece":0.0465,"cap/cardinality/jev/k4.nll":0.4411,"cap/cardinality/jev/k4.p50ms":135.1242,"cap/cardinality/jev/k4.p95ms":295.1202,"cap/cardinality/jev/k4.recall1":0.905,"cap/cardinality/jev/k4.recall20":1.0,"cap/cardinality/jev/k4.recall5":1.0,"cap/cardinality/jev/k4.rejected":0.0,"cap/cardinality/jev/k4.unanswered":0,"cap/cardinality/jev/k4.usd":0.0062,"cap/cardinality/jev/k77.accall":0.8425,"cap/cardinality/jev/k77.accuracy":0.8425,"cap/cardinality/jev/k77.bi.accall":0.8075,"cap/cardinality/jev/k77.bi.accuracy":0.8075,"cap/cardinality/jev/k77.bi.brier":0.2997,"cap/cardinality/jev/k77.bi.ece":0.0736,"cap/cardinality/jev/k77.bi.nll":2.2036,"cap/cardinality/jev/k77.bi.p50ms":136.1763,"cap/cardinality/jev/k77.bi.p95ms":208.5744,"cap/cardinality/jev/k77.bi.recall1":0.805,"cap/cardinality/jev/k77.bi.recall20":0.9325,"cap/cardinality/jev/k77.bi.recall5":0.93,"cap/cardinality/jev/k77.bi.rejected":0.0,"cap/cardinality/jev/k77.bi.unanswered":0,"cap/cardinality/jev/k77.bi.usd":0.0109,"cap/cardinality/jev/k77.brier":0.2574,"cap/cardinality/jev/k77.ece":0.0682,"cap/cardinality/jev/k77.nll":1.2902,"cap/cardinality/jev/k77.p50ms":135.1938,"cap/cardinality/jev/k77.p95ms":200.8454,"cap/cardinality/jev/k77.recall1":0.8425,"cap/cardinality/jev/k77.recall20":0.9675,"cap/cardinality/jev/k77.recall5":0.96,"cap/cardinality/jev/k77.rejected":0.0,"cap/cardinality/jev/k77.sl.accall":0.0775,"cap/cardinality/jev/k77.sl.accuracy":0.0775,"cap/cardinality/jev/k77.sl.brier":1.4965,"cap/cardinality/jev/k77.sl.ece":0.6704,"cap/cardinality/jev/k77.sl.nll":25.4291,"cap/cardinality/jev/k77.sl.p50ms":137.1797,"cap/cardinality/jev/k77.sl.p95ms":341.8403,"cap/cardinality/jev/k77.sl.recall1":0.0775,"cap/cardinality/jev/k77.sl.recall20":0.08,"cap/cardinality/jev/k77.sl.recall5":0.0775,"cap/cardinality/jev/k77.sl.rejected":0.0,"cap/cardinality/jev/k77.sl.unanswered":0,"cap/cardinality/jev/k77.sl.usd":0.011,"cap/cardinality/jev/k77.unanswered":0,"cap/cardinality/jev/k77.usd":0.0272,"cap/cardinality/kai/k1000.accall":0.005,"cap/cardinality/kai/k1000.accuracy":0.005,"cap/cardinality/kai/k1000.bi.accall":0.03,"cap/cardinality/kai/k1000.bi.accuracy":0.03,"cap/cardinality/kai/k1000.bi.brier":1.0824,"cap/cardinality/kai/k1000.bi.ece":0.253,"cap/cardinality/kai/k1000.bi.msperq":2.97,"cap/cardinality/kai/k1000.bi.nll":11.9627,"cap/cardinality/kai/k1000.bi.recall1":0.03,"cap/cardinality/kai/k1000.bi.recall20":0.655,"cap/cardinality/kai/k1000.bi.recall5":0.135,"cap/cardinality/kai/k1000.bi.rejected":0.0,"cap/cardinality/kai/k1000.bi.unanswered":0,"cap/cardinality/kai/k1000.brier":0.9982,"cap/cardinality/kai/k1000.ece":0.0261,"cap/cardinality/kai/k1000.msperq":32.01,"cap/cardinality/kai/k1000.nll":6.6967,"cap/cardinality/kai/k1000.recall1":0.005,"cap/cardinality/kai/k1000.recall20":0.1,"cap/cardinality/kai/k1000.recall5":0.035,"cap/cardinality/kai/k1000.rejected":0.0,"cap/cardinality/kai/k1000.sl.accall":0.0,"cap/cardinality/kai/k1000.sl.accuracy":0.0,"cap/cardinality/kai/k1000.sl.brier":1.118,"cap/cardinality/kai/k1000.sl.ece":0.2307,"cap/cardinality/kai/k1000.sl.msperq":2.885,"cap/cardinality/kai/k1000.sl.nll":26.6337,"cap/cardinality/kai/k1000.sl.recall1":0.0,"cap/cardinality/kai/k1000.sl.recall20":0.04,"cap/cardinality/kai/k1000.sl.recall5":0.025,"cap/cardinality/kai/k1000.sl.rejected":0.0,"cap/cardinality/kai/k1000.sl.unanswered":0,"cap/cardinality/kai/k1000.unanswered":0,"cap/cardinality/kai/k10000.accall":0.0,"cap/cardinality/kai/k10000.accuracy":0.0,"cap/cardinality/kai/k10000.bi.accall":0.02,"cap/cardinality/kai/k10000.bi.accuracy":0.02,"cap/cardinality/kai/k10000.bi.brier":1.0923,"cap/cardinality/kai/k10000.bi.ece":0.2318,"cap/cardinality/kai/k10000.bi.msperq":3.58,"cap/cardinality/kai/k10000.bi.nll":17.9489,"cap/cardinality/kai/k10000.bi.recall1":0.02,"cap/cardinality/kai/k10000.bi.recall20":0.405,"cap/cardinality/kai/k10000.bi.recall5":0.075,"cap/cardinality/kai/k10000.bi.rejected":0.0,"cap/cardinality/kai/k10000.bi.unanswered":0,"cap/cardinality/kai/k10000.brier":1.0,"cap/cardinality/kai/k10000.ece":0.01,"cap/cardinality/kai/k10000.msperq":206.56,"cap/cardinality/kai/k10000.nll":8.7704,"cap/cardinality/kai/k10000.recall1":0.0,"cap/cardinality/kai/k10000.recall20":0.035,"cap/cardinality/kai/k10000.recall5":0.01,"cap/cardinality/kai/k10000.rejected":0.0,"cap/cardinality/kai/k10000.sl.accall":0.0,"cap/cardinality/kai/k10000.sl.accuracy":0.0,"cap/cardinality/kai/k10000.sl.brier":1.1302,"cap/cardinality/kai/k10000.sl.ece":0.2365,"cap/cardinality/kai/k10000.sl.msperq":3.56,"cap/cardinality/kai/k10000.sl.nll":27.2571,"cap/cardinality/kai/k10000.sl.recall1":0.0,"cap/cardinality/kai/k10000.sl.recall20":0.015,"cap/cardinality/kai/k10000.sl.recall5":0.01,"cap/cardinality/kai/k10000.sl.rejected":0.0,"cap/cardinality/kai/k10000.sl.unanswered":0,"cap/cardinality/kai/k10000.unanswered":0,"cap/cardinality/kai/k100000.accall":0.0,"cap/cardinality/kai/k100000.accuracy":0.0,"cap/cardinality/kai/k100000.bi.accall":0.0,"cap/cardinality/kai/k100000.bi.accuracy":0.0,"cap/cardinality/kai/k100000.bi.brier":1.0965,"cap/cardinality/kai/k100000.bi.ece":0.1965,"cap/cardinality/kai/k100000.bi.msperq":11.82,"cap/cardinality/kai/k100000.bi.nll":21.4707,"cap/cardinality/kai/k100000.bi.recall1":0.0,"cap/cardinality/kai/k100000.bi.recall20":0.26,"cap/cardinality/kai/k100000.bi.recall5":0.04,"cap/cardinality/kai/k100000.bi.rejected":0.0,"cap/cardinality/kai/k100000.bi.unanswered":0,"cap/cardinality/kai/k100000.brier":1.001,"cap/cardinality/kai/k100000.ece":0.0047,"cap/cardinality/kai/k100000.msperq":3706.4,"cap/cardinality/kai/k100000.nll":11.2211,"cap/cardinality/kai/k100000.recall1":0.0,"cap/cardinality/kai/k100000.recall20":0.0,"cap/cardinality/kai/k100000.recall5":0.0,"cap/cardinality/kai/k100000.rejected":0.0,"cap/cardinality/kai/k100000.sl.accall":0.0,"cap/cardinality/kai/k100000.sl.accuracy":0.0,"cap/cardinality/kai/k100000.sl.brier":1.1271,"cap/cardinality/kai/k100000.sl.ece":0.2351,"cap/cardinality/kai/k100000.sl.msperq":37.92,"cap/cardinality/kai/k100000.sl.nll":27.1246,"cap/cardinality/kai/k100000.sl.recall1":0.0,"cap/cardinality/kai/k100000.sl.recall20":0.02,"cap/cardinality/kai/k100000.sl.recall5":0.02,"cap/cardinality/kai/k100000.sl.rejected":0.0,"cap/cardinality/kai/k100000.sl.unanswered":0,"cap/cardinality/kai/k100000.unanswered":0,"cap/cardinality/kai/k150.accall":0.9075,"cap/cardinality/kai/k150.accuracy":0.9075,"cap/cardinality/kai/k150.bi.accall":0.8025,"cap/cardinality/kai/k150.bi.accuracy":0.8025,"cap/cardinality/kai/k150.bi.brier":0.3219,"cap/cardinality/kai/k150.bi.ece":0.1376,"cap/cardinality/kai/k150.bi.msperq":1.1825,"cap/cardinality/kai/k150.bi.nll":4.2481,"cap/cardinality/kai/k150.bi.recall1":0.8025,"cap/cardinality/kai/k150.bi.recall20":0.8575,"cap/cardinality/kai/k150.bi.recall5":0.8525,"cap/cardinality/kai/k150.bi.rejected":0.0,"cap/cardinality/kai/k150.bi.unanswered":0,"cap/cardinality/kai/k150.brier":0.15,"cap/cardinality/kai/k150.ece":0.0409,"cap/cardinality/kai/k150.msperq":10.71,"cap/cardinality/kai/k150.nll":0.5465,"cap/cardinality/kai/k150.recall1":0.9075,"cap/cardinality/kai/k150.recall20":0.9875,"cap/cardinality/kai/k150.recall5":0.9675,"cap/cardinality/kai/k150.rejected":0.0,"cap/cardinality/kai/k150.sl.accall":0.19,"cap/cardinality/kai/k150.sl.accuracy":0.19,"cap/cardinality/kai/k150.sl.brier":1.3418,"cap/cardinality/kai/k150.sl.ece":0.6058,"cap/cardinality/kai/k150.sl.msperq":1.305,"cap/cardinality/kai/k150.sl.nll":22.1634,"cap/cardinality/kai/k150.sl.recall1":0.19,"cap/cardinality/kai/k150.sl.recall20":0.2,"cap/cardinality/kai/k150.sl.recall5":0.1975,"cap/cardinality/kai/k150.sl.rejected":0.0,"cap/cardinality/kai/k150.sl.unanswered":0,"cap/cardinality/kai/k150.unanswered":0,"cap/cardinality/kai/k16.accall":0.935,"cap/cardinality/kai/k16.accuracy":0.935,"cap/cardinality/kai/k16.brier":0.1135,"cap/cardinality/kai/k16.ece":0.0516,"cap/cardinality/kai/k16.msperq":1.4975,"cap/cardinality/kai/k16.nll":0.5631,"cap/cardinality/kai/k16.recall1":0.935,"cap/cardinality/kai/k16.recall20":1.0,"cap/cardinality/kai/k16.recall5":0.99,"cap/cardinality/kai/k16.rejected":0.0,"cap/cardinality/kai/k16.unanswered":0,"cap/cardinality/kai/k4.accall":0.9675,"cap/cardinality/kai/k4.accuracy":0.9675,"cap/cardinality/kai/k4.brier":0.0564,"cap/cardinality/kai/k4.ece":0.028,"cap/cardinality/kai/k4.msperq":2.6775,"cap/cardinality/kai/k4.nll":0.2471,"cap/cardinality/kai/k4.recall1":0.9675,"cap/cardinality/kai/k4.recall20":1.0,"cap/cardinality/kai/k4.recall5":1.0,"cap/cardinality/kai/k4.rejected":0.0,"cap/cardinality/kai/k4.unanswered":0,"cap/cardinality/kai/k77.accall":0.9075,"cap/cardinality/kai/k77.accuracy":0.9075,"cap/cardinality/kai/k77.bi.accall":0.8875,"cap/cardinality/kai/k77.bi.accuracy":0.8875,"cap/cardinality/kai/k77.bi.brier":0.1987,"cap/cardinality/kai/k77.bi.ece":0.09,"cap/cardinality/kai/k77.bi.msperq":1.5175,"cap/cardinality/kai/k77.bi.nll":1.7013,"cap/cardinality/kai/k77.bi.recall1":0.8875,"cap/cardinality/kai/k77.bi.recall20":0.965,"cap/cardinality/kai/k77.bi.recall5":0.9525,"cap/cardinality/kai/k77.bi.rejected":0.0,"cap/cardinality/kai/k77.bi.unanswered":0,"cap/cardinality/kai/k77.brier":0.1638,"cap/cardinality/kai/k77.ece":0.0747,"cap/cardinality/kai/k77.msperq":5.695,"cap/cardinality/kai/k77.nll":0.7448,"cap/cardinality/kai/k77.recall1":0.9075,"cap/cardinality/kai/k77.recall20":0.9925,"cap/cardinality/kai/k77.recall5":0.9725,"cap/cardinality/kai/k77.rejected":0.0,"cap/cardinality/kai/k77.sl.accall":0.075,"cap/cardinality/kai/k77.sl.accuracy":0.075,"cap/cardinality/kai/k77.sl.brier":1.568,"cap/cardinality/kai/k77.sl.ece":0.7244,"cap/cardinality/kai/k77.sl.msperq":1.5575,"cap/cardinality/kai/k77.sl.nll":25.4405,"cap/cardinality/kai/k77.sl.recall1":0.075,"cap/cardinality/kai/k77.sl.recall20":0.08,"cap/cardinality/kai/k77.sl.recall5":0.0775,"cap/cardinality/kai/k77.sl.rejected":0.0,"cap/cardinality/kai/k77.sl.unanswered":0,"cap/cardinality/kai/k77.unanswered":0,"cap/cardinality/laya/k1000.accall":0.0,"cap/cardinality/laya/k1000.bi.accall":0.425,"cap/cardinality/laya/k1000.bi.accuracy":0.425,"cap/cardinality/laya/k1000.bi.brier":0.9681,"cap/cardinality/laya/k1000.bi.ece":0.4263,"cap/cardinality/laya/k1000.bi.msperq":103.895,"cap/cardinality/laya/k1000.bi.nll":10.9297,"cap/cardinality/laya/k1000.bi.recall1":0.425,"cap/cardinality/laya/k1000.bi.recall20":0.655,"cap/cardinality/laya/k1000.bi.recall5":0.58,"cap/cardinality/laya/k1000.bi.unanswered":0,"cap/cardinality/laya/k1000.optionscut":0.0,"cap/cardinality/laya/k1000.rejected":1.0,"cap/cardinality/laya/k1000.sl.accall":0.03,"cap/cardinality/laya/k1000.sl.accuracy":0.03,"cap/cardinality/laya/k1000.sl.brier":1.4916,"cap/cardinality/laya/k1000.sl.ece":0.6185,"cap/cardinality/laya/k1000.sl.msperq":99.02,"cap/cardinality/laya/k1000.sl.nll":26.605,"cap/cardinality/laya/k1000.sl.recall1":0.03,"cap/cardinality/laya/k1000.sl.recall20":0.04,"cap/cardinality/laya/k1000.sl.recall5":0.04,"cap/cardinality/laya/k1000.sl.unanswered":0,"cap/cardinality/laya/k1000.statecut":0.0,"cap/cardinality/laya/k1000.unanswered":200,"cap/cardinality/laya/k1000.wide.accall":0.02,"cap/cardinality/laya/k1000.wide.accuracy":0.02,"cap/cardinality/laya/k1000.wide.brier":1.1571,"cap/cardinality/laya/k1000.wide.ece":0.284,"cap/cardinality/laya/k1000.wide.msperq":12943.335,"cap/cardinality/laya/k1000.wide.nll":8.4174,"cap/cardinality/laya/k1000.wide.optionscut":0.0,"cap/cardinality/laya/k1000.wide.recall1":0.02,"cap/cardinality/laya/k1000.wide.recall20":0.135,"cap/cardinality/laya/k1000.wide.recall5":0.055,"cap/cardinality/laya/k1000.wide.rejected":0.0,"cap/cardinality/laya/k1000.wide.statecut":0.0,"cap/cardinality/laya/k1000.wide.unanswered":0,"cap/cardinality/laya/k10000.accall":0.0,"cap/cardinality/laya/k10000.bi.accall":0.205,"cap/cardinality/laya/k10000.bi.accuracy":0.205,"cap/cardinality/laya/k10000.bi.brier":1.3592,"cap/cardinality/laya/k10000.bi.ece":0.6352,"cap/cardinality/laya/k10000.bi.msperq":103.435,"cap/cardinality/laya/k10000.bi.nll":17.5739,"cap/cardinality/laya/k10000.bi.recall1":0.205,"cap/cardinality/laya/k10000.bi.recall20":0.405,"cap/cardinality/laya/k10000.bi.recall5":0.35,"cap/cardinality/laya/k10000.bi.unanswered":0,"cap/cardinality/laya/k10000.optionscut":0.0,"cap/cardinality/laya/k10000.rejected":1.0,"cap/cardinality/laya/k10000.sl.accall":0.01,"cap/cardinality/laya/k10000.sl.accuracy":0.01,"cap/cardinality/laya/k10000.sl.brier":1.4228,"cap/cardinality/laya/k10000.sl.ece":0.5454,"cap/cardinality/laya/k10000.sl.msperq":102.41,"cap/cardinality/laya/k10000.sl.nll":27.2361,"cap/cardinality/laya/k10000.sl.recall1":0.01,"cap/cardinality/laya/k10000.sl.recall20":0.015,"cap/cardinality/laya/k10000.sl.recall5":0.015,"cap/cardinality/laya/k10000.sl.unanswered":0,"cap/cardinality/laya/k10000.statecut":0.0,"cap/cardinality/laya/k10000.unanswered":200,"cap/cardinality/laya/k100000.accall":0.0,"cap/cardinality/laya/k100000.bi.accall":0.04,"cap/cardinality/laya/k100000.bi.accuracy":0.04,"cap/cardinality/laya/k100000.bi.brier":1.6497,"cap/cardinality/laya/k100000.bi.ece":0.7569,"cap/cardinality/laya/k100000.bi.msperq":105.04,"cap/cardinality/laya/k100000.bi.nll":22.0356,"cap/cardinality/laya/k100000.bi.recall1":0.04,"cap/cardinality/laya/k100000.bi.recall20":0.26,"cap/cardinality/laya/k100000.bi.recall5":0.14,"cap/cardinality/laya/k100000.bi.unanswered":0,"cap/cardinality/laya/k100000.optionscut":0.0,"cap/cardinality/laya/k100000.rejected":1.0,"cap/cardinality/laya/k100000.sl.accall":0.02,"cap/cardinality/laya/k100000.sl.accuracy":0.02,"cap/cardinality/laya/k100000.sl.brier":1.4937,"cap/cardinality/laya/k100000.sl.ece":0.6194,"cap/cardinality/laya/k100000.sl.msperq":115.76,"cap/cardinality/laya/k100000.sl.nll":27.0784,"cap/cardinality/laya/k100000.sl.recall1":0.02,"cap/cardinality/laya/k100000.sl.recall20":0.02,"cap/cardinality/laya/k100000.sl.recall5":0.02,"cap/cardinality/laya/k100000.sl.unanswered":0,"cap/cardinality/laya/k100000.statecut":0.0,"cap/cardinality/laya/k100000.unanswered":50,"cap/cardinality/laya/k150.accall":0.61,"cap/cardinality/laya/k150.accuracy":0.61,"cap/cardinality/laya/k150.bi.accall":0.7775,"cap/cardinality/laya/k150.bi.accuracy":0.7775,"cap/cardinality/laya/k150.bi.brier":0.395,"cap/cardinality/laya/k150.bi.ece":0.1881,"cap/cardinality/laya/k150.bi.msperq":42.53,"cap/cardinality/laya/k150.bi.nll":4.5791,"cap/cardinality/laya/k150.bi.recall1":0.7775,"cap/cardinality/laya/k150.bi.recall20":0.8575,"cap/cardinality/laya/k150.bi.recall5":0.8575,"cap/cardinality/laya/k150.bi.unanswered":0,"cap/cardinality/laya/k150.brier":0.6982,"cap/cardinality/laya/k150.ece":0.3258,"cap/cardinality/laya/k150.msperq":396.03,"cap/cardinality/laya/k150.nll":4.1294,"cap/cardinality/laya/k150.optionscut":1.0,"cap/cardinality/laya/k150.recall1":0.61,"cap/cardinality/laya/k150.recall20":0.975,"cap/cardinality/laya/k150.recall5":0.92,"cap/cardinality/laya/k150.rejected":0.0,"cap/cardinality/laya/k150.sl.accall":0.1925,"cap/cardinality/laya/k150.sl.accuracy":0.1925,"cap/cardinality/laya/k150.sl.brier":1.3844,"cap/cardinality/laya/k150.sl.ece":0.6286,"cap/cardinality/laya/k150.sl.msperq":52.81,"cap/cardinality/laya/k150.sl.nll":22.1297,"cap/cardinality/laya/k150.sl.recall1":0.1925,"cap/cardinality/laya/k150.sl.recall20":0.2,"cap/cardinality/laya/k150.sl.recall5":0.2,"cap/cardinality/laya/k150.sl.unanswered":0,"cap/cardinality/laya/k150.statecut":0.0,"cap/cardinality/laya/k150.unanswered":0,"cap/cardinality/laya/k150.wide.accall":0.6575,"cap/cardinality/laya/k150.wide.accuracy":0.6575,"cap/cardinality/laya/k150.wide.brier":0.62,"cap/cardinality/laya/k150.wide.ece":0.2878,"cap/cardinality/laya/k150.wide.msperq":182.3875,"cap/cardinality/laya/k150.wide.nll":3.6941,"cap/cardinality/laya/k150.wide.optionscut":0.0,"cap/cardinality/laya/k150.wide.recall1":0.6575,"cap/cardinality/laya/k150.wide.recall20":0.9825,"cap/cardinality/laya/k150.wide.recall5":0.95,"cap/cardinality/laya/k150.wide.rejected":0.0,"cap/cardinality/laya/k150.wide.statecut":0.0,"cap/cardinality/laya/k150.wide.unanswered":0,"cap/cardinality/laya/k16.accall":0.67,"cap/cardinality/laya/k16.accuracy":0.67,"cap/cardinality/laya/k16.brier":0.5738,"cap/cardinality/laya/k16.ece":0.25,"cap/cardinality/laya/k16.msperq":56.925,"cap/cardinality/laya/k16.nll":2.4747,"cap/cardinality/laya/k16.optionscut":0.0,"cap/cardinality/laya/k16.recall1":0.67,"cap/cardinality/laya/k16.recall20":1.0,"cap/cardinality/laya/k16.recall5":0.8925,"cap/cardinality/laya/k16.rejected":0.0,"cap/cardinality/laya/k16.statecut":0.0,"cap/cardinality/laya/k16.unanswered":0,"cap/cardinality/laya/k4.accall":0.7775,"cap/cardinality/laya/k4.accuracy":0.7775,"cap/cardinality/laya/k4.brier":0.3229,"cap/cardinality/laya/k4.ece":0.0603,"cap/cardinality/laya/k4.msperq":54.8175,"cap/cardinality/laya/k4.nll":0.6512,"cap/cardinality/laya/k4.optionscut":0.0,"cap/cardinality/laya/k4.recall1":0.7775,"cap/cardinality/laya/k4.recall20":1.0,"cap/cardinality/laya/k4.recall5":1.0,"cap/cardinality/laya/k4.rejected":0.0,"cap/cardinality/laya/k4.statecut":0.0,"cap/cardinality/laya/k4.unanswered":0,"cap/cardinality/laya/k77.accall":0.425,"cap/cardinality/laya/k77.accuracy":0.425,"cap/cardinality/laya/k77.bi.accall":0.57,"cap/cardinality/laya/k77.bi.accuracy":0.57,"cap/cardinality/laya/k77.bi.brier":0.7035,"cap/cardinality/laya/k77.bi.ece":0.2981,"cap/cardinality/laya/k77.bi.msperq":74.7825,"cap/cardinality/laya/k77.bi.nll":3.5176,"cap/cardinality/laya/k77.bi.recall1":0.57,"cap/cardinality/laya/k77.bi.recall20":0.965,"cap/cardinality/laya/k77.bi.recall5":0.8175,"cap/cardinality/laya/k77.bi.unanswered":0,"cap/cardinality/laya/k77.brier":0.9246,"cap/cardinality/laya/k77.ece":0.3809,"cap/cardinality/laya/k77.msperq":196.645,"cap/cardinality/laya/k77.nll":4.7968,"cap/cardinality/laya/k77.optionscut":1.0,"cap/cardinality/laya/k77.recall1":0.425,"cap/cardinality/laya/k77.recall20":0.8775,"cap/cardinality/laya/k77.recall5":0.6975,"cap/cardinality/laya/k77.rejected":0.0,"cap/cardinality/laya/k77.sl.accall":0.075,"cap/cardinality/laya/k77.sl.accuracy":0.075,"cap/cardinality/laya/k77.sl.brier":1.3681,"cap/cardinality/laya/k77.sl.ece":0.55,"cap/cardinality/laya/k77.sl.msperq":75.49,"cap/cardinality/laya/k77.sl.nll":25.4353,"cap/cardinality/laya/k77.sl.recall1":0.075,"cap/cardinality/laya/k77.sl.recall20":0.08,"cap/cardinality/laya/k77.sl.recall5":0.08,"cap/cardinality/laya/k77.sl.unanswered":0,"cap/cardinality/laya/k77.statecut":0.0,"cap/cardinality/laya/k77.unanswered":0,"cap/cardinality/laya/k77.wide.accall":0.5325,"cap/cardinality/laya/k77.wide.accuracy":0.5325,"cap/cardinality/laya/k77.wide.brier":0.7646,"cap/cardinality/laya/k77.wide.ece":0.3024,"cap/cardinality/laya/k77.wide.msperq":239.45,"cap/cardinality/laya/k77.wide.nll":3.1271,"cap/cardinality/laya/k77.wide.optionscut":0.0,"cap/cardinality/laya/k77.wide.recall1":0.5325,"cap/cardinality/laya/k77.wide.recall20":0.9125,"cap/cardinality/laya/k77.wide.recall5":0.8275,"cap/cardinality/laya/k77.wide.rejected":0.0,"cap/cardinality/laya/k77.wide.statecut":0.0,"cap/cardinality/laya/k77.wide.unanswered":0},"detail":{"k1000":{"jev":{"n_errors":200,"error_kinds":{"HTTP 400":200},"errors":["0: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","1: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","2: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","3: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","4: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","5: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","6: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","7: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","8: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","9: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","10: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","11: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","12: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","13: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","14: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","15: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","16: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","17: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","18: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","19: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","20: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","21: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","22: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","23: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","24: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","25: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","26: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","27: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","28: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","29: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","30: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","31: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","32: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","33: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","34: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","35: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","36: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","37: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","38: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","39: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","40: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","41: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","42: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","43: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","44: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","45: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","46: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","47: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","48: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","49: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}"],"served":{},"cost_usd":0.0,"input_tokens":0,"metrics":{"questions":200,"unanswered":200,"accall":0.0}},"jev.bi":{"n_errors":0,"error_kinds":{},"errors":[],"served":{"typesafe/jev-1.13-20260917":200},"cost_usd":0.007075,"input_tokens":168460,"metrics":{"questions":200,"unanswered":0,"n":200,"accuracy":0.64,"ece":0.2198,"brier":0.5481,"nll":9.5674,"accall":0.64,"recall1":0.64,"recall5":0.655,"recall20":0.655}},"jev.sl":{"n_errors":0,"error_kinds":{},"errors":[],"served":{"typesafe/jev-1.13-20260917":200},"cost_usd":0.006791,"input_tokens":161696,"metrics":{"questions":200,"unanswered":0,"n":200,"accuracy":0.035,"ece":0.5853,"brier":1.4346,"nll":26.5488,"accall":0.035,"recall1":0.035,"recall5":0.04,"recall20":0.04}},"laya":{"fit":{"questions":200,"rejected":200,"options_cut":0,"state_cut":0},"dropped":200,"seconds":0.0,"metrics":{"questions":200,"unanswered":200,"accall":0.0}},"kai.bi":{"seconds":0.594,"metrics":{"questions":200,"unanswered":0,"n":200,"accuracy":0.03,"ece":0.253,"brier":1.0824,"nll":11.9627,"accall":0.03,"recall1":0.03,"recall5":0.135,"recall20":0.655}},"kai.sl":{"seconds":0.577,"metrics":{"questions":200,"unanswered":0,"n":200,"accuracy":0.0,"ece":0.2307,"brier":1.118,"nll":26.6337,"accall":0.0,"recall1":0.0,"recall5":0.025,"recall20":0.04}},"laya.bi":{"dropped":0,"seconds":20.779,"metrics":{"questions":200,"unanswered":0,"n":200,"accuracy":0.425,"ece":0.4263,"brier":0.9681,"nll":10.9297,"accall":0.425,"recall1":0.425,"recall5":0.58,"recall20":0.655}},"laya.sl":{"dropped":0,"seconds":19.804,"metrics":{"questions":200,"unanswered":0,"n":200,"accuracy":0.03,"ece":0.6185,"brier":1.4916,"nll":26.605,"accall":0.03,"recall1":0.03,"recall5":0.04,"recall20":0.04}},"laya.wide":{"fit":{"questions":200,"rejected":0,"options_cut":0,"state_cut":0},"dropped":0,"seconds":2588.667,"metrics":{"questions":200,"unanswered":0,"n":200,"accuracy":0.02,"ece":0.284,"brier":1.1571,"nll":8.4174,"accall":0.02,"recall1":0.02,"recall5":0.055,"recall20":0.135}},"kai":{"seconds":6.402,"metrics":{"questions":200,"unanswered":0,"n":200,"accuracy":0.005,"ece":0.0261,"brier":0.9982,"nll":6.6967,"accall":0.005,"recall1":0.005,"recall5":0.035,"recall20":0.1}}},"k10000":{"jev":{"n_errors":200,"error_kinds":{"HTTP 400":200},"errors":["0: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","1: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","2: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","3: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","4: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","5: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","6: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","7: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","8: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","9: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}"],"served":{},"cost_usd":0.0,"input_tokens":0,"unsent":190,"note":"10 sent, all refused (HTTP 400); the rest unsent","metrics":{"questions":200,"unanswered":200,"accall":0.0}},"jev.bi":{"n_errors":0,"error_kinds":{},"errors":[],"served":{"typesafe/jev-1.13-20260917":200},"cost_usd":0.007035,"input_tokens":167494,"metrics":{"questions":200,"unanswered":0,"n":200,"accuracy":0.4,"ece":0.3643,"brier":0.9115,"nll":16.4607,"accall":0.4,"recall1":0.4,"recall5":0.405,"recall20":0.405}},"jev.sl":{"n_errors":0,"error_kinds":{},"errors":[],"served":{"typesafe/jev-1.13-20260917":200},"cost_usd":0.006916,"input_tokens":164674,"metrics":{"questions":200,"unanswered":0,"n":200,"accuracy":0.015,"ece":0.5989,"brier":1.4585,"nll":27.2166,"accall":0.015,"recall1":0.015,"recall5":0.015,"recall20":0.015}},"kai.bi":{"seconds":0.716,"metrics":{"questions":200,"unanswered":0,"n":200,"accuracy":0.02,"ece":0.2318,"brier":1.0923,"nll":17.9489,"accall":0.02,"recall1":0.02,"recall5":0.075,"recall20":0.405}},"kai.sl":{"seconds":0.712,"metrics":{"questions":200,"unanswered":0,"n":200,"accuracy":0.0,"ece":0.2365,"brier":1.1302,"nll":27.2571,"accall":0.0,"recall1":0.0,"recall5":0.01,"recall20":0.015}},"laya.bi":{"dropped":0,"seconds":20.687,"metrics":{"questions":200,"unanswered":0,"n":200,"accuracy":0.205,"ece":0.6352,"brier":1.3592,"nll":17.5739,"accall":0.205,"recall1":0.205,"recall5":0.35,"recall20":0.405}},"laya.sl":{"dropped":0,"seconds":20.482,"metrics":{"questions":200,"unanswered":0,"n":200,"accuracy":0.01,"ece":0.5454,"brier":1.4228,"nll":27.2361,"accall":0.01,"recall1":0.01,"recall5":0.015,"recall20":0.015}},"laya":{"fit":{"questions":200,"rejected":200,"options_cut":0,"state_cut":0},"dropped":200,"seconds":0.0,"metrics":{"questions":200,"unanswered":200,"accall":0.0}},"kai":{"seconds":41.312,"metrics":{"questions":200,"unanswered":0,"n":200,"accuracy":0.0,"ece":0.01,"brier":1.0,"nll":8.7704,"accall":0.0,"recall1":0.0,"recall5":0.01,"recall20":0.035}}},"k100000":{"jev":{"n_errors":50,"error_kinds":{"HTTP 400":50},"errors":["0: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","1: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","2: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","3: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","4: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","5: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","6: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","7: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","8: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}","9: HTTP 400 HTTP 400: {\"detail\":\"Too many choices. Must have at most 255 choices.\"}"],"served":{},"cost_usd":0.0,"input_tokens":0,"unsent":40,"note":"10 sent, all refused (HTTP 400); the rest unsent","metrics":{"questions":50,"unanswered":50,"accall":0.0}},"jev.bi":{"n_errors":0,"error_kinds":{},"errors":[],"served":{"typesafe/jev-1.13-20260917":50},"cost_usd":0.001674,"input_tokens":39853,"metrics":{"questions":50,"unanswered":0,"n":50,"accuracy":0.26,"ece":0.3938,"brier":1.0509,"nll":20.4628,"accall":0.26,"recall1":0.26,"recall5":0.26,"recall20":0.26}},"jev.sl":{"n_errors":0,"error_kinds":{},"errors":[],"served":{"typesafe/jev-1.13-20260917":50},"cost_usd":0.001728,"input_tokens":41145,"metrics":{"questions":50,"unanswered":0,"n":50,"accuracy":0.02,"ece":0.6256,"brier":1.4745,"nll":27.0784,"accall":0.02,"recall1":0.02,"recall5":0.02,"recall20":0.02}},"kai.bi":{"seconds":0.591,"metrics":{"questions":50,"unanswered":0,"n":50,"accuracy":0.0,"ece":0.1965,"brier":1.0965,"nll":21.4707,"accall":0.0,"recall1":0.0,"recall5":0.04,"recall20":0.26}},"kai.sl":{"seconds":1.896,"metrics":{"questions":50,"unanswered":0,"n":50,"accuracy":0.0,"ece":0.2351,"brier":1.1271,"nll":27.1246,"accall":0.0,"recall1":0.0,"recall5":0.02,"recall20":0.02}},"laya.bi":{"dropped":0,"seconds":5.252,"metrics":{"questions":50,"unanswered":0,"n":50,"accuracy":0.04,"ece":0.7569,"brier":1.6497,"nll":22.0356,"accall":0.04,"recall1":0.04,"recall5":0.14,"recall20":0.26}},"laya.sl":{"dropped":0,"seconds":5.788,"metrics":{"questions":50,"unanswered":0,"n":50,"accuracy":0.02,"ece":0.6194,"brier":1.4937,"nll":27.0784,"accall":0.02,"recall1":0.02,"recall5":0.02,"recall20":0.02}},"laya":{"fit":{"questions":50,"rejected":50,"options_cut":0,"state_cut":0},"dropped":50,"seconds":0.0,"metrics":{"questions":50,"unanswered":50,"accall":0.0}},"kai":{"seconds":185.32,"metrics":{"questions":50,"unanswered":0,"n":50,"accuracy":0.0,"ece":0.0047,"brier":1.001,"nll":11.2211,"accall":0.0,"recall1":0.0,"recall5":0.0,"recall20":0.0}}},"k150":{"jev":{"n_errors":0,"error_kinds":{},"errors":[],"served":{"typesafe/jev-1.13-20260917":400},"cost_usd":0.040971,"input_tokens":975506,"metrics":{"questions":400,"unanswered":0,"n":400,"accuracy":0.94,"ece":0.0317,"brier":0.1074,"nll":0.4346,"accall":0.94,"recall1":0.9375,"recall5":0.99,"recall20":0.99}},"jev.bi":{"n_errors":0,"error_kinds":{},"errors":[],"served":{"typesafe/jev-1.13-20260917":400},"cost_usd":0.009705,"input_tokens":231074,"metrics":{"questions":400,"unanswered":0,"n":400,"accuracy":0.825,"ece":0.1044,"brier":0.2934,"nll":4.1037,"accall":0.825,"recall1":0.825,"recall5":0.855,"recall20":0.855}},"jev.sl":{"n_errors":0,"error_kinds":{},"errors":[],"served":{"typesafe/jev-1.13-20260917":400},"cost_usd":0.010185,"input_tokens":242510,"metrics":{"questions":400,"unanswered":0,"n":400,"accuracy":0.1925,"ece":0.5504,"brier":1.2677,"nll":22.189,"accall":0.1925,"recall1":0.1925,"recall5":0.1975,"recall20":0.1975}},"laya":{"fit":{"questions":400,"rejected":0,"options_cut":400,"state_cut":0},"dropped":0,"seconds":158.412,"metrics":{"questions":400,"unanswered":0,"n":400,"accuracy":0.61,"ece":0.3258,"brier":0.6982,"nll":4.1294,"accall":0.61,"recall1":0.61,"recall5":0.92,"recall20":0.975}},"laya.wide":{"fit":{"questions":400,"rejected":0,"options_cut":0,"state_cut":0},"dropped":0,"seconds":72.955,"metrics":{"questions":400,"unanswered":0,"n":400,"accuracy":0.6575,"ece":0.2878,"brier":0.62,"nll":3.6941,"accall":0.6575,"recall1":0.6575,"recall5":0.95,"recall20":0.9825}},"kai.bi":{"seconds":0.473,"metrics":{"questions":400,"unanswered":0,"n":400,"accuracy":0.8025,"ece":0.1376,"brier":0.3219,"nll":4.2481,"accall":0.8025,"recall1":0.8025,"recall5":0.8525,"recall20":0.8575}},"kai.sl":{"seconds":0.522,"metrics":{"questions":400,"unanswered":0,"n":400,"accuracy":0.19,"ece":0.6058,"brier":1.3418,"nll":22.1634,"accall":0.19,"recall1":0.19,"recall5":0.1975,"recall20":0.2}},"laya.bi":{"dropped":0,"seconds":17.012,"metrics":{"questions":400,"unanswered":0,"n":400,"accuracy":0.7775,"ece":0.1881,"brier":0.395,"nll":4.5791,"accall":0.7775,"recall1":0.7775,"recall5":0.8575,"recall20":0.8575}},"laya.sl":{"dropped":0,"seconds":21.124,"metrics":{"questions":400,"unanswered":0,"n":400,"accuracy":0.1925,"ece":0.6286,"brier":1.3844,"nll":22.1297,"accall":0.1925,"recall1":0.1925,"recall5":0.2,"recall20":0.2}},"kai":{"seconds":4.284,"metrics":{"questions":400,"unanswered":0,"n":400,"accuracy":0.9075,"ece":0.0409,"brier":0.15,"nll":0.5465,"accall":0.9075,"recall1":0.9075,"recall5":0.9675,"recall20":0.9875}}},"k16":{"jev":{"n_errors":0,"error_kinds":{},"errors":[],"served":{"typesafe/jev-1.13-20260917":400},"cost_usd":0.009654,"input_tokens":229856,"metrics":{"questions":400,"unanswered":0,"n":400,"accuracy":0.8525,"ece":0.067,"brier":0.2212,"nll":1.1469,"accall":0.8525,"recall1":0.8525,"recall5":0.965,"recall20":1.0}},"laya":{"fit":{"questions":400,"rejected":0,"options_cut":0,"state_cut":0},"dropped":0,"seconds":22.77,"metrics":{"questions":400,"unanswered":0,"n":400,"accuracy":0.67,"ece":0.25,"brier":0.5738,"nll":2.4747,"accall":0.67,"recall1":0.67,"recall5":0.8925,"recall20":1.0}},"kai":{"seconds":0.599,"metrics":{"questions":400,"unanswered":0,"n":400,"accuracy":0.935,"ece":0.0516,"brier":0.1135,"nll":0.5631,"accall":0.935,"recall1":0.935,"recall5":0.99,"recall20":1.0}}},"k4":{"jev":{"n_errors":0,"error_kinds":{},"errors":[],"served":{"typesafe/jev-1.13-20260917":400},"cost_usd":0.006198,"input_tokens":147562,"metrics":{"questions":400,"unanswered":0,"n":400,"accuracy":0.905,"ece":0.0465,"brier":0.1402,"nll":0.4411,"accall":0.905,"recall1":0.905,"recall5":1.0,"recall20":1.0}},"laya":{"fit":{"questions":400,"rejected":0,"options_cut":0,"state_cut":0},"dropped":0,"seconds":21.927,"metrics":{"questions":400,"unanswered":0,"n":400,"accuracy":0.7775,"ece":0.0603,"brier":0.3229,"nll":0.6512,"accall":0.7775,"recall1":0.7775,"recall5":1.0,"recall20":1.0}},"kai":{"seconds":1.071,"metrics":{"questions":400,"unanswered":0,"n":400,"accuracy":0.9675,"ece":0.028,"brier":0.0564,"nll":0.2471,"accall":0.9675,"recall1":0.9675,"recall5":1.0,"recall20":1.0}}},"k77":{"jev":{"n_errors":0,"error_kinds":{},"errors":[],"served":{"typesafe/jev-1.13-20260917":400},"cost_usd":0.027234,"input_tokens":648436,"metrics":{"questions":400,"unanswered":0,"n":400,"accuracy":0.8425,"ece":0.0682,"brier":0.2574,"nll":1.2902,"accall":0.8425,"recall1":0.8425,"recall5":0.96,"recall20":0.9675}},"jev.bi":{"n_errors":0,"error_kinds":{},"errors":[],"served":{"typesafe/jev-1.13-20260917":400},"cost_usd":0.010851,"input_tokens":258360,"metrics":{"questions":400,"unanswered":0,"n":400,"accuracy":0.8075,"ece":0.0736,"brier":0.2997,"nll":2.2036,"accall":0.8075,"recall1":0.805,"recall5":0.93,"recall20":0.9325}},"jev.sl":{"n_errors":0,"error_kinds":{},"errors":[],"served":{"typesafe/jev-1.13-20260917":400},"cost_usd":0.011044,"input_tokens":262964,"metrics":{"questions":400,"unanswered":0,"n":400,"accuracy":0.0775,"ece":0.6704,"brier":1.4965,"nll":25.4291,"accall":0.0775,"recall1":0.0775,"recall5":0.0775,"recall20":0.08}},"laya":{"fit":{"questions":400,"rejected":0,"options_cut":400,"state_cut":0},"dropped":0,"seconds":78.658,"metrics":{"questions":400,"unanswered":0,"n":400,"accuracy":0.425,"ece":0.3809,"brier":0.9246,"nll":4.7968,"accall":0.425,"recall1":0.425,"recall5":0.6975,"recall20":0.8775}},"laya.wide":{"fit":{"questions":400,"rejected":0,"options_cut":0,"state_cut":0},"dropped":0,"seconds":95.78,"metrics":{"questions":400,"unanswered":0,"n":400,"accuracy":0.5325,"ece":0.3024,"brier":0.7646,"nll":3.1271,"accall":0.5325,"recall1":0.5325,"recall5":0.8275,"recall20":0.9125}},"kai.bi":{"seconds":0.607,"metrics":{"questions":400,"unanswered":0,"n":400,"accuracy":0.8875,"ece":0.09,"brier":0.1987,"nll":1.7013,"accall":0.8875,"recall1":0.8875,"recall5":0.9525,"recall20":0.965}},"kai.sl":{"seconds":0.623,"metrics":{"questions":400,"unanswered":0,"n":400,"accuracy":0.075,"ece":0.7244,"brier":1.568,"nll":25.4405,"accall":0.075,"recall1":0.075,"recall5":0.0775,"recall20":0.08}},"laya.bi":{"dropped":0,"seconds":29.913,"metrics":{"questions":400,"unanswered":0,"n":400,"accuracy":0.57,"ece":0.2981,"brier":0.7035,"nll":3.5176,"accall":0.57,"recall1":0.57,"recall5":0.8175,"recall20":0.965}},"laya.sl":{"dropped":0,"seconds":30.196,"metrics":{"questions":400,"unanswered":0,"n":400,"accuracy":0.075,"ece":0.55,"brier":1.3681,"nll":25.4353,"accall":0.075,"recall1":0.075,"recall5":0.08,"recall20":0.08}},"kai":{"seconds":2.278,"metrics":{"questions":400,"unanswered":0,"n":400,"accuracy":0.9075,"ece":0.0747,"brier":0.1638,"nll":0.7448,"accall":0.9075,"recall1":0.9075,"recall5":0.9725,"recall20":0.9925}}}}}