benchmarks/decision/capability/results/questions.json

{"detail":{"jev":{"batch":[{"chunks":0,"errors":[],"input_tokens_per_call":808,"mean_ms":165.38819170000002,"n_errors":0,"p50_ms":154.33670899999998,"p95_ms":278.126917,"pairs":null,"peak_mb":null,"questions":1,"questions_per_s":30.231904397803508,"refused_whole":0,"rounds":10,"splits":0,"tokens":null,"usd_per_call":3.4e-05},{"chunks":0,"errors":[],"input_tokens_per_call":6469,"mean_ms":319.19968339999997,"n_errors":0,"p50_ms":271.427875,"p95_ms":640.581375,"pairs":null,"peak_mb":null,"questions":8,"questions_per_s":125.31340750070432,"refused_whole":0,"rounds":10,"splits":0,"tokens":null,"usd_per_call":0.000272},{"chunks":0,"errors":[],"input_tokens_per_call":30475,"mean_ms":725.0986085,"n_errors":0,"p50_ms":707.8235000000001,"p95_ms":841.8500829999999,"pairs":null,"peak_mb":null,"questions":32,"questions_per_s":220.65964287393888,"refused_whole":0,"rounds":10,"splits":0,"tokens":null,"usd_per_call":0.00128},{"chunks":0,"errors":[],"input_tokens_per_call":121320,"mean_ms":2548.9998917,"n_errors":0,"p50_ms":2582.889667,"p95_ms":2795.155458,"pairs":null,"peak_mb":null,"questions":128,"questions_per_s":251.07886512037706,"refused_whole":0,"rounds":10,"splits":0,"tokens":null,"usd_per_call":0.005095}],"n":[{"chunks":0,"errors":[],"input_tokens_per_call":676,"mean_ms":174.85520839999998,"n_errors":0,"p50_ms":133.350917,"p95_ms":545.836042,"pairs":null,"peak_mb":null,"questions":1,"questions_per_s":5.719017518267989,"refused_whole":0,"rounds":10,"splits":0,"tokens":null,"usd_per_call":2.8e-05},{"chunks":0,"errors":[],"input_tokens_per_call":3427,"mean_ms":158.98729570000003,"n_errors":0,"p50_ms":154.577291,"p95_ms":220.78924999999998,"pairs":null,"peak_mb":null,"questions":10,"questions_per_s":62.898107398904564,"refused_whole":0,"rounds":10,"splits":0,"tokens":null,"usd_per_call":0.000144},{"chunks":0,"errors":[],"input_tokens_per_call":15167,"mean_ms":214.37916240000004,"n_errors":0,"p50_ms":201.883125,"p95_ms":275.32433299999997,"pairs":null,"peak_mb":null,"questions":50,"questions_per_s":233.2316230749486,"refused_whole":0,"rounds":10,"splits":0,"tokens":null,"usd_per_call":0.000637},{"chunks":0,"errors":[],"input_tokens_per_call":30053,"mean_ms":374.30760810000004,"n_errors":0,"p50_ms":343.11575,"p95_ms":555.6970409999999,"pairs":null,"peak_mb":null,"questions":100,"questions_per_s":267.159945018494,"refused_whole":0,"rounds":10,"splits":0,"tokens":null,"usd_per_call":0.001262},{"chunks":44,"errors":[],"input_tokens_per_call":149928,"mean_ms":689.0805084000001,"n_errors":0,"p50_ms":514.985292,"p95_ms":2192.758791,"pairs":null,"peak_mb":null,"questions":500,"questions_per_s":725.6046193513257,"refused_whole":11,"rounds":10,"splits":0,"tokens":null,"usd_per_call":0.006297},{"chunks":77,"errors":[],"input_tokens_per_call":299532,"mean_ms":671.5430043,"n_errors":0,"p50_ms":602.163667,"p95_ms":1077.385125,"pairs":null,"peak_mb":null,"questions":1000,"questions_per_s":1489.1079105832923,"refused_whole":11,"rounds":10,"splits":0,"tokens":null,"usd_per_call":0.01258}]},"rocm-bf16.kai":{"batch1":{"mean_ms":29.1543049,"p50_ms":29.016461,"p95_ms":33.210536,"pairs":67834,"peak_mb":null,"questions":1,"questions_per_s":171.5012591502396,"rounds":10,"tokens":820},"batch128":{"mean_ms":1224.500818,"p50_ms":479.774138,"p95_ms":7309.201212999999,"pairs":71435842,"peak_mb":null,"questions":128,"questions_per_s":522.661961994704,"rounds":10,"tokens":212318},"batch32":{"mean_ms":1602.8935698,"p50_ms":1688.269221,"p95_ms":1994.593142,"pairs":16788372,"peak_mb":null,"questions":32,"questions_per_s":99.81947835748316,"rounds":10,"tokens":49154},"batch8":{"mean_ms":113.31446249999999,"p50_ms":115.25169100000001,"p95_ms":184.652477,"pairs":619456,"peak_mb":null,"questions":8,"questions_per_s":352.9999535584436,"rounds":10,"tokens":5212},"n1":{"mean_ms":23.132012000000003,"p50_ms":19.816133999999998,"p95_ms":28.971048,"pairs":15392,"peak_mb":null,"questions":1,"questions_per_s":43.23013493162635,"rounds":10,"tokens":306},"n10":{"mean_ms":51.223996299999996,"p50_ms":44.665290000000006,"p95_ms":105.34125499999999,"pairs":172006,"peak_mb":null,"questions":10,"questions_per_s":195.22100426202007,"rounds":10,"tokens":1960},"n100":{"mean_ms":310.02689810000004,"p50_ms":314.384393,"p95_ms":333.15693899999997,"pairs":1864462,"peak_mb":null,"questions":100,"questions_per_s":322.5526578914605,"rounds":10,"tokens":15378},"n1000":{"mean_ms":1970.0841838000001,"p50_ms":1980.9689449999998,"p95_ms":2013.52875,"pairs":14865507,"peak_mb":null,"questions":1000,"questions_per_s":507.59252230082296,"rounds":10,"tokens":123893},"n50":{"mean_ms":158.45214449999997,"p50_ms":139.494336,"p95_ms":213.667106,"pairs":841248,"peak_mb":null,"questions":50,"questions_per_s":315.5526872657757,"rounds":10,"tokens":7346},"n500":{"mean_ms":1068.4553783000003,"p50_ms":1050.289076,"p95_ms":1172.52825,"pairs":7688153,"peak_mb":null,"questions":500,"questions_per_s":467.9652610252576,"rounds":10,"tokens":63977}},"rocm-bf16.laya":{"batch1":{"dropped":0,"mean_ms":37.155764899999994,"p50_ms":36.790278,"p95_ms":38.962443,"pairs":131615,"peak_mb":null,"questions":1,"questions_per_s":134.5686197944481,"rounds":10,"tokens":807},"batch128":{"dropped":0,"mean_ms":7752.975996699999,"p50_ms":7108.831309,"p95_ms":9615.503542,"pairs":91789695,"peak_mb":null,"questions":128,"questions_per_s":82.54894640102222,"rounds":10,"tokens":240357},"batch32":{"dropped":0,"mean_ms":2058.5647483000002,"p50_ms":2091.5602409999997,"p95_ms":2506.491701,"pairs":23063450,"peak_mb":null,"questions":32,"questions_per_s":77.72405513702249,"rounds":10,"tokens":58160},"batch8":{"dropped":0,"mean_ms":288.26767600000005,"p50_ms":283.577223,"p95_ms":303.126831,"pairs":1197364,"peak_mb":null,"questions":8,"questions_per_s":138.75992117825933,"rounds":10,"tokens":6886},"n1":{"dropped":0,"mean_ms":16.3940527,"p50_ms":16.457254000000002,"p95_ms":18.321356,"pairs":82369,"peak_mb":null,"questions":1,"questions_per_s":60.997729987777824,"rounds":10,"tokens":287},"n10":{"dropped":0,"mean_ms":120.91995610000001,"p50_ms":116.200786,"p95_ms":173.314752,"pairs":635075,"peak_mb":null,"questions":10,"questions_per_s":82.69933534982485,"rounds":10,"tokens":2447},"n100":{"dropped":0,"mean_ms":1166.7112258000002,"p50_ms":1167.6768900000002,"p95_ms":1222.216384,"pairs":7989447,"peak_mb":null,"questions":100,"questions_per_s":85.71101210707147,"rounds":10,"tokens":27705},"n1000":{"dropped":0,"mean_ms":9933.384951700002,"p50_ms":9771.814519,"p95_ms":10374.57055,"pairs":72789260,"peak_mb":null,"questions":1000,"questions_per_s":100.6706178067588,"rounds":10,"tokens":264108},"n50":{"dropped":0,"mean_ms":570.8209101,"p50_ms":576.4678869999999,"p95_ms":592.33581,"pairs":3893671,"peak_mb":null,"questions":50,"questions_per_s":87.59314719434626,"rounds":10,"tokens":13717},"n500":{"dropped":0,"mean_ms":5051.367147,"p50_ms":5026.924848000001,"p95_ms":5386.425158999999,"pairs":36822493,"peak_mb":null,"questions":500,"questions_per_s":98.98310406855882,"rounds":10,"tokens":132841}}},"keys":{"cap/questions/jev/batch1.meanms":165.3882,"cap/questions/jev/batch1.nerrors":0,"cap/questions/jev/batch1.p50ms":154.3367,"cap/questions/jev/batch1.p95ms":278.1269,"cap/questions/jev/batch1.qpers":30.2319,"cap/questions/jev/batch1.refusedwhole":0,"cap/questions/jev/batch1.splits":0,"cap/questions/jev/batch1.usdpercall":0.0,"cap/questions/jev/batch128.meanms":2548.9999,"cap/questions/jev/batch128.nerrors":0,"cap/questions/jev/batch128.p50ms":2582.8897,"cap/questions/jev/batch128.p95ms":2795.1555,"cap/questions/jev/batch128.qpers":251.0789,"cap/questions/jev/batch128.refusedwhole":0,"cap/questions/jev/batch128.splits":0,"cap/questions/jev/batch128.usdpercall":0.0051,"cap/questions/jev/batch32.meanms":725.0986,"cap/questions/jev/batch32.nerrors":0,"cap/questions/jev/batch32.p50ms":707.8235,"cap/questions/jev/batch32.p95ms":841.8501,"cap/questions/jev/batch32.qpers":220.6596,"cap/questions/jev/batch32.refusedwhole":0,"cap/questions/jev/batch32.splits":0,"cap/questions/jev/batch32.usdpercall":0.0013,"cap/questions/jev/batch8.meanms":319.1997,"cap/questions/jev/batch8.nerrors":0,"cap/questions/jev/batch8.p50ms":271.4279,"cap/questions/jev/batch8.p95ms":640.5814,"cap/questions/jev/batch8.qpers":125.3134,"cap/questions/jev/batch8.refusedwhole":0,"cap/questions/jev/batch8.splits":0,"cap/questions/jev/batch8.usdpercall":0.0003,"cap/questions/jev/n1.meanms":174.8552,"cap/questions/jev/n1.nerrors":0,"cap/questions/jev/n1.p50ms":133.3509,"cap/questions/jev/n1.p95ms":545.836,"cap/questions/jev/n1.qpers":5.719,"cap/questions/jev/n1.refusedwhole":0,"cap/questions/jev/n1.splits":0,"cap/questions/jev/n1.usdpercall":0.0,"cap/questions/jev/n10.meanms":158.9873,"cap/questions/jev/n10.nerrors":0,"cap/questions/jev/n10.p50ms":154.5773,"cap/questions/jev/n10.p95ms":220.7892,"cap/questions/jev/n10.qpers":62.8981,"cap/questions/jev/n10.refusedwhole":0,"cap/questions/jev/n10.splits":0,"cap/questions/jev/n10.usdpercall":0.0001,"cap/questions/jev/n100.meanms":374.3076,"cap/questions/jev/n100.nerrors":0,"cap/questions/jev/n100.p50ms":343.1157,"cap/questions/jev/n100.p95ms":555.697,"cap/questions/jev/n100.qpers":267.1599,"cap/questions/jev/n100.refusedwhole":0,"cap/questions/jev/n100.splits":0,"cap/questions/jev/n100.usdpercall":0.0013,"cap/questions/jev/n1000.meanms":671.543,"cap/questions/jev/n1000.nerrors":0,"cap/questions/jev/n1000.p50ms":602.1637,"cap/questions/jev/n1000.p95ms":1077.3851,"cap/questions/jev/n1000.qpers":1489.1079,"cap/questions/jev/n1000.refusedwhole":11,"cap/questions/jev/n1000.splits":0,"cap/questions/jev/n1000.usdpercall":0.0126,"cap/questions/jev/n50.meanms":214.3792,"cap/questions/jev/n50.nerrors":0,"cap/questions/jev/n50.p50ms":201.8831,"cap/questions/jev/n50.p95ms":275.3243,"cap/questions/jev/n50.qpers":233.2316,"cap/questions/jev/n50.refusedwhole":0,"cap/questions/jev/n50.splits":0,"cap/questions/jev/n50.usdpercall":0.0006,"cap/questions/jev/n500.meanms":689.0805,"cap/questions/jev/n500.nerrors":0,"cap/questions/jev/n500.p50ms":514.9853,"cap/questions/jev/n500.p95ms":2192.7588,"cap/questions/jev/n500.qpers":725.6046,"cap/questions/jev/n500.refusedwhole":11,"cap/questions/jev/n500.splits":0,"cap/questions/jev/n500.usdpercall":0.0063,"cap/questions/laya/rocm-bf16.batch1.dropped":0,"cap/questions/laya/rocm-bf16.batch1.meanms":37.1558,"cap/questions/laya/rocm-bf16.batch1.p50ms":36.7903,"cap/questions/laya/rocm-bf16.batch1.p95ms":38.9624,"cap/questions/laya/rocm-bf16.batch1.pairs":131615,"cap/questions/laya/rocm-bf16.batch1.qpers":134.5686,"cap/questions/laya/rocm-bf16.batch1.rounds":10,"cap/questions/laya/rocm-bf16.batch1.tokens":807,"cap/questions/laya/rocm-bf16.batch128.dropped":0,"cap/questions/laya/rocm-bf16.batch128.meanms":7752.976,"cap/questions/laya/rocm-bf16.batch128.p50ms":7108.8313,"cap/questions/laya/rocm-bf16.batch128.p95ms":9615.5035,"cap/questions/laya/rocm-bf16.batch128.pairs":91789695,"cap/questions/laya/rocm-bf16.batch128.qpers":82.5489,"cap/questions/laya/rocm-bf16.batch128.rounds":10,"cap/questions/laya/rocm-bf16.batch128.tokens":240357,"cap/questions/laya/rocm-bf16.batch32.dropped":0,"cap/questions/laya/rocm-bf16.batch32.meanms":2058.5647,"cap/questions/laya/rocm-bf16.batch32.p50ms":2091.5602,"cap/questions/laya/rocm-bf16.batch32.p95ms":2506.4917,"cap/questions/laya/rocm-bf16.batch32.pairs":23063450,"cap/questions/laya/rocm-bf16.batch32.qpers":77.7241,"cap/questions/laya/rocm-bf16.batch32.rounds":10,"cap/questions/laya/rocm-bf16.batch32.tokens":58160,"cap/questions/laya/rocm-bf16.batch8.dropped":0,"cap/questions/laya/rocm-bf16.batch8.meanms":288.2677,"cap/questions/laya/rocm-bf16.batch8.p50ms":283.5772,"cap/questions/laya/rocm-bf16.batch8.p95ms":303.1268,"cap/questions/laya/rocm-bf16.batch8.pairs":1197364,"cap/questions/laya/rocm-bf16.batch8.qpers":138.7599,"cap/questions/laya/rocm-bf16.batch8.rounds":10,"cap/questions/laya/rocm-bf16.batch8.tokens":6886,"cap/questions/laya/rocm-bf16.checkpoint":"laya-multilingual: hanzoai/kai-1-multilingual@3119843b, encoder 768 wide, 22 layers","cap/questions/laya/rocm-bf16.n1.dropped":0,"cap/questions/laya/rocm-bf16.n1.meanms":16.3941,"cap/questions/laya/rocm-bf16.n1.p50ms":16.4573,"cap/questions/laya/rocm-bf16.n1.p95ms":18.3214,"cap/questions/laya/rocm-bf16.n1.pairs":82369,"cap/questions/laya/rocm-bf16.n1.qpers":60.9977,"cap/questions/laya/rocm-bf16.n1.rounds":10,"cap/questions/laya/rocm-bf16.n1.tokens":287,"cap/questions/laya/rocm-bf16.n10.dropped":0,"cap/questions/laya/rocm-bf16.n10.meanms":120.92,"cap/questions/laya/rocm-bf16.n10.p50ms":116.2008,"cap/questions/laya/rocm-bf16.n10.p95ms":173.3148,"cap/questions/laya/rocm-bf16.n10.pairs":635075,"cap/questions/laya/rocm-bf16.n10.qpers":82.6993,"cap/questions/laya/rocm-bf16.n10.rounds":10,"cap/questions/laya/rocm-bf16.n10.tokens":2447,"cap/questions/laya/rocm-bf16.n100.dropped":0,"cap/questions/laya/rocm-bf16.n100.meanms":1166.7112,"cap/questions/laya/rocm-bf16.n100.p50ms":1167.6769,"cap/questions/laya/rocm-bf16.n100.p95ms":1222.2164,"cap/questions/laya/rocm-bf16.n100.pairs":7989447,"cap/questions/laya/rocm-bf16.n100.qpers":85.711,"cap/questions/laya/rocm-bf16.n100.rounds":10,"cap/questions/laya/rocm-bf16.n100.tokens":27705,"cap/questions/laya/rocm-bf16.n1000.dropped":0,"cap/questions/laya/rocm-bf16.n1000.meanms":9933.385,"cap/questions/laya/rocm-bf16.n1000.p50ms":9771.8145,"cap/questions/laya/rocm-bf16.n1000.p95ms":10374.5706,"cap/questions/laya/rocm-bf16.n1000.pairs":72789260,"cap/questions/laya/rocm-bf16.n1000.qpers":100.6706,"cap/questions/laya/rocm-bf16.n1000.rounds":10,"cap/questions/laya/rocm-bf16.n1000.tokens":264108,"cap/questions/laya/rocm-bf16.n50.dropped":0,"cap/questions/laya/rocm-bf16.n50.meanms":570.8209,"cap/questions/laya/rocm-bf16.n50.p50ms":576.4679,"cap/questions/laya/rocm-bf16.n50.p95ms":592.3358,"cap/questions/laya/rocm-bf16.n50.pairs":3893671,"cap/questions/laya/rocm-bf16.n50.qpers":87.5931,"cap/questions/laya/rocm-bf16.n50.rounds":10,"cap/questions/laya/rocm-bf16.n50.tokens":13717,"cap/questions/laya/rocm-bf16.n500.dropped":0,"cap/questions/laya/rocm-bf16.n500.meanms":5051.3671,"cap/questions/laya/rocm-bf16.n500.p50ms":5026.9248,"cap/questions/laya/rocm-bf16.n500.p95ms":5386.4252,"cap/questions/laya/rocm-bf16.n500.pairs":36822493,"cap/questions/laya/rocm-bf16.n500.qpers":98.9831,"cap/questions/laya/rocm-bf16.n500.rounds":10,"cap/questions/laya/rocm-bf16.n500.tokens":132841},"meta":{"run 2026-09-27 10:47:01 +0000":{"host":"network","jev":{"model":"typesafe/jev-1.13","price_per_input_token":4.2e-08,"url":"https://openrouter.ai/api/alpha/decisions"},"pool":{"cases":400,"order":"shuffled, seed 13","questions":130,"states":400},"rounds":10},"run 2026-09-28 12:09:15 +0000":{"at":"2026-09-28 12:09:15 +0000","host":"amd64-host","laya":{"checkpoint":"multilingual","checkpoints":[{"id":"laya","repo":"hanzoai/kai-1","revision":"b50502c28537df49a3621f6fa543f9e8521e8a9c"},{"id":"laya-multilingual","repo":"hanzoai/kai-1-multilingual","revision":"3119843b5261e34903f93d03813467524c57982e"},{"id":"laya-agent","repo":"hanzoai/kai-1-agent","revision":"e3e416499b8a983d2aa0bcc684d5457ad6bfd6a1"}],"device":"rocm","dtype":"bf16","runtime":"bench (hanzo-ml, kai::joint)"},"load":[1.9833984375,2.6103515625,3.13232421875],"loadavg":{"after":[1.9833984375,2.6103515625,3.13232421875],"before":[1.71630859375,4.23828125,3.73046875]},"machine":"x86_64","pool":{"cases":400,"order":"shuffled, seed 13","questions":130,"states":400},"rounds":10,"runtime":"bench (hanzo-ml)","threads":{"cores":16,"rayon":null}}}}