benchmarks/decision/capability/results/orig.json

{"meta":{"run 2026-09-27 09:50:11 +0000":{"host":"arm64-mac","machine":"aarch64","runtime":"bench (hanzo-ml)","load":[18.05126953125,22.583984375,25.33251953125],"at":"2026-09-27 09:50:11 +0000"},"run 2026-09-27 14:36:21 +0000":{"host":"arm64-mac","machine":"aarch64","runtime":"bench (hanzo-ml)","load":[7.59814453125,10.54541015625,11.11572265625],"at":"2026-09-27 14:36:21 +0000"},"run 2026-09-28 03:19:56 +0000":{"host":"arm64-host","machine":"aarch64","runtime":"bench (hanzo-ml)","load":[2.78564453125,2.4892578125,2.5146484375],"at":"2026-09-28 03:19:56 +0000"}},"keys":{"cap/orig/jev/agnews.accuracy":0.86,"cap/orig/jev/agnews.aurc":0.0832,"cap/orig/jev/agnews.brier":0.2323,"cap/orig/jev/agnews.ece":0.1023,"cap/orig/jev/agnews.macrof1":0.8485,"cap/orig/jev/agnews.nll":1.805,"cap/orig/jev/agnews.unanswered":0,"cap/orig/jev/agnews.zeroprob":0.0575,"cap/orig/jev/banking77full.accuracy":0.835,"cap/orig/jev/banking77full.aurc":0.0579,"cap/orig/jev/banking77full.brier":0.257,"cap/orig/jev/banking77full.ece":0.0729,"cap/orig/jev/banking77full.macrof1":0.3137,"cap/orig/jev/banking77full.nll":1.3471,"cap/orig/jev/banking77full.unanswered":0,"cap/orig/jev/banking77full.zeroprob":0.035,"cap/orig/jev/beats.kai":16,"cap/orig/jev/beats.laya":56,"cap/orig/jev/emailspam.accuracy":0.9775,"cap/orig/jev/emailspam.aurc":0.0007,"cap/orig/jev/emailspam.brier":0.0379,"cap/orig/jev/emailspam.ece":0.0604,"cap/orig/jev/emailspam.macrof1":0.9774,"cap/orig/jev/emailspam.nll":0.0967,"cap/orig/jev/emailspam.unanswered":0,"cap/orig/jev/emailspam.zeroprob":0.0,"cap/orig/jev/emotion.accuracy":0.6025,"cap/orig/jev/emotion.aurc":0.2311,"cap/orig/jev/emotion.brier":0.6464,"cap/orig/jev/emotion.ece":0.2795,"cap/orig/jev/emotion.macrof1":0.5155,"cap/orig/jev/emotion.nll":4.6832,"cap/orig/jev/emotion.unanswered":0,"cap/orig/jev/emotion.zeroprob":0.1425,"cap/orig/jev/guardrailsjailbreak.accuracy":0.94,"cap/orig/jev/guardrailsjailbreak.aurc":0.012,"cap/orig/jev/guardrailsjailbreak.brier":0.1044,"cap/orig/jev/guardrailsjailbreak.ece":0.0469,"cap/orig/jev/guardrailsjailbreak.macrof1":0.9183,"cap/orig/jev/guardrailsjailbreak.nll":0.1891,"cap/orig/jev/guardrailsjailbreak.unanswered":0,"cap/orig/jev/guardrailsjailbreak.zeroprob":0.0,"cap/orig/jev/massive.above3xrandom":51,"cap/orig/jev/massive.languages":51,"cap/orig/jev/massive.macroaccuracy":0.8902,"cap/orig/jev/massive.macroece":0.0638,"cap/orig/jev/massive.nonenglishmacro":0.8896,"cap/orig/jev/modelroutingdomain.accuracy":0.9774,"cap/orig/jev/modelroutingdomain.aurc":0.0028,"cap/orig/jev/modelroutingdomain.brier":0.04,"cap/orig/jev/modelroutingdomain.ece":0.0138,"cap/orig/jev/modelroutingdomain.macrof1":0.5915,"cap/orig/jev/modelroutingdomain.nll":0.0847,"cap/orig/jev/modelroutingdomain.unanswered":0,"cap/orig/jev/modelroutingdomain.zeroprob":0.0,"cap/orig/jev/moderationtoxicity.accuracy":0.6625,"cap/orig/jev/moderationtoxicity.aurc":0.1382,"cap/orig/jev/moderationtoxicity.brier":0.4316,"cap/orig/jev/moderationtoxicity.ece":0.1768,"cap/orig/jev/moderationtoxicity.macrof1":0.6285,"cap/orig/jev/moderationtoxicity.nll":0.6192,"cap/orig/jev/moderationtoxicity.unanswered":0,"cap/orig/jev/moderationtoxicity.zeroprob":0.0,"cap/orig/jev/phishing.accuracy":0.9,"cap/orig/jev/phishing.aurc":0.0208,"cap/orig/jev/phishing.brier":0.1523,"cap/orig/jev/phishing.ece":0.0388,"cap/orig/jev/phishing.macrof1":0.8874,"cap/orig/jev/phishing.nll":0.2495,"cap/orig/jev/phishing.unanswered":0,"cap/orig/jev/phishing.zeroprob":0.0,"cap/orig/jev/ragrelevance.accuracy":0.62,"cap/orig/jev/ragrelevance.aurc":0.3309,"cap/orig/jev/ragrelevance.brier":0.6282,"cap/orig/jev/ragrelevance.ece":0.2788,"cap/orig/jev/ragrelevance.macrof1":0.5924,"cap/orig/jev/ragrelevance.nll":1.1129,"cap/orig/jev/ragrelevance.unanswered":0,"cap/orig/jev/ragrelevance.zeroprob":0.0,"cap/orig/jev/supporttriage.accuracy":0.365,"cap/orig/jev/supporttriage.aurc":0.4868,"cap/orig/jev/supporttriage.brier":1.0557,"cap/orig/jev/supporttriage.ece":0.4827,"cap/orig/jev/supporttriage.macrof1":0.3682,"cap/orig/jev/supporttriage.nll":10.1202,"cap/orig/jev/supporttriage.unanswered":0,"cap/orig/jev/supporttriage.zeroprob":0.335,"cap/orig/jev/typeddecisions.accuracy":0.7355,"cap/orig/jev/typeddecisions.aurc":0.1277,"cap/orig/jev/typeddecisions.brier":0.3658,"cap/orig/jev/typeddecisions.ece":0.0472,"cap/orig/jev/typeddecisions.macrof1":0.7381,"cap/orig/jev/typeddecisions.nll":0.7067,"cap/orig/jev/typeddecisions.unanswered":0,"cap/orig/jev/typeddecisions.zeroprob":0.003,"cap/orig/laya/agnews.accuracy":0.95,"cap/orig/laya/agnews.aurc":0.0067,"cap/orig/laya/agnews.brier":0.0808,"cap/orig/laya/agnews.ece":0.0315,"cap/orig/laya/agnews.macrof1":0.9439,"cap/orig/laya/agnews.nll":0.1636,"cap/orig/laya/agnews.unanswered":0,"cap/orig/laya/agnews.zeroprob":0.0,"cap/orig/laya/banking77full.accuracy":0.425,"cap/orig/laya/banking77full.aurc":0.3559,"cap/orig/laya/banking77full.brier":0.9246,"cap/orig/laya/banking77full.ece":0.3809,"cap/orig/laya/banking77full.macrof1":0.1121,"cap/orig/laya/banking77full.nll":4.7967,"cap/orig/laya/banking77full.unanswered":0,"cap/orig/laya/banking77full.zeroprob":0.0975,"cap/orig/laya/beats.jev":6,"cap/orig/laya/beats.kai":3,"cap/orig/laya/emailspam.accuracy":0.9975,"cap/orig/laya/emailspam.aurc":0.0,"cap/orig/laya/emailspam.brier":0.0052,"cap/orig/laya/emailspam.ece":0.0108,"cap/orig/laya/emailspam.macrof1":0.9975,"cap/orig/laya/emailspam.nll":0.0126,"cap/orig/laya/emailspam.unanswered":0,"cap/orig/laya/emailspam.zeroprob":0.0,"cap/orig/laya/emotion.accuracy":0.595,"cap/orig/laya/emotion.aurc":0.2655,"cap/orig/laya/emotion.brier":0.6963,"cap/orig/laya/emotion.ece":0.3063,"cap/orig/laya/emotion.macrof1":0.4752,"cap/orig/laya/emotion.nll":2.0194,"cap/orig/laya/emotion.unanswered":0,"cap/orig/laya/emotion.zeroprob":0.0,"cap/orig/laya/guardrailsjailbreak.accuracy":0.705,"cap/orig/laya/guardrailsjailbreak.aurc":0.2304,"cap/orig/laya/guardrailsjailbreak.brier":0.5427,"cap/orig/laya/guardrailsjailbreak.ece":0.2617,"cap/orig/laya/guardrailsjailbreak.macrof1":0.6818,"cap/orig/laya/guardrailsjailbreak.nll":1.9121,"cap/orig/laya/guardrailsjailbreak.unanswered":0,"cap/orig/laya/guardrailsjailbreak.zeroprob":0.0025,"cap/orig/laya/massive.above3xrandom":48,"cap/orig/laya/massive.languages":51,"cap/orig/laya/massive.macroaccuracy":0.3822,"cap/orig/laya/massive.macroece":0.4184,"cap/orig/laya/massive.nonenglishmacro":0.3734,"cap/orig/laya/modelroutingdomain.accuracy":0.6391,"cap/orig/laya/modelroutingdomain.aurc":0.1758,"cap/orig/laya/modelroutingdomain.brier":0.4938,"cap/orig/laya/modelroutingdomain.ece":0.0889,"cap/orig/laya/modelroutingdomain.macrof1":0.3199,"cap/orig/laya/modelroutingdomain.nll":1.089,"cap/orig/laya/modelroutingdomain.unanswered":0,"cap/orig/laya/modelroutingdomain.zeroprob":0.0,"cap/orig/laya/moderationtoxicity.accuracy":0.53,"cap/orig/laya/moderationtoxicity.aurc":0.2811,"cap/orig/laya/moderationtoxicity.brier":0.6015,"cap/orig/laya/moderationtoxicity.ece":0.2967,"cap/orig/laya/moderationtoxicity.macrof1":0.4003,"cap/orig/laya/moderationtoxicity.nll":0.8856,"cap/orig/laya/moderationtoxicity.unanswered":0,"cap/orig/laya/moderationtoxicity.zeroprob":0.0,"cap/orig/laya/phishing.accuracy":0.9825,"cap/orig/laya/phishing.aurc":0.0005,"cap/orig/laya/phishing.brier":0.026,"cap/orig/laya/phishing.ece":0.0101,"cap/orig/laya/phishing.macrof1":0.9815,"cap/orig/laya/phishing.nll":0.0439,"cap/orig/laya/phishing.unanswered":0,"cap/orig/laya/phishing.zeroprob":0.0,"cap/orig/laya/ragrelevance.accuracy":0.625,"cap/orig/laya/ragrelevance.aurc":0.2795,"cap/orig/laya/ragrelevance.brier":0.4656,"cap/orig/laya/ragrelevance.ece":0.1175,"cap/orig/laya/ragrelevance.macrof1":0.6234,"cap/orig/laya/ragrelevance.nll":0.671,"cap/orig/laya/ragrelevance.unanswered":0,"cap/orig/laya/ragrelevance.zeroprob":0.0,"cap/orig/laya/supporttriage.accuracy":0.5025,"cap/orig/laya/supporttriage.aurc":0.3208,"cap/orig/laya/supporttriage.brier":0.6438,"cap/orig/laya/supporttriage.ece":0.0907,"cap/orig/laya/supporttriage.macrof1":0.4802,"cap/orig/laya/supporttriage.nll":1.4237,"cap/orig/laya/supporttriage.unanswered":0,"cap/orig/laya/supporttriage.zeroprob":0.0,"cap/orig/laya/typeddecisions.accuracy":0.766,"cap/orig/laya/typeddecisions.aurc":0.0988,"cap/orig/laya/typeddecisions.brier":0.3996,"cap/orig/laya/typeddecisions.ece":0.2132,"cap/orig/laya/typeddecisions.macrof1":0.7503,"cap/orig/laya/typeddecisions.nll":0.7069,"cap/orig/laya/typeddecisions.unanswered":0,"cap/orig/laya/typeddecisions.zeroprob":0.0},"detail":{"sources":{"laya":{"bundle":"hanzoai/kai-1","revision":"b50502c28537df49a3621f6fa543f9e8521e8a9c","runtime":"laya 0.3.20","device":"cpu","dtype":"f32"},"jev":{"model":"typesafe/jev-1.13","served":["typesafe/jev-1.13-20260917"]}}}}