benchmarks/decision/capability/results/conformal.json

{"meta":{"run 2026-09-27 11:15:11 +0000":{"host":"arm64-mac","machine":"aarch64","runtime":"bench (hanzo-ml)","load":[20.61572265625,16.89453125,17.15087890625],"at":"2026-09-27 11:15:11 +0000","splits":20},"run 2026-09-27 14:44:27 +0000":{"host":"arm64-mac","machine":"aarch64","runtime":"bench (hanzo-ml)","load":[10.6669921875,9.20849609375,10.0126953125],"at":"2026-09-27 14:44:27 +0000","splits":20}},"keys":{"cap/conformal/jev/apps.accuracy":0.7739,"cap/conformal/jev/apps.aurc":0.1185,"cap/conformal/jev/apps.brier":0.3587,"cap/conformal/jev/apps.conf90.cover":0.9075,"cap/conformal/jev/apps.conf90.size":1.5938,"cap/conformal/jev/apps.conf95.cover":1.0,"cap/conformal/jev/apps.conf95.size":11.4111,"cap/conformal/jev/apps.ece":0.1209,"cap/conformal/jev/apps.n":3999,"cap/conformal/jev/apps.nll":2.0313,"cap/conformal/jev/apps.risk50":0.1047,"cap/conformal/jev/apps.risk75":0.1471,"cap/conformal/jev/apps.risk90":0.1916,"cap/conformal/jev/apps.unanswered":0,"cap/conformal/jev/aurc":0.061,"cap/conformal/jev/choice.accuracy":0.885,"cap/conformal/jev/choice.aurc":0.0304,"cap/conformal/jev/choice.brier":0.1815,"cap/conformal/jev/choice.conf90.cover":0.9054,"cap/conformal/jev/choice.conf90.size":1.0577,"cap/conformal/jev/choice.conf95.cover":0.9537,"cap/conformal/jev/choice.conf95.size":1.3462,"cap/conformal/jev/choice.ece":0.0446,"cap/conformal/jev/choice.n":1600,"cap/conformal/jev/choice.nll":0.8282,"cap/conformal/jev/choice.risk50":0.0145,"cap/conformal/jev/choice.risk75":0.0383,"cap/conformal/jev/choice.risk90":0.0733,"cap/conformal/jev/choice.unanswered":0,"cap/conformal/jev/cov90":0.9007,"cap/conformal/jev/cov95":0.9522,"cap/conformal/jev/ece":0.0492,"cap/conformal/jev/joint.accuracy":0.837,"cap/conformal/jev/joint.aurc":0.0438,"cap/conformal/jev/joint.brier":0.2224,"cap/conformal/jev/joint.conf90.cover":0.9032,"cap/conformal/jev/joint.conf90.size":1.1401,"cap/conformal/jev/joint.conf95.cover":0.953,"cap/conformal/jev/joint.conf95.size":1.3056,"cap/conformal/jev/joint.ece":0.0375,"cap/conformal/jev/joint.n":2000,"cap/conformal/jev/joint.nll":0.3634,"cap/conformal/jev/joint.risk50":0.0096,"cap/conformal/jev/joint.risk75":0.0899,"cap/conformal/jev/joint.risk90":0.1308,"cap/conformal/jev/joint.unanswered":0,"cap/conformal/jev/massive.accuracy":0.8902,"cap/conformal/jev/massive.aurc":0.0162,"cap/conformal/jev/massive.brier":0.1606,"cap/conformal/jev/massive.conf90.cover":0.9027,"cap/conformal/jev/massive.conf90.size":1.0288,"cap/conformal/jev/massive.conf95.cover":0.9538,"cap/conformal/jev/massive.conf95.size":1.4405,"cap/conformal/jev/massive.ece":0.0184,"cap/conformal/jev/massive.n":5099,"cap/conformal/jev/massive.nll":0.7755,"cap/conformal/jev/massive.risk50":0.0007,"cap/conformal/jev/massive.risk75":0.023,"cap/conformal/jev/massive.risk90":0.0547,"cap/conformal/jev/massive.unanswered":1,"cap/conformal/jev/orig.accuracy":0.8204,"cap/conformal/jev/orig.aurc":0.061,"cap/conformal/jev/orig.brier":0.2689,"cap/conformal/jev/orig.conf90.cover":0.9007,"cap/conformal/jev/orig.conf90.size":1.3043,"cap/conformal/jev/orig.conf95.cover":0.9522,"cap/conformal/jev/orig.conf95.size":1.9188,"cap/conformal/jev/orig.ece":0.0492,"cap/conformal/jev/orig.n":11098,"cap/conformal/jev/orig.nll":1.2156,"cap/conformal/jev/orig.risk50":0.0475,"cap/conformal/jev/orig.risk75":0.0918,"cap/conformal/jev/orig.risk90":0.1365,"cap/conformal/jev/orig.unanswered":1,"cap/conformal/jev/size90":1.3043,"cap/conformal/jev/size95":1.9188,"cap/conformal/jev/td.accuracy":0.7355,"cap/conformal/jev/td.aurc":0.1277,"cap/conformal/jev/td.brier":0.3658,"cap/conformal/jev/td.conf90.cover":0.9005,"cap/conformal/jev/td.conf90.size":1.5369,"cap/conformal/jev/td.conf95.cover":0.9521,"cap/conformal/jev/td.conf95.size":1.9118,"cap/conformal/jev/td.ece":0.0472,"cap/conformal/jev/td.n":2000,"cap/conformal/jev/td.nll":0.7067,"cap/conformal/jev/td.risk50":0.1142,"cap/conformal/jev/td.risk75":0.1859,"cap/conformal/jev/td.risk90":0.2314,"cap/conformal/jev/td.unanswered":0,"cap/conformal/laya/apps.accuracy":0.6952,"cap/conformal/laya/apps.aurc":0.1489,"cap/conformal/laya/apps.brier":0.448,"cap/conformal/laya/apps.conf90.cover":0.9004,"cap/conformal/laya/apps.conf90.size":2.1446,"cap/conformal/laya/apps.conf95.cover":0.9501,"cap/conformal/laya/apps.conf95.size":4.391,"cap/conformal/laya/apps.ece":0.1455,"cap/conformal/laya/apps.n":3999,"cap/conformal/laya/apps.nll":1.3018,"cap/conformal/laya/apps.risk50":0.1406,"cap/conformal/laya/apps.risk75":0.2071,"cap/conformal/laya/apps.risk90":0.2659,"cap/conformal/laya/apps.unanswered":0,"cap/conformal/laya/aurc":0.2999,"cap/conformal/laya/choice.accuracy":0.6206,"cap/conformal/laya/choice.aurc":0.2383,"cap/conformal/laya/choice.brier":0.6299,"cap/conformal/laya/choice.conf90.cover":0.9057,"cap/conformal/laya/choice.conf90.size":14.6707,"cap/conformal/laya/choice.conf95.cover":0.9544,"cap/conformal/laya/choice.conf95.size":26.0801,"cap/conformal/laya/choice.ece":0.2255,"cap/conformal/laya/choice.n":1600,"cap/conformal/laya/choice.nll":3.013,"cap/conformal/laya/choice.risk50":0.2712,"cap/conformal/laya/choice.risk75":0.3042,"cap/conformal/laya/choice.risk90":0.3451,"cap/conformal/laya/choice.unanswered":0,"cap/conformal/laya/cov90":0.9015,"cap/conformal/laya/cov95":0.9505,"cap/conformal/laya/ece":0.202,"cap/conformal/laya/joint.accuracy":0.336,"cap/conformal/laya/joint.aurc":0.6183,"cap/conformal/laya/joint.brier":0.7183,"cap/conformal/laya/joint.conf90.cover":0.8979,"cap/conformal/laya/joint.conf90.size":2.9477,"cap/conformal/laya/joint.conf95.cover":0.9457,"cap/conformal/laya/joint.conf95.size":3.1041,"cap/conformal/laya/joint.ece":0.0908,"cap/conformal/laya/joint.n":2000,"cap/conformal/laya/joint.nll":1.2587,"cap/conformal/laya/joint.risk50":0.626,"cap/conformal/laya/joint.risk75":0.644,"cap/conformal/laya/joint.risk90":0.6628,"cap/conformal/laya/joint.unanswered":0,"cap/conformal/laya/massive.accuracy":0.3822,"cap/conformal/laya/massive.aurc":0.4376,"cap/conformal/laya/massive.brier":0.987,"cap/conformal/laya/massive.conf90.cover":0.9015,"cap/conformal/laya/massive.conf90.size":13.9462,"cap/conformal/laya/massive.conf95.cover":0.9498,"cap/conformal/laya/massive.conf95.size":16.6606,"cap/conformal/laya/massive.ece":0.4096,"cap/conformal/laya/massive.n":5100,"cap/conformal/laya/massive.nll":3.8282,"cap/conformal/laya/massive.risk50":0.4651,"cap/conformal/laya/massive.risk75":0.5535,"cap/conformal/laya/massive.risk90":0.5891,"cap/conformal/laya/massive.unanswered":0,"cap/conformal/laya/orig.accuracy":0.5641,"cap/conformal/laya/orig.aurc":0.2999,"cap/conformal/laya/orig.brier":0.687,"cap/conformal/laya/orig.conf90.cover":0.9015,"cap/conformal/laya/orig.conf90.size":6.8007,"cap/conformal/laya/orig.conf95.cover":0.9505,"cap/conformal/laya/orig.conf95.size":9.6469,"cap/conformal/laya/orig.ece":0.202,"cap/conformal/laya/orig.n":11099,"cap/conformal/laya/orig.nll":2.3555,"cap/conformal/laya/orig.risk50":0.3296,"cap/conformal/laya/orig.risk75":0.3871,"cap/conformal/laya/orig.risk90":0.4064,"cap/conformal/laya/orig.unanswered":0,"cap/conformal/laya/size90":6.8007,"cap/conformal/laya/size95":9.6469,"cap/conformal/laya/td.accuracy":0.766,"cap/conformal/laya/td.aurc":0.0988,"cap/conformal/laya/td.brier":0.3996,"cap/conformal/laya/td.conf90.cover":0.9009,"cap/conformal/laya/td.conf90.size":1.4342,"cap/conformal/laya/td.conf95.cover":0.9514,"cap/conformal/laya/td.conf95.size":1.6718,"cap/conformal/laya/td.ece":0.2132,"cap/conformal/laya/td.n":2000,"cap/conformal/laya/td.nll":0.7069,"cap/conformal/laya/td.risk50":0.097,"cap/conformal/laya/td.risk75":0.1647,"cap/conformal/laya/td.risk90":0.2044,"cap/conformal/laya/td.unanswered":0},"detail":{"apps":{"laya":{"n":3999,"accuracy":0.6952,"macro_f1":0.1939,"ece":0.1455,"brier":0.448,"nll":1.3018,"mean_confidence":0.8339,"acc_at_50_coverage":0.8594,"aurc":0.1489,"risk_at_coverage":{"0.1":0.0426,"0.25":0.0731,"0.5":0.1406,"0.75":0.2071,"0.9":0.2659,"1.0":0.3048},"conf90.cover":0.9004000000000001,"conf90.size":2.1446250000000004,"conf95.cover":0.9501249999999999,"conf95.size":4.391,"unanswered":0},"jev":{"n":3999,"accuracy":0.7739,"macro_f1":0.3658,"ece":0.1209,"brier":0.3587,"nll":2.0313,"mean_confidence":0.8942,"acc_at_50_coverage":0.8914,"aurc":0.1185,"risk_at_coverage":{"0.1":0.0707,"0.25":0.0795,"0.5":0.1047,"0.75":0.1471,"0.9":0.1916,"1.0":0.2261},"conf90.cover":0.907475,"conf90.size":1.5938,"conf95.cover":1.0,"conf95.size":11.411075,"unanswered":0}},"orig":{"laya":{"n":11099,"accuracy":0.5641,"macro_f1":0.2029,"ece":0.202,"brier":0.687,"nll":2.3555,"mean_confidence":0.7639,"acc_at_50_coverage":0.6704,"aurc":0.2999,"risk_at_coverage":{"0.1":0.0992,"0.25":0.2343,"0.5":0.3296,"0.75":0.3871,"0.9":0.4064,"1.0":0.4359},"conf90.cover":0.9015405405405407,"conf90.size":6.800666666666667,"conf95.cover":0.9505225225225227,"conf95.size":9.6469009009009,"unanswered":0},"jev":{"n":11098,"accuracy":0.8204,"macro_f1":0.513,"ece":0.0492,"brier":0.2689,"nll":1.2156,"mean_confidence":0.8696,"acc_at_50_coverage":0.9506,"aurc":0.061,"risk_at_coverage":{"0.1":0.0182,"0.25":0.0182,"0.5":0.0475,"0.75":0.0918,"0.9":0.1365,"1.0":0.1796},"conf90.cover":0.9007478825013517,"conf90.size":1.3042980717246349,"conf95.cover":0.95218057307623,"conf95.size":1.9187691475941608,"unanswered":1}},"td":{"laya":{"n":2000,"accuracy":0.766,"macro_f1":0.7503,"ece":0.2132,"brier":0.3996,"nll":0.7069,"mean_confidence":0.5528,"acc_at_50_coverage":0.903,"aurc":0.0988,"risk_at_coverage":{"0.1":0.005,"0.25":0.028,"0.5":0.097,"0.75":0.1647,"0.9":0.2044,"1.0":0.234},"conf90.cover":0.9009,"conf90.size":1.4342000000000001,"conf95.cover":0.9514499999999998,"conf95.size":1.67175,"unanswered":0},"jev":{"n":2000,"accuracy":0.7355,"macro_f1":0.7381,"ece":0.0472,"brier":0.3658,"nll":0.7067,"mean_confidence":0.7567,"acc_at_50_coverage":0.885,"aurc":0.1277,"risk_at_coverage":{"0.1":0.025,"0.25":0.0769,"0.5":0.1142,"0.75":0.1859,"0.9":0.2314,"1.0":0.2645},"conf90.cover":0.9004999999999999,"conf90.size":1.5369000000000002,"conf95.cover":0.9521499999999998,"conf95.size":1.9118499999999998,"unanswered":0}},"massive":{"laya":{"n":5100,"accuracy":0.3822,"macro_f1":0.3629,"ece":0.4096,"brier":0.987,"nll":3.8282,"mean_confidence":0.7917,"acc_at_50_coverage":0.5349,"aurc":0.4376,"risk_at_coverage":{"0.1":0.2725,"0.25":0.342,"0.5":0.4651,"0.75":0.5535,"0.9":0.5891,"1.0":0.6178},"conf90.cover":0.9015098039215687,"conf90.size":13.946235294117646,"conf95.cover":0.9498235294117647,"conf95.size":16.660627450980392,"unanswered":0},"jev":{"n":5099,"accuracy":0.8902,"macro_f1":0.8851,"ece":0.0184,"brier":0.1606,"nll":0.7755,"mean_confidence":0.8946,"acc_at_50_coverage":0.9992,"aurc":0.0162,"risk_at_coverage":{"0.1":0.0007,"0.25":0.0007,"0.5":0.0007,"0.75":0.023,"0.9":0.0547,"1.0":0.1098},"conf90.cover":0.9027254901960786,"conf90.size":1.0288431372549018,"conf95.cover":0.953843137254902,"conf95.size":1.4404509803921566,"unanswered":1}},"joint":{"laya":{"n":2000,"accuracy":0.336,"macro_f1":0.3372,"ece":0.0908,"brier":0.7183,"nll":1.2587,"mean_confidence":0.424,"acc_at_50_coverage":0.374,"aurc":0.6183,"risk_at_coverage":{"0.1":0.62,"0.25":0.55,"0.5":0.626,"0.75":0.644,"0.9":0.6628,"1.0":0.664},"conf90.cover":0.8979000000000001,"conf90.size":2.9476999999999998,"conf95.cover":0.9456999999999999,"conf95.size":3.1041,"unanswered":0},"jev":{"n":2000,"accuracy":0.837,"macro_f1":0.8933,"ece":0.0375,"brier":0.2224,"nll":0.3634,"mean_confidence":0.8188,"acc_at_50_coverage":0.992,"aurc":0.0438,"risk_at_coverage":{"0.1":0.0,"0.25":0.0,"0.5":0.0096,"0.75":0.0899,"0.9":0.1308,"1.0":0.163},"conf90.cover":0.9032,"conf90.size":1.14005,"conf95.cover":0.95305,"conf95.size":1.3055999999999999,"unanswered":0}},"choice":{"laya":{"n":1600,"accuracy":0.6206,"macro_f1":0.5009,"ece":0.2255,"brier":0.6299,"nll":3.013,"mean_confidence":0.8437,"acc_at_50_coverage":0.7288,"aurc":0.2383,"risk_at_coverage":{"0.1":0.0875,"0.25":0.1875,"0.5":0.2712,"0.75":0.3042,"0.9":0.3451,"1.0":0.3794},"conf90.cover":0.9056875,"conf90.size":14.6706875,"conf95.cover":0.9544374999999998,"conf95.size":26.080125,"unanswered":0},"jev":{"n":1600,"accuracy":0.885,"macro_f1":0.8901,"ece":0.0446,"brier":0.1815,"nll":0.8282,"mean_confidence":0.9233,"acc_at_50_coverage":0.9838,"aurc":0.0304,"risk_at_coverage":{"0.1":0.0145,"0.25":0.0145,"0.5":0.0145,"0.75":0.0383,"0.9":0.0733,"1.0":0.115},"conf90.cover":0.9054375,"conf90.size":1.05775,"conf95.cover":0.9536875,"conf95.size":1.3461874999999999,"unanswered":0}}}}