mirror of
https://github.com/hanzoai/papers.git
synced 2026-10-30 14:58:42 +00:00
zeekay
ab5afcf18d
Hanzo CI/CD / test (linux-amd64) (push) Skipped
Hanzo CI/CD / test (linux-arm64) (push) Skipped
Hanzo CI/CD / test-advisory (push) Skipped
Build LaTeX Papers / build-pdfs (push) Failing after 31s
ci / papers (push) Failing after 23s
Hanzo CI/CD / test-amd64 (push) Successful in 1m14s
Hanzo CI/CD / cicd (push) Successful in 1m36s
CI/CD / cicd (push) Successful in 2m55s
27 lines
2.1 KiB
TeX
27 lines
2.1 KiB
TeX
% harness.tex: written by `bench tex` (hanzoai/decision); Kai runs/a7, weights 0834a74f, metal bf16; harness a1a4a73; benchmarks 039fbfe
|
|
% Laya: the checkpoint its router picks per case; typed decisions on laya-agent (kai-1-agent), the typed-decisions checkpoint Laya ships and its router does not pick.
|
|
\begin{tabular}{@{}l@{\quad}ccc@{\quad}ccc@{\quad}ccc@{}}
|
|
\toprule
|
|
& \multicolumn{3}{c}{Accuracy} & \multicolumn{3}{c}{Brier} & \multicolumn{3}{c}{ECE} \\
|
|
Suite & Kai & Laya & Jev & Kai & Laya & Jev & Kai & Laya & Jev \\
|
|
\midrule
|
|
AG News & 0.943 & \textbf{0.950} & 0.860 & 0.091 & 0.081 & 0.232 & 0.034 & 0.032 & 0.102 \\
|
|
DAIR Emotion & \textbf{0.935} & 0.595 & 0.603 & 0.080 & 0.696 & 0.646 & 0.019 & 0.306 & 0.280 \\
|
|
Banking77 & \textbf{0.907} & 0.425 & 0.835 & 0.165 & 0.925 & 0.257 & 0.076 & 0.381 & 0.073 \\
|
|
Support triage & \textbf{0.443} & 0.502$^\dagger$ & 0.365 & 0.975 & 0.644 & 1.056 & 0.441 & 0.091 & 0.483 \\
|
|
Email spam & \textbf{0.998} & \textbf{0.998} & 0.978 & 0.005 & 0.005 & 0.038 & 0.003 & 0.011 & 0.060 \\
|
|
Phishing & \textbf{0.990} & 0.983$^\dagger$ & 0.900 & 0.020 & 0.026 & 0.152 & 0.010 & 0.010 & 0.039 \\
|
|
Jailbreak & 0.920 & 0.705 & \textbf{0.940} & 0.147 & 0.543 & 0.104 & 0.072 & 0.262 & 0.047 \\
|
|
Toxicity & \textbf{0.802} & 0.530 & 0.662 & 0.384 & 0.602 & 0.432 & 0.191 & 0.297 & 0.177 \\
|
|
RAG relevance & \textbf{0.675} & 0.625 & 0.620 & 0.419 & 0.466 & 0.628 & 0.029 & 0.117 & 0.279 \\
|
|
Model routing & \textbf{1.000} & 0.639 & 0.977 & 0.000 & 0.494 & 0.040 & 0.000 & 0.089 & 0.014 \\
|
|
Typed decisions & 0.759 & \textbf{0.766} & 0.736 & 0.388 & 0.400 & 0.366 & 0.172 & 0.213 & 0.047 \\
|
|
MASSIVE (51 languages, macro) & \textbf{0.909} & 0.382 & 0.890 & 0.136 & 0.987 & 0.161 & 0.060 & 0.418 & 0.064 \\
|
|
\midrule
|
|
Macro, 11 suites & 0.852 & 0.702 & 0.770 & 0.243 & 0.444 & 0.359 & 0.095 & 0.164 & 0.145 \\
|
|
Clean macro, 9 suites & 0.882 & 0.693 & 0.801 & & & & & & \\
|
|
Best of three, 12 suites & 9 & 3 & 1 & & & & & & \\
|
|
\bottomrule
|
|
\multicolumn{10}{@{}l@{}}{\footnotesize\begin{tabular}[t]{@{}l@{}}Kai runs/a7, weights 0834a74f, metal bf16 \\ harness a1a4a73, benchmarks 039fbfe\end{tabular}} \\
|
|
\end{tabular}
|