mirror of
https://github.com/hanzoai/papers.git
synced 2026-10-22 17:20:17 +00:00
Kai: the result tables from stage a6 on the frozen harness, the capability suites and speed on dgx.
Hanzo CI/CD / test (linux-amd64) (push) Skipped
Hanzo CI/CD / test (linux-arm64) (push) Skipped
Hanzo CI/CD / test-advisory (push) Skipped
ci / papers (push) Failing after 23s
Build LaTeX Papers / build-pdfs (push) Failing after 28s
Hanzo CI/CD / test-amd64 (push) Successful in 1m28s
Hanzo CI/CD / cicd (push) Successful in 1m59s
CI/CD / cicd (push) Successful in 3m32s
Hanzo CI/CD / test (linux-amd64) (push) Skipped
Hanzo CI/CD / test (linux-arm64) (push) Skipped
Hanzo CI/CD / test-advisory (push) Skipped
ci / papers (push) Failing after 23s
Build LaTeX Papers / build-pdfs (push) Failing after 28s
Hanzo CI/CD / test-amd64 (push) Successful in 1m28s
Hanzo CI/CD / cicd (push) Successful in 1m59s
CI/CD / cicd (push) Successful in 3m32s
Claude-Session: https://claude.ai/code/session_018Gt5TvrE2rxXuZikWSauv3
This commit is contained in:
@@ -1,3 +1,45 @@
|
||||
% tables/capability.tex: stub. `bench tex --results <benchmarks>/decision --out <paper>/kai/tables`
|
||||
% (hanzoai/decision) overwrites this file with the table once the final checkpoint is scored.
|
||||
\pending{Cardinality accuracy at 4 to 100{,}000 options and share rejected; joint accuracy, exact match and rule violations, as answered and projected; permutation flips and largest probability change; \op{noul} swap mirrors; media; conformal coverage and set size at 90\%; refresh questions asked; offline operation. For Kai, Laya and Jev.}
|
||||
% capability.tex: written by `bench tex` (hanzoai/decision); Kai runs/a6, weights a211cc70, cpu f32; harness a1a4a73; benchmarks e418767
|
||||
% Kai: joint rows are its refinement over three passes (projected: its own selection and projection); Laya's joint rows are its typed-decisions checkpoint; media is Laya as routed. Rejected: questions a backend refuses, over all asked.
|
||||
\begin{tabular}{@{}lccc@{}}
|
||||
\toprule
|
||||
& Kai & Laya & Jev \\
|
||||
\midrule
|
||||
\multicolumn{4}{@{}l}{\emph{Cardinality}} \\
|
||||
Accuracy, $K = 4$ & 0.685 & 0.777 & 0.905 \\
|
||||
Accuracy, $K = 16$ & 0.395 & 0.670 & 0.853 \\
|
||||
Accuracy, $K = 77$ & 0.255 & 0.425 & 0.843 \\
|
||||
Accuracy, $K = 150$ & 0.287 & 0.610 & 0.940 \\
|
||||
Accuracy, $K = 1{,}000$ & 0.005 & 0.000 & 0.000 \\
|
||||
Accuracy, $K = 10{,}000$ & 0.000 & 0.000 & 0.000 \\
|
||||
Accuracy, $K = 100{,}000$ & 0.000 & 0.000 & 0.000 \\
|
||||
Rejected, $K = 1{,}000$ & 0.000 & 1.000 & 1.000 \\
|
||||
Rejected, $K = 10{,}000$ & 0.000 & 1.000 & 1.000 \\
|
||||
Rejected, $K = 100{,}000$ & 0.000 & 1.000 & 1.000 \\
|
||||
\midrule
|
||||
\multicolumn{4}{@{}l}{\emph{Joint coherence, five dependent questions}} \\
|
||||
Accuracy & 0.306 & 0.336 & 0.837 \\
|
||||
Exact & 0.000 & 0.005 & 0.438 \\
|
||||
Violations & 1.000 & 0.637 & 0.247 \\
|
||||
Accuracy, projected & 0.341 & 0.328 & 0.879 \\
|
||||
Exact, projected & 0.010 & 0.040 & 0.560 \\
|
||||
\midrule
|
||||
\multicolumn{4}{@{}l}{\emph{Invariance}} \\
|
||||
Permutation, flip & 0.000 & 0.255 & 0.060 \\
|
||||
Permutation, max $\Delta p$ & 0.000 & 0.996 & 0.470 \\
|
||||
Noul sides swapped, mirror & 0.915 & 0.205 & 0.062 \\
|
||||
Noul negated, mirror & 0.430 & 0.300 & 0.882 \\
|
||||
\midrule
|
||||
\multicolumn{4}{@{}l}{\emph{Media, accuracy}} \\
|
||||
Human activity (HAR) & 0.160 & 0.203 & 0.153 \\
|
||||
Turbofan life (C-MAPSS) & 0.330 & 0.170 & 0.300 \\
|
||||
Valve sound (MIMII) & 0.500 & 0.500 & 0.840 \\
|
||||
\midrule
|
||||
\multicolumn{4}{@{}l}{\emph{Conformal sets, 90\% target}} \\
|
||||
Coverage & 0.901 & 0.901 & 0.901 \\
|
||||
Mean set size & 2.498 & 6.801 & 1.304 \\
|
||||
\midrule
|
||||
Refresh: questions asked / full & 6 / 101 & 100 / 100 & 100 / 100 \\
|
||||
Runs offline & yes & yes & no \\
|
||||
\bottomrule
|
||||
\multicolumn{4}{@{}l@{}}{\footnotesize\begin{tabular}[t]{@{}l@{}}Kai runs/a6, weights a211cc70, cpu f32 \\ harness a1a4a73, benchmarks e418767\end{tabular}} \\
|
||||
\end{tabular}
|
||||
|
||||
+15
-17
@@ -1,26 +1,24 @@
|
||||
% harness.tex: written by `bench tex` (hanzoai/decision); Kai runs/a5, weights df1c16a2, metal bf16; harness 735e240; benchmarks c68a38b
|
||||
% harness.tex: written by `bench tex` (hanzoai/decision); Kai runs/a6, weights a211cc70, cpu f32; harness a1a4a73; benchmarks e418767
|
||||
% Laya: the checkpoint its router picks per case; typed decisions on laya-agent (kai-1-agent), the typed-decisions checkpoint Laya ships and its router does not pick.
|
||||
\begin{tabular}{@{}l@{\quad}ccc@{\quad}ccc@{\quad}ccc@{}}
|
||||
\toprule
|
||||
& \multicolumn{3}{c}{Accuracy} & \multicolumn{3}{c}{Brier} & \multicolumn{3}{c}{ECE} \\
|
||||
Suite & Kai & Laya & Jev & Kai & Laya & Jev & Kai & Laya & Jev \\
|
||||
\midrule
|
||||
AG News & \textbf{0.950} & \textbf{0.950} & 0.860 & 0.099 & 0.081 & 0.232 & 0.070 & 0.032 & 0.102 \\
|
||||
DAIR Emotion & \textbf{0.925} & 0.595 & 0.603 & 0.089 & 0.696 & 0.646 & 0.041 & 0.306 & 0.280 \\
|
||||
Banking77 & \textbf{0.907} & 0.425 & 0.835 & 0.155 & 0.925 & 0.257 & 0.065 & 0.381 & 0.073 \\
|
||||
Support triage & \textbf{0.490} & 0.502$^\dagger$ & 0.365 & 0.705 & 0.644 & 1.056 & 0.161 & 0.091 & 0.483 \\
|
||||
Email spam & 0.995 & \textbf{0.998} & 0.978 & 0.004 & 0.005 & 0.038 & 0.004 & 0.011 & 0.060 \\
|
||||
Phishing & \textbf{0.990} & 0.983$^\dagger$ & 0.900 & 0.020 & 0.026 & 0.152 & 0.010 & 0.010 & 0.039 \\
|
||||
Jailbreak & 0.915 & 0.705 & \textbf{0.940} & 0.143 & 0.543 & 0.104 & 0.055 & 0.262 & 0.047 \\
|
||||
Toxicity & \textbf{0.820} & 0.530 & 0.662 & 0.339 & 0.602 & 0.432 & 0.167 & 0.297 & 0.177 \\
|
||||
RAG relevance & \textbf{0.667} & 0.625 & 0.620 & 0.418 & 0.466 & 0.628 & 0.037 & 0.117 & 0.279 \\
|
||||
Model routing & \textbf{1.000} & 0.639 & 0.977 & 0.000 & 0.494 & 0.040 & 0.002 & 0.089 & 0.014 \\
|
||||
Typed decisions & 0.705 & \textbf{0.766} & 0.736 & 0.449 & 0.400 & 0.366 & 0.171 & 0.213 & 0.047 \\
|
||||
MASSIVE (51 languages, macro) & 0.888 & 0.382 & \textbf{0.890} & 0.157 & 0.987 & 0.161 & 0.066 & 0.418 & 0.064 \\
|
||||
AG News & 0.943 & 0.950 & 0.860 & 0.092 & 0.081 & 0.232 & 0.027 & 0.032 & 0.102 \\
|
||||
DAIR Emotion & 0.940 & 0.595 & 0.603 & 0.080 & 0.696 & 0.646 & 0.022 & 0.306 & 0.280 \\
|
||||
Banking77 & 0.922 & 0.425 & 0.835 & 0.143 & 0.925 & 0.257 & 0.067 & 0.381 & 0.073 \\
|
||||
Support triage & 0.427 & 0.502 & 0.365 & 0.962 & 0.644 & 1.056 & 0.441 & 0.091 & 0.483 \\
|
||||
Email spam & 0.998 & 0.998 & 0.978 & 0.004 & 0.005 & 0.038 & 0.002 & 0.011 & 0.060 \\
|
||||
Phishing & 0.993 & 0.983 & 0.900 & 0.015 & 0.026 & 0.152 & 0.007 & 0.010 & 0.039 \\
|
||||
Jailbreak & 0.920 & 0.705 & 0.940 & 0.142 & 0.543 & 0.104 & 0.067 & 0.262 & 0.047 \\
|
||||
Toxicity & 0.807 & 0.530 & 0.662 & 0.370 & 0.602 & 0.432 & 0.185 & 0.297 & 0.177 \\
|
||||
RAG relevance & 0.650 & 0.625 & 0.620 & 0.413 & 0.466 & 0.628 & 0.055 & 0.117 & 0.279 \\
|
||||
Model routing & 1.000 & 0.639 & 0.977 & 0.000 & 0.494 & 0.040 & 0.000 & 0.089 & 0.014 \\
|
||||
Typed decisions & 0.751 & 0.766 & 0.736 & 0.427 & 0.400 & 0.366 & 0.209 & 0.213 & 0.047 \\
|
||||
MASSIVE (51 languages, macro) & 0.910 & 0.382 & 0.890 & 0.136 & 0.987 & 0.161 & 0.058 & 0.418 & 0.064 \\
|
||||
\midrule
|
||||
Macro, 11 suites & 0.851 & 0.702 & 0.770 & 0.220 & 0.444 & 0.359 & 0.071 & 0.164 & 0.145 \\
|
||||
Clean macro, 9 suites & 0.876 & 0.693 & 0.801 & & & & & & \\
|
||||
Best of three, 12 suites & 8 & 3 & 2 & & & & & & \\
|
||||
Macro, 11 suites & 0.850 & 0.702 & 0.770 & 0.241 & 0.444 & 0.359 & 0.099 & 0.164 & 0.145 \\
|
||||
\bottomrule
|
||||
\multicolumn{10}{@{}l@{}}{\footnotesize\begin{tabular}[t]{@{}l@{}}Kai runs/a5, weights df1c16a2, metal bf16 \\ harness 735e240, benchmarks c68a38b\end{tabular}} \\
|
||||
\multicolumn{10}{@{}l@{}}{\footnotesize\begin{tabular}[t]{@{}l@{}}Kai runs/a6, weights a211cc70, cpu f32 \\ harness a1a4a73, benchmarks e418767\end{tabular}} \\
|
||||
\end{tabular}
|
||||
|
||||
+28
-28
@@ -1,35 +1,35 @@
|
||||
% languages.tex: written by `bench tex` (hanzoai/decision); Kai runs/a5, weights df1c16a2, metal bf16; harness 735e240; benchmarks c68a38b
|
||||
% languages.tex: written by `bench tex` (hanzoai/decision); Kai runs/a6, weights a211cc70, cpu f32; harness a1a4a73; benchmarks e418767
|
||||
% MASSIVE intent accuracy by language, 20 options, 100 utterances each; Laya as its router picks.
|
||||
\begin{tabular}{@{}lccc@{\qquad}lccc@{}}
|
||||
\toprule
|
||||
Language & Kai & Laya & Jev & Language & Kai & Laya & Jev \\
|
||||
\midrule
|
||||
af & 0.890 & 0.340 & 0.870 & ko & 0.920 & 0.470 & 0.940 \\
|
||||
am & 0.840 & 0.150 & 0.850 & lv & 0.850 & 0.300 & 0.850 \\
|
||||
ar & 0.880 & 0.460 & 0.870 & ml & 0.820 & 0.280 & 0.910 \\
|
||||
az & 0.920 & 0.360 & 0.880 & mn & 0.880 & 0.160 & 0.840 \\
|
||||
bn & 0.870 & 0.450 & 0.860 & ms & 0.920 & 0.270 & 0.920 \\
|
||||
cy & 0.820 & 0.160 & 0.760 & my & 0.900 & 0.160 & 0.890 \\
|
||||
da & 0.910 & 0.460 & 0.880 & nb & 0.910 & 0.500 & 0.860 \\
|
||||
de & 0.920 & 0.530 & 0.939 & nl & 0.920 & 0.400 & 0.900 \\
|
||||
el & 0.900 & 0.440 & 0.940 & pl & 0.940 & 0.470 & 0.920 \\
|
||||
en & 0.920 & 0.820 & 0.920 & pt & 0.920 & 0.440 & 0.900 \\
|
||||
es & 0.900 & 0.530 & 0.930 & ro & 0.900 & 0.370 & 0.940 \\
|
||||
fa & 0.930 & 0.510 & 0.950 & ru & 0.960 & 0.570 & 0.940 \\
|
||||
fi & 0.920 & 0.300 & 0.890 & sl & 0.900 & 0.330 & 0.890 \\
|
||||
fr & 0.920 & 0.600 & 0.900 & sq & 0.880 & 0.290 & 0.760 \\
|
||||
he & 0.870 & 0.370 & 0.940 & sv & 0.880 & 0.480 & 0.940 \\
|
||||
hi & 0.930 & 0.460 & 0.950 & sw & 0.810 & 0.130 & 0.700 \\
|
||||
hu & 0.870 & 0.340 & 0.920 & ta & 0.840 & 0.310 & 0.920 \\
|
||||
hy & 0.840 & 0.250 & 0.840 & te & 0.830 & 0.220 & 0.910 \\
|
||||
id & 0.920 & 0.360 & 0.930 & th & 0.930 & 0.480 & 0.940 \\
|
||||
is & 0.860 & 0.330 & 0.840 & tl & 0.860 & 0.300 & 0.850 \\
|
||||
it & 0.940 & 0.470 & 0.960 & tr & 0.880 & 0.410 & 0.920 \\
|
||||
ja & 0.940 & 0.640 & 0.970 & ur & 0.860 & 0.420 & 0.900 \\
|
||||
jv & 0.830 & 0.160 & 0.750 & vi & 0.890 & 0.330 & 0.890 \\
|
||||
ka & 0.790 & 0.150 & 0.880 & zh-CN & 0.930 & 0.650 & 0.940 \\
|
||||
km & 0.800 & 0.200 & 0.810 & zh-TW & 0.970 & 0.610 & 0.930 \\
|
||||
kn & 0.870 & 0.300 & 0.870 & & & & \\
|
||||
af & 0.900 & 0.340 & 0.870 & ko & 0.920 & 0.470 & 0.940 \\
|
||||
am & 0.850 & 0.150 & 0.850 & lv & 0.890 & 0.300 & 0.850 \\
|
||||
ar & 0.930 & 0.460 & 0.870 & ml & 0.870 & 0.280 & 0.910 \\
|
||||
az & 0.950 & 0.360 & 0.880 & mn & 0.880 & 0.160 & 0.840 \\
|
||||
bn & 0.900 & 0.450 & 0.860 & ms & 0.920 & 0.270 & 0.920 \\
|
||||
cy & 0.860 & 0.160 & 0.760 & my & 0.950 & 0.160 & 0.890 \\
|
||||
da & 0.920 & 0.460 & 0.880 & nb & 0.920 & 0.500 & 0.860 \\
|
||||
de & 0.920 & 0.530 & 0.939 & nl & 0.930 & 0.400 & 0.900 \\
|
||||
el & 0.930 & 0.440 & 0.940 & pl & 0.930 & 0.470 & 0.920 \\
|
||||
en & 0.940 & 0.820 & 0.920 & pt & 0.920 & 0.440 & 0.900 \\
|
||||
es & 0.920 & 0.530 & 0.930 & ro & 0.950 & 0.370 & 0.940 \\
|
||||
fa & 0.950 & 0.510 & 0.950 & ru & 0.980 & 0.570 & 0.940 \\
|
||||
fi & 0.930 & 0.300 & 0.890 & sl & 0.890 & 0.330 & 0.890 \\
|
||||
fr & 0.930 & 0.600 & 0.900 & sq & 0.920 & 0.290 & 0.760 \\
|
||||
he & 0.910 & 0.370 & 0.940 & sv & 0.930 & 0.480 & 0.940 \\
|
||||
hi & 0.920 & 0.460 & 0.950 & sw & 0.830 & 0.130 & 0.700 \\
|
||||
hu & 0.870 & 0.340 & 0.920 & ta & 0.910 & 0.310 & 0.920 \\
|
||||
hy & 0.870 & 0.250 & 0.840 & te & 0.860 & 0.220 & 0.910 \\
|
||||
id & 0.930 & 0.360 & 0.930 & th & 0.930 & 0.480 & 0.940 \\
|
||||
is & 0.870 & 0.330 & 0.840 & tl & 0.890 & 0.300 & 0.850 \\
|
||||
it & 0.930 & 0.470 & 0.960 & tr & 0.900 & 0.410 & 0.920 \\
|
||||
ja & 0.970 & 0.640 & 0.970 & ur & 0.920 & 0.420 & 0.900 \\
|
||||
jv & 0.870 & 0.160 & 0.750 & vi & 0.930 & 0.330 & 0.890 \\
|
||||
ka & 0.830 & 0.150 & 0.880 & zh-CN & 0.950 & 0.650 & 0.940 \\
|
||||
km & 0.850 & 0.200 & 0.810 & zh-TW & 0.960 & 0.610 & 0.930 \\
|
||||
kn & 0.900 & 0.300 & 0.870 & & & & \\
|
||||
\bottomrule
|
||||
\multicolumn{8}{@{}l@{}}{\footnotesize\begin{tabular}[t]{@{}l@{}}Kai runs/a5, weights df1c16a2, metal bf16 \\ harness 735e240, benchmarks c68a38b\end{tabular}} \\
|
||||
\multicolumn{8}{@{}l@{}}{\footnotesize\begin{tabular}[t]{@{}l@{}}Kai runs/a6, weights a211cc70, cpu f32 \\ harness a1a4a73, benchmarks e418767\end{tabular}} \\
|
||||
\end{tabular}
|
||||
|
||||
+25
-3
@@ -1,3 +1,25 @@
|
||||
% tables/speed.tex: stub. `bench tex --results <benchmarks>/decision --out <paper>/kai/tables`
|
||||
% (hanzoai/decision) overwrites this file with the table once the final checkpoint is scored.
|
||||
\pending{Latency per call (p50, p95) and questions per second at 1, 10, 50, 100, 500 and 1{,}000 questions per state, on Metal and on CPU, for Kai, Laya and Jev.}
|
||||
% speed.tex: written by `bench tex` (hanzoai/decision); Kai runs/a, encoder 768 wide, 22 layers; timed F32 on Metal and on the CPU, Kai's CPU encoder projections Q8_0; harness a1a4a73; benchmarks e418767
|
||||
% Kai and Laya on this runtime (hanzo-ml), same device and dtype (F32; on the CPU Kai's encoder projections are Q8_0, as Decider::load serves them), their calls interleaved; the last row names each checkpoint and its encoder. Jev is one network run, the same cells in both blocks.
|
||||
\begin{tabular}{@{}r@{\quad}cc@{\quad}cc@{\quad}cc@{\quad}ccc@{}}
|
||||
\toprule
|
||||
& \multicolumn{2}{c}{Kai} & \multicolumn{2}{c}{Laya} & \multicolumn{2}{c}{Jev, network} & \multicolumn{3}{c}{Questions/s} \\
|
||||
$N$ & p50 & p95 & p50 & p95 & p50 & p95 & Kai & Laya & Jev \\
|
||||
\midrule
|
||||
\multicolumn{10}{@{}l}{\emph{Metal}} \\
|
||||
1 & 31.9 & 46.0 & 28.9 & 133 & 133 & 546 & 27.5 & 26.5 & 5.72 \\
|
||||
10 & 129 & 205 & 214 & 314 & 155 & 221 & 74.5 & 44.5 & 62.9 \\
|
||||
50 & 555 & 750 & 1{,}097 & 1{,}246 & 202 & 275 & 85.9 & 44.3 & 233 \\
|
||||
100 & 1{,}214 & 1{,}403 & 2{,}362 & 2{,}587 & 343 & 556 & 80.4 & 42.4 & 267 \\
|
||||
500 & 5{,}235 & 6{,}079 & 10{,}133 & 10{,}828 & 515 & 2{,}193 & 93.0 & 49.4 & 726 \\
|
||||
1{,}000 & 10{,}239 & 11{,}433 & 19{,}616 & 20{,}714 & 602 & 1{,}077 & 94.7 & 50.7 & 1{,}489 \\
|
||||
\midrule
|
||||
\multicolumn{10}{@{}l}{\emph{CPU}} \\
|
||||
1 & 162 & 179 & 387 & 409 & 133 & 546 & 6.92 & 2.96 & 5.72 \\
|
||||
10 & 1{,}024 & 1{,}409 & 4{,}337 & 6{,}282 & 155 & 221 & 9.04 & 2.01 & 62.9 \\
|
||||
50 & 4{,}778 & 4{,}862 & 20{,}607 & 21{,}073 & 202 & 275 & 10.6 & 2.44 & 233 \\
|
||||
100 & 8{,}974 & 9{,}379 & 39{,}938 & 42{,}672 & 343 & 556 & 11.2 & 2.46 & 267 \\
|
||||
500 & 43{,}152 & 44{,}820 & 178{,}544 & 182{,}739 & 515 & 2{,}193 & 11.7 & 2.80 & 726 \\
|
||||
1{,}000 & 80{,}165 & 80{,}165 & 329{,}783 & 329{,}783 & 602 & 1{,}077 & 12.5 & 3.03 & 1{,}489 \\
|
||||
\bottomrule
|
||||
\multicolumn{10}{@{}l@{}}{\footnotesize\begin{tabular}[t]{@{}l@{}}Kai runs/a, encoder 768 wide, 22 layers \\ harness a1a4a73, benchmarks e418767 \\ timed F32 on Metal and on the CPU, Kai's CPU encoder projections Q8\_0 \\ Laya laya-multilingual: hanzoai/kai-1-multilingual@3119843b, encoder 768 wide, 22 layers \\ timed calls per N: Metal 10; CPU 3, 1 at N = 1,000 \\ dgx (NVIDIA GB10, CUDA)\end{tabular}} \\
|
||||
\end{tabular}
|
||||
|
||||
Reference in New Issue
Block a user