mirror of
https://github.com/hanzoai/papers.git
synced 2026-10-29 07:34:31 +00:00
zeekay
ab5afcf18d
Hanzo CI/CD / test (linux-amd64) (push) Skipped
Hanzo CI/CD / test (linux-arm64) (push) Skipped
Hanzo CI/CD / test-advisory (push) Skipped
Build LaTeX Papers / build-pdfs (push) Failing after 31s
ci / papers (push) Failing after 23s
Hanzo CI/CD / test-amd64 (push) Successful in 1m14s
Hanzo CI/CD / cicd (push) Successful in 1m36s
CI/CD / cicd (push) Successful in 2m55s
46 lines
2.1 KiB
TeX
46 lines
2.1 KiB
TeX
% capability.tex: written by `bench tex` (hanzoai/decision); Kai runs/a7, weights 0834a74f, metal bf16; harness a1a4a73; benchmarks 039fbfe
|
|
% Kai: joint rows are its refinement over three passes (projected: its own selection and projection); Laya's joint rows are its typed-decisions checkpoint; media is Laya as routed. Rejected: questions a backend refuses, over all asked.
|
|
\begin{tabular}{@{}lccc@{}}
|
|
\toprule
|
|
& Kai & Laya & Jev \\
|
|
\midrule
|
|
\multicolumn{4}{@{}l}{\emph{Cardinality}} \\
|
|
Accuracy, $K = 4$ & 0.685 & 0.777 & 0.905 \\
|
|
Accuracy, $K = 16$ & 0.395 & 0.670 & 0.853 \\
|
|
Accuracy, $K = 77$ & 0.255 & 0.425 & 0.843 \\
|
|
Accuracy, $K = 150$ & 0.287 & 0.610 & 0.940 \\
|
|
Accuracy, $K = 1{,}000$ & 0.005 & 0.000 & 0.000 \\
|
|
Accuracy, $K = 10{,}000$ & 0.000 & 0.000 & 0.000 \\
|
|
Accuracy, $K = 100{,}000$ & 0.000 & 0.000 & 0.000 \\
|
|
Rejected, $K = 1{,}000$ & 0.000 & 1.000 & 1.000 \\
|
|
Rejected, $K = 10{,}000$ & 0.000 & 1.000 & 1.000 \\
|
|
Rejected, $K = 100{,}000$ & 0.000 & 1.000 & 1.000 \\
|
|
\midrule
|
|
\multicolumn{4}{@{}l}{\emph{Joint coherence, five dependent questions}} \\
|
|
Accuracy & 0.306 & 0.336 & 0.837 \\
|
|
Exact & 0.000 & 0.005 & 0.438 \\
|
|
Violations & 1.000 & 0.637 & 0.247 \\
|
|
Accuracy, projected & 0.341 & 0.328 & 0.879 \\
|
|
Exact, projected & 0.010 & 0.040 & 0.560 \\
|
|
\midrule
|
|
\multicolumn{4}{@{}l}{\emph{Invariance}} \\
|
|
Permutation, flip & 0.000 & 0.255 & 0.060 \\
|
|
Permutation, max $\Delta p$ & 0.000 & 0.996 & 0.470 \\
|
|
Noul sides swapped, mirror & 0.915 & 0.205 & 0.062 \\
|
|
Noul negated, mirror & 0.430 & 0.300 & 0.882 \\
|
|
\midrule
|
|
\multicolumn{4}{@{}l}{\emph{Media, accuracy}} \\
|
|
Human activity (HAR) & 0.160 & 0.203 & 0.153 \\
|
|
Turbofan life (C-MAPSS) & 0.330 & 0.170 & 0.300 \\
|
|
Valve sound (MIMII) & 0.500 & 0.500 & 0.840 \\
|
|
\midrule
|
|
\multicolumn{4}{@{}l}{\emph{Conformal sets, 90\% target}} \\
|
|
Coverage & 0.901 & 0.901 & 0.901 \\
|
|
Mean set size & 2.498 & 6.801 & 1.304 \\
|
|
\midrule
|
|
Refresh: questions asked / full & 6 / 101 & 100 / 100 & 100 / 100 \\
|
|
Runs offline & yes & yes & no \\
|
|
\bottomrule
|
|
\multicolumn{4}{@{}l@{}}{\footnotesize\begin{tabular}[t]{@{}l@{}}Kai runs/a7, weights 0834a74f, metal bf16 \\ harness a1a4a73, benchmarks 039fbfe\end{tabular}} \\
|
|
\end{tabular}
|