papers/hanzo-aso/sections/09-experimental-results.tex

\section{Experimental Results}

\label{sec:experiments}

\subsection{Benchmarks and Setup}

We evaluate ASO on three code generation benchmarks:

\begin{itemize}[leftmargin=1.5em]
    \item \textbf{HumanEval}~\cite{chen2021evaluating}: 164 Python programming
          problems with unit tests. Metric: pass@1.
    \item \textbf{MBPP}~\cite{austin2021program}: 974 Python programming
          problems from crowd-sourcing. Metric: pass@1.
    \item \textbf{SWE-bench Verified}~\cite{jimenez2024swebench}: 500
          real-world GitHub issues with test-based verification. Metric:
          resolved rate.
\end{itemize}

\paragraph{Base Models.}
We use Claude 3.5 Sonnet as the primary base model ($\pi_\theta$) for all
experiments. For comparison, we also report results with GPT-4 and Zen-72B.

\paragraph{Hyperparameters.}
Group size $G = 4$, maximum iterations $K = 3$, reward weights
$\alpha = 0.7$, $\beta = 0.3$, PoE temperature $\tau = 0.7$ (generation) /
$\tau = 1.2$ (exploration), quality threshold $q_{\min} = 0.5$.

\subsection{HumanEval Results}

\begin{table}[h]
\centering
\begin{tabular}{lccc}
\toprule
Method & Base Model & pass@1 & $\Delta$ \\
\midrule
Zero-shot & Claude 3.5 Sonnet & 63.7\% & -- \\
Few-shot (3 examples) & Claude 3.5 Sonnet & 66.2\% & +2.5\% \\
Self-refine (3 iter.) & Claude 3.5 Sonnet & 68.1\% & +4.4\% \\
\textbf{ASO (ours)} & Claude 3.5 Sonnet & \textbf{72.8\%} & \textbf{+9.1\%} \\
\midrule
Zero-shot & GPT-4 & 67.0\% & -- \\
\textbf{ASO (ours)} & GPT-4 & \textbf{74.3\%} & \textbf{+7.3\%} \\
\midrule
Zero-shot & Zen-72B & 58.4\% & -- \\
\textbf{ASO (ours)} & Zen-72B & \textbf{66.9\%} & \textbf{+8.5\%} \\
\bottomrule
\end{tabular}
\caption{HumanEval pass@1 results. ASO consistently improves over zero-shot
baselines by 7--9 percentage points across different base models.}
\label{tab:humaneval}
\end{table}

\subsection{MBPP Results}

\begin{table}[h]
\centering
\begin{tabular}{lccc}
\toprule
Method & Base Model & pass@1 & $\Delta$ \\
\midrule
Zero-shot & Claude 3.5 Sonnet & 61.2\% & -- \\
Few-shot (3 examples) & Claude 3.5 Sonnet & 63.8\% & +2.6\% \\
Self-refine (3 iter.) & Claude 3.5 Sonnet & 65.1\% & +3.9\% \\
\textbf{ASO (ours)} & Claude 3.5 Sonnet & \textbf{68.4\%} & \textbf{+7.2\%} \\
\midrule
Zero-shot & GPT-4 & 64.2\% & -- \\
\textbf{ASO (ours)} & GPT-4 & \textbf{71.1\%} & \textbf{+6.9\%} \\
\bottomrule
\end{tabular}
\caption{MBPP pass@1 results. ASO provides consistent improvements across
both models and benchmark types.}
\label{tab:mbpp}
\end{table}

\subsection{SWE-bench Verified Results}

\begin{table}[h]
\centering
\begin{tabular}{lcc}
\toprule
Method & Resolved Rate & Avg.\ Iterations \\
\midrule
GPT-4 (zero-shot) & 8.3\% & 1.0 \\
Claude 3.5 Sonnet (agentic) & 12.5\% & 2.3 \\
AutoCodeRover~\cite{zhang2024autocoderover} & 14.7\% & 3.1 \\
SWE-agent~\cite{yang2024sweagent} & 15.9\% & 2.8 \\
Aider~\cite{aider2024} & 16.1\% & 2.5 \\
\textbf{Hanzo Dev (ASO)} & \textbf{18.2\%} & \textbf{2.4} \\
\bottomrule
\end{tabular}
\caption{SWE-bench Verified results (500 issues). Hanzo Dev uses TF-GRPO with
group size 4 and maximum 3 iterations.}
\label{tab:swebench}
\end{table}

\subsection{Ablation Studies}

\begin{table}[h]
\centering
\begin{tabular}{lccc}
\toprule
Configuration & HumanEval & MBPP & SWE-bench \\
\midrule
Full ASO & \textbf{72.8\%} & \textbf{68.4\%} & \textbf{18.2\%} \\
\midrule
$-$ TF-GRPO (single-shot) & 65.4\% & 62.7\% & 13.1\% \\
$-$ PoE (advantages as ICL) & 69.1\% & 65.8\% & 15.3\% \\
$-$ Epistemic term ($\beta = 0$) & 71.2\% & 67.1\% & 16.8\% \\
$-$ 1-bit compression (full precision) & 72.9\% & 68.5\% & 18.3\% \\
$-$ Context matching & 70.3\% & 66.2\% & 16.1\% \\
\midrule
TF-GRPO only (no PoE, no priors) & 67.8\% & 64.3\% & 14.5\% \\
PoE only (random priors) & 64.1\% & 61.5\% & 12.8\% \\
\bottomrule
\end{tabular}
\caption{Ablation results. Each row removes one component from the full ASO
system. 1-bit compression incurs negligible loss ($\leq 0.1\%$), confirming
the theoretical bound in Lemma~\ref{lem:quant-error}.}
\label{tab:ablation}
\end{table}

\subsection{Comparison with Fine-Tuning}

\begin{table}[h]
\centering
\begin{tabular}{lcccc}
\toprule
Method & SWE-bench & Cost & GPU Hours & Forgetting Risk \\
\midrule
LoRA fine-tuning & 22.1\% & \$15{,}000 & 120 & Medium \\
QLoRA fine-tuning & 20.8\% & \$10{,}000 & 80 & Medium \\
Full RLHF & 24.5\% & \$80{,}000 & 800 & High \\
DPO & 21.3\% & \$12{,}000 & 100 & Medium \\
\textbf{ASO (ours)} & \textbf{18.2\%} & \textbf{\$18} & \textbf{0} & \textbf{None} \\
\bottomrule
\end{tabular}
\caption{Comparison with parameter-update methods. ASO achieves 74--83\% of
fine-tuning performance at 0.02--0.2\% of the cost with zero forgetting risk.}
\label{tab:finetuning-comparison}
\end{table}

\subsection{Prior Transfer Analysis}

We investigate whether priors learned on one benchmark transfer to others:

\begin{table}[h]
\centering
\begin{tabular}{lccc}
\toprule
Prior Source & HumanEval & MBPP & SWE-bench \\
\midrule
None (zero-shot) & 63.7\% & 61.2\% & 12.5\% \\
HumanEval priors & \textbf{72.8\%} & 65.1\% & 14.8\% \\
MBPP priors & 67.3\% & \textbf{68.4\%} & 14.2\% \\
SWE-bench priors & 66.9\% & 64.8\% & \textbf{18.2\%} \\
All priors (merged) & 71.4\% & 67.9\% & 17.8\% \\
\bottomrule
\end{tabular}
\caption{Cross-benchmark prior transfer. Priors provide the largest benefit on
their source benchmark but also transfer partially (2--4\%) to related tasks.}
\label{tab:transfer}
\end{table}

\subsection{Scaling Analysis}

We study how performance scales with group size $G$ and iteration count $K$:

\begin{table}[h]
\centering
\begin{tabular}{ccccc}
\toprule
$G$ & $K$ & SWE-bench & Cost & Time \\
\midrule
2 & 1 & 14.1\% & \$4 & 0.5h \\
4 & 1 & 15.6\% & \$7 & 1.0h \\
4 & 3 & 18.2\% & \$18 & 2.5h \\
8 & 3 & 19.1\% & \$35 & 4.5h \\
8 & 5 & 19.4\% & \$55 & 7.0h \\
16 & 5 & 19.7\% & \$105 & 13h \\
\bottomrule
\end{tabular}
\caption{Performance scaling with group size and iteration count. Diminishing
returns are evident beyond $G=4$, $K=3$, which is our default configuration.}
\label{tab:scaling}
\end{table}

% ============================================================================
% 10. RELATED WORK
% ============================================================================