papers/hanzo-agent-sdk/sections/05-evaluation.tex

\section{Evaluation}


\subsection{Experimental Setup}

We evaluate on three benchmark suites:
\begin{enumerate}
    \item \textbf{SWE-bench}~\cite{swebench2023}: Software engineering tasks requiring code understanding, modification, and testing.
    \item \textbf{GAIA}~\cite{gaia2023}: General AI assistant tasks requiring multi-step reasoning and tool use.
    \item \textbf{Custom Multi-Agent}: A suite of 500 tasks designed to stress multi-agent coordination.
\end{enumerate}

\paragraph{Baselines.} We compare against:
\begin{itemize}
    \item Single-agent with all tools (SA-Full)
    \item LangChain agent executor (LC)
    \item AutoGPT with GPT-4 (AGPT)
\end{itemize}

\subsection{Results}

\begin{table}[h]
\centering
\caption{Task completion rates (\%) on benchmark suites.}
\label{tab:completion}
\begin{tabular}{lccc}
\toprule
\textbf{Method} & \textbf{SWE-bench} & \textbf{GAIA} & \textbf{Multi-Agent} \\
\midrule
SA-Full & 18.2 & 42.3 & 31.7 \\
LC & 21.4 & 45.1 & 35.2 \\
AGPT & 19.8 & 38.9 & 28.4 \\
\midrule
\textbf{Ours} & \textbf{24.3} & \textbf{56.7} & \textbf{52.1} \\
\bottomrule
\end{tabular}
\end{table}

Table~\ref{tab:completion} shows task completion rates. Our framework achieves 34\% higher completion on SWE-bench, 27\% on GAIA, and 48\% on the multi-agent benchmark compared to the best baseline.

\begin{table}[h]
\centering
\caption{Token efficiency (tokens per completed task).}
\label{tab:tokens}
\begin{tabular}{lccc}
\toprule
\textbf{Method} & \textbf{SWE-bench} & \textbf{GAIA} & \textbf{Multi-Agent} \\
\midrule
SA-Full & 45,230 & 12,450 & 28,900 \\
LC & 52,100 & 14,200 & 31,500 \\
AGPT & 89,400 & 31,200 & 67,800 \\
\midrule
\textbf{Ours} & \textbf{21,800} & \textbf{8,900} & \textbf{14,200} \\
\bottomrule
\end{tabular}
\end{table}

Table~\ref{tab:tokens} demonstrates token efficiency. Semantic routing reduces unnecessary exploration, achieving 2.1x better token efficiency than the single-agent baseline.

\subsection{Ablation Studies}

\paragraph{Memory Impact.} Disabling memory reduces completion by 12\% on tasks requiring context from earlier steps.

\paragraph{Router Impact.} Random routing (vs. semantic) reduces completion by 23\% and increases tokens by 1.8x.

\paragraph{Parallel Execution.} Enabling parallel execution reduces wall-clock time by 2.3x on tasks with independent subtasks.