papers/hanzo-federated-agents/sections/09-experimental-evaluation.tex

\section{Experimental Evaluation}

\label{sec:evaluation}

\subsection{Simulation Setup}

We evaluate FAI via a discrete-event simulator implementing the full protocol
stack. Each simulated agent node runs local GRPO and DSO on a fixed task
distribution drawn from SWE-bench Verified~\cite{jimenez2024swebench}
(software engineering tasks).

\paragraph{Network Configurations.}
We evaluate three fleet sizes: $N \in \{10, 50, 100\}$ nodes. For each,
we fix gossip fanout $k_{\text{peers}} = 8$, broadcast interval 60\,s,
group size $g = 128$, and filter threshold $\tau_{\text{filter}} = 2.0$.

\paragraph{Baselines.}
\begin{itemize}[leftmargin=1.5em]
    \item \textbf{Isolated ASO}: Each node runs ASO independently with no
          sharing. Represents the single-agent baseline.
    \item \textbf{Centralized DSO}: All experiences are aggregated at a
          central server (no Byzantine robustness, no compression).
          Represents the oracle upper bound.
    \item \textbf{FAI (ours)}: Full federated protocol with BitDelta and DeltaSoup.
    \item \textbf{FAI (no DeltaSoup)}: FAI without Byzantine-robust filtering,
          using naive averaging instead.
\end{itemize}

\subsection{Task Success Rate Results}

\begin{table}[H]
\centering
\begin{tabular}{lcccc}
\toprule
\multirow{2}{*}{Method} & \multicolumn{3}{c}{Fleet Size ($N$)} & \multirow{2}{*}{\shortstack{Improvement\\over Isolated}} \\
\cmidrule(lr){2-4}
& $N=10$ & $N=50$ & $N=100$ & \\
\midrule
Isolated ASO & 18.2\% & 18.2\% & 18.2\% & -- \\
Centralized DSO & 22.1\% & 23.4\% & 24.0\% & up to $+5.8\%$ \\
\midrule
FAI, no DeltaSoup & 20.1\% & 21.3\% & 21.9\% & up to $+3.7\%$ \\
\textbf{FAI (ours)} & \textbf{21.8\%} & \textbf{22.9\%} & \textbf{23.7\%} & up to $\mathbf{+5.5\%}$ \\
\bottomrule
\end{tabular}
\caption{Task success rate (SWE-bench resolved rate) by method and fleet size,
with 0\% Byzantine nodes. FAI approaches the centralized oracle while
maintaining full decentralization. DeltaSoup provides $+1.7\%$ improvement
over naive averaging.}
\label{tab:main-results}
\end{table}

\subsection{Byzantine Robustness}

\begin{table}[H]
\centering
\begin{tabular}{lcccc}
\toprule
Method & $f=0$ & $f=N/10$ & $f=N/4$ & $f=N/3$ \\
\midrule
FAI, no DeltaSoup & 23.7\% & 21.8\% & 19.4\% & 18.1\% \\
\textbf{FAI (ours)} & \textbf{23.7\%} & \textbf{23.1\%} & \textbf{22.8\%} & \textbf{22.0\%} \\
\bottomrule
\end{tabular}
\caption{Task success rate at $N=100$ with varying Byzantine fraction.
DeltaSoup maintains 92.8\% of clean performance at $f=N/3$, while naive
averaging degrades to isolated-agent performance at $f=N/3$.}
\label{tab:byzantine}
\end{table}

\subsection{Compression Efficiency}

\begin{table}[H]
\centering
\begin{tabular}{lccc}
\toprule
Representation & Bytes/experience & Total ($M=100$) & Compression \\
\midrule
Float32 embedding ($d=768$) & 3,072 & 307,200 & $1\times$ \\
BitDelta ($g=128$) & 288 & 28,800 & $10.7\times$ \\
BitDelta + metadata & 352 & 35,200 & $8.7\times$ \\
\bottomrule
\end{tabular}
\caption{Storage and bandwidth costs per experience and per batch. The
``+ metadata'' row includes text pattern (avg.\ 128 bytes), quality score
(4 bytes), domain tag (8 bytes), and node ID (32 bytes).}
\label{tab:compression}
\end{table}

\subsection{Cosine Similarity Preservation}

\begin{table}[H]
\centering
\begin{tabular}{lccc}
\toprule
Group size $g$ & Mean cosine sim. & Min cosine sim. & Compression ratio \\
\midrule
1 & 0.704 & 0.582 & $32\times$ \\
16 & 0.861 & 0.793 & $16\times$ \\
64 & 0.934 & 0.901 & $12.1\times$ \\
\textbf{128} & \textbf{0.957} & \textbf{0.931} & $10.7\times$ \\
256 & 0.971 & 0.952 & $9.1\times$ \\
\midrule
Uncompressed & 1.000 & 1.000 & $1\times$ \\
\bottomrule
\end{tabular}
\caption{Cosine similarity between original and reconstructed embeddings by
group size, averaged over 1,000 experiences from a sentence-transformer
encoder. Group size $g=128$ provides the best compression-fidelity tradeoff.}
\label{tab:cosine}
\end{table}

\subsection{Convergence Speed}

\begin{table}[H]
\centering
\begin{tabular}{lccc}
\toprule
Fleet Size & Rounds to 90\% of Centralized & Final Quality ($T=100$) & Bandwidth/Round \\
\midrule
$N=10$ & 12 & 98.6\% of centralized & 29\,KB/node \\
$N=50$ & 18 & 97.9\% of centralized & 29\,KB/node \\
$N=100$ & 23 & 98.8\% of centralized & 29\,KB/node \\
\bottomrule
\end{tabular}
\caption{Convergence speed and final quality relative to the centralized oracle.
Bandwidth per node per round is constant regardless of fleet size (gossip fanout
$k=8$, 100 experiences, BitDelta compressed).}
\label{tab:convergence}
\end{table}

% ============================================================================
% 10. DISCUSSION AND FUTURE WORK
% ============================================================================