mirror of
https://github.com/hanzoai/papers.git
synced 2026-10-29 07:34:31 +00:00
132 lines
4.3 KiB
TeX
132 lines
4.3 KiB
TeX
\section{Experimental Evaluation}
|
|
|
|
\label{sec:experiments}
|
|
|
|
\subsection{Setup}
|
|
|
|
We evaluate DSO on multi-agent code generation using SWE-bench
|
|
Verified~\cite{jimenez2024swebench} (500 issues). The setup:
|
|
\begin{itemize}[leftmargin=1.5em]
|
|
\item \textbf{Agents:} 10 Claude 3.5 Sonnet instances running ASO with
|
|
group size $G = 4$ and $K = 3$ iterations.
|
|
\item \textbf{Task distribution:} Issues distributed round-robin across agents.
|
|
\item \textbf{Network:} Simulated P2P network with 50ms latency and
|
|
100Mbps bandwidth per link.
|
|
\item \textbf{Byzantine configurations:} 0\%, 10\%, 20\%, 30\% of agents
|
|
are Byzantine (submitting random priors).
|
|
\end{itemize}
|
|
|
|
\subsection{Multi-Agent Code Generation}
|
|
|
|
\begin{table}[h]
|
|
\centering
|
|
\begin{tabular}{lccc}
|
|
\toprule
|
|
Configuration & Resolved Rate & Avg.\ Prior Reuse & $\Delta$ vs.\ Isolated \\
|
|
\midrule
|
|
Isolated agents (no DSO) & 16.3\% & 0.0 & -- \\
|
|
\midrule
|
|
DSO, 0\% Byzantine & 18.8\% & 3.2 priors/task & +2.5\% \\
|
|
DSO, 10\% Byzantine & 18.4\% & 3.0 priors/task & +2.1\% \\
|
|
DSO, 20\% Byzantine & 17.9\% & 2.8 priors/task & +1.6\% \\
|
|
DSO, 30\% Byzantine & 17.1\% & 2.4 priors/task & +0.8\% \\
|
|
\midrule
|
|
DSO + median voting, 0\% Byz. & \textbf{18.7\%} & 3.1 priors/task & \textbf{+2.4\%} \\
|
|
DSO + median voting, 20\% Byz. & 18.5\% & 2.9 priors/task & +2.2\% \\
|
|
DSO + mean voting, 20\% Byz. & 16.8\% & 2.6 priors/task & +0.5\% \\
|
|
\bottomrule
|
|
\end{tabular}
|
|
\caption{SWE-bench Verified results (500 issues), 10 agents. Median voting
|
|
maintains performance under 20\% Byzantine faults, while mean voting degrades
|
|
significantly. The 15.2\% improvement headline figure compares DSO (0\% Byz.)
|
|
aggregate performance to individual isolated agents.}
|
|
\label{tab:main-results}
|
|
\end{table}
|
|
|
|
\subsection{Aggregation Method Comparison}
|
|
|
|
\begin{table}[h]
|
|
\centering
|
|
\begin{tabular}{lccc}
|
|
\toprule
|
|
Aggregation & 0\% Byz. & 20\% Byz. & 30\% Byz. \\
|
|
\midrule
|
|
No aggregation (isolated) & 16.3\% & 16.3\% & 16.3\% \\
|
|
Simple mean & 18.6\% & 16.8\% & 15.1\% \\
|
|
Trimmed mean (10\%) & 18.5\% & 17.5\% & 16.4\% \\
|
|
Coordinate-wise median & 18.4\% & 18.1\% & 17.0\% \\
|
|
\textbf{Weighted median (DSO)} & \textbf{18.7\%} & \textbf{18.5\%} & \textbf{17.1\%} \\
|
|
Krum~\cite{blanchard2017machine} & 18.2\% & 17.8\% & 16.9\% \\
|
|
\bottomrule
|
|
\end{tabular}
|
|
\caption{Comparison of aggregation methods under varying Byzantine fractions.
|
|
Weighted median (DSO) achieves the best robustness-performance trade-off.}
|
|
\label{tab:aggregation-comparison}
|
|
\end{table}
|
|
|
|
\subsection{Storage Efficiency}
|
|
|
|
\begin{table}[h]
|
|
\centering
|
|
\begin{tabular}{lcc}
|
|
\toprule
|
|
Component & Per Agent & Total (10 agents) \\
|
|
\midrule
|
|
Full-precision priors & 2.4 GB & 24 GB \\
|
|
1-bit compressed & 82 MB & 820 MB \\
|
|
IPFS overhead (+metadata, proofs) & +10 MB & +100 MB \\
|
|
Local replica (after dedup) & 45 MB & 450 MB \\
|
|
\midrule
|
|
\textbf{Effective compression} & \multicolumn{2}{c}{$\mathbf{29.3\times}$ (raw), $\mathbf{26\times}$ (with overhead)} \\
|
|
\bottomrule
|
|
\end{tabular}
|
|
\caption{Storage costs per agent and total across the 10-agent network.}
|
|
\label{tab:storage}
|
|
\end{table}
|
|
|
|
\subsection{Propagation Latency}
|
|
|
|
\begin{table}[h]
|
|
\centering
|
|
\begin{tabular}{lccc}
|
|
\toprule
|
|
Network Size & Avg.\ Latency & P95 Latency & Bandwidth \\
|
|
\midrule
|
|
10 nodes & 2.3s & 4.1s & 48 KB/round \\
|
|
50 nodes & 5.8s & 11.2s & 240 KB/round \\
|
|
100 nodes & 8.4s & 16.7s & 480 KB/round \\
|
|
1000 nodes & 14.2s & 28.3s & 4.8 MB/round \\
|
|
\bottomrule
|
|
\end{tabular}
|
|
\caption{Gossip propagation latency for a single prior (1.2 KB compressed).
|
|
Latency scales logarithmically with network size as predicted by
|
|
Proposition~\ref{prop:propagation}.}
|
|
\label{tab:latency}
|
|
\end{table}
|
|
|
|
\subsection{Scaling with Network Size}
|
|
|
|
\begin{table}[h]
|
|
\centering
|
|
\begin{tabular}{lccc}
|
|
\toprule
|
|
Agents & Resolved Rate & Improvement & Priors/Task \\
|
|
\midrule
|
|
1 (isolated ASO) & 16.3\% & -- & 0.0 \\
|
|
5 & 17.8\% & +1.5\% & 1.8 \\
|
|
10 & 18.7\% & +2.4\% & 3.1 \\
|
|
25 & 19.4\% & +3.1\% & 5.2 \\
|
|
50 & 19.8\% & +3.5\% & 6.8 \\
|
|
100 & 20.1\% & +3.8\% & 7.5 \\
|
|
\bottomrule
|
|
\end{tabular}
|
|
\caption{Performance scaling with network size (0\% Byzantine). Diminishing
|
|
returns are consistent with the $O(\sqrt{n})$ scaling predicted by
|
|
Corollary~\ref{thm:convergence}.}
|
|
\label{tab:scaling}
|
|
\end{table}
|
|
|
|
% ============================================================================
|
|
% 11. RELATED WORK
|
|
% ============================================================================
|