papers/hanzo-aso/sections/99-extended-ablation-group-size-sensitivity.tex

\section{Extended Ablation: Group Size Sensitivity}

\label{app:group-size}

\begin{table}[h]
\centering
\begin{tabular}{ccccc}
\toprule
$G$ & HumanEval & MBPP & SWE-bench & Cost/issue \\
\midrule
1 & 65.4\% & 62.7\% & 13.1\% & \$1.20 \\
2 & 68.9\% & 65.1\% & 15.4\% & \$2.40 \\
4 & 72.8\% & 68.4\% & 18.2\% & \$4.80 \\
8 & 73.6\% & 69.1\% & 19.1\% & \$9.60 \\
16 & 74.1\% & 69.4\% & 19.5\% & \$19.20 \\
\bottomrule
\end{tabular}
\caption{Performance vs.\ group size with $K=3$ iterations. Diminishing
returns beyond $G=4$ justify our default choice.}
\label{tab:group-sensitivity}
\end{table}

% ============================================================================
% REFERENCES
% ============================================================================
\bibliographystyle{plainnat}
\begin{thebibliography}{30}

\bibitem[Aider(2024)]{aider2024}
Aider.
\newblock Aider: AI pair programming in your terminal.
\newblock \url{https://aider.chat}, 2024.

\bibitem[Austin et~al.(2021)]{austin2021program}
Austin, J., Odena, A., Nye, M., Bosma, M., Michalewski, H., Dohan, D., Jiang,
  E., Cai, C., Terry, M., Le, Q., and Sutton, C.
\newblock Program synthesis with large language models.
\newblock \emph{arXiv preprint arXiv:2108.07732}, 2021.

\bibitem[Brown et~al.(2020)]{brown2020language}
Brown, T., Mann, B., Ryder, N., Subbiah, M., et~al.
\newblock Language models are few-shot learners.
\newblock \emph{NeurIPS}, 33:1877--1901, 2020.

\bibitem[Chen et~al.(2021)]{chen2021evaluating}
Chen, M., Tworek, J., Jun, H., Yuan, Q., et~al.
\newblock Evaluating large language models trained on code.
\newblock \emph{arXiv preprint arXiv:2107.03374}, 2021.

\bibitem[Anthropic(2024)]{anthropic2024claude}
Anthropic.
\newblock The Claude model family.
\newblock Technical report, Anthropic, 2024.

\bibitem[Cursor(2024)]{cursor2024}
Cursor.
\newblock Cursor: The AI-first code editor.
\newblock \url{https://cursor.com}, 2024.

\bibitem[Dettmers et~al.(2024)]{dettmers2024qlora}
Dettmers, T., Pagnoni, A., Holtzman, A., and Zettlemoyer, L.
\newblock QLoRA: Efficient finetuning of quantized language models.
\newblock \emph{NeurIPS}, 2024.

\bibitem[Du et~al.(2023)]{du2023reducing}
Du, Y., Li, S., Torralba, A., Tenenbaum, J.~B., and Mordatch, I.
\newblock Reducing language model hallucination with product of experts.
\newblock \emph{arXiv preprint arXiv:2310.06827}, 2023.

\bibitem[Frantar et~al.(2023)]{frantar2023gptq}
Frantar, E., Ashkboos, S., Hoefler, T., and Alistarh, D.
\newblock GPTQ: Accurate post-training quantization for generative pre-trained
  transformers.
\newblock \emph{ICLR}, 2023.

\bibitem[Friston et~al.(2017)]{friston2017active}
Friston, K., FitzGerald, T., Rigoli, F., Schwartenbeck, P., and Pezzulo, G.
\newblock Active inference: A process theory.
\newblock \emph{Neural Computation}, 29(1):1--49, 2017.

\bibitem[Hinton(2002)]{hinton2002training}
Hinton, G.~E.
\newblock Training products of experts by minimizing contrastive divergence.
\newblock \emph{Neural Computation}, 14(8):1771--1800, 2002.

\bibitem[Hu et~al.(2022)]{hu2022lora}
Hu, E.~J., Shen, Y., Wallis, P., Allen-Zhu, Z., Li, Y., Wang, S., Wang, L.,
  and Chen, W.
\newblock LoRA: Low-rank adaptation of large language models.
\newblock \emph{ICLR}, 2022.

\bibitem[Jim{\'e}nez et~al.(2024)]{jimenez2024swebench}
Jim{\'e}nez, C.~E., Yang, J., Wettig, A., Yao, S., Pei, K., Press, O., and
  Narasimhan, K.
\newblock SWE-bench: Can language models resolve real-world {GitHub} issues?
\newblock \emph{ICLR}, 2024.

\bibitem[Kirkpatrick et~al.(2017)]{kirkpatrick2017overcoming}
Kirkpatrick, J., Pascanu, R., Rabinowitz, N., et~al.
\newblock Overcoming catastrophic forgetting in neural networks.
\newblock \emph{PNAS}, 114(13):3521--3526, 2017.

\bibitem[Lester et~al.(2021)]{lester2021power}
Lester, B., Al-Rfou, R., and Constant, N.
\newblock The power of scale for parameter-efficient prompt tuning.
\newblock \emph{EMNLP}, 2021.

\bibitem[Lewis et~al.(2020)]{lewis2020retrieval}
Lewis, P., Perez, E., Piktus, A., Petroni, F., et~al.
\newblock Retrieval-augmented generation for knowledge-intensive NLP tasks.
\newblock \emph{NeurIPS}, 2020.

\bibitem[Li \& Liang(2021)]{li2021prefix}
Li, X.~L. and Liang, P.
\newblock Prefix-tuning: Optimizing continuous prompts for generation.
\newblock \emph{ACL}, 2021.

\bibitem[Lin et~al.(2024)]{lin2024awq}
Lin, J., Tang, J., Tang, H., Yang, S., Dang, X., and Han, S.
\newblock AWQ: Activation-aware weight quantization for LLM compression and
  acceleration.
\newblock \emph{MLSys}, 2024.

\bibitem[Liu et~al.(2024a)]{liu2024bitdelta}
Liu, J., Desai, A., Liao, L., Wang, Y., and Xie, E.
\newblock BitDelta: Your fine-tune may only be worth one bit.
\newblock \emph{arXiv preprint arXiv:2402.10193}, 2024.

\bibitem[Liu et~al.(2024b)]{liu2024tuning}
Liu, A., Han, X., Wang, Y., et~al.
\newblock Tuning language models by proxy.
\newblock \emph{arXiv preprint arXiv:2401.08565}, 2024.

\bibitem[Madaan et~al.(2023)]{madaan2023selfrefine}
Madaan, A., Tandon, N., Gupta, P., et~al.
\newblock Self-refine: Iterative refinement with self-feedback.
\newblock \emph{NeurIPS}, 2023.

\bibitem[OpenAI(2023)]{openai2023gpt4}
OpenAI.
\newblock GPT-4 technical report.
\newblock \emph{arXiv preprint arXiv:2303.08774}, 2023.

\bibitem[Ouyang et~al.(2022)]{ouyang2022training}
Ouyang, L., Wu, J., Jiang, X., Almeida, D., et~al.
\newblock Training language models to follow instructions with human feedback.
\newblock \emph{NeurIPS}, 2022.

\bibitem[Parr et~al.(2022)]{parr2022active}
Parr, T., Pezzulo, G., and Friston, K.~J.
\newblock \emph{Active Inference: The Free Energy Principle in Mind, Brain, and
  Behavior}.
\newblock MIT Press, 2022.

\bibitem[Rafailov et~al.(2024)]{rafailov2024direct}
Rafailov, R., Sharma, A., Mitchell, E., et~al.
\newblock Direct preference optimization: Your language model is secretly a
  reward model.
\newblock \emph{NeurIPS}, 2024.

\bibitem[Sajid et~al.(2021)]{sajid2021active}
Sajid, N., Ball, P.~J., Parr, T., and Friston, K.~J.
\newblock Active inference: Demystified and compared.
\newblock \emph{Neural Computation}, 33(3):674--712, 2021.

\bibitem[Shao et~al.(2024)]{shao2024deepseekmath}
Shao, Z., Wang, P., Zhu, Q., et~al.
\newblock DeepSeekMath: Pushing the limits of mathematical reasoning in open
  language models.
\newblock \emph{arXiv preprint arXiv:2402.03300}, 2024.

\bibitem[Wei et~al.(2022)]{wei2022chain}
Wei, J., Wang, X., Schuurmans, D., Bosma, M., et~al.
\newblock Chain-of-thought prompting elicits reasoning in large language models.
\newblock \emph{NeurIPS}, 2022.

\bibitem[Williams(1992)]{williams1992simple}
Williams, R.~J.
\newblock Simple statistical gradient-following algorithms for connectionist
  reinforcement learning.
\newblock \emph{Machine Learning}, 8(3):229--256, 1992.

\bibitem[Yang et~al.(2024)]{yang2024sweagent}
Yang, J., Jim{\'e}nez, C.~E., Wettig, A., et~al.
\newblock SWE-agent: Agent-computer interfaces enable automated software
  engineering.
\newblock \emph{arXiv preprint arXiv:2405.15793}, 2024.

\bibitem[Zhang et~al.(2024)]{zhang2024autocoderover}
Zhang, Y., Ruan, W., Fan, Z., and Chen, A.
\newblock AutoCodeRover: Autonomous program improvement.
\newblock \emph{arXiv preprint arXiv:2404.05427}, 2024.

\end{thebibliography}