papers/hanzo-agent-grpo/sections/99-conclusion.tex

\section{Conclusion}

\label{sec:conclusion}

We have presented Agent-GRPO, a training-free continuous learning framework
that extends Group-Relative Policy Optimization to the agent tool-use setting.
The framework formalizes agent trajectories as typed tuples carrying rich
intermediate signals, adapts the YouTou-Agent three-stage extraction pipeline
to this richer domain, and constructs a semantic experience library whose
entries measurably improve future agent performance through context injection.

The theoretical analysis establishes that the experience library improves
monotonically under mild regularity conditions, with a sample complexity bound
of $O(K_x/\varepsilon^2)$ sessions to achieve $\varepsilon$-improvement.
Empirically, Agent-GRPO achieves a 20.1 percentage point improvement in task
completion rate over a zero-shot baseline on HB-Bench, at a total adaptation
cost below \$20---orders of magnitude below any gradient-based alternative.

The framework integrates naturally with the four-loop self-improvement
architecture of the Hanzo Dev agent, closing the feedback loop between
telemetry, user corrections, session reflection, and library maintenance.
This integration positions Agent-GRPO as the practical foundation for
continuously improving, training-free tool-using agents at production scale.

Code and benchmark data are available at \url{https://github.com/hanzoai/agent-grpo}.

% ============================================================================
% REFERENCES
% ============================================================================
\bibliographystyle{plainnat}
\begin{thebibliography}{99}

\bibitem[Benet(2014)]{benet2014ipfs}
Benet, J. (2014).
\newblock IPFS -- Content Addressed, Versioned, P2P File System.
\newblock \emph{arXiv:1407.3561}.

\bibitem[Brown et al.(2020)]{brown2020language}
Brown, T., Mann, B., Ryder, N., et al. (2020).
\newblock Language models are few-shot learners.
\newblock \emph{Advances in Neural Information Processing Systems (NeurIPS)}, 33, 1877--1901.

\bibitem[De Lange et al.(2021)]{de2021continual}
De Lange, M., Aljundi, R., Masana, M., et al. (2021).
\newblock A continual learning survey: Defying forgetting in classification tasks.
\newblock \emph{IEEE Transactions on Pattern Analysis and Machine Intelligence}, 44(7), 3366--3385.

\bibitem[Hanzo AI Research(2026a)]{hanzo2026aso}
Hanzo AI Research. (2026a).
\newblock Active Semantic Optimization: Training-Free Adaptation via Bayesian
  Product-of-Experts Decoding.
\newblock \emph{Hanzo AI Technical Report HIP-002}, February 2026.

\bibitem[Hanzo AI Research(2026b)]{hanzo2026dso}
Hanzo AI Research. (2026b).
\newblock Decentralized Semantic Optimization: Byzantine-Robust Prior
  Aggregation for Collective Model Adaptation.
\newblock \emph{Hanzo AI Technical Report ZIP-001}, February 2026.

\bibitem[Hinton(2002)]{hinton2002training}
Hinton, G. E. (2002).
\newblock Training products of experts by minimizing contrastive divergence.
\newblock \emph{Neural Computation}, 14(8), 1771--1800.

\bibitem[Hu et al.(2022)]{hu2022lora}
Hu, E. J., Shen, Y., Wallis, P., Allen-Zhu, Z., Li, Y., Wang, S., Wang, L., and Chen, W. (2022).
\newblock LoRA: Low-Rank Adaptation of Large Language Models.
\newblock \emph{International Conference on Learning Representations (ICLR)}.

\bibitem[Jimenez et al.(2024)]{jimenez2024swebench}
Jimenez, C. E., Yang, J., Wettig, A., et al. (2024).
\newblock SWE-bench: Can Language Models Resolve Real-World GitHub Issues?
\newblock \emph{International Conference on Learning Representations (ICLR)}.

\bibitem[Kirkpatrick et al.(2017)]{kirkpatrick2017overcoming}
Kirkpatrick, J., Pascanu, R., Rabinowitz, N., et al. (2017).
\newblock Overcoming catastrophic forgetting in neural networks.
\newblock \emph{Proceedings of the National Academy of Sciences}, 114(13), 3521--3526.

\bibitem[Lamport et al.(1982)]{lamport1982byzantine}
Lamport, L., Shostak, R., and Pease, M. (1982).
\newblock The Byzantine generals problem.
\newblock \emph{ACM Transactions on Programming Languages and Systems}, 4(3), 382--401.

\bibitem[Lewis et al.(2020)]{lewis2020retrieval}
Lewis, P., Perez, E., Piktus, A., et al. (2020).
\newblock Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks.
\newblock \emph{NeurIPS}, 33, 9459--9474.

\bibitem[Malkov and Yashunin(2018)]{malkov2018efficient}
Malkov, Y. A. and Yashunin, D. A. (2018).
\newblock Efficient and robust approximate nearest neighbor search using
  Hierarchical Navigable Small World graphs.
\newblock \emph{IEEE Transactions on Pattern Analysis and Machine Intelligence},
  42(4), 824--836.

\bibitem[Mitchell et al.(2022)]{mitchell2022memory}
Mitchell, E., Lin, C., Bosselut, A., Finn, C., and Manning, C. D. (2022).
\newblock Memory-Based Model Editing at Scale.
\newblock \emph{International Conference on Machine Learning (ICML)}.

\bibitem[Ouyang et al.(2022)]{ouyang2022training}
Ouyang, L., Wu, J., Jiang, X., et al. (2022).
\newblock Training language models to follow instructions with human feedback.
\newblock \emph{NeurIPS}, 35, 27730--27744.

\bibitem[Park et al.(2023)]{park2023generative}
Park, J. S., O'Brien, J. C., Cai, C. J., Morris, M. R., Liang, P., and Bernstein, M. S. (2023).
\newblock Generative agents: Interactive simulacra of human behavior.
\newblock \emph{ACM Symposium on User Interface Software and Technology (UIST)}.

\bibitem[Reimers and Gurevych(2019)]{reimers2019sentence}
Reimers, N. and Gurevych, I. (2019).
\newblock Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks.
\newblock \emph{Conference on Empirical Methods in Natural Language Processing (EMNLP)}.

\bibitem[Schick et al.(2023)]{schick2023toolformer}
Schick, T., Dwivedi-Yu, J., Dess{\`{i}}, R., et al. (2023).
\newblock Toolformer: Language models can teach themselves to use tools.
\newblock \emph{NeurIPS}, 36.

\bibitem[Shao et al.(2024)]{shao2024deepseekmath}
Shao, Z., Wang, P., Zhu, Q., et al. (2024).
\newblock DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open
  Language Models.
\newblock \emph{arXiv:2402.03300}.

\bibitem[Shinn et al.(2023)]{shinn2023reflexion}
Shinn, N., Cassano, F., Gopinath, A., Narasimhan, K., and Yao, S. (2023).
\newblock Reflexion: Language Agents with Verbal Reinforcement Learning.
\newblock \emph{NeurIPS}, 36.

\bibitem[Tencent YouTou Lab(2025)]{tencent2025youtoagent}
Tencent YouTou Lab. (2025).
\newblock YouTou-Agent: Training-Free Experience Extraction for Large Language
  Model Agents.
\newblock \emph{arXiv:2510.08191v1}.

\bibitem[Wang et al.(2023)]{wang2023voyager}
Wang, G., Xie, Y., Jiang, Y., et al. (2023).
\newblock Voyager: An Open-Ended Embodied Agent with Large Language Models.
\newblock \emph{arXiv:2305.16291}.

\bibitem[Williams(1992)]{williams1992simple}
Williams, R. J. (1992).
\newblock Simple statistical gradient-following algorithms for connectionist
  reinforcement learning.
\newblock \emph{Machine Learning}, 8(3--4), 229--256.

\bibitem[Yang et al.(2024)]{yang2024intercode}
Yang, J., Prabhakar, A., Narasimhan, K., and Yao, S. (2024).
\newblock InterCode: Standardizing and Benchmarking Interactive Coding with
  Execution Feedback.
\newblock \emph{NeurIPS}, 36.

\bibitem[Yao et al.(2023)]{yao2023react}
Yao, S., Zhao, J., Yu, D., et al. (2023).
\newblock ReAct: Synergizing Reasoning and Acting in Language Models.
\newblock \emph{International Conference on Learning Representations (ICLR)}.

\bibitem[Zhong et al.(2024)]{zhong2024memorybank}
Zhong, W., Guo, L., Gao, Q., Ye, H., and Wang, Y. (2024).
\newblock MemoryBank: Enhancing Large Language Models with Long-Term Memory.
\newblock \emph{Proceedings of the AAAI Conference on Artificial Intelligence},
  38(17), 19724--19731.

\end{thebibliography}