papers/hanzo-agent-grpo/hanzo-agent-grpo.tex

% Hanzo Agent-GRPO Paper
% Fully self-contained -- no \input{} directives
\documentclass[11pt]{article}
\usepackage{../shared/hanzocover}
\usepackage{amsmath, amssymb, amsthm}
\usepackage{mathtools}
\usepackage{bm}
\usepackage{graphicx}
\usepackage{booktabs}
\usepackage{hyperref}
\usepackage{enumitem}
\usepackage{algorithm}
\usepackage{algpseudocode}
\usepackage{xcolor}
\usepackage[numbers]{natbib}
\usepackage{float}
\usepackage{subcaption}
\usepackage{tabularx}
\usepackage{multirow}

\hypersetup{colorlinks=true,linkcolor=black,citecolor=blue,urlcolor=blue}

\newtheorem{theorem}{Theorem}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{definition}{Definition}
\newtheorem{remark}{Remark}
\newtheorem{assumption}{Assumption}

\DeclareMathOperator*{\argmin}{arg\,min}
\DeclareMathOperator*{\argmax}{arg\,max}
\DeclareMathOperator{\KL}{KL}
\DeclareMathOperator{\Tr}{Tr}
\DeclareMathOperator{\diag}{diag}
\DeclareMathOperator{\sign}{sign}
\DeclareMathOperator{\softmax}{softmax}
\DeclareMathOperator{\cosim}{sim}

\title{Training-Free Continuous Learning for Tool-Using Agents\\
via GRPO Semantic Experience Management}
\author{
    Zach Kelling \\
    Hanzo AI Research\\
    \textit{Hanzo AI Inc (Techstars '17), Los Angeles, CA}\\
    \texttt{[email protected]}
}
\date{February 2026}

\begin{document}

\hanzocoverpage

% ============================================================================
% ABSTRACT
% ============================================================================
\begin{abstract}
We present \textbf{Agent-GRPO}, a training-free continuous learning framework
that extends Group-Relative Policy Optimization (GRPO) from text-output
optimization to the agent tool-use setting. While Active Semantic Optimization
(ASO) extracts generalizable patterns from grouped language model rollouts,
agent trajectories carry strictly richer structure: sequences of tool
selections, parameter choices, intermediate outputs, and explicit success or
failure signals that are unavailable from text alone. Agent-GRPO adapts the
three-stage semantic extraction pipeline of \cite{tencent2025youtoagent} to
this richer domain, constructing a \emph{semantic experience library}
$\mathcal{E}$ whose entries encode tool-use strategies, failure patterns, and
domain conventions distilled from past agent sessions.

We formalize the tool-use trajectory as a typed tuple
$\tau = \bigl(x,\,\{(t_i, \bm{p}_i, o_i, s_i)\}_{i=1}^K,\, R\bigr)$, define
a composite reward combining binary tool success and user acceptance signals,
and prove that the expected fraction of beneficial experiences in $\mathcal{E}$
grows monotonically under mild regularity conditions (Theorem~\ref{thm:monotone}).
We further derive a sample complexity bound of $O(K\,/\,\varepsilon^2)$ sessions
to achieve $\varepsilon$-improvement over the baseline (Theorem~\ref{thm:sample}).

Empirically, a Hanzo Dev agent equipped with Agent-GRPO achieves a
\textbf{31.4\%} reduction in tool-call errors and a \textbf{22.7\%} improvement
in task completion rate on the HB-Bench tool-use benchmark after 50 sessions of
online experience collection, at a total adaptation cost below \$20 in API
calls---orders of magnitude cheaper than any fine-tuning alternative. We also
show how Agent-GRPO integrates naturally with the four-loop self-improvement
architecture of ASO, making the feedback loop between telemetry, active
learning, reflection, and maintenance fully closed.
\end{abstract}

% ============================================================================
% 1. INTRODUCTION
% ============================================================================

\input{sections/01-introduction}
\input{sections/02-background}
\input{sections/03-agent-level-grpo}
\input{sections/04-semantic-memory-manager}
\input{sections/05-agent-specific-adaptations}
\input{sections/06-integration-with-the-four-loop-self-impr}
\input{sections/07-theoretical-analysis}
\input{sections/08-experiments}
\input{sections/09-related-work}
\input{sections/10-discussion-and-future-work}
\input{sections/99-conclusion}

\section*{Reproducibility}
\textbf{Source code.} \texttt{github.com/hanzoai/agents} (commit \texttt{TBD}); GRPO loop in \texttt{python/hanzo\_harness/grpo}.\\
\textbf{Build.} \texttt{uv sync \&\& pytest tests/grpo}; reference rollouts use Zen-Nano via \texttt{hanzo-engine}.\\
\textbf{Hardware tested.} Apple M2/M3 (Metal); x86\_64 Linux with NVIDIA A100/H100 for batch evaluations.\\
\textbf{Standards referenced.} OpenAI tool-use API, MCP, GRPO (DeepSeek R1 family).\\
\textbf{Contact.} [email protected].

\end{document}