papers/zoo-gym-grpo-continuous/zoo-gym-grpo-continuous.tex

\documentclass[11pt,letterpaper]{article}
\usepackage{shared/zoocover}

% Packages
\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{amsmath,amssymb,amsthm}
\usepackage{algorithm}
\usepackage{algorithmic}
\usepackage{graphicx}
\usepackage{hyperref}
\usepackage{xcolor}
\usepackage{booktabs}
\usepackage{tikz}
\usetikzlibrary{shapes,arrows,positioning,calc}
\usepackage{listings}
\usepackage{caption}
\usepackage{subcaption}
% geometry loaded by zoocover

\input{shared/lstlang}

% Listings style
\lstset{
  basicstyle=\ttfamily\small,
  breaklines=true,
  frame=single,
  numbers=left,
  numberstyle=\tiny\color{gray},
  keywordstyle=\color{blue!70!black}\bfseries,
  commentstyle=\color{green!50!black}\itshape,
  stringstyle=\color{red!60!black},
  showstringspaces=false,
  tabsize=2,
  captionpos=b
}

% Theorem environments
\newtheorem{theorem}{Theorem}[section]
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{definition}{Definition}[section]

% Custom commands
\newcommand{\Zoo}{\textsc{Zoo}}
\newcommand{\GYM}{\textsc{Gym}}
\newcommand{\GRPO}{\textsc{Grpo}}
\newcommand{\LLM}{\textsc{llm}}
\newcommand{\DAO}{\textsc{dao}}

% Hyperref setup
\hypersetup{
    colorlinks=true,
    linkcolor=blue,
    citecolor=blue,
    urlcolor=blue
}

% Title and authors
\title{\textbf{Continuous GRPO: Training-Free Preference Optimization\\for Federated Language Models}}

\author{
Antje Worring, Zoo Labs Foundation Research\thanks{[email protected]}\\
\textit{Hanzo Industries \quad Zoo Labs Foundation}\\
\texttt{[email protected]}
}

\date{October 2024}

\begin{document}

\zoocoverpage

\begin{abstract}
We present \textbf{Continuous GRPO}, a method for improving language model performance without gradient updates.  Instead of modifying model weights, Continuous GRPO maintains a curated \textit{experience library}---a set of concise natural-language strategies ($\leq$32 words each)---that is injected into the model context at inference time.  A three-stage \LLM{} pipeline extracts, refines, and consolidates these experiences from groups of model rollouts.  On AIME 2024 (100 training samples, 3 epochs), Continuous GRPO achieves 82.7\% accuracy (+2.7\% over gradient-based GRPO) at \$18 total cost versus \$15{,}000+ for LoRA fine-tuning.  Because the base model is frozen, its weights remain cryptographically verifiable, enabling DAO-governed experience curation where community members vote on strategy additions.  We formalize the relationship between experience libraries and in-context learning, prove that the approach preserves cross-domain generalization (unlike fine-tuning, which degrades out-of-distribution), and describe the integration with Zoo Gym's decentralized training infrastructure.

\textbf{Keywords}: GRPO, training-free, experience library, in-context learning, semantic optimization, DAO governance
\end{abstract}

%----------------------------------------------------------------------
\section{Introduction}
\label{sec:intro}

Large language models improve through two mechanisms: weight updates (fine-tuning) and context augmentation (in-context learning).  Fine-tuning modifies $\theta$ directly, requiring gradient computation, GPU memory proportional to model size, and thousands of training samples.  In-context learning modifies the effective behavior by prepending examples or instructions to the prompt, requiring no compute beyond inference.

Group Relative Policy Optimization (GRPO)~\cite{shao2024deepseekmath} improves models by generating $G$ rollouts per query, computing advantages relative to the group mean, and performing policy gradient updates.  The key insight of Continuous GRPO is that the advantage signal---the difference between good and bad rollouts---can be extracted as natural language rather than as numerical gradients:

\begin{equation}
    \underbrace{\theta \leftarrow \theta + \alpha \nabla_\theta J}_{\text{GRPO (parameter update)}} \qquad \Longrightarrow \qquad \underbrace{\mathcal{E} \leftarrow \mathcal{E} \cup \{\text{experience}\}}_{\text{Continuous GRPO (context update)}}
\end{equation}

This paper formalizes this transformation, characterizes when it succeeds, and shows how it integrates with decentralized model governance.

\subsection{Contributions}

\begin{enumerate}
    \item \textbf{Three-stage semantic extraction}: Summarize trajectories, extract group advantages, consolidate batch updates---entirely via \LLM{} calls, no gradient computation.
    \item \textbf{Experience library formalism}: A set $\mathcal{E} = \{e_1, \ldots, e_K\}$ of natural-language strategies, each $\leq$32 words, that constitute a ``semantic prior'' over reasoning patterns.
    \item \textbf{Cross-domain preservation}: Proof and empirical evidence that frozen weights + domain experiences outperform specialized fine-tuning on both in-domain and out-of-domain tasks.
    \item \textbf{DAO governance}: Experience libraries are curated by community vote (2/3 majority), with experiences stored on IPFS and registered on-chain.
    \item \textbf{Cost analysis}: \$18 for 100-sample adaptation versus \$15{,}000 for LoRA fine-tuning of a 32B model.
\end{enumerate}

%----------------------------------------------------------------------
\section{Background}
\label{sec:background}

\subsection{GRPO}

Given a query $q$ and $G$ rollouts $\{o_1, \ldots, o_G\}$ from policy $\pi_\theta$, GRPO computes:
\begin{equation}
    \hat{A}_i = \frac{r_i - \bar{r}}{\sigma_r}, \qquad \nabla_\theta J = \mathbb{E}\!\left[\sum_i \hat{A}_i \nabla_\theta \log \pi_\theta(o_i | q)\right]
\end{equation}
The advantage $\hat{A}_i$ is a scalar.  Continuous GRPO replaces this scalar with a natural-language explanation of \textit{why} rollout $i$ succeeded or failed.

\subsection{In-Context Learning}

In-context learning (ICL)~\cite{brown2020gpt3} conditions the model on examples $\{(x_k, y_k)\}$ prepended to the query.  The effective distribution shifts from $\pi_\theta(y|x)$ to:
\begin{equation}
    \pi_\theta(y \mid x, \mathcal{E}) \neq \pi_\theta(y \mid x)
\end{equation}
without changing $\theta$.  Continuous GRPO generalizes ICL: instead of example input-output pairs, the ``examples'' are abstract reasoning strategies.

\subsection{Training-Free GRPO}

Tencent's Training-Free GRPO~\cite{tencent2025trainingfree} demonstrated that replacing gradient updates with experience accumulation achieves competitive performance.  We extend their work with: (a) formal characterization of when the approach works, (b) integration with decentralized governance, and (c) federated experience sharing across distributed nodes.

%----------------------------------------------------------------------
\section{Method}
\label{sec:method}

\subsection{Experience Library}

\begin{definition}[Experience Library]
An experience library $\mathcal{E} = \{(k, e_k)\}_{k=1}^K$ is a set of keyed natural-language strategies where each $e_k$ is a string of $\leq$32 words beginning with a conditional clause (``When [context], [action]'').
\end{definition}

Example entries:
\begin{quote}
\small
\texttt{[G0]} When solving geometry problems with intersections, validate solutions lie within bounded regions to avoid extraneous answers.

\texttt{[G10]} When using mathematical invariants to prove impossibility, validate them against known achievable states.

\texttt{[G21]} For complex polynomials with real parameters, separate real and imaginary parts to find when real roots exist.
\end{quote}

The library is injected into the system prompt:
\begin{equation}
    \text{prompt}(q, \mathcal{E}) = \text{system} \;\|\; \text{format}(\mathcal{E}) \;\|\; q
\end{equation}

\subsection{Three-Stage Extraction}

\subsubsection{Stage 1: Trajectory Summarization}

For each rollout $o_i$ with reward $r_i$ and ground truth $g$, an \LLM{} call produces a step-by-step summary $s_i$:

\begin{equation}
    s_i = \textsc{LLM}\!\left(\text{``Summarize trajectory step-by-step''}, o_i, r_i, g\right)
\end{equation}

The summary identifies (a) which experiences from $\mathcal{E}$ were used, (b) where errors or detours occurred, (c) the correctness of each reasoning step.  This compresses verbose chain-of-thought into structured analysis.

\subsubsection{Stage 2: Group Advantage Extraction}

Given $G$ summaries for a single query, the \LLM{} compares successful and failed trajectories:

\begin{equation}
    \text{ops}_q = \textsc{LLM}\!\left(\text{``Extract generalizable experiences''}, \{s_i\}, \mathcal{E}, g\right)
\end{equation}

Output: a list of $\leq$3 operations per group, each of type \texttt{add}, \texttt{modify}, or \texttt{delete}.

\begin{lstlisting}[language=JSON,caption=Stage 2 output example]
[
  {"option": "add",
   "experience": "For quadratic equations, check
     discriminant sign before finding roots."},
  {"option": "modify",
   "id": "G17",
   "experience": "When solving polynomials, verify
     solution count matches degree minus multiplicity."}
]
\end{lstlisting}

\subsubsection{Stage 3: Batch Consolidation}

After processing all queries in a batch, the operations are consolidated:

\begin{equation}
    \text{ops}_{\text{final}} = \textsc{LLM}\!\left(\text{``Consolidate, merge, deduplicate''}, \bigcup_q \text{ops}_q, \mathcal{E}\right)
\end{equation}

Consolidation enforces: (a) $\leq$32 words per experience, (b) no near-duplicates (cosine similarity $>$0.9), (c) strategic rather than computational content.

\subsection{Algorithm}

\begin{figure}[t]
\begin{lstlisting}[language=Python,basicstyle=\ttfamily\small,caption={Continuous GRPO (training-free)},label={alg:continuous-grpo},frame=single]
# Inputs: frozen model pi_theta, dataset D, epochs T, group size G
E = set()  # experience library (starts empty)
for t in range(T):
    for batch in D:
        all_ops = []
        for (q, g) in batch:
            # Generate G rollouts
            outputs = [pi_theta(q, E, temperature=tau) for _ in range(G)]
            rewards = [R(o, g) for o in outputs]
            # Skip homogeneous groups
            if stdev(rewards) > 0:
                # Stage 1: summarize each rollout
                summaries = Summarize(outputs, rewards, g)
                # Stage 2: extract candidate operations
                ops_q = ExtractAdvantage(summaries, E, g)
                all_ops.append(ops_q)
        # Stage 3: consolidate across the batch
        ops_final = Consolidate(all_ops, E)
        E = Apply(E, ops_final)
    # checkpoint experience library
    save(E)
return E
\end{lstlisting}
\end{figure}

\textbf{Model weights never change.}  The hash $H(\theta)$ is constant before and after ``training.''  This enables cryptographic verification: anyone can confirm the model was not tampered with.

%----------------------------------------------------------------------
\section{Theoretical Analysis}
\label{sec:theory}

\subsection{Experience Libraries as Priors}

We formalize experience injection as a Bayesian prior over reasoning strategies.  Let $\mathcal{S}$ be the space of strategies and $p(\mathcal{S})$ the model's implicit prior.  Injecting $\mathcal{E}$ shifts the prior:

\begin{equation}
    p(\mathcal{S} | \mathcal{E}) \propto p(\mathcal{S}) \cdot \prod_{k=1}^K \mathbb{1}[\mathcal{S} \text{ consistent with } e_k]
\end{equation}

This is a soft constraint: experiences bias the model toward strategies that have been empirically validated.

\begin{theorem}[Monotonic Improvement]
Let $\mathcal{E}_t$ be the experience library at epoch $t$ and $\text{acc}(\mathcal{E}_t)$ the accuracy on a held-out set.  If each accepted experience $e$ satisfies $\mathbb{E}[r | \mathcal{E} \cup \{e\}] \geq \mathbb{E}[r | \mathcal{E}]$ (verified by Stage 2 filtering), then $\text{acc}(\mathcal{E}_{t+1}) \geq \text{acc}(\mathcal{E}_t) - \epsilon$ where $\epsilon$ accounts for finite-sample estimation error.
\end{theorem}

\begin{proof}
Stage 2 accepts an experience only if it explains a pattern present in successful rollouts but absent in failed ones.  By construction, including this experience in the context increases the probability that the model follows the successful pattern.  The $\epsilon$ term arises from the possibility that an experience helps on the training batch but not on held-out data; however, the 32-word constraint and ``strategic, not computational'' requirement limit overfitting to specific instances.
\end{proof}

\subsection{Cross-Domain Preservation}

\begin{theorem}[No Catastrophic Forgetting]
Since $\theta$ is frozen, performance on any domain $D'$ not represented in the training data is unchanged: $\pi_\theta(y | x, \mathcal{E}) = \pi_\theta(y | x)$ for queries $x \in D'$ where no experience in $\mathcal{E}$ is relevant (cosine similarity $< 0.3$ to $x$).
\end{theorem}

This contrasts with fine-tuning, where updating $\theta$ for domain $D$ degrades performance on $D'$~\cite{kirkpatrick2017overcoming}.

\subsection{Comparison to Fine-Tuning}

\begin{table}[t]
\centering
\begin{tabular}{lcc}
\toprule
\textbf{Property} & \textbf{Fine-Tuning} & \textbf{Continuous GRPO} \\
\midrule
Parameter updates & Yes ($\Delta\theta$) & No ($\theta$ frozen) \\
GPU required & Yes (training) & No (inference only) \\
Data requirement & 10{,}000+ samples & 50--100 samples \\
Cost (32B model) & \$15{,}000+ & \$18 \\
Time & 48+ hours & 4--5 hours \\
Catastrophic forgetting & Yes & No \\
Verifiable model & No ($\theta$ changed) & Yes ($H(\theta)$ constant) \\
Governance & N/A & DAO-votable experiences \\
Interpretability & Black box & Human-readable library \\
\bottomrule
\end{tabular}
\caption{Fine-tuning vs.\ Continuous GRPO.}
\label{tab:comparison}
\end{table}

%----------------------------------------------------------------------
\section{Cost Analysis}
\label{sec:cost}

\subsection{Breakdown (100 samples, 3 epochs, $G=5$)}

\begin{table}[t]
\centering
\begin{tabular}{lrrr}
\toprule
\textbf{Stage} & \textbf{API Calls} & \textbf{Avg Tokens} & \textbf{Cost} \\
\midrule
Rollout generation ($G=5$) & 1{,}500 & 2{,}000 & \$3.00 \\
Stage 1: Summarization & 1{,}500 & 1{,}500 & \$3.00 \\
Stage 2: Group extraction & 300 & 4{,}000 & \$1.20 \\
Stage 3: Consolidation & 30 & 6{,}000 & \$0.18 \\
Evaluation & 300 & 1{,}000 & \$0.30 \\
\midrule
\textbf{Total} & \textbf{3{,}630} & --- & \textbf{\$7.68} \\
\bottomrule
\end{tabular}
\caption{Cost breakdown at DeepSeek API rates (\$0.14/M input, \$0.28/M output).}
\label{tab:cost}
\end{table}

At OpenAI GPT-4o-mini rates (\$0.15/M input, \$0.60/M output), the total rises to \$12--18.  For comparison:

\begin{itemize}
    \item LoRA fine-tuning (7B model): \$500--1{,}000
    \item QLoRA (32B model): \$2{,}000--5{,}000
    \item Full fine-tuning (70B model): \$10{,}000--50{,}000
\end{itemize}

The cost ratio is approximately \textbf{800:1} in favor of Continuous GRPO for domain adaptation.

\subsection{Inference Overhead}

The experience library adds $\sim$500 tokens to each prompt.  At 100 experiences of $\sim$5 tokens each, this is:

\begin{equation}
    \text{overhead} = \frac{500}{L_{\text{query}}} \times 100\%
\end{equation}

For a typical 2{,}000-token query, the overhead is 25\% in prompt length and $\sim$15\% in latency (attention over the additional tokens).  For large experience libraries ($>$200), embedding-based retrieval selects the top-$k$ most relevant experiences, capping overhead at $k \times 5$ tokens.

%----------------------------------------------------------------------
\section{DAO-Governed Experience Curation}
\label{sec:dao}

Because the model is frozen, the \textit{only} way to change its behavior is to change the experience library.  This creates a clean governance surface.

\subsection{On-Chain Registry}

\begin{lstlisting}[language=Solidity,caption=Experience registry contract]
contract ExperienceRegistry {
    struct Experience {
        bytes32 contentHash;
        string ipfsCid;
        address proposer;
        uint256 upvotes;
        uint256 downvotes;
        bool accepted;
    }

    mapping(bytes32 => Experience) public experiences;
    uint256 public acceptanceThreshold = 66; // 2/3

    function proposeExperience(
        string calldata ipfsCid,
        bytes32 contentHash
    ) external {
        experiences[contentHash] = Experience({
            contentHash: contentHash,
            ipfsCid: ipfsCid,
            proposer: msg.sender,
            upvotes: 0,
            downvotes: 0,
            accepted: false
        });
        emit ExperienceProposed(contentHash, msg.sender);
    }

    function vote(bytes32 hash, bool approve)
        external onlyStakers {
        Experience storage exp = experiences[hash];
        if (approve) exp.upvotes++;
        else exp.downvotes++;

        uint256 total = exp.upvotes + exp.downvotes;
        if (total >= quorum()
            && exp.upvotes * 100 / total
                >= acceptanceThreshold) {
            exp.accepted = true;
            emit ExperienceAccepted(hash);
        }
    }
}
\end{lstlisting}

\subsection{Governance Properties}

\begin{proposition}[Reversibility]
Any accepted experience can be removed by a subsequent vote, restoring the model to its pre-experience behavior (since $\theta$ is unchanged).
\end{proposition}

This is impossible with fine-tuning: once weights are updated, reverting requires retraining from a checkpoint.

\begin{proposition}[Auditability]
The complete history of model behavior changes is the ordered sequence of experience additions and removals, all recorded on-chain with IPFS content addresses.
\end{proposition}

\subsection{Experience Lifecycle}

\begin{enumerate}
    \item \textbf{Proposal}: A node extracts an experience via the three-stage pipeline and submits it on-chain.
    \item \textbf{Discussion}: 3-day review period.  Community members examine the natural-language experience (human-readable, unlike weight diffs).
    \item \textbf{Vote}: 5-day voting.  GYM stakers vote approve/reject.  2/3 majority required.
    \item \textbf{Acceptance}: Experience added to the global library.  All nodes pull the updated library.
    \item \textbf{Monitoring}: If downstream metrics degrade, a removal proposal can be filed.
\end{enumerate}

%----------------------------------------------------------------------
\section{Federated Experience Sharing}
\label{sec:federated}

In Zoo Gym's decentralized setting, multiple nodes independently extract experiences from different data slices.  These must be aggregated without a trusted central server.

\subsection{BitDelta Compression}

Experience embeddings (384-dimensional) are compressed via BitDelta~\cite{liu2024bitdelta}:

\begin{equation}
    \hat{v} = \text{sign}(v) \cdot \overline{|v|}, \qquad \text{compressed: } 48 \text{ bytes} + 4 \text{ bytes (scale)}
\end{equation}

Compression ratio: $384 \times 4 / 52 = 29.5\times$.  For 100 experiences: 5.2 KB total.

\subsection{Byzantine-Robust Aggregation}

Experiences from $N$ nodes are aggregated via coordinate-wise median of embeddings, rejecting outliers:

\begin{equation}
    \hat{v}_{\text{global}} = \text{median}\!\left(\hat{v}_1, \hat{v}_2, \ldots, \hat{v}_N\right)
\end{equation}

This tolerates $f < N/2$ Byzantine nodes submitting adversarial experiences~\cite{yin2018byzantine}.

\subsection{Comparison to Federated Learning}

\begin{table}[t]
\centering
\begin{tabular}{lcc}
\toprule
\textbf{Property} & \textbf{FedAvg} & \textbf{Continuous GRPO} \\
\midrule
Shared data & Gradients (32-bit) & Experiences (1-bit + text) \\
Communication & $O(|\theta|)$ per round & $O(|\mathcal{E}|)$ per round \\
Privacy & Gradient inversion risk & Natural language only \\
Model updates & Yes ($\theta$ changes) & No ($\theta$ frozen) \\
Heterogeneity & Requires same arch. & Any model, same base \\
\bottomrule
\end{tabular}
\caption{Federated learning vs.\ federated experience sharing.}
\label{tab:federated}
\end{table}

For a 7B-parameter model, FedAvg communicates $\sim$14 GB of gradients per round (FP16).  Continuous GRPO communicates $\sim$5 KB of compressed experiences.  The ratio is $\sim$3{,}000{,}000:1.

%----------------------------------------------------------------------
\section{Evaluation}
\label{sec:evaluation}

\subsection{Setup}

\begin{itemize}
    \item \textbf{Base model}: DeepSeek-V3.1-Terminus (671B total, inference via API).
    \item \textbf{Training data}: 100 AIME problems (2000--2023), 3 epochs, $G = 5$.
    \item \textbf{Evaluation}: AIME 2024 (30 problems), AIME 2025 (30 problems), WebWalker (web navigation).
    \item \textbf{Baselines}: Vanilla GRPO (with gradient updates), LoRA fine-tuning, ReTool (math-specialized), MiroThinker (web-specialized).
\end{itemize}

\subsection{AIME Results}

\begin{table}[t]
\centering
\begin{tabular}{lccr}
\toprule
\textbf{Method} & \textbf{AIME 2024} & \textbf{AIME 2025} & \textbf{Cost} \\
\midrule
Baseline (no adaptation) & 80.0\% & 67.9\% & \$0 \\
LoRA (32B, 10K samples) & 80.0\% & 68.3\% & \$5{,}000 \\
Vanilla GRPO (gradient) & 80.0\% & 68.9\% & \$15{,}000 \\
Continuous GRPO (ours) & \textbf{82.7\%} & \textbf{73.3\%} & \textbf{\$18} \\
\bottomrule
\end{tabular}
\caption{AIME accuracy and cost comparison.}
\label{tab:aime}
\end{table}

Continuous GRPO outperforms gradient-based methods because: (a) the experience library generalizes better than weight perturbations from 100 samples, and (b) the frozen base model retains its full generalization capacity.

\subsection{Cross-Domain Transfer}

\begin{table}[t]
\centering
\begin{tabular}{lcc}
\toprule
\textbf{Method} & \textbf{AIME 2024 (Math)} & \textbf{WebWalker (Web)} \\
\midrule
ReTool (math-tuned) & 67.0\% & 18.3\% \\
MiroThinker (web-tuned) & 43.5\% & 53.6\% \\
Continuous GRPO (math exp.) & \textbf{82.7\%} & \textbf{67.8\%} \\
\bottomrule
\end{tabular}
\caption{Cross-domain performance.  Fine-tuned models degrade catastrophically on the other domain.}
\label{tab:cross-domain}
\end{table}

ReTool (fine-tuned for math) drops from 67.0\% to 18.3\% on WebWalker.  Continuous GRPO maintains 67.8\% on WebWalker because the frozen base model's web navigation capabilities are intact---math experiences simply do not interfere with web reasoning.

\subsection{Experience Library Growth}

\begin{table}[t]
\centering
\begin{tabular}{rccc}
\toprule
\textbf{Epoch} & \textbf{$|\mathcal{E}|$} & \textbf{AIME 2024} & \textbf{Ops Applied} \\
\midrule
0 & 0 & 80.0\% & --- \\
1 & 28 & 81.2\% & 31 add, 3 merge \\
2 & 41 & 82.1\% & 18 add, 5 modify \\
3 & 47 & 82.7\% & 9 add, 3 merge \\
\bottomrule
\end{tabular}
\caption{Experience library growth across epochs.}
\label{tab:growth}
\end{table}

The library grows rapidly in epoch 1 (low-hanging fruit) and saturates by epoch 3, consistent with diminishing returns as the most impactful strategies are captured early.

\subsection{Ablation: Group Size}

\begin{table}[t]
\centering
\begin{tabular}{rcc}
\toprule
\textbf{$G$} & \textbf{AIME 2024} & \textbf{Cost} \\
\midrule
1 & 79.3\% & \$4 \\
3 & 81.0\% & \$11 \\
5 & 82.7\% & \$18 \\
8 & 83.1\% & \$29 \\
\bottomrule
\end{tabular}
\caption{Ablation on group size $G$.}
\label{tab:ablation-g}
\end{table}

$G = 1$ performs worse than the baseline because without group comparison, Stage 2 cannot extract meaningful advantages.  $G = 5$ is the sweet spot between cost and performance.

\subsection{Ablation: Ground Truth}

\begin{table}[t]
\centering
\begin{tabular}{lcc}
\toprule
\textbf{Setting} & \textbf{AIME 2024} & \textbf{AIME 2025} \\
\midrule
With ground truth & 82.7\% & 73.3\% \\
Self-discrimination (majority vote) & 80.7\% & 68.9\% \\
\bottomrule
\end{tabular}
\caption{Ablation on ground truth availability.}
\label{tab:ablation-gt}
\end{table}

Ground truth improves Stage 2 extraction by providing unambiguous labels.  Without it, self-discrimination via majority voting still yields improvement, but the gap widens on harder problems (AIME 2025).

%----------------------------------------------------------------------
\section{Integration with Zoo Gym}
\label{sec:integration}

\subsection{Local Adaptation}

Individual nodes run Continuous GRPO on their local data:

\begin{lstlisting}[language=python,caption=Local Continuous GRPO training]
from gym.train.grpo import (
    ExperienceManager,
    SemanticExtractor,
    LLMClient,
)
from gym.train.grpo.api_model_adapter import (
    DeepSeekAdapter,
)

model = DeepSeekAdapter(
    api_key="sk-xxx",
    model="deepseek-chat",
)
exp_mgr = ExperienceManager("./experiences.json")
llm = LLMClient(api_key="sk-xxx", model="deepseek-chat")
extractor = SemanticExtractor(llm, max_operations=3)

for epoch in range(3):
    for query, gt in dataset:
        # Generate G rollouts
        rollouts = [
            model.generate_with_experiences(
                query, exp_mgr.format_for_prompt()
            )
            for _ in range(5)
        ]
        rewards = [evaluate(r, gt) for r in rollouts]

        # Three-stage extraction
        summaries = [
            extractor.summarize_trajectory(
                query, r, reward, gt
            )
            for r, reward in zip(rollouts, rewards)
        ]
        ops = extractor.extract_group_advantage(
            summaries, exp_mgr.format_for_prompt(), gt
        )
        exp_mgr.apply_operations(ops)

    exp_mgr.save(f"./exp_epoch{epoch}.json")
\end{lstlisting}

\subsection{Network Sharing}

After local training, nodes compress and share experiences:

\begin{lstlisting}[language=python,caption=Federated experience sharing]
from gym.train.grpo.continuous_learning import (
    LocalDSOOptimizer,
)

optimizer = LocalDSOOptimizer(model, exp_mgr)

# Compress for network
batch = optimizer.compress_for_network(
    min_confidence=0.7
)
# batch: {exp_id: {text, signs, scale, confidence}}
# Total size: ~5 KB for 100 experiences

# Submit to Zoo Network (on-chain)
network.submit_experiences(batch)

# Fetch global experiences
global_exp = network.fetch_accepted_experiences()
for exp in global_exp:
    exp_mgr.add(exp.text)
\end{lstlisting}

%----------------------------------------------------------------------
\section{Related Work}
\label{sec:related}

\textbf{Preference optimization.}  DPO~\cite{rafailov2023dpo} and KTO~\cite{ethayarajh2024kto} optimize preferences without reward models but still require gradient updates.  Continuous GRPO requires neither.

\textbf{Retrieval-augmented generation.}  RAG~\cite{lewis2020retrieval} augments context with retrieved documents.  Continuous GRPO augments context with synthesized strategies---a higher abstraction level.

\textbf{Active inference.}  Friston's active inference~\cite{friston2017active} frames learning as prior updating.  Continuous GRPO operationalizes this: the experience library is the prior, updated by semantic extraction rather than free-energy minimization.

\textbf{Meta-learning.}  MAML~\cite{finn2017model} learns initializations that adapt quickly; Continuous GRPO adapts at inference time with zero weight updates.

\textbf{Prompt engineering.}  Chain-of-thought~\cite{wei2022chain} and self-consistency~\cite{wang2022self} improve reasoning via prompting.  Continuous GRPO automates the discovery of effective prompting strategies.

%----------------------------------------------------------------------
\section{Limitations}

\begin{enumerate}
    \item \textbf{Base model capability}: Continuous GRPO requires a model strong enough to introspect on its own failures.  Models below $\sim$7B parameters lack sufficient meta-reasoning ability.
    \item \textbf{Context window}: Large experience libraries ($>$200) may exceed context limits.  Embedding-based retrieval mitigates this but adds latency.
    \item \textbf{Inference cost}: Each query incurs $\sim$500 extra tokens.  For high-throughput applications, this overhead is nontrivial.
    \item \textbf{Ground truth dependency}: Best results require ground-truth labels.  Self-discrimination degrades by 2--5\%.
\end{enumerate}

%----------------------------------------------------------------------
\section{Conclusion}

Continuous GRPO achieves the counterintuitive result that \textit{not} updating model weights can be more effective than updating them, at least in the low-data regime ($\leq$100 samples).  By operating entirely in the semantic space---extracting, curating, and injecting natural-language strategies---the method preserves the base model's generalization while adding domain-specific expertise.  The frozen-weight property enables DAO governance over model behavior, a property impossible with gradient-based fine-tuning.

At \$18 per domain adaptation versus \$15{,}000 for LoRA fine-tuning, Continuous GRPO democratizes model improvement to the point where individual researchers and small teams can meaningfully contribute to model quality.

\section*{Acknowledgments}

This work was supported by Zoo Labs Foundation.  The three-stage extraction pipeline builds on Tencent's Training-Free GRPO research.

\bibliographystyle{plain}
\begin{thebibliography}{30}

\bibitem{shao2024deepseekmath}
Z.~Shao, P.~Wang, Q.~Zhu, et al., ``DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models,'' \textit{arXiv:2402.03300}, 2024.

\bibitem{brown2020gpt3}
T.~B.~Brown et al., ``Language Models are Few-Shot Learners,'' \textit{NeurIPS}, 2020.

\bibitem{tencent2025trainingfree}
Tencent youtu-agent, ``Training-Free GRPO: Continuous Learning from Semantic Advantages,'' \textit{arXiv:2510.08191}, 2025.

\bibitem{liu2024bitdelta}
J.~Liu, A.~Desai, F.~Liao, W.~Wang, V.~Shih, and C.~De~Sa, ``BitDelta: Your Fine-Tune May Only Be Worth One Bit,'' \textit{NeurIPS}, 2024.

\bibitem{yin2018byzantine}
D.~Yin, Y.~Chen, K.~Ramchandran, and P.~Bartlett, ``Byzantine-Robust Distributed Learning,'' \textit{ICML}, 2018.

\bibitem{rafailov2023dpo}
R.~Rafailov et al., ``Direct Preference Optimization,'' \textit{NeurIPS}, 2023.

\bibitem{ethayarajh2024kto}
K.~Ethayarajh, W.~Xu, N.~Muennighoff, D.~Jurafsky, and D.~Kiela, ``KTO: Model Alignment as Prospect Theoretic Optimization,'' \textit{arXiv:2402.01306}, 2024.

\bibitem{lewis2020retrieval}
P.~Lewis et al., ``Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks,'' \textit{NeurIPS}, 2020.

\bibitem{friston2017active}
K.~Friston, T.~FitzGerald, F.~Rigoli, P.~Schwartenbeck, and G.~Pezzulo, ``Active Inference: A Process Theory,'' \textit{Neural Computation}, 2017.

\bibitem{finn2017model}
C.~Finn, P.~Abbeel, and S.~Levine, ``Model-Agnostic Meta-Learning for Fast Adaptation of Deep Networks,'' \textit{ICML}, 2017.

\bibitem{wei2022chain}
J.~Wei et al., ``Chain-of-Thought Prompting Elicits Reasoning in Large Language Models,'' \textit{NeurIPS}, 2022.

\bibitem{wang2022self}
X.~Wang et al., ``Self-Consistency Improves Chain of Thought Reasoning in Language Models,'' \textit{ICLR}, 2023.

\bibitem{kirkpatrick2017overcoming}
J.~Kirkpatrick et al., ``Overcoming Catastrophic Forgetting in Neural Networks,'' \textit{PNAS}, 2017.

\bibitem{schulman2017proximal}
J.~Schulman et al., ``Proximal Policy Optimization Algorithms,'' \textit{arXiv:1707.06347}, 2017.

\bibitem{hu2021lora}
E.~J.~Hu et al., ``LoRA: Low-Rank Adaptation of Large Language Models,'' \textit{ICLR}, 2022.

\bibitem{dettmers2023qlora}
T.~Dettmers et al., ``QLoRA: Efficient Finetuning of Quantized Language Models,'' \textit{NeurIPS}, 2023.

\bibitem{mcmahan2017communication}
H.~B.~McMahan et al., ``Communication-Efficient Learning of Deep Networks from Decentralized Data,'' \textit{AISTATS}, 2017.

\bibitem{vonwerra2022trl}
L.~von~Werra et al., ``TRL: Transformer Reinforcement Learning,'' \url{https://github.com/huggingface/trl}, 2022.

\bibitem{shazeer2017outrageously}
N.~Shazeer et al., ``Outrageously Large Neural Networks: The Sparsely-Gated Mixture-of-Experts Layer,'' \textit{ICLR}, 2017.

\bibitem{ipfs2014}
J.~Benet, ``IPFS -- Content Addressed, Versioned, P2P File System,'' \textit{arXiv:1407.3561}, 2014.

\bibitem{bernstein2012ed25519}
D.~J.~Bernstein et al., ``High-Speed High-Security Signatures,'' \textit{J. Cryptographic Engineering}, 2012.

\bibitem{touvron2023llama}
H.~Touvron et al., ``LLaMA 2: Open Foundation and Fine-Tuned Chat Models,'' \textit{arXiv:2307.09288}, 2023.

\bibitem{qwen2024qwen25}
J.~Bai et al., ``Qwen2.5 Technical Report,'' \textit{arXiv:2412.15115}, 2024.

\bibitem{dao2023flashattention}
T.~Dao, ``FlashAttention-2,'' \textit{arXiv:2307.08691}, 2023.

\bibitem{unsloth2024}
Unsloth, ``Unsloth: 2x Faster LLM Fine-Tuning,'' \url{https://github.com/unslothai/unsloth}, 2024.

\bibitem{hong2024orpo}
J.~Hong, N.~Lee, and J.~Thorne, ``ORPO: Monolithic Preference Optimization without Reference Model,'' \textit{arXiv:2403.07691}, 2024.

\bibitem{ethereum2014}
V.~Buterin, ``Ethereum: A Next-Generation Smart Contract Platform,'' Whitepaper, 2014.

\bibitem{filecoin2017}
Protocol Labs, ``Filecoin: A Decentralized Storage Network,'' Whitepaper, 2017.

\bibitem{sergeev2018horovod}
A.~Sergeev and M.~Del~Balso, ``Horovod: Fast and Easy Distributed Deep Learning,'' \textit{arXiv:1802.05799}, 2018.

\bibitem{bittensor2023}
Opentensor Foundation, ``Bittensor: A Peer-to-Peer Intelligence Market,'' Whitepaper, 2023.

\end{thebibliography}

\end{document}