papers/sections/tf-grpo.tex

% Training-Free GRPO section
\section{Training-Free GRPO (TF-GRPO)}
\subsection{Group-Relative Objective}
For a group of rollouts \(\{y^{(i)}\}_{i=1}^G\), we compute a group-relative advantage combining extrinsic reward \(r^{(i)}\) and epistemic utility \(u^{(i)}\):
\begin{equation}\label{eq:tf-grpo-objective}
g^{(i)} = \alpha r^{(i)} + \beta u^{(i)},
\end{equation}
where \(\alpha, \beta \ge 0\) are hyperparameters. The advantages are centered and whitened within the group:
\begin{equation}
A^{(i)} = \frac{g^{(i)} - \bar g}{\sigma_g + \epsilon},
\end{equation}
where \(\bar g = \frac{1}{G}\sum_i g^{(i)}\) and \(\sigma_g^2 = \frac{1}{G}\sum_i (g^{(i)} - \bar g)^2\).

\subsection{Epistemic Utility (Expected Free Energy)}
Following Active Inference, the epistemic term quantifies information gain:
\begin{equation}
u^{(i)} = \mathbb{H}[p(s)] - \mathbb{E}_{s\sim p(s\mid y^{(i)})}[\mathbb{H}[p(s\mid y^{(i)})]],
\end{equation}
where \(s\) represents latent states or model beliefs. This encourages exploration of high-information-gain trajectories.

\subsection{Zero-Training Adaptation}
Unlike standard GRPO which updates model parameters \(\theta\), TF-GRPO extracts \emph{semantic advantages} as token-level or embedding-level priors \(E\) that modify decoding without gradient updates. These priors are compressed and shared across nodes (see \S\ref{sec:bitdelta}).