papers/hanzo-aso/sections/02-background.tex

\section{Background}

\label{sec:background}

\subsection{Autoregressive Language Models}

An autoregressive language model $\pi_\theta$ defines a distribution over token
sequences $y = (y_1, \ldots, y_T)$ conditioned on input $x$:
\begin{equation}
    \pi_\theta(y \mid x) = \prod_{t=1}^{T} \pi_\theta(y_t \mid x, y_{<t}),
\end{equation}
where $y_{<t} = (y_1, \ldots, y_{t-1})$ and each factor is a categorical
distribution over vocabulary $\mathcal{V}$ with $|\mathcal{V}| = V$. The logits
$\ell_t \in \mathbb{R}^V$ are transformed via softmax:
\begin{equation}
    \pi_\theta(y_t = v \mid x, y_{<t}) = \frac{\exp(\ell_{t,v})}{\sum_{v'} \exp(\ell_{t,v'})}.
\end{equation}

\subsection{Group-Relative Policy Optimization}

GRPO~\cite{shao2024deepseekmath} extends REINFORCE~\cite{williams1992simple}
by computing advantages relative to a group of rollouts rather than a learned
baseline. Given group $\{y^{(i)}\}_{i=1}^G$ with rewards $\{r^{(i)}\}_{i=1}^G$,
the advantage is:
\begin{equation}
    A^{(i)} = \frac{r^{(i)} - \bar{r}}{\sigma_r + \epsilon},
    \quad \bar{r} = \frac{1}{G}\sum_{i=1}^G r^{(i)},
    \quad \sigma_r = \sqrt{\frac{1}{G}\sum_{i=1}^G (r^{(i)} - \bar{r})^2}.
\end{equation}
Standard GRPO then performs a gradient update on $\theta$. Our key departure is
to \emph{extract} these advantages as structured text rather than
\emph{differentiate} through them.

\subsection{Product-of-Experts}

A Product-of-Experts (PoE) model~\cite{hinton2002training} combines $M$ expert
distributions multiplicatively:
\begin{equation}
    p_{\text{PoE}}(y) = \frac{1}{Z} \prod_{m=0}^{M} \phi_m(y),
    \quad Z = \sum_{y'} \prod_{m=0}^{M} \phi_m(y'),
\end{equation}
where $\phi_0$ is the base model and $\phi_1, \ldots, \phi_M$ are expert
factors. Unlike Mixture-of-Experts (MoE), PoE implements a logical AND: an
output must be plausible under \emph{all} experts to receive high probability.
This makes PoE naturally suited to incorporating constraints and
preferences~\cite{du2023reducing}.

\subsection{Active Inference and Expected Free Energy}

Active inference~\cite{friston2017active} frames decision-making as minimizing
expected free energy (EFE):
\begin{equation}
    G(\pi) = \underbrace{\mathbb{E}_\pi[\KL[q(s \mid o) \| p(s)]]}_{\text{epistemic value (information gain)}}
           + \underbrace{\mathbb{E}_\pi[-\ln p(o)]}_{\text{pragmatic value (reward)}}.
\end{equation}
ASO incorporates both terms: extrinsic rewards (test pass rates) provide
pragmatic value, while exploration bonuses provide epistemic value.

% ============================================================================
% 3. TRAINING-FREE GRPO (TF-GRPO)
% ============================================================================