Files
zeekay 004979143d fix(branding): move decorator to external /zen-decorate.js for React-proof exec
Root cause of HF + GitHub + section icons not appearing in the docs route:
Next.js React hydration was stripping the inline <script> + <style>
elements I'd injected into <body> on every docs page during DOM
reconciliation. The script never registered its setInterval.

Move the decorator to /zen-decorate.js loaded via <script defer src>
in <head>. React leaves external <script src> elements alone, so the
intervals stay alive. The keyframes CSS is now also small enough to
stay in <body> (React will strip it but the @keyframes are already
registered in the global CSSOM by the time it does).
2026-05-19 20:08:58 -07:00

161 lines
64 KiB
HTML
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html><!--i_dnJM_MIpJSOCQWNJVMq--><html lang="en" class="dark"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="stylesheet" href="/_next/static/chunks/f2332aac77592f9d.css" data-precedence="next"/><link rel="preload" as="script" fetchPriority="low" href="/_next/static/chunks/e83606e8fa9cc796.js"/><script src="/_next/static/chunks/36d6595f0156cd7e.js" async=""></script><script src="/_next/static/chunks/040e9cea20a8d9c7.js" async=""></script><script src="/_next/static/chunks/d4dffb5a0973d49c.js" async=""></script><script src="/_next/static/chunks/turbopack-967f27a9fd33556d.js" async=""></script><script src="/_next/static/chunks/59d0ad1b64f8544e.js" async=""></script><script src="/_next/static/chunks/4d80e004cf4896dd.js" async=""></script><script src="/_next/static/chunks/350ee4303b732916.js" async=""></script><script src="/_next/static/chunks/8de849ca74fc071f.js" async=""></script><script src="/_next/static/chunks/e62b91212ee7f8ff.js" async=""></script><script src="/_next/static/chunks/f19fe44237e54646.js" async=""></script><script src="/_next/static/chunks/cb0a883bafeb6805.js" async=""></script><meta name="theme-color" media="(prefers-color-scheme: dark)" content="#0A0A0A"/><meta name="theme-color" media="(prefers-color-scheme: light)" content="#fff"/><title>GRPO: Group Relative Policy Optimization — Zen LM Blog | Zen LM</title><meta name="description" content="Introducing GRPO, a new approach to reinforcement learning from human feedback that improves sample efficiency and alignment stability."/><meta property="og:title" content="Zen LM - Open Foundation Models"/><meta property="og:description" content="Zen AI model family by Hanzo AI. 14 frontier models from 4B to 1T+ parameters for code, reasoning, vision, and multimodal tasks."/><meta property="og:url" content="https://zenlm.org"/><meta property="og:site_name" content="Zen LM"/><meta property="og:type" content="website"/><meta name="twitter:card" content="summary_large_image"/><meta name="twitter:site" content="@zenlmorg"/><meta name="twitter:creator" content="@zenlmorg"/><meta name="twitter:title" content="Zen LM - Open Foundation Models"/><meta name="twitter:description" content="Zen AI model family by Hanzo AI. 14 frontier models from 4B to 1T+ parameters for code, reasoning, vision, and multimodal tasks."/><script src="/_next/static/chunks/a6dad97d9634a72d.js" noModule=""></script><script defer src="/zen-decorate.js"></script></head><body class="antialiased"><div hidden=""><!--$--><!--/$--></div><script>((a,b,c,d,e,f,g,h)=>{let i=document.documentElement,j=["light","dark"];function k(b){var c;(Array.isArray(a)?a:[a]).forEach(a=>{let c="class"===a,d=c&&f?e.map(a=>f[a]||a):e;c?(i.classList.remove(...d),i.classList.add(f&&f[b]?f[b]:b)):i.setAttribute(a,b)}),c=b,h&&j.includes(c)&&(i.style.colorScheme=c)}if(d)k(d);else try{let a=localStorage.getItem(b)||c,d=g&&"system"===a?window.matchMedia("(prefers-color-scheme: dark)").matches?"dark":"light":a;k(d)}catch(a){}})("class","theme","system",null,["light","dark"],null,true,true)</script><!--$--><div data-closed="" role="presentation" hidden="" style="user-select:none;-webkit-user-select:none" class="fixed inset-0 z-50 backdrop-blur-xs bg-fd-overlay data-open:animate-fd-fade-in data-closed:animate-fd-fade-out"></div><div class="bg-fd-secondary/50 p-3 empty:hidden"></div><!--/$--><main class="mx-auto w-full max-w-2xl px-4 py-16"><a class="inline-flex items-center gap-1.5 text-sm text-fd-muted-foreground hover:text-fd-foreground mb-10 transition-colors" href="/blog">← Back to Blog</a><div class="mb-8"><time class="text-xs font-mono text-fd-muted-foreground uppercase tracking-wider">September 18, 2022</time><h1 class="text-3xl font-bold mt-2 mb-3">GRPO: Group Relative Policy Optimization</h1><p class="text-fd-muted-foreground text-lg mb-4">Introducing GRPO, a new approach to reinforcement learning from human feedback that improves sample efficiency and alignment stability.</p><div class="flex items-center gap-3 pt-4 border-t border-fd-border"><span class="text-sm text-fd-muted-foreground">By <!-- -->Zach Kelling</span><div class="flex gap-1.5 ml-auto"><span class="text-[10px] font-mono uppercase tracking-wider bg-fd-muted text-fd-muted-foreground px-1.5 py-0.5 rounded">Research</span><span class="text-[10px] font-mono uppercase tracking-wider bg-fd-muted text-fd-muted-foreground px-1.5 py-0.5 rounded">Alignment</span><span class="text-[10px] font-mono uppercase tracking-wider bg-fd-muted text-fd-muted-foreground px-1.5 py-0.5 rounded">Training</span></div></div></div><div class="prose dark:prose-invert max-w-none"><p>Reinforcement learning from human feedback (RLHF) has become central to aligning language models with human preferences. But current methods like PPO are sample-inefficient and unstable. Today we introduce Group Relative Policy Optimization (GRPO), a new approach that addresses these limitations.</p>
<h2 class="flex scroll-m-28 flex-row items-center gap-2" id="the-rlhf-challenge"><a data-card="" href="#the-rlhf-challenge" class="peer">The RLHF Challenge</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h2>
<p>Standard RLHF follows three steps:</p>
<ol>
<li>Train a reward model on human preference data</li>
<li>Use the reward model to provide training signal</li>
<li>Optimize the policy with reinforcement learning (typically PPO)</li>
</ol>
<p>Step 3 is problematic. PPO requires careful hyperparameter tuning, extensive sampling, and still often produces unstable training dynamics.</p>
<h2 class="flex scroll-m-28 flex-row items-center gap-2" id="key-insight-relative-comparisons"><a data-card="" href="#key-insight-relative-comparisons" class="peer">Key Insight: Relative Comparisons</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h2>
<p>Humans naturally make relative judgments. “Response A is better than B” comes more easily than “Response A scores 7.3.” Yet reward models output absolute scores that discard this relational structure.</p>
<p>GRPO preserves relative comparisons throughout training.</p>
<h2 class="flex scroll-m-28 flex-row items-center gap-2" id="the-grpo-algorithm"><a data-card="" href="#the-grpo-algorithm" class="peer">The GRPO Algorithm</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h2>
<p>Instead of optimizing absolute rewards, GRPO optimizes within groups of sampled responses.</p>
<h3 class="flex scroll-m-28 flex-row items-center gap-2" id="sampling"><a data-card="" href="#sampling" class="peer">Sampling</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h3>
<p>For each prompt $x$, sample a group of $k$ responses:</p>
<p><code>G = \{y_1, y_2, …, y_k\} \sim \pi_\theta(y|x)</code></p>
<h3 class="flex scroll-m-28 flex-row items-center gap-2" id="ranking"><a data-card="" href="#ranking" class="peer">Ranking</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h3>
<p>Rank responses within the group using the reward model:</p>
<p><code>r_i = R(x, y_i)</code> <code>\text\{rank\}(y_i) = |\{j : r_j &gt; r_i\}| + 1</code></p>
<h3 class="flex scroll-m-28 flex-row items-center gap-2" id="relative-advantage"><a data-card="" href="#relative-advantage" class="peer">Relative Advantage</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h3>
<p>Compute advantages relative to the group:</p>
<p><code>A_i = \frac\{r_i - \mu_G\}\{\sigma_G\}</code></p>
<p>where $\mu_G$ and $\sigma_G$ are the group mean and standard deviation.</p>
<h3 class="flex scroll-m-28 flex-row items-center gap-2" id="policy-update"><a data-card="" href="#policy-update" class="peer">Policy Update</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h3>
<p>Update the policy to increase probability of high-ranked responses:</p>
<p><code>\mathcal\{L\}(\theta) = -\mathbb\{E\}_\{y \sim G\}\left[\min\left(\frac\{\pi_ \theta(y|x)\}\{\pi_\{\text\{old\}\}(y|x)\} A, \text\{clip\}\left(\frac\{\pi_\theta(y|x)\}\{\pi_\{\text\{old\}\}(y|x)\}, 1-\epsilon, 1+\epsilon\right) A\right)\right]</code></p>
<p>The clipping stabilizes training, similar to PPO, but the relative advantages provide better signal.</p>
<h2 class="flex scroll-m-28 flex-row items-center gap-2" id="why-grpo-works"><a data-card="" href="#why-grpo-works" class="peer">Why GRPO Works</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h2>
<h3 class="flex scroll-m-28 flex-row items-center gap-2" id="robust-to-reward-scale"><a data-card="" href="#robust-to-reward-scale" class="peer">Robust to Reward Scale</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h3>
<p>Absolute reward values vary across prompts and reward model calibration. Normalization within groups removes this sensitivity.</p>
<h3 class="flex scroll-m-28 flex-row items-center gap-2" id="better-gradient-signal"><a data-card="" href="#better-gradient-signal" class="peer">Better Gradient Signal</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h3>
<p>Groups provide multiple comparison points per prompt. This reduces variance and improves sample efficiency.</p>
<h3 class="flex scroll-m-28 flex-row items-center gap-2" id="natural-curriculum"><a data-card="" href="#natural-curriculum" class="peer">Natural Curriculum</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h3>
<p>Hard prompts where all responses score poorly still provide useful gradients. The best-in-group response gets positive advantage even if absolute rewards are low.</p>
<h3 class="flex scroll-m-28 flex-row items-center gap-2" id="reduced-reward-hacking"><a data-card="" href="#reduced-reward-hacking" class="peer">Reduced Reward Hacking</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h3>
<p>Optimizing relative rankings is harder to game than optimizing absolute scores. The model must genuinely improve, not find reward model exploits.</p>
<h2 class="flex scroll-m-28 flex-row items-center gap-2" id="experimental-results"><a data-card="" href="#experimental-results" class="peer">Experimental Results</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h2>
<p>We compared GRPO to PPO on alignment benchmarks:</p>
<div class="relative overflow-auto prose-no-margin my-6"><table><thead><tr><th>Method</th><th>Helpfulness</th><th>Harmlessness</th><th>Honesty</th><th>Samples Required</th></tr></thead><tbody><tr><td>PPO</td><td>78.2</td><td>82.1</td><td>75.4</td><td>100K</td></tr><tr><td>GRPO</td><td>81.7</td><td>84.3</td><td>79.1</td><td>35K</td></tr></tbody></table></div>
<p>GRPO achieves better alignment with 3x fewer samples.</p>
<p>Training dynamics also improve significantly:</p>
<ul>
<li><strong>Stability</strong> : GRPO loss curves show less variance</li>
<li><strong>Convergence</strong> : Reaches final performance 2x faster</li>
<li><strong>Robustness</strong> : Less sensitive to learning rate choice</li>
</ul>
<h2 class="flex scroll-m-28 flex-row items-center gap-2" id="implementation-details"><a data-card="" href="#implementation-details" class="peer">Implementation Details</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h2>
<p>Key hyperparameters:</p>
<ul>
<li><strong>Group size</strong> ($k$): 8-16 works well</li>
<li><strong>Clipping</strong> ($\epsilon$): 0.1-0.2</li>
<li><strong>KL penalty</strong> : Lower than PPO (0.01 vs 0.1)</li>
</ul>
<p>The larger group size compared to PPO’s typical 2-response setup is essential. More comparisons mean better gradient estimates.</p>
<h2 class="flex scroll-m-28 flex-row items-center gap-2" id="code-release"><a data-card="" href="#code-release" class="peer">Code Release</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h2>
<p>We’re releasing our GRPO implementation integrated with the Zen training framework:</p>
<figure dir="ltr" class="my-4 bg-fd-card rounded-xl shiki relative border shadow-sm not-prose overflow-hidden text-sm shiki shiki-themes github-light github-dark" style="--shiki-light:#24292e;--shiki-dark:#e1e4e8;--shiki-light-bg:#fff;--shiki-dark-bg:#24292e" tabindex="-1"><div class="empty:hidden absolute top-3 right-2 z-2 backdrop-blur-lg rounded-lg text-fd-muted-foreground"><button type="button" class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors duration-100 disabled:pointer-events-none disabled:opacity-50 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-fd-ring p-1 [&amp;_svg]:size-4 hover:text-fd-accent-foreground data-checked:text-fd-accent-foreground" aria-label="Copy Text"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-clipboard" aria-hidden="true"><rect width="8" height="4" x="8" y="2" rx="1" ry="1"></rect><path d="M16 4h2a2 2 0 0 1 2 2v14a2 2 0 0 1-2 2H6a2 2 0 0 1-2-2V6a2 2 0 0 1 2-2h2"></path></svg></button></div><div role="region" tabindex="0" class="text-[0.8125rem] py-3.5 overflow-auto max-h-[600px] fd-scroll-container focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-inset focus-visible:ring-fd-ring" style="--padding-right:calc(var(--spacing) * 8)"><pre class="min-w-full w-max *:flex *:flex-col"><code><span class="line"></span>
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583"> from</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> zen.alignment </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">import</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> GRPOTrainer</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> </span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> trainer </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> GRPOTrainer(</span></span>
<span class="line"><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70"> model</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">model,</span></span>
<span class="line"><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70"> reward_model</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">reward_model,</span></span>
<span class="line"><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70"> group_size</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF">12</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">,</span></span>
<span class="line"><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70"> clip_epsilon</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF">0.15</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">,</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> )</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> </span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> trainer.train(prompts)</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> </span></span></code></pre></div></figure>
<p>Full documentation and examples at github.com/zoo-labs/zen-align.</p>
<h2 class="flex scroll-m-28 flex-row items-center gap-2" id="whats-next"><a data-card="" href="#whats-next" class="peer">What’s Next</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h2>
<p>GRPO opens several research directions:</p>
<ul>
<li><strong>Multi-objective GRPO</strong> : Separate groups for different alignment dimensions</li>
<li><strong>Online preference learning</strong> : Update reward model during training</li>
<li><strong>Constitutional GRPO</strong> : Use principles instead of learned rewards</li>
</ul>
<p>Alignment is the central challenge of AI development. GRPO is one step toward making it more reliable.</p>
<hr/>
<p><em>Zach Kelling is a co-founder of Zoo Labs Foundation.</em></p></div></main><!--$--><!--/$--><script src="/_next/static/chunks/e83606e8fa9cc796.js" id="_R_" async=""></script><script>(self.__next_f=self.__next_f||[]).push([0])</script><script>self.__next_f.push([1,"1:\"$Sreact.fragment\"\n2:I[106,[\"/_next/static/chunks/59d0ad1b64f8544e.js\"],\"RootProvider\"]\n3:I[53113,[\"/_next/static/chunks/4d80e004cf4896dd.js\",\"/_next/static/chunks/350ee4303b732916.js\"],\"default\"]\n4:I[73211,[\"/_next/static/chunks/4d80e004cf4896dd.js\",\"/_next/static/chunks/350ee4303b732916.js\"],\"default\"]\n5:I[10086,[\"/_next/static/chunks/59d0ad1b64f8544e.js\",\"/_next/static/chunks/8de849ca74fc071f.js\",\"/_next/static/chunks/e62b91212ee7f8ff.js\",\"/_next/static/chunks/f19fe44237e54646.js\",\"/_next/static/chunks/cb0a883bafeb6805.js\"],\"\"]\n7:I[89923,[\"/_next/static/chunks/4d80e004cf4896dd.js\",\"/_next/static/chunks/350ee4303b732916.js\"],\"OutletBoundary\"]\n8:\"$Sreact.suspense\"\na:I[89923,[\"/_next/static/chunks/4d80e004cf4896dd.js\",\"/_next/static/chunks/350ee4303b732916.js\"],\"ViewportBoundary\"]\nc:I[89923,[\"/_next/static/chunks/4d80e004cf4896dd.js\",\"/_next/static/chunks/350ee4303b732916.js\"],\"MetadataBoundary\"]\ne:I[6998,[\"/_next/static/chunks/4d80e004cf4896dd.js\",\"/_next/static/chunks/350ee4303b732916.js\"],\"default\"]\n:HL[\"/_next/static/chunks/f2332aac77592f9d.css\",\"style\"]\n"])</script><script>self.__next_f.push([1,"0:{\"P\":null,\"b\":\"i-dnJM_MIpJSOCQWNJVMq\",\"c\":[\"\",\"blog\",\"grpo\"],\"q\":\"\",\"i\":false,\"f\":[[[\"\",{\"children\":[\"blog\",{\"children\":[[\"slug\",\"grpo\",\"d\"],{\"children\":[\"__PAGE__\",{}]}]}]},\"$undefined\",\"$undefined\",true],[[\"$\",\"$1\",\"c\",{\"children\":[[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/chunks/f2332aac77592f9d.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\",\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-0\",{\"src\":\"/_next/static/chunks/59d0ad1b64f8544e.js\",\"async\":true,\"nonce\":\"$undefined\"}]],[\"$\",\"html\",null,{\"lang\":\"en\",\"className\":\"dark\",\"suppressHydrationWarning\":true,\"children\":[\"$\",\"body\",null,{\"className\":\"antialiased\",\"children\":[\"$\",\"$L2\",null,{\"children\":[\"$\",\"$L3\",null,{\"parallelRouterKey\":\"children\",\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L4\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":[[\"$\",\"main\",null,{\"className\":\"flex min-h-screen flex-col items-center justify-center px-4 text-center\",\"children\":[[\"$\",\"div\",null,{\"className\":\"mb-8 opacity-20\",\"children\":[\"$\",\"svg\",null,{\"width\":\"120\",\"height\":\"120\",\"viewBox\":\"0 0 120 120\",\"fill\":\"none\",\"aria-hidden\":\"true\",\"children\":[\"$\",\"circle\",null,{\"cx\":\"60\",\"cy\":\"60\",\"r\":\"50\",\"stroke\":\"currentColor\",\"strokeWidth\":\"3\",\"strokeLinecap\":\"round\",\"strokeDasharray\":\"280 40\"}]}]}],[\"$\",\"p\",null,{\"className\":\"text-xs font-mono uppercase tracking-[0.3em] text-fd-muted-foreground mb-4\",\"children\":\"404\"}],[\"$\",\"h1\",null,{\"className\":\"text-3xl font-semibold mb-3\",\"children\":\"Page not found\"}],[\"$\",\"p\",null,{\"className\":\"text-fd-muted-foreground max-w-sm mb-10\",\"children\":\"This page doesn't exist, or it may have moved. Try the documentation or head home.\"}],[\"$\",\"div\",null,{\"className\":\"flex flex-wrap gap-3 justify-center\",\"children\":[[\"$\",\"$L5\",null,{\"href\":\"/\",\"className\":\"inline-flex items-center gap-2 rounded-lg bg-fd-primary text-fd-primary-foreground px-4 py-2 text-sm font-medium hover:opacity-90 transition\",\"children\":\"Go home\"}],[\"$\",\"$L5\",null,{\"href\":\"/docs\",\"className\":\"inline-flex items-center gap-2 rounded-lg border border-fd-border bg-fd-background px-4 py-2 text-sm font-medium hover:bg-fd-muted transition\",\"children\":\"Documentation\"}],[\"$\",\"$L5\",null,{\"href\":\"/docs/models\",\"className\":\"inline-flex items-center gap-2 rounded-lg border border-fd-border bg-fd-background px-4 py-2 text-sm font-medium hover:bg-fd-muted transition\",\"children\":\"Browse models\"}]]}],[\"$\",\"p\",null,{\"className\":\"mt-16 text-xs font-mono text-fd-muted-foreground opacity-50\",\"children\":\"zenlm.org\"}]]}],[]],\"forbidden\":\"$undefined\",\"unauthorized\":\"$undefined\"}]}]}]}]]}],{\"children\":[[\"$\",\"$1\",\"c\",{\"children\":[null,[\"$\",\"$L3\",null,{\"parallelRouterKey\":\"children\",\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L4\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":\"$undefined\",\"forbidden\":\"$undefined\",\"unauthorized\":\"$undefined\"}]]}],{\"children\":[[\"$\",\"$1\",\"c\",{\"children\":[null,[\"$\",\"$L3\",null,{\"parallelRouterKey\":\"children\",\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L4\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":\"$undefined\",\"forbidden\":\"$undefined\",\"unauthorized\":\"$undefined\"}]]}],{\"children\":[[\"$\",\"$1\",\"c\",{\"children\":[\"$L6\",[[\"$\",\"script\",\"script-0\",{\"src\":\"/_next/static/chunks/8de849ca74fc071f.js\",\"async\":true,\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-1\",{\"src\":\"/_next/static/chunks/e62b91212ee7f8ff.js\",\"async\":true,\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-2\",{\"src\":\"/_next/static/chunks/f19fe44237e54646.js\",\"async\":true,\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-3\",{\"src\":\"/_next/static/chunks/cb0a883bafeb6805.js\",\"async\":true,\"nonce\":\"$undefined\"}]],[\"$\",\"$L7\",null,{\"children\":[\"$\",\"$8\",null,{\"name\":\"Next.MetadataOutlet\",\"children\":\"$@9\"}]}]]}],{},null,false,false]},null,false,false]},null,false,false]},null,false,false],[\"$\",\"$1\",\"h\",{\"children\":[null,[\"$\",\"$La\",null,{\"children\":\"$Lb\"}],[\"$\",\"div\",null,{\"hidden\":true,\"children\":[\"$\",\"$Lc\",null,{\"children\":[\"$\",\"$8\",null,{\"name\":\"Next.Metadata\",\"children\":\"$Ld\"}]}]}],null]}],false]],\"m\":\"$undefined\",\"G\":[\"$e\",[]],\"S\":true}\n"])</script><script>self.__next_f.push([1,"6:[\"$\",\"main\",null,{\"className\":\"mx-auto w-full max-w-2xl px-4 py-16\",\"children\":[[\"$\",\"$L5\",null,{\"href\":\"/blog\",\"className\":\"inline-flex items-center gap-1.5 text-sm text-fd-muted-foreground hover:text-fd-foreground mb-10 transition-colors\",\"children\":\"← Back to Blog\"}],[\"$\",\"div\",null,{\"className\":\"mb-8\",\"children\":[[\"$\",\"time\",null,{\"className\":\"text-xs font-mono text-fd-muted-foreground uppercase tracking-wider\",\"children\":\"September 18, 2022\"}],[\"$\",\"h1\",null,{\"className\":\"text-3xl font-bold mt-2 mb-3\",\"children\":\"GRPO: Group Relative Policy Optimization\"}],[\"$\",\"p\",null,{\"className\":\"text-fd-muted-foreground text-lg mb-4\",\"children\":\"Introducing GRPO, a new approach to reinforcement learning from human feedback that improves sample efficiency and alignment stability.\"}],[\"$\",\"div\",null,{\"className\":\"flex items-center gap-3 pt-4 border-t border-fd-border\",\"children\":[[\"$\",\"span\",null,{\"className\":\"text-sm text-fd-muted-foreground\",\"children\":[\"By \",\"Zach Kelling\"]}],[\"$\",\"div\",null,{\"className\":\"flex gap-1.5 ml-auto\",\"children\":[[\"$\",\"span\",\"Research\",{\"className\":\"text-[10px] font-mono uppercase tracking-wider bg-fd-muted text-fd-muted-foreground px-1.5 py-0.5 rounded\",\"children\":\"Research\"}],[\"$\",\"span\",\"Alignment\",{\"className\":\"text-[10px] font-mono uppercase tracking-wider bg-fd-muted text-fd-muted-foreground px-1.5 py-0.5 rounded\",\"children\":\"Alignment\"}],[\"$\",\"span\",\"Training\",{\"className\":\"text-[10px] font-mono uppercase tracking-wider bg-fd-muted text-fd-muted-foreground px-1.5 py-0.5 rounded\",\"children\":\"Training\"}]]}]]}]]}],[\"$\",\"div\",null,{\"className\":\"prose dark:prose-invert max-w-none\",\"children\":[[\"$\",\"p\",null,{\"children\":\"Reinforcement learning from human feedback (RLHF) has become central to aligning language models with human preferences. But current methods like PPO are sample-inefficient and unstable. Today we introduce Group Relative Policy Optimization (GRPO), a new approach that addresses these limitations.\"}],\"\\n\",[\"$\",\"h2\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"the-rlhf-challenge\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#the-rlhf-challenge\",\"className\":\"peer\",\"children\":\"The RLHF Challenge\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}],\"\\n\",[\"$\",\"p\",null,{\"children\":\"Standard RLHF follows three steps:\"}],\"\\n\",[\"$\",\"ol\",null,{\"children\":[\"\\n\",[\"$\",\"li\",null,{\"children\":\"Train a reward model on human preference data\"}],\"\\n\",[\"$\",\"li\",null,{\"children\":\"Use the reward model to provide training signal\"}],\"\\n\",[\"$\",\"li\",null,{\"children\":\"Optimize the policy with reinforcement learning (typically PPO)\"}],\"\\n\"]}],\"\\n\",[\"$\",\"p\",null,{\"children\":\"Step 3 is problematic. PPO requires careful hyperparameter tuning, extensive sampling, and still often produces unstable training dynamics.\"}],\"\\n\",[\"$\",\"h2\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"key-insight-relative-comparisons\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#key-insight-relative-comparisons\",\"className\":\"peer\",\"children\":\"Key Insight: Relative Comparisons\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],\"$Lf\",\"$undefined\"]}]]}],\"\\n\",\"$L10\",\"\\n\",\"$L11\",\"\\n\",\"$L12\",\"\\n\",\"$L13\",\"\\n\",\"$L14\",\"\\n\",\"$L15\",\"\\n\",\"$L16\",\"\\n\",\"$L17\",\"\\n\",\"$L18\",\"\\n\",\"$L19\",\"\\n\",\"$L1a\",\"\\n\",\"$L1b\",\"\\n\",\"$L1c\",\"\\n\",\"$L1d\",\"\\n\",\"$L1e\",\"\\n\",\"$L1f\",\"\\n\",\"$L20\",\"\\n\",\"$L21\",\"\\n\",\"$L22\",\"\\n\",\"$L23\",\"\\n\",\"$L24\",\"\\n\",\"$L25\",\"\\n\",\"$L26\",\"\\n\",\"$L27\",\"\\n\",\"$L28\",\"\\n\",\"$L29\",\"\\n\",\"$L2a\",\"\\n\",\"$L2b\",\"\\n\",\"$L2c\",\"\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\",\"$L2d\",\"\\n\",\"$L2e\",\"\\n\",\"$L2f\",\"\\n\",\"$L30\",\"\\n\",\"$L31\",\"\\n\",\"$L32\",\"\\n\",\"$L33\",\"\\n\",\"$L34\",\"\\n\",\"$L35\",\"\\n\",\"$L36\",\"\\n\",\"$L37\",\"\\n\",\"$L38\",\"\\n\",\"$L39\",\"\\n\",\"$L3a\",\"\\n\",\"$L3b\",\"\\n\",\"$L3c\",\"\\n\",\"$L3d\",\"\\n\",\"$L3e\"]}]]}]\n"])</script><script>self.__next_f.push([1,"3f:I[51504,[\"/_next/static/chunks/59d0ad1b64f8544e.js\",\"/_next/static/chunks/8de849ca74fc071f.js\",\"/_next/static/chunks/e62b91212ee7f8ff.js\",\"/_next/static/chunks/f19fe44237e54646.js\",\"/_next/static/chunks/cb0a883bafeb6805.js\"],\"CodeBlock\"]\n41:I[51504,[\"/_next/static/chunks/59d0ad1b64f8544e.js\",\"/_next/static/chunks/8de849ca74fc071f.js\",\"/_next/static/chunks/e62b91212ee7f8ff.js\",\"/_next/static/chunks/f19fe44237e54646.js\",\"/_next/static/chunks/cb0a883bafeb6805.js\"],\"Pre\"]\nf:[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}]\n10:[\"$\",\"p\",null,{\"children\":\"Humans naturally make relative judgments. “Response A is better than B” comes more easily than “Response A scores 7.3.” Yet reward models output absolute scores that discard this relational structure.\"}]\n11:[\"$\",\"p\",null,{\"children\":\"GRPO preserves relative comparisons throughout training.\"}]\n12:[\"$\",\"h2\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"the-grpo-algorithm\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#the-grpo-algorithm\",\"className\":\"peer\",\"children\":\"The GRPO Algorithm\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n13:[\"$\",\"p\",null,{\"children\":\"Instead of optimizing absolute rewards, GRPO optimizes within groups of sampled responses.\"}]\n14:[\"$\",\"h3\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"sampling\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#sampling\",\"className\":\"peer\",\"children\":\"Sampling\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n15:[\"$\",\"p\",null,{\"children\":\"For each prompt $x$, sample a group of $k$ responses:\"}]\n16:[\"$\",\"p\",null,{\"children\":[\"$\",\"code\",null,{\"children\":\"G = \\\\{y_1, y_2, …, y_k\\\\} \\\\sim \\\\pi_\\\\theta(y|x)\"}]}]\n17:[\"$\",\"h3\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"ranking\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#ranking\",\"className\":\"peer\",\"children\":\"Ranking\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n18:[\"$\",\"p\",null,{\"children\":\"Rank responses within the group using the reward model:\"}]\n19:[\"$\",\"p\",null,{\"children\":[[\"$\",\"code\",null,{\"children\":\"r_i = R(x, y_i)\"}],\" \",[\"$\",\"code\",null,{\"children\":\"\\\\text\\\\{rank\\\\}(y_i) = |\\\\{j : r_j \u003e r_i\\\\}| + 1\"}]]}]\n1a:[\"$\",\"h3\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"relative-advantage\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#relative-advantage\",\"className\":\"peer\",\"children\":\"Relative Advantage\"}],[\"$\",\"svg\",null,{\"re"])</script><script>self.__next_f.push([1,"f\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n1b:[\"$\",\"p\",null,{\"children\":\"Compute advantages relative to the group:\"}]\n1c:[\"$\",\"p\",null,{\"children\":[\"$\",\"code\",null,{\"children\":\"A_i = \\\\frac\\\\{r_i - \\\\mu_G\\\\}\\\\{\\\\sigma_G\\\\}\"}]}]\n1d:[\"$\",\"p\",null,{\"children\":\"where $\\\\mu_G$ and $\\\\sigma_G$ are the group mean and standard deviation.\"}]\n1e:[\"$\",\"h3\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"policy-update\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#policy-update\",\"className\":\"peer\",\"children\":\"Policy Update\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n1f:[\"$\",\"p\",null,{\"children\":\"Update the policy to increase probability of high-ranked responses:\"}]\n20:[\"$\",\"p\",null,{\"children\":[\"$\",\"code\",null,{\"children\":\"\\\\mathcal\\\\{L\\\\}(\\\\theta) = -\\\\mathbb\\\\{E\\\\}_\\\\{y \\\\sim G\\\\}\\\\left[\\\\min\\\\left(\\\\frac\\\\{\\\\pi_ \\\\theta(y|x)\\\\}\\\\{\\\\pi_\\\\{\\\\text\\\\{old\\\\}\\\\}(y|x)\\\\} A, \\\\text\\\\{clip\\\\}\\\\left(\\\\frac\\\\{\\\\pi_\\\\theta(y|x)\\\\}\\\\{\\\\pi_\\\\{\\\\text\\\\{old\\\\}\\\\}(y|x)\\\\}, 1-\\\\epsilon, 1+\\\\epsilon\\\\right) A\\\\right)\\\\right]\"}]}]\n21:[\"$\",\"p\",null,{\"children\":\"The clipping stabilizes training, similar to PPO, but the relative advantages provide better signal.\"}]\n22:[\"$\",\"h2\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"why-grpo-works\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#why-grpo-works\",\"className\":\"peer\",\"children\":\"Why GRPO Works\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n23:[\"$\",\"h3\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"robust-to-reward-scale\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#robust-to-reward-scale\",\"className\":\"peer\",\"children\":\"Robust to Reward Scale\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n24:[\"$\",\"p\",null,{\"children\":\"Absolute reward values vary across prompts and reward model calibration. Normalization within groups removes this sensitivity.\"}]\n25:[\"$\",\"h3\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\""])</script><script>self.__next_f.push([1,"better-gradient-signal\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#better-gradient-signal\",\"className\":\"peer\",\"children\":\"Better Gradient Signal\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n26:[\"$\",\"p\",null,{\"children\":\"Groups provide multiple comparison points per prompt. This reduces variance and improves sample efficiency.\"}]\n27:[\"$\",\"h3\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"natural-curriculum\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#natural-curriculum\",\"className\":\"peer\",\"children\":\"Natural Curriculum\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n28:[\"$\",\"p\",null,{\"children\":\"Hard prompts where all responses score poorly still provide useful gradients. The best-in-group response gets positive advantage even if absolute rewards are low.\"}]\n29:[\"$\",\"h3\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"reduced-reward-hacking\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#reduced-reward-hacking\",\"className\":\"peer\",\"children\":\"Reduced Reward Hacking\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n2a:[\"$\",\"p\",null,{\"children\":\"Optimizing relative rankings is harder to game than optimizing absolute scores. The model must genuinely improve, not find reward model exploits.\"}]\n2b:[\"$\",\"h2\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"experimental-results\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#experimental-results\",\"className\":\"peer\",\"children\":\"Experimental Results\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n2c:[\"$\",\"p\",null,{\"children\":\"We compared GRPO to PPO on alignment benchmarks:\"}]\n2d:[\"$\",\"div\",null,{\"className\":\"relative overflow-auto prose-no-margin my-6\",\"children\":[\"$\",\"table\",null,{\"children\":[[\"$\",\"thead\",null,{\"children\":[\"$\",\"tr\",null,{\"children\":[[\"$\",\"th\",null,{\"children\":\"Method\"}],[\"$\",\"th\",null,{\"children\":\"Helpfulness\"}],[\"$\",\"th\",null,{\"children\":\"Harmlessness\"}],[\"$\",\"th\",null,{\""])</script><script>self.__next_f.push([1,"children\":\"Honesty\"}],[\"$\",\"th\",null,{\"children\":\"Samples Required\"}]]}]}],[\"$\",\"tbody\",null,{\"children\":[[\"$\",\"tr\",null,{\"children\":[[\"$\",\"td\",null,{\"children\":\"PPO\"}],[\"$\",\"td\",null,{\"children\":\"78.2\"}],[\"$\",\"td\",null,{\"children\":\"82.1\"}],[\"$\",\"td\",null,{\"children\":\"75.4\"}],[\"$\",\"td\",null,{\"children\":\"100K\"}]]}],[\"$\",\"tr\",null,{\"children\":[[\"$\",\"td\",null,{\"children\":\"GRPO\"}],[\"$\",\"td\",null,{\"children\":\"81.7\"}],[\"$\",\"td\",null,{\"children\":\"84.3\"}],[\"$\",\"td\",null,{\"children\":\"79.1\"}],[\"$\",\"td\",null,{\"children\":\"35K\"}]]}]]}]]}]}]\n2e:[\"$\",\"p\",null,{\"children\":\"GRPO achieves better alignment with 3x fewer samples.\"}]\n2f:[\"$\",\"p\",null,{\"children\":\"Training dynamics also improve significantly:\"}]\n30:[\"$\",\"ul\",null,{\"children\":[\"\\n\",[\"$\",\"li\",null,{\"children\":[[\"$\",\"strong\",null,{\"children\":\"Stability\"}],\" : GRPO loss curves show less variance\"]}],\"\\n\",[\"$\",\"li\",null,{\"children\":[[\"$\",\"strong\",null,{\"children\":\"Convergence\"}],\" : Reaches final performance 2x faster\"]}],\"\\n\",[\"$\",\"li\",null,{\"children\":[[\"$\",\"strong\",null,{\"children\":\"Robustness\"}],\" : Less sensitive to learning rate choice\"]}],\"\\n\"]}]\n31:[\"$\",\"h2\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"implementation-details\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#implementation-details\",\"className\":\"peer\",\"children\":\"Implementation Details\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n32:[\"$\",\"p\",null,{\"children\":\"Key hyperparameters:\"}]\n33:[\"$\",\"ul\",null,{\"children\":[\"\\n\",[\"$\",\"li\",null,{\"children\":[[\"$\",\"strong\",null,{\"children\":\"Group size\"}],\" ($k$): 8-16 works well\"]}],\"\\n\",[\"$\",\"li\",null,{\"children\":[[\"$\",\"strong\",null,{\"children\":\"Clipping\"}],\" ($\\\\epsilon$): 0.1-0.2\"]}],\"\\n\",[\"$\",\"li\",null,{\"children\":[[\"$\",\"strong\",null,{\"children\":\"KL penalty\"}],\" : Lower than PPO (0.01 vs 0.1)\"]}],\"\\n\"]}]\n34:[\"$\",\"p\",null,{\"children\":\"The larger group size compared to PPO’s typical 2-response setup is essential. More comparisons mean better gradient estimates.\"}]\n35:[\"$\",\"h2\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"code-release\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#code-release\",\"className\":\"peer\",\"children\":\"Code Release\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n36:[\"$\",\"p\",null,{\"children\":\"We’re releasing our GRPO implementation integrated with the Zen training framework:\"}]\n40:T5c0,"])</script><script>self.__next_f.push([1,"\u003csvg viewBox=\"0 0 24 24\"\u003e\u003cpath d=\"M14.25.18l.9.2.73.26.59.3.45.32.34.34.25.34.16.33.1.3.04.26.02.2-.01.13V8.5l-.05.63-.13.55-.21.46-.26.38-.3.31-.33.25-.35.19-.35.14-.33.1-.3.07-.26.04-.21.02H8.77l-.69.05-.59.14-.5.22-.41.27-.33.32-.27.35-.2.36-.15.37-.1.35-.07.32-.04.27-.02.21v3.06H3.17l-.21-.03-.28-.07-.32-.12-.35-.18-.36-.26-.36-.36-.35-.46-.32-.59-.28-.73-.21-.88-.14-1.05-.05-1.23.06-1.22.16-1.04.24-.87.32-.71.36-.57.4-.44.42-.33.42-.24.4-.16.36-.1.32-.05.24-.01h.16l.06.01h8.16v-.83H6.18l-.01-2.75-.02-.37.05-.34.11-.31.17-.28.25-.26.31-.23.38-.2.44-.18.51-.15.58-.12.64-.1.71-.06.77-.04.84-.02 1.27.05zm-6.3 1.98l-.23.33-.08.41.08.41.23.34.33.22.41.09.41-.09.33-.22.23-.34.08-.41-.08-.41-.23-.33-.33-.22-.41-.09-.41.09zm13.09 3.95l.28.06.32.12.35.18.36.27.36.35.35.47.32.59.28.73.21.88.14 1.04.05 1.23-.06 1.23-.16 1.04-.24.86-.32.71-.36.57-.4.45-.42.33-.42.24-.4.16-.36.09-.32.05-.24.02-.16-.01h-8.22v.82h5.84l.01 2.76.02.36-.05.34-.11.31-.17.29-.25.25-.31.24-.38.2-.44.17-.51.15-.58.13-.64.09-.71.07-.77.04-.84.01-1.27-.04-1.07-.14-.9-.2-.73-.25-.59-.3-.45-.33-.34-.34-.25-.34-.16-.33-.1-.3-.04-.25-.02-.2.01-.13v-5.34l.05-.64.13-.54.21-.46.26-.38.3-.32.33-.24.35-.2.35-.14.33-.1.3-.06.26-.04.21-.02.13-.01h5.84l.69-.05.59-.14.5-.21.41-.28.33-.32.27-.35.2-.36.15-.36.1-.35.07-.32.04-.28.02-.21V6.07h2.09l.14.01zm-6.47 14.25l-.23.33-.08.41.08.41.23.33.33.23.41.08.41-.08.33-.23.23-.33.08-.41-.08-.41-.23-.33-.33-.23-.41-.08-.41.08z\" fill=\"currentColor\" /\u003e\u003c/svg\u003e"])</script><script>self.__next_f.push([1,"37:[\"$\",\"$L3f\",null,{\"className\":\"shiki shiki-themes github-light github-dark\",\"style\":{\"--shiki-light\":\"#24292e\",\"--shiki-dark\":\"#e1e4e8\",\"--shiki-light-bg\":\"#fff\",\"--shiki-dark-bg\":\"#24292e\"},\"tabIndex\":\"0\",\"icon\":\"$40\",\"children\":[\"$\",\"$L41\",null,{\"children\":[\"$\",\"code\",null,{\"children\":[[\"$\",\"span\",null,{\"className\":\"line\"}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\" from\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" zen.alignment \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"import\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" GRPOTrainer\"}]]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" \"}]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" trainer \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"=\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" GRPOTrainer(\"}]]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#E36209\",\"--shiki-dark\":\"#FFAB70\"},\"children\":\" model\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"=\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\"model,\"}]]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#E36209\",\"--shiki-dark\":\"#FFAB70\"},\"children\":\" reward_model\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"=\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\"reward_model,\"}]]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#E36209\",\"--shiki-dark\":\"#FFAB70\"},\"children\":\" group_size\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"=\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#005CC5\",\"--shiki-dark\":\"#79B8FF\"},\"children\":\"12\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\",\"}]]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#E36209\",\"--shiki-dark\":\"#FFAB70\"},\"children\":\" clip_epsilon\"}],\"$L42\",\"$L43\",\"$L44\"]}],\"\\n\",\"$L45\",\"\\n\",\"$L46\",\"\\n\",\"$L47\",\"\\n\",\"$L48\"]}]}]}]\n"])</script><script>self.__next_f.push([1,"38:[\"$\",\"p\",null,{\"children\":\"Full documentation and examples at github.com/zoo-labs/zen-align.\"}]\n39:[\"$\",\"h2\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"whats-next\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#whats-next\",\"className\":\"peer\",\"children\":\"What’s Next\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n3a:[\"$\",\"p\",null,{\"children\":\"GRPO opens several research directions:\"}]\n3b:[\"$\",\"ul\",null,{\"children\":[\"\\n\",[\"$\",\"li\",null,{\"children\":[[\"$\",\"strong\",null,{\"children\":\"Multi-objective GRPO\"}],\" : Separate groups for different alignment dimensions\"]}],\"\\n\",[\"$\",\"li\",null,{\"children\":[[\"$\",\"strong\",null,{\"children\":\"Online preference learning\"}],\" : Update reward model during training\"]}],\"\\n\",[\"$\",\"li\",null,{\"children\":[[\"$\",\"strong\",null,{\"children\":\"Constitutional GRPO\"}],\" : Use principles instead of learned rewards\"]}],\"\\n\"]}]\n3c:[\"$\",\"p\",null,{\"children\":\"Alignment is the central challenge of AI development. GRPO is one step toward making it more reliable.\"}]\n3d:[\"$\",\"hr\",null,{}]\n3e:[\"$\",\"p\",null,{\"children\":[\"$\",\"em\",null,{\"children\":\"Zach Kelling is a co-founder of Zoo Labs Foundation.\"}]}]\n"])</script><script>self.__next_f.push([1,"42:[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"=\"}]\n43:[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#005CC5\",\"--shiki-dark\":\"#79B8FF\"},\"children\":\"0.15\"}]\n44:[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\",\"}]\n45:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" )\"}]}]\n46:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" \"}]}]\n47:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" trainer.train(prompts)\"}]}]\n48:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" \"}]}]\n"])</script><script>self.__next_f.push([1,"b:[[\"$\",\"meta\",\"0\",{\"charSet\":\"utf-8\"}],[\"$\",\"meta\",\"1\",{\"name\":\"viewport\",\"content\":\"width=device-width, initial-scale=1\"}],[\"$\",\"meta\",\"2\",{\"name\":\"theme-color\",\"media\":\"(prefers-color-scheme: dark)\",\"content\":\"#0A0A0A\"}],[\"$\",\"meta\",\"3\",{\"name\":\"theme-color\",\"media\":\"(prefers-color-scheme: light)\",\"content\":\"#fff\"}]]\n"])</script><script>self.__next_f.push([1,"9:null\nd:[[\"$\",\"title\",\"0\",{\"children\":\"GRPO: Group Relative Policy Optimization — Zen LM Blog | Zen LM\"}],[\"$\",\"meta\",\"1\",{\"name\":\"description\",\"content\":\"Introducing GRPO, a new approach to reinforcement learning from human feedback that improves sample efficiency and alignment stability.\"}],[\"$\",\"meta\",\"2\",{\"property\":\"og:title\",\"content\":\"Zen LM - Open Foundation Models\"}],[\"$\",\"meta\",\"3\",{\"property\":\"og:description\",\"content\":\"Zen AI model family by Hanzo AI. 14 frontier models from 4B to 1T+ parameters for code, reasoning, vision, and multimodal tasks.\"}],[\"$\",\"meta\",\"4\",{\"property\":\"og:url\",\"content\":\"https://zenlm.org\"}],[\"$\",\"meta\",\"5\",{\"property\":\"og:site_name\",\"content\":\"Zen LM\"}],[\"$\",\"meta\",\"6\",{\"property\":\"og:type\",\"content\":\"website\"}],[\"$\",\"meta\",\"7\",{\"name\":\"twitter:card\",\"content\":\"summary_large_image\"}],[\"$\",\"meta\",\"8\",{\"name\":\"twitter:site\",\"content\":\"@zenlmorg\"}],[\"$\",\"meta\",\"9\",{\"name\":\"twitter:creator\",\"content\":\"@zenlmorg\"}],[\"$\",\"meta\",\"10\",{\"name\":\"twitter:title\",\"content\":\"Zen LM - Open Foundation Models\"}],[\"$\",\"meta\",\"11\",{\"name\":\"twitter:description\",\"content\":\"Zen AI model family by Hanzo AI. 14 frontier models from 4B to 1T+ parameters for code, reasoning, vision, and multimodal tasks.\"}]]\n"])</script><script>
(function(){
// After React renders the lotus emoji from RSC payload, walk the DOM
// and replace it with the enso SVG. Polls so subsequent React re-renders
// also get cleaned up.
function swap(){
// 1. Replace lotus + ' Zen LM' text inside <a> brand links
document.querySelectorAll('a').forEach(function(a){
if (a.dataset.zenLotusSwapped) return;
var t = (a.textContent || '');
if (!/.*Zen LM|Zen LM.*/.test(t)) return;
// Find the lotus text node specifically
var walker = document.createTreeWalker(a, NodeFilter.SHOW_TEXT, null);
var node;
while ((node = walker.nextNode())) {
if (node.nodeValue && node.nodeValue.indexOf('') !== -1) {
node.nodeValue = node.nodeValue.replace(/\s?/g, '');
}
}
// Inject enso SVG at start
if (!a.querySelector('svg[aria-label="Zen LM"]')) {
var ns = 'http://www.w3.org/2000/svg';
var svg = document.createElementNS(ns, 'svg');
svg.setAttribute('viewBox', '0 0 512 512');
svg.setAttribute('fill', 'none');
svg.setAttribute('width', '18');
svg.setAttribute('height', '18');
svg.setAttribute('aria-label', 'Zen LM');
svg.style.cssText = 'display:inline-block;vertical-align:-2px;margin-right:6px;flex-shrink:0';
var p = document.createElementNS(ns, 'path');
p.setAttribute('d', 'M 256 84 A 172 172 0 1 0 256 428 A 172 172 0 1 0 256 84 Z');
p.setAttribute('stroke', 'currentColor');
p.setAttribute('stroke-width', '26');
p.setAttribute('stroke-linecap', 'round');
p.setAttribute('stroke-linejoin', 'round');
svg.appendChild(p);
a.insertBefore(svg, a.firstChild);
}
a.dataset.zenLotusSwapped = '1';
});
}
function start(){ swap(); setInterval(swap, 600); }
if (document.readyState !== 'loading') start();
else document.addEventListener('DOMContentLoaded', start);
})();
</script>
<style id="zenlm-brand-anim">
@keyframes zenlm-brand-draw { to { stroke-dashoffset: 0; } }
svg.zenlm-brand-svg .zenlm-brand-path {
stroke-dasharray: 1100;
stroke-dashoffset: 1100;
animation: zenlm-brand-draw 1.4s cubic-bezier(.45,0,.2,1) 0.1s 1 forwards;
}
.zen-link-icon { display: inline-block; vertical-align: middle; flex-shrink: 0; }
</style></body></html>