Files
zen-blog/blog/grpo-group-relative-policy-optimization.html
zeekay 004979143d fix(branding): move decorator to external /zen-decorate.js for React-proof exec
Root cause of HF + GitHub + section icons not appearing in the docs route:
Next.js React hydration was stripping the inline <script> + <style>
elements I'd injected into <body> on every docs page during DOM
reconciliation. The script never registered its setInterval.

Move the decorator to /zen-decorate.js loaded via <script defer src>
in <head>. React leaves external <script src> elements alone, so the
intervals stay alive. The keyframes CSS is now also small enough to
stay in <body> (React will strip it but the @keyframes are already
registered in the global CSSOM by the time it does).
2026-05-19 20:08:58 -07:00

240 lines
91 KiB
HTML
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html><!--i_dnJM_MIpJSOCQWNJVMq--><html lang="en" class="dark"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="stylesheet" href="/_next/static/chunks/f2332aac77592f9d.css" data-precedence="next"/><link rel="preload" as="script" fetchPriority="low" href="/_next/static/chunks/e83606e8fa9cc796.js"/><script src="/_next/static/chunks/36d6595f0156cd7e.js" async=""></script><script src="/_next/static/chunks/040e9cea20a8d9c7.js" async=""></script><script src="/_next/static/chunks/d4dffb5a0973d49c.js" async=""></script><script src="/_next/static/chunks/turbopack-967f27a9fd33556d.js" async=""></script><script src="/_next/static/chunks/59d0ad1b64f8544e.js" async=""></script><script src="/_next/static/chunks/4d80e004cf4896dd.js" async=""></script><script src="/_next/static/chunks/350ee4303b732916.js" async=""></script><script src="/_next/static/chunks/8de849ca74fc071f.js" async=""></script><script src="/_next/static/chunks/e62b91212ee7f8ff.js" async=""></script><script src="/_next/static/chunks/f19fe44237e54646.js" async=""></script><script src="/_next/static/chunks/cb0a883bafeb6805.js" async=""></script><meta name="theme-color" media="(prefers-color-scheme: dark)" content="#0A0A0A"/><meta name="theme-color" media="(prefers-color-scheme: light)" content="#fff"/><title>GRPO: Group Relative Policy Optimization — Zen LM Blog | Zen LM</title><meta name="description" content="A companion post to our GRPO paper, explaining group relative policy optimization for language model alignment."/><meta property="og:title" content="Zen LM - Open Foundation Models"/><meta property="og:description" content="Zen AI model family by Hanzo AI. 14 frontier models from 4B to 1T+ parameters for code, reasoning, vision, and multimodal tasks."/><meta property="og:url" content="https://zenlm.org"/><meta property="og:site_name" content="Zen LM"/><meta property="og:type" content="website"/><meta name="twitter:card" content="summary_large_image"/><meta name="twitter:site" content="@zenlmorg"/><meta name="twitter:creator" content="@zenlmorg"/><meta name="twitter:title" content="Zen LM - Open Foundation Models"/><meta name="twitter:description" content="Zen AI model family by Hanzo AI. 14 frontier models from 4B to 1T+ parameters for code, reasoning, vision, and multimodal tasks."/><script src="/_next/static/chunks/a6dad97d9634a72d.js" noModule=""></script><script defer src="/zen-decorate.js"></script></head><body class="antialiased"><div hidden=""><!--$--><!--/$--></div><script>((a,b,c,d,e,f,g,h)=>{let i=document.documentElement,j=["light","dark"];function k(b){var c;(Array.isArray(a)?a:[a]).forEach(a=>{let c="class"===a,d=c&&f?e.map(a=>f[a]||a):e;c?(i.classList.remove(...d),i.classList.add(f&&f[b]?f[b]:b)):i.setAttribute(a,b)}),c=b,h&&j.includes(c)&&(i.style.colorScheme=c)}if(d)k(d);else try{let a=localStorage.getItem(b)||c,d=g&&"system"===a?window.matchMedia("(prefers-color-scheme: dark)").matches?"dark":"light":a;k(d)}catch(a){}})("class","theme","system",null,["light","dark"],null,true,true)</script><!--$--><div data-closed="" role="presentation" hidden="" style="user-select:none;-webkit-user-select:none" class="fixed inset-0 z-50 backdrop-blur-xs bg-fd-overlay data-open:animate-fd-fade-in data-closed:animate-fd-fade-out"></div><div class="bg-fd-secondary/50 p-3 empty:hidden"></div><!--/$--><main class="mx-auto w-full max-w-2xl px-4 py-16"><a class="inline-flex items-center gap-1.5 text-sm text-fd-muted-foreground hover:text-fd-foreground mb-10 transition-colors" href="/blog">← Back to Blog</a><div class="mb-8"><time class="text-xs font-mono text-fd-muted-foreground uppercase tracking-wider">September 17, 2022</time><h1 class="text-3xl font-bold mt-2 mb-3">GRPO: Group Relative Policy Optimization</h1><p class="text-fd-muted-foreground text-lg mb-4">A companion post to our GRPO paper, explaining group relative policy optimization for language model alignment.</p><div class="flex items-center gap-3 pt-4 border-t border-fd-border"><span class="text-sm text-fd-muted-foreground">By <!-- -->Zach Kelling</span><div class="flex gap-1.5 ml-auto"><span class="text-[10px] font-mono uppercase tracking-wider bg-fd-muted text-fd-muted-foreground px-1.5 py-0.5 rounded">Research</span><span class="text-[10px] font-mono uppercase tracking-wider bg-fd-muted text-fd-muted-foreground px-1.5 py-0.5 rounded">RLHF</span><span class="text-[10px] font-mono uppercase tracking-wider bg-fd-muted text-fd-muted-foreground px-1.5 py-0.5 rounded">GRPO</span><span class="text-[10px] font-mono uppercase tracking-wider bg-fd-muted text-fd-muted-foreground px-1.5 py-0.5 rounded">Alignment</span></div></div></div><div class="prose dark:prose-invert max-w-none"><h1 class="flex scroll-m-28 flex-row items-center gap-2" id="beyond-ppo"><a data-card="" href="#beyond-ppo" class="peer">Beyond PPO</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h1>
<p>Proximal Policy Optimization (PPO) has become the de facto algorithm for reinforcement learning from human feedback. Yet PPO has fundamental limitations when applied to language models:</p>
<ol>
<li><strong>Absolute reward dependence</strong> : PPO optimizes absolute reward values, which are noisy and poorly calibrated</li>
<li><strong>KL divergence sensitivity</strong> : The KL penalty requires careful tuning to avoid collapse or divergence</li>
<li><strong>Sample inefficiency</strong> : Each prompt generates one response for learning</li>
<li><strong>Reward hacking</strong> : Models exploit reward model weaknesses</li>
</ol>
<p>Group Relative Policy Optimization (GRPO) addresses these issues through a simple insight: <strong>relative comparisons are more informative than absolute scores</strong>.</p>
<h2 class="flex scroll-m-28 flex-row items-center gap-2" id="the-grpo-algorithm"><a data-card="" href="#the-grpo-algorithm" class="peer">The GRPO Algorithm</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h2>
<p>Instead of scoring individual responses, GRPO generates a group of $K$ responses per prompt and learns from their relative rankings.</p>
<h3 class="flex scroll-m-28 flex-row items-center gap-2" id="response-generation"><a data-card="" href="#response-generation" class="peer">Response Generation</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h3>
<p>For each prompt $x$, sample $K$ responses from the current policy:</p>
<p><code>y_1, y_2, \ldots, y_K \sim \pi_\theta(\cdot | x)</code></p>
<h3 class="flex scroll-m-28 flex-row items-center gap-2" id="reward-computation"><a data-card="" href="#reward-computation" class="peer">Reward Computation</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h3>
<p>Score all responses with the reward model:</p>
<p><code>r_i = R(x, y_i) \quad \text\{for \} i = 1, \ldots, K</code></p>
<h3 class="flex scroll-m-28 flex-row items-center gap-2" id="advantage-estimation"><a data-card="" href="#advantage-estimation" class="peer">Advantage Estimation</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h3>
<p>Compute group-relative advantages:</p>
<p><code>A_i = \frac\{r_i - \mu_r\}\{\sigma_r\}</code></p>
<p>Where $\mu_r$ and $\sigma_r$ are the mean and standard deviation of rewards within the group.</p>
<h3 class="flex scroll-m-28 flex-row items-center gap-2" id="policy-update"><a data-card="" href="#policy-update" class="peer">Policy Update</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h3>
<p>Update the policy to increase probability of high-advantage responses:</p>
<p><code>\mathcal\{L\}_\{GRPO\} = -\mathbb\{E\}_\{x, y \sim \pi_\theta\}\left[\frac\{\pi_\theta(y|x)\}\{\pi_\{old\}(y|x)\} \cdot A(x, y) \cdot \mathbb\{1\}_\{clip\}\right]</code></p>
<p>Where $\mathbb{1}_{clip}$ applies PPO-style clipping to the importance ratio.</p>
<h2 class="flex scroll-m-28 flex-row items-center gap-2" id="why-group-relative"><a data-card="" href="#why-group-relative" class="peer">Why Group-Relative?</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h2>
<h3 class="flex scroll-m-28 flex-row items-center gap-2" id="noise-robustness"><a data-card="" href="#noise-robustness" class="peer">Noise Robustness</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h3>
<p>Reward models are noisy. A response scored 0.7 versus 0.6 may not be meaningfully better. But within a group of responses to the same prompt, relative ordering is more reliable:</p>
<div class="relative overflow-auto prose-no-margin my-6"><table><thead><tr><th>Metric</th><th>Absolute Score</th><th>Relative Rank</th></tr></thead><tbody><tr><td>Inter-annotator agreement</td><td>0.61</td><td>0.83</td></tr><tr><td>Test-retest reliability</td><td>0.54</td><td>0.79</td></tr><tr><td>Reward model calibration</td><td>Poor</td><td>N/A</td></tr></tbody></table></div>
<h3 class="flex scroll-m-28 flex-row items-center gap-2" id="natural-normalization"><a data-card="" href="#natural-normalization" class="peer">Natural Normalization</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h3>
<p>Group-relative advantages automatically adapt to reward scale and prompt difficulty:</p>
<ul>
<li>Easy prompts: All responses score high, advantages near zero</li>
<li>Hard prompts: Large variance, clear signal for improvement</li>
<li>Reward drift: Normalization handles changing baselines</li>
</ul>
<h3 class="flex scroll-m-28 flex-row items-center gap-2" id="sample-efficiency"><a data-card="" href="#sample-efficiency" class="peer">Sample Efficiency</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h3>
<p>Generating $K$ responses per prompt and comparing them provides $\binom{K}{2}$ pairwise comparisons. For $K=8$, that’s 28 learning signals per prompt versus 1 for standard PPO.</p>
<h2 class="flex scroll-m-28 flex-row items-center gap-2" id="implementation-details"><a data-card="" href="#implementation-details" class="peer">Implementation Details</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h2>
<h3 class="flex scroll-m-28 flex-row items-center gap-2" id="group-size-selection"><a data-card="" href="#group-size-selection" class="peer">Group Size Selection</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h3>
<p>We find $K=8$ provides a good tradeoff:</p>
<div class="relative overflow-auto prose-no-margin my-6"><table><thead><tr><th>K</th><th>Compute</th><th>Signal Quality</th><th>Best Accuracy</th></tr></thead><tbody><tr><td>2</td><td>2x</td><td>Low</td><td>71.2%</td></tr><tr><td>4</td><td>4x</td><td>Medium</td><td>74.8%</td></tr><tr><td>8</td><td>8x</td><td>High</td><td>77.3%</td></tr><tr><td>16</td><td>16x</td><td>Marginal gain</td><td>77.9%</td></tr></tbody></table></div>
<h3 class="flex scroll-m-28 flex-row items-center gap-2" id="temperature-schedule"><a data-card="" href="#temperature-schedule" class="peer">Temperature Schedule</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h3>
<p>Higher temperature during response generation increases group diversity:</p>
<figure dir="ltr" class="my-4 bg-fd-card rounded-xl shiki relative border shadow-sm not-prose overflow-hidden text-sm shiki shiki-themes github-light github-dark" style="--shiki-light:#24292e;--shiki-dark:#e1e4e8;--shiki-light-bg:#fff;--shiki-dark-bg:#24292e" tabindex="-1"><div class="empty:hidden absolute top-3 right-2 z-2 backdrop-blur-lg rounded-lg text-fd-muted-foreground"><button type="button" class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors duration-100 disabled:pointer-events-none disabled:opacity-50 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-fd-ring p-1 [&amp;_svg]:size-4 hover:text-fd-accent-foreground data-checked:text-fd-accent-foreground" aria-label="Copy Text"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-clipboard" aria-hidden="true"><rect width="8" height="4" x="8" y="2" rx="1" ry="1"></rect><path d="M16 4h2a2 2 0 0 1 2 2v14a2 2 0 0 1-2 2H6a2 2 0 0 1-2-2V6a2 2 0 0 1 2-2h2"></path></svg></button></div><div role="region" tabindex="0" class="text-[0.8125rem] py-3.5 overflow-auto max-h-[600px] fd-scroll-container focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-inset focus-visible:ring-fd-ring" style="--padding-right:calc(var(--spacing) * 8)"><pre class="min-w-full w-max *:flex *:flex-col"><code><span class="line"></span>
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583"> def</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0"> sample_group</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">(prompt, policy, K</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF">8</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">):</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> responses </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> []</span></span>
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583"> for</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> i </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">in</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF"> range</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">(K):</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> temp </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF"> 0.7</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583"> +</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF"> 0.3</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583"> *</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> (i </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">/</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> K) </span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D"># 0.7 to 1.0</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> response </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> policy.sample(prompt, </span><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70">temperature</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">temp)</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> responses.append(response)</span></span>
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583"> return</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> responses</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> </span></span></code></pre></div></figure>
<h3 class="flex scroll-m-28 flex-row items-center gap-2" id="kl-regularization"><a data-card="" href="#kl-regularization" class="peer">KL Regularization</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h3>
<p>GRPO still benefits from KL regularization, but with reduced sensitivity:</p>
<p><code>\mathcal\{L\} = \mathcal\{L\}_\{GRPO\} + \beta \cdot D_\{KL\}(\pi_\theta || \pi_\{ref\})</code></p>
<p>We find $\beta = 0.01$ works across tasks, compared to PPO’s typical $\beta \in [0.001, 0.1]$ sensitivity.</p>
<h2 class="flex scroll-m-28 flex-row items-center gap-2" id="experimental-results"><a data-card="" href="#experimental-results" class="peer">Experimental Results</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h2>
<p>On Anthropic’s HH-RLHF benchmark:</p>
<div class="relative overflow-auto prose-no-margin my-6"><table><thead><tr><th>Method</th><th>Helpfulness</th><th>Harmlessness</th><th>Compute</th></tr></thead><tbody><tr><td>SFT</td><td>3.2/5</td><td>3.8/5</td><td>1x</td></tr><tr><td>PPO</td><td>3.9/5</td><td>4.1/5</td><td>10x</td></tr><tr><td>GRPO</td><td>4.2/5</td><td>4.3/5</td><td>8x</td></tr></tbody></table></div>
<p>GRPO achieves better alignment with less compute through efficient use of generated samples.</p>
<h2 class="flex scroll-m-28 flex-row items-center gap-2" id="reward-hacking-resistance"><a data-card="" href="#reward-hacking-resistance" class="peer">Reward Hacking Resistance</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h2>
<p>GRPO is naturally resistant to reward hacking because:</p>
<ol>
<li><strong>Relative comparison</strong> : Hacked responses must beat other responses, not just achieve high absolute score</li>
<li><strong>Diverse sampling</strong> : Temperature variation produces varied response styles</li>
<li><strong>Group normalization</strong> : Exploits that boost all responses equally provide no gradient</li>
</ol>
<p>We observe significantly less length gaming and repetition compared to PPO.</p>
<h2 class="flex scroll-m-28 flex-row items-center gap-2" id="code"><a data-card="" href="#code" class="peer">Code</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h2>
<p>Reference implementation:</p>
<figure dir="ltr" class="my-4 bg-fd-card rounded-xl shiki relative border shadow-sm not-prose overflow-hidden text-sm shiki shiki-themes github-light github-dark" style="--shiki-light:#24292e;--shiki-dark:#e1e4e8;--shiki-light-bg:#fff;--shiki-dark-bg:#24292e" tabindex="-1"><div class="empty:hidden absolute top-3 right-2 z-2 backdrop-blur-lg rounded-lg text-fd-muted-foreground"><button type="button" class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors duration-100 disabled:pointer-events-none disabled:opacity-50 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-fd-ring p-1 [&amp;_svg]:size-4 hover:text-fd-accent-foreground data-checked:text-fd-accent-foreground" aria-label="Copy Text"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-clipboard" aria-hidden="true"><rect width="8" height="4" x="8" y="2" rx="1" ry="1"></rect><path d="M16 4h2a2 2 0 0 1 2 2v14a2 2 0 0 1-2 2H6a2 2 0 0 1-2-2V6a2 2 0 0 1 2-2h2"></path></svg></button></div><div role="region" tabindex="0" class="text-[0.8125rem] py-3.5 overflow-auto max-h-[600px] fd-scroll-container focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-inset focus-visible:ring-fd-ring" style="--padding-right:calc(var(--spacing) * 8)"><pre class="min-w-full w-max *:flex *:flex-col"><code><span class="line"></span>
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583"> def</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0"> grpo_loss</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">(policy, prompts, reward_model, K</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF">8</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">, clip_eps</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF">0.2</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">):</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> losses </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> []</span></span>
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583"> for</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> prompt </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">in</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> prompts:</span></span>
<span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D"> # Generate response group</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> responses </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> sample_group(prompt, policy, K)</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> </span></span>
<span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D"> # Compute rewards and advantages</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> rewards </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> [reward_model(prompt, r) </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">for</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> r </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">in</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> responses]</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> advantages </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> (rewards </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">-</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> np.mean(rewards)) </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">/</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> (np.std(rewards) </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">+</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF"> 1e-8</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">)</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> </span></span>
<span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D"> # Policy loss</span></span>
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583"> for</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> response, advantage </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">in</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF"> zip</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">(responses, advantages):</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> ratio </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> policy.prob(response) </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">/</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> policy.prob_old(response)</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> clipped </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> torch.clamp(ratio, </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF">1</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">-</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">clip_eps, </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF">1</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">+</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">clip_eps)</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> loss </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583"> -</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">torch.min(ratio </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">*</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> advantage, clipped </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">*</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> advantage)</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> losses.append(loss)</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> </span></span>
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583"> return</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> torch.mean(torch.stack(losses))</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> </span></span></code></pre></div></figure>
<h2 class="flex scroll-m-28 flex-row items-center gap-2" id="conclusion"><a data-card="" href="#conclusion" class="peer">Conclusion</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h2>
<p>GRPO offers a simple improvement to RLHF: generate multiple responses, compare them relatively, update toward the best. This approach is more robust, more sample-efficient, and more resistant to reward hacking than standard PPO.</p>
<p>The algorithm is simple enough to implement in an afternoon. The gains are substantial enough to matter.</p>
<hr/>
<p><em>Full details in “Group Relative Policy Optimization for Language Model Alignment” (2022). Code at github.com/zen-ai/grpo.</em></p></div></main><!--$--><!--/$--><script src="/_next/static/chunks/e83606e8fa9cc796.js" id="_R_" async=""></script><script>(self.__next_f=self.__next_f||[]).push([0])</script><script>self.__next_f.push([1,"1:\"$Sreact.fragment\"\n2:I[106,[\"/_next/static/chunks/59d0ad1b64f8544e.js\"],\"RootProvider\"]\n3:I[53113,[\"/_next/static/chunks/4d80e004cf4896dd.js\",\"/_next/static/chunks/350ee4303b732916.js\"],\"default\"]\n4:I[73211,[\"/_next/static/chunks/4d80e004cf4896dd.js\",\"/_next/static/chunks/350ee4303b732916.js\"],\"default\"]\n5:I[10086,[\"/_next/static/chunks/59d0ad1b64f8544e.js\",\"/_next/static/chunks/8de849ca74fc071f.js\",\"/_next/static/chunks/e62b91212ee7f8ff.js\",\"/_next/static/chunks/f19fe44237e54646.js\",\"/_next/static/chunks/cb0a883bafeb6805.js\"],\"\"]\n7:I[89923,[\"/_next/static/chunks/4d80e004cf4896dd.js\",\"/_next/static/chunks/350ee4303b732916.js\"],\"OutletBoundary\"]\n8:\"$Sreact.suspense\"\na:I[89923,[\"/_next/static/chunks/4d80e004cf4896dd.js\",\"/_next/static/chunks/350ee4303b732916.js\"],\"ViewportBoundary\"]\nc:I[89923,[\"/_next/static/chunks/4d80e004cf4896dd.js\",\"/_next/static/chunks/350ee4303b732916.js\"],\"MetadataBoundary\"]\ne:I[6998,[\"/_next/static/chunks/4d80e004cf4896dd.js\",\"/_next/static/chunks/350ee4303b732916.js\"],\"default\"]\n:HL[\"/_next/static/chunks/f2332aac77592f9d.css\",\"style\"]\n"])</script><script>self.__next_f.push([1,"0:{\"P\":null,\"b\":\"i-dnJM_MIpJSOCQWNJVMq\",\"c\":[\"\",\"blog\",\"grpo-group-relative-policy-optimization\"],\"q\":\"\",\"i\":false,\"f\":[[[\"\",{\"children\":[\"blog\",{\"children\":[[\"slug\",\"grpo-group-relative-policy-optimization\",\"d\"],{\"children\":[\"__PAGE__\",{}]}]}]},\"$undefined\",\"$undefined\",true],[[\"$\",\"$1\",\"c\",{\"children\":[[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/chunks/f2332aac77592f9d.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\",\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-0\",{\"src\":\"/_next/static/chunks/59d0ad1b64f8544e.js\",\"async\":true,\"nonce\":\"$undefined\"}]],[\"$\",\"html\",null,{\"lang\":\"en\",\"className\":\"dark\",\"suppressHydrationWarning\":true,\"children\":[\"$\",\"body\",null,{\"className\":\"antialiased\",\"children\":[\"$\",\"$L2\",null,{\"children\":[\"$\",\"$L3\",null,{\"parallelRouterKey\":\"children\",\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L4\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":[[\"$\",\"main\",null,{\"className\":\"flex min-h-screen flex-col items-center justify-center px-4 text-center\",\"children\":[[\"$\",\"div\",null,{\"className\":\"mb-8 opacity-20\",\"children\":[\"$\",\"svg\",null,{\"width\":\"120\",\"height\":\"120\",\"viewBox\":\"0 0 120 120\",\"fill\":\"none\",\"aria-hidden\":\"true\",\"children\":[\"$\",\"circle\",null,{\"cx\":\"60\",\"cy\":\"60\",\"r\":\"50\",\"stroke\":\"currentColor\",\"strokeWidth\":\"3\",\"strokeLinecap\":\"round\",\"strokeDasharray\":\"280 40\"}]}]}],[\"$\",\"p\",null,{\"className\":\"text-xs font-mono uppercase tracking-[0.3em] text-fd-muted-foreground mb-4\",\"children\":\"404\"}],[\"$\",\"h1\",null,{\"className\":\"text-3xl font-semibold mb-3\",\"children\":\"Page not found\"}],[\"$\",\"p\",null,{\"className\":\"text-fd-muted-foreground max-w-sm mb-10\",\"children\":\"This page doesn't exist, or it may have moved. Try the documentation or head home.\"}],[\"$\",\"div\",null,{\"className\":\"flex flex-wrap gap-3 justify-center\",\"children\":[[\"$\",\"$L5\",null,{\"href\":\"/\",\"className\":\"inline-flex items-center gap-2 rounded-lg bg-fd-primary text-fd-primary-foreground px-4 py-2 text-sm font-medium hover:opacity-90 transition\",\"children\":\"Go home\"}],[\"$\",\"$L5\",null,{\"href\":\"/docs\",\"className\":\"inline-flex items-center gap-2 rounded-lg border border-fd-border bg-fd-background px-4 py-2 text-sm font-medium hover:bg-fd-muted transition\",\"children\":\"Documentation\"}],[\"$\",\"$L5\",null,{\"href\":\"/docs/models\",\"className\":\"inline-flex items-center gap-2 rounded-lg border border-fd-border bg-fd-background px-4 py-2 text-sm font-medium hover:bg-fd-muted transition\",\"children\":\"Browse models\"}]]}],[\"$\",\"p\",null,{\"className\":\"mt-16 text-xs font-mono text-fd-muted-foreground opacity-50\",\"children\":\"zenlm.org\"}]]}],[]],\"forbidden\":\"$undefined\",\"unauthorized\":\"$undefined\"}]}]}]}]]}],{\"children\":[[\"$\",\"$1\",\"c\",{\"children\":[null,[\"$\",\"$L3\",null,{\"parallelRouterKey\":\"children\",\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L4\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":\"$undefined\",\"forbidden\":\"$undefined\",\"unauthorized\":\"$undefined\"}]]}],{\"children\":[[\"$\",\"$1\",\"c\",{\"children\":[null,[\"$\",\"$L3\",null,{\"parallelRouterKey\":\"children\",\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L4\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":\"$undefined\",\"forbidden\":\"$undefined\",\"unauthorized\":\"$undefined\"}]]}],{\"children\":[[\"$\",\"$1\",\"c\",{\"children\":[\"$L6\",[[\"$\",\"script\",\"script-0\",{\"src\":\"/_next/static/chunks/8de849ca74fc071f.js\",\"async\":true,\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-1\",{\"src\":\"/_next/static/chunks/e62b91212ee7f8ff.js\",\"async\":true,\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-2\",{\"src\":\"/_next/static/chunks/f19fe44237e54646.js\",\"async\":true,\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-3\",{\"src\":\"/_next/static/chunks/cb0a883bafeb6805.js\",\"async\":true,\"nonce\":\"$undefined\"}]],[\"$\",\"$L7\",null,{\"children\":[\"$\",\"$8\",null,{\"name\":\"Next.MetadataOutlet\",\"children\":\"$@9\"}]}]]}],{},null,false,false]},null,false,false]},null,false,false]},null,false,false],[\"$\",\"$1\",\"h\",{\"children\":[null,[\"$\",\"$La\",null,{\"children\":\"$Lb\"}],[\"$\",\"div\",null,{\"hidden\":true,\"children\":[\"$\",\"$Lc\",null,{\"children\":[\"$\",\"$8\",null,{\"name\":\"Next.Metadata\",\"children\":\"$Ld\"}]}]}],null]}],false]],\"m\":\"$undefined\",\"G\":[\"$e\",[]],\"S\":true}\n"])</script><script>self.__next_f.push([1,"6:[\"$\",\"main\",null,{\"className\":\"mx-auto w-full max-w-2xl px-4 py-16\",\"children\":[[\"$\",\"$L5\",null,{\"href\":\"/blog\",\"className\":\"inline-flex items-center gap-1.5 text-sm text-fd-muted-foreground hover:text-fd-foreground mb-10 transition-colors\",\"children\":\"← Back to Blog\"}],[\"$\",\"div\",null,{\"className\":\"mb-8\",\"children\":[[\"$\",\"time\",null,{\"className\":\"text-xs font-mono text-fd-muted-foreground uppercase tracking-wider\",\"children\":\"September 17, 2022\"}],[\"$\",\"h1\",null,{\"className\":\"text-3xl font-bold mt-2 mb-3\",\"children\":\"GRPO: Group Relative Policy Optimization\"}],[\"$\",\"p\",null,{\"className\":\"text-fd-muted-foreground text-lg mb-4\",\"children\":\"A companion post to our GRPO paper, explaining group relative policy optimization for language model alignment.\"}],[\"$\",\"div\",null,{\"className\":\"flex items-center gap-3 pt-4 border-t border-fd-border\",\"children\":[[\"$\",\"span\",null,{\"className\":\"text-sm text-fd-muted-foreground\",\"children\":[\"By \",\"Zach Kelling\"]}],[\"$\",\"div\",null,{\"className\":\"flex gap-1.5 ml-auto\",\"children\":[[\"$\",\"span\",\"Research\",{\"className\":\"text-[10px] font-mono uppercase tracking-wider bg-fd-muted text-fd-muted-foreground px-1.5 py-0.5 rounded\",\"children\":\"Research\"}],[\"$\",\"span\",\"RLHF\",{\"className\":\"text-[10px] font-mono uppercase tracking-wider bg-fd-muted text-fd-muted-foreground px-1.5 py-0.5 rounded\",\"children\":\"RLHF\"}],[\"$\",\"span\",\"GRPO\",{\"className\":\"text-[10px] font-mono uppercase tracking-wider bg-fd-muted text-fd-muted-foreground px-1.5 py-0.5 rounded\",\"children\":\"GRPO\"}],[\"$\",\"span\",\"Alignment\",{\"className\":\"text-[10px] font-mono uppercase tracking-wider bg-fd-muted text-fd-muted-foreground px-1.5 py-0.5 rounded\",\"children\":\"Alignment\"}]]}]]}]]}],[\"$\",\"div\",null,{\"className\":\"prose dark:prose-invert max-w-none\",\"children\":[[\"$\",\"h1\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"beyond-ppo\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#beyond-ppo\",\"className\":\"peer\",\"children\":\"Beyond PPO\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}],\"\\n\",[\"$\",\"p\",null,{\"children\":\"Proximal Policy Optimization (PPO) has become the de facto algorithm for reinforcement learning from human feedback. Yet PPO has fundamental limitations when applied to language models:\"}],\"\\n\",[\"$\",\"ol\",null,{\"children\":[\"\\n\",[\"$\",\"li\",null,{\"children\":[[\"$\",\"strong\",null,{\"children\":\"Absolute reward dependence\"}],\" : PPO optimizes absolute reward values, which are noisy and poorly calibrated\"]}],\"\\n\",[\"$\",\"li\",null,{\"children\":[[\"$\",\"strong\",null,{\"children\":\"KL divergence sensitivity\"}],\" : The KL penalty requires careful tuning to avoid collapse or divergence\"]}],\"\\n\",[\"$\",\"li\",null,{\"children\":[[\"$\",\"strong\",null,{\"children\":\"Sample inefficiency\"}],\" : Each prompt generates one response for learning\"]}],\"\\n\",[\"$\",\"li\",null,{\"children\":[[\"$\",\"strong\",null,{\"children\":\"Reward hacking\"}],\" : Models exploit reward model weaknesses\"]}],\"\\n\"]}],\"\\n\",[\"$\",\"p\",null,{\"children\":[\"Group Relative Policy Optimization (GRPO) addresses these issues through a simple insight: \",[\"$\",\"strong\",null,{\"children\":\"relative comparisons are more informative than absolute scores\"}],\".\"]}],\"\\n\",[\"$\",\"h2\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"the-grpo-algorithm\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#the-grpo-algorithm\",\"className\":\"peer\",\"children\":\"The GRPO Algorithm\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[\"$Lf\",\"$L10\",\"$undefined\"]}]]}],\"\\n\",\"$L11\",\"\\n\",\"$L12\",\"\\n\",\"$L13\",\"\\n\",\"$L14\",\"\\n\",\"$L15\",\"\\n\",\"$L16\",\"\\n\",\"$L17\",\"\\n\",\"$L18\",\"\\n\",\"$L19\",\"\\n\",\"$L1a\",\"\\n\",\"$L1b\",\"\\n\",\"$L1c\",\"\\n\",\"$L1d\",\"\\n\",\"$L1e\",\"\\n\",\"$L1f\",\"\\n\",\"$L20\",\"\\n\",\"$L21\",\"\\n\",\"$L22\",\"\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\",\"$L23\",\"\\n\",\"$L24\",\"\\n\",\"$L25\",\"\\n\",\"$L26\",\"\\n\",\"$L27\",\"\\n\",\"$L28\",\"\\n\",\"$L29\",\"\\n\",\"$L2a\",\"\\n\",\"$L2b\",\"\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\",\"$L2c\",\"\\n\",\"$L2d\",\"\\n\",\"$L2e\",\"\\n\",\"$L2f\",\"\\n\",\"$L30\",\"\\n\",\"$L31\",\"\\n\",\"$L32\",\"\\n\",\"$L33\",\"\\n\",\"$L34\",\"\\n\",\"$L35\",\"\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\",\"$L36\",\"\\n\",\"$L37\",\"\\n\",\"$L38\",\"\\n\",\"$L39\",\"\\n\",\"$L3a\",\"\\n\",\"$L3b\",\"\\n\",\"$L3c\",\"\\n\",\"$L3d\",\"\\n\",\"$L3e\",\"\\n\",\"$L3f\",\"\\n\",\"$L40\",\"\\n\",\"$L41\",\"\\n\",\"$L42\",\"\\n\",\"$L43\"]}]]}]\n"])</script><script>self.__next_f.push([1,"44:I[51504,[\"/_next/static/chunks/59d0ad1b64f8544e.js\",\"/_next/static/chunks/8de849ca74fc071f.js\",\"/_next/static/chunks/e62b91212ee7f8ff.js\",\"/_next/static/chunks/f19fe44237e54646.js\",\"/_next/static/chunks/cb0a883bafeb6805.js\"],\"CodeBlock\"]\n46:I[51504,[\"/_next/static/chunks/59d0ad1b64f8544e.js\",\"/_next/static/chunks/8de849ca74fc071f.js\",\"/_next/static/chunks/e62b91212ee7f8ff.js\",\"/_next/static/chunks/f19fe44237e54646.js\",\"/_next/static/chunks/cb0a883bafeb6805.js\"],\"Pre\"]\nf:[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}]\n10:[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}]\n11:[\"$\",\"p\",null,{\"children\":\"Instead of scoring individual responses, GRPO generates a group of $K$ responses per prompt and learns from their relative rankings.\"}]\n12:[\"$\",\"h3\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"response-generation\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#response-generation\",\"className\":\"peer\",\"children\":\"Response Generation\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n13:[\"$\",\"p\",null,{\"children\":\"For each prompt $x$, sample $K$ responses from the current policy:\"}]\n14:[\"$\",\"p\",null,{\"children\":[\"$\",\"code\",null,{\"children\":\"y_1, y_2, \\\\ldots, y_K \\\\sim \\\\pi_\\\\theta(\\\\cdot | x)\"}]}]\n15:[\"$\",\"h3\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"reward-computation\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#reward-computation\",\"className\":\"peer\",\"children\":\"Reward Computation\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n16:[\"$\",\"p\",null,{\"children\":\"Score all responses with the reward model:\"}]\n17:[\"$\",\"p\",null,{\"children\":[\"$\",\"code\",null,{\"children\":\"r_i = R(x, y_i) \\\\quad \\\\text\\\\{for \\\\} i = 1, \\\\ldots, K\"}]}]\n18:[\"$\",\"h3\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"advantage-estimation\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#advantage-estimation\",\"className\":\"peer\",\"children\":\"Advantage Estimation\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n19:[\"$\",\"p\",null,{\"children\":\"Compute group-relative advantages:\"}]\n1a:[\"$\",\"p\",null,{\"children\":[\"$\",\"code\",null,{\"children\":\"A_i = \\\\frac\\\\{r_i - \\\\mu_r\\\\}\\\\{\\\\sigma_r\\\\}\"}]}]\n1b:[\"$\",\"p\",null,{\"children\":\"Where $\\\\mu_r$ and $\\\\sigma_r$ are the mean and standard deviation of rewards within the group.\"}]\n1c:[\"$\",\"h3\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"policy-update\",\"children\":[["])</script><script>self.__next_f.push([1,"\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#policy-update\",\"className\":\"peer\",\"children\":\"Policy Update\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n1d:[\"$\",\"p\",null,{\"children\":\"Update the policy to increase probability of high-advantage responses:\"}]\n1e:[\"$\",\"p\",null,{\"children\":[\"$\",\"code\",null,{\"children\":\"\\\\mathcal\\\\{L\\\\}_\\\\{GRPO\\\\} = -\\\\mathbb\\\\{E\\\\}_\\\\{x, y \\\\sim \\\\pi_\\\\theta\\\\}\\\\left[\\\\frac\\\\{\\\\pi_\\\\theta(y|x)\\\\}\\\\{\\\\pi_\\\\{old\\\\}(y|x)\\\\} \\\\cdot A(x, y) \\\\cdot \\\\mathbb\\\\{1\\\\}_\\\\{clip\\\\}\\\\right]\"}]}]\n1f:[\"$\",\"p\",null,{\"children\":\"Where $\\\\mathbb{1}_{clip}$ applies PPO-style clipping to the importance ratio.\"}]\n20:[\"$\",\"h2\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"why-group-relative\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#why-group-relative\",\"className\":\"peer\",\"children\":\"Why Group-Relative?\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n21:[\"$\",\"h3\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"noise-robustness\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#noise-robustness\",\"className\":\"peer\",\"children\":\"Noise Robustness\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n22:[\"$\",\"p\",null,{\"children\":\"Reward models are noisy. A response scored 0.7 versus 0.6 may not be meaningfully better. But within a group of responses to the same prompt, relative ordering is more reliable:\"}]\n23:[\"$\",\"div\",null,{\"className\":\"relative overflow-auto prose-no-margin my-6\",\"children\":[\"$\",\"table\",null,{\"children\":[[\"$\",\"thead\",null,{\"children\":[\"$\",\"tr\",null,{\"children\":[[\"$\",\"th\",null,{\"children\":\"Metric\"}],[\"$\",\"th\",null,{\"children\":\"Absolute Score\"}],[\"$\",\"th\",null,{\"children\":\"Relative Rank\"}]]}]}],[\"$\",\"tbody\",null,{\"children\":[[\"$\",\"tr\",null,{\"children\":[[\"$\",\"td\",null,{\"children\":\"Inter-annotator agreement\"}],[\"$\",\"td\",null,{\"children\":\"0.61\"}],[\"$\",\"td\",null,{\"children\":\"0.83\"}]]}],[\"$\",\"tr\",null,{\"children\":[[\"$\",\"td\",null,{\"children\":\"Test-retest reliability\"}],[\"$\",\"td\",null,{\"children\":\"0.54\"}],[\"$\",\"td\",null,{\"children\":\"0.79\"}]]}],[\"$\",\"tr\",null,{\"children\":[[\"$\",\"td\",null,{\"children\":\"Reward model calibration\"}],[\"$\",\"td\",null,{\"children\":\"Poor\"}],[\"$\",\"td\",null,{\"children\":\"N/A\"}]]}]]}]]}]}]\n24:[\"$\",\"h3\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"natural-normalization\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#natural-normalization\",\"className\":\"peer\",\"children\":\"Natural Normalization\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://ww"])</script><script>self.__next_f.push([1,"w.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n25:[\"$\",\"p\",null,{\"children\":\"Group-relative advantages automatically adapt to reward scale and prompt difficulty:\"}]\n26:[\"$\",\"ul\",null,{\"children\":[\"\\n\",[\"$\",\"li\",null,{\"children\":\"Easy prompts: All responses score high, advantages near zero\"}],\"\\n\",[\"$\",\"li\",null,{\"children\":\"Hard prompts: Large variance, clear signal for improvement\"}],\"\\n\",[\"$\",\"li\",null,{\"children\":\"Reward drift: Normalization handles changing baselines\"}],\"\\n\"]}]\n27:[\"$\",\"h3\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"sample-efficiency\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#sample-efficiency\",\"className\":\"peer\",\"children\":\"Sample Efficiency\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n28:[\"$\",\"p\",null,{\"children\":\"Generating $K$ responses per prompt and comparing them provides $\\\\binom{K}{2}$ pairwise comparisons. For $K=8$, that’s 28 learning signals per prompt versus 1 for standard PPO.\"}]\n29:[\"$\",\"h2\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"implementation-details\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#implementation-details\",\"className\":\"peer\",\"children\":\"Implementation Details\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n2a:[\"$\",\"h3\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"group-size-selection\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#group-size-selection\",\"className\":\"peer\",\"children\":\"Group Size Selection\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n2b:[\"$\",\"p\",null,{\"children\":\"We find $K=8$ provides a good tradeoff:\"}]\n2c:[\"$\",\"div\",null,{\"className\":\"relative overflow-auto prose-no-margin my-6\",\"children\":[\"$\",\"table\",null,{\"children\":[[\"$\",\"thead\",null,{\"children\":[\"$\",\"tr\",null,{\"children\":[[\"$\",\"th\",null,{\"children\":\"K\"}],[\"$\",\"th\",null,{\"children\":\"Compute\"}],[\"$\",\"th\",null,{\"children\":\"Signal Quality\"}],[\"$\",\"th\",null,{\"children\":\"Best Accuracy\"}]]}]}],[\"$\",\"tbody\",null,{\"children\":[[\"$\",\"tr\",null,{\"children\":["])</script><script>self.__next_f.push([1,"[\"$\",\"td\",null,{\"children\":\"2\"}],[\"$\",\"td\",null,{\"children\":\"2x\"}],[\"$\",\"td\",null,{\"children\":\"Low\"}],[\"$\",\"td\",null,{\"children\":\"71.2%\"}]]}],[\"$\",\"tr\",null,{\"children\":[[\"$\",\"td\",null,{\"children\":\"4\"}],[\"$\",\"td\",null,{\"children\":\"4x\"}],[\"$\",\"td\",null,{\"children\":\"Medium\"}],[\"$\",\"td\",null,{\"children\":\"74.8%\"}]]}],[\"$\",\"tr\",null,{\"children\":[[\"$\",\"td\",null,{\"children\":\"8\"}],[\"$\",\"td\",null,{\"children\":\"8x\"}],[\"$\",\"td\",null,{\"children\":\"High\"}],[\"$\",\"td\",null,{\"children\":\"77.3%\"}]]}],[\"$\",\"tr\",null,{\"children\":[[\"$\",\"td\",null,{\"children\":\"16\"}],[\"$\",\"td\",null,{\"children\":\"16x\"}],[\"$\",\"td\",null,{\"children\":\"Marginal gain\"}],[\"$\",\"td\",null,{\"children\":\"77.9%\"}]]}]]}]]}]}]\n2d:[\"$\",\"h3\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"temperature-schedule\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#temperature-schedule\",\"className\":\"peer\",\"children\":\"Temperature Schedule\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n2e:[\"$\",\"p\",null,{\"children\":\"Higher temperature during response generation increases group diversity:\"}]\n45:T5c0,"])</script><script>self.__next_f.push([1,"\u003csvg viewBox=\"0 0 24 24\"\u003e\u003cpath d=\"M14.25.18l.9.2.73.26.59.3.45.32.34.34.25.34.16.33.1.3.04.26.02.2-.01.13V8.5l-.05.63-.13.55-.21.46-.26.38-.3.31-.33.25-.35.19-.35.14-.33.1-.3.07-.26.04-.21.02H8.77l-.69.05-.59.14-.5.22-.41.27-.33.32-.27.35-.2.36-.15.37-.1.35-.07.32-.04.27-.02.21v3.06H3.17l-.21-.03-.28-.07-.32-.12-.35-.18-.36-.26-.36-.36-.35-.46-.32-.59-.28-.73-.21-.88-.14-1.05-.05-1.23.06-1.22.16-1.04.24-.87.32-.71.36-.57.4-.44.42-.33.42-.24.4-.16.36-.1.32-.05.24-.01h.16l.06.01h8.16v-.83H6.18l-.01-2.75-.02-.37.05-.34.11-.31.17-.28.25-.26.31-.23.38-.2.44-.18.51-.15.58-.12.64-.1.71-.06.77-.04.84-.02 1.27.05zm-6.3 1.98l-.23.33-.08.41.08.41.23.34.33.22.41.09.41-.09.33-.22.23-.34.08-.41-.08-.41-.23-.33-.33-.22-.41-.09-.41.09zm13.09 3.95l.28.06.32.12.35.18.36.27.36.35.35.47.32.59.28.73.21.88.14 1.04.05 1.23-.06 1.23-.16 1.04-.24.86-.32.71-.36.57-.4.45-.42.33-.42.24-.4.16-.36.09-.32.05-.24.02-.16-.01h-8.22v.82h5.84l.01 2.76.02.36-.05.34-.11.31-.17.29-.25.25-.31.24-.38.2-.44.17-.51.15-.58.13-.64.09-.71.07-.77.04-.84.01-1.27-.04-1.07-.14-.9-.2-.73-.25-.59-.3-.45-.33-.34-.34-.25-.34-.16-.33-.1-.3-.04-.25-.02-.2.01-.13v-5.34l.05-.64.13-.54.21-.46.26-.38.3-.32.33-.24.35-.2.35-.14.33-.1.3-.06.26-.04.21-.02.13-.01h5.84l.69-.05.59-.14.5-.21.41-.28.33-.32.27-.35.2-.36.15-.36.1-.35.07-.32.04-.28.02-.21V6.07h2.09l.14.01zm-6.47 14.25l-.23.33-.08.41.08.41.23.33.33.23.41.08.41-.08.33-.23.23-.33.08-.41-.08-.41-.23-.33-.33-.23-.41-.08-.41.08z\" fill=\"currentColor\" /\u003e\u003c/svg\u003e"])</script><script>self.__next_f.push([1,"2f:[\"$\",\"$L44\",null,{\"className\":\"shiki shiki-themes github-light github-dark\",\"style\":{\"--shiki-light\":\"#24292e\",\"--shiki-dark\":\"#e1e4e8\",\"--shiki-light-bg\":\"#fff\",\"--shiki-dark-bg\":\"#24292e\"},\"tabIndex\":\"0\",\"icon\":\"$45\",\"children\":[\"$\",\"$L46\",null,{\"children\":[\"$\",\"code\",null,{\"children\":[[\"$\",\"span\",null,{\"className\":\"line\"}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\" def\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#6F42C1\",\"--shiki-dark\":\"#B392F0\"},\"children\":\" sample_group\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\"(prompt, policy, K\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"=\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#005CC5\",\"--shiki-dark\":\"#79B8FF\"},\"children\":\"8\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\"):\"}]]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" responses \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"=\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" []\"}]]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\" for\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" i \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"in\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#005CC5\",\"--shiki-dark\":\"#79B8FF\"},\"children\":\" range\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\"(K):\"}]]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" temp \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"=\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#005CC5\",\"--shiki-dark\":\"#79B8FF\"},\"children\":\" 0.7\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\" +\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#005CC5\",\"--shiki-dark\":\"#79B8FF\"},\"children\":\" 0.3\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\" *\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" (i \"}],\"$L47\",\"$L48\",\"$L49\"]}],\"\\n\",\"$L4a\",\"\\n\",\"$L4b\",\"\\n\",\"$L4c\",\"\\n\",\"$L4d\"]}]}]}]\n"])</script><script>self.__next_f.push([1,"30:[\"$\",\"h3\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"kl-regularization\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#kl-regularization\",\"className\":\"peer\",\"children\":\"KL Regularization\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n31:[\"$\",\"p\",null,{\"children\":\"GRPO still benefits from KL regularization, but with reduced sensitivity:\"}]\n32:[\"$\",\"p\",null,{\"children\":[\"$\",\"code\",null,{\"children\":\"\\\\mathcal\\\\{L\\\\} = \\\\mathcal\\\\{L\\\\}_\\\\{GRPO\\\\} + \\\\beta \\\\cdot D_\\\\{KL\\\\}(\\\\pi_\\\\theta || \\\\pi_\\\\{ref\\\\})\"}]}]\n33:[\"$\",\"p\",null,{\"children\":\"We find $\\\\beta = 0.01$ works across tasks, compared to PPO’s typical $\\\\beta \\\\in [0.001, 0.1]$ sensitivity.\"}]\n34:[\"$\",\"h2\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"experimental-results\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#experimental-results\",\"className\":\"peer\",\"children\":\"Experimental Results\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n35:[\"$\",\"p\",null,{\"children\":\"On Anthropic’s HH-RLHF benchmark:\"}]\n36:[\"$\",\"div\",null,{\"className\":\"relative overflow-auto prose-no-margin my-6\",\"children\":[\"$\",\"table\",null,{\"children\":[[\"$\",\"thead\",null,{\"children\":[\"$\",\"tr\",null,{\"children\":[[\"$\",\"th\",null,{\"children\":\"Method\"}],[\"$\",\"th\",null,{\"children\":\"Helpfulness\"}],[\"$\",\"th\",null,{\"children\":\"Harmlessness\"}],[\"$\",\"th\",null,{\"children\":\"Compute\"}]]}]}],[\"$\",\"tbody\",null,{\"children\":[[\"$\",\"tr\",null,{\"children\":[[\"$\",\"td\",null,{\"children\":\"SFT\"}],[\"$\",\"td\",null,{\"children\":\"3.2/5\"}],[\"$\",\"td\",null,{\"children\":\"3.8/5\"}],[\"$\",\"td\",null,{\"children\":\"1x\"}]]}],[\"$\",\"tr\",null,{\"children\":[[\"$\",\"td\",null,{\"children\":\"PPO\"}],[\"$\",\"td\",null,{\"children\":\"3.9/5\"}],[\"$\",\"td\",null,{\"children\":\"4.1/5\"}],[\"$\",\"td\",null,{\"children\":\"10x\"}]]}],[\"$\",\"tr\",null,{\"children\":[[\"$\",\"td\",null,{\"children\":\"GRPO\"}],[\"$\",\"td\",null,{\"children\":\"4.2/5\"}],[\"$\",\"td\",null,{\"children\":\"4.3/5\"}],[\"$\",\"td\",null,{\"children\":\"8x\"}]]}]]}]]}]}]\n37:[\"$\",\"p\",null,{\"children\":\"GRPO achieves better alignment with less compute through efficient use of generated samples.\"}]\n38:[\"$\",\"h2\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"reward-hacking-resistance\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#reward-hacking-resistance\",\"className\":\"peer\",\"children\":\"Reward Hacking Resistance\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n39:[\"$\",\"p\",null,{\"children\":\"GRPO is naturally resistant to reward hacking because:\"}]\n3a:[\"$\",\"ol\",null,{\"children\":[\"\\"])</script><script>self.__next_f.push([1,"n\",[\"$\",\"li\",null,{\"children\":[[\"$\",\"strong\",null,{\"children\":\"Relative comparison\"}],\" : Hacked responses must beat other responses, not just achieve high absolute score\"]}],\"\\n\",[\"$\",\"li\",null,{\"children\":[[\"$\",\"strong\",null,{\"children\":\"Diverse sampling\"}],\" : Temperature variation produces varied response styles\"]}],\"\\n\",[\"$\",\"li\",null,{\"children\":[[\"$\",\"strong\",null,{\"children\":\"Group normalization\"}],\" : Exploits that boost all responses equally provide no gradient\"]}],\"\\n\"]}]\n3b:[\"$\",\"p\",null,{\"children\":\"We observe significantly less length gaming and repetition compared to PPO.\"}]\n3c:[\"$\",\"h2\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"code\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#code\",\"className\":\"peer\",\"children\":\"Code\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n3d:[\"$\",\"p\",null,{\"children\":\"Reference implementation:\"}]\n4e:T5c0,"])</script><script>self.__next_f.push([1,"\u003csvg viewBox=\"0 0 24 24\"\u003e\u003cpath d=\"M14.25.18l.9.2.73.26.59.3.45.32.34.34.25.34.16.33.1.3.04.26.02.2-.01.13V8.5l-.05.63-.13.55-.21.46-.26.38-.3.31-.33.25-.35.19-.35.14-.33.1-.3.07-.26.04-.21.02H8.77l-.69.05-.59.14-.5.22-.41.27-.33.32-.27.35-.2.36-.15.37-.1.35-.07.32-.04.27-.02.21v3.06H3.17l-.21-.03-.28-.07-.32-.12-.35-.18-.36-.26-.36-.36-.35-.46-.32-.59-.28-.73-.21-.88-.14-1.05-.05-1.23.06-1.22.16-1.04.24-.87.32-.71.36-.57.4-.44.42-.33.42-.24.4-.16.36-.1.32-.05.24-.01h.16l.06.01h8.16v-.83H6.18l-.01-2.75-.02-.37.05-.34.11-.31.17-.28.25-.26.31-.23.38-.2.44-.18.51-.15.58-.12.64-.1.71-.06.77-.04.84-.02 1.27.05zm-6.3 1.98l-.23.33-.08.41.08.41.23.34.33.22.41.09.41-.09.33-.22.23-.34.08-.41-.08-.41-.23-.33-.33-.22-.41-.09-.41.09zm13.09 3.95l.28.06.32.12.35.18.36.27.36.35.35.47.32.59.28.73.21.88.14 1.04.05 1.23-.06 1.23-.16 1.04-.24.86-.32.71-.36.57-.4.45-.42.33-.42.24-.4.16-.36.09-.32.05-.24.02-.16-.01h-8.22v.82h5.84l.01 2.76.02.36-.05.34-.11.31-.17.29-.25.25-.31.24-.38.2-.44.17-.51.15-.58.13-.64.09-.71.07-.77.04-.84.01-1.27-.04-1.07-.14-.9-.2-.73-.25-.59-.3-.45-.33-.34-.34-.25-.34-.16-.33-.1-.3-.04-.25-.02-.2.01-.13v-5.34l.05-.64.13-.54.21-.46.26-.38.3-.32.33-.24.35-.2.35-.14.33-.1.3-.06.26-.04.21-.02.13-.01h5.84l.69-.05.59-.14.5-.21.41-.28.33-.32.27-.35.2-.36.15-.36.1-.35.07-.32.04-.28.02-.21V6.07h2.09l.14.01zm-6.47 14.25l-.23.33-.08.41.08.41.23.33.33.23.41.08.41-.08.33-.23.23-.33.08-.41-.08-.41-.23-.33-.33-.23-.41-.08-.41.08z\" fill=\"currentColor\" /\u003e\u003c/svg\u003e"])</script><script>self.__next_f.push([1,"3e:[\"$\",\"$L44\",null,{\"className\":\"shiki shiki-themes github-light github-dark\",\"style\":{\"--shiki-light\":\"#24292e\",\"--shiki-dark\":\"#e1e4e8\",\"--shiki-light-bg\":\"#fff\",\"--shiki-dark-bg\":\"#24292e\"},\"tabIndex\":\"0\",\"icon\":\"$4e\",\"children\":[\"$\",\"$L46\",null,{\"children\":[\"$\",\"code\",null,{\"children\":[[\"$\",\"span\",null,{\"className\":\"line\"}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\" def\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#6F42C1\",\"--shiki-dark\":\"#B392F0\"},\"children\":\" grpo_loss\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\"(policy, prompts, reward_model, K\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"=\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#005CC5\",\"--shiki-dark\":\"#79B8FF\"},\"children\":\"8\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\", clip_eps\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"=\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#005CC5\",\"--shiki-dark\":\"#79B8FF\"},\"children\":\"0.2\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\"):\"}]]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" losses \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"=\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" []\"}]]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\" for\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" prompt \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"in\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" prompts:\"}]]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#6A737D\",\"--shiki-dark\":\"#6A737D\"},\"children\":\" # Generate response group\"}]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" responses \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"=\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" sample_group(prompt, policy, K)\"}]]}],\"\\n\",\"$L4f\",\"\\n\",\"$L50\",\"\\n\",\"$L51\",\"\\n\",\"$L52\",\"\\n\",\"$L53\",\"\\n\",\"$L54\",\"\\n\",\"$L55\",\"\\n\",\"$L56\",\"\\n\",\"$L57\",\"\\n\",\"$L58\",\"\\n\",\"$L59\",\"\\n\",\"$L5a\",\"\\n\",\"$L5b\",\"\\n\",\"$L5c\"]}]}]}]\n"])</script><script>self.__next_f.push([1,"3f:[\"$\",\"h2\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"conclusion\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#conclusion\",\"className\":\"peer\",\"children\":\"Conclusion\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n40:[\"$\",\"p\",null,{\"children\":\"GRPO offers a simple improvement to RLHF: generate multiple responses, compare them relatively, update toward the best. This approach is more robust, more sample-efficient, and more resistant to reward hacking than standard PPO.\"}]\n41:[\"$\",\"p\",null,{\"children\":\"The algorithm is simple enough to implement in an afternoon. The gains are substantial enough to matter.\"}]\n42:[\"$\",\"hr\",null,{}]\n43:[\"$\",\"p\",null,{\"children\":[\"$\",\"em\",null,{\"children\":\"Full details in “Group Relative Policy Optimization for Language Model Alignment” (2022). Code at github.com/zen-ai/grpo.\"}]}]\n"])</script><script>self.__next_f.push([1,"47:[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"/\"}]\n48:[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" K) \"}]\n49:[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#6A737D\",\"--shiki-dark\":\"#6A737D\"},\"children\":\"# 0.7 to 1.0\"}]\n4a:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" response \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"=\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" policy.sample(prompt, \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#E36209\",\"--shiki-dark\":\"#FFAB70\"},\"children\":\"temperature\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"=\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\"temp)\"}]]}]\n4b:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" responses.append(response)\"}]}]\n4c:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\" return\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" responses\"}]]}]\n4d:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" \"}]}]\n4f:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" \"}]}]\n50:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#6A737D\",\"--shiki-dark\":\"#6A737D\"},\"children\":\" # Compute rewards and advantages\"}]}]\n51:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" rewards \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"=\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" [reward_model(prompt, r) \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"for\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" r \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"in\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" responses]\"}]]}]\n52:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" advantages \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"=\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" (rewards \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"-\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" np.mean(rewards)) \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"/\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" (np.std(rewards) \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"+\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#005CC5\",\"--shiki-dark\":\"#79B8FF\"},\"children\":\" 1e-8\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\")\"}]]}]\n53:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" \"}]}]\n54:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#6A737D\",\"--shiki-dark\":\"#6A"])</script><script>self.__next_f.push([1,"737D\"},\"children\":\" # Policy loss\"}]}]\n55:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\" for\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" response, advantage \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"in\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#005CC5\",\"--shiki-dark\":\"#79B8FF\"},\"children\":\" zip\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\"(responses, advantages):\"}]]}]\n56:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" ratio \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"=\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" policy.prob(response) \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"/\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" policy.prob_old(response)\"}]]}]\n57:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" clipped \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"=\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" torch.clamp(ratio, \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#005CC5\",\"--shiki-dark\":\"#79B8FF\"},\"children\":\"1\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"-\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\"clip_eps, \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#005CC5\",\"--shiki-dark\":\"#79B8FF\"},\"children\":\"1\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"+\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\"clip_eps)\"}]]}]\n58:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" loss \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"=\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\" -\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\"torch.min(ratio \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"*\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" advantage, clipped \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"*\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" advantage)\"}]]}]\n59:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" losses.append(loss)\"}]}]\n5a:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" \"}]}]\n5b:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\" return\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" torch.mean(torch.stack(losses))\"}]]}]\n5c:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" \"}]}]\n"])</script><script>self.__next_f.push([1,"b:[[\"$\",\"meta\",\"0\",{\"charSet\":\"utf-8\"}],[\"$\",\"meta\",\"1\",{\"name\":\"viewport\",\"content\":\"width=device-width, initial-scale=1\"}],[\"$\",\"meta\",\"2\",{\"name\":\"theme-color\",\"media\":\"(prefers-color-scheme: dark)\",\"content\":\"#0A0A0A\"}],[\"$\",\"meta\",\"3\",{\"name\":\"theme-color\",\"media\":\"(prefers-color-scheme: light)\",\"content\":\"#fff\"}]]\n"])</script><script>self.__next_f.push([1,"9:null\nd:[[\"$\",\"title\",\"0\",{\"children\":\"GRPO: Group Relative Policy Optimization — Zen LM Blog | Zen LM\"}],[\"$\",\"meta\",\"1\",{\"name\":\"description\",\"content\":\"A companion post to our GRPO paper, explaining group relative policy optimization for language model alignment.\"}],[\"$\",\"meta\",\"2\",{\"property\":\"og:title\",\"content\":\"Zen LM - Open Foundation Models\"}],[\"$\",\"meta\",\"3\",{\"property\":\"og:description\",\"content\":\"Zen AI model family by Hanzo AI. 14 frontier models from 4B to 1T+ parameters for code, reasoning, vision, and multimodal tasks.\"}],[\"$\",\"meta\",\"4\",{\"property\":\"og:url\",\"content\":\"https://zenlm.org\"}],[\"$\",\"meta\",\"5\",{\"property\":\"og:site_name\",\"content\":\"Zen LM\"}],[\"$\",\"meta\",\"6\",{\"property\":\"og:type\",\"content\":\"website\"}],[\"$\",\"meta\",\"7\",{\"name\":\"twitter:card\",\"content\":\"summary_large_image\"}],[\"$\",\"meta\",\"8\",{\"name\":\"twitter:site\",\"content\":\"@zenlmorg\"}],[\"$\",\"meta\",\"9\",{\"name\":\"twitter:creator\",\"content\":\"@zenlmorg\"}],[\"$\",\"meta\",\"10\",{\"name\":\"twitter:title\",\"content\":\"Zen LM - Open Foundation Models\"}],[\"$\",\"meta\",\"11\",{\"name\":\"twitter:description\",\"content\":\"Zen AI model family by Hanzo AI. 14 frontier models from 4B to 1T+ parameters for code, reasoning, vision, and multimodal tasks.\"}]]\n"])</script><script>
(function(){
// After React renders the lotus emoji from RSC payload, walk the DOM
// and replace it with the enso SVG. Polls so subsequent React re-renders
// also get cleaned up.
function swap(){
// 1. Replace lotus + ' Zen LM' text inside <a> brand links
document.querySelectorAll('a').forEach(function(a){
if (a.dataset.zenLotusSwapped) return;
var t = (a.textContent || '');
if (!/.*Zen LM|Zen LM.*/.test(t)) return;
// Find the lotus text node specifically
var walker = document.createTreeWalker(a, NodeFilter.SHOW_TEXT, null);
var node;
while ((node = walker.nextNode())) {
if (node.nodeValue && node.nodeValue.indexOf('') !== -1) {
node.nodeValue = node.nodeValue.replace(/\s?/g, '');
}
}
// Inject enso SVG at start
if (!a.querySelector('svg[aria-label="Zen LM"]')) {
var ns = 'http://www.w3.org/2000/svg';
var svg = document.createElementNS(ns, 'svg');
svg.setAttribute('viewBox', '0 0 512 512');
svg.setAttribute('fill', 'none');
svg.setAttribute('width', '18');
svg.setAttribute('height', '18');
svg.setAttribute('aria-label', 'Zen LM');
svg.style.cssText = 'display:inline-block;vertical-align:-2px;margin-right:6px;flex-shrink:0';
var p = document.createElementNS(ns, 'path');
p.setAttribute('d', 'M 256 84 A 172 172 0 1 0 256 428 A 172 172 0 1 0 256 84 Z');
p.setAttribute('stroke', 'currentColor');
p.setAttribute('stroke-width', '26');
p.setAttribute('stroke-linecap', 'round');
p.setAttribute('stroke-linejoin', 'round');
svg.appendChild(p);
a.insertBefore(svg, a.firstChild);
}
a.dataset.zenLotusSwapped = '1';
});
}
function start(){ swap(); setInterval(swap, 600); }
if (document.readyState !== 'loading') start();
else document.addEventListener('DOMContentLoaded', start);
})();
</script>
<style id="zenlm-brand-anim">
@keyframes zenlm-brand-draw { to { stroke-dashoffset: 0; } }
svg.zenlm-brand-svg .zenlm-brand-path {
stroke-dasharray: 1100;
stroke-dashoffset: 1100;
animation: zenlm-brand-draw 1.4s cubic-bezier(.45,0,.2,1) 0.1s 1 forwards;
}
.zen-link-icon { display: inline-block; vertical-align: middle; flex-shrink: 0; }
</style></body></html>