mirror of
https://github.com/zenlm/zen-blog.git
synced 2026-11-03 14:11:45 +00:00
Root cause of HF + GitHub + section icons not appearing in the docs route: Next.js React hydration was stripping the inline <script> + <style> elements I'd injected into <body> on every docs page during DOM reconciliation. The script never registered its setInterval. Move the decorator to /zen-decorate.js loaded via <script defer src> in <head>. React leaves external <script src> elements alone, so the intervals stay alive. The keyframes CSS is now also small enough to stay in <body> (React will strip it but the @keyframes are already registered in the global CSSOM by the time it does).
91 lines
42 KiB
HTML
91 lines
42 KiB
HTML
<!DOCTYPE html><!--i_dnJM_MIpJSOCQWNJVMq--><html lang="en" class="dark"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="stylesheet" href="/_next/static/chunks/f2332aac77592f9d.css" data-precedence="next"/><link rel="preload" as="script" fetchPriority="low" href="/_next/static/chunks/e83606e8fa9cc796.js"/><script src="/_next/static/chunks/36d6595f0156cd7e.js" async=""></script><script src="/_next/static/chunks/040e9cea20a8d9c7.js" async=""></script><script src="/_next/static/chunks/d4dffb5a0973d49c.js" async=""></script><script src="/_next/static/chunks/turbopack-967f27a9fd33556d.js" async=""></script><script src="/_next/static/chunks/59d0ad1b64f8544e.js" async=""></script><script src="/_next/static/chunks/4d80e004cf4896dd.js" async=""></script><script src="/_next/static/chunks/350ee4303b732916.js" async=""></script><script src="/_next/static/chunks/8de849ca74fc071f.js" async=""></script><script src="/_next/static/chunks/e62b91212ee7f8ff.js" async=""></script><script src="/_next/static/chunks/f19fe44237e54646.js" async=""></script><script src="/_next/static/chunks/cb0a883bafeb6805.js" async=""></script><meta name="theme-color" media="(prefers-color-scheme: dark)" content="#0A0A0A"/><meta name="theme-color" media="(prefers-color-scheme: light)" content="#fff"/><title>Global-batch load balance almost free lunch to improve your MoE LLM training — Zen LM Blog | Zen LM</title><meta name="description" content="GITHUB HUGGING FACE MODELSCOPE DISCORD"/><meta property="og:title" content="Zen LM - Open Foundation Models"/><meta property="og:description" content="Zen AI model family by Hanzo AI. 14 frontier models from 4B to 1T+ parameters for code, reasoning, vision, and multimodal tasks."/><meta property="og:url" content="https://zenlm.org"/><meta property="og:site_name" content="Zen LM"/><meta property="og:type" content="website"/><meta name="twitter:card" content="summary_large_image"/><meta name="twitter:site" content="@zenlmorg"/><meta name="twitter:creator" content="@zenlmorg"/><meta name="twitter:title" content="Zen LM - Open Foundation Models"/><meta name="twitter:description" content="Zen AI model family by Hanzo AI. 14 frontier models from 4B to 1T+ parameters for code, reasoning, vision, and multimodal tasks."/><script src="/_next/static/chunks/a6dad97d9634a72d.js" noModule=""></script><script defer src="/zen-decorate.js"></script></head><body class="antialiased"><div hidden=""><!--$--><!--/$--></div><script>((a,b,c,d,e,f,g,h)=>{let i=document.documentElement,j=["light","dark"];function k(b){var c;(Array.isArray(a)?a:[a]).forEach(a=>{let c="class"===a,d=c&&f?e.map(a=>f[a]||a):e;c?(i.classList.remove(...d),i.classList.add(f&&f[b]?f[b]:b)):i.setAttribute(a,b)}),c=b,h&&j.includes(c)&&(i.style.colorScheme=c)}if(d)k(d);else try{let a=localStorage.getItem(b)||c,d=g&&"system"===a?window.matchMedia("(prefers-color-scheme: dark)").matches?"dark":"light":a;k(d)}catch(a){}})("class","theme","system",null,["light","dark"],null,true,true)</script><!--$--><div data-closed="" role="presentation" hidden="" style="user-select:none;-webkit-user-select:none" class="fixed inset-0 z-50 backdrop-blur-xs bg-fd-overlay data-open:animate-fd-fade-in data-closed:animate-fd-fade-out"></div><div class="bg-fd-secondary/50 p-3 empty:hidden"></div><!--/$--><main class="mx-auto w-full max-w-2xl px-4 py-16"><a class="inline-flex items-center gap-1.5 text-sm text-fd-muted-foreground hover:text-fd-foreground mb-10 transition-colors" href="/blog">← Back to Blog</a><div class="mb-8"><time class="text-xs font-mono text-fd-muted-foreground uppercase tracking-wider">January 20, 2025</time><h1 class="text-3xl font-bold mt-2 mb-3">Global-batch load balance almost free lunch to improve your MoE LLM training</h1><p class="text-fd-muted-foreground text-lg mb-4">GITHUB HUGGING FACE MODELSCOPE DISCORD</p><div class="flex items-center gap-3 pt-4 border-t border-fd-border"><span class="text-sm text-fd-muted-foreground">By <!-- -->Zen LM Team</span><div class="flex gap-1.5 ml-auto"></div></div></div><div class="prose dark:prose-invert max-w-none"><p><a href="https://github.com/QwenLM/zen-Math" rel="noreferrer noopener" target="_blank">GITHUB</a> <a href="https://huggingface.co/collections/Qwen/qwen25-math-66eaa240a1b7d5ee65f1da3e" rel="noreferrer noopener" target="_blank">HUGGING FACE</a> <a href="https://modelscope.cn/organization/qwen" rel="noreferrer noopener" target="_blank">MODELSCOPE</a> <a href="https://discord.gg/yPEP2vHTu4" rel="noreferrer noopener" target="_blank">DISCORD</a></p>
|
|
<h2 class="flex scroll-m-28 flex-row items-center gap-2" id="background"><a data-card="" href="#background" class="peer">Background</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h2>
|
|
<p>The Mixture-of-Experts (MoEs) architecture has become a popular model-parameter-scale-up technique. Typically, one MoE layer consists of a router (often parameterized as one single Linear layer) and a group of experts (for transformer-based models, each expert is one feedforward layer). Given an input, only a subset of experts will be activated, and then their outputs will be aggregated based on the scores the router assigned. Specifically,</p>
|
|
<p><code>\mathbb\{y\}=\sum_\{i\in N_E,g_i\in\operatorname\{topK\}\}g_i(\mathbb\{x\})E_i(\mathbb\{x\})</code></p>
|
|
<h3 class="flex scroll-m-28 flex-row items-center gap-2" id="load-balancing-loss"><a data-card="" href="#load-balancing-loss" class="peer">Load Balancing loss</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h3>
|
|
<p>Load balancing loss is an essential regularization technique in training MoE-based networks, and high-level intuition encourages the balanced activation of all experts. It can be calculated as:</p>
|
|
<p><code>L_\{\text\{balance\}\}=N_E \sum_\{i=1\}^\{N_E\} f_ip_i</code></p>
|
|
<p>where $f_i$ is the activation frequency of the expert $E_i$, and the $p_i$ is the the average gating score that the expert $E_i$ is assigned.</p>
|
|
<p>However, most existing MoE training frameworks (e.g., Megatron-core), implement <strong>micro-batch</strong> level balance, which means the $L_{\text{balance}}$ is calculated within every micro-batch and is then averaged on the global batch level.</p>
|
|
<p>Our key point is that this implementation could be problematic if one micro-batch does not contain diverse data. For instance, imagine one micro-batch only contains some code data; the aforementioned load-balancing loss still pushes the router to distribute these code tokens to all experts uniformly, potentially hurting the model performance and preventing expert specialization.</p>
|
|
<p>This situation is even more common in training MoE-based LLMs: the data in one micro-batch is often from the same domain. This partially explains why most existing open-source MoE-based LLMs do not achieve notable expert specialization.</p>
|
|
<p>This drawback motivates us to extend the current method to the global-batch level balance.</p>
|
|
<h2 class="flex scroll-m-28 flex-row items-center gap-2" id="from-micro-batch-balance-to-global-batch-balance"><a data-card="" href="#from-micro-batch-balance-to-global-batch-balance" class="peer">From micro-batch balance to global-batch balance</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h2>
|
|
<p>One easy way to calculate global-batch balance loss is to 1) Synchronize expert selection frequency $f_i$ across all parallel groups; 2) Calculate the load-balancing loss in each parallel group (e.g., one GPU); 3) Aggregate the loss across all micro-batches.</p>
|
|
<p>Specifically,</p>
|
|
<p><code>L_\{\text\{global\}\}=N_E\sum_\{i=1\}^\{N_E\}f_i^\{\text\{global\}\}p_i^\{\text\{global\}\}=N_E\sum_\{i=1\}^\{n_E\}f_i^\{\text\{global\}\}\cdot(\frac\{1\}\{N_p\}\sum_\{j=1\}^\{N_p\}p_j)= \frac\{1\}\{N_P\} \sum_\{j=1\}^\{N_p\}(N_E \sum_\{i=1\}^\{N_E\} \bar\{f_i\} \cdot P^j_i)</code></p>
|
|
<p>Note that the expert selection frequency is just one expert-num-dimentional vector! It is almost free to synchronize them across micro-batches.</p>
|
|
<h2 class="flex scroll-m-28 flex-row items-center gap-2" id="results-more-performant-and-interpretable-moe"><a data-card="" href="#results-more-performant-and-interpretable-moe" class="peer">Results: More Performant and Interpretable MoE</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h2>
|
|
<p>We experiment with three MoE configs (3.4B with 0.6B activated, 15B with 2.54B activated, and 43B with 6.6B activated) and two data configs (120B tokens and 400B tokens). The results are shown in the following figure and chart.</p>
|
|
<p>In short, compared to the micro-batch-level loss, the global batch one achieves better performance in all settings (model, data, and tasks). More importantly, the MoE model achieves significant domain specialization with the global batch balance. In Figure (b) left, almost all experts are uniformly activated regardless of the domain. But in Figure (b) right, some experts are frequently activated by a specific domain, demonstrating their specialization.</p>
|
|
<p>We further compare the model performance with balance batch size on a 3.4B with 0.6B activated model. The ptr-training PPL decreases rapidly from balance BSZ 2 to 128, and gradually saturates after 128. In the current mainstream MoE framework, even with cross expert-parallel groups communication, the balance BSZ is generally between 8 and 16 for larger models, which further reflects the significance of our method.</p>
|
|
<p>Using global batch balance may result in a degradation in micro-batch balance, potentially affecting the computational efficiency of MoE. We further experimented with the effect of adding micro-batch balance loss on top of the global-batch balance loss (with constant weights of 0.01 of the global batch loss). It can be seen that adding local equalization improves the speed of the model (from 1.64 to 1.59 seconds per update step) while the effectiveness of the model is almost unaffected.</p>
|
|
<h2 class="flex scroll-m-28 flex-row items-center gap-2" id="conclusion"><a data-card="" href="#conclusion" class="peer">Conclusion</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h2>
|
|
<p>In summary, we investigate the challenges associated with the LBL in training MoEs models. By introducing global-batch balance loss, we achieve improved performance and foster expert specialization within the MoE model. We believe this advancement addresses an essential limitation in existing MoE training, offering a novel perspective for MoEs model optimization. Though mainly experimenting with language-based tasks, we hope our work could pave the way for training more substantial and specialised MoE models in various domains.</p>
|
|
<h1 class="flex scroll-m-28 flex-row items-center gap-2" id="citation"><a data-card="" href="#citation" class="peer">Citation</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h1>
|
|
<p>If you find our work helpful, feel free to give us a citation.</p>
|
|
<figure dir="ltr" class="my-4 bg-fd-card rounded-xl shiki relative border shadow-sm not-prose overflow-hidden text-sm shiki shiki-themes github-light github-dark" style="--shiki-light:#24292e;--shiki-dark:#e1e4e8;--shiki-light-bg:#fff;--shiki-dark-bg:#24292e" tabindex="-1"><div class="empty:hidden absolute top-3 right-2 z-2 backdrop-blur-lg rounded-lg text-fd-muted-foreground"><button type="button" class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors duration-100 disabled:pointer-events-none disabled:opacity-50 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-fd-ring p-1 [&_svg]:size-4 hover:text-fd-accent-foreground data-checked:text-fd-accent-foreground" aria-label="Copy Text"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-clipboard" aria-hidden="true"><rect width="8" height="4" x="8" y="2" rx="1" ry="1"></rect><path d="M16 4h2a2 2 0 0 1 2 2v14a2 2 0 0 1-2 2H6a2 2 0 0 1-2-2V6a2 2 0 0 1 2-2h2"></path></svg></button></div><div role="region" tabindex="0" class="text-[0.8125rem] py-3.5 overflow-auto max-h-[600px] fd-scroll-container focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-inset focus-visible:ring-fd-ring" style="--padding-right:calc(var(--spacing) * 8)"><pre class="min-w-full w-max *:flex *:flex-col"><code><span class="line"><span></span></span>
|
|
<span class="line"><span> @article{qiu2025demonsdetailimplementingload,</span></span>
|
|
<span class="line"><span> title={Demons in the Detail: On Implementing Load Balancing Loss for Training Specialized Mixture-of-Expert Models}, </span></span>
|
|
<span class="line"><span> author={</span></span>
|
|
<span class="line"><span> Zihan Qiu and Zeyu Huang and Bo Zheng and Kaiyue Wen and Zekun Wang and Rui Men and Ivan Titov and Dayiheng Liu and Jingren Zhou and Junyang Lin</span></span>
|
|
<span class="line"><span> },</span></span>
|
|
<span class="line"><span> journal={arXiv preprint arXiv:2501.11873},</span></span>
|
|
<span class="line"><span> year={2025}</span></span>
|
|
<span class="line"><span> }</span></span>
|
|
<span class="line"><span> </span></span></code></pre></div></figure></div></main><!--$--><!--/$--><script src="/_next/static/chunks/e83606e8fa9cc796.js" id="_R_" async=""></script><script>(self.__next_f=self.__next_f||[]).push([0])</script><script>self.__next_f.push([1,"1:\"$Sreact.fragment\"\n2:I[106,[\"/_next/static/chunks/59d0ad1b64f8544e.js\"],\"RootProvider\"]\n3:I[53113,[\"/_next/static/chunks/4d80e004cf4896dd.js\",\"/_next/static/chunks/350ee4303b732916.js\"],\"default\"]\n4:I[73211,[\"/_next/static/chunks/4d80e004cf4896dd.js\",\"/_next/static/chunks/350ee4303b732916.js\"],\"default\"]\n5:I[10086,[\"/_next/static/chunks/59d0ad1b64f8544e.js\",\"/_next/static/chunks/8de849ca74fc071f.js\",\"/_next/static/chunks/e62b91212ee7f8ff.js\",\"/_next/static/chunks/f19fe44237e54646.js\",\"/_next/static/chunks/cb0a883bafeb6805.js\"],\"\"]\n7:I[89923,[\"/_next/static/chunks/4d80e004cf4896dd.js\",\"/_next/static/chunks/350ee4303b732916.js\"],\"OutletBoundary\"]\n8:\"$Sreact.suspense\"\na:I[89923,[\"/_next/static/chunks/4d80e004cf4896dd.js\",\"/_next/static/chunks/350ee4303b732916.js\"],\"ViewportBoundary\"]\nc:I[89923,[\"/_next/static/chunks/4d80e004cf4896dd.js\",\"/_next/static/chunks/350ee4303b732916.js\"],\"MetadataBoundary\"]\ne:I[6998,[\"/_next/static/chunks/4d80e004cf4896dd.js\",\"/_next/static/chunks/350ee4303b732916.js\"],\"default\"]\n:HL[\"/_next/static/chunks/f2332aac77592f9d.css\",\"style\"]\n"])</script><script>self.__next_f.push([1,"0:{\"P\":null,\"b\":\"i-dnJM_MIpJSOCQWNJVMq\",\"c\":[\"\",\"blog\",\"global-load-balance\"],\"q\":\"\",\"i\":false,\"f\":[[[\"\",{\"children\":[\"blog\",{\"children\":[[\"slug\",\"global-load-balance\",\"d\"],{\"children\":[\"__PAGE__\",{}]}]}]},\"$undefined\",\"$undefined\",true],[[\"$\",\"$1\",\"c\",{\"children\":[[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/chunks/f2332aac77592f9d.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\",\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-0\",{\"src\":\"/_next/static/chunks/59d0ad1b64f8544e.js\",\"async\":true,\"nonce\":\"$undefined\"}]],[\"$\",\"html\",null,{\"lang\":\"en\",\"className\":\"dark\",\"suppressHydrationWarning\":true,\"children\":[\"$\",\"body\",null,{\"className\":\"antialiased\",\"children\":[\"$\",\"$L2\",null,{\"children\":[\"$\",\"$L3\",null,{\"parallelRouterKey\":\"children\",\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L4\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":[[\"$\",\"main\",null,{\"className\":\"flex min-h-screen flex-col items-center justify-center px-4 text-center\",\"children\":[[\"$\",\"div\",null,{\"className\":\"mb-8 opacity-20\",\"children\":[\"$\",\"svg\",null,{\"width\":\"120\",\"height\":\"120\",\"viewBox\":\"0 0 120 120\",\"fill\":\"none\",\"aria-hidden\":\"true\",\"children\":[\"$\",\"circle\",null,{\"cx\":\"60\",\"cy\":\"60\",\"r\":\"50\",\"stroke\":\"currentColor\",\"strokeWidth\":\"3\",\"strokeLinecap\":\"round\",\"strokeDasharray\":\"280 40\"}]}]}],[\"$\",\"p\",null,{\"className\":\"text-xs font-mono uppercase tracking-[0.3em] text-fd-muted-foreground mb-4\",\"children\":\"404\"}],[\"$\",\"h1\",null,{\"className\":\"text-3xl font-semibold mb-3\",\"children\":\"Page not found\"}],[\"$\",\"p\",null,{\"className\":\"text-fd-muted-foreground max-w-sm mb-10\",\"children\":\"This page doesn't exist, or it may have moved. Try the documentation or head home.\"}],[\"$\",\"div\",null,{\"className\":\"flex flex-wrap gap-3 justify-center\",\"children\":[[\"$\",\"$L5\",null,{\"href\":\"/\",\"className\":\"inline-flex items-center gap-2 rounded-lg bg-fd-primary text-fd-primary-foreground px-4 py-2 text-sm font-medium hover:opacity-90 transition\",\"children\":\"Go home\"}],[\"$\",\"$L5\",null,{\"href\":\"/docs\",\"className\":\"inline-flex items-center gap-2 rounded-lg border border-fd-border bg-fd-background px-4 py-2 text-sm font-medium hover:bg-fd-muted transition\",\"children\":\"Documentation\"}],[\"$\",\"$L5\",null,{\"href\":\"/docs/models\",\"className\":\"inline-flex items-center gap-2 rounded-lg border border-fd-border bg-fd-background px-4 py-2 text-sm font-medium hover:bg-fd-muted transition\",\"children\":\"Browse models\"}]]}],[\"$\",\"p\",null,{\"className\":\"mt-16 text-xs font-mono text-fd-muted-foreground opacity-50\",\"children\":\"zenlm.org\"}]]}],[]],\"forbidden\":\"$undefined\",\"unauthorized\":\"$undefined\"}]}]}]}]]}],{\"children\":[[\"$\",\"$1\",\"c\",{\"children\":[null,[\"$\",\"$L3\",null,{\"parallelRouterKey\":\"children\",\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L4\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":\"$undefined\",\"forbidden\":\"$undefined\",\"unauthorized\":\"$undefined\"}]]}],{\"children\":[[\"$\",\"$1\",\"c\",{\"children\":[null,[\"$\",\"$L3\",null,{\"parallelRouterKey\":\"children\",\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L4\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":\"$undefined\",\"forbidden\":\"$undefined\",\"unauthorized\":\"$undefined\"}]]}],{\"children\":[[\"$\",\"$1\",\"c\",{\"children\":[\"$L6\",[[\"$\",\"script\",\"script-0\",{\"src\":\"/_next/static/chunks/8de849ca74fc071f.js\",\"async\":true,\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-1\",{\"src\":\"/_next/static/chunks/e62b91212ee7f8ff.js\",\"async\":true,\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-2\",{\"src\":\"/_next/static/chunks/f19fe44237e54646.js\",\"async\":true,\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-3\",{\"src\":\"/_next/static/chunks/cb0a883bafeb6805.js\",\"async\":true,\"nonce\":\"$undefined\"}]],[\"$\",\"$L7\",null,{\"children\":[\"$\",\"$8\",null,{\"name\":\"Next.MetadataOutlet\",\"children\":\"$@9\"}]}]]}],{},null,false,false]},null,false,false]},null,false,false]},null,false,false],[\"$\",\"$1\",\"h\",{\"children\":[null,[\"$\",\"$La\",null,{\"children\":\"$Lb\"}],[\"$\",\"div\",null,{\"hidden\":true,\"children\":[\"$\",\"$Lc\",null,{\"children\":[\"$\",\"$8\",null,{\"name\":\"Next.Metadata\",\"children\":\"$Ld\"}]}]}],null]}],false]],\"m\":\"$undefined\",\"G\":[\"$e\",[]],\"S\":true}\n"])</script><script>self.__next_f.push([1,"f:I[48068,[\"/_next/static/chunks/59d0ad1b64f8544e.js\",\"/_next/static/chunks/8de849ca74fc071f.js\",\"/_next/static/chunks/e62b91212ee7f8ff.js\",\"/_next/static/chunks/f19fe44237e54646.js\",\"/_next/static/chunks/cb0a883bafeb6805.js\"],\"default\"]\n"])</script><script>self.__next_f.push([1,"6:[\"$\",\"main\",null,{\"className\":\"mx-auto w-full max-w-2xl px-4 py-16\",\"children\":[[\"$\",\"$L5\",null,{\"href\":\"/blog\",\"className\":\"inline-flex items-center gap-1.5 text-sm text-fd-muted-foreground hover:text-fd-foreground mb-10 transition-colors\",\"children\":\"← Back to Blog\"}],[\"$\",\"div\",null,{\"className\":\"mb-8\",\"children\":[[\"$\",\"time\",null,{\"className\":\"text-xs font-mono text-fd-muted-foreground uppercase tracking-wider\",\"children\":\"January 20, 2025\"}],[\"$\",\"h1\",null,{\"className\":\"text-3xl font-bold mt-2 mb-3\",\"children\":\"Global-batch load balance almost free lunch to improve your MoE LLM training\"}],[\"$\",\"p\",null,{\"className\":\"text-fd-muted-foreground text-lg mb-4\",\"children\":\"GITHUB HUGGING FACE MODELSCOPE DISCORD\"}],[\"$\",\"div\",null,{\"className\":\"flex items-center gap-3 pt-4 border-t border-fd-border\",\"children\":[[\"$\",\"span\",null,{\"className\":\"text-sm text-fd-muted-foreground\",\"children\":[\"By \",\"Zen LM Team\"]}],[\"$\",\"div\",null,{\"className\":\"flex gap-1.5 ml-auto\",\"children\":[]}]]}]]}],[\"$\",\"div\",null,{\"className\":\"prose dark:prose-invert max-w-none\",\"children\":[[\"$\",\"p\",null,{\"children\":[[\"$\",\"$Lf\",null,{\"href\":\"https://github.com/QwenLM/zen-Math\",\"children\":\"GITHUB\"}],\" \",[\"$\",\"$Lf\",null,{\"href\":\"https://huggingface.co/collections/Qwen/qwen25-math-66eaa240a1b7d5ee65f1da3e\",\"children\":\"HUGGING FACE\"}],\" \",[\"$\",\"$Lf\",null,{\"href\":\"https://modelscope.cn/organization/qwen\",\"children\":\"MODELSCOPE\"}],\" \",[\"$\",\"$Lf\",null,{\"href\":\"https://discord.gg/yPEP2vHTu4\",\"children\":\"DISCORD\"}]]}],\"\\n\",[\"$\",\"h2\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"background\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#background\",\"className\":\"peer\",\"children\":\"Background\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}],\"\\n\",[\"$\",\"p\",null,{\"children\":\"The Mixture-of-Experts (MoEs) architecture has become a popular model-parameter-scale-up technique. Typically, one MoE layer consists of a router (often parameterized as one single Linear layer) and a group of experts (for transformer-based models, each expert is one feedforward layer). Given an input, only a subset of experts will be activated, and then their outputs will be aggregated based on the scores the router assigned. Specifically,\"}],\"\\n\",[\"$\",\"p\",null,{\"children\":[\"$\",\"code\",null,{\"children\":\"\\\\mathbb\\\\{y\\\\}=\\\\sum_\\\\{i\\\\in N_E,g_i\\\\in\\\\operatorname\\\\{topK\\\\}\\\\}g_i(\\\\mathbb\\\\{x\\\\})E_i(\\\\mathbb\\\\{x\\\\})\"}]}],\"\\n\",[\"$\",\"h3\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"load-balancing-loss\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#load-balancing-loss\",\"className\":\"peer\",\"children\":\"Load Balancing loss\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}],\"\\n\",[\"$\",\"p\",null,{\"children\":\"Load balancing loss is an essential regularization technique in training MoE-based networks, and high-level intuition encourages the balanced activation of all experts. It can be calculated as:\"}],\"\\n\",[\"$\",\"p\",null,{\"children\":[\"$\",\"code\",null,{\"children\":\"L_\\\\{\\\\text\\\\{balance\\\\}\\\\}=N_E \\\\sum_\\\\{i=1\\\\}^\\\\{N_E\\\\} f_ip_i\"}]}],\"\\n\",\"$L10\",\"\\n\",\"$L11\",\"\\n\",\"$L12\",\"\\n\",\"$L13\",\"\\n\",\"$L14\",\"\\n\",\"$L15\",\"\\n\",\"$L16\",\"\\n\",\"$L17\",\"\\n\",\"$L18\",\"\\n\",\"$L19\",\"\\n\",\"$L1a\",\"\\n\",\"$L1b\",\"\\n\",\"$L1c\",\"\\n\",\"$L1d\",\"\\n\",\"$L1e\",\"\\n\",\"$L1f\",\"\\n\",\"$L20\",\"\\n\",\"$L21\",\"\\n\",\"$L22\",\"\\n\",\"$L23\"]}]]}]\n"])</script><script>self.__next_f.push([1,"24:I[51504,[\"/_next/static/chunks/59d0ad1b64f8544e.js\",\"/_next/static/chunks/8de849ca74fc071f.js\",\"/_next/static/chunks/e62b91212ee7f8ff.js\",\"/_next/static/chunks/f19fe44237e54646.js\",\"/_next/static/chunks/cb0a883bafeb6805.js\"],\"CodeBlock\"]\n25:I[51504,[\"/_next/static/chunks/59d0ad1b64f8544e.js\",\"/_next/static/chunks/8de849ca74fc071f.js\",\"/_next/static/chunks/e62b91212ee7f8ff.js\",\"/_next/static/chunks/f19fe44237e54646.js\",\"/_next/static/chunks/cb0a883bafeb6805.js\"],\"Pre\"]\n10:[\"$\",\"p\",null,{\"children\":\"where $f_i$ is the activation frequency of the expert $E_i$, and the $p_i$ is the the average gating score that the expert $E_i$ is assigned.\"}]\n11:[\"$\",\"p\",null,{\"children\":[\"However, most existing MoE training frameworks (e.g., Megatron-core), implement \",[\"$\",\"strong\",null,{\"children\":\"micro-batch\"}],\" level balance, which means the $L_{\\\\text{balance}}$ is calculated within every micro-batch and is then averaged on the global batch level.\"]}]\n12:[\"$\",\"p\",null,{\"children\":\"Our key point is that this implementation could be problematic if one micro-batch does not contain diverse data. For instance, imagine one micro-batch only contains some code data; the aforementioned load-balancing loss still pushes the router to distribute these code tokens to all experts uniformly, potentially hurting the model performance and preventing expert specialization.\"}]\n13:[\"$\",\"p\",null,{\"children\":\"This situation is even more common in training MoE-based LLMs: the data in one micro-batch is often from the same domain. This partially explains why most existing open-source MoE-based LLMs do not achieve notable expert specialization.\"}]\n14:[\"$\",\"p\",null,{\"children\":\"This drawback motivates us to extend the current method to the global-batch level balance.\"}]\n15:[\"$\",\"h2\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"from-micro-batch-balance-to-global-batch-balance\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#from-micro-batch-balance-to-global-batch-balance\",\"className\":\"peer\",\"children\":\"From micro-batch balance to global-batch balance\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n16:[\"$\",\"p\",null,{\"children\":\"One easy way to calculate global-batch balance loss is to 1) Synchronize expert selection frequency $f_i$ across all parallel groups; 2) Calculate the load-balancing loss in each parallel group (e.g., one GPU); 3) Aggregate the loss across all micro-batches.\"}]\n17:[\"$\",\"p\",null,{\"children\":\"Specifically,\"}]\n18:[\"$\",\"p\",null,{\"children\":[\"$\",\"code\",null,{\"children\":\"L_\\\\{\\\\text\\\\{global\\\\}\\\\}=N_E\\\\sum_\\\\{i=1\\\\}^\\\\{N_E\\\\}f_i^\\\\{\\\\text\\\\{global\\\\}\\\\}p_i^\\\\{\\\\text\\\\{global\\\\}\\\\}=N_E\\\\sum_\\\\{i=1\\\\}^\\\\{n_E\\\\}f_i^\\\\{\\\\text\\\\{global\\\\}\\\\}\\\\cdot(\\\\frac\\\\{1\\\\}\\\\{N_p\\\\}\\\\sum_\\\\{j=1\\\\}^\\\\{N_p\\\\}p_j)= \\\\frac\\\\{1\\\\}\\\\{N_P\\\\} \\\\sum_\\\\{j=1\\\\}^\\\\{N_p\\\\}(N_E \\\\sum_\\\\{i=1\\\\}^\\\\{N_E\\\\} \\\\bar\\\\{f_i\\\\} \\\\cdot P^j_i)\"}]}]\n19:[\"$\",\"p\",null,{\"children\":\"Note that the expert selection frequency is just one expert-num-dimentional vector! It is almost free to synchronize them across micro-batches.\"}]\n1a:[\"$\",\"h2\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"results-more-performant-and-interpretable-moe\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#results-more-performant-and-interpretable-moe\",\"className\":\"peer\",\"children\":\"Results: More Performant and Interpretable MoE\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLin"])</script><script>self.__next_f.push([1,"ejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n1b:[\"$\",\"p\",null,{\"children\":\"We experiment with three MoE configs (3.4B with 0.6B activated, 15B with 2.54B activated, and 43B with 6.6B activated) and two data configs (120B tokens and 400B tokens). The results are shown in the following figure and chart.\"}]\n1c:[\"$\",\"p\",null,{\"children\":\"In short, compared to the micro-batch-level loss, the global batch one achieves better performance in all settings (model, data, and tasks). More importantly, the MoE model achieves significant domain specialization with the global batch balance. In Figure (b) left, almost all experts are uniformly activated regardless of the domain. But in Figure (b) right, some experts are frequently activated by a specific domain, demonstrating their specialization.\"}]\n1d:[\"$\",\"p\",null,{\"children\":\"We further compare the model performance with balance batch size on a 3.4B with 0.6B activated model. The ptr-training PPL decreases rapidly from balance BSZ 2 to 128, and gradually saturates after 128. In the current mainstream MoE framework, even with cross expert-parallel groups communication, the balance BSZ is generally between 8 and 16 for larger models, which further reflects the significance of our method.\"}]\n1e:[\"$\",\"p\",null,{\"children\":\"Using global batch balance may result in a degradation in micro-batch balance, potentially affecting the computational efficiency of MoE. We further experimented with the effect of adding micro-batch balance loss on top of the global-batch balance loss (with constant weights of 0.01 of the global batch loss). It can be seen that adding local equalization improves the speed of the model (from 1.64 to 1.59 seconds per update step) while the effectiveness of the model is almost unaffected.\"}]\n1f:[\"$\",\"h2\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"conclusion\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#conclusion\",\"className\":\"peer\",\"children\":\"Conclusion\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n20:[\"$\",\"p\",null,{\"children\":\"In summary, we investigate the challenges associated with the LBL in training MoEs models. By introducing global-batch balance loss, we achieve improved performance and foster expert specialization within the MoE model. We believe this advancement addresses an essential limitation in existing MoE training, offering a novel perspective for MoEs model optimization. Though mainly experimenting with language-based tasks, we hope our work could pave the way for training more substantial and specialised MoE models in various domains.\"}]\n21:[\"$\",\"h1\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"citation\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#citation\",\"className\":\"peer\",\"children\":\"Citation\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-"])</script><script>self.__next_f.push([1,"7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n22:[\"$\",\"p\",null,{\"children\":\"If you find our work helpful, feel free to give us a citation.\"}]\n"])</script><script>self.__next_f.push([1,"23:[\"$\",\"$L24\",null,{\"className\":\"shiki shiki-themes github-light github-dark\",\"style\":{\"--shiki-light\":\"#24292e\",\"--shiki-dark\":\"#e1e4e8\",\"--shiki-light-bg\":\"#fff\",\"--shiki-dark-bg\":\"#24292e\"},\"tabIndex\":\"0\",\"icon\":\"\u003csvg viewBox=\\\"0 0 24 24\\\"\u003e\u003cpath d=\\\"M 6,1 C 4.354992,1 3,2.354992 3,4 v 16 c 0,1.645008 1.354992,3 3,3 h 12 c 1.645008,0 3,-1.354992 3,-3 V 8 7 A 1.0001,1.0001 0 0 0 20.707031,6.2929687 l -5,-5 A 1.0001,1.0001 0 0 0 15,1 h -1 z m 0,2 h 7 v 3 c 0,1.645008 1.354992,3 3,3 h 3 v 11 c 0,0.564129 -0.435871,1 -1,1 H 6 C 5.4358712,21 5,20.564129 5,20 V 4 C 5,3.4358712 5.4358712,3 6,3 Z M 15,3.4140625 18.585937,7 H 16 C 15.435871,7 15,6.5641288 15,6 Z\\\" fill=\\\"currentColor\\\" /\u003e\u003c/svg\u003e\",\"children\":[\"$\",\"$L25\",null,{\"children\":[\"$\",\"code\",null,{\"children\":[[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{}]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"children\":\" @article{qiu2025demonsdetailimplementingload,\"}]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"children\":\" title={Demons in the Detail: On Implementing Load Balancing Loss for Training Specialized Mixture-of-Expert Models}, \"}]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"children\":\" author={\"}]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"children\":\" Zihan Qiu and Zeyu Huang and Bo Zheng and Kaiyue Wen and Zekun Wang and Rui Men and Ivan Titov and Dayiheng Liu and Jingren Zhou and Junyang Lin\"}]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"children\":\" },\"}]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"children\":\" journal={arXiv preprint arXiv:2501.11873},\"}]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"children\":\" year={2025}\"}]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"children\":\" }\"}]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"children\":\" \"}]}]]}]}]}]\n"])</script><script>self.__next_f.push([1,"b:[[\"$\",\"meta\",\"0\",{\"charSet\":\"utf-8\"}],[\"$\",\"meta\",\"1\",{\"name\":\"viewport\",\"content\":\"width=device-width, initial-scale=1\"}],[\"$\",\"meta\",\"2\",{\"name\":\"theme-color\",\"media\":\"(prefers-color-scheme: dark)\",\"content\":\"#0A0A0A\"}],[\"$\",\"meta\",\"3\",{\"name\":\"theme-color\",\"media\":\"(prefers-color-scheme: light)\",\"content\":\"#fff\"}]]\n"])</script><script>self.__next_f.push([1,"9:null\nd:[[\"$\",\"title\",\"0\",{\"children\":\"Global-batch load balance almost free lunch to improve your MoE LLM training — Zen LM Blog | Zen LM\"}],[\"$\",\"meta\",\"1\",{\"name\":\"description\",\"content\":\"GITHUB HUGGING FACE MODELSCOPE DISCORD\"}],[\"$\",\"meta\",\"2\",{\"property\":\"og:title\",\"content\":\"Zen LM - Open Foundation Models\"}],[\"$\",\"meta\",\"3\",{\"property\":\"og:description\",\"content\":\"Zen AI model family by Hanzo AI. 14 frontier models from 4B to 1T+ parameters for code, reasoning, vision, and multimodal tasks.\"}],[\"$\",\"meta\",\"4\",{\"property\":\"og:url\",\"content\":\"https://zenlm.org\"}],[\"$\",\"meta\",\"5\",{\"property\":\"og:site_name\",\"content\":\"Zen LM\"}],[\"$\",\"meta\",\"6\",{\"property\":\"og:type\",\"content\":\"website\"}],[\"$\",\"meta\",\"7\",{\"name\":\"twitter:card\",\"content\":\"summary_large_image\"}],[\"$\",\"meta\",\"8\",{\"name\":\"twitter:site\",\"content\":\"@zenlmorg\"}],[\"$\",\"meta\",\"9\",{\"name\":\"twitter:creator\",\"content\":\"@zenlmorg\"}],[\"$\",\"meta\",\"10\",{\"name\":\"twitter:title\",\"content\":\"Zen LM - Open Foundation Models\"}],[\"$\",\"meta\",\"11\",{\"name\":\"twitter:description\",\"content\":\"Zen AI model family by Hanzo AI. 14 frontier models from 4B to 1T+ parameters for code, reasoning, vision, and multimodal tasks.\"}]]\n"])</script><script>
|
|
(function(){
|
|
// After React renders the lotus emoji from RSC payload, walk the DOM
|
|
// and replace it with the enso SVG. Polls so subsequent React re-renders
|
|
// also get cleaned up.
|
|
function swap(){
|
|
// 1. Replace lotus + ' Zen LM' text inside <a> brand links
|
|
document.querySelectorAll('a').forEach(function(a){
|
|
if (a.dataset.zenLotusSwapped) return;
|
|
var t = (a.textContent || '');
|
|
if (!/.*Zen LM|Zen LM.*/.test(t)) return;
|
|
// Find the lotus text node specifically
|
|
var walker = document.createTreeWalker(a, NodeFilter.SHOW_TEXT, null);
|
|
var node;
|
|
while ((node = walker.nextNode())) {
|
|
if (node.nodeValue && node.nodeValue.indexOf('') !== -1) {
|
|
node.nodeValue = node.nodeValue.replace(/\s?/g, '');
|
|
}
|
|
}
|
|
// Inject enso SVG at start
|
|
if (!a.querySelector('svg[aria-label="Zen LM"]')) {
|
|
var ns = 'http://www.w3.org/2000/svg';
|
|
var svg = document.createElementNS(ns, 'svg');
|
|
svg.setAttribute('viewBox', '0 0 512 512');
|
|
svg.setAttribute('fill', 'none');
|
|
svg.setAttribute('width', '18');
|
|
svg.setAttribute('height', '18');
|
|
svg.setAttribute('aria-label', 'Zen LM');
|
|
svg.style.cssText = 'display:inline-block;vertical-align:-2px;margin-right:6px;flex-shrink:0';
|
|
var p = document.createElementNS(ns, 'path');
|
|
p.setAttribute('d', 'M 256 84 A 172 172 0 1 0 256 428 A 172 172 0 1 0 256 84 Z');
|
|
p.setAttribute('stroke', 'currentColor');
|
|
p.setAttribute('stroke-width', '26');
|
|
p.setAttribute('stroke-linecap', 'round');
|
|
p.setAttribute('stroke-linejoin', 'round');
|
|
svg.appendChild(p);
|
|
a.insertBefore(svg, a.firstChild);
|
|
}
|
|
a.dataset.zenLotusSwapped = '1';
|
|
});
|
|
}
|
|
function start(){ swap(); setInterval(swap, 600); }
|
|
if (document.readyState !== 'loading') start();
|
|
else document.addEventListener('DOMContentLoaded', start);
|
|
})();
|
|
</script>
|
|
|
|
<style id="zenlm-brand-anim">
|
|
@keyframes zenlm-brand-draw { to { stroke-dashoffset: 0; } }
|
|
svg.zenlm-brand-svg .zenlm-brand-path {
|
|
stroke-dasharray: 1100;
|
|
stroke-dashoffset: 1100;
|
|
animation: zenlm-brand-draw 1.4s cubic-bezier(.45,0,.2,1) 0.1s 1 forwards;
|
|
}
|
|
.zen-link-icon { display: inline-block; vertical-align: middle; flex-shrink: 0; }
|
|
</style></body></html> |