Files

115 lines
49 KiB
Plaintext
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
1:"$Sreact.fragment"
2:I[106,["/_next/static/chunks/59d0ad1b64f8544e.js"],"RootProvider"]
3:I[53113,["/_next/static/chunks/4d80e004cf4896dd.js","/_next/static/chunks/350ee4303b732916.js"],"default"]
4:I[73211,["/_next/static/chunks/4d80e004cf4896dd.js","/_next/static/chunks/350ee4303b732916.js"],"default"]
5:I[10086,["/_next/static/chunks/59d0ad1b64f8544e.js","/_next/static/chunks/8de849ca74fc071f.js","/_next/static/chunks/e62b91212ee7f8ff.js","/_next/static/chunks/f19fe44237e54646.js","/_next/static/chunks/cb0a883bafeb6805.js"],""]
7:I[89923,["/_next/static/chunks/4d80e004cf4896dd.js","/_next/static/chunks/350ee4303b732916.js"],"OutletBoundary"]
8:"$Sreact.suspense"
a:I[89923,["/_next/static/chunks/4d80e004cf4896dd.js","/_next/static/chunks/350ee4303b732916.js"],"ViewportBoundary"]
c:I[89923,["/_next/static/chunks/4d80e004cf4896dd.js","/_next/static/chunks/350ee4303b732916.js"],"MetadataBoundary"]
e:I[6998,["/_next/static/chunks/4d80e004cf4896dd.js","/_next/static/chunks/350ee4303b732916.js"],"default"]
:HL["/_next/static/chunks/f2332aac77592f9d.css","style"]
0:{"P":null,"b":"i-dnJM_MIpJSOCQWNJVMq","c":["","blog","drop-upcycling-zen-mode"],"q":"","i":false,"f":[[["",{"children":["blog",{"children":[["slug","drop-upcycling-zen-mode","d"],{"children":["__PAGE__",{}]}]}]},"$undefined","$undefined",true],[["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/_next/static/chunks/f2332aac77592f9d.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","script","script-0",{"src":"/_next/static/chunks/59d0ad1b64f8544e.js","async":true,"nonce":"$undefined"}]],["$","html",null,{"lang":"en","className":"dark","suppressHydrationWarning":true,"children":["$","body",null,{"className":"antialiased","children":["$","$L2",null,{"children":["$","$L3",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L4",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","main",null,{"className":"flex min-h-screen flex-col items-center justify-center px-4 text-center","children":[["$","div",null,{"className":"mb-8 opacity-20","children":["$","svg",null,{"width":"120","height":"120","viewBox":"0 0 120 120","fill":"none","aria-hidden":"true","children":["$","circle",null,{"cx":"60","cy":"60","r":"50","stroke":"currentColor","strokeWidth":"3","strokeLinecap":"round","strokeDasharray":"280 40"}]}]}],["$","p",null,{"className":"text-xs font-mono uppercase tracking-[0.3em] text-fd-muted-foreground mb-4","children":"404"}],["$","h1",null,{"className":"text-3xl font-semibold mb-3","children":"Page not found"}],["$","p",null,{"className":"text-fd-muted-foreground max-w-sm mb-10","children":"This page doesn't exist, or it may have moved. Try the documentation or head home."}],["$","div",null,{"className":"flex flex-wrap gap-3 justify-center","children":[["$","$L5",null,{"href":"/","className":"inline-flex items-center gap-2 rounded-lg bg-fd-primary text-fd-primary-foreground px-4 py-2 text-sm font-medium hover:opacity-90 transition","children":"Go home"}],["$","$L5",null,{"href":"/docs","className":"inline-flex items-center gap-2 rounded-lg border border-fd-border bg-fd-background px-4 py-2 text-sm font-medium hover:bg-fd-muted transition","children":"Documentation"}],["$","$L5",null,{"href":"/docs/models","className":"inline-flex items-center gap-2 rounded-lg border border-fd-border bg-fd-background px-4 py-2 text-sm font-medium hover:bg-fd-muted transition","children":"Browse models"}]]}],["$","p",null,{"className":"mt-16 text-xs font-mono text-fd-muted-foreground opacity-50","children":"zenlm.org"}]]}],[]],"forbidden":"$undefined","unauthorized":"$undefined"}]}]}]}]]}],{"children":[["$","$1","c",{"children":[null,["$","$L3",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L4",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","forbidden":"$undefined","unauthorized":"$undefined"}]]}],{"children":[["$","$1","c",{"children":[null,["$","$L3",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L4",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","forbidden":"$undefined","unauthorized":"$undefined"}]]}],{"children":[["$","$1","c",{"children":["$L6",[["$","script","script-0",{"src":"/_next/static/chunks/8de849ca74fc071f.js","async":true,"nonce":"$undefined"}],["$","script","script-1",{"src":"/_next/static/chunks/e62b91212ee7f8ff.js","async":true,"nonce":"$undefined"}],["$","script","script-2",{"src":"/_next/static/chunks/f19fe44237e54646.js","async":true,"nonce":"$undefined"}],["$","script","script-3",{"src":"/_next/static/chunks/cb0a883bafeb6805.js","async":true,"nonce":"$undefined"}]],["$","$L7",null,{"children":["$","$8",null,{"name":"Next.MetadataOutlet","children":"$@9"}]}]]}],{},null,false,false]},null,false,false]},null,false,false]},null,false,false],["$","$1","h",{"children":[null,["$","$La",null,{"children":"$Lb"}],["$","div",null,{"hidden":true,"children":["$","$Lc",null,{"children":["$","$8",null,{"name":"Next.Metadata","children":"$Ld"}]}]}],null]}],false]],"m":"$undefined","G":["$e",[]],"S":true}
f:I[48068,["/_next/static/chunks/59d0ad1b64f8544e.js","/_next/static/chunks/8de849ca74fc071f.js","/_next/static/chunks/e62b91212ee7f8ff.js","/_next/static/chunks/f19fe44237e54646.js","/_next/static/chunks/cb0a883bafeb6805.js"],"default"]
6:["$","main",null,{"className":"mx-auto w-full max-w-2xl px-4 py-16","children":[["$","$L5",null,{"href":"/blog","className":"inline-flex items-center gap-1.5 text-sm text-fd-muted-foreground hover:text-fd-foreground mb-10 transition-colors","children":"← Back to Blog"}],["$","div",null,{"className":"mb-8","children":[["$","time",null,{"className":"text-xs font-mono text-fd-muted-foreground uppercase tracking-wider","children":"February 27, 2026"}],["$","h1",null,{"className":"text-3xl font-bold mt-2 mb-3","children":"Drop-Upcycling and the Birth of Zen MoDE Architecture"}],["$","p",null,{"className":"text-fd-muted-foreground text-lg mb-4","children":"How Drop-Upcycling (arXiv:2502.19261) transforms dense checkpoints into MoE models at 1/4 training cost, and how it shapes Zen MoDE — our Mixture of Distilled Experts architecture."}],["$","div",null,{"className":"flex items-center gap-3 pt-4 border-t border-fd-border","children":[["$","span",null,{"className":"text-sm text-fd-muted-foreground","children":["By ","Zen LM Team"]}],["$","div",null,{"className":"flex gap-1.5 ml-auto","children":[["$","span","Research",{"className":"text-[10px] font-mono uppercase tracking-wider bg-fd-muted text-fd-muted-foreground px-1.5 py-0.5 rounded","children":"Research"}],["$","span","MoE",{"className":"text-[10px] font-mono uppercase tracking-wider bg-fd-muted text-fd-muted-foreground px-1.5 py-0.5 rounded","children":"MoE"}],["$","span","Architecture",{"className":"text-[10px] font-mono uppercase tracking-wider bg-fd-muted text-fd-muted-foreground px-1.5 py-0.5 rounded","children":"Architecture"}],["$","span","Drop-Upcycling",{"className":"text-[10px] font-mono uppercase tracking-wider bg-fd-muted text-fd-muted-foreground px-1.5 py-0.5 rounded","children":"Drop-Upcycling"}],["$","span","Zen MoDE",{"className":"text-[10px] font-mono uppercase tracking-wider bg-fd-muted text-fd-muted-foreground px-1.5 py-0.5 rounded","children":"Zen MoDE"}]]}]]}]]}],["$","div",null,{"className":"prose dark:prose-invert max-w-none","children":[["$","p",null,{"children":[["$","$Lf",null,{"href":"https://arxiv.org/abs/2502.19261","children":"DROP-UPCYCLING PAPER"}]," ",["$","$Lf",null,{"href":"https://huggingface.co/zenlm","children":"ZEN MODELS"}]," ",["$","$Lf",null,{"href":"https://github.com/zenlm","children":"ZEN CODE"}]]}],"\n",["$","p",null,{"children":"Mixture of Experts (MoE) is the architecture that makes trillion-parameter models economically viable. By routing each token through a small subset of expert networks rather than the full parameter set, MoE achieves large-model quality at dense-model inference cost. The problem: training an MoE from scratch is expensive. You are paying for both the scale and the specialization overhead."}],"\n",["$","p",null,{"children":["Drop-Upcycling is a technique that converts a trained dense checkpoint into an MoE at roughly 1/4 the training cost of building the MoE from scratch. It is one of the foundational techniques behind ",["$","strong",null,{"children":"Zen MoDE"}]," — our Mixture of Distilled Experts architecture. This post explains how it works, why it works, and how we apply it at three scales."]}],"\n",["$","h2",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"why-naive-expert-cloning-fails","children":[["$","a",null,{"data-card":"","href":"#why-naive-expert-cloning-fails","className":"peer","children":"Why Naive Expert Cloning Fails"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],"$L10","$undefined"]}]]}],"\n","$L11","\n","$L12","\n","$L13","\n","$L14","\n","$L15","\n","$L16","\n","$L17","\n","$L18","\n","$L19","\n","$L1a","\n","$L1b","\n","$L1c","\n","$L1d","\n","$L1e","\n","$L1f","\n","$L20","\n","$L21","\n","$L22","\n","$L23","\n","$L24","\n","$L25","\n","$L26","\n","$L27","\n","$L28","\n","$L29","\n","$L2a","\n","$L2b","\n","$L2c","\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n","$L2d","\n","$L2e","\n","$L2f","\n","$L30","\n","$L31","\n","$L32","\n","$L33","\n","$L34","\n","$L35","\n","$L36","\n","$L37","\n","$L38"]}]]}]
39:I[51504,["/_next/static/chunks/59d0ad1b64f8544e.js","/_next/static/chunks/8de849ca74fc071f.js","/_next/static/chunks/e62b91212ee7f8ff.js","/_next/static/chunks/f19fe44237e54646.js","/_next/static/chunks/cb0a883bafeb6805.js"],"CodeBlock"]
3a:I[51504,["/_next/static/chunks/59d0ad1b64f8544e.js","/_next/static/chunks/8de849ca74fc071f.js","/_next/static/chunks/e62b91212ee7f8ff.js","/_next/static/chunks/f19fe44237e54646.js","/_next/static/chunks/cb0a883bafeb6805.js"],"Pre"]
10:["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}]
11:["$","p",null,{"children":["The obvious approach to dense-to-MoE conversion: clone the dense FFN block N times to create N experts, initialize a router, and continue training. This costs almost nothing upfront. The problem reveals itself within a few thousand steps: ",["$","strong",null,{"children":"weight correlation collapse"}],"."]}]
12:["$","p",null,{"children":"When all experts start from identical weights, they receive identical gradients on every token that routes to multiple of them simultaneously. The router has no signal to differentiate them. Gradient updates push all experts in the same direction. Within tens of thousands of steps, the experts have converged to nearly identical weights despite being nominally separate. The MoE behaves like a dense model with routing overhead and no specialization benefit."}]
13:["$","p",null,{"children":"This is not a training instability — it is a symmetry problem. Identical initialization creates a saddle point in the loss landscape where all expert-breaking perturbations are equally likely but none are preferred. The model sits at the saddle indefinitely."}]
14:["$","h2",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"drop-upcycling-breaking-symmetry-with-structured-noise","children":[["$","a",null,{"data-card":"","href":"#drop-upcycling-breaking-symmetry-with-structured-noise","className":"peer","children":"Drop-Upcycling: Breaking Symmetry With Structured Noise"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
15:["$","p",null,{"children":[["$","strong",null,{"children":"Paper"}]," : arXiv:2502.19261"]}]
16:["$","p",null,{"children":"Drop-Upcycling solves the symmetry problem by deliberately damaging each expert’s initialization in a structured way. For each expert i, randomly select p% of the FFN rows and reinitialize them from a normal distribution:"}]
17:["$","$L39",null,{"className":"shiki shiki-themes github-light github-dark","style":{"--shiki-light":"#24292e","--shiki-dark":"#e1e4e8","--shiki-light-bg":"#fff","--shiki-dark-bg":"#24292e"},"tabIndex":"0","icon":"<svg viewBox=\"0 0 24 24\"><path d=\"M 6,1 C 4.354992,1 3,2.354992 3,4 v 16 c 0,1.645008 1.354992,3 3,3 h 12 c 1.645008,0 3,-1.354992 3,-3 V 8 7 A 1.0001,1.0001 0 0 0 20.707031,6.2929687 l -5,-5 A 1.0001,1.0001 0 0 0 15,1 h -1 z m 0,2 h 7 v 3 c 0,1.645008 1.354992,3 3,3 h 3 v 11 c 0,0.564129 -0.435871,1 -1,1 H 6 C 5.4358712,21 5,20.564129 5,20 V 4 C 5,3.4358712 5.4358712,3 6,3 Z M 15,3.4140625 18.585937,7 H 16 C 15.435871,7 15,6.5641288 15,6 Z\" fill=\"currentColor\" /></svg>","children":["$","$L3a",null,{"children":["$","code",null,{"children":[["$","span",null,{"className":"line","children":["$","span",null,{}]}],"\n",["$","span",null,{"className":"line","children":["$","span",null,{"children":" w_j^(i) = N(0, σ²) if j ∈ dropped_rows(i)"}]}],"\n",["$","span",null,{"className":"line","children":["$","span",null,{"children":" w_j^(i) = w_j^dense otherwise"}]}],"\n",["$","span",null,{"className":"line","children":["$","span",null,{"children":" "}]}]]}]}]}]
18:["$","p",null,{"children":"The dropped set is different for each expert (sampled independently). This breaks the symmetry: experts start from the same functional foundation but with different “holes” in their weight matrices."}]
3b:T5c0,<svg viewBox="0 0 24 24"><path d="M14.25.18l.9.2.73.26.59.3.45.32.34.34.25.34.16.33.1.3.04.26.02.2-.01.13V8.5l-.05.63-.13.55-.21.46-.26.38-.3.31-.33.25-.35.19-.35.14-.33.1-.3.07-.26.04-.21.02H8.77l-.69.05-.59.14-.5.22-.41.27-.33.32-.27.35-.2.36-.15.37-.1.35-.07.32-.04.27-.02.21v3.06H3.17l-.21-.03-.28-.07-.32-.12-.35-.18-.36-.26-.36-.36-.35-.46-.32-.59-.28-.73-.21-.88-.14-1.05-.05-1.23.06-1.22.16-1.04.24-.87.32-.71.36-.57.4-.44.42-.33.42-.24.4-.16.36-.1.32-.05.24-.01h.16l.06.01h8.16v-.83H6.18l-.01-2.75-.02-.37.05-.34.11-.31.17-.28.25-.26.31-.23.38-.2.44-.18.51-.15.58-.12.64-.1.71-.06.77-.04.84-.02 1.27.05zm-6.3 1.98l-.23.33-.08.41.08.41.23.34.33.22.41.09.41-.09.33-.22.23-.34.08-.41-.08-.41-.23-.33-.33-.22-.41-.09-.41.09zm13.09 3.95l.28.06.32.12.35.18.36.27.36.35.35.47.32.59.28.73.21.88.14 1.04.05 1.23-.06 1.23-.16 1.04-.24.86-.32.71-.36.57-.4.45-.42.33-.42.24-.4.16-.36.09-.32.05-.24.02-.16-.01h-8.22v.82h5.84l.01 2.76.02.36-.05.34-.11.31-.17.29-.25.25-.31.24-.38.2-.44.17-.51.15-.58.13-.64.09-.71.07-.77.04-.84.01-1.27-.04-1.07-.14-.9-.2-.73-.25-.59-.3-.45-.33-.34-.34-.25-.34-.16-.33-.1-.3-.04-.25-.02-.2.01-.13v-5.34l.05-.64.13-.54.21-.46.26-.38.3-.32.33-.24.35-.2.35-.14.33-.1.3-.06.26-.04.21-.02.13-.01h5.84l.69-.05.59-.14.5-.21.41-.28.33-.32.27-.35.2-.36.15-.36.1-.35.07-.32.04-.28.02-.21V6.07h2.09l.14.01zm-6.47 14.25l-.23.33-.08.41.08.41.23.33.33.23.41.08.41-.08.33-.23.23-.33.08-.41-.08-.41-.23-.33-.33-.23-.41-.08-.41.08z" fill="currentColor" /></svg>19:["$","$L39",null,{"className":"shiki shiki-themes github-light github-dark","style":{"--shiki-light":"#24292e","--shiki-dark":"#e1e4e8","--shiki-light-bg":"#fff","--shiki-dark-bg":"#24292e"},"tabIndex":"0","icon":"$3b","children":["$","$L3a",null,{"children":["$","code",null,{"children":[["$","span",null,{"className":"line"}],"\n",["$","span",null,{"className":"line","children":[["$","span",null,{"style":{"--shiki-light":"#D73A49","--shiki-dark":"#F97583"},"children":" import"}],["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":" torch"}]]}],"\n",["$","span",null,{"className":"line","children":[["$","span",null,{"style":{"--shiki-light":"#D73A49","--shiki-dark":"#F97583"},"children":" import"}],["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":" torch.nn "}],["$","span",null,{"style":{"--shiki-light":"#D73A49","--shiki-dark":"#F97583"},"children":"as"}],["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":" nn"}]]}],"\n",["$","span",null,{"className":"line","children":[["$","span",null,{"style":{"--shiki-light":"#D73A49","--shiki-dark":"#F97583"},"children":" from"}],["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":" typing "}],["$","span",null,{"style":{"--shiki-light":"#D73A49","--shiki-dark":"#F97583"},"children":"import"}],["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":" List"}]]}],"\n",["$","span",null,{"className":"line","children":["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":" "}]}],"\n",["$","span",null,{"className":"line","children":[["$","span",null,{"style":{"--shiki-light":"#D73A49","--shiki-dark":"#F97583"},"children":" def"}],["$","span",null,{"style":{"--shiki-light":"#6F42C1","--shiki-dark":"#B392F0"},"children":" drop_upcycle"}],["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":"("}]]}],"\n",["$","span",null,{"className":"line","children":["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":" dense_ffn: nn.Linear,"}]}],"\n",["$","span",null,{"className":"line","children":[["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":" n_experts: "}],["$","span",null,{"style":{"--shiki-light":"#005CC5","--shiki-dark":"#79B8FF"},"children":"int"}],["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":","}]]}],"\n",["$","span",null,{"className":"line","children":[["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":" drop_rate: "}],"$L3c","$L3d","$L3e","$L3f"]}],"\n","$L40","\n","$L41","\n","$L42","\n","$L43","\n","$L44","\n","$L45","\n","$L46","\n","$L47","\n","$L48","\n","$L49","\n","$L4a","\n","$L4b","\n","$L4c","\n","$L4d","\n","$L4e","\n","$L4f","\n","$L50","\n","$L51","\n","$L52","\n","$L53","\n","$L54","\n","$L55","\n","$L56","\n","$L57","\n","$L58","\n","$L59","\n","$L5a","\n","$L5b","\n","$L5c","\n","$L5d","\n","$L5e","\n","$L5f","\n","$L60","\n","$L61","\n","$L62","\n","$L63","\n","$L64","\n","$L65","\n","$L66","\n","$L67","\n","$L68","\n","$L69","\n","$L6a","\n","$L6b","\n","$L6c","\n","$L6d","\n","$L6e","\n","$L6f","\n","$L70","\n","$L71"]}]}]}]
1a:["$","p",null,{"children":["The ",["$","code",null,{"children":"drop_rate"}]," hyperparameter is the key dial. Too low (< 5%) and experts remain too correlated. Too high (> 30%) and you lose the functional initialization benefit — the expert essentially starts from random weights. The sweet spot we found empirically: ",["$","strong",null,{"children":"8-12% for general language models, 15% for code-specialized models"}]," (where more aggressive diversity is needed to separate syntax vs. semantics experts)."]}]
1b:["$","h2",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"training-dynamics-implicit-specialization-signal","children":[["$","a",null,{"data-card":"","href":"#training-dynamics-implicit-specialization-signal","className":"peer","children":"Training Dynamics: Implicit Specialization Signal"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
1c:["$","p",null,{"children":"Why do dropped experts specialize rather than just learn to patch their own holes? The mechanism is elegant."}]
1d:["$","p",null,{"children":"After the first few training steps, the “intact” experts (those with more of the original dense weights) perform better on common tokens — they have a head start. The router, which is optimizing for overall performance, learns to send common tokens to the better-performing intact experts. The dropped experts receive a different token distribution: harder tokens, rarer constructs, edge cases that the intact experts handle poorly."}]
1e:["$","p",null,{"children":["This is the ",["$","strong",null,{"children":"implicit specialization signal"}]," : experts do not specialize by design, they specialize by default. Each expert optimizes for the token distribution it actually receives, and that distribution is different for each expert because their relative competencies differ. By 50K training steps, the expert specialization is measurable:"]}]
1f:["$","ul",null,{"children":["\n",["$","li",null,{"children":"Intact experts (low drop rate) converge toward high-frequency, syntactic functions"}],"\n",["$","li",null,{"children":"Heavily-dropped experts develop novel representations for rare or complex tokens"}],"\n"]}]
20:["$","p",null,{"children":"The dense-to-MoE transition effectively turns a capability gap (some experts start worse) into a specialization signal (those experts become domain-specific)."}]
21:["$","h2",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"results-at-scale","children":[["$","a",null,{"data-card":"","href":"#results-at-scale","className":"peer","children":"Results at Scale"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
22:["$","p",null,{"children":"On the primary benchmark suite (comparing Drop-Upcycled MoE vs. MoE trained from scratch):"}]
23:["$","ul",null,{"children":["\n",["$","li",null,{"children":[["$","strong",null,{"children":"5.9B active parameters"}]," : Drop-Upcycled MoE achieves 13B-equivalent quality at 1/4 the training FLOPs"]}],"\n",["$","li",null,{"children":[["$","strong",null,{"children":"MMLU"}]," : Drop-Upcycling reaches 75.4 vs. from-scratch MoE at 74.8 (at same FLOP budget)"]}],"\n",["$","li",null,{"children":[["$","strong",null,{"children":"HumanEval"}]," : 68.2 vs. 65.1 — Drop-Upcycling is better here because code has cleaner specialization axes"]}],"\n",["$","li",null,{"children":[["$","strong",null,{"children":"Training efficiency"}]," : 4x speedup to target quality vs. from-scratch MoE"]}],"\n"]}]
24:["$","p",null,{"children":"The 1/4 FLOP claim requires context: the dense checkpoint training cost is amortized. If you already have the dense model (which you do, because you trained it first), the incremental cost to get an MoE is roughly 1/4 of a from-scratch MoE run. The total cost (dense + MoE) is higher than from-scratch, but for organizations that already have dense checkpoints — which is everyone — the marginal cost argument is what matters."}]
25:["$","h2",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"zen-mode-three-scales-of-application","children":[["$","a",null,{"data-card":"","href":"#zen-mode-three-scales-of-application","className":"peer","children":"Zen MoDE: Three Scales of Application"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
26:["$","p",null,{"children":"Zen MoDE (Mixture of Distilled Experts) applies Drop-Upcycling at three scales:"}]
27:["$","p",null,{"children":[["$","strong",null,{"children":"zen4-mini (4B total, 4B active)"}]," — Dense. No upcycling needed at this scale; the routing overhead would dominate the compute savings. Zen4-mini uses a dense Qwen3 base."]}]
28:["$","p",null,{"children":[["$","strong",null,{"children":"zen4-max (30B total, 3B active)"}]," — 16 experts, 2 active per token. Drop-Upcycled from an 8B dense checkpoint. Drop rate: 8%. Router: learned top-2 routing with load balancing. The transition from 8B dense to 30B MoE takes 200M training tokens, roughly 3 days on 32×H100."]}]
29:["$","p",null,{"children":[["$","strong",null,{"children":"zen4-ultra (1T total, 32B active)"}]," — 384 experts, 8 active per token. This is our frontier model based on the Kimi K2.5 architecture. The upcycling here was done by the upstream team; we train behavioral adapters on top using GT-QLoRA (see the companion post on that technique)."]}]
2a:["$","h2",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"what-do-the-experts-actually-learn","children":[["$","a",null,{"data-card":"","href":"#what-do-the-experts-actually-learn","className":"peer","children":"What Do the Experts Actually Learn?"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
2b:["$","p",null,{"children":"We ran expert attribution analysis on zen4-max after 500M post-upcycling training tokens. The methodology: for each expert, collect the 10K tokens that activate it most strongly and analyze the distribution."}]
2c:["$","p",null,{"children":"The results cluster into recognizable domains:"}]
2d:["$","div",null,{"className":"relative overflow-auto prose-no-margin my-6","children":["$","table",null,{"children":[["$","thead",null,{"children":["$","tr",null,{"children":[["$","th",null,{"children":"Expert Group"}],["$","th",null,{"children":"Token Characteristics"}]]}]}],["$","tbody",null,{"children":[["$","tr",null,{"children":[["$","td",null,{"children":"Experts 0-3"}],["$","td",null,{"children":"High-frequency English function words, punctuation"}]]}],["$","tr",null,{"children":[["$","td",null,{"children":"Experts 4-6"}],["$","td",null,{"children":"Code tokens: brackets, operators, keywords"}]]}],["$","tr",null,{"children":[["$","td",null,{"children":"Experts 7-9"}],["$","td",null,{"children":"Mathematical notation, numerals, equations"}]]}],["$","tr",null,{"children":[["$","td",null,{"children":"Experts 10-12"}],["$","td",null,{"children":"Multilingual tokens (Chinese, Arabic, Cyrillic)"}]]}],["$","tr",null,{"children":[["$","td",null,{"children":"Experts 13-15"}],["$","td",null,{"children":"Rare English words, technical terminology"}]]}]]}]]}]}]
2e:["$","p",null,{"children":"This is not designed specialization — it emerged from the implicit signal described above. The router discovered that routing code tokens to experts 4-6 produces better outputs than routing them to experts 0-3. No explicit supervision was provided."}]
2f:["$","h2",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"q-galore-for-moe-training-efficiency","children":[["$","a",null,{"data-card":"","href":"#q-galore-for-moe-training-efficiency","className":"peer","children":"Q-GaLore for MoE Training Efficiency"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
30:["$","p",null,{"children":"Training a 30B MoE requires careful memory management. We use Q-GaLore (Quantized Gradient Low-Rank Projection) for the upcycling phase:"}]
31:["$","ul",null,{"children":["\n",["$","li",null,{"children":"Gradient projection: instead of storing full gradients for all parameters, project them into a low-rank subspace (rank 128 for most layers)"}],"\n",["$","li",null,{"children":"Quantize the projected gradients to INT8 before accumulation"}],"\n",["$","li",null,{"children":"Result: 50% memory reduction vs. standard LoRA, with +5.19 MMLU points vs. QLoRA on equivalent compute"}],"\n"]}]
32:["$","p",null,{"children":"The memory savings matter because upcycling requires loading both the dense checkpoint (for initialization) and the growing MoE checkpoint (for training) simultaneously. Q-GaLore makes this tractable on 8×A100 80GB configurations where it would otherwise OOM."}]
33:["$","h2",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"research-frontier-progressive-router-pruning","children":[["$","a",null,{"data-card":"","href":"#research-frontier-progressive-router-pruning","className":"peer","children":"Research Frontier: Progressive Router Pruning"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
34:["$","p",null,{"children":"The current Drop-Upcycling approach creates experts of fixed capacity. An open question we are actively investigating: can you progressively prune the router to identify which experts are actually being used, then collapse unused capacity back into the shared parameters?"}]
35:["$","p",null,{"children":"Early results suggest that after 500M training tokens, roughly 20-30% of experts receive less than 2% of routing probability across the evaluation corpus. These “dormant” experts can be pruned and their parameters absorbed back into the shared FFN without measurable quality degradation. This gives a dynamic MoE that starts dense, develops expert structure, and then self-compresses to its natural capacity."}]
36:["$","p",null,{"children":"The mechanism matters for continual learning: as new domains are added, dormant experts can be “awakened” and repurposed for the new domain rather than creating new experts from scratch. This connects Drop-Upcycling directly to the SuRe + OPCM continual learning stack described in our companion post."}]
37:["$","hr",null,{}]
38:["$","p",null,{"children":["$","em",null,{"children":"Zen LM is a joint initiative of Hanzo AI Inc. (Techstars ‘17) and Zoo Labs Foundation (501c3)."}]}]
3c:["$","span",null,{"style":{"--shiki-light":"#005CC5","--shiki-dark":"#79B8FF"},"children":"float"}]
3d:["$","span",null,{"style":{"--shiki-light":"#D73A49","--shiki-dark":"#F97583"},"children":" ="}]
3e:["$","span",null,{"style":{"--shiki-light":"#005CC5","--shiki-dark":"#79B8FF"},"children":" 0.1"}]
3f:["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":","}]
40:["$","span",null,{"className":"line","children":[["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":" init_std: "}],["$","span",null,{"style":{"--shiki-light":"#005CC5","--shiki-dark":"#79B8FF"},"children":"float"}],["$","span",null,{"style":{"--shiki-light":"#D73A49","--shiki-dark":"#F97583"},"children":" ="}],["$","span",null,{"style":{"--shiki-light":"#005CC5","--shiki-dark":"#79B8FF"},"children":" 0.02"}],["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":","}]]}]
41:["$","span",null,{"className":"line","children":["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":" ) -> List[nn.Linear]:"}]}]
42:["$","span",null,{"className":"line","children":["$","span",null,{"style":{"--shiki-light":"#032F62","--shiki-dark":"#9ECBFF"},"children":" \"\"\""}]}]
43:["$","span",null,{"className":"line","children":["$","span",null,{"style":{"--shiki-light":"#032F62","--shiki-dark":"#9ECBFF"},"children":" Convert a single dense FFN layer into n_experts expert layers"}]}]
44:["$","span",null,{"className":"line","children":["$","span",null,{"style":{"--shiki-light":"#032F62","--shiki-dark":"#9ECBFF"},"children":" using Drop-Upcycling initialization."}]}]
45:["$","span",null,{"className":"line","children":["$","span",null,{"style":{"--shiki-light":"#032F62","--shiki-dark":"#9ECBFF"},"children":" "}]}]
46:["$","span",null,{"className":"line","children":["$","span",null,{"style":{"--shiki-light":"#032F62","--shiki-dark":"#9ECBFF"},"children":" Args:"}]}]
47:["$","span",null,{"className":"line","children":["$","span",null,{"style":{"--shiki-light":"#032F62","--shiki-dark":"#9ECBFF"},"children":" dense_ffn: Pretrained dense FFN layer"}]}]
48:["$","span",null,{"className":"line","children":["$","span",null,{"style":{"--shiki-light":"#032F62","--shiki-dark":"#9ECBFF"},"children":" n_experts: Number of MoE experts to create"}]}]
49:["$","span",null,{"className":"line","children":["$","span",null,{"style":{"--shiki-light":"#032F62","--shiki-dark":"#9ECBFF"},"children":" drop_rate: Fraction of rows to reinitialize per expert"}]}]
4a:["$","span",null,{"className":"line","children":["$","span",null,{"style":{"--shiki-light":"#032F62","--shiki-dark":"#9ECBFF"},"children":" init_std: Standard deviation for reinitialized rows"}]}]
4b:["$","span",null,{"className":"line","children":["$","span",null,{"style":{"--shiki-light":"#032F62","--shiki-dark":"#9ECBFF"},"children":" "}]}]
4c:["$","span",null,{"className":"line","children":["$","span",null,{"style":{"--shiki-light":"#032F62","--shiki-dark":"#9ECBFF"},"children":" Returns:"}]}]
4d:["$","span",null,{"className":"line","children":["$","span",null,{"style":{"--shiki-light":"#032F62","--shiki-dark":"#9ECBFF"},"children":" List of n_experts initialized expert layers"}]}]
4e:["$","span",null,{"className":"line","children":["$","span",null,{"style":{"--shiki-light":"#032F62","--shiki-dark":"#9ECBFF"},"children":" \"\"\""}]}]
4f:["$","span",null,{"className":"line","children":[["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":" d_out, d_in "}],["$","span",null,{"style":{"--shiki-light":"#D73A49","--shiki-dark":"#F97583"},"children":"="}],["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":" dense_ffn.weight.shape"}]]}]
50:["$","span",null,{"className":"line","children":[["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":" n_drop "}],["$","span",null,{"style":{"--shiki-light":"#D73A49","--shiki-dark":"#F97583"},"children":"="}],["$","span",null,{"style":{"--shiki-light":"#005CC5","--shiki-dark":"#79B8FF"},"children":" max"}],["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":"("}],["$","span",null,{"style":{"--shiki-light":"#005CC5","--shiki-dark":"#79B8FF"},"children":"1"}],["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":", "}],["$","span",null,{"style":{"--shiki-light":"#005CC5","--shiki-dark":"#79B8FF"},"children":"int"}],["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":"(d_out "}],["$","span",null,{"style":{"--shiki-light":"#D73A49","--shiki-dark":"#F97583"},"children":"*"}],["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":" drop_rate))"}]]}]
51:["$","span",null,{"className":"line","children":[["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":" experts "}],["$","span",null,{"style":{"--shiki-light":"#D73A49","--shiki-dark":"#F97583"},"children":"="}],["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":" []"}]]}]
52:["$","span",null,{"className":"line","children":["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":" "}]}]
53:["$","span",null,{"className":"line","children":[["$","span",null,{"style":{"--shiki-light":"#D73A49","--shiki-dark":"#F97583"},"children":" for"}],["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":" i "}],["$","span",null,{"style":{"--shiki-light":"#D73A49","--shiki-dark":"#F97583"},"children":"in"}],["$","span",null,{"style":{"--shiki-light":"#005CC5","--shiki-dark":"#79B8FF"},"children":" range"}],["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":"(n_experts):"}]]}]
54:["$","span",null,{"className":"line","children":[["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":" expert "}],["$","span",null,{"style":{"--shiki-light":"#D73A49","--shiki-dark":"#F97583"},"children":"="}],["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":" nn.Linear(d_in, d_out, "}],["$","span",null,{"style":{"--shiki-light":"#E36209","--shiki-dark":"#FFAB70"},"children":"bias"}],["$","span",null,{"style":{"--shiki-light":"#D73A49","--shiki-dark":"#F97583"},"children":"="}],["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":"dense_ffn.bias "}],["$","span",null,{"style":{"--shiki-light":"#D73A49","--shiki-dark":"#F97583"},"children":"is"}],["$","span",null,{"style":{"--shiki-light":"#D73A49","--shiki-dark":"#F97583"},"children":" not"}],["$","span",null,{"style":{"--shiki-light":"#005CC5","--shiki-dark":"#79B8FF"},"children":" None"}],["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":")"}]]}]
55:["$","span",null,{"className":"line","children":[["$","span",null,{"style":{"--shiki-light":"#D73A49","--shiki-dark":"#F97583"},"children":" with"}],["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":" torch.no_grad():"}]]}]
56:["$","span",null,{"className":"line","children":["$","span",null,{"style":{"--shiki-light":"#6A737D","--shiki-dark":"#6A737D"},"children":" # Start from dense weights"}]}]
57:["$","span",null,{"className":"line","children":["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":" expert.weight.copy_(dense_ffn.weight)"}]}]
58:["$","span",null,{"className":"line","children":[["$","span",null,{"style":{"--shiki-light":"#D73A49","--shiki-dark":"#F97583"},"children":" if"}],["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":" dense_ffn.bias "}],["$","span",null,{"style":{"--shiki-light":"#D73A49","--shiki-dark":"#F97583"},"children":"is"}],["$","span",null,{"style":{"--shiki-light":"#D73A49","--shiki-dark":"#F97583"},"children":" not"}],["$","span",null,{"style":{"--shiki-light":"#005CC5","--shiki-dark":"#79B8FF"},"children":" None"}],["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":":"}]]}]
59:["$","span",null,{"className":"line","children":["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":" expert.bias.copy_(dense_ffn.bias)"}]}]
5a:["$","span",null,{"className":"line","children":["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":" "}]}]
5b:["$","span",null,{"className":"line","children":["$","span",null,{"style":{"--shiki-light":"#6A737D","--shiki-dark":"#6A737D"},"children":" # Drop a unique random subset of rows"}]}]
5c:["$","span",null,{"className":"line","children":[["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":" drop_indices "}],["$","span",null,{"style":{"--shiki-light":"#D73A49","--shiki-dark":"#F97583"},"children":"="}],["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":" torch.randperm(d_out)[:n_drop]"}]]}]
5d:["$","span",null,{"className":"line","children":[["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":" expert.weight[drop_indices] "}],["$","span",null,{"style":{"--shiki-light":"#D73A49","--shiki-dark":"#F97583"},"children":"="}],["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":" torch.randn(n_drop, d_in) "}],["$","span",null,{"style":{"--shiki-light":"#D73A49","--shiki-dark":"#F97583"},"children":"*"}],["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":" init_std"}]]}]
5e:["$","span",null,{"className":"line","children":[["$","span",null,{"style":{"--shiki-light":"#D73A49","--shiki-dark":"#F97583"},"children":" if"}],["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":" dense_ffn.bias "}],["$","span",null,{"style":{"--shiki-light":"#D73A49","--shiki-dark":"#F97583"},"children":"is"}],["$","span",null,{"style":{"--shiki-light":"#D73A49","--shiki-dark":"#F97583"},"children":" not"}],["$","span",null,{"style":{"--shiki-light":"#005CC5","--shiki-dark":"#79B8FF"},"children":" None"}],["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":":"}]]}]
5f:["$","span",null,{"className":"line","children":[["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":" expert.bias[drop_indices] "}],["$","span",null,{"style":{"--shiki-light":"#D73A49","--shiki-dark":"#F97583"},"children":"="}],["$","span",null,{"style":{"--shiki-light":"#005CC5","--shiki-dark":"#79B8FF"},"children":" 0.0"}]]}]
60:["$","span",null,{"className":"line","children":["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":" "}]}]
61:["$","span",null,{"className":"line","children":["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":" experts.append(expert)"}]}]
62:["$","span",null,{"className":"line","children":["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":" "}]}]
63:["$","span",null,{"className":"line","children":[["$","span",null,{"style":{"--shiki-light":"#D73A49","--shiki-dark":"#F97583"},"children":" return"}],["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":" experts"}]]}]
64:["$","span",null,{"className":"line","children":["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":" "}]}]
65:["$","span",null,{"className":"line","children":["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":" "}]}]
66:["$","span",null,{"className":"line","children":[["$","span",null,{"style":{"--shiki-light":"#D73A49","--shiki-dark":"#F97583"},"children":" def"}],["$","span",null,{"style":{"--shiki-light":"#6F42C1","--shiki-dark":"#B392F0"},"children":" upcycle_transformer_block"}],["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":"("}]]}]
67:["$","span",null,{"className":"line","children":["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":" dense_block,"}]}]
68:["$","span",null,{"className":"line","children":[["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":" n_experts: "}],["$","span",null,{"style":{"--shiki-light":"#005CC5","--shiki-dark":"#79B8FF"},"children":"int"}],["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":","}]]}]
69:["$","span",null,{"className":"line","children":[["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":" drop_rate: "}],["$","span",null,{"style":{"--shiki-light":"#005CC5","--shiki-dark":"#79B8FF"},"children":"float"}],["$","span",null,{"style":{"--shiki-light":"#D73A49","--shiki-dark":"#F97583"},"children":" ="}],["$","span",null,{"style":{"--shiki-light":"#005CC5","--shiki-dark":"#79B8FF"},"children":" 0.1"}],["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":","}]]}]
6a:["$","span",null,{"className":"line","children":[["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":" ) -> "}],["$","span",null,{"style":{"--shiki-light":"#005CC5","--shiki-dark":"#79B8FF"},"children":"dict"}],["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":":"}]]}]
6b:["$","span",null,{"className":"line","children":["$","span",null,{"style":{"--shiki-light":"#032F62","--shiki-dark":"#9ECBFF"},"children":" \"\"\"Upcycle a full transformer FFN block into MoE experts.\"\"\""}]}]
6c:["$","span",null,{"className":"line","children":[["$","span",null,{"style":{"--shiki-light":"#D73A49","--shiki-dark":"#F97583"},"children":" return"}],["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":" {"}]]}]
6d:["$","span",null,{"className":"line","children":[["$","span",null,{"style":{"--shiki-light":"#032F62","--shiki-dark":"#9ECBFF"},"children":" 'gate_proj'"}],["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":": drop_upcycle(dense_block.gate_proj, n_experts, drop_rate),"}]]}]
6e:["$","span",null,{"className":"line","children":[["$","span",null,{"style":{"--shiki-light":"#032F62","--shiki-dark":"#9ECBFF"},"children":" 'up_proj'"}],["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":": drop_upcycle(dense_block.up_proj, n_experts, drop_rate),"}]]}]
6f:["$","span",null,{"className":"line","children":[["$","span",null,{"style":{"--shiki-light":"#032F62","--shiki-dark":"#9ECBFF"},"children":" 'down_proj'"}],["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":": drop_upcycle(dense_block.down_proj, n_experts, drop_rate),"}]]}]
70:["$","span",null,{"className":"line","children":["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":" }"}]}]
71:["$","span",null,{"className":"line","children":["$","span",null,{"style":{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},"children":" "}]}]
b:[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","meta","2",{"name":"theme-color","media":"(prefers-color-scheme: dark)","content":"#0A0A0A"}],["$","meta","3",{"name":"theme-color","media":"(prefers-color-scheme: light)","content":"#fff"}]]
9:null
d:[["$","title","0",{"children":"Drop-Upcycling and the Birth of Zen MoDE Architecture — Zen LM Blog | Zen LM"}],["$","meta","1",{"name":"description","content":"How Drop-Upcycling (arXiv:2502.19261) transforms dense checkpoints into MoE models at 1/4 training cost, and how it shapes Zen MoDE — our Mixture of Distilled Experts architecture."}],["$","meta","2",{"property":"og:title","content":"Zen LM - Open Foundation Models"}],["$","meta","3",{"property":"og:description","content":"Zen AI model family by Hanzo AI. 14 frontier models from 4B to 1T+ parameters for code, reasoning, vision, and multimodal tasks."}],["$","meta","4",{"property":"og:url","content":"https://zenlm.org"}],["$","meta","5",{"property":"og:site_name","content":"Zen LM"}],["$","meta","6",{"property":"og:type","content":"website"}],["$","meta","7",{"name":"twitter:card","content":"summary_large_image"}],["$","meta","8",{"name":"twitter:site","content":"@zenlmorg"}],["$","meta","9",{"name":"twitter:creator","content":"@zenlmorg"}],["$","meta","10",{"name":"twitter:title","content":"Zen LM - Open Foundation Models"}],["$","meta","11",{"name":"twitter:description","content":"Zen AI model family by Hanzo AI. 14 frontier models from 4B to 1T+ parameters for code, reasoning, vision, and multimodal tasks."}]]