Files

40 lines
21 KiB
Plaintext

1:"$Sreact.fragment"
2:I[106,["/_next/static/chunks/59d0ad1b64f8544e.js"],"RootProvider"]
3:I[53113,["/_next/static/chunks/4d80e004cf4896dd.js","/_next/static/chunks/350ee4303b732916.js"],"default"]
4:I[73211,["/_next/static/chunks/4d80e004cf4896dd.js","/_next/static/chunks/350ee4303b732916.js"],"default"]
5:I[10086,["/_next/static/chunks/59d0ad1b64f8544e.js","/_next/static/chunks/8de849ca74fc071f.js","/_next/static/chunks/e62b91212ee7f8ff.js","/_next/static/chunks/f19fe44237e54646.js","/_next/static/chunks/cb0a883bafeb6805.js"],""]
7:I[89923,["/_next/static/chunks/4d80e004cf4896dd.js","/_next/static/chunks/350ee4303b732916.js"],"OutletBoundary"]
8:"$Sreact.suspense"
a:I[89923,["/_next/static/chunks/4d80e004cf4896dd.js","/_next/static/chunks/350ee4303b732916.js"],"ViewportBoundary"]
c:I[89923,["/_next/static/chunks/4d80e004cf4896dd.js","/_next/static/chunks/350ee4303b732916.js"],"MetadataBoundary"]
e:I[6998,["/_next/static/chunks/4d80e004cf4896dd.js","/_next/static/chunks/350ee4303b732916.js"],"default"]
:HL["/_next/static/chunks/f2332aac77592f9d.css","style"]
0:{"P":null,"b":"i-dnJM_MIpJSOCQWNJVMq","c":["","blog","global-load-balance"],"q":"","i":false,"f":[[["",{"children":["blog",{"children":[["slug","global-load-balance","d"],{"children":["__PAGE__",{}]}]}]},"$undefined","$undefined",true],[["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/_next/static/chunks/f2332aac77592f9d.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","script","script-0",{"src":"/_next/static/chunks/59d0ad1b64f8544e.js","async":true,"nonce":"$undefined"}]],["$","html",null,{"lang":"en","className":"dark","suppressHydrationWarning":true,"children":["$","body",null,{"className":"antialiased","children":["$","$L2",null,{"children":["$","$L3",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L4",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","main",null,{"className":"flex min-h-screen flex-col items-center justify-center px-4 text-center","children":[["$","div",null,{"className":"mb-8 opacity-20","children":["$","svg",null,{"width":"120","height":"120","viewBox":"0 0 120 120","fill":"none","aria-hidden":"true","children":["$","circle",null,{"cx":"60","cy":"60","r":"50","stroke":"currentColor","strokeWidth":"3","strokeLinecap":"round","strokeDasharray":"280 40"}]}]}],["$","p",null,{"className":"text-xs font-mono uppercase tracking-[0.3em] text-fd-muted-foreground mb-4","children":"404"}],["$","h1",null,{"className":"text-3xl font-semibold mb-3","children":"Page not found"}],["$","p",null,{"className":"text-fd-muted-foreground max-w-sm mb-10","children":"This page doesn't exist, or it may have moved. Try the documentation or head home."}],["$","div",null,{"className":"flex flex-wrap gap-3 justify-center","children":[["$","$L5",null,{"href":"/","className":"inline-flex items-center gap-2 rounded-lg bg-fd-primary text-fd-primary-foreground px-4 py-2 text-sm font-medium hover:opacity-90 transition","children":"Go home"}],["$","$L5",null,{"href":"/docs","className":"inline-flex items-center gap-2 rounded-lg border border-fd-border bg-fd-background px-4 py-2 text-sm font-medium hover:bg-fd-muted transition","children":"Documentation"}],["$","$L5",null,{"href":"/docs/models","className":"inline-flex items-center gap-2 rounded-lg border border-fd-border bg-fd-background px-4 py-2 text-sm font-medium hover:bg-fd-muted transition","children":"Browse models"}]]}],["$","p",null,{"className":"mt-16 text-xs font-mono text-fd-muted-foreground opacity-50","children":"zenlm.org"}]]}],[]],"forbidden":"$undefined","unauthorized":"$undefined"}]}]}]}]]}],{"children":[["$","$1","c",{"children":[null,["$","$L3",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L4",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","forbidden":"$undefined","unauthorized":"$undefined"}]]}],{"children":[["$","$1","c",{"children":[null,["$","$L3",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L4",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","forbidden":"$undefined","unauthorized":"$undefined"}]]}],{"children":[["$","$1","c",{"children":["$L6",[["$","script","script-0",{"src":"/_next/static/chunks/8de849ca74fc071f.js","async":true,"nonce":"$undefined"}],["$","script","script-1",{"src":"/_next/static/chunks/e62b91212ee7f8ff.js","async":true,"nonce":"$undefined"}],["$","script","script-2",{"src":"/_next/static/chunks/f19fe44237e54646.js","async":true,"nonce":"$undefined"}],["$","script","script-3",{"src":"/_next/static/chunks/cb0a883bafeb6805.js","async":true,"nonce":"$undefined"}]],["$","$L7",null,{"children":["$","$8",null,{"name":"Next.MetadataOutlet","children":"$@9"}]}]]}],{},null,false,false]},null,false,false]},null,false,false]},null,false,false],["$","$1","h",{"children":[null,["$","$La",null,{"children":"$Lb"}],["$","div",null,{"hidden":true,"children":["$","$Lc",null,{"children":["$","$8",null,{"name":"Next.Metadata","children":"$Ld"}]}]}],null]}],false]],"m":"$undefined","G":["$e",[]],"S":true}
f:I[48068,["/_next/static/chunks/59d0ad1b64f8544e.js","/_next/static/chunks/8de849ca74fc071f.js","/_next/static/chunks/e62b91212ee7f8ff.js","/_next/static/chunks/f19fe44237e54646.js","/_next/static/chunks/cb0a883bafeb6805.js"],"default"]
6:["$","main",null,{"className":"mx-auto w-full max-w-2xl px-4 py-16","children":[["$","$L5",null,{"href":"/blog","className":"inline-flex items-center gap-1.5 text-sm text-fd-muted-foreground hover:text-fd-foreground mb-10 transition-colors","children":"← Back to Blog"}],["$","div",null,{"className":"mb-8","children":[["$","time",null,{"className":"text-xs font-mono text-fd-muted-foreground uppercase tracking-wider","children":"January 20, 2025"}],["$","h1",null,{"className":"text-3xl font-bold mt-2 mb-3","children":"Global-batch load balance almost free lunch to improve your MoE LLM training"}],["$","p",null,{"className":"text-fd-muted-foreground text-lg mb-4","children":"GITHUB HUGGING FACE MODELSCOPE DISCORD"}],["$","div",null,{"className":"flex items-center gap-3 pt-4 border-t border-fd-border","children":[["$","span",null,{"className":"text-sm text-fd-muted-foreground","children":["By ","Zen LM Team"]}],["$","div",null,{"className":"flex gap-1.5 ml-auto","children":[]}]]}]]}],["$","div",null,{"className":"prose dark:prose-invert max-w-none","children":[["$","p",null,{"children":[["$","$Lf",null,{"href":"https://github.com/QwenLM/zen-Math","children":"GITHUB"}]," ",["$","$Lf",null,{"href":"https://huggingface.co/collections/Qwen/qwen25-math-66eaa240a1b7d5ee65f1da3e","children":"HUGGING FACE"}]," ",["$","$Lf",null,{"href":"https://modelscope.cn/organization/qwen","children":"MODELSCOPE"}]," ",["$","$Lf",null,{"href":"https://discord.gg/yPEP2vHTu4","children":"DISCORD"}]]}],"\n",["$","h2",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"background","children":[["$","a",null,{"data-card":"","href":"#background","className":"peer","children":"Background"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}],"\n",["$","p",null,{"children":"The Mixture-of-Experts (MoEs) architecture has become a popular model-parameter-scale-up technique. Typically, one MoE layer consists of a router (often parameterized as one single Linear layer) and a group of experts (for transformer-based models, each expert is one feedforward layer). Given an input, only a subset of experts will be activated, and then their outputs will be aggregated based on the scores the router assigned. Specifically,"}],"\n",["$","p",null,{"children":["$","code",null,{"children":"\\mathbb\\{y\\}=\\sum_\\{i\\in N_E,g_i\\in\\operatorname\\{topK\\}\\}g_i(\\mathbb\\{x\\})E_i(\\mathbb\\{x\\})"}]}],"\n",["$","h3",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"load-balancing-loss","children":[["$","a",null,{"data-card":"","href":"#load-balancing-loss","className":"peer","children":"Load Balancing loss"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}],"\n",["$","p",null,{"children":"Load balancing loss is an essential regularization technique in training MoE-based networks, and high-level intuition encourages the balanced activation of all experts. It can be calculated as:"}],"\n",["$","p",null,{"children":["$","code",null,{"children":"L_\\{\\text\\{balance\\}\\}=N_E \\sum_\\{i=1\\}^\\{N_E\\} f_ip_i"}]}],"\n","$L10","\n","$L11","\n","$L12","\n","$L13","\n","$L14","\n","$L15","\n","$L16","\n","$L17","\n","$L18","\n","$L19","\n","$L1a","\n","$L1b","\n","$L1c","\n","$L1d","\n","$L1e","\n","$L1f","\n","$L20","\n","$L21","\n","$L22","\n","$L23"]}]]}]
24:I[51504,["/_next/static/chunks/59d0ad1b64f8544e.js","/_next/static/chunks/8de849ca74fc071f.js","/_next/static/chunks/e62b91212ee7f8ff.js","/_next/static/chunks/f19fe44237e54646.js","/_next/static/chunks/cb0a883bafeb6805.js"],"CodeBlock"]
25:I[51504,["/_next/static/chunks/59d0ad1b64f8544e.js","/_next/static/chunks/8de849ca74fc071f.js","/_next/static/chunks/e62b91212ee7f8ff.js","/_next/static/chunks/f19fe44237e54646.js","/_next/static/chunks/cb0a883bafeb6805.js"],"Pre"]
10:["$","p",null,{"children":"where $f_i$ is the activation frequency of the expert $E_i$, and the $p_i$ is the the average gating score that the expert $E_i$ is assigned."}]
11:["$","p",null,{"children":["However, most existing MoE training frameworks (e.g., Megatron-core), implement ",["$","strong",null,{"children":"micro-batch"}]," level balance, which means the $L_{\\text{balance}}$ is calculated within every micro-batch and is then averaged on the global batch level."]}]
12:["$","p",null,{"children":"Our key point is that this implementation could be problematic if one micro-batch does not contain diverse data. For instance, imagine one micro-batch only contains some code data; the aforementioned load-balancing loss still pushes the router to distribute these code tokens to all experts uniformly, potentially hurting the model performance and preventing expert specialization."}]
13:["$","p",null,{"children":"This situation is even more common in training MoE-based LLMs: the data in one micro-batch is often from the same domain. This partially explains why most existing open-source MoE-based LLMs do not achieve notable expert specialization."}]
14:["$","p",null,{"children":"This drawback motivates us to extend the current method to the global-batch level balance."}]
15:["$","h2",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"from-micro-batch-balance-to-global-batch-balance","children":[["$","a",null,{"data-card":"","href":"#from-micro-batch-balance-to-global-batch-balance","className":"peer","children":"From micro-batch balance to global-batch balance"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
16:["$","p",null,{"children":"One easy way to calculate global-batch balance loss is to 1) Synchronize expert selection frequency $f_i$ across all parallel groups; 2) Calculate the load-balancing loss in each parallel group (e.g., one GPU); 3) Aggregate the loss across all micro-batches."}]
17:["$","p",null,{"children":"Specifically,"}]
18:["$","p",null,{"children":["$","code",null,{"children":"L_\\{\\text\\{global\\}\\}=N_E\\sum_\\{i=1\\}^\\{N_E\\}f_i^\\{\\text\\{global\\}\\}p_i^\\{\\text\\{global\\}\\}=N_E\\sum_\\{i=1\\}^\\{n_E\\}f_i^\\{\\text\\{global\\}\\}\\cdot(\\frac\\{1\\}\\{N_p\\}\\sum_\\{j=1\\}^\\{N_p\\}p_j)= \\frac\\{1\\}\\{N_P\\} \\sum_\\{j=1\\}^\\{N_p\\}(N_E \\sum_\\{i=1\\}^\\{N_E\\} \\bar\\{f_i\\} \\cdot P^j_i)"}]}]
19:["$","p",null,{"children":"Note that the expert selection frequency is just one expert-num-dimentional vector! It is almost free to synchronize them across micro-batches."}]
1a:["$","h2",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"results-more-performant-and-interpretable-moe","children":[["$","a",null,{"data-card":"","href":"#results-more-performant-and-interpretable-moe","className":"peer","children":"Results: More Performant and Interpretable MoE"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
1b:["$","p",null,{"children":"We experiment with three MoE configs (3.4B with 0.6B activated, 15B with 2.54B activated, and 43B with 6.6B activated) and two data configs (120B tokens and 400B tokens). The results are shown in the following figure and chart."}]
1c:["$","p",null,{"children":"In short, compared to the micro-batch-level loss, the global batch one achieves better performance in all settings (model, data, and tasks). More importantly, the MoE model achieves significant domain specialization with the global batch balance. In Figure (b) left, almost all experts are uniformly activated regardless of the domain. But in Figure (b) right, some experts are frequently activated by a specific domain, demonstrating their specialization."}]
1d:["$","p",null,{"children":"We further compare the model performance with balance batch size on a 3.4B with 0.6B activated model. The ptr-training PPL decreases rapidly from balance BSZ 2 to 128, and gradually saturates after 128. In the current mainstream MoE framework, even with cross expert-parallel groups communication, the balance BSZ is generally between 8 and 16 for larger models, which further reflects the significance of our method."}]
1e:["$","p",null,{"children":"Using global batch balance may result in a degradation in micro-batch balance, potentially affecting the computational efficiency of MoE. We further experimented with the effect of adding micro-batch balance loss on top of the global-batch balance loss (with constant weights of 0.01 of the global batch loss). It can be seen that adding local equalization improves the speed of the model (from 1.64 to 1.59 seconds per update step) while the effectiveness of the model is almost unaffected."}]
1f:["$","h2",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"conclusion","children":[["$","a",null,{"data-card":"","href":"#conclusion","className":"peer","children":"Conclusion"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
20:["$","p",null,{"children":"In summary, we investigate the challenges associated with the LBL in training MoEs models. By introducing global-batch balance loss, we achieve improved performance and foster expert specialization within the MoE model. We believe this advancement addresses an essential limitation in existing MoE training, offering a novel perspective for MoEs model optimization. Though mainly experimenting with language-based tasks, we hope our work could pave the way for training more substantial and specialised MoE models in various domains."}]
21:["$","h1",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"citation","children":[["$","a",null,{"data-card":"","href":"#citation","className":"peer","children":"Citation"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
22:["$","p",null,{"children":"If you find our work helpful, feel free to give us a citation."}]
23:["$","$L24",null,{"className":"shiki shiki-themes github-light github-dark","style":{"--shiki-light":"#24292e","--shiki-dark":"#e1e4e8","--shiki-light-bg":"#fff","--shiki-dark-bg":"#24292e"},"tabIndex":"0","icon":"<svg viewBox=\"0 0 24 24\"><path d=\"M 6,1 C 4.354992,1 3,2.354992 3,4 v 16 c 0,1.645008 1.354992,3 3,3 h 12 c 1.645008,0 3,-1.354992 3,-3 V 8 7 A 1.0001,1.0001 0 0 0 20.707031,6.2929687 l -5,-5 A 1.0001,1.0001 0 0 0 15,1 h -1 z m 0,2 h 7 v 3 c 0,1.645008 1.354992,3 3,3 h 3 v 11 c 0,0.564129 -0.435871,1 -1,1 H 6 C 5.4358712,21 5,20.564129 5,20 V 4 C 5,3.4358712 5.4358712,3 6,3 Z M 15,3.4140625 18.585937,7 H 16 C 15.435871,7 15,6.5641288 15,6 Z\" fill=\"currentColor\" /></svg>","children":["$","$L25",null,{"children":["$","code",null,{"children":[["$","span",null,{"className":"line","children":["$","span",null,{}]}],"\n",["$","span",null,{"className":"line","children":["$","span",null,{"children":" @article{qiu2025demonsdetailimplementingload,"}]}],"\n",["$","span",null,{"className":"line","children":["$","span",null,{"children":" title={Demons in the Detail: On Implementing Load Balancing Loss for Training Specialized Mixture-of-Expert Models}, "}]}],"\n",["$","span",null,{"className":"line","children":["$","span",null,{"children":" author={"}]}],"\n",["$","span",null,{"className":"line","children":["$","span",null,{"children":" Zihan Qiu and Zeyu Huang and Bo Zheng and Kaiyue Wen and Zekun Wang and Rui Men and Ivan Titov and Dayiheng Liu and Jingren Zhou and Junyang Lin"}]}],"\n",["$","span",null,{"className":"line","children":["$","span",null,{"children":" },"}]}],"\n",["$","span",null,{"className":"line","children":["$","span",null,{"children":" journal={arXiv preprint arXiv:2501.11873},"}]}],"\n",["$","span",null,{"className":"line","children":["$","span",null,{"children":" year={2025}"}]}],"\n",["$","span",null,{"className":"line","children":["$","span",null,{"children":" }"}]}],"\n",["$","span",null,{"className":"line","children":["$","span",null,{"children":" "}]}]]}]}]}]
b:[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","meta","2",{"name":"theme-color","media":"(prefers-color-scheme: dark)","content":"#0A0A0A"}],["$","meta","3",{"name":"theme-color","media":"(prefers-color-scheme: light)","content":"#fff"}]]
9:null
d:[["$","title","0",{"children":"Global-batch load balance almost free lunch to improve your MoE LLM training — Zen LM Blog | Zen LM"}],["$","meta","1",{"name":"description","content":"GITHUB HUGGING FACE MODELSCOPE DISCORD"}],["$","meta","2",{"property":"og:title","content":"Zen LM - Open Foundation Models"}],["$","meta","3",{"property":"og:description","content":"Zen AI model family by Hanzo AI. 14 frontier models from 4B to 1T+ parameters for code, reasoning, vision, and multimodal tasks."}],["$","meta","4",{"property":"og:url","content":"https://zenlm.org"}],["$","meta","5",{"property":"og:site_name","content":"Zen LM"}],["$","meta","6",{"property":"og:type","content":"website"}],["$","meta","7",{"name":"twitter:card","content":"summary_large_image"}],["$","meta","8",{"name":"twitter:site","content":"@zenlmorg"}],["$","meta","9",{"name":"twitter:creator","content":"@zenlmorg"}],["$","meta","10",{"name":"twitter:title","content":"Zen LM - Open Foundation Models"}],["$","meta","11",{"name":"twitter:description","content":"Zen AI model family by Hanzo AI. 14 frontier models from 4B to 1T+ parameters for code, reasoning, vision, and multimodal tasks."}]]