Files
zen-blog/blog/qwen2.5-omni.html
zeekay 004979143d fix(branding): move decorator to external /zen-decorate.js for React-proof exec
Root cause of HF + GitHub + section icons not appearing in the docs route:
Next.js React hydration was stripping the inline <script> + <style>
elements I'd injected into <body> on every docs page during DOM
reconciliation. The script never registered its setInterval.

Move the decorator to /zen-decorate.js loaded via <script defer src>
in <head>. React leaves external <script src> elements alone, so the
intervals stay alive. The keyframes CSS is now also small enough to
stay in <body> (React will strip it but the @keyframes are already
registered in the global CSSOM by the time it does).
2026-05-19 20:08:58 -07:00

81 lines
32 KiB
HTML
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html><!--i_dnJM_MIpJSOCQWNJVMq--><html lang="en" class="dark"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="stylesheet" href="/_next/static/chunks/f2332aac77592f9d.css" data-precedence="next"/><link rel="preload" as="script" fetchPriority="low" href="/_next/static/chunks/e83606e8fa9cc796.js"/><script src="/_next/static/chunks/36d6595f0156cd7e.js" async=""></script><script src="/_next/static/chunks/040e9cea20a8d9c7.js" async=""></script><script src="/_next/static/chunks/d4dffb5a0973d49c.js" async=""></script><script src="/_next/static/chunks/turbopack-967f27a9fd33556d.js" async=""></script><script src="/_next/static/chunks/59d0ad1b64f8544e.js" async=""></script><script src="/_next/static/chunks/4d80e004cf4896dd.js" async=""></script><script src="/_next/static/chunks/350ee4303b732916.js" async=""></script><script src="/_next/static/chunks/8de849ca74fc071f.js" async=""></script><script src="/_next/static/chunks/e62b91212ee7f8ff.js" async=""></script><script src="/_next/static/chunks/f19fe44237e54646.js" async=""></script><script src="/_next/static/chunks/cb0a883bafeb6805.js" async=""></script><meta name="theme-color" media="(prefers-color-scheme: dark)" content="#0A0A0A"/><meta name="theme-color" media="(prefers-color-scheme: light)" content="#fff"/><title>zen Omni: See, Hear, Talk, Write, Do It All! — Zen LM Blog | Zen LM</title><meta name="description" content="QWEN CHAT HUGGING FACE MODELSCOPE DASHSCOPE GITHUB PAPER DEMO DISCORD"/><meta property="og:title" content="Zen LM - Open Foundation Models"/><meta property="og:description" content="Zen AI model family by Hanzo AI. 14 frontier models from 4B to 1T+ parameters for code, reasoning, vision, and multimodal tasks."/><meta property="og:url" content="https://zenlm.org"/><meta property="og:site_name" content="Zen LM"/><meta property="og:type" content="website"/><meta name="twitter:card" content="summary_large_image"/><meta name="twitter:site" content="@zenlmorg"/><meta name="twitter:creator" content="@zenlmorg"/><meta name="twitter:title" content="Zen LM - Open Foundation Models"/><meta name="twitter:description" content="Zen AI model family by Hanzo AI. 14 frontier models from 4B to 1T+ parameters for code, reasoning, vision, and multimodal tasks."/><script src="/_next/static/chunks/a6dad97d9634a72d.js" noModule=""></script><script defer src="/zen-decorate.js"></script></head><body class="antialiased"><div hidden=""><!--$--><!--/$--></div><script>((a,b,c,d,e,f,g,h)=>{let i=document.documentElement,j=["light","dark"];function k(b){var c;(Array.isArray(a)?a:[a]).forEach(a=>{let c="class"===a,d=c&&f?e.map(a=>f[a]||a):e;c?(i.classList.remove(...d),i.classList.add(f&&f[b]?f[b]:b)):i.setAttribute(a,b)}),c=b,h&&j.includes(c)&&(i.style.colorScheme=c)}if(d)k(d);else try{let a=localStorage.getItem(b)||c,d=g&&"system"===a?window.matchMedia("(prefers-color-scheme: dark)").matches?"dark":"light":a;k(d)}catch(a){}})("class","theme","system",null,["light","dark"],null,true,true)</script><!--$--><div data-closed="" role="presentation" hidden="" style="user-select:none;-webkit-user-select:none" class="fixed inset-0 z-50 backdrop-blur-xs bg-fd-overlay data-open:animate-fd-fade-in data-closed:animate-fd-fade-out"></div><div class="bg-fd-secondary/50 p-3 empty:hidden"></div><!--/$--><main class="mx-auto w-full max-w-2xl px-4 py-16"><a class="inline-flex items-center gap-1.5 text-sm text-fd-muted-foreground hover:text-fd-foreground mb-10 transition-colors" href="/blog">← Back to Blog</a><div class="mb-8"><time class="text-xs font-mono text-fd-muted-foreground uppercase tracking-wider">March 26, 2025</time><h1 class="text-3xl font-bold mt-2 mb-3">zen Omni: See, Hear, Talk, Write, Do It All!</h1><p class="text-fd-muted-foreground text-lg mb-4">QWEN CHAT HUGGING FACE MODELSCOPE DASHSCOPE GITHUB PAPER DEMO DISCORD</p><div class="flex items-center gap-3 pt-4 border-t border-fd-border"><span class="text-sm text-fd-muted-foreground">By <!-- -->Zen LM Team</span><div class="flex gap-1.5 ml-auto"></div></div></div><div class="prose dark:prose-invert max-w-none"><p><a href="https://chat.qwenlm.ai" rel="noreferrer noopener" target="_blank">QWEN CHAT</a> <a href="https://huggingface.co/Qwen/zen-Omni-7B" rel="noreferrer noopener" target="_blank">HUGGING FACE</a> <a href="https://modelscope.cn/models/Qwen/zen-Omni-7B" rel="noreferrer noopener" target="_blank">MODELSCOPE</a> <a href="https://help.aliyun.com/zh/model-studio/user-guide/qwen-omni" rel="noreferrer noopener" target="_blank">DASHSCOPE</a> <a href="https://github.com/QwenLM/zen-Omni" rel="noreferrer noopener" target="_blank">GITHUB</a> <a href="https://github.com/QwenLM/zen-Omni/blob/main/assets/zen_Omni.pdf" rel="noreferrer noopener" target="_blank">PAPER</a> <a href="https://huggingface.co/spaces/Qwen/zen-Omni-7B-Demo" rel="noreferrer noopener" target="_blank">DEMO</a> <a href="https://discord.com/invite/yPEP2vHTu4" rel="noreferrer noopener" target="_blank">DISCORD</a></p>
<p>We release <strong>zen-Omni</strong> , the new flagship end-to-end multimodal model in the Qwen series. Designed for comprehensive multimodal perception, it seamlessly processes diverse inputs including text, images, audio, and video, while delivering real-time streaming responses through both text generation and natural speech synthesis. To try the latest model, feel free to visit <a href="https://chat.qwenlm.ai" rel="noreferrer noopener" target="_blank">Qwen Chat</a> and choose zen-Omni-7B. The model is now openly available on <a href="https://huggingface.co/Qwen/zen-Omni-7B" rel="noreferrer noopener" target="_blank">Hugging Face</a>, <a href="https://modelscope.cn/models/Qwen/zen-Omni-7B" rel="noreferrer noopener" target="_blank">ModelScope</a>, <a href="https://help.aliyun.com/zh/model-studio/user-guide/qwen-omni" rel="noreferrer noopener" target="_blank">DashScope</a>,and <a href="https://github.com/QwenLM/zen-Omni" rel="noreferrer noopener" target="_blank">GitHub</a>, with technical documentation available in our <a href="https://github.com/QwenLM/zen-Omni/assets/zen_Omni.pdf" rel="noreferrer noopener" target="_blank">Paper</a>. Experience interactive capabilities through our <a href="https://huggingface.co/spaces/Qwen/zen-Omni-7B-Demo" rel="noreferrer noopener" target="_blank">Demo</a> or join our <a href="https://discord.gg/yPEP2vHTu4" rel="noreferrer noopener" target="_blank">Discord</a> for discussions.</p>
<p>Key Features:</p>
<ul>
<li>
<p><strong>Omni and Novel Architecture</strong> : We propose Thinker-Talker architecture, an end-to-end multimodal model designed to perceive diverse modalities, including text, images, audio, and video, while simultaneously generating text and natural speech responses in a streaming manner. We prpose a novel position embedding, named TMRoPE (Time-aligned Multimodal RoPE), to synchronize the timestamps of video inputs with audio.</p>
</li>
<li>
<p><strong>Real-Time Voice and Video Chat</strong> : Architecture Designed for fully real-time interactions, supporting chunked input and immediate output.</p>
</li>
<li>
<p><strong>Natural and Robust Speech Generation</strong> : Surpassing many existing streaming and non-streaming alternatives, demonstrating superior robustness and naturalness in speech generation.</p>
</li>
<li>
<p><strong>Strong Performance Across Modalities</strong> : Exhibiting exceptional performance across all modalities when benchmarked against similarly sized single-modality models. zen-Omni outperforms the similarly sized zen-Audio in audio capabilities and achieves comparable performance to zen-VL-7B.</p>
</li>
<li>
<p><strong>Excellent End-to-End Speech Instruction Following</strong> : zen-Omni shows performance in end-to-end speech instruction following that rivals its effectiveness with text inputs, evidenced by benchmarks such as MMLU and GSM8K.</p>
</li>
</ul>
<h2 class="flex scroll-m-28 flex-row items-center gap-2" id="architecture"><a data-card="" href="#architecture" class="peer">Architecture</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h2>
<p>zen-Omni employs Thinker-Talker architecture. Thinker functions like a brain, responsible for processing and understanding inputs from text, audio and video modalities, generating high-level representations and corresponding text. Talker operates like a human mouth, taking in the high-level representations and text produced by the Thinker in a streaming manner, and outputting discrete tokens of speech fluidly. Thinker is a Transformer decoder, accompanied by encoders for audio and image that facilitate information extraction. In contrast, Talker is designed as a dual-track autoregressive Transformer Decoder architecture. During both training and inference, Talker directly receives high-dimensional representations from Thinker and shares all of Thinker’s historical context information. Consequently, the entire architecture operates as a cohesive single model, enabling end-to-end training and inference.</p>
<h2 class="flex scroll-m-28 flex-row items-center gap-2" id="performance"><a data-card="" href="#performance" class="peer">Performance</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h2>
<p>We conducted a comprehensive evaluation of zen-Omni, which demonstrates strong performance across all modalities when compared to similarly sized single-modality models and closed-source models like zen-VL-7B, zen-Audio, and Gemini-1.5-pro. In tasks requiring the integration of multiple modalities, such as OmniBench, zen-Omni achieves state-of-the-art performance. Furthermore, in single-modality tasks, it excels in areas including speech recognition (Common Voice), translation (CoVoST2), audio understanding (MMAU), image reasoning (MMMU, MMStar), video understanding (MVBench), and speech generation (Seed-tts-eval and subjective naturalness).</p>
<h2 class="flex scroll-m-28 flex-row items-center gap-2" id="whats-next"><a data-card="" href="#whats-next" class="peer">What’s Next</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h2>
<p>We are eager to hear your feedback and see the innovative applications you create with zen-Omni. In the near future, our goal is to enhance our model’s ability to follow voice commands and improve audio-visual collaborative understanding. Additionally, we strive to integrate more modalities towards an omni-model!</p></div></main><!--$--><!--/$--><script src="/_next/static/chunks/e83606e8fa9cc796.js" id="_R_" async=""></script><script>(self.__next_f=self.__next_f||[]).push([0])</script><script>self.__next_f.push([1,"1:\"$Sreact.fragment\"\n2:I[106,[\"/_next/static/chunks/59d0ad1b64f8544e.js\"],\"RootProvider\"]\n3:I[53113,[\"/_next/static/chunks/4d80e004cf4896dd.js\",\"/_next/static/chunks/350ee4303b732916.js\"],\"default\"]\n4:I[73211,[\"/_next/static/chunks/4d80e004cf4896dd.js\",\"/_next/static/chunks/350ee4303b732916.js\"],\"default\"]\n5:I[10086,[\"/_next/static/chunks/59d0ad1b64f8544e.js\",\"/_next/static/chunks/8de849ca74fc071f.js\",\"/_next/static/chunks/e62b91212ee7f8ff.js\",\"/_next/static/chunks/f19fe44237e54646.js\",\"/_next/static/chunks/cb0a883bafeb6805.js\"],\"\"]\n7:I[89923,[\"/_next/static/chunks/4d80e004cf4896dd.js\",\"/_next/static/chunks/350ee4303b732916.js\"],\"OutletBoundary\"]\n8:\"$Sreact.suspense\"\na:I[89923,[\"/_next/static/chunks/4d80e004cf4896dd.js\",\"/_next/static/chunks/350ee4303b732916.js\"],\"ViewportBoundary\"]\nc:I[89923,[\"/_next/static/chunks/4d80e004cf4896dd.js\",\"/_next/static/chunks/350ee4303b732916.js\"],\"MetadataBoundary\"]\ne:I[6998,[\"/_next/static/chunks/4d80e004cf4896dd.js\",\"/_next/static/chunks/350ee4303b732916.js\"],\"default\"]\n:HL[\"/_next/static/chunks/f2332aac77592f9d.css\",\"style\"]\n"])</script><script>self.__next_f.push([1,"0:{\"P\":null,\"b\":\"i-dnJM_MIpJSOCQWNJVMq\",\"c\":[\"\",\"blog\",\"qwen2.5-omni\"],\"q\":\"\",\"i\":false,\"f\":[[[\"\",{\"children\":[\"blog\",{\"children\":[[\"slug\",\"qwen2.5-omni\",\"d\"],{\"children\":[\"__PAGE__\",{}]}]}]},\"$undefined\",\"$undefined\",true],[[\"$\",\"$1\",\"c\",{\"children\":[[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/chunks/f2332aac77592f9d.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\",\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-0\",{\"src\":\"/_next/static/chunks/59d0ad1b64f8544e.js\",\"async\":true,\"nonce\":\"$undefined\"}]],[\"$\",\"html\",null,{\"lang\":\"en\",\"className\":\"dark\",\"suppressHydrationWarning\":true,\"children\":[\"$\",\"body\",null,{\"className\":\"antialiased\",\"children\":[\"$\",\"$L2\",null,{\"children\":[\"$\",\"$L3\",null,{\"parallelRouterKey\":\"children\",\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L4\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":[[\"$\",\"main\",null,{\"className\":\"flex min-h-screen flex-col items-center justify-center px-4 text-center\",\"children\":[[\"$\",\"div\",null,{\"className\":\"mb-8 opacity-20\",\"children\":[\"$\",\"svg\",null,{\"width\":\"120\",\"height\":\"120\",\"viewBox\":\"0 0 120 120\",\"fill\":\"none\",\"aria-hidden\":\"true\",\"children\":[\"$\",\"circle\",null,{\"cx\":\"60\",\"cy\":\"60\",\"r\":\"50\",\"stroke\":\"currentColor\",\"strokeWidth\":\"3\",\"strokeLinecap\":\"round\",\"strokeDasharray\":\"280 40\"}]}]}],[\"$\",\"p\",null,{\"className\":\"text-xs font-mono uppercase tracking-[0.3em] text-fd-muted-foreground mb-4\",\"children\":\"404\"}],[\"$\",\"h1\",null,{\"className\":\"text-3xl font-semibold mb-3\",\"children\":\"Page not found\"}],[\"$\",\"p\",null,{\"className\":\"text-fd-muted-foreground max-w-sm mb-10\",\"children\":\"This page doesn't exist, or it may have moved. Try the documentation or head home.\"}],[\"$\",\"div\",null,{\"className\":\"flex flex-wrap gap-3 justify-center\",\"children\":[[\"$\",\"$L5\",null,{\"href\":\"/\",\"className\":\"inline-flex items-center gap-2 rounded-lg bg-fd-primary text-fd-primary-foreground px-4 py-2 text-sm font-medium hover:opacity-90 transition\",\"children\":\"Go home\"}],[\"$\",\"$L5\",null,{\"href\":\"/docs\",\"className\":\"inline-flex items-center gap-2 rounded-lg border border-fd-border bg-fd-background px-4 py-2 text-sm font-medium hover:bg-fd-muted transition\",\"children\":\"Documentation\"}],[\"$\",\"$L5\",null,{\"href\":\"/docs/models\",\"className\":\"inline-flex items-center gap-2 rounded-lg border border-fd-border bg-fd-background px-4 py-2 text-sm font-medium hover:bg-fd-muted transition\",\"children\":\"Browse models\"}]]}],[\"$\",\"p\",null,{\"className\":\"mt-16 text-xs font-mono text-fd-muted-foreground opacity-50\",\"children\":\"zenlm.org\"}]]}],[]],\"forbidden\":\"$undefined\",\"unauthorized\":\"$undefined\"}]}]}]}]]}],{\"children\":[[\"$\",\"$1\",\"c\",{\"children\":[null,[\"$\",\"$L3\",null,{\"parallelRouterKey\":\"children\",\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L4\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":\"$undefined\",\"forbidden\":\"$undefined\",\"unauthorized\":\"$undefined\"}]]}],{\"children\":[[\"$\",\"$1\",\"c\",{\"children\":[null,[\"$\",\"$L3\",null,{\"parallelRouterKey\":\"children\",\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L4\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":\"$undefined\",\"forbidden\":\"$undefined\",\"unauthorized\":\"$undefined\"}]]}],{\"children\":[[\"$\",\"$1\",\"c\",{\"children\":[\"$L6\",[[\"$\",\"script\",\"script-0\",{\"src\":\"/_next/static/chunks/8de849ca74fc071f.js\",\"async\":true,\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-1\",{\"src\":\"/_next/static/chunks/e62b91212ee7f8ff.js\",\"async\":true,\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-2\",{\"src\":\"/_next/static/chunks/f19fe44237e54646.js\",\"async\":true,\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-3\",{\"src\":\"/_next/static/chunks/cb0a883bafeb6805.js\",\"async\":true,\"nonce\":\"$undefined\"}]],[\"$\",\"$L7\",null,{\"children\":[\"$\",\"$8\",null,{\"name\":\"Next.MetadataOutlet\",\"children\":\"$@9\"}]}]]}],{},null,false,false]},null,false,false]},null,false,false]},null,false,false],[\"$\",\"$1\",\"h\",{\"children\":[null,[\"$\",\"$La\",null,{\"children\":\"$Lb\"}],[\"$\",\"div\",null,{\"hidden\":true,\"children\":[\"$\",\"$Lc\",null,{\"children\":[\"$\",\"$8\",null,{\"name\":\"Next.Metadata\",\"children\":\"$Ld\"}]}]}],null]}],false]],\"m\":\"$undefined\",\"G\":[\"$e\",[]],\"S\":true}\n"])</script><script>self.__next_f.push([1,"f:I[48068,[\"/_next/static/chunks/59d0ad1b64f8544e.js\",\"/_next/static/chunks/8de849ca74fc071f.js\",\"/_next/static/chunks/e62b91212ee7f8ff.js\",\"/_next/static/chunks/f19fe44237e54646.js\",\"/_next/static/chunks/cb0a883bafeb6805.js\"],\"default\"]\n"])</script><script>self.__next_f.push([1,"6:[\"$\",\"main\",null,{\"className\":\"mx-auto w-full max-w-2xl px-4 py-16\",\"children\":[[\"$\",\"$L5\",null,{\"href\":\"/blog\",\"className\":\"inline-flex items-center gap-1.5 text-sm text-fd-muted-foreground hover:text-fd-foreground mb-10 transition-colors\",\"children\":\"← Back to Blog\"}],[\"$\",\"div\",null,{\"className\":\"mb-8\",\"children\":[[\"$\",\"time\",null,{\"className\":\"text-xs font-mono text-fd-muted-foreground uppercase tracking-wider\",\"children\":\"March 26, 2025\"}],[\"$\",\"h1\",null,{\"className\":\"text-3xl font-bold mt-2 mb-3\",\"children\":\"zen Omni: See, Hear, Talk, Write, Do It All!\"}],[\"$\",\"p\",null,{\"className\":\"text-fd-muted-foreground text-lg mb-4\",\"children\":\"QWEN CHAT HUGGING FACE MODELSCOPE DASHSCOPE GITHUB PAPER DEMO DISCORD\"}],[\"$\",\"div\",null,{\"className\":\"flex items-center gap-3 pt-4 border-t border-fd-border\",\"children\":[[\"$\",\"span\",null,{\"className\":\"text-sm text-fd-muted-foreground\",\"children\":[\"By \",\"Zen LM Team\"]}],[\"$\",\"div\",null,{\"className\":\"flex gap-1.5 ml-auto\",\"children\":[]}]]}]]}],[\"$\",\"div\",null,{\"className\":\"prose dark:prose-invert max-w-none\",\"children\":[[\"$\",\"p\",null,{\"children\":[[\"$\",\"$Lf\",null,{\"href\":\"https://chat.qwenlm.ai\",\"children\":\"QWEN CHAT\"}],\" \",[\"$\",\"$Lf\",null,{\"href\":\"https://huggingface.co/Qwen/zen-Omni-7B\",\"children\":\"HUGGING FACE\"}],\" \",[\"$\",\"$Lf\",null,{\"href\":\"https://modelscope.cn/models/Qwen/zen-Omni-7B\",\"children\":\"MODELSCOPE\"}],\" \",[\"$\",\"$Lf\",null,{\"href\":\"https://help.aliyun.com/zh/model-studio/user-guide/qwen-omni\",\"children\":\"DASHSCOPE\"}],\" \",[\"$\",\"$Lf\",null,{\"href\":\"https://github.com/QwenLM/zen-Omni\",\"children\":\"GITHUB\"}],\" \",[\"$\",\"$Lf\",null,{\"href\":\"https://github.com/QwenLM/zen-Omni/blob/main/assets/zen_Omni.pdf\",\"children\":\"PAPER\"}],\" \",[\"$\",\"$Lf\",null,{\"href\":\"https://huggingface.co/spaces/Qwen/zen-Omni-7B-Demo\",\"children\":\"DEMO\"}],\" \",[\"$\",\"$Lf\",null,{\"href\":\"https://discord.com/invite/yPEP2vHTu4\",\"children\":\"DISCORD\"}]]}],\"\\n\",[\"$\",\"p\",null,{\"children\":[\"We release \",[\"$\",\"strong\",null,{\"children\":\"zen-Omni\"}],\" , the new flagship end-to-end multimodal model in the Qwen series. Designed for comprehensive multimodal perception, it seamlessly processes diverse inputs including text, images, audio, and video, while delivering real-time streaming responses through both text generation and natural speech synthesis. To try the latest model, feel free to visit \",[\"$\",\"$Lf\",null,{\"href\":\"https://chat.qwenlm.ai\",\"children\":\"Qwen Chat\"}],\" and choose zen-Omni-7B. The model is now openly available on \",[\"$\",\"$Lf\",null,{\"href\":\"https://huggingface.co/Qwen/zen-Omni-7B\",\"children\":\"Hugging Face\"}],\", \",[\"$\",\"$Lf\",null,{\"href\":\"https://modelscope.cn/models/Qwen/zen-Omni-7B\",\"children\":\"ModelScope\"}],\", \",[\"$\",\"$Lf\",null,{\"href\":\"https://help.aliyun.com/zh/model-studio/user-guide/qwen-omni\",\"children\":\"DashScope\"}],\",and \",[\"$\",\"$Lf\",null,{\"href\":\"https://github.com/QwenLM/zen-Omni\",\"children\":\"GitHub\"}],\", with technical documentation available in our \",[\"$\",\"$Lf\",null,{\"href\":\"https://github.com/QwenLM/zen-Omni/assets/zen_Omni.pdf\",\"children\":\"Paper\"}],\". Experience interactive capabilities through our \",[\"$\",\"$Lf\",null,{\"href\":\"https://huggingface.co/spaces/Qwen/zen-Omni-7B-Demo\",\"children\":\"Demo\"}],\" or join our \",[\"$\",\"$Lf\",null,{\"href\":\"https://discord.gg/yPEP2vHTu4\",\"children\":\"Discord\"}],\" for discussions.\"]}],\"\\n\",[\"$\",\"p\",null,{\"children\":\"Key Features:\"}],\"\\n\",[\"$\",\"ul\",null,{\"children\":[\"\\n\",[\"$\",\"li\",null,{\"children\":[\"\\n\",[\"$\",\"p\",null,{\"children\":[[\"$\",\"strong\",null,{\"children\":\"Omni and Novel Architecture\"}],\" : We propose Thinker-Talker architecture, an end-to-end multimodal model designed to perceive diverse modalities, including text, images, audio, and video, while simultaneously generating text and natural speech responses in a streaming manner. We prpose a novel position embedding, named TMRoPE (Time-aligned Multimodal RoPE), to synchronize the timestamps of video inputs with audio.\"]}],\"\\n\"]}],\"\\n\",[\"$\",\"li\",null,{\"children\":[\"\\n\",[\"$\",\"p\",null,{\"children\":[[\"$\",\"strong\",null,{\"children\":\"Real-Time Voice and Video Chat\"}],\" : Architecture Designed for fully real-time interactions, supporting chunked input and immediate output.\"]}],\"\\n\"]}],\"\\n\",\"$L10\",\"\\n\",\"$L11\",\"\\n\",\"$L12\",\"\\n\"]}],\"\\n\",\"$L13\",\"\\n\",\"$L14\",\"\\n\",\"$L15\",\"\\n\",\"$L16\",\"\\n\",\"$L17\",\"\\n\",\"$L18\"]}]]}]\n"])</script><script>self.__next_f.push([1,"10:[\"$\",\"li\",null,{\"children\":[\"\\n\",[\"$\",\"p\",null,{\"children\":[[\"$\",\"strong\",null,{\"children\":\"Natural and Robust Speech Generation\"}],\" : Surpassing many existing streaming and non-streaming alternatives, demonstrating superior robustness and naturalness in speech generation.\"]}],\"\\n\"]}]\n11:[\"$\",\"li\",null,{\"children\":[\"\\n\",[\"$\",\"p\",null,{\"children\":[[\"$\",\"strong\",null,{\"children\":\"Strong Performance Across Modalities\"}],\" : Exhibiting exceptional performance across all modalities when benchmarked against similarly sized single-modality models. zen-Omni outperforms the similarly sized zen-Audio in audio capabilities and achieves comparable performance to zen-VL-7B.\"]}],\"\\n\"]}]\n12:[\"$\",\"li\",null,{\"children\":[\"\\n\",[\"$\",\"p\",null,{\"children\":[[\"$\",\"strong\",null,{\"children\":\"Excellent End-to-End Speech Instruction Following\"}],\" : zen-Omni shows performance in end-to-end speech instruction following that rivals its effectiveness with text inputs, evidenced by benchmarks such as MMLU and GSM8K.\"]}],\"\\n\"]}]\n13:[\"$\",\"h2\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"architecture\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#architecture\",\"className\":\"peer\",\"children\":\"Architecture\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n14:[\"$\",\"p\",null,{\"children\":\"zen-Omni employs Thinker-Talker architecture. Thinker functions like a brain, responsible for processing and understanding inputs from text, audio and video modalities, generating high-level representations and corresponding text. Talker operates like a human mouth, taking in the high-level representations and text produced by the Thinker in a streaming manner, and outputting discrete tokens of speech fluidly. Thinker is a Transformer decoder, accompanied by encoders for audio and image that facilitate information extraction. In contrast, Talker is designed as a dual-track autoregressive Transformer Decoder architecture. During both training and inference, Talker directly receives high-dimensional representations from Thinker and shares all of Thinker’s historical context information. Consequently, the entire architecture operates as a cohesive single model, enabling end-to-end training and inference.\"}]\n15:[\"$\",\"h2\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"performance\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#performance\",\"className\":\"peer\",\"children\":\"Performance\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n16:[\"$\",\"p\",null,{\"children\":\"We conducted a comprehensive evaluation of zen-Omni, which demonstrates strong performance across all modalities when compared to similarly sized single-modality models and closed-source models like zen-VL-7B, zen-Audio, and Gemini-1.5-pro. In tasks requiring the integration of multiple modalities, such as OmniBench, zen-Omni achieves state-of-the-art performance. Furthermore, in single-modality tasks, it excels in areas including speech recognition (Common Voice), translation (CoVoST2), audio understanding (MMAU), im"])</script><script>self.__next_f.push([1,"age reasoning (MMMU, MMStar), video understanding (MVBench), and speech generation (Seed-tts-eval and subjective naturalness).\"}]\n17:[\"$\",\"h2\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"whats-next\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#whats-next\",\"className\":\"peer\",\"children\":\"What’s Next\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n18:[\"$\",\"p\",null,{\"children\":\"We are eager to hear your feedback and see the innovative applications you create with zen-Omni. In the near future, our goal is to enhance our model’s ability to follow voice commands and improve audio-visual collaborative understanding. Additionally, we strive to integrate more modalities towards an omni-model!\"}]\n"])</script><script>self.__next_f.push([1,"b:[[\"$\",\"meta\",\"0\",{\"charSet\":\"utf-8\"}],[\"$\",\"meta\",\"1\",{\"name\":\"viewport\",\"content\":\"width=device-width, initial-scale=1\"}],[\"$\",\"meta\",\"2\",{\"name\":\"theme-color\",\"media\":\"(prefers-color-scheme: dark)\",\"content\":\"#0A0A0A\"}],[\"$\",\"meta\",\"3\",{\"name\":\"theme-color\",\"media\":\"(prefers-color-scheme: light)\",\"content\":\"#fff\"}]]\n"])</script><script>self.__next_f.push([1,"9:null\nd:[[\"$\",\"title\",\"0\",{\"children\":\"zen Omni: See, Hear, Talk, Write, Do It All! — Zen LM Blog | Zen LM\"}],[\"$\",\"meta\",\"1\",{\"name\":\"description\",\"content\":\"QWEN CHAT HUGGING FACE MODELSCOPE DASHSCOPE GITHUB PAPER DEMO DISCORD\"}],[\"$\",\"meta\",\"2\",{\"property\":\"og:title\",\"content\":\"Zen LM - Open Foundation Models\"}],[\"$\",\"meta\",\"3\",{\"property\":\"og:description\",\"content\":\"Zen AI model family by Hanzo AI. 14 frontier models from 4B to 1T+ parameters for code, reasoning, vision, and multimodal tasks.\"}],[\"$\",\"meta\",\"4\",{\"property\":\"og:url\",\"content\":\"https://zenlm.org\"}],[\"$\",\"meta\",\"5\",{\"property\":\"og:site_name\",\"content\":\"Zen LM\"}],[\"$\",\"meta\",\"6\",{\"property\":\"og:type\",\"content\":\"website\"}],[\"$\",\"meta\",\"7\",{\"name\":\"twitter:card\",\"content\":\"summary_large_image\"}],[\"$\",\"meta\",\"8\",{\"name\":\"twitter:site\",\"content\":\"@zenlmorg\"}],[\"$\",\"meta\",\"9\",{\"name\":\"twitter:creator\",\"content\":\"@zenlmorg\"}],[\"$\",\"meta\",\"10\",{\"name\":\"twitter:title\",\"content\":\"Zen LM - Open Foundation Models\"}],[\"$\",\"meta\",\"11\",{\"name\":\"twitter:description\",\"content\":\"Zen AI model family by Hanzo AI. 14 frontier models from 4B to 1T+ parameters for code, reasoning, vision, and multimodal tasks.\"}]]\n"])</script><script>
(function(){
// After React renders the lotus emoji from RSC payload, walk the DOM
// and replace it with the enso SVG. Polls so subsequent React re-renders
// also get cleaned up.
function swap(){
// 1. Replace lotus + ' Zen LM' text inside <a> brand links
document.querySelectorAll('a').forEach(function(a){
if (a.dataset.zenLotusSwapped) return;
var t = (a.textContent || '');
if (!/.*Zen LM|Zen LM.*/.test(t)) return;
// Find the lotus text node specifically
var walker = document.createTreeWalker(a, NodeFilter.SHOW_TEXT, null);
var node;
while ((node = walker.nextNode())) {
if (node.nodeValue && node.nodeValue.indexOf('') !== -1) {
node.nodeValue = node.nodeValue.replace(/\s?/g, '');
}
}
// Inject enso SVG at start
if (!a.querySelector('svg[aria-label="Zen LM"]')) {
var ns = 'http://www.w3.org/2000/svg';
var svg = document.createElementNS(ns, 'svg');
svg.setAttribute('viewBox', '0 0 512 512');
svg.setAttribute('fill', 'none');
svg.setAttribute('width', '18');
svg.setAttribute('height', '18');
svg.setAttribute('aria-label', 'Zen LM');
svg.style.cssText = 'display:inline-block;vertical-align:-2px;margin-right:6px;flex-shrink:0';
var p = document.createElementNS(ns, 'path');
p.setAttribute('d', 'M 256 84 A 172 172 0 1 0 256 428 A 172 172 0 1 0 256 84 Z');
p.setAttribute('stroke', 'currentColor');
p.setAttribute('stroke-width', '26');
p.setAttribute('stroke-linecap', 'round');
p.setAttribute('stroke-linejoin', 'round');
svg.appendChild(p);
a.insertBefore(svg, a.firstChild);
}
a.dataset.zenLotusSwapped = '1';
});
}
function start(){ swap(); setInterval(swap, 600); }
if (document.readyState !== 'loading') start();
else document.addEventListener('DOMContentLoaded', start);
})();
</script>
<style id="zenlm-brand-anim">
@keyframes zenlm-brand-draw { to { stroke-dashoffset: 0; } }
svg.zenlm-brand-svg .zenlm-brand-path {
stroke-dasharray: 1100;
stroke-dashoffset: 1100;
animation: zenlm-brand-draw 1.4s cubic-bezier(.45,0,.2,1) 0.1s 1 forwards;
}
.zen-link-icon { display: inline-block; vertical-align: middle; flex-shrink: 0; }
</style></body></html>