Files
zen-blog/blog/qwen2.5-omni.txt

27 lines
16 KiB
Plaintext
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
1:"$Sreact.fragment"
2:I[106,["/_next/static/chunks/59d0ad1b64f8544e.js"],"RootProvider"]
3:I[53113,["/_next/static/chunks/4d80e004cf4896dd.js","/_next/static/chunks/350ee4303b732916.js"],"default"]
4:I[73211,["/_next/static/chunks/4d80e004cf4896dd.js","/_next/static/chunks/350ee4303b732916.js"],"default"]
5:I[10086,["/_next/static/chunks/59d0ad1b64f8544e.js","/_next/static/chunks/8de849ca74fc071f.js","/_next/static/chunks/e62b91212ee7f8ff.js","/_next/static/chunks/f19fe44237e54646.js","/_next/static/chunks/cb0a883bafeb6805.js"],""]
7:I[89923,["/_next/static/chunks/4d80e004cf4896dd.js","/_next/static/chunks/350ee4303b732916.js"],"OutletBoundary"]
8:"$Sreact.suspense"
a:I[89923,["/_next/static/chunks/4d80e004cf4896dd.js","/_next/static/chunks/350ee4303b732916.js"],"ViewportBoundary"]
c:I[89923,["/_next/static/chunks/4d80e004cf4896dd.js","/_next/static/chunks/350ee4303b732916.js"],"MetadataBoundary"]
e:I[6998,["/_next/static/chunks/4d80e004cf4896dd.js","/_next/static/chunks/350ee4303b732916.js"],"default"]
:HL["/_next/static/chunks/f2332aac77592f9d.css","style"]
0:{"P":null,"b":"i-dnJM_MIpJSOCQWNJVMq","c":["","blog","qwen2.5-omni"],"q":"","i":false,"f":[[["",{"children":["blog",{"children":[["slug","qwen2.5-omni","d"],{"children":["__PAGE__",{}]}]}]},"$undefined","$undefined",true],[["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/_next/static/chunks/f2332aac77592f9d.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","script","script-0",{"src":"/_next/static/chunks/59d0ad1b64f8544e.js","async":true,"nonce":"$undefined"}]],["$","html",null,{"lang":"en","className":"dark","suppressHydrationWarning":true,"children":["$","body",null,{"className":"antialiased","children":["$","$L2",null,{"children":["$","$L3",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L4",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","main",null,{"className":"flex min-h-screen flex-col items-center justify-center px-4 text-center","children":[["$","div",null,{"className":"mb-8 opacity-20","children":["$","svg",null,{"width":"120","height":"120","viewBox":"0 0 120 120","fill":"none","aria-hidden":"true","children":["$","circle",null,{"cx":"60","cy":"60","r":"50","stroke":"currentColor","strokeWidth":"3","strokeLinecap":"round","strokeDasharray":"280 40"}]}]}],["$","p",null,{"className":"text-xs font-mono uppercase tracking-[0.3em] text-fd-muted-foreground mb-4","children":"404"}],["$","h1",null,{"className":"text-3xl font-semibold mb-3","children":"Page not found"}],["$","p",null,{"className":"text-fd-muted-foreground max-w-sm mb-10","children":"This page doesn't exist, or it may have moved. Try the documentation or head home."}],["$","div",null,{"className":"flex flex-wrap gap-3 justify-center","children":[["$","$L5",null,{"href":"/","className":"inline-flex items-center gap-2 rounded-lg bg-fd-primary text-fd-primary-foreground px-4 py-2 text-sm font-medium hover:opacity-90 transition","children":"Go home"}],["$","$L5",null,{"href":"/docs","className":"inline-flex items-center gap-2 rounded-lg border border-fd-border bg-fd-background px-4 py-2 text-sm font-medium hover:bg-fd-muted transition","children":"Documentation"}],["$","$L5",null,{"href":"/docs/models","className":"inline-flex items-center gap-2 rounded-lg border border-fd-border bg-fd-background px-4 py-2 text-sm font-medium hover:bg-fd-muted transition","children":"Browse models"}]]}],["$","p",null,{"className":"mt-16 text-xs font-mono text-fd-muted-foreground opacity-50","children":"zenlm.org"}]]}],[]],"forbidden":"$undefined","unauthorized":"$undefined"}]}]}]}]]}],{"children":[["$","$1","c",{"children":[null,["$","$L3",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L4",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","forbidden":"$undefined","unauthorized":"$undefined"}]]}],{"children":[["$","$1","c",{"children":[null,["$","$L3",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L4",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","forbidden":"$undefined","unauthorized":"$undefined"}]]}],{"children":[["$","$1","c",{"children":["$L6",[["$","script","script-0",{"src":"/_next/static/chunks/8de849ca74fc071f.js","async":true,"nonce":"$undefined"}],["$","script","script-1",{"src":"/_next/static/chunks/e62b91212ee7f8ff.js","async":true,"nonce":"$undefined"}],["$","script","script-2",{"src":"/_next/static/chunks/f19fe44237e54646.js","async":true,"nonce":"$undefined"}],["$","script","script-3",{"src":"/_next/static/chunks/cb0a883bafeb6805.js","async":true,"nonce":"$undefined"}]],["$","$L7",null,{"children":["$","$8",null,{"name":"Next.MetadataOutlet","children":"$@9"}]}]]}],{},null,false,false]},null,false,false]},null,false,false]},null,false,false],["$","$1","h",{"children":[null,["$","$La",null,{"children":"$Lb"}],["$","div",null,{"hidden":true,"children":["$","$Lc",null,{"children":["$","$8",null,{"name":"Next.Metadata","children":"$Ld"}]}]}],null]}],false]],"m":"$undefined","G":["$e",[]],"S":true}
f:I[48068,["/_next/static/chunks/59d0ad1b64f8544e.js","/_next/static/chunks/8de849ca74fc071f.js","/_next/static/chunks/e62b91212ee7f8ff.js","/_next/static/chunks/f19fe44237e54646.js","/_next/static/chunks/cb0a883bafeb6805.js"],"default"]
6:["$","main",null,{"className":"mx-auto w-full max-w-2xl px-4 py-16","children":[["$","$L5",null,{"href":"/blog","className":"inline-flex items-center gap-1.5 text-sm text-fd-muted-foreground hover:text-fd-foreground mb-10 transition-colors","children":"← Back to Blog"}],["$","div",null,{"className":"mb-8","children":[["$","time",null,{"className":"text-xs font-mono text-fd-muted-foreground uppercase tracking-wider","children":"March 26, 2025"}],["$","h1",null,{"className":"text-3xl font-bold mt-2 mb-3","children":"zen Omni: See, Hear, Talk, Write, Do It All!"}],["$","p",null,{"className":"text-fd-muted-foreground text-lg mb-4","children":"QWEN CHAT HUGGING FACE MODELSCOPE DASHSCOPE GITHUB PAPER DEMO DISCORD"}],["$","div",null,{"className":"flex items-center gap-3 pt-4 border-t border-fd-border","children":[["$","span",null,{"className":"text-sm text-fd-muted-foreground","children":["By ","Zen LM Team"]}],["$","div",null,{"className":"flex gap-1.5 ml-auto","children":[]}]]}]]}],["$","div",null,{"className":"prose dark:prose-invert max-w-none","children":[["$","p",null,{"children":[["$","$Lf",null,{"href":"https://chat.qwenlm.ai","children":"QWEN CHAT"}]," ",["$","$Lf",null,{"href":"https://huggingface.co/Qwen/zen-Omni-7B","children":"HUGGING FACE"}]," ",["$","$Lf",null,{"href":"https://modelscope.cn/models/Qwen/zen-Omni-7B","children":"MODELSCOPE"}]," ",["$","$Lf",null,{"href":"https://help.aliyun.com/zh/model-studio/user-guide/qwen-omni","children":"DASHSCOPE"}]," ",["$","$Lf",null,{"href":"https://github.com/QwenLM/zen-Omni","children":"GITHUB"}]," ",["$","$Lf",null,{"href":"https://github.com/QwenLM/zen-Omni/blob/main/assets/zen_Omni.pdf","children":"PAPER"}]," ",["$","$Lf",null,{"href":"https://huggingface.co/spaces/Qwen/zen-Omni-7B-Demo","children":"DEMO"}]," ",["$","$Lf",null,{"href":"https://discord.com/invite/yPEP2vHTu4","children":"DISCORD"}]]}],"\n",["$","p",null,{"children":["We release ",["$","strong",null,{"children":"zen-Omni"}]," , the new flagship end-to-end multimodal model in the Qwen series. Designed for comprehensive multimodal perception, it seamlessly processes diverse inputs including text, images, audio, and video, while delivering real-time streaming responses through both text generation and natural speech synthesis. To try the latest model, feel free to visit ",["$","$Lf",null,{"href":"https://chat.qwenlm.ai","children":"Qwen Chat"}]," and choose zen-Omni-7B. The model is now openly available on ",["$","$Lf",null,{"href":"https://huggingface.co/Qwen/zen-Omni-7B","children":"Hugging Face"}],", ",["$","$Lf",null,{"href":"https://modelscope.cn/models/Qwen/zen-Omni-7B","children":"ModelScope"}],", ",["$","$Lf",null,{"href":"https://help.aliyun.com/zh/model-studio/user-guide/qwen-omni","children":"DashScope"}],",and ",["$","$Lf",null,{"href":"https://github.com/QwenLM/zen-Omni","children":"GitHub"}],", with technical documentation available in our ",["$","$Lf",null,{"href":"https://github.com/QwenLM/zen-Omni/assets/zen_Omni.pdf","children":"Paper"}],". Experience interactive capabilities through our ",["$","$Lf",null,{"href":"https://huggingface.co/spaces/Qwen/zen-Omni-7B-Demo","children":"Demo"}]," or join our ",["$","$Lf",null,{"href":"https://discord.gg/yPEP2vHTu4","children":"Discord"}]," for discussions."]}],"\n",["$","p",null,{"children":"Key Features:"}],"\n",["$","ul",null,{"children":["\n",["$","li",null,{"children":["\n",["$","p",null,{"children":[["$","strong",null,{"children":"Omni and Novel Architecture"}]," : We propose Thinker-Talker architecture, an end-to-end multimodal model designed to perceive diverse modalities, including text, images, audio, and video, while simultaneously generating text and natural speech responses in a streaming manner. We prpose a novel position embedding, named TMRoPE (Time-aligned Multimodal RoPE), to synchronize the timestamps of video inputs with audio."]}],"\n"]}],"\n",["$","li",null,{"children":["\n",["$","p",null,{"children":[["$","strong",null,{"children":"Real-Time Voice and Video Chat"}]," : Architecture Designed for fully real-time interactions, supporting chunked input and immediate output."]}],"\n"]}],"\n","$L10","\n","$L11","\n","$L12","\n"]}],"\n","$L13","\n","$L14","\n","$L15","\n","$L16","\n","$L17","\n","$L18"]}]]}]
10:["$","li",null,{"children":["\n",["$","p",null,{"children":[["$","strong",null,{"children":"Natural and Robust Speech Generation"}]," : Surpassing many existing streaming and non-streaming alternatives, demonstrating superior robustness and naturalness in speech generation."]}],"\n"]}]
11:["$","li",null,{"children":["\n",["$","p",null,{"children":[["$","strong",null,{"children":"Strong Performance Across Modalities"}]," : Exhibiting exceptional performance across all modalities when benchmarked against similarly sized single-modality models. zen-Omni outperforms the similarly sized zen-Audio in audio capabilities and achieves comparable performance to zen-VL-7B."]}],"\n"]}]
12:["$","li",null,{"children":["\n",["$","p",null,{"children":[["$","strong",null,{"children":"Excellent End-to-End Speech Instruction Following"}]," : zen-Omni shows performance in end-to-end speech instruction following that rivals its effectiveness with text inputs, evidenced by benchmarks such as MMLU and GSM8K."]}],"\n"]}]
13:["$","h2",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"architecture","children":[["$","a",null,{"data-card":"","href":"#architecture","className":"peer","children":"Architecture"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
14:["$","p",null,{"children":"zen-Omni employs Thinker-Talker architecture. Thinker functions like a brain, responsible for processing and understanding inputs from text, audio and video modalities, generating high-level representations and corresponding text. Talker operates like a human mouth, taking in the high-level representations and text produced by the Thinker in a streaming manner, and outputting discrete tokens of speech fluidly. Thinker is a Transformer decoder, accompanied by encoders for audio and image that facilitate information extraction. In contrast, Talker is designed as a dual-track autoregressive Transformer Decoder architecture. During both training and inference, Talker directly receives high-dimensional representations from Thinker and shares all of Thinker’s historical context information. Consequently, the entire architecture operates as a cohesive single model, enabling end-to-end training and inference."}]
15:["$","h2",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"performance","children":[["$","a",null,{"data-card":"","href":"#performance","className":"peer","children":"Performance"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
16:["$","p",null,{"children":"We conducted a comprehensive evaluation of zen-Omni, which demonstrates strong performance across all modalities when compared to similarly sized single-modality models and closed-source models like zen-VL-7B, zen-Audio, and Gemini-1.5-pro. In tasks requiring the integration of multiple modalities, such as OmniBench, zen-Omni achieves state-of-the-art performance. Furthermore, in single-modality tasks, it excels in areas including speech recognition (Common Voice), translation (CoVoST2), audio understanding (MMAU), image reasoning (MMMU, MMStar), video understanding (MVBench), and speech generation (Seed-tts-eval and subjective naturalness)."}]
17:["$","h2",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"whats-next","children":[["$","a",null,{"data-card":"","href":"#whats-next","className":"peer","children":"What’s Next"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
18:["$","p",null,{"children":"We are eager to hear your feedback and see the innovative applications you create with zen-Omni. In the near future, our goal is to enhance our model’s ability to follow voice commands and improve audio-visual collaborative understanding. Additionally, we strive to integrate more modalities towards an omni-model!"}]
b:[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","meta","2",{"name":"theme-color","media":"(prefers-color-scheme: dark)","content":"#0A0A0A"}],["$","meta","3",{"name":"theme-color","media":"(prefers-color-scheme: light)","content":"#fff"}]]
9:null
d:[["$","title","0",{"children":"zen Omni: See, Hear, Talk, Write, Do It All! — Zen LM Blog | Zen LM"}],["$","meta","1",{"name":"description","content":"QWEN CHAT HUGGING FACE MODELSCOPE DASHSCOPE GITHUB PAPER DEMO DISCORD"}],["$","meta","2",{"property":"og:title","content":"Zen LM - Open Foundation Models"}],["$","meta","3",{"property":"og:description","content":"Zen AI model family by Hanzo AI. 14 frontier models from 4B to 1T+ parameters for code, reasoning, vision, and multimodal tasks."}],["$","meta","4",{"property":"og:url","content":"https://zenlm.org"}],["$","meta","5",{"property":"og:site_name","content":"Zen LM"}],["$","meta","6",{"property":"og:type","content":"website"}],["$","meta","7",{"name":"twitter:card","content":"summary_large_image"}],["$","meta","8",{"name":"twitter:site","content":"@zenlmorg"}],["$","meta","9",{"name":"twitter:creator","content":"@zenlmorg"}],["$","meta","10",{"name":"twitter:title","content":"Zen LM - Open Foundation Models"}],["$","meta","11",{"name":"twitter:description","content":"Zen AI model family by Hanzo AI. 14 frontier models from 4B to 1T+ parameters for code, reasoning, vision, and multimodal tasks."}]]