Files
zen-blog/blog/qwen2.5-omni/__next.blog.$d$slug.__PAGE__.txt

22 lines
11 KiB
Plaintext
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
1:"$Sreact.fragment"
2:I[10086,["/_next/static/chunks/59d0ad1b64f8544e.js","/_next/static/chunks/8de849ca74fc071f.js","/_next/static/chunks/e62b91212ee7f8ff.js","/_next/static/chunks/f19fe44237e54646.js","/_next/static/chunks/cb0a883bafeb6805.js"],""]
3:I[48068,["/_next/static/chunks/59d0ad1b64f8544e.js","/_next/static/chunks/8de849ca74fc071f.js","/_next/static/chunks/e62b91212ee7f8ff.js","/_next/static/chunks/f19fe44237e54646.js","/_next/static/chunks/cb0a883bafeb6805.js"],"default"]
12:I[89923,["/_next/static/chunks/4d80e004cf4896dd.js","/_next/static/chunks/350ee4303b732916.js"],"OutletBoundary"]
13:"$Sreact.suspense"
0:{"buildId":"i-dnJM_MIpJSOCQWNJVMq","rsc":["$","$1","c",{"children":[["$","main",null,{"className":"mx-auto w-full max-w-2xl px-4 py-16","children":[["$","$L2",null,{"href":"/blog","className":"inline-flex items-center gap-1.5 text-sm text-fd-muted-foreground hover:text-fd-foreground mb-10 transition-colors","children":"← Back to Blog"}],["$","div",null,{"className":"mb-8","children":[["$","time",null,{"className":"text-xs font-mono text-fd-muted-foreground uppercase tracking-wider","children":"March 26, 2025"}],["$","h1",null,{"className":"text-3xl font-bold mt-2 mb-3","children":"zen Omni: See, Hear, Talk, Write, Do It All!"}],["$","p",null,{"className":"text-fd-muted-foreground text-lg mb-4","children":"QWEN CHAT HUGGING FACE MODELSCOPE DASHSCOPE GITHUB PAPER DEMO DISCORD"}],["$","div",null,{"className":"flex items-center gap-3 pt-4 border-t border-fd-border","children":[["$","span",null,{"className":"text-sm text-fd-muted-foreground","children":["By ","Zen LM Team"]}],["$","div",null,{"className":"flex gap-1.5 ml-auto","children":[]}]]}]]}],["$","div",null,{"className":"prose dark:prose-invert max-w-none","children":[["$","p",null,{"children":[["$","$L3",null,{"href":"https://chat.qwenlm.ai","children":"QWEN CHAT"}]," ",["$","$L3",null,{"href":"https://huggingface.co/Qwen/zen-Omni-7B","children":"HUGGING FACE"}]," ",["$","$L3",null,{"href":"https://modelscope.cn/models/Qwen/zen-Omni-7B","children":"MODELSCOPE"}]," ",["$","$L3",null,{"href":"https://help.aliyun.com/zh/model-studio/user-guide/qwen-omni","children":"DASHSCOPE"}]," ",["$","$L3",null,{"href":"https://github.com/QwenLM/zen-Omni","children":"GITHUB"}]," ",["$","$L3",null,{"href":"https://github.com/QwenLM/zen-Omni/blob/main/assets/zen_Omni.pdf","children":"PAPER"}]," ",["$","$L3",null,{"href":"https://huggingface.co/spaces/Qwen/zen-Omni-7B-Demo","children":"DEMO"}]," ",["$","$L3",null,{"href":"https://discord.com/invite/yPEP2vHTu4","children":"DISCORD"}]]}],"\n",["$","p",null,{"children":["We release ",["$","strong",null,{"children":"zen-Omni"}]," , the new flagship end-to-end multimodal model in the Qwen series. Designed for comprehensive multimodal perception, it seamlessly processes diverse inputs including text, images, audio, and video, while delivering real-time streaming responses through both text generation and natural speech synthesis. To try the latest model, feel free to visit ",["$","$L3",null,{"href":"https://chat.qwenlm.ai","children":"Qwen Chat"}]," and choose zen-Omni-7B. The model is now openly available on ",["$","$L3",null,{"href":"https://huggingface.co/Qwen/zen-Omni-7B","children":"Hugging Face"}],", ",["$","$L3",null,{"href":"https://modelscope.cn/models/Qwen/zen-Omni-7B","children":"ModelScope"}],", ",["$","$L3",null,{"href":"https://help.aliyun.com/zh/model-studio/user-guide/qwen-omni","children":"DashScope"}],",and ",["$","$L3",null,{"href":"https://github.com/QwenLM/zen-Omni","children":"GitHub"}],", with technical documentation available in our ",["$","$L3",null,{"href":"https://github.com/QwenLM/zen-Omni/assets/zen_Omni.pdf","children":"Paper"}],". Experience interactive capabilities through our ",["$","$L3",null,{"href":"https://huggingface.co/spaces/Qwen/zen-Omni-7B-Demo","children":"Demo"}]," or join our ",["$","$L3",null,{"href":"https://discord.gg/yPEP2vHTu4","children":"Discord"}]," for discussions."]}],"\n",["$","p",null,{"children":"Key Features:"}],"\n",["$","ul",null,{"children":["\n",["$","li",null,{"children":["\n",["$","p",null,{"children":[["$","strong",null,{"children":"Omni and Novel Architecture"}]," : We propose Thinker-Talker architecture, an end-to-end multimodal model designed to perceive diverse modalities, including text, images, audio, and video, while simultaneously generating text and natural speech responses in a streaming manner. We prpose a novel position embedding, named TMRoPE (Time-aligned Multimodal RoPE), to synchronize the timestamps of video inputs with audio."]}],"\n"]}],"\n",["$","li",null,{"children":["\n",["$","p",null,{"children":[["$","strong",null,{"children":"Real-Time Voice and Video Chat"}]," : Architecture Designed for fully real-time interactions, supporting chunked input and immediate output."]}],"\n"]}],"\n","$L4","\n","$L5","\n","$L6","\n"]}],"\n","$L7","\n","$L8","\n","$L9","\n","$La","\n","$Lb","\n","$Lc"]}]]}],["$Ld","$Le","$Lf","$L10"],"$L11"]}],"loading":null,"isPartial":false}
4:["$","li",null,{"children":["\n",["$","p",null,{"children":[["$","strong",null,{"children":"Natural and Robust Speech Generation"}]," : Surpassing many existing streaming and non-streaming alternatives, demonstrating superior robustness and naturalness in speech generation."]}],"\n"]}]
5:["$","li",null,{"children":["\n",["$","p",null,{"children":[["$","strong",null,{"children":"Strong Performance Across Modalities"}]," : Exhibiting exceptional performance across all modalities when benchmarked against similarly sized single-modality models. zen-Omni outperforms the similarly sized zen-Audio in audio capabilities and achieves comparable performance to zen-VL-7B."]}],"\n"]}]
6:["$","li",null,{"children":["\n",["$","p",null,{"children":[["$","strong",null,{"children":"Excellent End-to-End Speech Instruction Following"}]," : zen-Omni shows performance in end-to-end speech instruction following that rivals its effectiveness with text inputs, evidenced by benchmarks such as MMLU and GSM8K."]}],"\n"]}]
7:["$","h2",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"architecture","children":[["$","a",null,{"data-card":"","href":"#architecture","className":"peer","children":"Architecture"}],["$","svg",null,{"xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
8:["$","p",null,{"children":"zen-Omni employs Thinker-Talker architecture. Thinker functions like a brain, responsible for processing and understanding inputs from text, audio and video modalities, generating high-level representations and corresponding text. Talker operates like a human mouth, taking in the high-level representations and text produced by the Thinker in a streaming manner, and outputting discrete tokens of speech fluidly. Thinker is a Transformer decoder, accompanied by encoders for audio and image that facilitate information extraction. In contrast, Talker is designed as a dual-track autoregressive Transformer Decoder architecture. During both training and inference, Talker directly receives high-dimensional representations from Thinker and shares all of Thinker’s historical context information. Consequently, the entire architecture operates as a cohesive single model, enabling end-to-end training and inference."}]
9:["$","h2",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"performance","children":[["$","a",null,{"data-card":"","href":"#performance","className":"peer","children":"Performance"}],["$","svg",null,{"xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
a:["$","p",null,{"children":"We conducted a comprehensive evaluation of zen-Omni, which demonstrates strong performance across all modalities when compared to similarly sized single-modality models and closed-source models like zen-VL-7B, zen-Audio, and Gemini-1.5-pro. In tasks requiring the integration of multiple modalities, such as OmniBench, zen-Omni achieves state-of-the-art performance. Furthermore, in single-modality tasks, it excels in areas including speech recognition (Common Voice), translation (CoVoST2), audio understanding (MMAU), image reasoning (MMMU, MMStar), video understanding (MVBench), and speech generation (Seed-tts-eval and subjective naturalness)."}]
b:["$","h2",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"whats-next","children":[["$","a",null,{"data-card":"","href":"#whats-next","className":"peer","children":"What’s Next"}],["$","svg",null,{"xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
c:["$","p",null,{"children":"We are eager to hear your feedback and see the innovative applications you create with zen-Omni. In the near future, our goal is to enhance our model’s ability to follow voice commands and improve audio-visual collaborative understanding. Additionally, we strive to integrate more modalities towards an omni-model!"}]
d:["$","script","script-0",{"src":"/_next/static/chunks/8de849ca74fc071f.js","async":true}]
e:["$","script","script-1",{"src":"/_next/static/chunks/e62b91212ee7f8ff.js","async":true}]
f:["$","script","script-2",{"src":"/_next/static/chunks/f19fe44237e54646.js","async":true}]
10:["$","script","script-3",{"src":"/_next/static/chunks/cb0a883bafeb6805.js","async":true}]
11:["$","$L12",null,{"children":["$","$13",null,{"name":"Next.MetadataOutlet","children":"$@14"}]}]
14:null