Files
zen-blog/blog/qwen2.5-1m.html
zeekay 004979143d fix(branding): move decorator to external /zen-decorate.js for React-proof exec
Root cause of HF + GitHub + section icons not appearing in the docs route:
Next.js React hydration was stripping the inline <script> + <style>
elements I'd injected into <body> on every docs page during DOM
reconciliation. The script never registered its setInterval.

Move the decorator to /zen-decorate.js loaded via <script defer src>
in <head>. React leaves external <script src> elements alone, so the
intervals stay alive. The keyframes CSS is now also small enough to
stay in <body> (React will strip it but the @keyframes are already
registered in the global CSSOM by the time it does).
2026-05-19 20:08:58 -07:00

238 lines
107 KiB
HTML
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html><!--i_dnJM_MIpJSOCQWNJVMq--><html lang="en" class="dark"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="stylesheet" href="/_next/static/chunks/f2332aac77592f9d.css" data-precedence="next"/><link rel="preload" as="script" fetchPriority="low" href="/_next/static/chunks/e83606e8fa9cc796.js"/><script src="/_next/static/chunks/36d6595f0156cd7e.js" async=""></script><script src="/_next/static/chunks/040e9cea20a8d9c7.js" async=""></script><script src="/_next/static/chunks/d4dffb5a0973d49c.js" async=""></script><script src="/_next/static/chunks/turbopack-967f27a9fd33556d.js" async=""></script><script src="/_next/static/chunks/59d0ad1b64f8544e.js" async=""></script><script src="/_next/static/chunks/4d80e004cf4896dd.js" async=""></script><script src="/_next/static/chunks/350ee4303b732916.js" async=""></script><script src="/_next/static/chunks/8de849ca74fc071f.js" async=""></script><script src="/_next/static/chunks/e62b91212ee7f8ff.js" async=""></script><script src="/_next/static/chunks/f19fe44237e54646.js" async=""></script><script src="/_next/static/chunks/cb0a883bafeb6805.js" async=""></script><meta name="theme-color" media="(prefers-color-scheme: dark)" content="#0A0A0A"/><meta name="theme-color" media="(prefers-color-scheme: light)" content="#fff"/><title>zen-1M: Deploy Your Own Qwen with Context Length up to 1M Tokens — Zen LM Blog | Zen LM</title><meta name="description" content="Tech Report HuggingFace ModelScope Qwen Chat HuggingFace Demo ModelScope Demo DISCORD"/><meta property="og:title" content="Zen LM - Open Foundation Models"/><meta property="og:description" content="Zen AI model family by Hanzo AI. 14 frontier models from 4B to 1T+ parameters for code, reasoning, vision, and multimodal tasks."/><meta property="og:url" content="https://zenlm.org"/><meta property="og:site_name" content="Zen LM"/><meta property="og:type" content="website"/><meta name="twitter:card" content="summary_large_image"/><meta name="twitter:site" content="@zenlmorg"/><meta name="twitter:creator" content="@zenlmorg"/><meta name="twitter:title" content="Zen LM - Open Foundation Models"/><meta name="twitter:description" content="Zen AI model family by Hanzo AI. 14 frontier models from 4B to 1T+ parameters for code, reasoning, vision, and multimodal tasks."/><script src="/_next/static/chunks/a6dad97d9634a72d.js" noModule=""></script><script defer src="/zen-decorate.js"></script></head><body class="antialiased"><div hidden=""><!--$--><!--/$--></div><script>((a,b,c,d,e,f,g,h)=>{let i=document.documentElement,j=["light","dark"];function k(b){var c;(Array.isArray(a)?a:[a]).forEach(a=>{let c="class"===a,d=c&&f?e.map(a=>f[a]||a):e;c?(i.classList.remove(...d),i.classList.add(f&&f[b]?f[b]:b)):i.setAttribute(a,b)}),c=b,h&&j.includes(c)&&(i.style.colorScheme=c)}if(d)k(d);else try{let a=localStorage.getItem(b)||c,d=g&&"system"===a?window.matchMedia("(prefers-color-scheme: dark)").matches?"dark":"light":a;k(d)}catch(a){}})("class","theme","system",null,["light","dark"],null,true,true)</script><!--$--><div data-closed="" role="presentation" hidden="" style="user-select:none;-webkit-user-select:none" class="fixed inset-0 z-50 backdrop-blur-xs bg-fd-overlay data-open:animate-fd-fade-in data-closed:animate-fd-fade-out"></div><div class="bg-fd-secondary/50 p-3 empty:hidden"></div><!--/$--><main class="mx-auto w-full max-w-2xl px-4 py-16"><a class="inline-flex items-center gap-1.5 text-sm text-fd-muted-foreground hover:text-fd-foreground mb-10 transition-colors" href="/blog">← Back to Blog</a><div class="mb-8"><time class="text-xs font-mono text-fd-muted-foreground uppercase tracking-wider">January 26, 2025</time><h1 class="text-3xl font-bold mt-2 mb-3">zen-1M: Deploy Your Own Qwen with Context Length up to 1M Tokens</h1><p class="text-fd-muted-foreground text-lg mb-4">Tech Report HuggingFace ModelScope Qwen Chat HuggingFace Demo ModelScope Demo DISCORD</p><div class="flex items-center gap-3 pt-4 border-t border-fd-border"><span class="text-sm text-fd-muted-foreground">By <!-- -->Zen LM Team</span><div class="flex gap-1.5 ml-auto"></div></div></div><div class="prose dark:prose-invert max-w-none"><p><a href="https://qianwen-res.oss-cn-beijing.aliyuncs.com/zen-1M/zen_1M_Technical_Report.pdf" rel="noreferrer noopener" target="_blank">Tech Report</a> <a href="https://huggingface.co/Qwen" rel="noreferrer noopener" target="_blank">HuggingFace</a> <a href="https://modelscope.cn/organization/qwen" rel="noreferrer noopener" target="_blank">ModelScope</a> <a href="https://chat.qwenlm.ai/" rel="noreferrer noopener" target="_blank">Qwen Chat</a> <a href="https://huggingface.co/spaces/Qwen/zen-1M-Demo" rel="noreferrer noopener" target="_blank">HuggingFace Demo</a> <a href="https://www.modelscope.cn/studios/Qwen/zen-1M-Demo" rel="noreferrer noopener" target="_blank">ModelScope Demo</a> <a href="https://discord.gg/yPEP2vHTu4" rel="noreferrer noopener" target="_blank">DISCORD</a></p>
<h1 class="flex scroll-m-28 flex-row items-center gap-2" id="introduction"><a data-card="" href="#introduction" class="peer">Introduction</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h1>
<p>Two months after upgrading <a href="../qwen3-turbo">zen-Turbo</a> to support context length up to one million tokens, we are back with the open-source zen-1M models and the corresponding inference framework support. Here’s what you can expect from this release:</p>
<ol>
<li>
<p><strong>Opensource Models:</strong> We’re releasing two new checkpoints, <strong>zen-7B-Instruct-1M</strong> and <strong>zen-14B-Instruct-1M</strong> , marking the first time we’ve upgraded our opensource Qwen models to handle 1M-token contexts.</p>
</li>
<li>
<p><strong>Inference Framework:</strong> To help developers deploy the zen-1M series models more efficiently, we’ve fully open-sourced our inference framework based on <a href="https://github.com/vllm-project/vllm" rel="noreferrer noopener" target="_blank">vLLM</a>. With integration with sparse attention methods, our framework can process 1M-token inputs <strong>3x to 7x</strong> faster.</p>
</li>
<li>
<p><strong><a href="https://qianwen-res.oss-cn-beijing.aliyuncs.com/zen-1M/zen_1M_Technical_Report.pdf" rel="noreferrer noopener" target="_blank">Technical Report</a>:</strong> We’re also sharing the technical details behind the zen-1M series, including design insights for training and inference frameworks, as well as ablation experiments.</p>
</li>
</ol>
<p>You can experience zen-1M models online by visiting our demo on <a href="https://huggingface.co/spaces/Qwen/zen-1M-Demo" rel="noreferrer noopener" target="_blank">Huggingface</a> and <a href="https://www.modelscope.cn/studios/Qwen/zen-1M-Demo" rel="noreferrer noopener" target="_blank">Modelscope</a>.</p>
<p>Additionally, we recently introduced <strong><a href="https://chat.qwenlm.ai/" rel="noreferrer noopener" target="_blank">Qwen Chat</a></strong> , an advanced AI assistant from the Qwen series. With Qwen Chat, you can engage in conversations, write code, perform searches, generate images and videos, and utilize various tools. Notably, Qwen Chat also features the zen-Turbo model, which supports long-context processing with a context length of up to 1M tokens.</p>
<h1 class="flex scroll-m-28 flex-row items-center gap-2" id="model-performance"><a data-card="" href="#model-performance" class="peer">Model Performance</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h1>
<p>Let’s start by diving into the performance of the zen-1M series models, covering both long-context and short text tasks.</p>
<h2 class="flex scroll-m-28 flex-row items-center gap-2" id="long-context-tasks"><a data-card="" href="#long-context-tasks" class="peer">Long-Context Tasks</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h2>
<p>First off, we evaluate the zen-1M models on the Passkey Retrieval task with a context length of 1 million tokens. The results show that these models can accurately retrieve hidden information from documents containing up to 1M tokens, with only minor errors observed in the 7B model.</p>
<p>For more complex long-context understanding tasks, we select <a href="https://github.com/hsiehjackson/RULER" rel="noreferrer noopener" target="_blank">RULER</a>, <a href="https://github.com/infinigence/LVEval" rel="noreferrer noopener" target="_blank">LV-Eval</a>, <a href="https://github.com/THUDM/LongAlign" rel="noreferrer noopener" target="_blank">LongbenchChat</a> used in <a href="../qwen3-turbo/#more-complex-long-text-tasks">this blog</a>.</p>
<p>From these results, we can draw a few key conclusions:</p>
<ul>
<li><strong>Significantly Superior to the 128k Version:</strong> The zen-1M series models significantly outperform their 128K counterparts in most long-context tasks, especially for sequences exceeding 64K in length.</li>
<li><strong>Notable Performance Advantage:</strong> The zen-14B-Instruct-1M model not only beats zen-Turbo but also consistently outperforms GPT-4o-mini across multiple datasets, offering a robust open-source alternative for long-context tasks.</li>
</ul>
<h2 class="flex scroll-m-28 flex-row items-center gap-2" id="short-context-tasks"><a data-card="" href="#short-context-tasks" class="peer">Short-Context Tasks</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h2>
<p>Besides performance on long sequences, we’re equally interested in how these models handle short sequences. So, we compare the zen-1M models and their 128K versions on widely used academic benchmarks, throwing in GPT-4o-mini for comparison.</p>
<p>Here’s what we find:</p>
<ul>
<li>Both zen-7B-Instruct-1M and zen-14B-Instruct-1M maintain performance on short text tasks that is similar to their 128K versions, ensuring the fundamental capabilities haven’t been compromised by the addition of long-sequence processing abilities.</li>
<li>Compared to GPT-4o-mini, both zen-14B-Instruct-1M and zen-Turbo achieve similar performance on short text tasks while supporting a context length that’s eight times longer.</li>
</ul>
<h1 class="flex scroll-m-28 flex-row items-center gap-2" id="key-techniques"><a data-card="" href="#key-techniques" class="peer">Key Techniques</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h1>
<p>Here, we’ll briefly introduce the key techniques behind building zen-1M. For more details, please check out our <a href="https://qianwen-res.oss-cn-beijing.aliyuncs.com/zen-1M/zen_1M_Technical_Report.pdf" rel="noreferrer noopener" target="_blank">technical report</a>.</p>
<h2 class="flex scroll-m-28 flex-row items-center gap-2" id="long-context-training"><a data-card="" href="#long-context-training" class="peer">Long-Context Training</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h2>
<p>Training with long sequences demands substantial computational resources, so we adopt a progressive approach to expand the context length for zen-1M through multiple stages:</p>
<ul>
<li>We begin with an intermediate checkpoint of pre-trained zen, which had a 4K token context length.</li>
<li><strong>In Pretraining</strong> , we gradually increase the context length from 4K to 256K tokens while using <a href="https://arxiv.org/abs/2309.16039" rel="noreferrer noopener" target="_blank">Adjusted Base Frequency</a>, raising the RoPE base from 10,000 to 10,000,000.</li>
<li><strong>In Supervised Fine-tuning</strong> , we split this into two stages to preserve performance on shorter sequences:
<ul>
<li><strong>Stage 1:</strong> Fine-tuned only on short instructions (up to 32K tokens) using the same data and steps as the 128K versions of zen.</li>
<li><strong>Stage 2:</strong> Mixed short (up to 32K) and long (up to 256K) instructions to enhance long-context task performance while maintaining short-task quality.</li>
</ul>
</li>
<li><strong>In Reinforcement Learning</strong> , we train models on short texts up to 8K tokens, which sufficiently improves alignment with human preferences and generalizes well to long-context tasks.</li>
</ul>
<p>The final instruction-tuned models are capable of handling sequences up to 256K tokens.</p>
<h2 class="flex scroll-m-28 flex-row items-center gap-2" id="length-extrapolation"><a data-card="" href="#length-extrapolation" class="peer">Length Extrapolation</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h2>
<p>During training, we develop an instruction-tuned model with a context length of 256K tokens. To extend this to 1M tokens, we employ length extrapolation techniques.</p>
<p>The degradation of LLMs based on RoPE in long-context tasks is mainly due to unseen, large relative positional distances between queries and keys in computing attention weight. We employ <a href="https://arxiv.org/abs/2402.17463" rel="noreferrer noopener" target="_blank"><strong>Dual Chunk Attention</strong></a> (DCA), which addresses this issue by remapping relative positions to smaller values, avoiding the large distances not seen during training.</p>
<p>We evaluat the zen-1M models and their 128K counterparts with and without the length extrapolation method. We can find:</p>
<p>Even models trained on just 32K tokens, such as the zen-7B-Instruct, achieve nearly perfect accuracy in passkey retrieval tasks with 1M-token contexts. This underscores the remarkable ability of DCA to extend supported context lengths, without any training required.</p>
<h2 class="flex scroll-m-28 flex-row items-center gap-2" id="sparse-attention"><a data-card="" href="#sparse-attention" class="peer">Sparse Attention</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h2>
<p>For long-context language models, inference speed is crucial for user experience. We introduce a sparse attention mechanism based on <a href="https://arxiv.org/abs/2407.02490" rel="noreferrer noopener" target="_blank"><strong>MInference</strong></a> to accelerate the prefill phase. Furthermore, we propose several improvements:</p>
<ul>
<li>
<p><strong>Integrating with Chunked Prefill:</strong> Directly processing sequences of 1M tokens results in substantial memory overhead to store the activations in MLP layers, consuming 71GB of VRAM in zen-7B. By integrating with chunk prefill with a chunk length of 32,768 tokens, activation VRAM usage is reduced by 96.7%, leading to a significant decrease in memory consumption.</p>
</li>
<li>
<p><strong>Integrating with Length Extrapolation:</strong> We integrate DCA with MInference in long-context processing, thereby enhancing inference efficiency and achieving greater accuracy.</p>
</li>
<li>
<p><strong>Sparsity Refinement on Long Sequences:</strong> MInference requires an offline search to determine the optimal sparsification configuration for each attention head. Due to the computational demand of full attention weights, this search is typically conducted on short sequences, which may not generalize well to longer sequences. We developed a method to refine the sparsification configuration specifically for sequences up to 1M tokens, which significantly reduces the accuracy loss brought by sparse attention.</p>
</li>
<li>
<p><strong>More Optimizations:</strong> We introduce additional optimizations, such as enhanced kernel efficiency and dynamic chunked pipeline parallelism, to fully unlock the potential of the entire framework.</p>
</li>
</ul>
<p>With these enhancements, our inference framework results in a 3.2x to 6.7x acceleration in the prefill speed across different model sizes and GPU devices for sequences of 1M token length.</p>
<h1 class="flex scroll-m-28 flex-row items-center gap-2" id="deploy-zen-1m-models-locally"><a data-card="" href="#deploy-zen-1m-models-locally" class="peer">Deploy zen-1M Models Locally</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h1>
<p>Here we provide step-by-step instructions for deploying the zen-1M models on your local devices.</p>
<h3 class="flex scroll-m-28 flex-row items-center gap-2" id="1-system-preparation"><a data-card="" href="#1-system-preparation" class="peer">1. System Preparation</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h3>
<p>To achieve the best performance, we recommend using GPUs with Ampere or Hopper architecture, which support optimized kernels.</p>
<p>Ensure your system meets the following requirements:</p>
<ul>
<li><strong>CUDA Version</strong> : 12.1 or 12.3</li>
<li><strong>Python Version</strong> : &gt;=3.9 and &lt;=3.12</li>
</ul>
<p>VRAM Requirement for processing 1 million-token sequences:</p>
<ul>
<li><strong>zen-7B-Instruct-1M</strong> : At least 120GB VRAM (total across GPUs).</li>
<li><strong>zen-14B-Instruct-1M</strong> : At least 320GB VRAM (total across GPUs).</li>
</ul>
<p>If your GPUs do not have sufficient VRAM, you can still use zen-1M models for shorter tasks.</p>
<h3 class="flex scroll-m-28 flex-row items-center gap-2" id="2-install-dependencies"><a data-card="" href="#2-install-dependencies" class="peer">2. Install Dependencies</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h3>
<p>For now, you need to clone the vLLM repository from our custom branch and install it manually. We are working on getting our branch merged into the main vLLM project.</p>
<figure dir="ltr" class="my-4 bg-fd-card rounded-xl shiki relative border shadow-sm not-prose overflow-hidden text-sm shiki shiki-themes github-light github-dark" style="--shiki-light:#24292e;--shiki-dark:#e1e4e8;--shiki-light-bg:#fff;--shiki-dark-bg:#24292e" tabindex="-1"><div class="empty:hidden absolute top-3 right-2 z-2 backdrop-blur-lg rounded-lg text-fd-muted-foreground"><button type="button" class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors duration-100 disabled:pointer-events-none disabled:opacity-50 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-fd-ring p-1 [&amp;_svg]:size-4 hover:text-fd-accent-foreground data-checked:text-fd-accent-foreground" aria-label="Copy Text"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-clipboard" aria-hidden="true"><rect width="8" height="4" x="8" y="2" rx="1" ry="1"></rect><path d="M16 4h2a2 2 0 0 1 2 2v14a2 2 0 0 1-2 2H6a2 2 0 0 1-2-2V6a2 2 0 0 1 2-2h2"></path></svg></button></div><div role="region" tabindex="0" class="text-[0.8125rem] py-3.5 overflow-auto max-h-[600px] fd-scroll-container focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-inset focus-visible:ring-fd-ring" style="--padding-right:calc(var(--spacing) * 8)"><pre class="min-w-full w-max *:flex *:flex-col"><code><span class="line"></span>
<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0"> git</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF"> clone</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF"> -b</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF"> dev/dual-chunk-attn</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF"> [email protected]:QwenLM/vllm.git</span></span>
<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF"> cd</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF"> vllm</span></span>
<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0"> pip</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF"> install</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF"> -e</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF"> .</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF"> -v</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> </span></span></code></pre></div></figure>
<h3 class="flex scroll-m-28 flex-row items-center gap-2" id="3-launch-openai-compatible-api-service"><a data-card="" href="#3-launch-openai-compatible-api-service" class="peer">3. Launch OpenAI-Compatible API Service</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h3>
<p>Use the following command to start the service, configuring it based on your hardware setup:</p>
<figure dir="ltr" class="my-4 bg-fd-card rounded-xl shiki relative border shadow-sm not-prose overflow-hidden text-sm shiki shiki-themes github-light github-dark" style="--shiki-light:#24292e;--shiki-dark:#e1e4e8;--shiki-light-bg:#fff;--shiki-dark-bg:#24292e" tabindex="-1"><div class="empty:hidden absolute top-3 right-2 z-2 backdrop-blur-lg rounded-lg text-fd-muted-foreground"><button type="button" class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors duration-100 disabled:pointer-events-none disabled:opacity-50 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-fd-ring p-1 [&amp;_svg]:size-4 hover:text-fd-accent-foreground data-checked:text-fd-accent-foreground" aria-label="Copy Text"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-clipboard" aria-hidden="true"><rect width="8" height="4" x="8" y="2" rx="1" ry="1"></rect><path d="M16 4h2a2 2 0 0 1 2 2v14a2 2 0 0 1-2 2H6a2 2 0 0 1-2-2V6a2 2 0 0 1 2-2h2"></path></svg></button></div><div role="region" tabindex="0" class="text-[0.8125rem] py-3.5 overflow-auto max-h-[600px] fd-scroll-container focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-inset focus-visible:ring-fd-ring" style="--padding-right:calc(var(--spacing) * 8)"><pre class="min-w-full w-max *:flex *:flex-col"><code><span class="line"><span></span></span>
<span class="line"><span> vllm serve Qwen/zen-7B-Instruct-1M \</span></span>
<span class="line"><span> --tensor-parallel-size 4 \</span></span>
<span class="line"><span> --max-model-len 1010000 \</span></span>
<span class="line"><span> --enable-chunked-prefill --max-num-batched-tokens 131072 \</span></span>
<span class="line"><span> --enforce-eager \</span></span>
<span class="line"><span> --max-num-seqs 1</span></span>
<span class="line"><span> </span></span>
<span class="line"><span> # --quantization fp8 # Enabling FP8 quantization for model weights can reduce memory usage.</span></span>
<span class="line"><span> </span></span></code></pre></div></figure>
<p>If you encounter any issues, please refer to the <a href="https://huggingface.co/Qwen/zen-7B-Instruct-1M#troubleshooting" rel="noreferrer noopener" target="_blank">Troubleshooting</a> section for more information.</p>
<p><strong>Parameter Explanations:</strong></p>
<ul>
<li>
<p><strong><code>--tensor-parallel-size</code></strong></p>
<ul>
<li>Set to the number of GPUs you are using. Max 4 GPUs for the 7B model, and 8 GPUs for the 14B model.</li>
</ul>
</li>
<li>
<p><strong><code>--max-model-len</code></strong></p>
<ul>
<li>Defines the maximum input sequence length. Reduce this value if you encounter Out of Memory issues.</li>
</ul>
</li>
<li>
<p><strong><code>--max-num-batched-tokens</code></strong></p>
<ul>
<li>Sets the chunk size in Chunked Prefill. A smaller value reduces activation memory usage but may slow down inference.</li>
<li>Recommend 131072 for optimal performance.</li>
</ul>
</li>
<li>
<p><strong><code>--max-num-seqs</code></strong></p>
<ul>
<li>Limits concurrent sequences processed.</li>
</ul>
</li>
</ul>
<h3 class="flex scroll-m-28 flex-row items-center gap-2" id="4-interact-with-the-model"><a data-card="" href="#4-interact-with-the-model" class="peer">4. Interact with the Model</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h3>
<p>You can interact with the deployed model using one of the following methods:</p>
<p><strong>Option 1. Using Curl</strong></p>
<figure dir="ltr" class="my-4 bg-fd-card rounded-xl shiki relative border shadow-sm not-prose overflow-hidden text-sm shiki shiki-themes github-light github-dark" style="--shiki-light:#24292e;--shiki-dark:#e1e4e8;--shiki-light-bg:#fff;--shiki-dark-bg:#24292e" tabindex="-1"><div class="empty:hidden absolute top-3 right-2 z-2 backdrop-blur-lg rounded-lg text-fd-muted-foreground"><button type="button" class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors duration-100 disabled:pointer-events-none disabled:opacity-50 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-fd-ring p-1 [&amp;_svg]:size-4 hover:text-fd-accent-foreground data-checked:text-fd-accent-foreground" aria-label="Copy Text"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-clipboard" aria-hidden="true"><rect width="8" height="4" x="8" y="2" rx="1" ry="1"></rect><path d="M16 4h2a2 2 0 0 1 2 2v14a2 2 0 0 1-2 2H6a2 2 0 0 1-2-2V6a2 2 0 0 1 2-2h2"></path></svg></button></div><div role="region" tabindex="0" class="text-[0.8125rem] py-3.5 overflow-auto max-h-[600px] fd-scroll-container focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-inset focus-visible:ring-fd-ring" style="--padding-right:calc(var(--spacing) * 8)"><pre class="min-w-full w-max *:flex *:flex-col"><code><span class="line"></span>
<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0"> curl</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF"> http://localhost:8000/v1/chat/completions</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF"> \</span></span>
<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF"> -H</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF"> &quot;Content-Type: application/json&quot;</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF"> \</span></span>
<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF"> -d</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF"> &#x27;{</span></span>
<span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF"> &quot;model&quot;: &quot;Qwen/zen-7B-Instruct-1M&quot;,</span></span>
<span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF"> &quot;messages&quot;: [</span></span>
<span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF"> {&quot;role&quot;: &quot;user&quot;, &quot;content&quot;: &quot;Tell me something about large language models.&quot;}</span></span>
<span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF"> ],</span></span>
<span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF"> &quot;temperature&quot;: 0.7,</span></span>
<span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF"> &quot;top_p&quot;: 0.8,</span></span>
<span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF"> &quot;repetition_penalty&quot;: 1.05,</span></span>
<span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF"> &quot;max_tokens&quot;: 512</span></span>
<span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF"> }&#x27;</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> </span></span></code></pre></div></figure>
<p><strong>Option 2. Using Python</strong></p>
<figure dir="ltr" class="my-4 bg-fd-card rounded-xl shiki relative border shadow-sm not-prose overflow-hidden text-sm shiki shiki-themes github-light github-dark" style="--shiki-light:#24292e;--shiki-dark:#e1e4e8;--shiki-light-bg:#fff;--shiki-dark-bg:#24292e" tabindex="-1"><div class="empty:hidden absolute top-3 right-2 z-2 backdrop-blur-lg rounded-lg text-fd-muted-foreground"><button type="button" class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors duration-100 disabled:pointer-events-none disabled:opacity-50 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-fd-ring p-1 [&amp;_svg]:size-4 hover:text-fd-accent-foreground data-checked:text-fd-accent-foreground" aria-label="Copy Text"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-clipboard" aria-hidden="true"><rect width="8" height="4" x="8" y="2" rx="1" ry="1"></rect><path d="M16 4h2a2 2 0 0 1 2 2v14a2 2 0 0 1-2 2H6a2 2 0 0 1-2-2V6a2 2 0 0 1 2-2h2"></path></svg></button></div><div role="region" tabindex="0" class="text-[0.8125rem] py-3.5 overflow-auto max-h-[600px] fd-scroll-container focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-inset focus-visible:ring-fd-ring" style="--padding-right:calc(var(--spacing) * 8)"><pre class="min-w-full w-max *:flex *:flex-col"><code><span class="line"></span>
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583"> from</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> openai </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">import</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> OpenAI</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> </span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> openai_api_key </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF"> &quot;EMPTY&quot;</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> openai_api_base </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF"> &quot;http://localhost:8000/v1&quot;</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> </span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> client </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> OpenAI(</span></span>
<span class="line"><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70"> api_key</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">openai_api_key,</span></span>
<span class="line"><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70"> base_url</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">openai_api_base,</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> )</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> </span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> prompt </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> (</span></span>
<span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF"> &quot;There is an important info hidden inside a lot of irrelevant text. &quot;</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583"> +</span></span>
<span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF"> &quot;Find it and memorize it. I will quiz you about the important information there.</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF">\n\n</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF">&quot;</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583"> +</span></span>
<span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF"> &quot;The pass key is 28884. Remember it. 28884 is the pass key.</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF">\n</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF">&quot;</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583"> +</span></span>
<span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF"> &quot;The grass is green. The sky is blue. The sun is yellow. Here we go. There and back again. &quot;</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583"> *</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF"> 800</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583"> +</span></span>
<span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF"> &quot;</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF">\n</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF">What is the pass key?&quot;</span></span>
<span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D"> # The prompt is approximately 20k tokens long. You can try longer prompts by increasing the multiplier.</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> )</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> </span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> chat_response </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> client.chat.completions.create(</span></span>
<span class="line"><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70"> model</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF">&quot;Qwen/zen-7B-Instruct-1M&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">,</span></span>
<span class="line"><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70"> messages</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">[{</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF">&quot;role&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF">&quot;user&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">, </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF">&quot;content&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">: prompt}],</span></span>
<span class="line"><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70"> temperature</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF">0</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">,</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> )</span></span>
<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF"> print</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF">&quot;Chat response:&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">, chat_response.choices[</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF">0</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">].message.content)</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> </span></span></code></pre></div></figure>
<p><strong>Other Options</strong></p>
<p>For more advanced use cases, consider exploring frameworks like <a href="https://github.com/QwenLM/Qwen-Agent/tree/main" rel="noreferrer noopener" target="_blank">Qwen-Agent</a>, which enable the model to read PDF files and perform other specialized tasks.</p>
<h1 class="flex scroll-m-28 flex-row items-center gap-2" id="whats-next"><a data-card="" href="#whats-next" class="peer">What’s Next?</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h1>
<p>We recognize that long-context models still have a lot of room for improvement. Our goal is to build models that excel in both short and long-context tasks, making sure they bring real value to practical, long-context scenarios. We’re diving deep into more efficient training methods, model architectures, and inference methods to make them deployable effectively and perform exceptionally well even in environments with limited resources. We’re confident that all these efforts will open up a whole new world of possibilities for long-context models, expanding their use across a much broader range of applications. Stay tuned as we keep pushing the boundaries!</p></div></main><!--$--><!--/$--><script src="/_next/static/chunks/e83606e8fa9cc796.js" id="_R_" async=""></script><script>(self.__next_f=self.__next_f||[]).push([0])</script><script>self.__next_f.push([1,"1:\"$Sreact.fragment\"\n2:I[106,[\"/_next/static/chunks/59d0ad1b64f8544e.js\"],\"RootProvider\"]\n3:I[53113,[\"/_next/static/chunks/4d80e004cf4896dd.js\",\"/_next/static/chunks/350ee4303b732916.js\"],\"default\"]\n4:I[73211,[\"/_next/static/chunks/4d80e004cf4896dd.js\",\"/_next/static/chunks/350ee4303b732916.js\"],\"default\"]\n5:I[10086,[\"/_next/static/chunks/59d0ad1b64f8544e.js\",\"/_next/static/chunks/8de849ca74fc071f.js\",\"/_next/static/chunks/e62b91212ee7f8ff.js\",\"/_next/static/chunks/f19fe44237e54646.js\",\"/_next/static/chunks/cb0a883bafeb6805.js\"],\"\"]\n7:I[89923,[\"/_next/static/chunks/4d80e004cf4896dd.js\",\"/_next/static/chunks/350ee4303b732916.js\"],\"OutletBoundary\"]\n8:\"$Sreact.suspense\"\na:I[89923,[\"/_next/static/chunks/4d80e004cf4896dd.js\",\"/_next/static/chunks/350ee4303b732916.js\"],\"ViewportBoundary\"]\nc:I[89923,[\"/_next/static/chunks/4d80e004cf4896dd.js\",\"/_next/static/chunks/350ee4303b732916.js\"],\"MetadataBoundary\"]\ne:I[6998,[\"/_next/static/chunks/4d80e004cf4896dd.js\",\"/_next/static/chunks/350ee4303b732916.js\"],\"default\"]\n:HL[\"/_next/static/chunks/f2332aac77592f9d.css\",\"style\"]\n"])</script><script>self.__next_f.push([1,"0:{\"P\":null,\"b\":\"i-dnJM_MIpJSOCQWNJVMq\",\"c\":[\"\",\"blog\",\"qwen2.5-1m\"],\"q\":\"\",\"i\":false,\"f\":[[[\"\",{\"children\":[\"blog\",{\"children\":[[\"slug\",\"qwen2.5-1m\",\"d\"],{\"children\":[\"__PAGE__\",{}]}]}]},\"$undefined\",\"$undefined\",true],[[\"$\",\"$1\",\"c\",{\"children\":[[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/chunks/f2332aac77592f9d.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\",\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-0\",{\"src\":\"/_next/static/chunks/59d0ad1b64f8544e.js\",\"async\":true,\"nonce\":\"$undefined\"}]],[\"$\",\"html\",null,{\"lang\":\"en\",\"className\":\"dark\",\"suppressHydrationWarning\":true,\"children\":[\"$\",\"body\",null,{\"className\":\"antialiased\",\"children\":[\"$\",\"$L2\",null,{\"children\":[\"$\",\"$L3\",null,{\"parallelRouterKey\":\"children\",\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L4\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":[[\"$\",\"main\",null,{\"className\":\"flex min-h-screen flex-col items-center justify-center px-4 text-center\",\"children\":[[\"$\",\"div\",null,{\"className\":\"mb-8 opacity-20\",\"children\":[\"$\",\"svg\",null,{\"width\":\"120\",\"height\":\"120\",\"viewBox\":\"0 0 120 120\",\"fill\":\"none\",\"aria-hidden\":\"true\",\"children\":[\"$\",\"circle\",null,{\"cx\":\"60\",\"cy\":\"60\",\"r\":\"50\",\"stroke\":\"currentColor\",\"strokeWidth\":\"3\",\"strokeLinecap\":\"round\",\"strokeDasharray\":\"280 40\"}]}]}],[\"$\",\"p\",null,{\"className\":\"text-xs font-mono uppercase tracking-[0.3em] text-fd-muted-foreground mb-4\",\"children\":\"404\"}],[\"$\",\"h1\",null,{\"className\":\"text-3xl font-semibold mb-3\",\"children\":\"Page not found\"}],[\"$\",\"p\",null,{\"className\":\"text-fd-muted-foreground max-w-sm mb-10\",\"children\":\"This page doesn't exist, or it may have moved. Try the documentation or head home.\"}],[\"$\",\"div\",null,{\"className\":\"flex flex-wrap gap-3 justify-center\",\"children\":[[\"$\",\"$L5\",null,{\"href\":\"/\",\"className\":\"inline-flex items-center gap-2 rounded-lg bg-fd-primary text-fd-primary-foreground px-4 py-2 text-sm font-medium hover:opacity-90 transition\",\"children\":\"Go home\"}],[\"$\",\"$L5\",null,{\"href\":\"/docs\",\"className\":\"inline-flex items-center gap-2 rounded-lg border border-fd-border bg-fd-background px-4 py-2 text-sm font-medium hover:bg-fd-muted transition\",\"children\":\"Documentation\"}],[\"$\",\"$L5\",null,{\"href\":\"/docs/models\",\"className\":\"inline-flex items-center gap-2 rounded-lg border border-fd-border bg-fd-background px-4 py-2 text-sm font-medium hover:bg-fd-muted transition\",\"children\":\"Browse models\"}]]}],[\"$\",\"p\",null,{\"className\":\"mt-16 text-xs font-mono text-fd-muted-foreground opacity-50\",\"children\":\"zenlm.org\"}]]}],[]],\"forbidden\":\"$undefined\",\"unauthorized\":\"$undefined\"}]}]}]}]]}],{\"children\":[[\"$\",\"$1\",\"c\",{\"children\":[null,[\"$\",\"$L3\",null,{\"parallelRouterKey\":\"children\",\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L4\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":\"$undefined\",\"forbidden\":\"$undefined\",\"unauthorized\":\"$undefined\"}]]}],{\"children\":[[\"$\",\"$1\",\"c\",{\"children\":[null,[\"$\",\"$L3\",null,{\"parallelRouterKey\":\"children\",\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L4\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":\"$undefined\",\"forbidden\":\"$undefined\",\"unauthorized\":\"$undefined\"}]]}],{\"children\":[[\"$\",\"$1\",\"c\",{\"children\":[\"$L6\",[[\"$\",\"script\",\"script-0\",{\"src\":\"/_next/static/chunks/8de849ca74fc071f.js\",\"async\":true,\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-1\",{\"src\":\"/_next/static/chunks/e62b91212ee7f8ff.js\",\"async\":true,\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-2\",{\"src\":\"/_next/static/chunks/f19fe44237e54646.js\",\"async\":true,\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-3\",{\"src\":\"/_next/static/chunks/cb0a883bafeb6805.js\",\"async\":true,\"nonce\":\"$undefined\"}]],[\"$\",\"$L7\",null,{\"children\":[\"$\",\"$8\",null,{\"name\":\"Next.MetadataOutlet\",\"children\":\"$@9\"}]}]]}],{},null,false,false]},null,false,false]},null,false,false]},null,false,false],[\"$\",\"$1\",\"h\",{\"children\":[null,[\"$\",\"$La\",null,{\"children\":\"$Lb\"}],[\"$\",\"div\",null,{\"hidden\":true,\"children\":[\"$\",\"$Lc\",null,{\"children\":[\"$\",\"$8\",null,{\"name\":\"Next.Metadata\",\"children\":\"$Ld\"}]}]}],null]}],false]],\"m\":\"$undefined\",\"G\":[\"$e\",[]],\"S\":true}\n"])</script><script>self.__next_f.push([1,"f:I[48068,[\"/_next/static/chunks/59d0ad1b64f8544e.js\",\"/_next/static/chunks/8de849ca74fc071f.js\",\"/_next/static/chunks/e62b91212ee7f8ff.js\",\"/_next/static/chunks/f19fe44237e54646.js\",\"/_next/static/chunks/cb0a883bafeb6805.js\"],\"default\"]\n"])</script><script>self.__next_f.push([1,"6:[\"$\",\"main\",null,{\"className\":\"mx-auto w-full max-w-2xl px-4 py-16\",\"children\":[[\"$\",\"$L5\",null,{\"href\":\"/blog\",\"className\":\"inline-flex items-center gap-1.5 text-sm text-fd-muted-foreground hover:text-fd-foreground mb-10 transition-colors\",\"children\":\"← Back to Blog\"}],[\"$\",\"div\",null,{\"className\":\"mb-8\",\"children\":[[\"$\",\"time\",null,{\"className\":\"text-xs font-mono text-fd-muted-foreground uppercase tracking-wider\",\"children\":\"January 26, 2025\"}],[\"$\",\"h1\",null,{\"className\":\"text-3xl font-bold mt-2 mb-3\",\"children\":\"zen-1M: Deploy Your Own Qwen with Context Length up to 1M Tokens\"}],[\"$\",\"p\",null,{\"className\":\"text-fd-muted-foreground text-lg mb-4\",\"children\":\"Tech Report HuggingFace ModelScope Qwen Chat HuggingFace Demo ModelScope Demo DISCORD\"}],[\"$\",\"div\",null,{\"className\":\"flex items-center gap-3 pt-4 border-t border-fd-border\",\"children\":[[\"$\",\"span\",null,{\"className\":\"text-sm text-fd-muted-foreground\",\"children\":[\"By \",\"Zen LM Team\"]}],[\"$\",\"div\",null,{\"className\":\"flex gap-1.5 ml-auto\",\"children\":[]}]]}]]}],[\"$\",\"div\",null,{\"className\":\"prose dark:prose-invert max-w-none\",\"children\":[[\"$\",\"p\",null,{\"children\":[[\"$\",\"$Lf\",null,{\"href\":\"https://qianwen-res.oss-cn-beijing.aliyuncs.com/zen-1M/zen_1M_Technical_Report.pdf\",\"children\":\"Tech Report\"}],\" \",[\"$\",\"$Lf\",null,{\"href\":\"https://huggingface.co/Qwen\",\"children\":\"HuggingFace\"}],\" \",[\"$\",\"$Lf\",null,{\"href\":\"https://modelscope.cn/organization/qwen\",\"children\":\"ModelScope\"}],\" \",[\"$\",\"$Lf\",null,{\"href\":\"https://chat.qwenlm.ai/\",\"children\":\"Qwen Chat\"}],\" \",[\"$\",\"$Lf\",null,{\"href\":\"https://huggingface.co/spaces/Qwen/zen-1M-Demo\",\"children\":\"HuggingFace Demo\"}],\" \",[\"$\",\"$Lf\",null,{\"href\":\"https://www.modelscope.cn/studios/Qwen/zen-1M-Demo\",\"children\":\"ModelScope Demo\"}],\" \",[\"$\",\"$Lf\",null,{\"href\":\"https://discord.gg/yPEP2vHTu4\",\"children\":\"DISCORD\"}]]}],\"\\n\",[\"$\",\"h1\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"introduction\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#introduction\",\"className\":\"peer\",\"children\":\"Introduction\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}],\"\\n\",[\"$\",\"p\",null,{\"children\":[\"Two months after upgrading \",[\"$\",\"$Lf\",null,{\"href\":\"../qwen3-turbo\",\"children\":\"zen-Turbo\"}],\" to support context length up to one million tokens, we are back with the open-source zen-1M models and the corresponding inference framework support. Here’s what you can expect from this release:\"]}],\"\\n\",[\"$\",\"ol\",null,{\"children\":[\"\\n\",[\"$\",\"li\",null,{\"children\":[\"\\n\",[\"$\",\"p\",null,{\"children\":[[\"$\",\"strong\",null,{\"children\":\"Opensource Models:\"}],\" We’re releasing two new checkpoints, \",[\"$\",\"strong\",null,{\"children\":\"zen-7B-Instruct-1M\"}],\" and \",[\"$\",\"strong\",null,{\"children\":\"zen-14B-Instruct-1M\"}],\" , marking the first time we’ve upgraded our opensource Qwen models to handle 1M-token contexts.\"]}],\"\\n\"]}],\"\\n\",[\"$\",\"li\",null,{\"children\":[\"\\n\",[\"$\",\"p\",null,{\"children\":[[\"$\",\"strong\",null,{\"children\":\"Inference Framework:\"}],\" To help developers deploy the zen-1M series models more efficiently, we’ve fully open-sourced our inference framework based on \",[\"$\",\"$Lf\",null,{\"href\":\"https://github.com/vllm-project/vllm\",\"children\":\"vLLM\"}],\". With integration with sparse attention methods, our framework can process 1M-token inputs \",[\"$\",\"strong\",null,{\"children\":\"3x to 7x\"}],\" faster.\"]}],\"\\n\"]}],\"\\n\",[\"$\",\"li\",null,{\"children\":[\"\\n\",[\"$\",\"p\",null,{\"children\":[[\"$\",\"strong\",null,{\"children\":[[\"$\",\"$Lf\",null,{\"href\":\"https://qianwen-res.oss-cn-beijing.aliyuncs.com/zen-1M/zen_1M_Technical_Report.pdf\",\"children\":\"Technical Report\"}],\":\"]}],\" We’re also sharing the technical details behind the zen-1M series, including design insights for training and inference frameworks, as well as ablation experiments.\"]}],\"\\n\"]}],\"\\n\"]}],\"\\n\",\"$L10\",\"\\n\",\"$L11\",\"\\n\",\"$L12\",\"\\n\",\"$L13\",\"\\n\",\"$L14\",\"\\n\",\"$L15\",\"\\n\",\"$L16\",\"\\n\",\"$L17\",\"\\n\",\"$L18\",\"\\n\",\"$L19\",\"\\n\",\"$L1a\",\"\\n\",\"$L1b\",\"\\n\",\"$L1c\",\"\\n\",\"$L1d\",\"\\n\",\"$L1e\",\"\\n\",\"$L1f\",\"\\n\",\"$L20\",\"\\n\",\"$L21\",\"\\n\",\"$L22\",\"\\n\",\"$L23\",\"\\n\",\"$L24\",\"\\n\",\"$L25\",\"\\n\",\"$L26\",\"\\n\",\"$L27\",\"\\n\",\"$L28\",\"\\n\",\"$L29\",\"\\n\",\"$L2a\",\"\\n\",\"$L2b\",\"\\n\",\"$L2c\",\"\\n\",\"$L2d\",\"\\n\",\"$L2e\",\"\\n\",\"$L2f\",\"\\n\",\"$L30\",\"\\n\",\"$L31\",\"\\n\",\"$L32\",\"\\n\",\"$L33\",\"\\n\",\"$L34\",\"\\n\",\"$L35\",\"\\n\",\"$L36\",\"\\n\",\"$L37\",\"\\n\",\"$L38\",\"\\n\",\"$L39\",\"\\n\",\"$L3a\",\"\\n\",\"$L3b\",\"\\n\",\"$L3c\",\"\\n\",\"$L3d\",\"\\n\",\"$L3e\",\"\\n\",\"$L3f\",\"\\n\",\"$L40\",\"\\n\",\"$L41\",\"\\n\",\"$L42\",\"\\n\",\"$L43\",\"\\n\",\"$L44\",\"\\n\",\"$L45\",\"\\n\",\"$L46\",\"\\n\",\"$L47\"]}]]}]\n"])</script><script>self.__next_f.push([1,"48:I[51504,[\"/_next/static/chunks/59d0ad1b64f8544e.js\",\"/_next/static/chunks/8de849ca74fc071f.js\",\"/_next/static/chunks/e62b91212ee7f8ff.js\",\"/_next/static/chunks/f19fe44237e54646.js\",\"/_next/static/chunks/cb0a883bafeb6805.js\"],\"CodeBlock\"]\n49:I[51504,[\"/_next/static/chunks/59d0ad1b64f8544e.js\",\"/_next/static/chunks/8de849ca74fc071f.js\",\"/_next/static/chunks/e62b91212ee7f8ff.js\",\"/_next/static/chunks/f19fe44237e54646.js\",\"/_next/static/chunks/cb0a883bafeb6805.js\"],\"Pre\"]\n10:[\"$\",\"p\",null,{\"children\":[\"You can experience zen-1M models online by visiting our demo on \",[\"$\",\"$Lf\",null,{\"href\":\"https://huggingface.co/spaces/Qwen/zen-1M-Demo\",\"children\":\"Huggingface\"}],\" and \",[\"$\",\"$Lf\",null,{\"href\":\"https://www.modelscope.cn/studios/Qwen/zen-1M-Demo\",\"children\":\"Modelscope\"}],\".\"]}]\n11:[\"$\",\"p\",null,{\"children\":[\"Additionally, we recently introduced \",[\"$\",\"strong\",null,{\"children\":[\"$\",\"$Lf\",null,{\"href\":\"https://chat.qwenlm.ai/\",\"children\":\"Qwen Chat\"}]}],\" , an advanced AI assistant from the Qwen series. With Qwen Chat, you can engage in conversations, write code, perform searches, generate images and videos, and utilize various tools. Notably, Qwen Chat also features the zen-Turbo model, which supports long-context processing with a context length of up to 1M tokens.\"]}]\n12:[\"$\",\"h1\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"model-performance\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#model-performance\",\"className\":\"peer\",\"children\":\"Model Performance\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n13:[\"$\",\"p\",null,{\"children\":\"Let’s start by diving into the performance of the zen-1M series models, covering both long-context and short text tasks.\"}]\n14:[\"$\",\"h2\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"long-context-tasks\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#long-context-tasks\",\"className\":\"peer\",\"children\":\"Long-Context Tasks\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n15:[\"$\",\"p\",null,{\"children\":\"First off, we evaluate the zen-1M models on the Passkey Retrieval task with a context length of 1 million tokens. The results show that these models can accurately retrieve hidden information from documents containing up to 1M tokens, with only minor errors observed in the 7B model.\"}]\n16:[\"$\",\"p\",null,{\"children\":[\"For more complex long-context understanding tasks, we select \",[\"$\",\"$Lf\",null,{\"href\":\"https://github.com/hsiehjackson/RULER\",\"children\":\"RULER\"}],\", \",[\"$\",\"$Lf\",null,{\"href\":\"https://github.com/infinigence/LVEval\",\"children\":\"LV-Eval\"}],\", \",[\"$\",\"$Lf\",null,{\"href\":\"https://github.com/THUDM/LongAlign\",\"children\":\"LongbenchChat\"}],\" used in \",[\"$\",\"$Lf\",null,{\"href\":\"../qwen3-turbo/#more-complex-long-text-tasks\",\"children\":\"this blog\"}],\".\"]}]\n17:[\"$\",\"p\",null,{\"children\":\"From these results, we can draw a few key conclusions:\"}]\n18:[\"$\",\"ul\",null,{\"children\":[\"\\n\",[\"$\",\"li\",null,{\"children\":[[\"$\",\"strong\",null,{\"children\":\"Significantly Superior to the 128k Version:\"}],\" The"])</script><script>self.__next_f.push([1," zen-1M series models significantly outperform their 128K counterparts in most long-context tasks, especially for sequences exceeding 64K in length.\"]}],\"\\n\",[\"$\",\"li\",null,{\"children\":[[\"$\",\"strong\",null,{\"children\":\"Notable Performance Advantage:\"}],\" The zen-14B-Instruct-1M model not only beats zen-Turbo but also consistently outperforms GPT-4o-mini across multiple datasets, offering a robust open-source alternative for long-context tasks.\"]}],\"\\n\"]}]\n19:[\"$\",\"h2\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"short-context-tasks\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#short-context-tasks\",\"className\":\"peer\",\"children\":\"Short-Context Tasks\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n1a:[\"$\",\"p\",null,{\"children\":\"Besides performance on long sequences, we’re equally interested in how these models handle short sequences. So, we compare the zen-1M models and their 128K versions on widely used academic benchmarks, throwing in GPT-4o-mini for comparison.\"}]\n1b:[\"$\",\"p\",null,{\"children\":\"Here’s what we find:\"}]\n1c:[\"$\",\"ul\",null,{\"children\":[\"\\n\",[\"$\",\"li\",null,{\"children\":\"Both zen-7B-Instruct-1M and zen-14B-Instruct-1M maintain performance on short text tasks that is similar to their 128K versions, ensuring the fundamental capabilities haven’t been compromised by the addition of long-sequence processing abilities.\"}],\"\\n\",[\"$\",\"li\",null,{\"children\":\"Compared to GPT-4o-mini, both zen-14B-Instruct-1M and zen-Turbo achieve similar performance on short text tasks while supporting a context length that’s eight times longer.\"}],\"\\n\"]}]\n1d:[\"$\",\"h1\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"key-techniques\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#key-techniques\",\"className\":\"peer\",\"children\":\"Key Techniques\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n1e:[\"$\",\"p\",null,{\"children\":[\"Here, we’ll briefly introduce the key techniques behind building zen-1M. For more details, please check out our \",[\"$\",\"$Lf\",null,{\"href\":\"https://qianwen-res.oss-cn-beijing.aliyuncs.com/zen-1M/zen_1M_Technical_Report.pdf\",\"children\":\"technical report\"}],\".\"]}]\n1f:[\"$\",\"h2\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"long-context-training\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#long-context-training\",\"className\":\"peer\",\"children\":\"Long-Context Training\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n20:[\"$\",\"p\",null,{\"children\":\"Training with long se"])</script><script>self.__next_f.push([1,"quences demands substantial computational resources, so we adopt a progressive approach to expand the context length for zen-1M through multiple stages:\"}]\n"])</script><script>self.__next_f.push([1,"21:[\"$\",\"ul\",null,{\"children\":[\"\\n\",[\"$\",\"li\",null,{\"children\":\"We begin with an intermediate checkpoint of pre-trained zen, which had a 4K token context length.\"}],\"\\n\",[\"$\",\"li\",null,{\"children\":[[\"$\",\"strong\",null,{\"children\":\"In Pretraining\"}],\" , we gradually increase the context length from 4K to 256K tokens while using \",[\"$\",\"$Lf\",null,{\"href\":\"https://arxiv.org/abs/2309.16039\",\"children\":\"Adjusted Base Frequency\"}],\", raising the RoPE base from 10,000 to 10,000,000.\"]}],\"\\n\",[\"$\",\"li\",null,{\"children\":[[\"$\",\"strong\",null,{\"children\":\"In Supervised Fine-tuning\"}],\" , we split this into two stages to preserve performance on shorter sequences:\\n\",[\"$\",\"ul\",null,{\"children\":[\"\\n\",[\"$\",\"li\",null,{\"children\":[[\"$\",\"strong\",null,{\"children\":\"Stage 1:\"}],\" Fine-tuned only on short instructions (up to 32K tokens) using the same data and steps as the 128K versions of zen.\"]}],\"\\n\",[\"$\",\"li\",null,{\"children\":[[\"$\",\"strong\",null,{\"children\":\"Stage 2:\"}],\" Mixed short (up to 32K) and long (up to 256K) instructions to enhance long-context task performance while maintaining short-task quality.\"]}],\"\\n\"]}],\"\\n\"]}],\"\\n\",[\"$\",\"li\",null,{\"children\":[[\"$\",\"strong\",null,{\"children\":\"In Reinforcement Learning\"}],\" , we train models on short texts up to 8K tokens, which sufficiently improves alignment with human preferences and generalizes well to long-context tasks.\"]}],\"\\n\"]}]\n"])</script><script>self.__next_f.push([1,"22:[\"$\",\"p\",null,{\"children\":\"The final instruction-tuned models are capable of handling sequences up to 256K tokens.\"}]\n23:[\"$\",\"h2\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"length-extrapolation\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#length-extrapolation\",\"className\":\"peer\",\"children\":\"Length Extrapolation\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n24:[\"$\",\"p\",null,{\"children\":\"During training, we develop an instruction-tuned model with a context length of 256K tokens. To extend this to 1M tokens, we employ length extrapolation techniques.\"}]\n25:[\"$\",\"p\",null,{\"children\":[\"The degradation of LLMs based on RoPE in long-context tasks is mainly due to unseen, large relative positional distances between queries and keys in computing attention weight. We employ \",[\"$\",\"$Lf\",null,{\"href\":\"https://arxiv.org/abs/2402.17463\",\"children\":[\"$\",\"strong\",null,{\"children\":\"Dual Chunk Attention\"}]}],\" (DCA), which addresses this issue by remapping relative positions to smaller values, avoiding the large distances not seen during training.\"]}]\n26:[\"$\",\"p\",null,{\"children\":\"We evaluat the zen-1M models and their 128K counterparts with and without the length extrapolation method. We can find:\"}]\n27:[\"$\",\"p\",null,{\"children\":\"Even models trained on just 32K tokens, such as the zen-7B-Instruct, achieve nearly perfect accuracy in passkey retrieval tasks with 1M-token contexts. This underscores the remarkable ability of DCA to extend supported context lengths, without any training required.\"}]\n28:[\"$\",\"h2\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"sparse-attention\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#sparse-attention\",\"className\":\"peer\",\"children\":\"Sparse Attention\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n29:[\"$\",\"p\",null,{\"children\":[\"For long-context language models, inference speed is crucial for user experience. We introduce a sparse attention mechanism based on \",[\"$\",\"$Lf\",null,{\"href\":\"https://arxiv.org/abs/2407.02490\",\"children\":[\"$\",\"strong\",null,{\"children\":\"MInference\"}]}],\" to accelerate the prefill phase. Furthermore, we propose several improvements:\"]}]\n"])</script><script>self.__next_f.push([1,"2a:[\"$\",\"ul\",null,{\"children\":[\"\\n\",[\"$\",\"li\",null,{\"children\":[\"\\n\",[\"$\",\"p\",null,{\"children\":[[\"$\",\"strong\",null,{\"children\":\"Integrating with Chunked Prefill:\"}],\" Directly processing sequences of 1M tokens results in substantial memory overhead to store the activations in MLP layers, consuming 71GB of VRAM in zen-7B. By integrating with chunk prefill with a chunk length of 32,768 tokens, activation VRAM usage is reduced by 96.7%, leading to a significant decrease in memory consumption.\"]}],\"\\n\"]}],\"\\n\",[\"$\",\"li\",null,{\"children\":[\"\\n\",[\"$\",\"p\",null,{\"children\":[[\"$\",\"strong\",null,{\"children\":\"Integrating with Length Extrapolation:\"}],\" We integrate DCA with MInference in long-context processing, thereby enhancing inference efficiency and achieving greater accuracy.\"]}],\"\\n\"]}],\"\\n\",[\"$\",\"li\",null,{\"children\":[\"\\n\",[\"$\",\"p\",null,{\"children\":[[\"$\",\"strong\",null,{\"children\":\"Sparsity Refinement on Long Sequences:\"}],\" MInference requires an offline search to determine the optimal sparsification configuration for each attention head. Due to the computational demand of full attention weights, this search is typically conducted on short sequences, which may not generalize well to longer sequences. We developed a method to refine the sparsification configuration specifically for sequences up to 1M tokens, which significantly reduces the accuracy loss brought by sparse attention.\"]}],\"\\n\"]}],\"\\n\",[\"$\",\"li\",null,{\"children\":[\"\\n\",[\"$\",\"p\",null,{\"children\":[[\"$\",\"strong\",null,{\"children\":\"More Optimizations:\"}],\" We introduce additional optimizations, such as enhanced kernel efficiency and dynamic chunked pipeline parallelism, to fully unlock the potential of the entire framework.\"]}],\"\\n\"]}],\"\\n\"]}]\n"])</script><script>self.__next_f.push([1,"2b:[\"$\",\"p\",null,{\"children\":\"With these enhancements, our inference framework results in a 3.2x to 6.7x acceleration in the prefill speed across different model sizes and GPU devices for sequences of 1M token length.\"}]\n2c:[\"$\",\"h1\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"deploy-zen-1m-models-locally\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#deploy-zen-1m-models-locally\",\"className\":\"peer\",\"children\":\"Deploy zen-1M Models Locally\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n2d:[\"$\",\"p\",null,{\"children\":\"Here we provide step-by-step instructions for deploying the zen-1M models on your local devices.\"}]\n2e:[\"$\",\"h3\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"1-system-preparation\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#1-system-preparation\",\"className\":\"peer\",\"children\":\"1. System Preparation\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n2f:[\"$\",\"p\",null,{\"children\":\"To achieve the best performance, we recommend using GPUs with Ampere or Hopper architecture, which support optimized kernels.\"}]\n30:[\"$\",\"p\",null,{\"children\":\"Ensure your system meets the following requirements:\"}]\n31:[\"$\",\"ul\",null,{\"children\":[\"\\n\",[\"$\",\"li\",null,{\"children\":[[\"$\",\"strong\",null,{\"children\":\"CUDA Version\"}],\" : 12.1 or 12.3\"]}],\"\\n\",[\"$\",\"li\",null,{\"children\":[[\"$\",\"strong\",null,{\"children\":\"Python Version\"}],\" : \u003e=3.9 and \u003c=3.12\"]}],\"\\n\"]}]\n32:[\"$\",\"p\",null,{\"children\":\"VRAM Requirement for processing 1 million-token sequences:\"}]\n33:[\"$\",\"ul\",null,{\"children\":[\"\\n\",[\"$\",\"li\",null,{\"children\":[[\"$\",\"strong\",null,{\"children\":\"zen-7B-Instruct-1M\"}],\" : At least 120GB VRAM (total across GPUs).\"]}],\"\\n\",[\"$\",\"li\",null,{\"children\":[[\"$\",\"strong\",null,{\"children\":\"zen-14B-Instruct-1M\"}],\" : At least 320GB VRAM (total across GPUs).\"]}],\"\\n\"]}]\n34:[\"$\",\"p\",null,{\"children\":\"If your GPUs do not have sufficient VRAM, you can still use zen-1M models for shorter tasks.\"}]\n35:[\"$\",\"h3\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"2-install-dependencies\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#2-install-dependencies\",\"className\":\"peer\",\"children\":\"2. Install Dependencies\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n36:[\"$\",\"p\",null,{\"children\":\"For now, you need to clone the vLLM repository from our custom branch and install it manually. We are working on getting our branch merged into the main vLLM project.\"}]\n"])</script><script>self.__next_f.push([1,"37:[\"$\",\"$L48\",null,{\"className\":\"shiki shiki-themes github-light github-dark\",\"style\":{\"--shiki-light\":\"#24292e\",\"--shiki-dark\":\"#e1e4e8\",\"--shiki-light-bg\":\"#fff\",\"--shiki-dark-bg\":\"#24292e\"},\"tabIndex\":\"0\",\"icon\":\"\u003csvg viewBox=\\\"0 0 24 24\\\"\u003e\u003cpath d=\\\"m 4,4 a 1,1 0 0 0 -0.7070312,0.2929687 1,1 0 0 0 0,1.4140625 L 8.5859375,11 3.2929688,16.292969 a 1,1 0 0 0 0,1.414062 1,1 0 0 0 1.4140624,0 l 5.9999998,-6 a 1.0001,1.0001 0 0 0 0,-1.414062 L 4.7070312,4.2929687 A 1,1 0 0 0 4,4 Z m 8,14 a 1,1 0 0 0 -1,1 1,1 0 0 0 1,1 h 8 a 1,1 0 0 0 1,-1 1,1 0 0 0 -1,-1 z\\\" fill=\\\"currentColor\\\" /\u003e\u003c/svg\u003e\",\"children\":[\"$\",\"$L49\",null,{\"children\":[\"$\",\"code\",null,{\"children\":[[\"$\",\"span\",null,{\"className\":\"line\"}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#6F42C1\",\"--shiki-dark\":\"#B392F0\"},\"children\":\" git\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#032F62\",\"--shiki-dark\":\"#9ECBFF\"},\"children\":\" clone\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#005CC5\",\"--shiki-dark\":\"#79B8FF\"},\"children\":\" -b\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#032F62\",\"--shiki-dark\":\"#9ECBFF\"},\"children\":\" dev/dual-chunk-attn\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#032F62\",\"--shiki-dark\":\"#9ECBFF\"},\"children\":\" [email protected]:QwenLM/vllm.git\"}]]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#005CC5\",\"--shiki-dark\":\"#79B8FF\"},\"children\":\" cd\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#032F62\",\"--shiki-dark\":\"#9ECBFF\"},\"children\":\" vllm\"}]]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#6F42C1\",\"--shiki-dark\":\"#B392F0\"},\"children\":\" pip\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#032F62\",\"--shiki-dark\":\"#9ECBFF\"},\"children\":\" install\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#005CC5\",\"--shiki-dark\":\"#79B8FF\"},\"children\":\" -e\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#032F62\",\"--shiki-dark\":\"#9ECBFF\"},\"children\":\" .\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#005CC5\",\"--shiki-dark\":\"#79B8FF\"},\"children\":\" -v\"}]]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" \"}]}]]}]}]}]\n"])</script><script>self.__next_f.push([1,"38:[\"$\",\"h3\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"3-launch-openai-compatible-api-service\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#3-launch-openai-compatible-api-service\",\"className\":\"peer\",\"children\":\"3. Launch OpenAI-Compatible API Service\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n39:[\"$\",\"p\",null,{\"children\":\"Use the following command to start the service, configuring it based on your hardware setup:\"}]\n"])</script><script>self.__next_f.push([1,"3a:[\"$\",\"$L48\",null,{\"className\":\"shiki shiki-themes github-light github-dark\",\"style\":{\"--shiki-light\":\"#24292e\",\"--shiki-dark\":\"#e1e4e8\",\"--shiki-light-bg\":\"#fff\",\"--shiki-dark-bg\":\"#24292e\"},\"tabIndex\":\"0\",\"icon\":\"\u003csvg viewBox=\\\"0 0 24 24\\\"\u003e\u003cpath d=\\\"M 6,1 C 4.354992,1 3,2.354992 3,4 v 16 c 0,1.645008 1.354992,3 3,3 h 12 c 1.645008,0 3,-1.354992 3,-3 V 8 7 A 1.0001,1.0001 0 0 0 20.707031,6.2929687 l -5,-5 A 1.0001,1.0001 0 0 0 15,1 h -1 z m 0,2 h 7 v 3 c 0,1.645008 1.354992,3 3,3 h 3 v 11 c 0,0.564129 -0.435871,1 -1,1 H 6 C 5.4358712,21 5,20.564129 5,20 V 4 C 5,3.4358712 5.4358712,3 6,3 Z M 15,3.4140625 18.585937,7 H 16 C 15.435871,7 15,6.5641288 15,6 Z\\\" fill=\\\"currentColor\\\" /\u003e\u003c/svg\u003e\",\"children\":[\"$\",\"$L49\",null,{\"children\":[\"$\",\"code\",null,{\"children\":[[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{}]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"children\":\" vllm serve Qwen/zen-7B-Instruct-1M \\\\\"}]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"children\":\" --tensor-parallel-size 4 \\\\\"}]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"children\":\" --max-model-len 1010000 \\\\\"}]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"children\":\" --enable-chunked-prefill --max-num-batched-tokens 131072 \\\\\"}]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"children\":\" --enforce-eager \\\\\"}]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"children\":\" --max-num-seqs 1\"}]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"children\":\" \"}]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"children\":\" # --quantization fp8 # Enabling FP8 quantization for model weights can reduce memory usage.\"}]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"children\":\" \"}]}]]}]}]}]\n"])</script><script>self.__next_f.push([1,"3b:[\"$\",\"p\",null,{\"children\":[\"If you encounter any issues, please refer to the \",[\"$\",\"$Lf\",null,{\"href\":\"https://huggingface.co/Qwen/zen-7B-Instruct-1M#troubleshooting\",\"children\":\"Troubleshooting\"}],\" section for more information.\"]}]\n3c:[\"$\",\"p\",null,{\"children\":[\"$\",\"strong\",null,{\"children\":\"Parameter Explanations:\"}]}]\n"])</script><script>self.__next_f.push([1,"3d:[\"$\",\"ul\",null,{\"children\":[\"\\n\",[\"$\",\"li\",null,{\"children\":[\"\\n\",[\"$\",\"p\",null,{\"children\":[\"$\",\"strong\",null,{\"children\":[\"$\",\"code\",null,{\"children\":\"--tensor-parallel-size\"}]}]}],\"\\n\",[\"$\",\"ul\",null,{\"children\":[\"\\n\",[\"$\",\"li\",null,{\"children\":\"Set to the number of GPUs you are using. Max 4 GPUs for the 7B model, and 8 GPUs for the 14B model.\"}],\"\\n\"]}],\"\\n\"]}],\"\\n\",[\"$\",\"li\",null,{\"children\":[\"\\n\",[\"$\",\"p\",null,{\"children\":[\"$\",\"strong\",null,{\"children\":[\"$\",\"code\",null,{\"children\":\"--max-model-len\"}]}]}],\"\\n\",[\"$\",\"ul\",null,{\"children\":[\"\\n\",[\"$\",\"li\",null,{\"children\":\"Defines the maximum input sequence length. Reduce this value if you encounter Out of Memory issues.\"}],\"\\n\"]}],\"\\n\"]}],\"\\n\",[\"$\",\"li\",null,{\"children\":[\"\\n\",[\"$\",\"p\",null,{\"children\":[\"$\",\"strong\",null,{\"children\":[\"$\",\"code\",null,{\"children\":\"--max-num-batched-tokens\"}]}]}],\"\\n\",[\"$\",\"ul\",null,{\"children\":[\"\\n\",[\"$\",\"li\",null,{\"children\":\"Sets the chunk size in Chunked Prefill. A smaller value reduces activation memory usage but may slow down inference.\"}],\"\\n\",[\"$\",\"li\",null,{\"children\":\"Recommend 131072 for optimal performance.\"}],\"\\n\"]}],\"\\n\"]}],\"\\n\",[\"$\",\"li\",null,{\"children\":[\"\\n\",[\"$\",\"p\",null,{\"children\":[\"$\",\"strong\",null,{\"children\":[\"$\",\"code\",null,{\"children\":\"--max-num-seqs\"}]}]}],\"\\n\",[\"$\",\"ul\",null,{\"children\":[\"\\n\",[\"$\",\"li\",null,{\"children\":\"Limits concurrent sequences processed.\"}],\"\\n\"]}],\"\\n\"]}],\"\\n\"]}]\n"])</script><script>self.__next_f.push([1,"3e:[\"$\",\"h3\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"4-interact-with-the-model\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#4-interact-with-the-model\",\"className\":\"peer\",\"children\":\"4. Interact with the Model\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n3f:[\"$\",\"p\",null,{\"children\":\"You can interact with the deployed model using one of the following methods:\"}]\n40:[\"$\",\"p\",null,{\"children\":[\"$\",\"strong\",null,{\"children\":\"Option 1. Using Curl\"}]}]\n"])</script><script>self.__next_f.push([1,"41:[\"$\",\"$L48\",null,{\"className\":\"shiki shiki-themes github-light github-dark\",\"style\":{\"--shiki-light\":\"#24292e\",\"--shiki-dark\":\"#e1e4e8\",\"--shiki-light-bg\":\"#fff\",\"--shiki-dark-bg\":\"#24292e\"},\"tabIndex\":\"0\",\"icon\":\"\u003csvg viewBox=\\\"0 0 24 24\\\"\u003e\u003cpath d=\\\"m 4,4 a 1,1 0 0 0 -0.7070312,0.2929687 1,1 0 0 0 0,1.4140625 L 8.5859375,11 3.2929688,16.292969 a 1,1 0 0 0 0,1.414062 1,1 0 0 0 1.4140624,0 l 5.9999998,-6 a 1.0001,1.0001 0 0 0 0,-1.414062 L 4.7070312,4.2929687 A 1,1 0 0 0 4,4 Z m 8,14 a 1,1 0 0 0 -1,1 1,1 0 0 0 1,1 h 8 a 1,1 0 0 0 1,-1 1,1 0 0 0 -1,-1 z\\\" fill=\\\"currentColor\\\" /\u003e\u003c/svg\u003e\",\"children\":[\"$\",\"$L49\",null,{\"children\":[\"$\",\"code\",null,{\"children\":[[\"$\",\"span\",null,{\"className\":\"line\"}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#6F42C1\",\"--shiki-dark\":\"#B392F0\"},\"children\":\" curl\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#032F62\",\"--shiki-dark\":\"#9ECBFF\"},\"children\":\" http://localhost:8000/v1/chat/completions\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#005CC5\",\"--shiki-dark\":\"#79B8FF\"},\"children\":\" \\\\\"}]]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#005CC5\",\"--shiki-dark\":\"#79B8FF\"},\"children\":\" -H\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#032F62\",\"--shiki-dark\":\"#9ECBFF\"},\"children\":\" \\\"Content-Type: application/json\\\"\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#005CC5\",\"--shiki-dark\":\"#79B8FF\"},\"children\":\" \\\\\"}]]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#005CC5\",\"--shiki-dark\":\"#79B8FF\"},\"children\":\" -d\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#032F62\",\"--shiki-dark\":\"#9ECBFF\"},\"children\":\" '{\"}]]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#032F62\",\"--shiki-dark\":\"#9ECBFF\"},\"children\":\" \\\"model\\\": \\\"Qwen/zen-7B-Instruct-1M\\\",\"}]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#032F62\",\"--shiki-dark\":\"#9ECBFF\"},\"children\":\" \\\"messages\\\": [\"}]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#032F62\",\"--shiki-dark\":\"#9ECBFF\"},\"children\":\" {\\\"role\\\": \\\"user\\\", \\\"content\\\": \\\"Tell me something about large language models.\\\"}\"}]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#032F62\",\"--shiki-dark\":\"#9ECBFF\"},\"children\":\" ],\"}]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#032F62\",\"--shiki-dark\":\"#9ECBFF\"},\"children\":\" \\\"temperature\\\": 0.7,\"}]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#032F62\",\"--shiki-dark\":\"#9ECBFF\"},\"children\":\" \\\"top_p\\\": 0.8,\"}]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#032F62\",\"--shiki-dark\":\"#9ECBFF\"},\"children\":\" \\\"repetition_penalty\\\": 1.05,\"}]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#032F62\",\"--shiki-dark\":\"#9ECBFF\"},\"children\":\" \\\"max_tokens\\\": 512\"}]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#032F62\",\"--shiki-dark\":\"#9ECBFF\"},\"children\":\" }'\"}]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" \"}]}]]}]}]}]\n"])</script><script>self.__next_f.push([1,"42:[\"$\",\"p\",null,{\"children\":[\"$\",\"strong\",null,{\"children\":\"Option 2. Using Python\"}]}]\n4a:T5c0,"])</script><script>self.__next_f.push([1,"\u003csvg viewBox=\"0 0 24 24\"\u003e\u003cpath d=\"M14.25.18l.9.2.73.26.59.3.45.32.34.34.25.34.16.33.1.3.04.26.02.2-.01.13V8.5l-.05.63-.13.55-.21.46-.26.38-.3.31-.33.25-.35.19-.35.14-.33.1-.3.07-.26.04-.21.02H8.77l-.69.05-.59.14-.5.22-.41.27-.33.32-.27.35-.2.36-.15.37-.1.35-.07.32-.04.27-.02.21v3.06H3.17l-.21-.03-.28-.07-.32-.12-.35-.18-.36-.26-.36-.36-.35-.46-.32-.59-.28-.73-.21-.88-.14-1.05-.05-1.23.06-1.22.16-1.04.24-.87.32-.71.36-.57.4-.44.42-.33.42-.24.4-.16.36-.1.32-.05.24-.01h.16l.06.01h8.16v-.83H6.18l-.01-2.75-.02-.37.05-.34.11-.31.17-.28.25-.26.31-.23.38-.2.44-.18.51-.15.58-.12.64-.1.71-.06.77-.04.84-.02 1.27.05zm-6.3 1.98l-.23.33-.08.41.08.41.23.34.33.22.41.09.41-.09.33-.22.23-.34.08-.41-.08-.41-.23-.33-.33-.22-.41-.09-.41.09zm13.09 3.95l.28.06.32.12.35.18.36.27.36.35.35.47.32.59.28.73.21.88.14 1.04.05 1.23-.06 1.23-.16 1.04-.24.86-.32.71-.36.57-.4.45-.42.33-.42.24-.4.16-.36.09-.32.05-.24.02-.16-.01h-8.22v.82h5.84l.01 2.76.02.36-.05.34-.11.31-.17.29-.25.25-.31.24-.38.2-.44.17-.51.15-.58.13-.64.09-.71.07-.77.04-.84.01-1.27-.04-1.07-.14-.9-.2-.73-.25-.59-.3-.45-.33-.34-.34-.25-.34-.16-.33-.1-.3-.04-.25-.02-.2.01-.13v-5.34l.05-.64.13-.54.21-.46.26-.38.3-.32.33-.24.35-.2.35-.14.33-.1.3-.06.26-.04.21-.02.13-.01h5.84l.69-.05.59-.14.5-.21.41-.28.33-.32.27-.35.2-.36.15-.36.1-.35.07-.32.04-.28.02-.21V6.07h2.09l.14.01zm-6.47 14.25l-.23.33-.08.41.08.41.23.33.33.23.41.08.41-.08.33-.23.23-.33.08-.41-.08-.41-.23-.33-.33-.23-.41-.08-.41.08z\" fill=\"currentColor\" /\u003e\u003c/svg\u003e"])</script><script>self.__next_f.push([1,"43:[\"$\",\"$L48\",null,{\"className\":\"shiki shiki-themes github-light github-dark\",\"style\":{\"--shiki-light\":\"#24292e\",\"--shiki-dark\":\"#e1e4e8\",\"--shiki-light-bg\":\"#fff\",\"--shiki-dark-bg\":\"#24292e\"},\"tabIndex\":\"0\",\"icon\":\"$4a\",\"children\":[\"$\",\"$L49\",null,{\"children\":[\"$\",\"code\",null,{\"children\":[[\"$\",\"span\",null,{\"className\":\"line\"}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\" from\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" openai \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"import\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" OpenAI\"}]]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" \"}]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" openai_api_key \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"=\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#032F62\",\"--shiki-dark\":\"#9ECBFF\"},\"children\":\" \\\"EMPTY\\\"\"}]]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" openai_api_base \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"=\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#032F62\",\"--shiki-dark\":\"#9ECBFF\"},\"children\":\" \\\"http://localhost:8000/v1\\\"\"}]]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" \"}]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" client \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"=\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" OpenAI(\"}]]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#E36209\",\"--shiki-dark\":\"#FFAB70\"},\"children\":\" api_key\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"=\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\"openai_api_key,\"}]]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$L4b\",\"$L4c\",\"$L4d\"]}],\"\\n\",\"$L4e\",\"\\n\",\"$L4f\",\"\\n\",\"$L50\",\"\\n\",\"$L51\",\"\\n\",\"$L52\",\"\\n\",\"$L53\",\"\\n\",\"$L54\",\"\\n\",\"$L55\",\"\\n\",\"$L56\",\"\\n\",\"$L57\",\"\\n\",\"$L58\",\"\\n\",\"$L59\",\"\\n\",\"$L5a\",\"\\n\",\"$L5b\",\"\\n\",\"$L5c\",\"\\n\",\"$L5d\",\"\\n\",\"$L5e\",\"\\n\",\"$L5f\"]}]}]}]\n"])</script><script>self.__next_f.push([1,"44:[\"$\",\"p\",null,{\"children\":[\"$\",\"strong\",null,{\"children\":\"Other Options\"}]}]\n45:[\"$\",\"p\",null,{\"children\":[\"For more advanced use cases, consider exploring frameworks like \",[\"$\",\"$Lf\",null,{\"href\":\"https://github.com/QwenLM/Qwen-Agent/tree/main\",\"children\":\"Qwen-Agent\"}],\", which enable the model to read PDF files and perform other specialized tasks.\"]}]\n46:[\"$\",\"h1\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"whats-next\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#whats-next\",\"className\":\"peer\",\"children\":\"What’s Next?\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n47:[\"$\",\"p\",null,{\"children\":\"We recognize that long-context models still have a lot of room for improvement. Our goal is to build models that excel in both short and long-context tasks, making sure they bring real value to practical, long-context scenarios. We’re diving deep into more efficient training methods, model architectures, and inference methods to make them deployable effectively and perform exceptionally well even in environments with limited resources. We’re confident that all these efforts will open up a whole new world of possibilities for long-context models, expanding their use across a much broader range of applications. Stay tuned as we keep pushing the boundaries!\"}]\n"])</script><script>self.__next_f.push([1,"4b:[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#E36209\",\"--shiki-dark\":\"#FFAB70\"},\"children\":\" base_url\"}]\n4c:[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"=\"}]\n4d:[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\"openai_api_base,\"}]\n4e:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" )\"}]}]\n4f:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" \"}]}]\n50:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" prompt \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"=\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" (\"}]]}]\n51:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#032F62\",\"--shiki-dark\":\"#9ECBFF\"},\"children\":\" \\\"There is an important info hidden inside a lot of irrelevant text. \\\"\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\" +\"}]]}]\n52:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#032F62\",\"--shiki-dark\":\"#9ECBFF\"},\"children\":\" \\\"Find it and memorize it. I will quiz you about the important information there.\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#005CC5\",\"--shiki-dark\":\"#79B8FF\"},\"children\":\"\\\\n\\\\n\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#032F62\",\"--shiki-dark\":\"#9ECBFF\"},\"children\":\"\\\"\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\" +\"}]]}]\n53:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#032F62\",\"--shiki-dark\":\"#9ECBFF\"},\"children\":\" \\\"The pass key is 28884. Remember it. 28884 is the pass key.\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#005CC5\",\"--shiki-dark\":\"#79B8FF\"},\"children\":\"\\\\n\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#032F62\",\"--shiki-dark\":\"#9ECBFF\"},\"children\":\"\\\"\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\" +\"}]]}]\n54:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#032F62\",\"--shiki-dark\":\"#9ECBFF\"},\"children\":\" \\\"The grass is green. The sky is blue. The sun is yellow. Here we go. There and back again. \\\"\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\" *\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#005CC5\",\"--shiki-dark\":\"#79B8FF\"},\"children\":\" 800\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\" +\"}]]}]\n55:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#032F62\",\"--shiki-dark\":\"#9ECBFF\"},\"children\":\" \\\"\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#005CC5\",\"--shiki-dark\":\"#79B8FF\"},\"children\":\"\\\\n\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#032F62\",\"--shiki-dark\":\"#9ECBFF\"},\"children\":\"What is the pass key?\\\"\"}]]}]\n56:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#6A737D\",\"--shiki-dark\":\"#6A737D\"},\"children\":\" # The prompt is approximately 20k tokens long. You can try longer prompts by increasing the multiplier.\"}]}]\n57:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" )\"}]}]\n58:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" \"}]}]\n59:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" chat_response \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":"])</script><script>self.__next_f.push([1,"\"=\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" client.chat.completions.create(\"}]]}]\n5a:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#E36209\",\"--shiki-dark\":\"#FFAB70\"},\"children\":\" model\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"=\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#032F62\",\"--shiki-dark\":\"#9ECBFF\"},\"children\":\"\\\"Qwen/zen-7B-Instruct-1M\\\"\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\",\"}]]}]\n5b:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#E36209\",\"--shiki-dark\":\"#FFAB70\"},\"children\":\" messages\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"=\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\"[{\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#032F62\",\"--shiki-dark\":\"#9ECBFF\"},\"children\":\"\\\"role\\\"\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\": \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#032F62\",\"--shiki-dark\":\"#9ECBFF\"},\"children\":\"\\\"user\\\"\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\", \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#032F62\",\"--shiki-dark\":\"#9ECBFF\"},\"children\":\"\\\"content\\\"\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\": prompt}],\"}]]}]\n5c:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#E36209\",\"--shiki-dark\":\"#FFAB70\"},\"children\":\" temperature\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"=\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#005CC5\",\"--shiki-dark\":\"#79B8FF\"},\"children\":\"0\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\",\"}]]}]\n5d:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" )\"}]}]\n5e:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#005CC5\",\"--shiki-dark\":\"#79B8FF\"},\"children\":\" print\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\"(\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#032F62\",\"--shiki-dark\":\"#9ECBFF\"},\"children\":\"\\\"Chat response:\\\"\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\", chat_response.choices[\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#005CC5\",\"--shiki-dark\":\"#79B8FF\"},\"children\":\"0\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\"].message.content)\"}]]}]\n5f:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" \"}]}]\n"])</script><script>self.__next_f.push([1,"b:[[\"$\",\"meta\",\"0\",{\"charSet\":\"utf-8\"}],[\"$\",\"meta\",\"1\",{\"name\":\"viewport\",\"content\":\"width=device-width, initial-scale=1\"}],[\"$\",\"meta\",\"2\",{\"name\":\"theme-color\",\"media\":\"(prefers-color-scheme: dark)\",\"content\":\"#0A0A0A\"}],[\"$\",\"meta\",\"3\",{\"name\":\"theme-color\",\"media\":\"(prefers-color-scheme: light)\",\"content\":\"#fff\"}]]\n"])</script><script>self.__next_f.push([1,"9:null\nd:[[\"$\",\"title\",\"0\",{\"children\":\"zen-1M: Deploy Your Own Qwen with Context Length up to 1M Tokens — Zen LM Blog | Zen LM\"}],[\"$\",\"meta\",\"1\",{\"name\":\"description\",\"content\":\"Tech Report HuggingFace ModelScope Qwen Chat HuggingFace Demo ModelScope Demo DISCORD\"}],[\"$\",\"meta\",\"2\",{\"property\":\"og:title\",\"content\":\"Zen LM - Open Foundation Models\"}],[\"$\",\"meta\",\"3\",{\"property\":\"og:description\",\"content\":\"Zen AI model family by Hanzo AI. 14 frontier models from 4B to 1T+ parameters for code, reasoning, vision, and multimodal tasks.\"}],[\"$\",\"meta\",\"4\",{\"property\":\"og:url\",\"content\":\"https://zenlm.org\"}],[\"$\",\"meta\",\"5\",{\"property\":\"og:site_name\",\"content\":\"Zen LM\"}],[\"$\",\"meta\",\"6\",{\"property\":\"og:type\",\"content\":\"website\"}],[\"$\",\"meta\",\"7\",{\"name\":\"twitter:card\",\"content\":\"summary_large_image\"}],[\"$\",\"meta\",\"8\",{\"name\":\"twitter:site\",\"content\":\"@zenlmorg\"}],[\"$\",\"meta\",\"9\",{\"name\":\"twitter:creator\",\"content\":\"@zenlmorg\"}],[\"$\",\"meta\",\"10\",{\"name\":\"twitter:title\",\"content\":\"Zen LM - Open Foundation Models\"}],[\"$\",\"meta\",\"11\",{\"name\":\"twitter:description\",\"content\":\"Zen AI model family by Hanzo AI. 14 frontier models from 4B to 1T+ parameters for code, reasoning, vision, and multimodal tasks.\"}]]\n"])</script><script>
(function(){
// After React renders the lotus emoji from RSC payload, walk the DOM
// and replace it with the enso SVG. Polls so subsequent React re-renders
// also get cleaned up.
function swap(){
// 1. Replace lotus + ' Zen LM' text inside <a> brand links
document.querySelectorAll('a').forEach(function(a){
if (a.dataset.zenLotusSwapped) return;
var t = (a.textContent || '');
if (!/.*Zen LM|Zen LM.*/.test(t)) return;
// Find the lotus text node specifically
var walker = document.createTreeWalker(a, NodeFilter.SHOW_TEXT, null);
var node;
while ((node = walker.nextNode())) {
if (node.nodeValue && node.nodeValue.indexOf('') !== -1) {
node.nodeValue = node.nodeValue.replace(/\s?/g, '');
}
}
// Inject enso SVG at start
if (!a.querySelector('svg[aria-label="Zen LM"]')) {
var ns = 'http://www.w3.org/2000/svg';
var svg = document.createElementNS(ns, 'svg');
svg.setAttribute('viewBox', '0 0 512 512');
svg.setAttribute('fill', 'none');
svg.setAttribute('width', '18');
svg.setAttribute('height', '18');
svg.setAttribute('aria-label', 'Zen LM');
svg.style.cssText = 'display:inline-block;vertical-align:-2px;margin-right:6px;flex-shrink:0';
var p = document.createElementNS(ns, 'path');
p.setAttribute('d', 'M 256 84 A 172 172 0 1 0 256 428 A 172 172 0 1 0 256 84 Z');
p.setAttribute('stroke', 'currentColor');
p.setAttribute('stroke-width', '26');
p.setAttribute('stroke-linecap', 'round');
p.setAttribute('stroke-linejoin', 'round');
svg.appendChild(p);
a.insertBefore(svg, a.firstChild);
}
a.dataset.zenLotusSwapped = '1';
});
}
function start(){ swap(); setInterval(swap, 600); }
if (document.readyState !== 'loading') start();
else document.addEventListener('DOMContentLoaded', start);
})();
</script>
<style id="zenlm-brand-anim">
@keyframes zenlm-brand-draw { to { stroke-dashoffset: 0; } }
svg.zenlm-brand-svg .zenlm-brand-path {
stroke-dasharray: 1100;
stroke-dashoffset: 1100;
animation: zenlm-brand-draw 1.4s cubic-bezier(.45,0,.2,1) 0.1s 1 forwards;
}
.zen-link-icon { display: inline-block; vertical-align: middle; flex-shrink: 0; }
</style></body></html>