mirror of
https://github.com/zenlm/zen.git
synced 2026-11-17 11:36:07 +00:00
At 390 pixels the blog's tables, the datasets tables and the model card specs ran past the right edge, and body's overflow-x clip cut them off rather than letting them scroll. Tables now scroll in their own box, and a spec column shrinks and wraps instead of pushing its neighbour out of the card.
148 lines
52 KiB
HTML
148 lines
52 KiB
HTML
<!DOCTYPE html><!--_BoRG_r6Z3v9udL0lI0J_--><html lang="en"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="stylesheet" href="/_next/static/css/1d1f6bc532e5f43f.css" data-precedence="next"/><link rel="stylesheet" href="/_next/static/css/eb87e4f7aea490c6.css" data-precedence="next"/><link rel="preload" as="script" fetchPriority="low" href="/_next/static/chunks/webpack-f43ad49c79db3299.js"/><script src="/_next/static/chunks/4bd1b696-c023c6e3521b1417.js" async=""></script><script src="/_next/static/chunks/255-566af14aff71e752.js" async=""></script><script src="/_next/static/chunks/main-app-7eaab9f523b3767f.js" async=""></script><script src="/_next/static/chunks/619-ba102abea3e3d0e4.js" async=""></script><script src="/_next/static/chunks/app/layout-425ca81883167c16.js" async=""></script><script src="/_next/static/chunks/app/blog/%5Bslug%5D/page-38f8c2a205f98d97.js" async=""></script><script src="/assets/js/main.js" async=""></script><link rel="icon" type="image/svg+xml" href="/favicon.svg"/><link rel="alternate icon" href="/favicon.png"/><title>Code with CodeQwen1.5 — Zen Blog</title><meta name="description" content="The advent of advanced programming tools, which harnesses the power of large language models (LLMs), has significantly enhanced programmer productivity and accuracy. Notwithstanding these advancements, dominant coding assistants like Github Copilot, built upon proprietary LLMs, pose notable challeng"/><meta name="keywords" content="AI, LLM, Agentic AI, Code Generation, Zen Coder, Multimodal, Open Source, Machine Learning"/><meta name="data-upstream" content="true"/><script src="/_next/static/chunks/polyfills-42372ed130431b0a.js" noModule=""></script></head><body><div hidden=""><!--$--><!--/$--></div><header><nav class="navbar"><div class="container"><div class="logo"><a href="/"><svg class="enso logo-img" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 100 100" width="34" height="34" fill="none" role="img" aria-label="Zen ensō"><defs><linearGradient id="_R_adb_" x1="12%" y1="0%" x2="88%" y2="100%"><stop offset="0%" stop-color="#ffffff"></stop><stop offset="50%" stop-color="#d4d4d8"></stop><stop offset="100%" stop-color="#7c7c82"></stop></linearGradient></defs><path class="enso-stroke" pathLength="1" d="M58 13 C38 9 17 22 13 44 C9 68 26 89 50 90 C74 91 91 74 89 50 C88 41 87 35 84 30" stroke="url(#_R_adb_)" stroke-width="7" stroke-linecap="round" stroke-linejoin="round"></path></svg><div class="logo-text"><h1>Zen LM</h1></div></a></div><div class="header-right"><div class="nav-links"><a class="" href="/">Home</a><div class="nav-item"><a class="" href="/models/">Models<svg class="nav-caret" width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M6 9l6 6 6-6"></path></svg></a><div class="nav-dropdown"><div class="nav-dropdown-inner"><a class="dd-item" href="/models/#zen6"><div class="dd-item-text"><span class="dd-item-label">Zen 6<span class="dd-tag">Available</span></span><span class="dd-item-desc">27B dense — text, images and video, 1M context with YaRN</span></div><svg class="dd-arrow" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M5 12h14M13 6l6 6-6 6"></path></svg></a><a class="dd-item" href="/models/#zen6"><div class="dd-item-text"><span class="dd-item-label">Zen 6 Flash<span class="dd-tag">Available</span></span><span class="dd-item-desc">The ternary Zen 6 — 5.95 GB, fits an Apple Silicon laptop</span></div><svg class="dd-arrow" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M5 12h14M13 6l6 6-6 6"></path></svg></a><a class="dd-item" href="https://hanzo.ai/research-access" target="_blank" rel="noopener noreferrer"><div class="dd-item-text"><span class="dd-item-label">Zen 7<span class="dd-tag">Research preview</span></span><span class="dd-item-desc">No weights yet — request access</span></div><svg class="dd-arrow" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M5 12h14M13 6l6 6-6 6"></path></svg></a><a class="dd-item" href="/models/#zen5"><div class="dd-item-text"><span class="dd-item-label">Earlier generations</span><span class="dd-item-desc">Zen 5, Zen 4 and Zen 3</span></div><svg class="dd-arrow" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M5 12h14M13 6l6 6-6 6"></path></svg></a><a class="dd-item" href="/models/#catalog"><div class="dd-item-text"><span class="dd-item-label">All models</span><span class="dd-item-desc">Browse the full catalog</span></div><svg class="dd-arrow" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M5 12h14M13 6l6 6-6 6"></path></svg></a></div></div></div><div class="nav-item"><a class="" href="/datasets/">Datasets<svg class="nav-caret" width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M6 9l6 6 6-6"></path></svg></a><div class="nav-dropdown"><div class="nav-dropdown-inner"><a class="dd-item" href="/datasets/"><div class="dd-item-text"><span class="dd-item-label">Zen Agentic Dataset<span class="dd-tag">New</span></span><span class="dd-item-desc">10B+ tokens of real tool use and reasoning</span></div><svg class="dd-arrow" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M5 12h14M13 6l6 6-6 6"></path></svg></a><a class="dd-item" href="/datasets/"><div class="dd-item-text"><span class="dd-item-label">Training Data</span><span class="dd-item-desc">Curation, sources, and methodology</span></div><svg class="dd-arrow" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M5 12h14M13 6l6 6-6 6"></path></svg></a><a class="dd-item" href="https://huggingface.co/datasets/hanzoai/zen-agentic-dataset" target="_blank" rel="noopener noreferrer"><div class="dd-item-text"><span class="dd-item-label">On HuggingFace</span><span class="dd-item-desc">Download and license the datasets</span></div><svg class="dd-arrow" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M5 12h14M13 6l6 6-6 6"></path></svg></a></div></div></div><div class="nav-item"><a class="" href="/research/">Research<svg class="nav-caret" width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M6 9l6 6 6-6"></path></svg></a><div class="nav-dropdown"><div class="nav-dropdown-inner"><a class="dd-item" href="/research/"><div class="dd-item-text"><span class="dd-item-label">Research Papers</span><span class="dd-item-desc">Technical reports across the Zen family</span></div><svg class="dd-arrow" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M5 12h14M13 6l6 6-6 6"></path></svg></a><a class="dd-item" href="https://papers.zenlm.org" target="_blank" rel="noopener noreferrer"><div class="dd-item-text"><span class="dd-item-label">Papers Archive</span><span class="dd-item-desc">Full paper archive — papers.zenlm.org</span></div><svg class="dd-arrow" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M5 12h14M13 6l6 6-6 6"></path></svg></a><a class="dd-item" href="/blog/"><div class="dd-item-text"><span class="dd-item-label">Blog</span><span class="dd-item-desc">Research notes, deep dives, and releases</span></div><svg class="dd-arrow" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M5 12h14M13 6l6 6-6 6"></path></svg></a></div></div></div><a class="active" href="/blog/">Blog</a></div><div class="logo-links"><a href="https://github.com/zenlm" target="_blank" rel="noopener noreferrer" class="icon-link" title="GitHub"><svg width="24" height="24" viewBox="0 0 24 24" fill="currentColor"><path d="M12 0c-6.626 0-12 5.373-12 12 0 5.302 3.438 9.8 8.207 11.387.599.111.793-.261.793-.577v-2.234c-3.338.726-4.033-1.416-4.033-1.416-.546-1.387-1.333-1.756-1.333-1.756-1.089-.745.083-.729.083-.729 1.205.084 1.839 1.237 1.839 1.237 1.07 1.834 2.807 1.304 3.492.997.107-.775.418-1.305.762-1.604-2.665-.305-5.467-1.334-5.467-5.931 0-1.311.469-2.381 1.236-3.221-.124-.303-.535-1.524.117-3.176 0 0 1.008-.322 3.301 1.23.957-.266 1.983-.399 3.003-.404 1.02.005 2.047.138 3.006.404 2.291-1.552 3.297-1.23 3.297-1.23.653 1.653.242 2.874.118 3.176.77.84 1.235 1.911 1.235 3.221 0 4.609-2.807 5.624-5.479 5.921.43.372.823 1.102.823 2.222v3.293c0 .319.192.694.801.576 4.765-1.589 8.199-6.086 8.199-11.386 0-6.627-5.373-12-12-12z"></path></svg></a><a href="https://huggingface.co/zenlm" target="_blank" rel="noopener noreferrer" class="icon-link" title="HuggingFace"><svg width="28" height="28" viewBox="0 0 32 32" fill="currentColor" aria-hidden="true"><circle cx="16" cy="16" r="13" fill="#d4d4d8"></circle><circle cx="11" cy="14" r="2" fill="#18181b"></circle><circle cx="21" cy="14" r="2" fill="#18181b"></circle><path d="M11 19 Q16 24 21 19" stroke="#18181b" stroke-width="1.6" fill="none" stroke-linecap="round"></path></svg></a></div><div class="nav-item try-item"><a href="https://hanzo.chat/?model=zen6" target="_blank" rel="noopener noreferrer" class="btn-try">Try Zen<svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M5 12h14M13 6l6 6-6 6"></path></svg></a><div class="nav-dropdown nav-dropdown-right"><div class="nav-dropdown-inner"><a class="dd-item" href="https://hanzo.chat/?model=zen6" target="_blank" rel="noopener noreferrer"><div class="dd-item-text"><span class="dd-item-label">Zen 6<span class="dd-tag">Available</span></span><span class="dd-item-desc">Chat with Zen 6</span></div><svg class="dd-arrow" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M5 12h14M13 6l6 6-6 6"></path></svg></a><a class="dd-item" href="https://hanzo.chat/?model=zen6-flash" target="_blank" rel="noopener noreferrer"><div class="dd-item-text"><span class="dd-item-label">Zen 6 Flash<span class="dd-tag">Available</span></span><span class="dd-item-desc">Chat with the ternary Zen 6</span></div><svg class="dd-arrow" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M5 12h14M13 6l6 6-6 6"></path></svg></a><a class="dd-item" href="https://api.hanzo.ai" target="_blank" rel="noopener noreferrer"><div class="dd-item-text"><span class="dd-item-label">Zen API</span><span class="dd-item-desc">OpenAI- & Anthropic-compatible. One key.</span></div><svg class="dd-arrow" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M5 12h14M13 6l6 6-6 6"></path></svg></a><a class="dd-item" href="https://hanzo.chat" target="_blank" rel="noopener noreferrer"><div class="dd-item-text"><span class="dd-item-label">Zen Chat</span><span class="dd-item-desc">Open the full chat experience</span></div><svg class="dd-arrow" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M5 12h14M13 6l6 6-6 6"></path></svg></a></div></div></div></div></div></nav></header><main><article class="blog-article" data-upstream=""><a class="blog-back" href="/blog/">← Blog</a><div class="blog-post-meta">April 15, 2024<!-- --> <!-- -->·<!-- --> <!-- -->7<!-- --> min read</div><h1 class="blog-post-title">Code with CodeQwen1.5</h1><p class="blog-post-lede">The advent of advanced programming tools, which harnesses the power of large language models (LLMs), has significantly enhanced programmer productivity and accuracy. Notwithstanding these advancements, dominant coding assistants like Github Copilot, built upon proprietary LLMs, pose notable challeng</p><div class="blog-prose"><p><a href="https://github.com/QwenLM/CodeQwen1.5">GITHUB</a>
|
||
<a href="https://huggingface.co/Qwen">HUGGING FACE</a>
|
||
<a href="https://modelscope.cn/organization/qwen">MODELSCOPE</a>
|
||
<a href="https://huggingface.co/spaces/Qwen/CodeQwen1.5-7b-Chat-demo">DEMO</a>
|
||
<a href="https://discord.gg/yPEP2vHTu4">DISCORD</a></p>
|
||
<h1 id="introduction">Introduction</h1>
|
||
<p>The advent of advanced programming tools, which harnesses the power of large language models (LLMs), has significantly enhanced programmer productivity and accuracy. Notwithstanding these advancements, dominant coding assistants like Github Copilot, built upon proprietary LLMs, pose notable challenges in terms of cost, privacy, security, and potential copyright infringement. Recognizing the imperative for a more transparent and accessible alternative, the open-source community has embarked on a concerted endeavor to develop open codeLLMs. This initiative has already given rise to several promising open-source models, including StarCoder2, CodeLlama, and DeepSeek-Coder, offering a path forward, albeit one that necessitates continued refinement.</p>
|
||
<figure><img src="https://qianwen-res.oss-accelerate-overseas.aliyuncs.com/assets/blog/codeqwen1.5/intro.png" alt="" loading="lazy"></figure>
|
||
<p>Today, we are delighted to introduce a new member of the Qwen1.5 open-source family, the CodeQwen1.5-7B, a specialized codeLLM built upon the Qwen1.5 language model. CodeQwen1.5-7B has been pretrained with around 3 trillion tokens of code-related data. It supports an extensive repertoire of 92 programming languages, and it exhibits exceptional capacity in long-context understanding and generation with the ability to process information of 64K tokens. In terms of performance, CodeQwen1.5 demonstrates impressive capabilities in basic code generation, long-context modeliing, code editation and SQL. We believe this model can significantly enhance developer productivity and streamline software development workflows within diverse technological environments.</p>
|
||
<h1 id="codeqwen-is-a-basic-coder">CodeQwen is a Basic Coder</h1>
|
||
<p>Code generation is a key competence for large language models, as they are tasked with translating natural language instructions into executable code with unwavering precision. CodeQwen1.5, with only 7 billion parameters, has surpassed larger models in basic code generation capabilities, further narrowing the gap in coding proficiency between GPT-4 and opensource code LLMs. We conducted a thorough evaluation on HumanEval and MBPP to provide a clear and fair comparison as follows.</p>
|
||
<table style="{{textAlign:" "center"}}="">
|
||
<tbody><tr style="{{fontWeight:" "bold"}}="">
|
||
<td style="{{textAlign:" "left"}}="">Model</td>
|
||
<td style="{{textAlign:" "left"}}="">Size</td>
|
||
<td>
|
||
<div>HumanEval</div>
|
||
<div class="cell-aux">0-shot</div>
|
||
</td>
|
||
<td>
|
||
<div>HumanEval+</div>
|
||
<div class="cell-aux">0-shot</div>
|
||
</td>
|
||
<td>
|
||
<div>MBPP</div>
|
||
<div class="cell-aux">0-shot</div>
|
||
</td>
|
||
<td>
|
||
<div>MBPP+</div>
|
||
<div class="cell-aux">0-shot</div>
|
||
</td>
|
||
<td>
|
||
<div>MBPP</div>
|
||
<div class="cell-aux">3-shot</div>
|
||
</td>
|
||
</tr>
|
||
<tr>
|
||
<td colspan="7"><b>Base Model</b></td>
|
||
</tr>
|
||
<tr>
|
||
<td style="{{textAlign:" "left"}}="">CodeLlama-Base</td>
|
||
<td style="{{textAlign:" "left"}}="">7B</td>
|
||
<td>33.5</td>
|
||
<td>25.6</td>
|
||
<td>52.1</td>
|
||
<td>41.6</td>
|
||
<td>38.6</td>
|
||
</tr>
|
||
<tr>
|
||
<td style="{{textAlign:" "left"}}="">StarCoder2</td>
|
||
<td style="{{textAlign:" "left"}}="">7B</td>
|
||
<td>35.4</td>
|
||
<td>29.9</td>
|
||
<td>54.4</td>
|
||
<td>45.6</td>
|
||
<td>51.0</td>
|
||
</tr>
|
||
<tr>
|
||
<td style="{{textAlign:" "left"}}="">DeepSeek-Coder-Base</td>
|
||
<td style="{{textAlign:" "left"}}="">6.7B</td>
|
||
<td>47.6</td>
|
||
<td>39.6</td>
|
||
<td>70.2</td>
|
||
<td>56.6</td>
|
||
<td>60.6</td>
|
||
</tr>
|
||
<tr>
|
||
<td style="{{textAlign:" "left"}}=""><b>CodeQwen1.5</b></td>
|
||
<td style="{{textAlign:" "left"}}="">7B</td>
|
||
<td>51.8</td>
|
||
<td>45.7</td>
|
||
<td>72.2</td>
|
||
<td>60.2</td>
|
||
<td>61.8</td>
|
||
</tr>
|
||
<tr>
|
||
<td colspan="7"><b>Chat Model</b></td>
|
||
</tr>
|
||
<tr>
|
||
<td style="{{textAlign:" "left"}}="">GPT-3.5-Turbo</td>
|
||
<td style="{{textAlign:" "left"}}="">-</td>
|
||
<td>76.8</td>
|
||
<td>70.7</td>
|
||
<td>82.5</td>
|
||
<td>69.7</td>
|
||
<td>70.8</td>
|
||
</tr>
|
||
<tr>
|
||
<td style="{{textAlign:" "left"}}="">GPT-4-Turbo (Nov 2023)</td>
|
||
<td style="{{textAlign:" "left"}}="">-</td>
|
||
<td>85.4</td>
|
||
<td>81.7</td>
|
||
<td>83.5</td>
|
||
<td>70.7</td>
|
||
<td>80.0</td>
|
||
</tr>
|
||
<tr>
|
||
<td style="{{textAlign:" "left"}}="">DeepSeek-Coder-Instruct</td>
|
||
<td style="{{textAlign:" "left"}}="">6.7B</td>
|
||
<td>78.6</td>
|
||
<td>70.1</td>
|
||
<td>73.2</td>
|
||
<td>63.4</td>
|
||
<td>65.4</td>
|
||
</tr>
|
||
<tr>
|
||
<td style="{{textAlign:" "left"}}=""><b>CodeQwen1.5-Chat</b></td>
|
||
<td style="{{textAlign:" "left"}}="">7B</td>
|
||
<td>83.5</td>
|
||
<td>78.7</td>
|
||
<td>77.7</td>
|
||
<td>67.2</td>
|
||
<td>70.6</td>
|
||
</tr>
|
||
</tbody></table>
|
||
<p>In addition to the widely recognized HumanEval and MBPP benchmarks, we explored LiveCodeBench. This benchmark assesses code performance by introducing fresh challenges sourced from coding competitions such as LeetCode, AtCoder, and CodeForces over time. Our evaluation of CodeQwen1.5 on LiveCodeBench spanned from September 1, 2023, to April 1, 2024. The findings indicate that CodeQwen1.5 ranks among the top open-access models currently available. Note: it is possible that the inclusion of LeetCode data in our pretraining corpus may contribute to the performance in LiveCodeBench.</p>
|
||
<figure><img src="https://qianwen-res.oss-accelerate-overseas.aliyuncs.com/assets/blog/codeqwen1.5/livecode.png" alt="" loading="lazy"></figure>
|
||
<p>The evaluations mentioned primarily revolve around Python capabilities; however, CodeQwen1.5 is not merely a Python specialist but also an expert across multiple programming languages. We conducted a comprehensive evaluation of CodeQwen1.5 in the eight mainstream languages featured in MultiPL-E, including Python, C++, Java, PHP, TypeScript, C#, Bash, and JavaScript. The results highlight the exceptional programming capabilities of CodeQwen1.5.</p>
|
||
<figure><img src="https://qianwen-res.oss-accelerate-overseas.aliyuncs.com/assets/blog/codeqwen1.5/radar-vertical.png" alt="" loading="lazy"></figure>
|
||
<h1 id="codeqwen-is-a-long-context-coder">CodeQwen is a Long Context Coder</h1>
|
||
<p>Long context capability is crucial for code LLMs, serving as the core skill for understanding repository-level code and becoming a code agent. However, current code models still have very limited support for length, which hinders their potential for practical application. CodeQwen1.5 aims to further advance the progress of open-source code models in long context modeling. To achieve this, we have collected and constructed long sequence code data at the repository level for pre-training. Through careful data proportioning and organization, we have enabled it to support input lengths of up to 64K tokens.</p>
|
||
<p><strong>Evaluation 1</strong>: We collected high-quality repo from GitHub Trending repositories on 2024-3-28 that were not included in CodeQwen1.5's training data to observe the effectiveness of long context modeling. The following figure demonstrates that as the sequence length increases, CodeQwen1.5's Perplexity (PPL) still manages to maintain a downward trend.</p>
|
||
<figure><img src="https://qianwen-res.oss-accelerate-overseas.aliyuncs.com/assets/blog/codeqwen1.5/longcontext-ppl.png" alt="" loading="lazy"></figure>
|
||
<p><strong>Evaluation 2</strong>: We created a synthetic task called <code>Needle in the Code</code>, inspired by popular long-context evaluations in the text domain. In this task, we inserted a very simple custom function at various positions within a longer codebase (we chose Megatron to honor its contributions to open-source LLMs!) and tested whether the model could replicate this function at the end of the codebase. The figure below shows that CodeQwen is capable of successfully completing this task within a 64k length range.</p>
|
||
<figure><img src="https://qianwen-res.oss-accelerate-overseas.aliyuncs.com/assets/blog/codeqwen1.5/longcontext-needle.png" alt="" loading="lazy"></figure>
|
||
<p>Both Evaluation 1 and Evaluation 2 serve as initial and foundational assessments. For the Chat model, we aim to evaluate its long context capabilities with more practical tasks. However, our objective is to examine the Chat model's capability to handle long contexts through more pragmatic, real-world evaluation tasks.</p>
|
||
<p><strong>Evaluation 3</strong>: SWE Bench is a benchmark designed to assess the ability of Large Language Models (LLMs) or agents to tackle practical software development challenges. It presents contestants with a code repository and an associated issue, tasking them with generating a commit patch that resolves the issue effectively. The benchmark uniquely emphasizes the long-context processing capabilities of code LLMs, necessitating both deep comprehension of the given codebase and the generation of extensive, unit-test-passing code.</p>
|
||
<figure><img src="https://qianwen-res.oss-accelerate-overseas.aliyuncs.com/assets/blog/codeqwen1.5/swe-bench.png" alt="" loading="lazy"></figure>
|
||
<p>Currently, participants in the SWE Bench competition predominantly are proprietary models. We introduce CodeQwen1.5 as an open-source model entry. Despite achieving a score of 0.89, CodeQwen1.5 surpasses ChatGPT-3.5, demonstrating the nascent yet promising competitiveness of open-source code models against their proprietary counterparts.</p>
|
||
<h1 id="codeqwen-is-a-debugger">CodeQwen is a Debugger</h1>
|
||
<p>An effective code assistant must demonstrate proficiency in both generating code in response to given specifications and adeptly modifying or debugging existing code to accommodate evolving requirements or rectify errors. In assessing CodeQwen1.5's proficiency in code modification tasks, we concentrated our evaluation on the CodeEditorBench suite, encompassing four distinct dimensions: Debugging, Translation, Language Switching, and Code Polishing. The results indicate that CodeQwen1.5 achieves the SOTA performance at the 7 billion parameter scale.</p>
|
||
<figure><img src="https://qianwen-res.oss-accelerate-overseas.aliyuncs.com/assets/blog/codeqwen1.5/codeedit.jpeg" alt="" loading="lazy"></figure>
|
||
<h1 id="codeqwen-is-a-sqler">CodeQwen is a SQLer</h1>
|
||
<p>CodeQwen1.5 serves as a solution to bridge the gap between non-programming professionals and efficient data interaction. It alleviates the steep learning curve associated with SQL by enabling users without coding expertise to query databases through natural language. We evaluated CodeQwen1.5-Chat's performance on two popular Text-to-SQL benchmarks, Spider and Bird. Experimental results pose CodeQwen1.5 a second position close to GPT-4 (results come from DIN-SQL, a SOTA prompting method). This outstanding performance is attributed to the utilization of synthetic data throughout both pre-training and fine-tuning stages. Synthetic data, characterized by its scalability, verifiability, and variety, emerges as a compelling area for future research due to its proven effectiveness in enhancing CodeQwen1.5's SQL capabilities.</p>
|
||
<figure><img src="https://qianwen-res.oss-accelerate-overseas.aliyuncs.com/assets/blog/codeqwen1.5/sql-score.png" alt="" loading="lazy"></figure>
|
||
<h1 id="develop-with-codeqwen15">Develop with CodeQwen1.5</h1>
|
||
<p>CodeQwen1.5 is part of the Qwen1.5 open-source family. We advise you to read our blog for <a href="https://qwenlm.github.io/blog/qwen1.5/">Qwen1.5</a> to figure out the usages with Transformers, vLLM, llama.cpp, Ollama, etc.</p>
|
||
<h1 id="conclusion">Conclusion</h1>
|
||
<p>We have released CodeQwen1.5-7B and CodeQwen1.5-7B-Chat, an open and versatile code LLM. The models are intended to aid progress in code assistance and code agents, benefiting the research community. We'll keep investing heavily in smart code development, with the ultimate goal of creating AI programmers.</p>
|
||
<h1 id="citation">Citation</h1>
|
||
<pre><code>@misc{codeqwen1.5,
|
||
title = {Code with CodeQwen1.5},
|
||
url = {https://qwenlm.github.io/blog/codeqwen1.5/},
|
||
author = {Qwen Team},
|
||
month = {April},
|
||
year = {2024}
|
||
}
|
||
</code></pre></div></article></main><!--$--><!--/$--><footer><div class="container"><div class="footer-content"><div class="footer-section"><h4>Zen LM</h4><p>Open models from Zoo Labs Foundation, a 501(c)(3) non-profit, for agentic coding on your own machine and marketing work. Served on api.hanzo.ai.</p></div><div class="footer-section"><h4>Zen 6</h4><ul><li><a href="/models/#zen6">Zen 6</a></li><li><a href="/models/#zen6">Zen 6 Flash</a></li><li><a href="https://hanzo.ai/research-access" target="_blank" rel="noopener noreferrer">Zen 7 research preview</a></li></ul></div><div class="footer-section"><h4>Earlier Generations</h4><ul><li><a href="/models/#zen5">Zen 5</a></li><li><a href="/models/#zen4">Zen 4</a></li><li><a href="/models/#zen3">Zen 3</a></li></ul></div><div class="footer-section"><h4>Resources</h4><ul><li><a href="/datasets/">Training Data</a></li><li><a href="https://huggingface.co/zenlm" target="_blank" rel="noopener noreferrer">HuggingFace</a></li><li><a href="https://github.com/zenlm" target="_blank" rel="noopener noreferrer">GitHub</a></li><li><a href="/research/">Research Papers</a></li><li><a href="https://api.hanzo.ai" target="_blank" rel="noopener noreferrer">Zen API</a></li></ul></div></div><div class="footer-bottom"><p>© <!-- -->2026<!-- --> Zoo Labs Foundation, a 501(c)(3) non-profit. Zen LM open models.</p></div></div></footer><script src="/_next/static/chunks/webpack-f43ad49c79db3299.js" id="_R_" async=""></script><script>(self.__next_f=self.__next_f||[]).push([0])</script><script>self.__next_f.push([1,"1:\"$Sreact.fragment\"\n2:I[6529,[\"619\",\"static/chunks/619-ba102abea3e3d0e4.js\",\"177\",\"static/chunks/app/layout-425ca81883167c16.js\"],\"default\"]\n3:I[9766,[],\"\"]\n4:I[8924,[],\"\"]\n5:I[2619,[\"619\",\"static/chunks/619-ba102abea3e3d0e4.js\",\"953\",\"static/chunks/app/blog/%5Bslug%5D/page-38f8c2a205f98d97.js\"],\"\"]\n6:I[7405,[\"619\",\"static/chunks/619-ba102abea3e3d0e4.js\",\"177\",\"static/chunks/app/layout-425ca81883167c16.js\"],\"default\"]\na:I[7150,[],\"\"]\n:HL[\"/_next/static/css/1d1f6bc532e5f43f.css\",\"style\"]\n:HL[\"/_next/static/css/eb87e4f7aea490c6.css\",\"style\"]\n"])</script><script>self.__next_f.push([1,"0:{\"P\":null,\"b\":\"-BoRG_r6Z3v9udL0lI0J_\",\"p\":\"\",\"c\":[\"\",\"blog\",\"codeqwen1.5\",\"\"],\"i\":false,\"f\":[[[\"\",{\"children\":[\"blog\",{\"children\":[[\"slug\",\"codeqwen1.5\",\"d\"],{\"children\":[\"__PAGE__\",{}]}]}]},\"$undefined\",\"$undefined\",true],[\"\",[\"$\",\"$1\",\"c\",{\"children\":[[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/css/1d1f6bc532e5f43f.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\",\"nonce\":\"$undefined\"}]],[\"$\",\"html\",null,{\"lang\":\"en\",\"children\":[[\"$\",\"head\",null,{\"children\":[[\"$\",\"link\",null,{\"rel\":\"icon\",\"type\":\"image/svg+xml\",\"href\":\"/favicon.svg\"}],[\"$\",\"link\",null,{\"rel\":\"alternate icon\",\"href\":\"/favicon.png\"}]]}],[\"$\",\"body\",null,{\"children\":[[\"$\",\"$L2\",null,{}],[\"$\",\"$L3\",null,{\"parallelRouterKey\":\"children\",\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L4\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":[[[\"$\",\"title\",null,{\"children\":\"404: This page could not be found.\"}],[\"$\",\"div\",null,{\"style\":{\"fontFamily\":\"system-ui,\\\"Segoe UI\\\",Roboto,Helvetica,Arial,sans-serif,\\\"Apple Color Emoji\\\",\\\"Segoe UI Emoji\\\"\",\"height\":\"100vh\",\"textAlign\":\"center\",\"display\":\"flex\",\"flexDirection\":\"column\",\"alignItems\":\"center\",\"justifyContent\":\"center\"},\"children\":[\"$\",\"div\",null,{\"children\":[[\"$\",\"style\",null,{\"dangerouslySetInnerHTML\":{\"__html\":\"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}\"}}],[\"$\",\"h1\",null,{\"className\":\"next-error-h1\",\"style\":{\"display\":\"inline-block\",\"margin\":\"0 20px 0 0\",\"padding\":\"0 23px 0 0\",\"fontSize\":24,\"fontWeight\":500,\"verticalAlign\":\"top\",\"lineHeight\":\"49px\"},\"children\":404}],[\"$\",\"div\",null,{\"style\":{\"display\":\"inline-block\"},\"children\":[\"$\",\"h2\",null,{\"style\":{\"fontSize\":14,\"fontWeight\":400,\"lineHeight\":\"49px\",\"margin\":0},\"children\":\"This page could not be found.\"}]}]]}]}]],[]],\"forbidden\":\"$undefined\",\"unauthorized\":\"$undefined\"}],[\"$\",\"footer\",null,{\"children\":[\"$\",\"div\",null,{\"className\":\"container\",\"children\":[[\"$\",\"div\",null,{\"className\":\"footer-content\",\"children\":[[\"$\",\"div\",null,{\"className\":\"footer-section\",\"children\":[[\"$\",\"h4\",null,{\"children\":\"Zen LM\"}],[\"$\",\"p\",null,{\"children\":\"Open models from Zoo Labs Foundation, a 501(c)(3) non-profit, for agentic coding on your own machine and marketing work. Served on api.hanzo.ai.\"}]]}],[\"$\",\"div\",null,{\"className\":\"footer-section\",\"children\":[[\"$\",\"h4\",null,{\"children\":\"Zen 6\"}],[\"$\",\"ul\",null,{\"children\":[[\"$\",\"li\",null,{\"children\":[\"$\",\"$L5\",null,{\"href\":\"/models#zen6\",\"children\":\"Zen 6\"}]}],[\"$\",\"li\",null,{\"children\":[\"$\",\"$L5\",null,{\"href\":\"/models#zen6\",\"children\":\"Zen 6 Flash\"}]}],[\"$\",\"li\",null,{\"children\":[\"$\",\"a\",null,{\"href\":\"https://hanzo.ai/research-access\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"children\":\"Zen 7 research preview\"}]}]]}]]}],[\"$\",\"div\",null,{\"className\":\"footer-section\",\"children\":[[\"$\",\"h4\",null,{\"children\":\"Earlier Generations\"}],[\"$\",\"ul\",null,{\"children\":[[\"$\",\"li\",null,{\"children\":[\"$\",\"$L5\",null,{\"href\":\"/models#zen5\",\"children\":\"Zen 5\"}]}],[\"$\",\"li\",null,{\"children\":[\"$\",\"$L5\",null,{\"href\":\"/models#zen4\",\"children\":\"Zen 4\"}]}],[\"$\",\"li\",null,{\"children\":[\"$\",\"$L5\",null,{\"href\":\"/models#zen3\",\"children\":\"Zen 3\"}]}]]}]]}],[\"$\",\"div\",null,{\"className\":\"footer-section\",\"children\":[[\"$\",\"h4\",null,{\"children\":\"Resources\"}],[\"$\",\"ul\",null,{\"children\":[[\"$\",\"li\",null,{\"children\":[\"$\",\"$L5\",null,{\"href\":\"/datasets\",\"children\":\"Training Data\"}]}],[\"$\",\"li\",null,{\"children\":[\"$\",\"a\",null,{\"href\":\"https://huggingface.co/zenlm\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"children\":\"HuggingFace\"}]}],[\"$\",\"li\",null,{\"children\":[\"$\",\"a\",null,{\"href\":\"https://github.com/zenlm\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"children\":\"GitHub\"}]}],[\"$\",\"li\",null,{\"children\":[\"$\",\"$L5\",null,{\"href\":\"/research\",\"children\":\"Research Papers\"}]}],[\"$\",\"li\",null,{\"children\":[\"$\",\"a\",null,{\"href\":\"https://api.hanzo.ai\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"children\":\"Zen API\"}]}]]}]]}]]}],[\"$\",\"div\",null,{\"className\":\"footer-bottom\",\"children\":[\"$\",\"p\",null,{\"children\":[\"© \",2026,\" Zoo Labs Foundation, a 501(c)(3) non-profit. Zen LM open models.\"]}]}]]}]}],[\"$\",\"$L6\",null,{}],[\"$\",\"script\",null,{\"src\":\"/assets/js/main.js\",\"async\":true}]]}]]}]]}],{\"children\":[\"blog\",[\"$\",\"$1\",\"c\",{\"children\":[null,[\"$\",\"$L3\",null,{\"parallelRouterKey\":\"children\",\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L4\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":\"$undefined\",\"forbidden\":\"$undefined\",\"unauthorized\":\"$undefined\"}]]}],{\"children\":[[\"slug\",\"codeqwen1.5\",\"d\"],[\"$\",\"$1\",\"c\",{\"children\":[null,[\"$\",\"$L3\",null,{\"parallelRouterKey\":\"children\",\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L4\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":\"$undefined\",\"forbidden\":\"$undefined\",\"unauthorized\":\"$undefined\"}]]}],{\"children\":[\"__PAGE__\",[\"$\",\"$1\",\"c\",{\"children\":[\"$L7\",[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/css/eb87e4f7aea490c6.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\",\"nonce\":\"$undefined\"}]],\"$L8\"]}],{},null,false]},null,false]},null,false]},null,false],\"$L9\",false]],\"m\":\"$undefined\",\"G\":[\"$a\",[]],\"s\":false,\"S\":true}\n"])</script><script>self.__next_f.push([1,"b:I[4431,[],\"OutletBoundary\"]\nd:I[5278,[],\"AsyncMetadataOutlet\"]\nf:I[4431,[],\"ViewportBoundary\"]\n11:I[4431,[],\"MetadataBoundary\"]\n12:\"$Sreact.suspense\"\n8:[\"$\",\"$Lb\",null,{\"children\":[\"$Lc\",[\"$\",\"$Ld\",null,{\"promise\":\"$@e\"}]]}]\n9:[\"$\",\"$1\",\"h\",{\"children\":[null,[[\"$\",\"$Lf\",null,{\"children\":\"$L10\"}],null],[\"$\",\"$L11\",null,{\"children\":[\"$\",\"div\",null,{\"hidden\":true,\"children\":[\"$\",\"$12\",null,{\"fallback\":null,\"children\":\"$L13\"}]}]}]]}]\n"])</script><script>self.__next_f.push([1,"14:T32c5,"])</script><script>self.__next_f.push([1,"\u003cp\u003e\u003ca href=\"https://github.com/QwenLM/CodeQwen1.5\"\u003eGITHUB\u003c/a\u003e\n\u003ca href=\"https://huggingface.co/Qwen\"\u003eHUGGING FACE\u003c/a\u003e\n\u003ca href=\"https://modelscope.cn/organization/qwen\"\u003eMODELSCOPE\u003c/a\u003e\n\u003ca href=\"https://huggingface.co/spaces/Qwen/CodeQwen1.5-7b-Chat-demo\"\u003eDEMO\u003c/a\u003e\n\u003ca href=\"https://discord.gg/yPEP2vHTu4\"\u003eDISCORD\u003c/a\u003e\u003c/p\u003e\n\u003ch1 id=\"introduction\"\u003eIntroduction\u003c/h1\u003e\n\u003cp\u003eThe advent of advanced programming tools, which harnesses the power of large language models (LLMs), has significantly enhanced programmer productivity and accuracy. Notwithstanding these advancements, dominant coding assistants like Github Copilot, built upon proprietary LLMs, pose notable challenges in terms of cost, privacy, security, and potential copyright infringement. Recognizing the imperative for a more transparent and accessible alternative, the open-source community has embarked on a concerted endeavor to develop open codeLLMs. This initiative has already given rise to several promising open-source models, including StarCoder2, CodeLlama, and DeepSeek-Coder, offering a path forward, albeit one that necessitates continued refinement.\u003c/p\u003e\n\u003cfigure\u003e\u003cimg src=\"https://qianwen-res.oss-accelerate-overseas.aliyuncs.com/assets/blog/codeqwen1.5/intro.png\" alt=\"\" loading=\"lazy\"\u003e\u003c/figure\u003e\n\u003cp\u003eToday, we are delighted to introduce a new member of the Qwen1.5 open-source family, the CodeQwen1.5-7B, a specialized codeLLM built upon the Qwen1.5 language model. CodeQwen1.5-7B has been pretrained with around 3 trillion tokens of code-related data. It supports an extensive repertoire of 92 programming languages, and it exhibits exceptional capacity in long-context understanding and generation with the ability to process information of 64K tokens. In terms of performance, CodeQwen1.5 demonstrates impressive capabilities in basic code generation, long-context modeliing, code editation and SQL. We believe this model can significantly enhance developer productivity and streamline software development workflows within diverse technological environments.\u003c/p\u003e\n\u003ch1 id=\"codeqwen-is-a-basic-coder\"\u003eCodeQwen is a Basic Coder\u003c/h1\u003e\n\u003cp\u003eCode generation is a key competence for large language models, as they are tasked with translating natural language instructions into executable code with unwavering precision. CodeQwen1.5, with only 7 billion parameters, has surpassed larger models in basic code generation capabilities, further narrowing the gap in coding proficiency between GPT-4 and opensource code LLMs. We conducted a thorough evaluation on HumanEval and MBPP to provide a clear and fair comparison as follows.\u003c/p\u003e\n\u003ctable style=\"{{textAlign:\" \u0026#x22;center\u0026#x22;}}=\"\"\u003e\n \u003ctbody\u003e\u003ctr style=\"{{fontWeight:\" \u0026#x22;bold\u0026#x22;}}=\"\"\u003e\n \u003ctd style=\"{{textAlign:\" \u0026#x22;left\u0026#x22;}}=\"\"\u003eModel\u003c/td\u003e\n \u003ctd style=\"{{textAlign:\" \u0026#x22;left\u0026#x22;}}=\"\"\u003eSize\u003c/td\u003e\n \u003ctd\u003e\n \u003cdiv\u003eHumanEval\u003c/div\u003e\n \u003cdiv class=\"cell-aux\"\u003e0-shot\u003c/div\u003e\n \u003c/td\u003e\n \u003ctd\u003e\n \u003cdiv\u003eHumanEval+\u003c/div\u003e\n \u003cdiv class=\"cell-aux\"\u003e0-shot\u003c/div\u003e\n \u003c/td\u003e\n \u003ctd\u003e\n \u003cdiv\u003eMBPP\u003c/div\u003e\n \u003cdiv class=\"cell-aux\"\u003e0-shot\u003c/div\u003e\n \u003c/td\u003e\n \u003ctd\u003e\n \u003cdiv\u003eMBPP+\u003c/div\u003e\n \u003cdiv class=\"cell-aux\"\u003e0-shot\u003c/div\u003e\n \u003c/td\u003e\n \u003ctd\u003e\n \u003cdiv\u003eMBPP\u003c/div\u003e\n \u003cdiv class=\"cell-aux\"\u003e3-shot\u003c/div\u003e\n \u003c/td\u003e\n \u003c/tr\u003e\n \u003ctr\u003e\n \u003ctd colspan=\"7\"\u003e\u003cb\u003eBase Model\u003c/b\u003e\u003c/td\u003e\n \u003c/tr\u003e\n \u003ctr\u003e\n \u003ctd style=\"{{textAlign:\" \u0026#x22;left\u0026#x22;}}=\"\"\u003eCodeLlama-Base\u003c/td\u003e\n \u003ctd style=\"{{textAlign:\" \u0026#x22;left\u0026#x22;}}=\"\"\u003e7B\u003c/td\u003e\n \u003ctd\u003e33.5\u003c/td\u003e\n \u003ctd\u003e25.6\u003c/td\u003e\n \u003ctd\u003e52.1\u003c/td\u003e\n \u003ctd\u003e41.6\u003c/td\u003e\n \u003ctd\u003e38.6\u003c/td\u003e\n \u003c/tr\u003e\n \u003ctr\u003e\n \u003ctd style=\"{{textAlign:\" \u0026#x22;left\u0026#x22;}}=\"\"\u003eStarCoder2\u003c/td\u003e\n \u003ctd style=\"{{textAlign:\" \u0026#x22;left\u0026#x22;}}=\"\"\u003e7B\u003c/td\u003e\n \u003ctd\u003e35.4\u003c/td\u003e\n \u003ctd\u003e29.9\u003c/td\u003e\n \u003ctd\u003e54.4\u003c/td\u003e\n \u003ctd\u003e45.6\u003c/td\u003e\n \u003ctd\u003e51.0\u003c/td\u003e\n \u003c/tr\u003e\n \u003ctr\u003e\n \u003ctd style=\"{{textAlign:\" \u0026#x22;left\u0026#x22;}}=\"\"\u003eDeepSeek-Coder-Base\u003c/td\u003e\n \u003ctd style=\"{{textAlign:\" \u0026#x22;left\u0026#x22;}}=\"\"\u003e6.7B\u003c/td\u003e\n \u003ctd\u003e47.6\u003c/td\u003e\n \u003ctd\u003e39.6\u003c/td\u003e\n \u003ctd\u003e70.2\u003c/td\u003e\n \u003ctd\u003e56.6\u003c/td\u003e\n \u003ctd\u003e60.6\u003c/td\u003e\n \u003c/tr\u003e\n \u003ctr\u003e\n \u003ctd style=\"{{textAlign:\" \u0026#x22;left\u0026#x22;}}=\"\"\u003e\u003cb\u003eCodeQwen1.5\u003c/b\u003e\u003c/td\u003e\n \u003ctd style=\"{{textAlign:\" \u0026#x22;left\u0026#x22;}}=\"\"\u003e7B\u003c/td\u003e\n \u003ctd\u003e51.8\u003c/td\u003e\n \u003ctd\u003e45.7\u003c/td\u003e\n \u003ctd\u003e72.2\u003c/td\u003e\n \u003ctd\u003e60.2\u003c/td\u003e\n \u003ctd\u003e61.8\u003c/td\u003e\n \u003c/tr\u003e\n \u003ctr\u003e\n \u003ctd colspan=\"7\"\u003e\u003cb\u003eChat Model\u003c/b\u003e\u003c/td\u003e\n \u003c/tr\u003e\n \u003ctr\u003e\n \u003ctd style=\"{{textAlign:\" \u0026#x22;left\u0026#x22;}}=\"\"\u003eGPT-3.5-Turbo\u003c/td\u003e\n \u003ctd style=\"{{textAlign:\" \u0026#x22;left\u0026#x22;}}=\"\"\u003e-\u003c/td\u003e\n \u003ctd\u003e76.8\u003c/td\u003e\n \u003ctd\u003e70.7\u003c/td\u003e\n \u003ctd\u003e82.5\u003c/td\u003e\n \u003ctd\u003e69.7\u003c/td\u003e\n \u003ctd\u003e70.8\u003c/td\u003e\n \u003c/tr\u003e\n \u003ctr\u003e\n \u003ctd style=\"{{textAlign:\" \u0026#x22;left\u0026#x22;}}=\"\"\u003eGPT-4-Turbo (Nov 2023)\u003c/td\u003e\n \u003ctd style=\"{{textAlign:\" \u0026#x22;left\u0026#x22;}}=\"\"\u003e-\u003c/td\u003e\n \u003ctd\u003e85.4\u003c/td\u003e\n \u003ctd\u003e81.7\u003c/td\u003e\n \u003ctd\u003e83.5\u003c/td\u003e\n \u003ctd\u003e70.7\u003c/td\u003e\n \u003ctd\u003e80.0\u003c/td\u003e\n \u003c/tr\u003e\n \u003ctr\u003e\n \u003ctd style=\"{{textAlign:\" \u0026#x22;left\u0026#x22;}}=\"\"\u003eDeepSeek-Coder-Instruct\u003c/td\u003e\n \u003ctd style=\"{{textAlign:\" \u0026#x22;left\u0026#x22;}}=\"\"\u003e6.7B\u003c/td\u003e\n \u003ctd\u003e78.6\u003c/td\u003e\n \u003ctd\u003e70.1\u003c/td\u003e\n \u003ctd\u003e73.2\u003c/td\u003e\n \u003ctd\u003e63.4\u003c/td\u003e\n \u003ctd\u003e65.4\u003c/td\u003e\n \u003c/tr\u003e\n \u003ctr\u003e\n \u003ctd style=\"{{textAlign:\" \u0026#x22;left\u0026#x22;}}=\"\"\u003e\u003cb\u003eCodeQwen1.5-Chat\u003c/b\u003e\u003c/td\u003e\n \u003ctd style=\"{{textAlign:\" \u0026#x22;left\u0026#x22;}}=\"\"\u003e7B\u003c/td\u003e\n \u003ctd\u003e83.5\u003c/td\u003e\n \u003ctd\u003e78.7\u003c/td\u003e\n \u003ctd\u003e77.7\u003c/td\u003e\n \u003ctd\u003e67.2\u003c/td\u003e\n \u003ctd\u003e70.6\u003c/td\u003e\n \u003c/tr\u003e\n\u003c/tbody\u003e\u003c/table\u003e\n\u003cp\u003eIn addition to the widely recognized HumanEval and MBPP benchmarks, we explored LiveCodeBench. This benchmark assesses code performance by introducing fresh challenges sourced from coding competitions such as LeetCode, AtCoder, and CodeForces over time. Our evaluation of CodeQwen1.5 on LiveCodeBench spanned from September 1, 2023, to April 1, 2024. The findings indicate that CodeQwen1.5 ranks among the top open-access models currently available. Note: it is possible that the inclusion of LeetCode data in our pretraining corpus may contribute to the performance in LiveCodeBench.\u003c/p\u003e\n\u003cfigure\u003e\u003cimg src=\"https://qianwen-res.oss-accelerate-overseas.aliyuncs.com/assets/blog/codeqwen1.5/livecode.png\" alt=\"\" loading=\"lazy\"\u003e\u003c/figure\u003e\n\u003cp\u003eThe evaluations mentioned primarily revolve around Python capabilities; however, CodeQwen1.5 is not merely a Python specialist but also an expert across multiple programming languages. We conducted a comprehensive evaluation of CodeQwen1.5 in the eight mainstream languages featured in MultiPL-E, including Python, C++, Java, PHP, TypeScript, C#, Bash, and JavaScript. The results highlight the exceptional programming capabilities of CodeQwen1.5.\u003c/p\u003e\n\u003cfigure\u003e\u003cimg src=\"https://qianwen-res.oss-accelerate-overseas.aliyuncs.com/assets/blog/codeqwen1.5/radar-vertical.png\" alt=\"\" loading=\"lazy\"\u003e\u003c/figure\u003e\n\u003ch1 id=\"codeqwen-is-a-long-context-coder\"\u003eCodeQwen is a Long Context Coder\u003c/h1\u003e\n\u003cp\u003eLong context capability is crucial for code LLMs, serving as the core skill for understanding repository-level code and becoming a code agent. However, current code models still have very limited support for length, which hinders their potential for practical application. CodeQwen1.5 aims to further advance the progress of open-source code models in long context modeling. To achieve this, we have collected and constructed long sequence code data at the repository level for pre-training. Through careful data proportioning and organization, we have enabled it to support input lengths of up to 64K tokens.\u003c/p\u003e\n\u003cp\u003e\u003cstrong\u003eEvaluation 1\u003c/strong\u003e: We collected high-quality repo from GitHub Trending repositories on 2024-3-28 that were not included in CodeQwen1.5's training data to observe the effectiveness of long context modeling. The following figure demonstrates that as the sequence length increases, CodeQwen1.5's Perplexity (PPL) still manages to maintain a downward trend.\u003c/p\u003e\n\u003cfigure\u003e\u003cimg src=\"https://qianwen-res.oss-accelerate-overseas.aliyuncs.com/assets/blog/codeqwen1.5/longcontext-ppl.png\" alt=\"\" loading=\"lazy\"\u003e\u003c/figure\u003e\n\u003cp\u003e\u003cstrong\u003eEvaluation 2\u003c/strong\u003e: We created a synthetic task called \u003ccode\u003eNeedle in the Code\u003c/code\u003e, inspired by popular long-context evaluations in the text domain. In this task, we inserted a very simple custom function at various positions within a longer codebase (we chose Megatron to honor its contributions to open-source LLMs!) and tested whether the model could replicate this function at the end of the codebase. The figure below shows that CodeQwen is capable of successfully completing this task within a 64k length range.\u003c/p\u003e\n\u003cfigure\u003e\u003cimg src=\"https://qianwen-res.oss-accelerate-overseas.aliyuncs.com/assets/blog/codeqwen1.5/longcontext-needle.png\" alt=\"\" loading=\"lazy\"\u003e\u003c/figure\u003e\n\u003cp\u003eBoth Evaluation 1 and Evaluation 2 serve as initial and foundational assessments. For the Chat model, we aim to evaluate its long context capabilities with more practical tasks. However, our objective is to examine the Chat model's capability to handle long contexts through more pragmatic, real-world evaluation tasks.\u003c/p\u003e\n\u003cp\u003e\u003cstrong\u003eEvaluation 3\u003c/strong\u003e: SWE Bench is a benchmark designed to assess the ability of Large Language Models (LLMs) or agents to tackle practical software development challenges. It presents contestants with a code repository and an associated issue, tasking them with generating a commit patch that resolves the issue effectively. The benchmark uniquely emphasizes the long-context processing capabilities of code LLMs, necessitating both deep comprehension of the given codebase and the generation of extensive, unit-test-passing code.\u003c/p\u003e\n\u003cfigure\u003e\u003cimg src=\"https://qianwen-res.oss-accelerate-overseas.aliyuncs.com/assets/blog/codeqwen1.5/swe-bench.png\" alt=\"\" loading=\"lazy\"\u003e\u003c/figure\u003e\n\u003cp\u003eCurrently, participants in the SWE Bench competition predominantly are proprietary models. We introduce CodeQwen1.5 as an open-source model entry. Despite achieving a score of 0.89, CodeQwen1.5 surpasses ChatGPT-3.5, demonstrating the nascent yet promising competitiveness of open-source code models against their proprietary counterparts.\u003c/p\u003e\n\u003ch1 id=\"codeqwen-is-a-debugger\"\u003eCodeQwen is a Debugger\u003c/h1\u003e\n\u003cp\u003eAn effective code assistant must demonstrate proficiency in both generating code in response to given specifications and adeptly modifying or debugging existing code to accommodate evolving requirements or rectify errors. In assessing CodeQwen1.5's proficiency in code modification tasks, we concentrated our evaluation on the CodeEditorBench suite, encompassing four distinct dimensions: Debugging, Translation, Language Switching, and Code Polishing. The results indicate that CodeQwen1.5 achieves the SOTA performance at the 7 billion parameter scale.\u003c/p\u003e\n\u003cfigure\u003e\u003cimg src=\"https://qianwen-res.oss-accelerate-overseas.aliyuncs.com/assets/blog/codeqwen1.5/codeedit.jpeg\" alt=\"\" loading=\"lazy\"\u003e\u003c/figure\u003e\n\u003ch1 id=\"codeqwen-is-a-sqler\"\u003eCodeQwen is a SQLer\u003c/h1\u003e\n\u003cp\u003eCodeQwen1.5 serves as a solution to bridge the gap between non-programming professionals and efficient data interaction. It alleviates the steep learning curve associated with SQL by enabling users without coding expertise to query databases through natural language. We evaluated CodeQwen1.5-Chat's performance on two popular Text-to-SQL benchmarks, Spider and Bird. Experimental results pose CodeQwen1.5 a second position close to GPT-4 (results come from DIN-SQL, a SOTA prompting method). This outstanding performance is attributed to the utilization of synthetic data throughout both pre-training and fine-tuning stages. Synthetic data, characterized by its scalability, verifiability, and variety, emerges as a compelling area for future research due to its proven effectiveness in enhancing CodeQwen1.5's SQL capabilities.\u003c/p\u003e\n\u003cfigure\u003e\u003cimg src=\"https://qianwen-res.oss-accelerate-overseas.aliyuncs.com/assets/blog/codeqwen1.5/sql-score.png\" alt=\"\" loading=\"lazy\"\u003e\u003c/figure\u003e\n\u003ch1 id=\"develop-with-codeqwen15\"\u003eDevelop with CodeQwen1.5\u003c/h1\u003e\n\u003cp\u003eCodeQwen1.5 is part of the Qwen1.5 open-source family. We advise you to read our blog for \u003ca href=\"https://qwenlm.github.io/blog/qwen1.5/\"\u003eQwen1.5\u003c/a\u003e to figure out the usages with Transformers, vLLM, llama.cpp, Ollama, etc.\u003c/p\u003e\n\u003ch1 id=\"conclusion\"\u003eConclusion\u003c/h1\u003e\n\u003cp\u003eWe have released CodeQwen1.5-7B and CodeQwen1.5-7B-Chat, an open and versatile code LLM. The models are intended to aid progress in code assistance and code agents, benefiting the research community. We'll keep investing heavily in smart code development, with the ultimate goal of creating AI programmers.\u003c/p\u003e\n\u003ch1 id=\"citation\"\u003eCitation\u003c/h1\u003e\n\u003cpre\u003e\u003ccode\u003e@misc{codeqwen1.5,\n title = {Code with CodeQwen1.5},\n url = {https://qwenlm.github.io/blog/codeqwen1.5/},\n author = {Qwen Team},\n month = {April},\n year = {2024}\n}\n\u003c/code\u003e\u003c/pre\u003e"])</script><script>self.__next_f.push([1,"7:[\"$\",\"main\",null,{\"children\":[\"$\",\"article\",null,{\"className\":\"blog-article\",\"data-upstream\":\"\",\"children\":[[\"$\",\"$L5\",null,{\"className\":\"blog-back\",\"href\":\"/blog\",\"children\":\"← Blog\"}],[\"$\",\"div\",null,{\"className\":\"blog-post-meta\",\"children\":[\"April 15, 2024\",\" \",\"·\",\" \",7,\" min read\"]}],[\"$\",\"h1\",null,{\"className\":\"blog-post-title\",\"children\":\"Code with CodeQwen1.5\"}],[\"$\",\"p\",null,{\"className\":\"blog-post-lede\",\"children\":\"The advent of advanced programming tools, which harnesses the power of large language models (LLMs), has significantly enhanced programmer productivity and accuracy. Notwithstanding these advancements, dominant coding assistants like Github Copilot, built upon proprietary LLMs, pose notable challeng\"}],[\"$\",\"div\",null,{\"className\":\"blog-prose\",\"dangerouslySetInnerHTML\":{\"__html\":\"$14\"}}]]}]}]\n"])</script><script>self.__next_f.push([1,"10:[[\"$\",\"meta\",\"0\",{\"charSet\":\"utf-8\"}],[\"$\",\"meta\",\"1\",{\"name\":\"viewport\",\"content\":\"width=device-width, initial-scale=1\"}]]\nc:null\n"])</script><script>self.__next_f.push([1,"e:{\"metadata\":[[\"$\",\"title\",\"0\",{\"children\":\"Code with CodeQwen1.5 — Zen Blog\"}],[\"$\",\"meta\",\"1\",{\"name\":\"description\",\"content\":\"The advent of advanced programming tools, which harnesses the power of large language models (LLMs), has significantly enhanced programmer productivity and accuracy. Notwithstanding these advancements, dominant coding assistants like Github Copilot, built upon proprietary LLMs, pose notable challeng\"}],[\"$\",\"meta\",\"2\",{\"name\":\"keywords\",\"content\":\"AI, LLM, Agentic AI, Code Generation, Zen Coder, Multimodal, Open Source, Machine Learning\"}],[\"$\",\"meta\",\"3\",{\"name\":\"data-upstream\",\"content\":\"true\"}]],\"error\":null,\"digest\":\"$undefined\"}\n"])</script><script>self.__next_f.push([1,"13:\"$e:metadata\"\n"])</script></body></html> |