Files
zeekay 1f308322ab Keep wide tables and specs inside a phone screen
At 390 pixels the blog's tables, the datasets tables and the model
card specs ran past the right edge, and body's overflow-x clip cut
them off rather than letting them scroll. Tables now scroll in their
own box, and a spec column shrinks and wraps instead of pushing its
neighbour out of the card.
2026-09-30 02:41:46 -07:00

160 lines
72 KiB
HTML

<!DOCTYPE html><!--_BoRG_r6Z3v9udL0lI0J_--><html lang="en"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="stylesheet" href="/_next/static/css/1d1f6bc532e5f43f.css" data-precedence="next"/><link rel="stylesheet" href="/_next/static/css/eb87e4f7aea490c6.css" data-precedence="next"/><link rel="preload" as="script" fetchPriority="low" href="/_next/static/chunks/webpack-f43ad49c79db3299.js"/><script src="/_next/static/chunks/4bd1b696-c023c6e3521b1417.js" async=""></script><script src="/_next/static/chunks/255-566af14aff71e752.js" async=""></script><script src="/_next/static/chunks/main-app-7eaab9f523b3767f.js" async=""></script><script src="/_next/static/chunks/619-ba102abea3e3d0e4.js" async=""></script><script src="/_next/static/chunks/app/layout-425ca81883167c16.js" async=""></script><script src="/_next/static/chunks/app/blog/%5Bslug%5D/page-38f8c2a205f98d97.js" async=""></script><script src="/assets/js/main.js" async=""></script><link rel="icon" type="image/svg+xml" href="/favicon.svg"/><link rel="alternate icon" href="/favicon.png"/><title>OFA: Towards Building a One-For-All Model — Zen Blog</title><meta name="description" content="2022 is a year of generalist models! With the bloom of multimodal pretraining, especially the unified model, we have witnessed the opportunity to building a generalist model that is capable of processing tasks of different modalities or multi-modalities! Thus, we propose OFA[^1], namely One-For-All,"/><meta name="keywords" content="AI, LLM, Agentic AI, Code Generation, Zen Coder, Multimodal, Open Source, Machine Learning"/><meta name="data-upstream" content="true"/><script src="/_next/static/chunks/polyfills-42372ed130431b0a.js" noModule=""></script></head><body><div hidden=""><!--$--><!--/$--></div><header><nav class="navbar"><div class="container"><div class="logo"><a href="/"><svg class="enso logo-img" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 100 100" width="34" height="34" fill="none" role="img" aria-label="Zen ensō"><defs><linearGradient id="_R_adb_" x1="12%" y1="0%" x2="88%" y2="100%"><stop offset="0%" stop-color="#ffffff"></stop><stop offset="50%" stop-color="#d4d4d8"></stop><stop offset="100%" stop-color="#7c7c82"></stop></linearGradient></defs><path class="enso-stroke" pathLength="1" d="M58 13 C38 9 17 22 13 44 C9 68 26 89 50 90 C74 91 91 74 89 50 C88 41 87 35 84 30" stroke="url(#_R_adb_)" stroke-width="7" stroke-linecap="round" stroke-linejoin="round"></path></svg><div class="logo-text"><h1>Zen LM</h1></div></a></div><div class="header-right"><div class="nav-links"><a class="" href="/">Home</a><div class="nav-item"><a class="" href="/models/">Models<svg class="nav-caret" width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M6 9l6 6 6-6"></path></svg></a><div class="nav-dropdown"><div class="nav-dropdown-inner"><a class="dd-item" href="/models/#zen6"><div class="dd-item-text"><span class="dd-item-label">Zen 6<span class="dd-tag">Available</span></span><span class="dd-item-desc">27B dense — text, images and video, 1M context with YaRN</span></div><svg class="dd-arrow" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M5 12h14M13 6l6 6-6 6"></path></svg></a><a class="dd-item" href="/models/#zen6"><div class="dd-item-text"><span class="dd-item-label">Zen 6 Flash<span class="dd-tag">Available</span></span><span class="dd-item-desc">The ternary Zen 6 — 5.95 GB, fits an Apple Silicon laptop</span></div><svg class="dd-arrow" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M5 12h14M13 6l6 6-6 6"></path></svg></a><a class="dd-item" href="https://hanzo.ai/research-access" target="_blank" rel="noopener noreferrer"><div class="dd-item-text"><span class="dd-item-label">Zen 7<span class="dd-tag">Research preview</span></span><span class="dd-item-desc">No weights yet — request access</span></div><svg class="dd-arrow" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M5 12h14M13 6l6 6-6 6"></path></svg></a><a class="dd-item" href="/models/#zen5"><div class="dd-item-text"><span class="dd-item-label">Earlier generations</span><span class="dd-item-desc">Zen 5, Zen 4 and Zen 3</span></div><svg class="dd-arrow" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M5 12h14M13 6l6 6-6 6"></path></svg></a><a class="dd-item" href="/models/#catalog"><div class="dd-item-text"><span class="dd-item-label">All models</span><span class="dd-item-desc">Browse the full catalog</span></div><svg class="dd-arrow" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M5 12h14M13 6l6 6-6 6"></path></svg></a></div></div></div><div class="nav-item"><a class="" href="/datasets/">Datasets<svg class="nav-caret" width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M6 9l6 6 6-6"></path></svg></a><div class="nav-dropdown"><div class="nav-dropdown-inner"><a class="dd-item" href="/datasets/"><div class="dd-item-text"><span class="dd-item-label">Zen Agentic Dataset<span class="dd-tag">New</span></span><span class="dd-item-desc">10B+ tokens of real tool use and reasoning</span></div><svg class="dd-arrow" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M5 12h14M13 6l6 6-6 6"></path></svg></a><a class="dd-item" href="/datasets/"><div class="dd-item-text"><span class="dd-item-label">Training Data</span><span class="dd-item-desc">Curation, sources, and methodology</span></div><svg class="dd-arrow" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M5 12h14M13 6l6 6-6 6"></path></svg></a><a class="dd-item" href="https://huggingface.co/datasets/hanzoai/zen-agentic-dataset" target="_blank" rel="noopener noreferrer"><div class="dd-item-text"><span class="dd-item-label">On HuggingFace</span><span class="dd-item-desc">Download and license the datasets</span></div><svg class="dd-arrow" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M5 12h14M13 6l6 6-6 6"></path></svg></a></div></div></div><div class="nav-item"><a class="" href="/research/">Research<svg class="nav-caret" width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M6 9l6 6 6-6"></path></svg></a><div class="nav-dropdown"><div class="nav-dropdown-inner"><a class="dd-item" href="/research/"><div class="dd-item-text"><span class="dd-item-label">Research Papers</span><span class="dd-item-desc">Technical reports across the Zen family</span></div><svg class="dd-arrow" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M5 12h14M13 6l6 6-6 6"></path></svg></a><a class="dd-item" href="https://papers.zenlm.org" target="_blank" rel="noopener noreferrer"><div class="dd-item-text"><span class="dd-item-label">Papers Archive</span><span class="dd-item-desc">Full paper archive — papers.zenlm.org</span></div><svg class="dd-arrow" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M5 12h14M13 6l6 6-6 6"></path></svg></a><a class="dd-item" href="/blog/"><div class="dd-item-text"><span class="dd-item-label">Blog</span><span class="dd-item-desc">Research notes, deep dives, and releases</span></div><svg class="dd-arrow" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M5 12h14M13 6l6 6-6 6"></path></svg></a></div></div></div><a class="active" href="/blog/">Blog</a></div><div class="logo-links"><a href="https://github.com/zenlm" target="_blank" rel="noopener noreferrer" class="icon-link" title="GitHub"><svg width="24" height="24" viewBox="0 0 24 24" fill="currentColor"><path d="M12 0c-6.626 0-12 5.373-12 12 0 5.302 3.438 9.8 8.207 11.387.599.111.793-.261.793-.577v-2.234c-3.338.726-4.033-1.416-4.033-1.416-.546-1.387-1.333-1.756-1.333-1.756-1.089-.745.083-.729.083-.729 1.205.084 1.839 1.237 1.839 1.237 1.07 1.834 2.807 1.304 3.492.997.107-.775.418-1.305.762-1.604-2.665-.305-5.467-1.334-5.467-5.931 0-1.311.469-2.381 1.236-3.221-.124-.303-.535-1.524.117-3.176 0 0 1.008-.322 3.301 1.23.957-.266 1.983-.399 3.003-.404 1.02.005 2.047.138 3.006.404 2.291-1.552 3.297-1.23 3.297-1.23.653 1.653.242 2.874.118 3.176.77.84 1.235 1.911 1.235 3.221 0 4.609-2.807 5.624-5.479 5.921.43.372.823 1.102.823 2.222v3.293c0 .319.192.694.801.576 4.765-1.589 8.199-6.086 8.199-11.386 0-6.627-5.373-12-12-12z"></path></svg></a><a href="https://huggingface.co/zenlm" target="_blank" rel="noopener noreferrer" class="icon-link" title="HuggingFace"><svg width="28" height="28" viewBox="0 0 32 32" fill="currentColor" aria-hidden="true"><circle cx="16" cy="16" r="13" fill="#d4d4d8"></circle><circle cx="11" cy="14" r="2" fill="#18181b"></circle><circle cx="21" cy="14" r="2" fill="#18181b"></circle><path d="M11 19 Q16 24 21 19" stroke="#18181b" stroke-width="1.6" fill="none" stroke-linecap="round"></path></svg></a></div><div class="nav-item try-item"><a href="https://hanzo.chat/?model=zen6" target="_blank" rel="noopener noreferrer" class="btn-try">Try Zen<svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M5 12h14M13 6l6 6-6 6"></path></svg></a><div class="nav-dropdown nav-dropdown-right"><div class="nav-dropdown-inner"><a class="dd-item" href="https://hanzo.chat/?model=zen6" target="_blank" rel="noopener noreferrer"><div class="dd-item-text"><span class="dd-item-label">Zen 6<span class="dd-tag">Available</span></span><span class="dd-item-desc">Chat with Zen 6</span></div><svg class="dd-arrow" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M5 12h14M13 6l6 6-6 6"></path></svg></a><a class="dd-item" href="https://hanzo.chat/?model=zen6-flash" target="_blank" rel="noopener noreferrer"><div class="dd-item-text"><span class="dd-item-label">Zen 6 Flash<span class="dd-tag">Available</span></span><span class="dd-item-desc">Chat with the ternary Zen 6</span></div><svg class="dd-arrow" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M5 12h14M13 6l6 6-6 6"></path></svg></a><a class="dd-item" href="https://api.hanzo.ai" target="_blank" rel="noopener noreferrer"><div class="dd-item-text"><span class="dd-item-label">Zen API</span><span class="dd-item-desc">OpenAI- &amp; Anthropic-compatible. One key.</span></div><svg class="dd-arrow" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M5 12h14M13 6l6 6-6 6"></path></svg></a><a class="dd-item" href="https://hanzo.chat" target="_blank" rel="noopener noreferrer"><div class="dd-item-text"><span class="dd-item-label">Zen Chat</span><span class="dd-item-desc">Open the full chat experience</span></div><svg class="dd-arrow" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M5 12h14M13 6l6 6-6 6"></path></svg></a></div></div></div></div></div></nav></header><main><article class="blog-article" data-upstream=""><a class="blog-back" href="/blog/">← Blog</a><div class="blog-post-meta">November 13, 2022<!-- --> <!-- -->·<!-- --> <!-- -->10<!-- --> min read</div><h1 class="blog-post-title">OFA: Towards Building a One-For-All Model</h1><p class="blog-post-lede">2022 is a year of generalist models! With the bloom of multimodal pretraining, especially the unified model, we have witnessed the opportunity to building a generalist model that is capable of processing tasks of different modalities or multi-modalities! Thus, we propose OFA[^1], namely One-For-All,</p><div class="blog-prose"><p><video src="https://qianwen-res.oss-accelerate-overseas.aliyuncs.com/assets/blog/ofa/demo.mp4" controls playsinline preload="metadata" style="width:100%"></video></p>
<p>2022 is a year of generalist models! With the bloom of multimodal pretraining, especially the unified model, we have witnessed the opportunity to building a generalist model that is capable of processing tasks of different modalities or multi-modalities! Thus, we propose OFA<sup><a href="#user-content-fn-1" id="user-content-fnref-1" data-footnote-ref="" aria-describedby="footnote-label">1</a></sup>, namely One-For-All, a unified multimodal pretrained model that unifies understanding and generation tasks concerning modalities into a single framework, and we pretrain OFA with the instruction-based multitask-pretraining that endows it with multiple capabilities. We opensourced both the pretrained and finetuned models to the community, hoping this pioneer work can help accelerate the development of generalist models.</p>
<p><a href="https://arxiv.org/abs/2202.03052">Paper</a>
<a href="https://github.com/OFA-Sys/OFA">Github</a>
<a href="https://www.modelscope.cn/models?name=ofa">ModelScope</a>
<a href="https://huggingface.co/spaces/OFA-Sys/OFA-Generic_Interface">Demo</a></p>
<h2 id="background">Background</h2>
<figure><img src="https://qianwen-res.oss-accelerate-overseas.aliyuncs.com/assets/blog/ofa/uniter.jpg" alt="" loading="lazy"></figure>
<p>Multimodal pretraining has been developing rapidly ever since the transfer of BERT<sup><a href="#user-content-fn-bert" id="user-content-fnref-bert" data-footnote-ref="" aria-describedby="footnote-label">2</a></sup> to cross-modal representation learning. Representative studies include UNITER<sup><a href="#user-content-fn-2" id="user-content-fnref-2" data-footnote-ref="" aria-describedby="footnote-label">3</a></sup>, VilBERT<sup><a href="#user-content-fn-3" id="user-content-fnref-3" data-footnote-ref="" aria-describedby="footnote-label">4</a></sup>, etc. These studies directly incorporate the Transformer-based BERT<sup><a href="#user-content-fn-bert" id="user-content-fnref-bert-2" data-footnote-ref="" aria-describedby="footnote-label">2</a></sup> to a single-stream or dual stream framework for multimodal pretraining, and transform the image to a sequence of object features to be concatenated with the word embeddings as the input of Transformer. Later in 2021, with the rise of Vision Transformer<sup><a href="#user-content-fn-5" id="user-content-fnref-5" data-footnote-ref="" aria-describedby="footnote-label">5</a></sup>, there came methods that got rid of object-level features, which depend on complex preprocessing pipelines, say Faster-RCNN<sup><a href="#user-content-fn-6" id="user-content-fnref-6" data-footnote-ref="" aria-describedby="footnote-label">6</a></sup>: For example, the simplest ViLT<sup><a href="#user-content-fn-7" id="user-content-fnref-7" data-footnote-ref="" aria-describedby="footnote-label">7</a></sup> based on patch projection, the CLIP-based<sup><a href="#user-content-fn-8" id="user-content-fnref-8" data-footnote-ref="" aria-describedby="footnote-label">8</a></sup> CLIP-ViL<sup><a href="#user-content-fn-9" id="user-content-fnref-9" data-footnote-ref="" aria-describedby="footnote-label">9</a></sup>, etc. One milestone after should be the proposal of SimVLM<sup><a href="#user-content-fn-10" id="user-content-fnref-10" data-footnote-ref="" aria-describedby="footnote-label">10</a></sup>, which leverages the T5/BART method for multimodal pretraining and achieves new SoTA in many tasks. These progress should be regarded as the foundation of unified multimodal pretrained models in 2022, including OFA of ours, Unified-IO<sup><a href="#user-content-fn-11" id="user-content-fnref-11" data-footnote-ref="" aria-describedby="footnote-label">11</a></sup>, Flamingo<sup><a href="#user-content-fn-12" id="user-content-fnref-12" data-footnote-ref="" aria-describedby="footnote-label">12</a></sup>, BeiT-3<sup><a href="#user-content-fn-13" id="user-content-fnref-13" data-footnote-ref="" aria-describedby="footnote-label">13</a></sup>, etc.</p>
<h2 id="method">Method</h2>
<p>What OFA wants to achieve is the unification of tasks, modalities, and architecture. We suppose there are three features for a unified model, i.e., task agnostic, modality agnostic, and task comprehensiveness. To further explain them, "task agnostic" indicates that the unified model should be able to accept tasks without modifying its own architecture and training methods, "modality agnostic" indicates that a unified model should accept inputs of different modalities without knowing what they are and designing complex preprocessing, and "task comprehensiveness" indicates that the unified model should learn as many tasks as possible so that it can transfer to unseen tasks with the composition of existing capabilities. Thus, we propose 3 types of unification for OFA, namely the unification of modalities, architecture, and tasks. Let's figure them out one by one.</p>
<p>For the unification of modalities, one key issue is the tokenization of inputs of different modalities, or to say, the discretization. Otherwise, there should be other solutions like diffusion models<sup><a href="#user-content-fn-14" id="user-content-fnref-14" data-footnote-ref="" aria-describedby="footnote-label">14</a></sup> for the generation. There is no need to change the tokenization for texts, but the images and bounding boxes need to be discretized. Owing to the success of vector quantization<sup><a href="#user-content-fn-15" id="user-content-fnref-15" data-footnote-ref="" aria-describedby="footnote-label">15</a></sup><sup><a href="#user-content-fn-16" id="user-content-fnref-16" data-footnote-ref="" aria-describedby="footnote-label">16</a></sup> and text-to-image generation with Transformer<sup><a href="#user-content-fn-17" id="user-content-fnref-17" data-footnote-ref="" aria-describedby="footnote-label">17</a></sup><sup><a href="#user-content-fn-18" id="user-content-fnref-18" data-footnote-ref="" aria-describedby="footnote-label">18</a></sup>, images can be represented with VQ tokens. Inspired by pix2seq<sup><a href="#user-content-fn-19" id="user-content-fnref-19" data-footnote-ref="" aria-describedby="footnote-label">19</a></sup>, bounding boxes can also be discretized with bins.</p>
<figure><img src="https://qianwen-res.oss-accelerate-overseas.aliyuncs.com/assets/blog/ofa/io.jpg" alt="" loading="lazy"></figure>
<p>We choose the universal Transformer encoder-decoder architecture, due to its successful usages in NLP unified models like T5<sup><a href="#user-content-fn-20" id="user-content-fnref-20" data-footnote-ref="" aria-describedby="footnote-label">20</a></sup>. Note that for the input of images to the Transformer, we use the first three blocks of ResNet. For the Transformer architecture, we modify the design by incorporating Normformer<sup><a href="#user-content-fn-21" id="user-content-fnref-21" data-footnote-ref="" aria-describedby="footnote-label">21</a></sup> for the training stability and transfer performance.</p>
<figure><img src="https://qianwen-res.oss-accelerate-overseas.aliyuncs.com/assets/blog/ofa/arc.jpg" alt="" loading="lazy"></figure>
<p>The multitask learning is the key innovation of OFA. Specifically, we pretrain the model with 8 tasks, including 5 vision-language tasks, 2 vision tasks, and 1 language task. The vision-language tasks include visual grounding, grounded captioning, visual question answering, image-text matching, and image captioning. The vision tasks include detection and image infilling. The language task is text infilling. To help the model differentiate tasks, we insert an instruction, which is simply a piece of text describing the task. Thus, we expect the model to perform zero-shot generation based on a new instruction indicating an unseen task.</p>
<figure><img src="https://qianwen-res.oss-accelerate-overseas.aliyuncs.com/assets/blog/ofa/task.jpg" alt="" loading="lazy"></figure>
<p>To make this research as reproducible as possible, our pretraining is dependent on public datasets. Therefore, we expect the researchers following this work can reproduce our results with our opensourced code.</p>
<p>We have released OFA models of 5 sizes, including OFA-Tiny (33M), OFA-Medium (93M), OFA-Base (180M), OFA-Large(470M), OFA-Huge (930M). See the table below for more statistics.</p>
<figure><img src="https://qianwen-res.oss-accelerate-overseas.aliyuncs.com/assets/blog/ofa/variants.jpg" alt="" loading="lazy"></figure>
<h2 id="experiments">Experiments</h2>
<p>We have conducted experiments on multiple cross-modal tasks and unimodal tasks. On vision-language understanding, we test the models on VQA and SNLI-VE. We find that the huge-size model can achieve a comparable performance to the 80B-parameter model Flamingo and the 2B-parameter model CoCa pretrained on 5B image-text pairs. Furthermore, we achieve the best performance on visual entailment. For vision-language generation, we focus on the classical image captioning, and our OFA achieves the SoTA performance in both setups of cross-entropy optimization and CIDEr optimization. Also, we have transformed the task of visual grounding to a generation task, and we find that even the base-size OFA can outperform the previous SoTA, and the scaling of model size consistently brings performance improvements. This shows the significance of the unification of modalities and tasks.</p>
<figure><img src="https://qianwen-res.oss-accelerate-overseas.aliyuncs.com/assets/blog/ofa/vqa.jpg" alt="" loading="lazy"></figure>
<figure><img src="https://qianwen-res.oss-accelerate-overseas.aliyuncs.com/assets/blog/ofa/caption.jpg" alt="" loading="lazy"></figure>
<figure><img src="https://qianwen-res.oss-accelerate-overseas.aliyuncs.com/assets/blog/ofa/vg.jpg" alt="" loading="lazy"></figure>
<p>Additionally, we test OFA on text-to-image generation, as we believe that the image infilling task in pretraining endows it with the capability to generate image codes. We show that OFA can achieve a low FID score in the evaluation, and further finetuning on a larger dataset can significantly boost its performance. See cases below.</p>
<figure><img src="https://qianwen-res.oss-accelerate-overseas.aliyuncs.com/assets/blog/ofa/t2i.jpg" alt="" loading="lazy"></figure>
<figure><img src="https://qianwen-res.oss-accelerate-overseas.aliyuncs.com/assets/blog/ofa/t2i_cases.jpg" alt="" loading="lazy"></figure>
<p>As to the unimodal tasks, we evaluate OFA on the GLUE benchmark for NLU, Gigaword summarization for NLG, and ImageNet classification for vision understanding. We show that OFA can be competitive with both RoBERTa and DeBERTa, and the previous multimodal pretrained model often falls far behind the SoTAs in NLU. Similarly, OFA can achieve good performance on NLG and outperform the previous best models. As to image classification, it can also achieve similar performance with the self-supervised vision models like BeiT<sup><a href="#user-content-fn-22" id="user-content-fnref-22" data-footnote-ref="" aria-describedby="footnote-label">22</a></sup> and MAE<sup><a href="#user-content-fn-23" id="user-content-fnref-23" data-footnote-ref="" aria-describedby="footnote-label">23</a></sup>.</p>
<p>We observe that OFA based on multitask pretraining demonstrates potential in transferring to unseen tasks and unseen domains. We show them with two cases below.</p>
<figure><img src="https://qianwen-res.oss-accelerate-overseas.aliyuncs.com/assets/blog/ofa/unseen_task.jpg" alt="" loading="lazy"></figure>
<p>The preceding case demonstrates the model's ability of compositional generalization by understanding the instruction and leveraging two learned capabilities to perform the new task. We set up a new task called <em>Grounded VQA</em>, which is a combination of VQA and grounded captioning. What we need to change is the instruction. The new task instruction with both question and region information directs the model to provide a correct answer.</p>
<figure><img src="https://qianwen-res.oss-accelerate-overseas.aliyuncs.com/assets/blog/ofa/unseen_domain.jpg" alt="" loading="lazy"></figure>
<p>Also, we find that OFA can transfer to unseen domains effectively. One example is the visual grounding on images of animation. OFA can perform well in this setup as it has been pretrained on some anime data and it has been pretrained on visual grounding on general-domain data. This again shows the compositional ability of the unified model.</p>
<h2 id="conclusion">Conclusion</h2>
<p>This is the starting point of our research for the technically "One-For-All" model, or to say, the generalist model. We show that this research direction is promising as Transformer is a really powerful architecture and tasks and modalities can be unified to a single training framework. Like GPT-3<sup><a href="#user-content-fn-24" id="user-content-fnref-24" data-footnote-ref="" aria-describedby="footnote-label">24</a></sup>, we believe that there will soon be a powerful foundation model in multimodal representation learning.</p>
<section data-footnotes="" class="footnotes"><h2 class="sr-only" id="footnote-label">Footnotes</h2>
<ol>
<li id="user-content-fn-1">
<p>Wang, P., Yang, A., Men, R., Lin, J., Bai, S., Li, Z., Ma, J., Zhou, C., Zhou, J., &#x26; Yang, H. (2022).
Unifying Architectures, Tasks, and Modalities Through a Simple Sequence-to-Sequence Learning Framework.
International Conference on Machine Learning. <a href="#user-content-fnref-1" data-footnote-backref="" aria-label="Back to reference 1" class="data-footnote-backref">↩</a></p>
</li>
<li id="user-content-fn-bert">
<p>Devlin, J., Chang, M., Lee, K., &#x26; Toutanova, K. (2019).
BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding.
arXiv, abs/1810.04805. <a href="#user-content-fnref-bert" data-footnote-backref="" aria-label="Back to reference 2" class="data-footnote-backref">↩</a> <a href="#user-content-fnref-bert-2" data-footnote-backref="" aria-label="Back to reference 2-2" class="data-footnote-backref">↩<sup>2</sup></a></p>
</li>
<li id="user-content-fn-2">
<p>Chen, Y., Li, L., Yu, L., Kholy, A.E., Ahmed, F., Gan, Z., Cheng, Y., &#x26; Liu, J. (2019).
UNITER: UNiversal Image-TExt Representation Learning.
European Conference on Computer Vision. <a href="#user-content-fnref-2" data-footnote-backref="" aria-label="Back to reference 3" class="data-footnote-backref">↩</a></p>
</li>
<li id="user-content-fn-3">
<p>Lu, J., Batra, D., Parikh, D., &#x26; Lee, S. (2019).
ViLBERT: Pretraining Task-Agnostic Visiolinguistic Representations for Vision-and-Language Tasks.
Neural Information Processing Systems. <a href="#user-content-fnref-3" data-footnote-backref="" aria-label="Back to reference 4" class="data-footnote-backref">↩</a></p>
</li>
<li id="user-content-fn-5">
<p>Dosovitskiy, A., Beyer, L., Kolesnikov, A., Weissenborn, D., Zhai, X., Unterthiner, T., Dehghani, M., Minderer, M., Heigold, G., Gelly, S., Uszkoreit, J., &#x26; Houlsby, N. (2020).
An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale.
arXiv, abs/2010.11929. <a href="#user-content-fnref-5" data-footnote-backref="" aria-label="Back to reference 5" class="data-footnote-backref">↩</a></p>
</li>
<li id="user-content-fn-6">
<p>Ren, S., He, K., Girshick, R.B., &#x26; Sun, J. (2015).
Faster R-CNN: Towards Real-Time Object Detection with Region Proposal Networks.
IEEE Transactions on Pattern Analysis and Machine Intelligence, 39, 1137-1149. <a href="#user-content-fnref-6" data-footnote-backref="" aria-label="Back to reference 6" class="data-footnote-backref">↩</a></p>
</li>
<li id="user-content-fn-7">
<p>Kim, W., Son, B., &#x26; Kim, I. (2021).
ViLT: Vision-and-Language Transformer Without Convolution or Region Supervision.
International Conference on Machine Learning. <a href="#user-content-fnref-7" data-footnote-backref="" aria-label="Back to reference 7" class="data-footnote-backref">↩</a></p>
</li>
<li id="user-content-fn-8">
<p>Radford, A., Kim, J.W., Hallacy, C., Ramesh, A., Goh, G., Agarwal, S., Sastry, G., Askell, A., Mishkin, P., Clark, J., Krueger, G., &#x26; Sutskever, I. (2021).
Learning Transferable Visual Models From Natural Language Supervision.
International Conference on Machine Learning. <a href="#user-content-fnref-8" data-footnote-backref="" aria-label="Back to reference 8" class="data-footnote-backref">↩</a></p>
</li>
<li id="user-content-fn-9">
<p>Shen, S., Li, L.H., Tan, H., Bansal, M., Rohrbach, A., Chang, K., Yao, Z., &#x26; Keutzer, K. (2021).
How Much Can CLIP Benefit Vision-and-Language Tasks?
arXiv, abs/2107.06383. <a href="#user-content-fnref-9" data-footnote-backref="" aria-label="Back to reference 9" class="data-footnote-backref">↩</a></p>
</li>
<li id="user-content-fn-10">
<p>Wang, Z., Yu, J., Yu, A.W., Dai, Z., Tsvetkov, Y., &#x26; Cao, Y. (2021).
SimVLM: Simple Visual Language Model Pretraining with Weak Supervision.
arXiv, abs/2108.10904. <a href="#user-content-fnref-10" data-footnote-backref="" aria-label="Back to reference 10" class="data-footnote-backref">↩</a></p>
</li>
<li id="user-content-fn-11">
<p>Lu, J., Clark, C., Zellers, R., Mottaghi, R., &#x26; Kembhavi, A. (2022).
Unified-IO: A Unified Model for Vision, Language, and Multi-Modal Tasks.
arXiv, abs/2206.08916. <a href="#user-content-fnref-11" data-footnote-backref="" aria-label="Back to reference 11" class="data-footnote-backref">↩</a></p>
</li>
<li id="user-content-fn-12">
<p>Alayrac, J., Donahue, J., Luc, P., Miech, A., Barr, I., Hasson, Y., Lenc, K., Mensch, A., Millican, K., Reynolds, M., Ring, R., Rutherford, E., Cabi, S., Han, T., Gong, Z., Samangooei, S., Monteiro, M., Menick, J., Borgeaud, S., Brock, A., Nematzadeh, A., Sharifzadeh, S., Binkowski, M., Barreira, R., Vinyals, O., Zisserman, A., &#x26; Simonyan, K. (2022).
Flamingo: a Visual Language Model for Few-Shot Learning.
arXiv, abs/2204.14198. <a href="#user-content-fnref-12" data-footnote-backref="" aria-label="Back to reference 12" class="data-footnote-backref">↩</a></p>
</li>
<li id="user-content-fn-13">
<p>Wang, W., Bao, H., Dong, L., Bjorck, J., Peng, Z., Liu, Q., Aggarwal, K., Mohammed, O., Singhal, S., Som, S., &#x26; Wei, F. (2022).
Image as a Foreign Language: BEiT Pretraining for All Vision and Vision-Language Tasks.
arXiv, abs/2208.10442. <a href="#user-content-fnref-13" data-footnote-backref="" aria-label="Back to reference 13" class="data-footnote-backref">↩</a></p>
</li>
<li id="user-content-fn-14">
<p>Ho, J., Jain, A., &#x26; Abbeel, P. (2020).
Denoising Diffusion Probabilistic Models.
arXiv, abs/2006.11239. <a href="#user-content-fnref-14" data-footnote-backref="" aria-label="Back to reference 14" class="data-footnote-backref">↩</a></p>
</li>
<li id="user-content-fn-15">
<p>Razavi, A., Oord, A.V., &#x26; Vinyals, O. (2019).
Generating Diverse High-Fidelity Images with VQ-VAE-2.
arXiv, abs/1906.00446. <a href="#user-content-fnref-15" data-footnote-backref="" aria-label="Back to reference 15" class="data-footnote-backref">↩</a></p>
</li>
<li id="user-content-fn-16">
<p>Esser, P., Rombach, R., &#x26; Ommer, B. (2020).
Taming Transformers for High-Resolution Image Synthesis.
2021 IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR), 12868-12878. <a href="#user-content-fnref-16" data-footnote-backref="" aria-label="Back to reference 16" class="data-footnote-backref">↩</a></p>
</li>
<li id="user-content-fn-17">
<p>Ramesh, A., Pavlov, M., Goh, G., Gray, S., Voss, C., Radford, A., Chen, M., &#x26; Sutskever, I. (2021).
Zero-Shot Text-to-Image Generation.
arXiv, abs/2102.12092. <a href="#user-content-fnref-17" data-footnote-backref="" aria-label="Back to reference 17" class="data-footnote-backref">↩</a></p>
</li>
<li id="user-content-fn-18">
<p>Ding, M., Yang, Z., Hong, W., Zheng, W., Zhou, C., Yin, D., Lin, J., Zou, X., Shao, Z., Yang, H., &#x26; Tang, J. (2021).
CogView: Mastering Text-to-Image Generation via Transformers.
Neural Information Processing Systems. <a href="#user-content-fnref-18" data-footnote-backref="" aria-label="Back to reference 18" class="data-footnote-backref">↩</a></p>
</li>
<li id="user-content-fn-19">
<p>Chen, T., Saxena, S., Li, L., Fleet, D.J., &#x26; Hinton, G.R. (2021).
Pix2seq: A Language Modeling Framework for Object Detection.
arXiv, abs/2109.10852. <a href="#user-content-fnref-19" data-footnote-backref="" aria-label="Back to reference 19" class="data-footnote-backref">↩</a></p>
</li>
<li id="user-content-fn-20">
<p>Raffel, C., Shazeer, N.M., Roberts, A., Lee, K., Narang, S., Matena, M., Zhou, Y., Li, W., &#x26; Liu, P.J. (2019).
Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer.
arXiv, abs/1910.10683. <a href="#user-content-fnref-20" data-footnote-backref="" aria-label="Back to reference 20" class="data-footnote-backref">↩</a></p>
</li>
<li id="user-content-fn-21">
<p>Shleifer, S., Weston, J., &#x26; Ott, M. (2021).
NormFormer: Improved Transformer Pretraining with Extra Normalization.
arXiv, abs/2110.09456. <a href="#user-content-fnref-21" data-footnote-backref="" aria-label="Back to reference 21" class="data-footnote-backref">↩</a></p>
</li>
<li id="user-content-fn-22">
<p>Bao, H., Dong, L., &#x26; Wei, F. (2021).
BEiT: BERT Pre-Training of Image Transformers.
arXiv, abs/2106.08254. <a href="#user-content-fnref-22" data-footnote-backref="" aria-label="Back to reference 22" class="data-footnote-backref">↩</a></p>
</li>
<li id="user-content-fn-23">
<p>He, K., Chen, X., Xie, S., Li, Y., Doll'ar, P., &#x26; Girshick, R.B. (2021).
Masked Autoencoders Are Scalable Vision Learners.
2022 IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR), 15979-15988. <a href="#user-content-fnref-23" data-footnote-backref="" aria-label="Back to reference 23" class="data-footnote-backref">↩</a></p>
</li>
<li id="user-content-fn-24">
<p>Brown, T.B., Mann, B., Ryder, N., Subbiah, M., Kaplan, J., Dhariwal, P., Neelakantan, A., Shyam, P., Sastry, G., Askell, A., Agarwal, S., Herbert-Voss, A., Krueger, G., Henighan, T.J., Child, R., Ramesh, A., Ziegler, D.M., Wu, J., Winter, C., Hesse, C., Chen, M., Sigler, E., Litwin, M., Gray, S., Chess, B., Clark, J., Berner, C., McCandlish, S., Radford, A., Sutskever, I., &#x26; Amodei, D. (2020).
Language Models are Few-Shot Learners.
arXiv, abs/2005.14165. <a href="#user-content-fnref-24" data-footnote-backref="" aria-label="Back to reference 24" class="data-footnote-backref">↩</a></p>
</li>
</ol>
</section></div></article></main><!--$--><!--/$--><footer><div class="container"><div class="footer-content"><div class="footer-section"><h4>Zen LM</h4><p>Open models from Zoo Labs Foundation, a 501(c)(3) non-profit, for agentic coding on your own machine and marketing work. Served on api.hanzo.ai.</p></div><div class="footer-section"><h4>Zen 6</h4><ul><li><a href="/models/#zen6">Zen 6</a></li><li><a href="/models/#zen6">Zen 6 Flash</a></li><li><a href="https://hanzo.ai/research-access" target="_blank" rel="noopener noreferrer">Zen 7 research preview</a></li></ul></div><div class="footer-section"><h4>Earlier Generations</h4><ul><li><a href="/models/#zen5">Zen 5</a></li><li><a href="/models/#zen4">Zen 4</a></li><li><a href="/models/#zen3">Zen 3</a></li></ul></div><div class="footer-section"><h4>Resources</h4><ul><li><a href="/datasets/">Training Data</a></li><li><a href="https://huggingface.co/zenlm" target="_blank" rel="noopener noreferrer">HuggingFace</a></li><li><a href="https://github.com/zenlm" target="_blank" rel="noopener noreferrer">GitHub</a></li><li><a href="/research/">Research Papers</a></li><li><a href="https://api.hanzo.ai" target="_blank" rel="noopener noreferrer">Zen API</a></li></ul></div></div><div class="footer-bottom"><p>© <!-- -->2026<!-- --> Zoo Labs Foundation, a 501(c)(3) non-profit. Zen LM open models.</p></div></div></footer><script src="/_next/static/chunks/webpack-f43ad49c79db3299.js" id="_R_" async=""></script><script>(self.__next_f=self.__next_f||[]).push([0])</script><script>self.__next_f.push([1,"1:\"$Sreact.fragment\"\n2:I[6529,[\"619\",\"static/chunks/619-ba102abea3e3d0e4.js\",\"177\",\"static/chunks/app/layout-425ca81883167c16.js\"],\"default\"]\n3:I[9766,[],\"\"]\n4:I[8924,[],\"\"]\n5:I[2619,[\"619\",\"static/chunks/619-ba102abea3e3d0e4.js\",\"953\",\"static/chunks/app/blog/%5Bslug%5D/page-38f8c2a205f98d97.js\"],\"\"]\n6:I[7405,[\"619\",\"static/chunks/619-ba102abea3e3d0e4.js\",\"177\",\"static/chunks/app/layout-425ca81883167c16.js\"],\"default\"]\na:I[7150,[],\"\"]\n:HL[\"/_next/static/css/1d1f6bc532e5f43f.css\",\"style\"]\n:HL[\"/_next/static/css/eb87e4f7aea490c6.css\",\"style\"]\n"])</script><script>self.__next_f.push([1,"0:{\"P\":null,\"b\":\"-BoRG_r6Z3v9udL0lI0J_\",\"p\":\"\",\"c\":[\"\",\"blog\",\"ofa\",\"\"],\"i\":false,\"f\":[[[\"\",{\"children\":[\"blog\",{\"children\":[[\"slug\",\"ofa\",\"d\"],{\"children\":[\"__PAGE__\",{}]}]}]},\"$undefined\",\"$undefined\",true],[\"\",[\"$\",\"$1\",\"c\",{\"children\":[[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/css/1d1f6bc532e5f43f.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\",\"nonce\":\"$undefined\"}]],[\"$\",\"html\",null,{\"lang\":\"en\",\"children\":[[\"$\",\"head\",null,{\"children\":[[\"$\",\"link\",null,{\"rel\":\"icon\",\"type\":\"image/svg+xml\",\"href\":\"/favicon.svg\"}],[\"$\",\"link\",null,{\"rel\":\"alternate icon\",\"href\":\"/favicon.png\"}]]}],[\"$\",\"body\",null,{\"children\":[[\"$\",\"$L2\",null,{}],[\"$\",\"$L3\",null,{\"parallelRouterKey\":\"children\",\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L4\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":[[[\"$\",\"title\",null,{\"children\":\"404: This page could not be found.\"}],[\"$\",\"div\",null,{\"style\":{\"fontFamily\":\"system-ui,\\\"Segoe UI\\\",Roboto,Helvetica,Arial,sans-serif,\\\"Apple Color Emoji\\\",\\\"Segoe UI Emoji\\\"\",\"height\":\"100vh\",\"textAlign\":\"center\",\"display\":\"flex\",\"flexDirection\":\"column\",\"alignItems\":\"center\",\"justifyContent\":\"center\"},\"children\":[\"$\",\"div\",null,{\"children\":[[\"$\",\"style\",null,{\"dangerouslySetInnerHTML\":{\"__html\":\"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}\"}}],[\"$\",\"h1\",null,{\"className\":\"next-error-h1\",\"style\":{\"display\":\"inline-block\",\"margin\":\"0 20px 0 0\",\"padding\":\"0 23px 0 0\",\"fontSize\":24,\"fontWeight\":500,\"verticalAlign\":\"top\",\"lineHeight\":\"49px\"},\"children\":404}],[\"$\",\"div\",null,{\"style\":{\"display\":\"inline-block\"},\"children\":[\"$\",\"h2\",null,{\"style\":{\"fontSize\":14,\"fontWeight\":400,\"lineHeight\":\"49px\",\"margin\":0},\"children\":\"This page could not be found.\"}]}]]}]}]],[]],\"forbidden\":\"$undefined\",\"unauthorized\":\"$undefined\"}],[\"$\",\"footer\",null,{\"children\":[\"$\",\"div\",null,{\"className\":\"container\",\"children\":[[\"$\",\"div\",null,{\"className\":\"footer-content\",\"children\":[[\"$\",\"div\",null,{\"className\":\"footer-section\",\"children\":[[\"$\",\"h4\",null,{\"children\":\"Zen LM\"}],[\"$\",\"p\",null,{\"children\":\"Open models from Zoo Labs Foundation, a 501(c)(3) non-profit, for agentic coding on your own machine and marketing work. Served on api.hanzo.ai.\"}]]}],[\"$\",\"div\",null,{\"className\":\"footer-section\",\"children\":[[\"$\",\"h4\",null,{\"children\":\"Zen 6\"}],[\"$\",\"ul\",null,{\"children\":[[\"$\",\"li\",null,{\"children\":[\"$\",\"$L5\",null,{\"href\":\"/models#zen6\",\"children\":\"Zen 6\"}]}],[\"$\",\"li\",null,{\"children\":[\"$\",\"$L5\",null,{\"href\":\"/models#zen6\",\"children\":\"Zen 6 Flash\"}]}],[\"$\",\"li\",null,{\"children\":[\"$\",\"a\",null,{\"href\":\"https://hanzo.ai/research-access\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"children\":\"Zen 7 research preview\"}]}]]}]]}],[\"$\",\"div\",null,{\"className\":\"footer-section\",\"children\":[[\"$\",\"h4\",null,{\"children\":\"Earlier Generations\"}],[\"$\",\"ul\",null,{\"children\":[[\"$\",\"li\",null,{\"children\":[\"$\",\"$L5\",null,{\"href\":\"/models#zen5\",\"children\":\"Zen 5\"}]}],[\"$\",\"li\",null,{\"children\":[\"$\",\"$L5\",null,{\"href\":\"/models#zen4\",\"children\":\"Zen 4\"}]}],[\"$\",\"li\",null,{\"children\":[\"$\",\"$L5\",null,{\"href\":\"/models#zen3\",\"children\":\"Zen 3\"}]}]]}]]}],[\"$\",\"div\",null,{\"className\":\"footer-section\",\"children\":[[\"$\",\"h4\",null,{\"children\":\"Resources\"}],[\"$\",\"ul\",null,{\"children\":[[\"$\",\"li\",null,{\"children\":[\"$\",\"$L5\",null,{\"href\":\"/datasets\",\"children\":\"Training Data\"}]}],[\"$\",\"li\",null,{\"children\":[\"$\",\"a\",null,{\"href\":\"https://huggingface.co/zenlm\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"children\":\"HuggingFace\"}]}],[\"$\",\"li\",null,{\"children\":[\"$\",\"a\",null,{\"href\":\"https://github.com/zenlm\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"children\":\"GitHub\"}]}],[\"$\",\"li\",null,{\"children\":[\"$\",\"$L5\",null,{\"href\":\"/research\",\"children\":\"Research Papers\"}]}],[\"$\",\"li\",null,{\"children\":[\"$\",\"a\",null,{\"href\":\"https://api.hanzo.ai\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"children\":\"Zen API\"}]}]]}]]}]]}],[\"$\",\"div\",null,{\"className\":\"footer-bottom\",\"children\":[\"$\",\"p\",null,{\"children\":[\"© \",2026,\" Zoo Labs Foundation, a 501(c)(3) non-profit. Zen LM open models.\"]}]}]]}]}],[\"$\",\"$L6\",null,{}],[\"$\",\"script\",null,{\"src\":\"/assets/js/main.js\",\"async\":true}]]}]]}]]}],{\"children\":[\"blog\",[\"$\",\"$1\",\"c\",{\"children\":[null,[\"$\",\"$L3\",null,{\"parallelRouterKey\":\"children\",\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L4\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":\"$undefined\",\"forbidden\":\"$undefined\",\"unauthorized\":\"$undefined\"}]]}],{\"children\":[[\"slug\",\"ofa\",\"d\"],[\"$\",\"$1\",\"c\",{\"children\":[null,[\"$\",\"$L3\",null,{\"parallelRouterKey\":\"children\",\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L4\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":\"$undefined\",\"forbidden\":\"$undefined\",\"unauthorized\":\"$undefined\"}]]}],{\"children\":[\"__PAGE__\",[\"$\",\"$1\",\"c\",{\"children\":[\"$L7\",[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/css/eb87e4f7aea490c6.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\",\"nonce\":\"$undefined\"}]],\"$L8\"]}],{},null,false]},null,false]},null,false]},null,false],\"$L9\",false]],\"m\":\"$undefined\",\"G\":[\"$a\",[]],\"s\":false,\"S\":true}\n"])</script><script>self.__next_f.push([1,"b:I[4431,[],\"OutletBoundary\"]\nd:I[5278,[],\"AsyncMetadataOutlet\"]\nf:I[4431,[],\"ViewportBoundary\"]\n11:I[4431,[],\"MetadataBoundary\"]\n12:\"$Sreact.suspense\"\n8:[\"$\",\"$Lb\",null,{\"children\":[\"$Lc\",[\"$\",\"$Ld\",null,{\"promise\":\"$@e\"}]]}]\n9:[\"$\",\"$1\",\"h\",{\"children\":[null,[[\"$\",\"$Lf\",null,{\"children\":\"$L10\"}],null],[\"$\",\"$L11\",null,{\"children\":[\"$\",\"div\",null,{\"hidden\":true,\"children\":[\"$\",\"$12\",null,{\"fallback\":null,\"children\":\"$L13\"}]}]}]]}]\n"])</script><script>self.__next_f.push([1,"14:T5818,"])</script><script>self.__next_f.push([1,"\u003cp\u003e\u003cvideo src=\"https://qianwen-res.oss-accelerate-overseas.aliyuncs.com/assets/blog/ofa/demo.mp4\" controls playsinline preload=\"metadata\" style=\"width:100%\"\u003e\u003c/video\u003e\u003c/p\u003e\n\u003cp\u003e2022 is a year of generalist models! With the bloom of multimodal pretraining, especially the unified model, we have witnessed the opportunity to building a generalist model that is capable of processing tasks of different modalities or multi-modalities! Thus, we propose OFA\u003csup\u003e\u003ca href=\"#user-content-fn-1\" id=\"user-content-fnref-1\" data-footnote-ref=\"\" aria-describedby=\"footnote-label\"\u003e1\u003c/a\u003e\u003c/sup\u003e, namely One-For-All, a unified multimodal pretrained model that unifies understanding and generation tasks concerning modalities into a single framework, and we pretrain OFA with the instruction-based multitask-pretraining that endows it with multiple capabilities. We opensourced both the pretrained and finetuned models to the community, hoping this pioneer work can help accelerate the development of generalist models.\u003c/p\u003e\n\u003cp\u003e\u003ca href=\"https://arxiv.org/abs/2202.03052\"\u003ePaper\u003c/a\u003e\n\u003ca href=\"https://github.com/OFA-Sys/OFA\"\u003eGithub\u003c/a\u003e\n\u003ca href=\"https://www.modelscope.cn/models?name=ofa\"\u003eModelScope\u003c/a\u003e\n\u003ca href=\"https://huggingface.co/spaces/OFA-Sys/OFA-Generic_Interface\"\u003eDemo\u003c/a\u003e\u003c/p\u003e\n\u003ch2 id=\"background\"\u003eBackground\u003c/h2\u003e\n\u003cfigure\u003e\u003cimg src=\"https://qianwen-res.oss-accelerate-overseas.aliyuncs.com/assets/blog/ofa/uniter.jpg\" alt=\"\" loading=\"lazy\"\u003e\u003c/figure\u003e\n\u003cp\u003eMultimodal pretraining has been developing rapidly ever since the transfer of BERT\u003csup\u003e\u003ca href=\"#user-content-fn-bert\" id=\"user-content-fnref-bert\" data-footnote-ref=\"\" aria-describedby=\"footnote-label\"\u003e2\u003c/a\u003e\u003c/sup\u003e to cross-modal representation learning. Representative studies include UNITER\u003csup\u003e\u003ca href=\"#user-content-fn-2\" id=\"user-content-fnref-2\" data-footnote-ref=\"\" aria-describedby=\"footnote-label\"\u003e3\u003c/a\u003e\u003c/sup\u003e, VilBERT\u003csup\u003e\u003ca href=\"#user-content-fn-3\" id=\"user-content-fnref-3\" data-footnote-ref=\"\" aria-describedby=\"footnote-label\"\u003e4\u003c/a\u003e\u003c/sup\u003e, etc. These studies directly incorporate the Transformer-based BERT\u003csup\u003e\u003ca href=\"#user-content-fn-bert\" id=\"user-content-fnref-bert-2\" data-footnote-ref=\"\" aria-describedby=\"footnote-label\"\u003e2\u003c/a\u003e\u003c/sup\u003e to a single-stream or dual stream framework for multimodal pretraining, and transform the image to a sequence of object features to be concatenated with the word embeddings as the input of Transformer. Later in 2021, with the rise of Vision Transformer\u003csup\u003e\u003ca href=\"#user-content-fn-5\" id=\"user-content-fnref-5\" data-footnote-ref=\"\" aria-describedby=\"footnote-label\"\u003e5\u003c/a\u003e\u003c/sup\u003e, there came methods that got rid of object-level features, which depend on complex preprocessing pipelines, say Faster-RCNN\u003csup\u003e\u003ca href=\"#user-content-fn-6\" id=\"user-content-fnref-6\" data-footnote-ref=\"\" aria-describedby=\"footnote-label\"\u003e6\u003c/a\u003e\u003c/sup\u003e: For example, the simplest ViLT\u003csup\u003e\u003ca href=\"#user-content-fn-7\" id=\"user-content-fnref-7\" data-footnote-ref=\"\" aria-describedby=\"footnote-label\"\u003e7\u003c/a\u003e\u003c/sup\u003e based on patch projection, the CLIP-based\u003csup\u003e\u003ca href=\"#user-content-fn-8\" id=\"user-content-fnref-8\" data-footnote-ref=\"\" aria-describedby=\"footnote-label\"\u003e8\u003c/a\u003e\u003c/sup\u003e CLIP-ViL\u003csup\u003e\u003ca href=\"#user-content-fn-9\" id=\"user-content-fnref-9\" data-footnote-ref=\"\" aria-describedby=\"footnote-label\"\u003e9\u003c/a\u003e\u003c/sup\u003e, etc. One milestone after should be the proposal of SimVLM\u003csup\u003e\u003ca href=\"#user-content-fn-10\" id=\"user-content-fnref-10\" data-footnote-ref=\"\" aria-describedby=\"footnote-label\"\u003e10\u003c/a\u003e\u003c/sup\u003e, which leverages the T5/BART method for multimodal pretraining and achieves new SoTA in many tasks. These progress should be regarded as the foundation of unified multimodal pretrained models in 2022, including OFA of ours, Unified-IO\u003csup\u003e\u003ca href=\"#user-content-fn-11\" id=\"user-content-fnref-11\" data-footnote-ref=\"\" aria-describedby=\"footnote-label\"\u003e11\u003c/a\u003e\u003c/sup\u003e, Flamingo\u003csup\u003e\u003ca href=\"#user-content-fn-12\" id=\"user-content-fnref-12\" data-footnote-ref=\"\" aria-describedby=\"footnote-label\"\u003e12\u003c/a\u003e\u003c/sup\u003e, BeiT-3\u003csup\u003e\u003ca href=\"#user-content-fn-13\" id=\"user-content-fnref-13\" data-footnote-ref=\"\" aria-describedby=\"footnote-label\"\u003e13\u003c/a\u003e\u003c/sup\u003e, etc.\u003c/p\u003e\n\u003ch2 id=\"method\"\u003eMethod\u003c/h2\u003e\n\u003cp\u003eWhat OFA wants to achieve is the unification of tasks, modalities, and architecture. We suppose there are three features for a unified model, i.e., task agnostic, modality agnostic, and task comprehensiveness. To further explain them, \"task agnostic\" indicates that the unified model should be able to accept tasks without modifying its own architecture and training methods, \"modality agnostic\" indicates that a unified model should accept inputs of different modalities without knowing what they are and designing complex preprocessing, and \"task comprehensiveness\" indicates that the unified model should learn as many tasks as possible so that it can transfer to unseen tasks with the composition of existing capabilities. Thus, we propose 3 types of unification for OFA, namely the unification of modalities, architecture, and tasks. Let's figure them out one by one.\u003c/p\u003e\n\u003cp\u003eFor the unification of modalities, one key issue is the tokenization of inputs of different modalities, or to say, the discretization. Otherwise, there should be other solutions like diffusion models\u003csup\u003e\u003ca href=\"#user-content-fn-14\" id=\"user-content-fnref-14\" data-footnote-ref=\"\" aria-describedby=\"footnote-label\"\u003e14\u003c/a\u003e\u003c/sup\u003e for the generation. There is no need to change the tokenization for texts, but the images and bounding boxes need to be discretized. Owing to the success of vector quantization\u003csup\u003e\u003ca href=\"#user-content-fn-15\" id=\"user-content-fnref-15\" data-footnote-ref=\"\" aria-describedby=\"footnote-label\"\u003e15\u003c/a\u003e\u003c/sup\u003e\u003csup\u003e\u003ca href=\"#user-content-fn-16\" id=\"user-content-fnref-16\" data-footnote-ref=\"\" aria-describedby=\"footnote-label\"\u003e16\u003c/a\u003e\u003c/sup\u003e and text-to-image generation with Transformer\u003csup\u003e\u003ca href=\"#user-content-fn-17\" id=\"user-content-fnref-17\" data-footnote-ref=\"\" aria-describedby=\"footnote-label\"\u003e17\u003c/a\u003e\u003c/sup\u003e\u003csup\u003e\u003ca href=\"#user-content-fn-18\" id=\"user-content-fnref-18\" data-footnote-ref=\"\" aria-describedby=\"footnote-label\"\u003e18\u003c/a\u003e\u003c/sup\u003e, images can be represented with VQ tokens. Inspired by pix2seq\u003csup\u003e\u003ca href=\"#user-content-fn-19\" id=\"user-content-fnref-19\" data-footnote-ref=\"\" aria-describedby=\"footnote-label\"\u003e19\u003c/a\u003e\u003c/sup\u003e, bounding boxes can also be discretized with bins.\u003c/p\u003e\n\u003cfigure\u003e\u003cimg src=\"https://qianwen-res.oss-accelerate-overseas.aliyuncs.com/assets/blog/ofa/io.jpg\" alt=\"\" loading=\"lazy\"\u003e\u003c/figure\u003e\n\u003cp\u003eWe choose the universal Transformer encoder-decoder architecture, due to its successful usages in NLP unified models like T5\u003csup\u003e\u003ca href=\"#user-content-fn-20\" id=\"user-content-fnref-20\" data-footnote-ref=\"\" aria-describedby=\"footnote-label\"\u003e20\u003c/a\u003e\u003c/sup\u003e. Note that for the input of images to the Transformer, we use the first three blocks of ResNet. For the Transformer architecture, we modify the design by incorporating Normformer\u003csup\u003e\u003ca href=\"#user-content-fn-21\" id=\"user-content-fnref-21\" data-footnote-ref=\"\" aria-describedby=\"footnote-label\"\u003e21\u003c/a\u003e\u003c/sup\u003e for the training stability and transfer performance.\u003c/p\u003e\n\u003cfigure\u003e\u003cimg src=\"https://qianwen-res.oss-accelerate-overseas.aliyuncs.com/assets/blog/ofa/arc.jpg\" alt=\"\" loading=\"lazy\"\u003e\u003c/figure\u003e\n\u003cp\u003eThe multitask learning is the key innovation of OFA. Specifically, we pretrain the model with 8 tasks, including 5 vision-language tasks, 2 vision tasks, and 1 language task. The vision-language tasks include visual grounding, grounded captioning, visual question answering, image-text matching, and image captioning. The vision tasks include detection and image infilling. The language task is text infilling. To help the model differentiate tasks, we insert an instruction, which is simply a piece of text describing the task. Thus, we expect the model to perform zero-shot generation based on a new instruction indicating an unseen task.\u003c/p\u003e\n\u003cfigure\u003e\u003cimg src=\"https://qianwen-res.oss-accelerate-overseas.aliyuncs.com/assets/blog/ofa/task.jpg\" alt=\"\" loading=\"lazy\"\u003e\u003c/figure\u003e\n\u003cp\u003eTo make this research as reproducible as possible, our pretraining is dependent on public datasets. Therefore, we expect the researchers following this work can reproduce our results with our opensourced code.\u003c/p\u003e\n\u003cp\u003eWe have released OFA models of 5 sizes, including OFA-Tiny (33M), OFA-Medium (93M), OFA-Base (180M), OFA-Large(470M), OFA-Huge (930M). See the table below for more statistics.\u003c/p\u003e\n\u003cfigure\u003e\u003cimg src=\"https://qianwen-res.oss-accelerate-overseas.aliyuncs.com/assets/blog/ofa/variants.jpg\" alt=\"\" loading=\"lazy\"\u003e\u003c/figure\u003e\n\u003ch2 id=\"experiments\"\u003eExperiments\u003c/h2\u003e\n\u003cp\u003eWe have conducted experiments on multiple cross-modal tasks and unimodal tasks. On vision-language understanding, we test the models on VQA and SNLI-VE. We find that the huge-size model can achieve a comparable performance to the 80B-parameter model Flamingo and the 2B-parameter model CoCa pretrained on 5B image-text pairs. Furthermore, we achieve the best performance on visual entailment. For vision-language generation, we focus on the classical image captioning, and our OFA achieves the SoTA performance in both setups of cross-entropy optimization and CIDEr optimization. Also, we have transformed the task of visual grounding to a generation task, and we find that even the base-size OFA can outperform the previous SoTA, and the scaling of model size consistently brings performance improvements. This shows the significance of the unification of modalities and tasks.\u003c/p\u003e\n\u003cfigure\u003e\u003cimg src=\"https://qianwen-res.oss-accelerate-overseas.aliyuncs.com/assets/blog/ofa/vqa.jpg\" alt=\"\" loading=\"lazy\"\u003e\u003c/figure\u003e\n\u003cfigure\u003e\u003cimg src=\"https://qianwen-res.oss-accelerate-overseas.aliyuncs.com/assets/blog/ofa/caption.jpg\" alt=\"\" loading=\"lazy\"\u003e\u003c/figure\u003e\n\u003cfigure\u003e\u003cimg src=\"https://qianwen-res.oss-accelerate-overseas.aliyuncs.com/assets/blog/ofa/vg.jpg\" alt=\"\" loading=\"lazy\"\u003e\u003c/figure\u003e\n\u003cp\u003eAdditionally, we test OFA on text-to-image generation, as we believe that the image infilling task in pretraining endows it with the capability to generate image codes. We show that OFA can achieve a low FID score in the evaluation, and further finetuning on a larger dataset can significantly boost its performance. See cases below.\u003c/p\u003e\n\u003cfigure\u003e\u003cimg src=\"https://qianwen-res.oss-accelerate-overseas.aliyuncs.com/assets/blog/ofa/t2i.jpg\" alt=\"\" loading=\"lazy\"\u003e\u003c/figure\u003e\n\u003cfigure\u003e\u003cimg src=\"https://qianwen-res.oss-accelerate-overseas.aliyuncs.com/assets/blog/ofa/t2i_cases.jpg\" alt=\"\" loading=\"lazy\"\u003e\u003c/figure\u003e\n\u003cp\u003eAs to the unimodal tasks, we evaluate OFA on the GLUE benchmark for NLU, Gigaword summarization for NLG, and ImageNet classification for vision understanding. We show that OFA can be competitive with both RoBERTa and DeBERTa, and the previous multimodal pretrained model often falls far behind the SoTAs in NLU. Similarly, OFA can achieve good performance on NLG and outperform the previous best models. As to image classification, it can also achieve similar performance with the self-supervised vision models like BeiT\u003csup\u003e\u003ca href=\"#user-content-fn-22\" id=\"user-content-fnref-22\" data-footnote-ref=\"\" aria-describedby=\"footnote-label\"\u003e22\u003c/a\u003e\u003c/sup\u003e and MAE\u003csup\u003e\u003ca href=\"#user-content-fn-23\" id=\"user-content-fnref-23\" data-footnote-ref=\"\" aria-describedby=\"footnote-label\"\u003e23\u003c/a\u003e\u003c/sup\u003e.\u003c/p\u003e\n\u003cp\u003eWe observe that OFA based on multitask pretraining demonstrates potential in transferring to unseen tasks and unseen domains. We show them with two cases below.\u003c/p\u003e\n\u003cfigure\u003e\u003cimg src=\"https://qianwen-res.oss-accelerate-overseas.aliyuncs.com/assets/blog/ofa/unseen_task.jpg\" alt=\"\" loading=\"lazy\"\u003e\u003c/figure\u003e\n\u003cp\u003eThe preceding case demonstrates the model's ability of compositional generalization by understanding the instruction and leveraging two learned capabilities to perform the new task. We set up a new task called \u003cem\u003eGrounded VQA\u003c/em\u003e, which is a combination of VQA and grounded captioning. What we need to change is the instruction. The new task instruction with both question and region information directs the model to provide a correct answer.\u003c/p\u003e\n\u003cfigure\u003e\u003cimg src=\"https://qianwen-res.oss-accelerate-overseas.aliyuncs.com/assets/blog/ofa/unseen_domain.jpg\" alt=\"\" loading=\"lazy\"\u003e\u003c/figure\u003e\n\u003cp\u003eAlso, we find that OFA can transfer to unseen domains effectively. One example is the visual grounding on images of animation. OFA can perform well in this setup as it has been pretrained on some anime data and it has been pretrained on visual grounding on general-domain data. This again shows the compositional ability of the unified model.\u003c/p\u003e\n\u003ch2 id=\"conclusion\"\u003eConclusion\u003c/h2\u003e\n\u003cp\u003eThis is the starting point of our research for the technically \"One-For-All\" model, or to say, the generalist model. We show that this research direction is promising as Transformer is a really powerful architecture and tasks and modalities can be unified to a single training framework. Like GPT-3\u003csup\u003e\u003ca href=\"#user-content-fn-24\" id=\"user-content-fnref-24\" data-footnote-ref=\"\" aria-describedby=\"footnote-label\"\u003e24\u003c/a\u003e\u003c/sup\u003e, we believe that there will soon be a powerful foundation model in multimodal representation learning.\u003c/p\u003e\n\u003csection data-footnotes=\"\" class=\"footnotes\"\u003e\u003ch2 class=\"sr-only\" id=\"footnote-label\"\u003eFootnotes\u003c/h2\u003e\n\u003col\u003e\n\u003cli id=\"user-content-fn-1\"\u003e\n\u003cp\u003eWang, P., Yang, A., Men, R., Lin, J., Bai, S., Li, Z., Ma, J., Zhou, C., Zhou, J., \u0026#x26; Yang, H. (2022).\nUnifying Architectures, Tasks, and Modalities Through a Simple Sequence-to-Sequence Learning Framework.\nInternational Conference on Machine Learning. \u003ca href=\"#user-content-fnref-1\" data-footnote-backref=\"\" aria-label=\"Back to reference 1\" class=\"data-footnote-backref\"\u003e↩\u003c/a\u003e\u003c/p\u003e\n\u003c/li\u003e\n\u003cli id=\"user-content-fn-bert\"\u003e\n\u003cp\u003eDevlin, J., Chang, M., Lee, K., \u0026#x26; Toutanova, K. (2019).\nBERT: Pre-training of Deep Bidirectional Transformers for Language Understanding.\narXiv, abs/1810.04805. \u003ca href=\"#user-content-fnref-bert\" data-footnote-backref=\"\" aria-label=\"Back to reference 2\" class=\"data-footnote-backref\"\u003e↩\u003c/a\u003e \u003ca href=\"#user-content-fnref-bert-2\" data-footnote-backref=\"\" aria-label=\"Back to reference 2-2\" class=\"data-footnote-backref\"\u003e↩\u003csup\u003e2\u003c/sup\u003e\u003c/a\u003e\u003c/p\u003e\n\u003c/li\u003e\n\u003cli id=\"user-content-fn-2\"\u003e\n\u003cp\u003eChen, Y., Li, L., Yu, L., Kholy, A.E., Ahmed, F., Gan, Z., Cheng, Y., \u0026#x26; Liu, J. (2019).\nUNITER: UNiversal Image-TExt Representation Learning.\nEuropean Conference on Computer Vision. \u003ca href=\"#user-content-fnref-2\" data-footnote-backref=\"\" aria-label=\"Back to reference 3\" class=\"data-footnote-backref\"\u003e↩\u003c/a\u003e\u003c/p\u003e\n\u003c/li\u003e\n\u003cli id=\"user-content-fn-3\"\u003e\n\u003cp\u003eLu, J., Batra, D., Parikh, D., \u0026#x26; Lee, S. (2019).\nViLBERT: Pretraining Task-Agnostic Visiolinguistic Representations for Vision-and-Language Tasks.\nNeural Information Processing Systems. \u003ca href=\"#user-content-fnref-3\" data-footnote-backref=\"\" aria-label=\"Back to reference 4\" class=\"data-footnote-backref\"\u003e↩\u003c/a\u003e\u003c/p\u003e\n\u003c/li\u003e\n\u003cli id=\"user-content-fn-5\"\u003e\n\u003cp\u003eDosovitskiy, A., Beyer, L., Kolesnikov, A., Weissenborn, D., Zhai, X., Unterthiner, T., Dehghani, M., Minderer, M., Heigold, G., Gelly, S., Uszkoreit, J., \u0026#x26; Houlsby, N. (2020).\nAn Image is Worth 16x16 Words: Transformers for Image Recognition at Scale.\narXiv, abs/2010.11929. \u003ca href=\"#user-content-fnref-5\" data-footnote-backref=\"\" aria-label=\"Back to reference 5\" class=\"data-footnote-backref\"\u003e↩\u003c/a\u003e\u003c/p\u003e\n\u003c/li\u003e\n\u003cli id=\"user-content-fn-6\"\u003e\n\u003cp\u003eRen, S., He, K., Girshick, R.B., \u0026#x26; Sun, J. (2015).\nFaster R-CNN: Towards Real-Time Object Detection with Region Proposal Networks.\nIEEE Transactions on Pattern Analysis and Machine Intelligence, 39, 1137-1149. \u003ca href=\"#user-content-fnref-6\" data-footnote-backref=\"\" aria-label=\"Back to reference 6\" class=\"data-footnote-backref\"\u003e↩\u003c/a\u003e\u003c/p\u003e\n\u003c/li\u003e\n\u003cli id=\"user-content-fn-7\"\u003e\n\u003cp\u003eKim, W., Son, B., \u0026#x26; Kim, I. (2021).\nViLT: Vision-and-Language Transformer Without Convolution or Region Supervision.\nInternational Conference on Machine Learning. \u003ca href=\"#user-content-fnref-7\" data-footnote-backref=\"\" aria-label=\"Back to reference 7\" class=\"data-footnote-backref\"\u003e↩\u003c/a\u003e\u003c/p\u003e\n\u003c/li\u003e\n\u003cli id=\"user-content-fn-8\"\u003e\n\u003cp\u003eRadford, A., Kim, J.W., Hallacy, C., Ramesh, A., Goh, G., Agarwal, S., Sastry, G., Askell, A., Mishkin, P., Clark, J., Krueger, G., \u0026#x26; Sutskever, I. (2021).\nLearning Transferable Visual Models From Natural Language Supervision.\nInternational Conference on Machine Learning. \u003ca href=\"#user-content-fnref-8\" data-footnote-backref=\"\" aria-label=\"Back to reference 8\" class=\"data-footnote-backref\"\u003e↩\u003c/a\u003e\u003c/p\u003e\n\u003c/li\u003e\n\u003cli id=\"user-content-fn-9\"\u003e\n\u003cp\u003eShen, S., Li, L.H., Tan, H., Bansal, M., Rohrbach, A., Chang, K., Yao, Z., \u0026#x26; Keutzer, K. (2021).\nHow Much Can CLIP Benefit Vision-and-Language Tasks?\narXiv, abs/2107.06383. \u003ca href=\"#user-content-fnref-9\" data-footnote-backref=\"\" aria-label=\"Back to reference 9\" class=\"data-footnote-backref\"\u003e↩\u003c/a\u003e\u003c/p\u003e\n\u003c/li\u003e\n\u003cli id=\"user-content-fn-10\"\u003e\n\u003cp\u003eWang, Z., Yu, J., Yu, A.W., Dai, Z., Tsvetkov, Y., \u0026#x26; Cao, Y. (2021).\nSimVLM: Simple Visual Language Model Pretraining with Weak Supervision.\narXiv, abs/2108.10904. \u003ca href=\"#user-content-fnref-10\" data-footnote-backref=\"\" aria-label=\"Back to reference 10\" class=\"data-footnote-backref\"\u003e↩\u003c/a\u003e\u003c/p\u003e\n\u003c/li\u003e\n\u003cli id=\"user-content-fn-11\"\u003e\n\u003cp\u003eLu, J., Clark, C., Zellers, R., Mottaghi, R., \u0026#x26; Kembhavi, A. (2022).\nUnified-IO: A Unified Model for Vision, Language, and Multi-Modal Tasks.\narXiv, abs/2206.08916. \u003ca href=\"#user-content-fnref-11\" data-footnote-backref=\"\" aria-label=\"Back to reference 11\" class=\"data-footnote-backref\"\u003e↩\u003c/a\u003e\u003c/p\u003e\n\u003c/li\u003e\n\u003cli id=\"user-content-fn-12\"\u003e\n\u003cp\u003eAlayrac, J., Donahue, J., Luc, P., Miech, A., Barr, I., Hasson, Y., Lenc, K., Mensch, A., Millican, K., Reynolds, M., Ring, R., Rutherford, E., Cabi, S., Han, T., Gong, Z., Samangooei, S., Monteiro, M., Menick, J., Borgeaud, S., Brock, A., Nematzadeh, A., Sharifzadeh, S., Binkowski, M., Barreira, R., Vinyals, O., Zisserman, A., \u0026#x26; Simonyan, K. (2022).\nFlamingo: a Visual Language Model for Few-Shot Learning.\narXiv, abs/2204.14198. \u003ca href=\"#user-content-fnref-12\" data-footnote-backref=\"\" aria-label=\"Back to reference 12\" class=\"data-footnote-backref\"\u003e↩\u003c/a\u003e\u003c/p\u003e\n\u003c/li\u003e\n\u003cli id=\"user-content-fn-13\"\u003e\n\u003cp\u003eWang, W., Bao, H., Dong, L., Bjorck, J., Peng, Z., Liu, Q., Aggarwal, K., Mohammed, O., Singhal, S., Som, S., \u0026#x26; Wei, F. (2022).\nImage as a Foreign Language: BEiT Pretraining for All Vision and Vision-Language Tasks.\narXiv, abs/2208.10442. \u003ca href=\"#user-content-fnref-13\" data-footnote-backref=\"\" aria-label=\"Back to reference 13\" class=\"data-footnote-backref\"\u003e↩\u003c/a\u003e\u003c/p\u003e\n\u003c/li\u003e\n\u003cli id=\"user-content-fn-14\"\u003e\n\u003cp\u003eHo, J., Jain, A., \u0026#x26; Abbeel, P. (2020).\nDenoising Diffusion Probabilistic Models.\narXiv, abs/2006.11239. \u003ca href=\"#user-content-fnref-14\" data-footnote-backref=\"\" aria-label=\"Back to reference 14\" class=\"data-footnote-backref\"\u003e↩\u003c/a\u003e\u003c/p\u003e\n\u003c/li\u003e\n\u003cli id=\"user-content-fn-15\"\u003e\n\u003cp\u003eRazavi, A., Oord, A.V., \u0026#x26; Vinyals, O. (2019).\nGenerating Diverse High-Fidelity Images with VQ-VAE-2.\narXiv, abs/1906.00446. \u003ca href=\"#user-content-fnref-15\" data-footnote-backref=\"\" aria-label=\"Back to reference 15\" class=\"data-footnote-backref\"\u003e↩\u003c/a\u003e\u003c/p\u003e\n\u003c/li\u003e\n\u003cli id=\"user-content-fn-16\"\u003e\n\u003cp\u003eEsser, P., Rombach, R., \u0026#x26; Ommer, B. (2020).\nTaming Transformers for High-Resolution Image Synthesis.\n2021 IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR), 12868-12878. \u003ca href=\"#user-content-fnref-16\" data-footnote-backref=\"\" aria-label=\"Back to reference 16\" class=\"data-footnote-backref\"\u003e↩\u003c/a\u003e\u003c/p\u003e\n\u003c/li\u003e\n\u003cli id=\"user-content-fn-17\"\u003e\n\u003cp\u003eRamesh, A., Pavlov, M., Goh, G., Gray, S., Voss, C., Radford, A., Chen, M., \u0026#x26; Sutskever, I. (2021).\nZero-Shot Text-to-Image Generation.\narXiv, abs/2102.12092. \u003ca href=\"#user-content-fnref-17\" data-footnote-backref=\"\" aria-label=\"Back to reference 17\" class=\"data-footnote-backref\"\u003e↩\u003c/a\u003e\u003c/p\u003e\n\u003c/li\u003e\n\u003cli id=\"user-content-fn-18\"\u003e\n\u003cp\u003eDing, M., Yang, Z., Hong, W., Zheng, W., Zhou, C., Yin, D., Lin, J., Zou, X., Shao, Z., Yang, H., \u0026#x26; Tang, J. (2021).\nCogView: Mastering Text-to-Image Generation via Transformers.\nNeural Information Processing Systems. \u003ca href=\"#user-content-fnref-18\" data-footnote-backref=\"\" aria-label=\"Back to reference 18\" class=\"data-footnote-backref\"\u003e↩\u003c/a\u003e\u003c/p\u003e\n\u003c/li\u003e\n\u003cli id=\"user-content-fn-19\"\u003e\n\u003cp\u003eChen, T., Saxena, S., Li, L., Fleet, D.J., \u0026#x26; Hinton, G.R. (2021).\nPix2seq: A Language Modeling Framework for Object Detection.\narXiv, abs/2109.10852. \u003ca href=\"#user-content-fnref-19\" data-footnote-backref=\"\" aria-label=\"Back to reference 19\" class=\"data-footnote-backref\"\u003e↩\u003c/a\u003e\u003c/p\u003e\n\u003c/li\u003e\n\u003cli id=\"user-content-fn-20\"\u003e\n\u003cp\u003eRaffel, C., Shazeer, N.M., Roberts, A., Lee, K., Narang, S., Matena, M., Zhou, Y., Li, W., \u0026#x26; Liu, P.J. (2019).\nExploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer.\narXiv, abs/1910.10683. \u003ca href=\"#user-content-fnref-20\" data-footnote-backref=\"\" aria-label=\"Back to reference 20\" class=\"data-footnote-backref\"\u003e↩\u003c/a\u003e\u003c/p\u003e\n\u003c/li\u003e\n\u003cli id=\"user-content-fn-21\"\u003e\n\u003cp\u003eShleifer, S., Weston, J., \u0026#x26; Ott, M. (2021).\nNormFormer: Improved Transformer Pretraining with Extra Normalization.\narXiv, abs/2110.09456. \u003ca href=\"#user-content-fnref-21\" data-footnote-backref=\"\" aria-label=\"Back to reference 21\" class=\"data-footnote-backref\"\u003e↩\u003c/a\u003e\u003c/p\u003e\n\u003c/li\u003e\n\u003cli id=\"user-content-fn-22\"\u003e\n\u003cp\u003eBao, H., Dong, L., \u0026#x26; Wei, F. (2021).\nBEiT: BERT Pre-Training of Image Transformers.\narXiv, abs/2106.08254. \u003ca href=\"#user-content-fnref-22\" data-footnote-backref=\"\" aria-label=\"Back to reference 22\" class=\"data-footnote-backref\"\u003e↩\u003c/a\u003e\u003c/p\u003e\n\u003c/li\u003e\n\u003cli id=\"user-content-fn-23\"\u003e\n\u003cp\u003eHe, K., Chen, X., Xie, S., Li, Y., Doll'ar, P., \u0026#x26; Girshick, R.B. (2021).\nMasked Autoencoders Are Scalable Vision Learners.\n2022 IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR), 15979-15988. \u003ca href=\"#user-content-fnref-23\" data-footnote-backref=\"\" aria-label=\"Back to reference 23\" class=\"data-footnote-backref\"\u003e↩\u003c/a\u003e\u003c/p\u003e\n\u003c/li\u003e\n\u003cli id=\"user-content-fn-24\"\u003e\n\u003cp\u003eBrown, T.B., Mann, B., Ryder, N., Subbiah, M., Kaplan, J., Dhariwal, P., Neelakantan, A., Shyam, P., Sastry, G., Askell, A., Agarwal, S., Herbert-Voss, A., Krueger, G., Henighan, T.J., Child, R., Ramesh, A., Ziegler, D.M., Wu, J., Winter, C., Hesse, C., Chen, M., Sigler, E., Litwin, M., Gray, S., Chess, B., Clark, J., Berner, C., McCandlish, S., Radford, A., Sutskever, I., \u0026#x26; Amodei, D. (2020).\nLanguage Models are Few-Shot Learners.\narXiv, abs/2005.14165. \u003ca href=\"#user-content-fnref-24\" data-footnote-backref=\"\" aria-label=\"Back to reference 24\" class=\"data-footnote-backref\"\u003e↩\u003c/a\u003e\u003c/p\u003e\n\u003c/li\u003e\n\u003c/ol\u003e\n\u003c/section\u003e"])</script><script>self.__next_f.push([1,"7:[\"$\",\"main\",null,{\"children\":[\"$\",\"article\",null,{\"className\":\"blog-article\",\"data-upstream\":\"\",\"children\":[[\"$\",\"$L5\",null,{\"className\":\"blog-back\",\"href\":\"/blog\",\"children\":\"← Blog\"}],[\"$\",\"div\",null,{\"className\":\"blog-post-meta\",\"children\":[\"November 13, 2022\",\" \",\"·\",\" \",10,\" min read\"]}],[\"$\",\"h1\",null,{\"className\":\"blog-post-title\",\"children\":\"OFA: Towards Building a One-For-All Model\"}],[\"$\",\"p\",null,{\"className\":\"blog-post-lede\",\"children\":\"2022 is a year of generalist models! With the bloom of multimodal pretraining, especially the unified model, we have witnessed the opportunity to building a generalist model that is capable of processing tasks of different modalities or multi-modalities! Thus, we propose OFA[^1], namely One-For-All,\"}],[\"$\",\"div\",null,{\"className\":\"blog-prose\",\"dangerouslySetInnerHTML\":{\"__html\":\"$14\"}}]]}]}]\n"])</script><script>self.__next_f.push([1,"10:[[\"$\",\"meta\",\"0\",{\"charSet\":\"utf-8\"}],[\"$\",\"meta\",\"1\",{\"name\":\"viewport\",\"content\":\"width=device-width, initial-scale=1\"}]]\nc:null\n"])</script><script>self.__next_f.push([1,"e:{\"metadata\":[[\"$\",\"title\",\"0\",{\"children\":\"OFA: Towards Building a One-For-All Model — Zen Blog\"}],[\"$\",\"meta\",\"1\",{\"name\":\"description\",\"content\":\"2022 is a year of generalist models! With the bloom of multimodal pretraining, especially the unified model, we have witnessed the opportunity to building a generalist model that is capable of processing tasks of different modalities or multi-modalities! Thus, we propose OFA[^1], namely One-For-All,\"}],[\"$\",\"meta\",\"2\",{\"name\":\"keywords\",\"content\":\"AI, LLM, Agentic AI, Code Generation, Zen Coder, Multimodal, Open Source, Machine Learning\"}],[\"$\",\"meta\",\"3\",{\"name\":\"data-upstream\",\"content\":\"true\"}]],\"error\":null,\"digest\":\"$undefined\"}\n"])</script><script>self.__next_f.push([1,"13:\"$e:metadata\"\n"])</script></body></html>