mirror of
https://github.com/zenlm/zen-blog.git
synced 2026-11-02 17:28:53 +00:00
Root cause of HF + GitHub + section icons not appearing in the docs route: Next.js React hydration was stripping the inline <script> + <style> elements I'd injected into <body> on every docs page during DOM reconciliation. The script never registered its setInterval. Move the decorator to /zen-decorate.js loaded via <script defer src> in <head>. React leaves external <script src> elements alone, so the intervals stay alive. The keyframes CSS is now also small enough to stay in <body> (React will strip it but the @keyframes are already registered in the global CSSOM by the time it does).
293 lines
70 KiB
HTML
293 lines
70 KiB
HTML
<!DOCTYPE html><!--i_dnJM_MIpJSOCQWNJVMq--><html lang="en" class="dark"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="stylesheet" href="/_next/static/chunks/f2332aac77592f9d.css" data-precedence="next"/><link rel="preload" as="script" fetchPriority="low" href="/_next/static/chunks/e83606e8fa9cc796.js"/><script src="/_next/static/chunks/36d6595f0156cd7e.js" async=""></script><script src="/_next/static/chunks/040e9cea20a8d9c7.js" async=""></script><script src="/_next/static/chunks/d4dffb5a0973d49c.js" async=""></script><script src="/_next/static/chunks/turbopack-967f27a9fd33556d.js" async=""></script><script src="/_next/static/chunks/59d0ad1b64f8544e.js" async=""></script><script src="/_next/static/chunks/4d80e004cf4896dd.js" async=""></script><script src="/_next/static/chunks/350ee4303b732916.js" async=""></script><script src="/_next/static/chunks/8de849ca74fc071f.js" async=""></script><script src="/_next/static/chunks/e62b91212ee7f8ff.js" async=""></script><script src="/_next/static/chunks/f19fe44237e54646.js" async=""></script><script src="/_next/static/chunks/cb0a883bafeb6805.js" async=""></script><meta name="theme-color" media="(prefers-color-scheme: dark)" content="#0A0A0A"/><meta name="theme-color" media="(prefers-color-scheme: light)" content="#fff"/><title>Embedding Spaces at 7680 Dimensions — Zen LM Blog | Zen LM</title><meta name="description" content="Exploring high-dimensional embedding spaces for semantic search and retrieval."/><meta property="og:title" content="Zen LM - Open Foundation Models"/><meta property="og:description" content="Zen AI model family by Hanzo AI. 14 frontier models from 4B to 1T+ parameters for code, reasoning, vision, and multimodal tasks."/><meta property="og:url" content="https://zenlm.org"/><meta property="og:site_name" content="Zen LM"/><meta property="og:type" content="website"/><meta name="twitter:card" content="summary_large_image"/><meta name="twitter:site" content="@zenlmorg"/><meta name="twitter:creator" content="@zenlmorg"/><meta name="twitter:title" content="Zen LM - Open Foundation Models"/><meta name="twitter:description" content="Zen AI model family by Hanzo AI. 14 frontier models from 4B to 1T+ parameters for code, reasoning, vision, and multimodal tasks."/><script src="/_next/static/chunks/a6dad97d9634a72d.js" noModule=""></script><script defer src="/zen-decorate.js"></script></head><body class="antialiased"><div hidden=""><!--$--><!--/$--></div><script>((a,b,c,d,e,f,g,h)=>{let i=document.documentElement,j=["light","dark"];function k(b){var c;(Array.isArray(a)?a:[a]).forEach(a=>{let c="class"===a,d=c&&f?e.map(a=>f[a]||a):e;c?(i.classList.remove(...d),i.classList.add(f&&f[b]?f[b]:b)):i.setAttribute(a,b)}),c=b,h&&j.includes(c)&&(i.style.colorScheme=c)}if(d)k(d);else try{let a=localStorage.getItem(b)||c,d=g&&"system"===a?window.matchMedia("(prefers-color-scheme: dark)").matches?"dark":"light":a;k(d)}catch(a){}})("class","theme","system",null,["light","dark"],null,true,true)</script><!--$--><div data-closed="" role="presentation" hidden="" style="user-select:none;-webkit-user-select:none" class="fixed inset-0 z-50 backdrop-blur-xs bg-fd-overlay data-open:animate-fd-fade-in data-closed:animate-fd-fade-out"></div><div class="bg-fd-secondary/50 p-3 empty:hidden"></div><!--/$--><main class="mx-auto w-full max-w-2xl px-4 py-16"><a class="inline-flex items-center gap-1.5 text-sm text-fd-muted-foreground hover:text-fd-foreground mb-10 transition-colors" href="/blog">← Back to Blog</a><div class="mb-8"><time class="text-xs font-mono text-fd-muted-foreground uppercase tracking-wider">December 4, 2022</time><h1 class="text-3xl font-bold mt-2 mb-3">Embedding Spaces at 7680 Dimensions</h1><p class="text-fd-muted-foreground text-lg mb-4">Exploring high-dimensional embedding spaces for semantic search and retrieval.</p><div class="flex items-center gap-3 pt-4 border-t border-fd-border"><span class="text-sm text-fd-muted-foreground">By <!-- -->Zach Kelling</span><div class="flex gap-1.5 ml-auto"><span class="text-[10px] font-mono uppercase tracking-wider bg-fd-muted text-fd-muted-foreground px-1.5 py-0.5 rounded">Research</span><span class="text-[10px] font-mono uppercase tracking-wider bg-fd-muted text-fd-muted-foreground px-1.5 py-0.5 rounded">Embeddings</span><span class="text-[10px] font-mono uppercase tracking-wider bg-fd-muted text-fd-muted-foreground px-1.5 py-0.5 rounded">Retrieval</span></div></div></div><div class="prose dark:prose-invert max-w-none"><h1 class="flex scroll-m-28 flex-row items-center gap-2" id="the-dimension-question"><a data-card="" href="#the-dimension-question" class="peer">The Dimension Question</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h1>
|
||
<p>How many dimensions does a text embedding need?</p>
|
||
<p>The field has settled on conventions: 768 for BERT-scale models, 1536 for OpenAI’s ada-002, 4096 for some recent models. But these choices reflect architectural constraints, not fundamental requirements.</p>
|
||
<p>We investigate what happens when we scale embedding dimensions to 7680—ten times the BERT baseline.</p>
|
||
<h2 class="flex scroll-m-28 flex-row items-center gap-2" id="why-higher-dimensions"><a data-card="" href="#why-higher-dimensions" class="peer">Why Higher Dimensions?</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h2>
|
||
<h3 class="flex scroll-m-28 flex-row items-center gap-2" id="capacity-arguments"><a data-card="" href="#capacity-arguments" class="peer">Capacity Arguments</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h3>
|
||
<p>A $d$-dimensional embedding space can represent $\mathcal{O}(e^d)$ nearly-orthogonal vectors. For semantic search, we want documents with different meanings to map to different regions. Higher dimensions provide more room.</p>
|
||
<h3 class="flex scroll-m-28 flex-row items-center gap-2" id="information-theoretic-view"><a data-card="" href="#information-theoretic-view" class="peer">Information-Theoretic View</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h3>
|
||
<p>Consider a corpus of $N$ documents, each with entropy $H$ bits. A $d$-dimensional float32 embedding stores $32d$ bits. For lossless encoding:</p>
|
||
<p><code>32d \geq N \cdot H</code></p>
|
||
<p>For a corpus of 1 billion documents with 100 bits of effective information each, we need:</p>
|
||
<p><code>d \geq \frac\{10^9 \cdot 100\}\{32\} \approx 3 \times 10^9</code></p>
|
||
<p>Clearly, embeddings are lossy compression. But higher dimensions reduce the loss.</p>
|
||
<h3 class="flex scroll-m-28 flex-row items-center gap-2" id="empirical-observations"><a data-card="" href="#empirical-observations" class="peer">Empirical Observations</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h3>
|
||
<p>Retrieval quality on our internal benchmarks plateaued around:</p>
|
||
<ul>
|
||
<li>768 dimensions: 71% recall@10</li>
|
||
<li>1536 dimensions: 78% recall@10</li>
|
||
<li>3072 dimensions: 83% recall@10</li>
|
||
<li>7680 dimensions: 87% recall@10</li>
|
||
</ul>
|
||
<p>Diminishing returns set in, but gains persist well beyond conventional wisdom.</p>
|
||
<h2 class="flex scroll-m-28 flex-row items-center gap-2" id="training-high-dimensional-embeddings"><a data-card="" href="#training-high-dimensional-embeddings" class="peer">Training High-Dimensional Embeddings</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h2>
|
||
<h3 class="flex scroll-m-28 flex-row items-center gap-2" id="architecture"><a data-card="" href="#architecture" class="peer">Architecture</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h3>
|
||
<p>We use a standard transformer encoder with a projection head:</p>
|
||
<figure dir="ltr" class="my-4 bg-fd-card rounded-xl shiki relative border shadow-sm not-prose overflow-hidden text-sm shiki shiki-themes github-light github-dark" style="--shiki-light:#24292e;--shiki-dark:#e1e4e8;--shiki-light-bg:#fff;--shiki-dark-bg:#24292e" tabindex="-1"><div class="empty:hidden absolute top-3 right-2 z-2 backdrop-blur-lg rounded-lg text-fd-muted-foreground"><button type="button" class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors duration-100 disabled:pointer-events-none disabled:opacity-50 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-fd-ring p-1 [&_svg]:size-4 hover:text-fd-accent-foreground data-checked:text-fd-accent-foreground" aria-label="Copy Text"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-clipboard" aria-hidden="true"><rect width="8" height="4" x="8" y="2" rx="1" ry="1"></rect><path d="M16 4h2a2 2 0 0 1 2 2v14a2 2 0 0 1-2 2H6a2 2 0 0 1-2-2V6a2 2 0 0 1 2-2h2"></path></svg></button></div><div role="region" tabindex="0" class="text-[0.8125rem] py-3.5 overflow-auto max-h-[600px] fd-scroll-container focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-inset focus-visible:ring-fd-ring" style="--padding-right:calc(var(--spacing) * 8)"><pre class="min-w-full w-max *:flex *:flex-col"><code><span class="line"><span></span></span>
|
||
<span class="line"><span> Input --> Transformer(L=12, H=768) --> Pool --> Linear(768, 7680) --> Normalize</span></span>
|
||
<span class="line"><span> </span></span></code></pre></div></figure>
|
||
<p>The projection head maps from the transformer’s hidden dimension to the embedding space. This decouples representation capacity from compute requirements.</p>
|
||
<h3 class="flex scroll-m-28 flex-row items-center gap-2" id="contrastive-learning"><a data-card="" href="#contrastive-learning" class="peer">Contrastive Learning</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h3>
|
||
<p>We train with InfoNCE loss over batches of (query, positive, negatives):</p>
|
||
<p><code>\mathcal\{L\} = -\log \frac\{\exp(q \cdot p^+ / \tau)\}\{\sum_\{i\} \exp(q \cdot p_i / \tau)\}</code></p>
|
||
<p>With temperature $\tau = 0.01$ for high-dimensional spaces (lower than typical).</p>
|
||
<h3 class="flex scroll-m-28 flex-row items-center gap-2" id="hard-negative-mining"><a data-card="" href="#hard-negative-mining" class="peer">Hard Negative Mining</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h3>
|
||
<p>High-dimensional spaces require harder negatives to provide gradient signal. Our mining strategy:</p>
|
||
<ol>
|
||
<li>Retrieve top-100 candidates via approximate nearest neighbor</li>
|
||
<li>Filter to exclude true positives</li>
|
||
<li>Sample negatives weighted by similarity (harder = more likely)</li>
|
||
</ol>
|
||
<p>This curriculum focuses training on the decision boundary.</p>
|
||
<h2 class="flex scroll-m-28 flex-row items-center gap-2" id="practical-considerations"><a data-card="" href="#practical-considerations" class="peer">Practical Considerations</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h2>
|
||
<h3 class="flex scroll-m-28 flex-row items-center gap-2" id="storage"><a data-card="" href="#storage" class="peer">Storage</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h3>
|
||
<p>7680-dimensional float32 embeddings require 30KB per vector. For 1 billion documents:</p>
|
||
<p><code>10^9 \times 30\text\{KB\} = 30\text\{TB\}</code></p>
|
||
<p>This is substantial but manageable with modern storage.</p>
|
||
<h3 class="flex scroll-m-28 flex-row items-center gap-2" id="quantization"><a data-card="" href="#quantization" class="peer">Quantization</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h3>
|
||
<p>We can reduce storage through quantization:</p>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<div class="relative overflow-auto prose-no-margin my-6"><table><thead><tr><th>Precision</th><th>Bytes/Vector</th><th>Recall@10</th></tr></thead><tbody><tr><td>float32</td><td>30,720</td><td>87.3%</td></tr><tr><td>float16</td><td>15,360</td><td>87.1%</td></tr><tr><td>int8</td><td>7,680</td><td>85.9%</td></tr><tr><td>binary</td><td>960</td><td>78.2%</td></tr></tbody></table></div>
|
||
<p>int8 quantization provides 4x compression with minimal quality loss.</p>
|
||
<h3 class="flex scroll-m-28 flex-row items-center gap-2" id="approximate-search"><a data-card="" href="#approximate-search" class="peer">Approximate Search</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h3>
|
||
<p>Exact nearest neighbor search in 7680 dimensions is expensive. We use hierarchical navigable small world (HNSW) graphs:</p>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<div class="relative overflow-auto prose-no-margin my-6"><table><thead><tr><th>Dimensions</th><th>Build Time</th><th>Query Time</th><th>Recall@10</th></tr></thead><tbody><tr><td>768</td><td>1x</td><td>1x</td><td>99.2%</td></tr><tr><td>7680</td><td>3.2x</td><td>2.8x</td><td>98.7%</td></tr></tbody></table></div>
|
||
<p>The overhead is sublinear in dimensionality due to efficient distance computations.</p>
|
||
<h2 class="flex scroll-m-28 flex-row items-center gap-2" id="benchmark-results"><a data-card="" href="#benchmark-results" class="peer">Benchmark Results</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h2>
|
||
<h3 class="flex scroll-m-28 flex-row items-center gap-2" id="ms-marco-passage-retrieval"><a data-card="" href="#ms-marco-passage-retrieval" class="peer">MS MARCO Passage Retrieval</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h3>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<div class="relative overflow-auto prose-no-margin my-6"><table><thead><tr><th>Model</th><th>Dimensions</th><th>MRR@10</th><th>Recall@100</th></tr></thead><tbody><tr><td>BM25</td><td>-</td><td>18.4</td><td>66.5</td></tr><tr><td>DPR</td><td>768</td><td>31.1</td><td>82.4</td></tr><tr><td>Contriever</td><td>768</td><td>32.8</td><td>84.1</td></tr><tr><td>Zen-Embed</td><td>7680</td><td>38.6</td><td>91.3</td></tr></tbody></table></div>
|
||
<h3 class="flex scroll-m-28 flex-row items-center gap-2" id="natural-questions"><a data-card="" href="#natural-questions" class="peer">Natural Questions</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h3>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<div class="relative overflow-auto prose-no-margin my-6"><table><thead><tr><th>Model</th><th>Dimensions</th><th>Top-20 Acc</th><th>Top-100 Acc</th></tr></thead><tbody><tr><td>DPR</td><td>768</td><td>78.4</td><td>85.4</td></tr><tr><td>Contriever</td><td>768</td><td>81.3</td><td>88.1</td></tr><tr><td>Zen-Embed</td><td>7680</td><td>86.7</td><td>93.2</td></tr></tbody></table></div>
|
||
<p>High-dimensional embeddings provide substantial gains on retrieval benchmarks.</p>
|
||
<h2 class="flex scroll-m-28 flex-row items-center gap-2" id="analysis"><a data-card="" href="#analysis" class="peer">Analysis</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h2>
|
||
<h3 class="flex scroll-m-28 flex-row items-center gap-2" id="what-do-extra-dimensions-encode"><a data-card="" href="#what-do-extra-dimensions-encode" class="peer">What Do Extra Dimensions Encode?</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h3>
|
||
<p>We analyze the learned embedding space through probing tasks:</p>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<div class="relative overflow-auto prose-no-margin my-6"><table><thead><tr><th>Property</th><th>768d Probe Acc</th><th>7680d Probe Acc</th></tr></thead><tbody><tr><td>Topic</td><td>84.2%</td><td>86.1%</td></tr><tr><td>Sentiment</td><td>91.3%</td><td>92.8%</td></tr><tr><td>Entity</td><td>67.4%</td><td>78.9%</td></tr><tr><td>Relation</td><td>52.1%</td><td>71.3%</td></tr></tbody></table></div>
|
||
<p>The largest gains are in entity and relation encoding—fine-grained semantic properties that require more capacity.</p>
|
||
<h3 class="flex scroll-m-28 flex-row items-center gap-2" id="nearest-neighbor-analysis"><a data-card="" href="#nearest-neighbor-analysis" class="peer">Nearest Neighbor Analysis</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h3>
|
||
<p>For the query “What causes inflation?”, nearest neighbors at different dimensions:</p>
|
||
<p><strong>768 dimensions:</strong></p>
|
||
<ol>
|
||
<li>What is inflation? (similar query)</li>
|
||
<li>How does inflation work? (similar query)</li>
|
||
<li>Inflation rates by country (tangential)</li>
|
||
</ol>
|
||
<p><strong>7680 dimensions:</strong></p>
|
||
<ol>
|
||
<li>Inflation is caused by… (direct answer)</li>
|
||
<li>The primary drivers of inflation include… (direct answer)</li>
|
||
<li>Central bank policies affect inflation through… (relevant detail)</li>
|
||
</ol>
|
||
<p>Higher dimensions better distinguish queries from answers.</p>
|
||
<h2 class="flex scroll-m-28 flex-row items-center gap-2" id="recommendations"><a data-card="" href="#recommendations" class="peer">Recommendations</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h2>
|
||
<p>Based on our experiments:</p>
|
||
<ol>
|
||
<li><strong>Try 3072+ dimensions</strong> if retrieval quality matters and storage is available</li>
|
||
<li><strong>Use int8 quantization</strong> for production deployments</li>
|
||
<li><strong>Invest in hard negative mining</strong> to realize the benefits of capacity</li>
|
||
<li><strong>Benchmark on your data</strong> —gains vary by domain</li>
|
||
</ol>
|
||
<p>Conventional embedding sizes are conventions, not laws. Question them.</p>
|
||
<hr/>
|
||
<p><em>Technical details in “Scaling Embedding Dimensions for Semantic Retrieval” (2022). Model weights available on Hugging Face.</em></p></div></main><!--$--><!--/$--><script src="/_next/static/chunks/e83606e8fa9cc796.js" id="_R_" async=""></script><script>(self.__next_f=self.__next_f||[]).push([0])</script><script>self.__next_f.push([1,"1:\"$Sreact.fragment\"\n2:I[106,[\"/_next/static/chunks/59d0ad1b64f8544e.js\"],\"RootProvider\"]\n3:I[53113,[\"/_next/static/chunks/4d80e004cf4896dd.js\",\"/_next/static/chunks/350ee4303b732916.js\"],\"default\"]\n4:I[73211,[\"/_next/static/chunks/4d80e004cf4896dd.js\",\"/_next/static/chunks/350ee4303b732916.js\"],\"default\"]\n5:I[10086,[\"/_next/static/chunks/59d0ad1b64f8544e.js\",\"/_next/static/chunks/8de849ca74fc071f.js\",\"/_next/static/chunks/e62b91212ee7f8ff.js\",\"/_next/static/chunks/f19fe44237e54646.js\",\"/_next/static/chunks/cb0a883bafeb6805.js\"],\"\"]\n7:I[89923,[\"/_next/static/chunks/4d80e004cf4896dd.js\",\"/_next/static/chunks/350ee4303b732916.js\"],\"OutletBoundary\"]\n8:\"$Sreact.suspense\"\na:I[89923,[\"/_next/static/chunks/4d80e004cf4896dd.js\",\"/_next/static/chunks/350ee4303b732916.js\"],\"ViewportBoundary\"]\nc:I[89923,[\"/_next/static/chunks/4d80e004cf4896dd.js\",\"/_next/static/chunks/350ee4303b732916.js\"],\"MetadataBoundary\"]\ne:I[6998,[\"/_next/static/chunks/4d80e004cf4896dd.js\",\"/_next/static/chunks/350ee4303b732916.js\"],\"default\"]\n:HL[\"/_next/static/chunks/f2332aac77592f9d.css\",\"style\"]\n"])</script><script>self.__next_f.push([1,"0:{\"P\":null,\"b\":\"i-dnJM_MIpJSOCQWNJVMq\",\"c\":[\"\",\"blog\",\"embedding-spaces-7680-dimensions\"],\"q\":\"\",\"i\":false,\"f\":[[[\"\",{\"children\":[\"blog\",{\"children\":[[\"slug\",\"embedding-spaces-7680-dimensions\",\"d\"],{\"children\":[\"__PAGE__\",{}]}]}]},\"$undefined\",\"$undefined\",true],[[\"$\",\"$1\",\"c\",{\"children\":[[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/chunks/f2332aac77592f9d.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\",\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-0\",{\"src\":\"/_next/static/chunks/59d0ad1b64f8544e.js\",\"async\":true,\"nonce\":\"$undefined\"}]],[\"$\",\"html\",null,{\"lang\":\"en\",\"className\":\"dark\",\"suppressHydrationWarning\":true,\"children\":[\"$\",\"body\",null,{\"className\":\"antialiased\",\"children\":[\"$\",\"$L2\",null,{\"children\":[\"$\",\"$L3\",null,{\"parallelRouterKey\":\"children\",\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L4\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":[[\"$\",\"main\",null,{\"className\":\"flex min-h-screen flex-col items-center justify-center px-4 text-center\",\"children\":[[\"$\",\"div\",null,{\"className\":\"mb-8 opacity-20\",\"children\":[\"$\",\"svg\",null,{\"width\":\"120\",\"height\":\"120\",\"viewBox\":\"0 0 120 120\",\"fill\":\"none\",\"aria-hidden\":\"true\",\"children\":[\"$\",\"circle\",null,{\"cx\":\"60\",\"cy\":\"60\",\"r\":\"50\",\"stroke\":\"currentColor\",\"strokeWidth\":\"3\",\"strokeLinecap\":\"round\",\"strokeDasharray\":\"280 40\"}]}]}],[\"$\",\"p\",null,{\"className\":\"text-xs font-mono uppercase tracking-[0.3em] text-fd-muted-foreground mb-4\",\"children\":\"404\"}],[\"$\",\"h1\",null,{\"className\":\"text-3xl font-semibold mb-3\",\"children\":\"Page not found\"}],[\"$\",\"p\",null,{\"className\":\"text-fd-muted-foreground max-w-sm mb-10\",\"children\":\"This page doesn't exist, or it may have moved. Try the documentation or head home.\"}],[\"$\",\"div\",null,{\"className\":\"flex flex-wrap gap-3 justify-center\",\"children\":[[\"$\",\"$L5\",null,{\"href\":\"/\",\"className\":\"inline-flex items-center gap-2 rounded-lg bg-fd-primary text-fd-primary-foreground px-4 py-2 text-sm font-medium hover:opacity-90 transition\",\"children\":\"Go home\"}],[\"$\",\"$L5\",null,{\"href\":\"/docs\",\"className\":\"inline-flex items-center gap-2 rounded-lg border border-fd-border bg-fd-background px-4 py-2 text-sm font-medium hover:bg-fd-muted transition\",\"children\":\"Documentation\"}],[\"$\",\"$L5\",null,{\"href\":\"/docs/models\",\"className\":\"inline-flex items-center gap-2 rounded-lg border border-fd-border bg-fd-background px-4 py-2 text-sm font-medium hover:bg-fd-muted transition\",\"children\":\"Browse models\"}]]}],[\"$\",\"p\",null,{\"className\":\"mt-16 text-xs font-mono text-fd-muted-foreground opacity-50\",\"children\":\"zenlm.org\"}]]}],[]],\"forbidden\":\"$undefined\",\"unauthorized\":\"$undefined\"}]}]}]}]]}],{\"children\":[[\"$\",\"$1\",\"c\",{\"children\":[null,[\"$\",\"$L3\",null,{\"parallelRouterKey\":\"children\",\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L4\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":\"$undefined\",\"forbidden\":\"$undefined\",\"unauthorized\":\"$undefined\"}]]}],{\"children\":[[\"$\",\"$1\",\"c\",{\"children\":[null,[\"$\",\"$L3\",null,{\"parallelRouterKey\":\"children\",\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L4\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":\"$undefined\",\"forbidden\":\"$undefined\",\"unauthorized\":\"$undefined\"}]]}],{\"children\":[[\"$\",\"$1\",\"c\",{\"children\":[\"$L6\",[[\"$\",\"script\",\"script-0\",{\"src\":\"/_next/static/chunks/8de849ca74fc071f.js\",\"async\":true,\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-1\",{\"src\":\"/_next/static/chunks/e62b91212ee7f8ff.js\",\"async\":true,\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-2\",{\"src\":\"/_next/static/chunks/f19fe44237e54646.js\",\"async\":true,\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-3\",{\"src\":\"/_next/static/chunks/cb0a883bafeb6805.js\",\"async\":true,\"nonce\":\"$undefined\"}]],[\"$\",\"$L7\",null,{\"children\":[\"$\",\"$8\",null,{\"name\":\"Next.MetadataOutlet\",\"children\":\"$@9\"}]}]]}],{},null,false,false]},null,false,false]},null,false,false]},null,false,false],[\"$\",\"$1\",\"h\",{\"children\":[null,[\"$\",\"$La\",null,{\"children\":\"$Lb\"}],[\"$\",\"div\",null,{\"hidden\":true,\"children\":[\"$\",\"$Lc\",null,{\"children\":[\"$\",\"$8\",null,{\"name\":\"Next.Metadata\",\"children\":\"$Ld\"}]}]}],null]}],false]],\"m\":\"$undefined\",\"G\":[\"$e\",[]],\"S\":true}\n"])</script><script>self.__next_f.push([1,"6:[\"$\",\"main\",null,{\"className\":\"mx-auto w-full max-w-2xl px-4 py-16\",\"children\":[[\"$\",\"$L5\",null,{\"href\":\"/blog\",\"className\":\"inline-flex items-center gap-1.5 text-sm text-fd-muted-foreground hover:text-fd-foreground mb-10 transition-colors\",\"children\":\"← Back to Blog\"}],[\"$\",\"div\",null,{\"className\":\"mb-8\",\"children\":[[\"$\",\"time\",null,{\"className\":\"text-xs font-mono text-fd-muted-foreground uppercase tracking-wider\",\"children\":\"December 4, 2022\"}],[\"$\",\"h1\",null,{\"className\":\"text-3xl font-bold mt-2 mb-3\",\"children\":\"Embedding Spaces at 7680 Dimensions\"}],[\"$\",\"p\",null,{\"className\":\"text-fd-muted-foreground text-lg mb-4\",\"children\":\"Exploring high-dimensional embedding spaces for semantic search and retrieval.\"}],[\"$\",\"div\",null,{\"className\":\"flex items-center gap-3 pt-4 border-t border-fd-border\",\"children\":[[\"$\",\"span\",null,{\"className\":\"text-sm text-fd-muted-foreground\",\"children\":[\"By \",\"Zach Kelling\"]}],[\"$\",\"div\",null,{\"className\":\"flex gap-1.5 ml-auto\",\"children\":[[\"$\",\"span\",\"Research\",{\"className\":\"text-[10px] font-mono uppercase tracking-wider bg-fd-muted text-fd-muted-foreground px-1.5 py-0.5 rounded\",\"children\":\"Research\"}],[\"$\",\"span\",\"Embeddings\",{\"className\":\"text-[10px] font-mono uppercase tracking-wider bg-fd-muted text-fd-muted-foreground px-1.5 py-0.5 rounded\",\"children\":\"Embeddings\"}],[\"$\",\"span\",\"Retrieval\",{\"className\":\"text-[10px] font-mono uppercase tracking-wider bg-fd-muted text-fd-muted-foreground px-1.5 py-0.5 rounded\",\"children\":\"Retrieval\"}]]}]]}]]}],[\"$\",\"div\",null,{\"className\":\"prose dark:prose-invert max-w-none\",\"children\":[[\"$\",\"h1\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"the-dimension-question\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#the-dimension-question\",\"className\":\"peer\",\"children\":\"The Dimension Question\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}],\"\\n\",[\"$\",\"p\",null,{\"children\":\"How many dimensions does a text embedding need?\"}],\"\\n\",[\"$\",\"p\",null,{\"children\":\"The field has settled on conventions: 768 for BERT-scale models, 1536 for OpenAI’s ada-002, 4096 for some recent models. But these choices reflect architectural constraints, not fundamental requirements.\"}],\"\\n\",[\"$\",\"p\",null,{\"children\":\"We investigate what happens when we scale embedding dimensions to 7680—ten times the BERT baseline.\"}],\"\\n\",[\"$\",\"h2\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"why-higher-dimensions\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#why-higher-dimensions\",\"className\":\"peer\",\"children\":\"Why Higher Dimensions?\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}],\"\\n\",[\"$\",\"h3\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"capacity-arguments\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#capacity-arguments\",\"className\":\"peer\",\"children\":\"Capacity Arguments\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[\"$Lf\",\"$L10\",\"$undefined\"]}]]}],\"\\n\",\"$L11\",\"\\n\",\"$L12\",\"\\n\",\"$L13\",\"\\n\",\"$L14\",\"\\n\",\"$L15\",\"\\n\",\"$L16\",\"\\n\",\"$L17\",\"\\n\",\"$L18\",\"\\n\",\"$L19\",\"\\n\",\"$L1a\",\"\\n\",\"$L1b\",\"\\n\",\"$L1c\",\"\\n\",\"$L1d\",\"\\n\",\"$L1e\",\"\\n\",\"$L1f\",\"\\n\",\"$L20\",\"\\n\",\"$L21\",\"\\n\",\"$L22\",\"\\n\",\"$L23\",\"\\n\",\"$L24\",\"\\n\",\"$L25\",\"\\n\",\"$L26\",\"\\n\",\"$L27\",\"\\n\",\"$L28\",\"\\n\",\"$L29\",\"\\n\",\"$L2a\",\"\\n\",\"$L2b\",\"\\n\",\"$L2c\",\"\\n\",\"$L2d\",\"\\n\",\"$L2e\",\"\\n\",\"$L2f\",\"\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\",\"$L30\",\"\\n\",\"$L31\",\"\\n\",\"$L32\",\"\\n\",\"$L33\",\"\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\",\"$L34\",\"\\n\",\"$L35\",\"\\n\",\"$L36\",\"\\n\",\"$L37\",\"\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\",\"$L38\",\"\\n\",\"$L39\",\"\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\",\"$L3a\",\"\\n\",\"$L3b\",\"\\n\",\"$L3c\",\"\\n\",\"$L3d\",\"\\n\",\"$L3e\",\"\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\",\"$L3f\",\"\\n\",\"$L40\",\"\\n\",\"$L41\",\"\\n\",\"$L42\",\"\\n\",\"$L43\",\"\\n\",\"$L44\",\"\\n\",\"$L45\",\"\\n\",\"$L46\",\"\\n\",\"$L47\",\"\\n\",\"$L48\",\"\\n\",\"$L49\",\"\\n\",\"$L4a\",\"\\n\",\"$L4b\",\"\\n\",\"$L4c\",\"\\n\",\"$L4d\"]}]]}]\n"])</script><script>self.__next_f.push([1,"4e:I[51504,[\"/_next/static/chunks/59d0ad1b64f8544e.js\",\"/_next/static/chunks/8de849ca74fc071f.js\",\"/_next/static/chunks/e62b91212ee7f8ff.js\",\"/_next/static/chunks/f19fe44237e54646.js\",\"/_next/static/chunks/cb0a883bafeb6805.js\"],\"CodeBlock\"]\n4f:I[51504,[\"/_next/static/chunks/59d0ad1b64f8544e.js\",\"/_next/static/chunks/8de849ca74fc071f.js\",\"/_next/static/chunks/e62b91212ee7f8ff.js\",\"/_next/static/chunks/f19fe44237e54646.js\",\"/_next/static/chunks/cb0a883bafeb6805.js\"],\"Pre\"]\nf:[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}]\n10:[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}]\n11:[\"$\",\"p\",null,{\"children\":\"A $d$-dimensional embedding space can represent $\\\\mathcal{O}(e^d)$ nearly-orthogonal vectors. For semantic search, we want documents with different meanings to map to different regions. Higher dimensions provide more room.\"}]\n12:[\"$\",\"h3\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"information-theoretic-view\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#information-theoretic-view\",\"className\":\"peer\",\"children\":\"Information-Theoretic View\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n13:[\"$\",\"p\",null,{\"children\":\"Consider a corpus of $N$ documents, each with entropy $H$ bits. A $d$-dimensional float32 embedding stores $32d$ bits. For lossless encoding:\"}]\n14:[\"$\",\"p\",null,{\"children\":[\"$\",\"code\",null,{\"children\":\"32d \\\\geq N \\\\cdot H\"}]}]\n15:[\"$\",\"p\",null,{\"children\":\"For a corpus of 1 billion documents with 100 bits of effective information each, we need:\"}]\n16:[\"$\",\"p\",null,{\"children\":[\"$\",\"code\",null,{\"children\":\"d \\\\geq \\\\frac\\\\{10^9 \\\\cdot 100\\\\}\\\\{32\\\\} \\\\approx 3 \\\\times 10^9\"}]}]\n17:[\"$\",\"p\",null,{\"children\":\"Clearly, embeddings are lossy compression. But higher dimensions reduce the loss.\"}]\n18:[\"$\",\"h3\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"empirical-observations\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#empirical-observations\",\"className\":\"peer\",\"children\":\"Empirical Observations\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n19:[\"$\",\"p\",null,{\"children\":\"Retrieval quality on our internal benchmarks plateaued around:\"}]\n1a:[\"$\",\"ul\",null,{\"children\":[\"\\n\",[\"$\",\"li\",null,{\"children\":\"768 dimensions: 71% recall@10\"}],\"\\n\",[\"$\",\"li\",null,{\"children\":\"1536 dimensions: 78% recall@10\"}],\"\\n\",[\"$\",\"li\",null,{\"children\":\"3072 dimensions: 83% recall@10\"}],\"\\n\",[\"$\",\"li\",null,{\"children\":\"7680 dimensions: 87% recall@10\"}],\"\\n\"]}]\n1b:[\"$\",\"p\",null,{\"children\":\"Diminishing returns set in, but gains persist well beyond conventional wisdom.\"}]\n1c:[\"$\",\"h2\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"training-high-dimensional-embeddings\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#training-high-dimensional-embeddings\",\"className\":\"peer\",\"children\":\"Training High-Dimensional Embeddings\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"vi"])</script><script>self.__next_f.push([1,"ewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n1d:[\"$\",\"h3\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"architecture\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#architecture\",\"className\":\"peer\",\"children\":\"Architecture\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n1e:[\"$\",\"p\",null,{\"children\":\"We use a standard transformer encoder with a projection head:\"}]\n1f:[\"$\",\"$L4e\",null,{\"className\":\"shiki shiki-themes github-light github-dark\",\"style\":{\"--shiki-light\":\"#24292e\",\"--shiki-dark\":\"#e1e4e8\",\"--shiki-light-bg\":\"#fff\",\"--shiki-dark-bg\":\"#24292e\"},\"tabIndex\":\"0\",\"icon\":\"\u003csvg viewBox=\\\"0 0 24 24\\\"\u003e\u003cpath d=\\\"M 6,1 C 4.354992,1 3,2.354992 3,4 v 16 c 0,1.645008 1.354992,3 3,3 h 12 c 1.645008,0 3,-1.354992 3,-3 V 8 7 A 1.0001,1.0001 0 0 0 20.707031,6.2929687 l -5,-5 A 1.0001,1.0001 0 0 0 15,1 h -1 z m 0,2 h 7 v 3 c 0,1.645008 1.354992,3 3,3 h 3 v 11 c 0,0.564129 -0.435871,1 -1,1 H 6 C 5.4358712,21 5,20.564129 5,20 V 4 C 5,3.4358712 5.4358712,3 6,3 Z M 15,3.4140625 18.585937,7 H 16 C 15.435871,7 15,6.5641288 15,6 Z\\\" fill=\\\"currentColor\\\" /\u003e\u003c/svg\u003e\",\"children\":[\"$\",\"$L4f\",null,{\"children\":[\"$\",\"code\",null,{\"children\":[[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{}]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"children\":\" Input --\u003e Transformer(L=12, H=768) --\u003e Pool --\u003e Linear(768, 7680) --\u003e Normalize\"}]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"children\":\" \"}]}]]}]}]}]\n20:[\"$\",\"p\",null,{\"children\":\"The projection head maps from the transformer’s hidden dimension to the embedding space. This decouples representation capacity from compute requirements.\"}]\n21:[\"$\",\"h3\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"contrastive-learning\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#contrastive-learning\",\"className\":\"peer\",\"children\":\"Contrastive Learning\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n22:[\"$\",\"p\",null,{\"children\":\"We train with InfoNCE loss over batches of (query, positive, negatives):\"}]\n23:[\"$\",\"p\",null,{\"children\":[\"$\",\"code\",null,{\"children\":\"\\\\mathcal\\\\{L\\\\} = -\\\\log \\\\frac\\\\{\\\\exp(q \\\\cdot p^+ / \\\\tau)\\\\}\\\\{\\\\sum_\\\\{i\\\\} \\\\exp(q \\\\cdot p_i / \\\\tau)\\\\}\"}]}]\n24:[\"$\",\"p\",null,{\"children\":\"With temperature $\\\\tau = 0.01$ for high-dimensional spaces (lower than typical).\"}]\n25:[\"$\",\"h3\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"hard-negative-mining\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#hard-negative-mining\",\"className\":\"peer\",\"children\":\"Hard N"])</script><script>self.__next_f.push([1,"egative Mining\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n26:[\"$\",\"p\",null,{\"children\":\"High-dimensional spaces require harder negatives to provide gradient signal. Our mining strategy:\"}]\n27:[\"$\",\"ol\",null,{\"children\":[\"\\n\",[\"$\",\"li\",null,{\"children\":\"Retrieve top-100 candidates via approximate nearest neighbor\"}],\"\\n\",[\"$\",\"li\",null,{\"children\":\"Filter to exclude true positives\"}],\"\\n\",[\"$\",\"li\",null,{\"children\":\"Sample negatives weighted by similarity (harder = more likely)\"}],\"\\n\"]}]\n28:[\"$\",\"p\",null,{\"children\":\"This curriculum focuses training on the decision boundary.\"}]\n29:[\"$\",\"h2\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"practical-considerations\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#practical-considerations\",\"className\":\"peer\",\"children\":\"Practical Considerations\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n2a:[\"$\",\"h3\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"storage\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#storage\",\"className\":\"peer\",\"children\":\"Storage\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n2b:[\"$\",\"p\",null,{\"children\":\"7680-dimensional float32 embeddings require 30KB per vector. For 1 billion documents:\"}]\n2c:[\"$\",\"p\",null,{\"children\":[\"$\",\"code\",null,{\"children\":\"10^9 \\\\times 30\\\\text\\\\{KB\\\\} = 30\\\\text\\\\{TB\\\\}\"}]}]\n2d:[\"$\",\"p\",null,{\"children\":\"This is substantial but manageable with modern storage.\"}]\n2e:[\"$\",\"h3\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"quantization\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#quantization\",\"className\":\"peer\",\"children\":\"Quantization\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n2f:[\"$\",\"p\",null,{\"children\":\"We can reduce storage through quantization:\"}]\n30:[\"$\",\"div\",null,{\"className\":\"relative overflow-auto prose-no-margin my-6\",\"children\":[\"$\",\"table\",null,{\"children\":[[\"$\",\"thead\",null,{\"children\":[\"$\",\"tr\",null,{\"children\":[[\"$\""])</script><script>self.__next_f.push([1,",\"th\",null,{\"children\":\"Precision\"}],[\"$\",\"th\",null,{\"children\":\"Bytes/Vector\"}],[\"$\",\"th\",null,{\"children\":\"Recall@10\"}]]}]}],[\"$\",\"tbody\",null,{\"children\":[[\"$\",\"tr\",null,{\"children\":[[\"$\",\"td\",null,{\"children\":\"float32\"}],[\"$\",\"td\",null,{\"children\":\"30,720\"}],[\"$\",\"td\",null,{\"children\":\"87.3%\"}]]}],[\"$\",\"tr\",null,{\"children\":[[\"$\",\"td\",null,{\"children\":\"float16\"}],[\"$\",\"td\",null,{\"children\":\"15,360\"}],[\"$\",\"td\",null,{\"children\":\"87.1%\"}]]}],[\"$\",\"tr\",null,{\"children\":[[\"$\",\"td\",null,{\"children\":\"int8\"}],[\"$\",\"td\",null,{\"children\":\"7,680\"}],[\"$\",\"td\",null,{\"children\":\"85.9%\"}]]}],[\"$\",\"tr\",null,{\"children\":[[\"$\",\"td\",null,{\"children\":\"binary\"}],[\"$\",\"td\",null,{\"children\":\"960\"}],[\"$\",\"td\",null,{\"children\":\"78.2%\"}]]}]]}]]}]}]\n31:[\"$\",\"p\",null,{\"children\":\"int8 quantization provides 4x compression with minimal quality loss.\"}]\n32:[\"$\",\"h3\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"approximate-search\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#approximate-search\",\"className\":\"peer\",\"children\":\"Approximate Search\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n33:[\"$\",\"p\",null,{\"children\":\"Exact nearest neighbor search in 7680 dimensions is expensive. We use hierarchical navigable small world (HNSW) graphs:\"}]\n34:[\"$\",\"div\",null,{\"className\":\"relative overflow-auto prose-no-margin my-6\",\"children\":[\"$\",\"table\",null,{\"children\":[[\"$\",\"thead\",null,{\"children\":[\"$\",\"tr\",null,{\"children\":[[\"$\",\"th\",null,{\"children\":\"Dimensions\"}],[\"$\",\"th\",null,{\"children\":\"Build Time\"}],[\"$\",\"th\",null,{\"children\":\"Query Time\"}],[\"$\",\"th\",null,{\"children\":\"Recall@10\"}]]}]}],[\"$\",\"tbody\",null,{\"children\":[[\"$\",\"tr\",null,{\"children\":[[\"$\",\"td\",null,{\"children\":\"768\"}],[\"$\",\"td\",null,{\"children\":\"1x\"}],[\"$\",\"td\",null,{\"children\":\"1x\"}],[\"$\",\"td\",null,{\"children\":\"99.2%\"}]]}],[\"$\",\"tr\",null,{\"children\":[[\"$\",\"td\",null,{\"children\":\"7680\"}],[\"$\",\"td\",null,{\"children\":\"3.2x\"}],[\"$\",\"td\",null,{\"children\":\"2.8x\"}],[\"$\",\"td\",null,{\"children\":\"98.7%\"}]]}]]}]]}]}]\n35:[\"$\",\"p\",null,{\"children\":\"The overhead is sublinear in dimensionality due to efficient distance computations.\"}]\n36:[\"$\",\"h2\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"benchmark-results\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#benchmark-results\",\"className\":\"peer\",\"children\":\"Benchmark Results\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n37:[\"$\",\"h3\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"ms-marco-passage-retrieval\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#ms-marco-passage-retrieval\",\"className\":\"peer\",\"children\":\"MS MARCO Passage Retrieval\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\","])</script><script>self.__next_f.push([1,"\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n38:[\"$\",\"div\",null,{\"className\":\"relative overflow-auto prose-no-margin my-6\",\"children\":[\"$\",\"table\",null,{\"children\":[[\"$\",\"thead\",null,{\"children\":[\"$\",\"tr\",null,{\"children\":[[\"$\",\"th\",null,{\"children\":\"Model\"}],[\"$\",\"th\",null,{\"children\":\"Dimensions\"}],[\"$\",\"th\",null,{\"children\":\"MRR@10\"}],[\"$\",\"th\",null,{\"children\":\"Recall@100\"}]]}]}],[\"$\",\"tbody\",null,{\"children\":[[\"$\",\"tr\",null,{\"children\":[[\"$\",\"td\",null,{\"children\":\"BM25\"}],[\"$\",\"td\",null,{\"children\":\"-\"}],[\"$\",\"td\",null,{\"children\":\"18.4\"}],[\"$\",\"td\",null,{\"children\":\"66.5\"}]]}],[\"$\",\"tr\",null,{\"children\":[[\"$\",\"td\",null,{\"children\":\"DPR\"}],[\"$\",\"td\",null,{\"children\":\"768\"}],[\"$\",\"td\",null,{\"children\":\"31.1\"}],[\"$\",\"td\",null,{\"children\":\"82.4\"}]]}],[\"$\",\"tr\",null,{\"children\":[[\"$\",\"td\",null,{\"children\":\"Contriever\"}],[\"$\",\"td\",null,{\"children\":\"768\"}],[\"$\",\"td\",null,{\"children\":\"32.8\"}],[\"$\",\"td\",null,{\"children\":\"84.1\"}]]}],[\"$\",\"tr\",null,{\"children\":[[\"$\",\"td\",null,{\"children\":\"Zen-Embed\"}],[\"$\",\"td\",null,{\"children\":\"7680\"}],[\"$\",\"td\",null,{\"children\":\"38.6\"}],[\"$\",\"td\",null,{\"children\":\"91.3\"}]]}]]}]]}]}]\n39:[\"$\",\"h3\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"natural-questions\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#natural-questions\",\"className\":\"peer\",\"children\":\"Natural Questions\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n3a:[\"$\",\"div\",null,{\"className\":\"relative overflow-auto prose-no-margin my-6\",\"children\":[\"$\",\"table\",null,{\"children\":[[\"$\",\"thead\",null,{\"children\":[\"$\",\"tr\",null,{\"children\":[[\"$\",\"th\",null,{\"children\":\"Model\"}],[\"$\",\"th\",null,{\"children\":\"Dimensions\"}],[\"$\",\"th\",null,{\"children\":\"Top-20 Acc\"}],[\"$\",\"th\",null,{\"children\":\"Top-100 Acc\"}]]}]}],[\"$\",\"tbody\",null,{\"children\":[[\"$\",\"tr\",null,{\"children\":[[\"$\",\"td\",null,{\"children\":\"DPR\"}],[\"$\",\"td\",null,{\"children\":\"768\"}],[\"$\",\"td\",null,{\"children\":\"78.4\"}],[\"$\",\"td\",null,{\"children\":\"85.4\"}]]}],[\"$\",\"tr\",null,{\"children\":[[\"$\",\"td\",null,{\"children\":\"Contriever\"}],[\"$\",\"td\",null,{\"children\":\"768\"}],[\"$\",\"td\",null,{\"children\":\"81.3\"}],[\"$\",\"td\",null,{\"children\":\"88.1\"}]]}],[\"$\",\"tr\",null,{\"children\":[[\"$\",\"td\",null,{\"children\":\"Zen-Embed\"}],[\"$\",\"td\",null,{\"children\":\"7680\"}],[\"$\",\"td\",null,{\"children\":\"86.7\"}],[\"$\",\"td\",null,{\"children\":\"93.2\"}]]}]]}]]}]}]\n3b:[\"$\",\"p\",null,{\"children\":\"High-dimensional embeddings provide substantial gains on retrieval benchmarks.\"}]\n3c:[\"$\",\"h2\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"analysis\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#analysis\",\"className\":\"peer\",\"children\":\"Analysis\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n3d:[\"$\",\"h3\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"what-do-extra-dimensions-encode\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#what-do-extra-dimensions-encode\","])</script><script>self.__next_f.push([1,"\"className\":\"peer\",\"children\":\"What Do Extra Dimensions Encode?\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n3e:[\"$\",\"p\",null,{\"children\":\"We analyze the learned embedding space through probing tasks:\"}]\n3f:[\"$\",\"div\",null,{\"className\":\"relative overflow-auto prose-no-margin my-6\",\"children\":[\"$\",\"table\",null,{\"children\":[[\"$\",\"thead\",null,{\"children\":[\"$\",\"tr\",null,{\"children\":[[\"$\",\"th\",null,{\"children\":\"Property\"}],[\"$\",\"th\",null,{\"children\":\"768d Probe Acc\"}],[\"$\",\"th\",null,{\"children\":\"7680d Probe Acc\"}]]}]}],[\"$\",\"tbody\",null,{\"children\":[[\"$\",\"tr\",null,{\"children\":[[\"$\",\"td\",null,{\"children\":\"Topic\"}],[\"$\",\"td\",null,{\"children\":\"84.2%\"}],[\"$\",\"td\",null,{\"children\":\"86.1%\"}]]}],[\"$\",\"tr\",null,{\"children\":[[\"$\",\"td\",null,{\"children\":\"Sentiment\"}],[\"$\",\"td\",null,{\"children\":\"91.3%\"}],[\"$\",\"td\",null,{\"children\":\"92.8%\"}]]}],[\"$\",\"tr\",null,{\"children\":[[\"$\",\"td\",null,{\"children\":\"Entity\"}],[\"$\",\"td\",null,{\"children\":\"67.4%\"}],[\"$\",\"td\",null,{\"children\":\"78.9%\"}]]}],[\"$\",\"tr\",null,{\"children\":[[\"$\",\"td\",null,{\"children\":\"Relation\"}],[\"$\",\"td\",null,{\"children\":\"52.1%\"}],[\"$\",\"td\",null,{\"children\":\"71.3%\"}]]}]]}]]}]}]\n40:[\"$\",\"p\",null,{\"children\":\"The largest gains are in entity and relation encoding—fine-grained semantic properties that require more capacity.\"}]\n41:[\"$\",\"h3\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"nearest-neighbor-analysis\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#nearest-neighbor-analysis\",\"className\":\"peer\",\"children\":\"Nearest Neighbor Analysis\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n42:[\"$\",\"p\",null,{\"children\":\"For the query “What causes inflation?”, nearest neighbors at different dimensions:\"}]\n43:[\"$\",\"p\",null,{\"children\":[\"$\",\"strong\",null,{\"children\":\"768 dimensions:\"}]}]\n44:[\"$\",\"ol\",null,{\"children\":[\"\\n\",[\"$\",\"li\",null,{\"children\":\"What is inflation? (similar query)\"}],\"\\n\",[\"$\",\"li\",null,{\"children\":\"How does inflation work? (similar query)\"}],\"\\n\",[\"$\",\"li\",null,{\"children\":\"Inflation rates by country (tangential)\"}],\"\\n\"]}]\n45:[\"$\",\"p\",null,{\"children\":[\"$\",\"strong\",null,{\"children\":\"7680 dimensions:\"}]}]\n46:[\"$\",\"ol\",null,{\"children\":[\"\\n\",[\"$\",\"li\",null,{\"children\":\"Inflation is caused by… (direct answer)\"}],\"\\n\",[\"$\",\"li\",null,{\"children\":\"The primary drivers of inflation include… (direct answer)\"}],\"\\n\",[\"$\",\"li\",null,{\"children\":\"Central bank policies affect inflation through… (relevant detail)\"}],\"\\n\"]}]\n47:[\"$\",\"p\",null,{\"children\":\"Higher dimensions better distinguish queries from answers.\"}]\n48:[\"$\",\"h2\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"recommendations\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#recommendations\",\"className\":\"peer\",\"children\":\"Recommendations\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink"])</script><script>self.__next_f.push([1,"-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n49:[\"$\",\"p\",null,{\"children\":\"Based on our experiments:\"}]\n4a:[\"$\",\"ol\",null,{\"children\":[\"\\n\",[\"$\",\"li\",null,{\"children\":[[\"$\",\"strong\",null,{\"children\":\"Try 3072+ dimensions\"}],\" if retrieval quality matters and storage is available\"]}],\"\\n\",[\"$\",\"li\",null,{\"children\":[[\"$\",\"strong\",null,{\"children\":\"Use int8 quantization\"}],\" for production deployments\"]}],\"\\n\",[\"$\",\"li\",null,{\"children\":[[\"$\",\"strong\",null,{\"children\":\"Invest in hard negative mining\"}],\" to realize the benefits of capacity\"]}],\"\\n\",[\"$\",\"li\",null,{\"children\":[[\"$\",\"strong\",null,{\"children\":\"Benchmark on your data\"}],\" —gains vary by domain\"]}],\"\\n\"]}]\n4b:[\"$\",\"p\",null,{\"children\":\"Conventional embedding sizes are conventions, not laws. Question them.\"}]\n4c:[\"$\",\"hr\",null,{}]\n4d:[\"$\",\"p\",null,{\"children\":[\"$\",\"em\",null,{\"children\":\"Technical details in “Scaling Embedding Dimensions for Semantic Retrieval” (2022). Model weights available on Hugging Face.\"}]}]\n"])</script><script>self.__next_f.push([1,"b:[[\"$\",\"meta\",\"0\",{\"charSet\":\"utf-8\"}],[\"$\",\"meta\",\"1\",{\"name\":\"viewport\",\"content\":\"width=device-width, initial-scale=1\"}],[\"$\",\"meta\",\"2\",{\"name\":\"theme-color\",\"media\":\"(prefers-color-scheme: dark)\",\"content\":\"#0A0A0A\"}],[\"$\",\"meta\",\"3\",{\"name\":\"theme-color\",\"media\":\"(prefers-color-scheme: light)\",\"content\":\"#fff\"}]]\n"])</script><script>self.__next_f.push([1,"9:null\nd:[[\"$\",\"title\",\"0\",{\"children\":\"Embedding Spaces at 7680 Dimensions — Zen LM Blog | Zen LM\"}],[\"$\",\"meta\",\"1\",{\"name\":\"description\",\"content\":\"Exploring high-dimensional embedding spaces for semantic search and retrieval.\"}],[\"$\",\"meta\",\"2\",{\"property\":\"og:title\",\"content\":\"Zen LM - Open Foundation Models\"}],[\"$\",\"meta\",\"3\",{\"property\":\"og:description\",\"content\":\"Zen AI model family by Hanzo AI. 14 frontier models from 4B to 1T+ parameters for code, reasoning, vision, and multimodal tasks.\"}],[\"$\",\"meta\",\"4\",{\"property\":\"og:url\",\"content\":\"https://zenlm.org\"}],[\"$\",\"meta\",\"5\",{\"property\":\"og:site_name\",\"content\":\"Zen LM\"}],[\"$\",\"meta\",\"6\",{\"property\":\"og:type\",\"content\":\"website\"}],[\"$\",\"meta\",\"7\",{\"name\":\"twitter:card\",\"content\":\"summary_large_image\"}],[\"$\",\"meta\",\"8\",{\"name\":\"twitter:site\",\"content\":\"@zenlmorg\"}],[\"$\",\"meta\",\"9\",{\"name\":\"twitter:creator\",\"content\":\"@zenlmorg\"}],[\"$\",\"meta\",\"10\",{\"name\":\"twitter:title\",\"content\":\"Zen LM - Open Foundation Models\"}],[\"$\",\"meta\",\"11\",{\"name\":\"twitter:description\",\"content\":\"Zen AI model family by Hanzo AI. 14 frontier models from 4B to 1T+ parameters for code, reasoning, vision, and multimodal tasks.\"}]]\n"])</script><script>
|
||
(function(){
|
||
// After React renders the lotus emoji from RSC payload, walk the DOM
|
||
// and replace it with the enso SVG. Polls so subsequent React re-renders
|
||
// also get cleaned up.
|
||
function swap(){
|
||
// 1. Replace lotus + ' Zen LM' text inside <a> brand links
|
||
document.querySelectorAll('a').forEach(function(a){
|
||
if (a.dataset.zenLotusSwapped) return;
|
||
var t = (a.textContent || '');
|
||
if (!/.*Zen LM|Zen LM.*/.test(t)) return;
|
||
// Find the lotus text node specifically
|
||
var walker = document.createTreeWalker(a, NodeFilter.SHOW_TEXT, null);
|
||
var node;
|
||
while ((node = walker.nextNode())) {
|
||
if (node.nodeValue && node.nodeValue.indexOf('') !== -1) {
|
||
node.nodeValue = node.nodeValue.replace(/\s?/g, '');
|
||
}
|
||
}
|
||
// Inject enso SVG at start
|
||
if (!a.querySelector('svg[aria-label="Zen LM"]')) {
|
||
var ns = 'http://www.w3.org/2000/svg';
|
||
var svg = document.createElementNS(ns, 'svg');
|
||
svg.setAttribute('viewBox', '0 0 512 512');
|
||
svg.setAttribute('fill', 'none');
|
||
svg.setAttribute('width', '18');
|
||
svg.setAttribute('height', '18');
|
||
svg.setAttribute('aria-label', 'Zen LM');
|
||
svg.style.cssText = 'display:inline-block;vertical-align:-2px;margin-right:6px;flex-shrink:0';
|
||
var p = document.createElementNS(ns, 'path');
|
||
p.setAttribute('d', 'M 256 84 A 172 172 0 1 0 256 428 A 172 172 0 1 0 256 84 Z');
|
||
p.setAttribute('stroke', 'currentColor');
|
||
p.setAttribute('stroke-width', '26');
|
||
p.setAttribute('stroke-linecap', 'round');
|
||
p.setAttribute('stroke-linejoin', 'round');
|
||
svg.appendChild(p);
|
||
a.insertBefore(svg, a.firstChild);
|
||
}
|
||
a.dataset.zenLotusSwapped = '1';
|
||
});
|
||
}
|
||
function start(){ swap(); setInterval(swap, 600); }
|
||
if (document.readyState !== 'loading') start();
|
||
else document.addEventListener('DOMContentLoaded', start);
|
||
})();
|
||
</script>
|
||
|
||
<style id="zenlm-brand-anim">
|
||
@keyframes zenlm-brand-draw { to { stroke-dashoffset: 0; } }
|
||
svg.zenlm-brand-svg .zenlm-brand-path {
|
||
stroke-dasharray: 1100;
|
||
stroke-dashoffset: 1100;
|
||
animation: zenlm-brand-draw 1.4s cubic-bezier(.45,0,.2,1) 0.1s 1 forwards;
|
||
}
|
||
.zen-link-icon { display: inline-block; vertical-align: middle; flex-shrink: 0; }
|
||
</style></body></html> |