Files
zen-blog/blog/embedding-spaces-7680-dimensions.txt

82 lines
37 KiB
Plaintext
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
1:"$Sreact.fragment"
2:I[106,["/_next/static/chunks/59d0ad1b64f8544e.js"],"RootProvider"]
3:I[53113,["/_next/static/chunks/4d80e004cf4896dd.js","/_next/static/chunks/350ee4303b732916.js"],"default"]
4:I[73211,["/_next/static/chunks/4d80e004cf4896dd.js","/_next/static/chunks/350ee4303b732916.js"],"default"]
5:I[10086,["/_next/static/chunks/59d0ad1b64f8544e.js","/_next/static/chunks/8de849ca74fc071f.js","/_next/static/chunks/e62b91212ee7f8ff.js","/_next/static/chunks/f19fe44237e54646.js","/_next/static/chunks/cb0a883bafeb6805.js"],""]
7:I[89923,["/_next/static/chunks/4d80e004cf4896dd.js","/_next/static/chunks/350ee4303b732916.js"],"OutletBoundary"]
8:"$Sreact.suspense"
a:I[89923,["/_next/static/chunks/4d80e004cf4896dd.js","/_next/static/chunks/350ee4303b732916.js"],"ViewportBoundary"]
c:I[89923,["/_next/static/chunks/4d80e004cf4896dd.js","/_next/static/chunks/350ee4303b732916.js"],"MetadataBoundary"]
e:I[6998,["/_next/static/chunks/4d80e004cf4896dd.js","/_next/static/chunks/350ee4303b732916.js"],"default"]
:HL["/_next/static/chunks/f2332aac77592f9d.css","style"]
0:{"P":null,"b":"i-dnJM_MIpJSOCQWNJVMq","c":["","blog","embedding-spaces-7680-dimensions"],"q":"","i":false,"f":[[["",{"children":["blog",{"children":[["slug","embedding-spaces-7680-dimensions","d"],{"children":["__PAGE__",{}]}]}]},"$undefined","$undefined",true],[["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/_next/static/chunks/f2332aac77592f9d.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","script","script-0",{"src":"/_next/static/chunks/59d0ad1b64f8544e.js","async":true,"nonce":"$undefined"}]],["$","html",null,{"lang":"en","className":"dark","suppressHydrationWarning":true,"children":["$","body",null,{"className":"antialiased","children":["$","$L2",null,{"children":["$","$L3",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L4",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","main",null,{"className":"flex min-h-screen flex-col items-center justify-center px-4 text-center","children":[["$","div",null,{"className":"mb-8 opacity-20","children":["$","svg",null,{"width":"120","height":"120","viewBox":"0 0 120 120","fill":"none","aria-hidden":"true","children":["$","circle",null,{"cx":"60","cy":"60","r":"50","stroke":"currentColor","strokeWidth":"3","strokeLinecap":"round","strokeDasharray":"280 40"}]}]}],["$","p",null,{"className":"text-xs font-mono uppercase tracking-[0.3em] text-fd-muted-foreground mb-4","children":"404"}],["$","h1",null,{"className":"text-3xl font-semibold mb-3","children":"Page not found"}],["$","p",null,{"className":"text-fd-muted-foreground max-w-sm mb-10","children":"This page doesn't exist, or it may have moved. Try the documentation or head home."}],["$","div",null,{"className":"flex flex-wrap gap-3 justify-center","children":[["$","$L5",null,{"href":"/","className":"inline-flex items-center gap-2 rounded-lg bg-fd-primary text-fd-primary-foreground px-4 py-2 text-sm font-medium hover:opacity-90 transition","children":"Go home"}],["$","$L5",null,{"href":"/docs","className":"inline-flex items-center gap-2 rounded-lg border border-fd-border bg-fd-background px-4 py-2 text-sm font-medium hover:bg-fd-muted transition","children":"Documentation"}],["$","$L5",null,{"href":"/docs/models","className":"inline-flex items-center gap-2 rounded-lg border border-fd-border bg-fd-background px-4 py-2 text-sm font-medium hover:bg-fd-muted transition","children":"Browse models"}]]}],["$","p",null,{"className":"mt-16 text-xs font-mono text-fd-muted-foreground opacity-50","children":"zenlm.org"}]]}],[]],"forbidden":"$undefined","unauthorized":"$undefined"}]}]}]}]]}],{"children":[["$","$1","c",{"children":[null,["$","$L3",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L4",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","forbidden":"$undefined","unauthorized":"$undefined"}]]}],{"children":[["$","$1","c",{"children":[null,["$","$L3",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L4",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","forbidden":"$undefined","unauthorized":"$undefined"}]]}],{"children":[["$","$1","c",{"children":["$L6",[["$","script","script-0",{"src":"/_next/static/chunks/8de849ca74fc071f.js","async":true,"nonce":"$undefined"}],["$","script","script-1",{"src":"/_next/static/chunks/e62b91212ee7f8ff.js","async":true,"nonce":"$undefined"}],["$","script","script-2",{"src":"/_next/static/chunks/f19fe44237e54646.js","async":true,"nonce":"$undefined"}],["$","script","script-3",{"src":"/_next/static/chunks/cb0a883bafeb6805.js","async":true,"nonce":"$undefined"}]],["$","$L7",null,{"children":["$","$8",null,{"name":"Next.MetadataOutlet","children":"$@9"}]}]]}],{},null,false,false]},null,false,false]},null,false,false]},null,false,false],["$","$1","h",{"children":[null,["$","$La",null,{"children":"$Lb"}],["$","div",null,{"hidden":true,"children":["$","$Lc",null,{"children":["$","$8",null,{"name":"Next.Metadata","children":"$Ld"}]}]}],null]}],false]],"m":"$undefined","G":["$e",[]],"S":true}
6:["$","main",null,{"className":"mx-auto w-full max-w-2xl px-4 py-16","children":[["$","$L5",null,{"href":"/blog","className":"inline-flex items-center gap-1.5 text-sm text-fd-muted-foreground hover:text-fd-foreground mb-10 transition-colors","children":"← Back to Blog"}],["$","div",null,{"className":"mb-8","children":[["$","time",null,{"className":"text-xs font-mono text-fd-muted-foreground uppercase tracking-wider","children":"December 4, 2022"}],["$","h1",null,{"className":"text-3xl font-bold mt-2 mb-3","children":"Embedding Spaces at 7680 Dimensions"}],["$","p",null,{"className":"text-fd-muted-foreground text-lg mb-4","children":"Exploring high-dimensional embedding spaces for semantic search and retrieval."}],["$","div",null,{"className":"flex items-center gap-3 pt-4 border-t border-fd-border","children":[["$","span",null,{"className":"text-sm text-fd-muted-foreground","children":["By ","Zach Kelling"]}],["$","div",null,{"className":"flex gap-1.5 ml-auto","children":[["$","span","Research",{"className":"text-[10px] font-mono uppercase tracking-wider bg-fd-muted text-fd-muted-foreground px-1.5 py-0.5 rounded","children":"Research"}],["$","span","Embeddings",{"className":"text-[10px] font-mono uppercase tracking-wider bg-fd-muted text-fd-muted-foreground px-1.5 py-0.5 rounded","children":"Embeddings"}],["$","span","Retrieval",{"className":"text-[10px] font-mono uppercase tracking-wider bg-fd-muted text-fd-muted-foreground px-1.5 py-0.5 rounded","children":"Retrieval"}]]}]]}]]}],["$","div",null,{"className":"prose dark:prose-invert max-w-none","children":[["$","h1",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"the-dimension-question","children":[["$","a",null,{"data-card":"","href":"#the-dimension-question","className":"peer","children":"The Dimension Question"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}],"\n",["$","p",null,{"children":"How many dimensions does a text embedding need?"}],"\n",["$","p",null,{"children":"The field has settled on conventions: 768 for BERT-scale models, 1536 for OpenAI’s ada-002, 4096 for some recent models. But these choices reflect architectural constraints, not fundamental requirements."}],"\n",["$","p",null,{"children":"We investigate what happens when we scale embedding dimensions to 7680—ten times the BERT baseline."}],"\n",["$","h2",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"why-higher-dimensions","children":[["$","a",null,{"data-card":"","href":"#why-higher-dimensions","className":"peer","children":"Why Higher Dimensions?"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}],"\n",["$","h3",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"capacity-arguments","children":[["$","a",null,{"data-card":"","href":"#capacity-arguments","className":"peer","children":"Capacity Arguments"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":["$Lf","$L10","$undefined"]}]]}],"\n","$L11","\n","$L12","\n","$L13","\n","$L14","\n","$L15","\n","$L16","\n","$L17","\n","$L18","\n","$L19","\n","$L1a","\n","$L1b","\n","$L1c","\n","$L1d","\n","$L1e","\n","$L1f","\n","$L20","\n","$L21","\n","$L22","\n","$L23","\n","$L24","\n","$L25","\n","$L26","\n","$L27","\n","$L28","\n","$L29","\n","$L2a","\n","$L2b","\n","$L2c","\n","$L2d","\n","$L2e","\n","$L2f","\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n","$L30","\n","$L31","\n","$L32","\n","$L33","\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n","$L34","\n","$L35","\n","$L36","\n","$L37","\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n","$L38","\n","$L39","\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n","$L3a","\n","$L3b","\n","$L3c","\n","$L3d","\n","$L3e","\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n","$L3f","\n","$L40","\n","$L41","\n","$L42","\n","$L43","\n","$L44","\n","$L45","\n","$L46","\n","$L47","\n","$L48","\n","$L49","\n","$L4a","\n","$L4b","\n","$L4c","\n","$L4d"]}]]}]
4e:I[51504,["/_next/static/chunks/59d0ad1b64f8544e.js","/_next/static/chunks/8de849ca74fc071f.js","/_next/static/chunks/e62b91212ee7f8ff.js","/_next/static/chunks/f19fe44237e54646.js","/_next/static/chunks/cb0a883bafeb6805.js"],"CodeBlock"]
4f:I[51504,["/_next/static/chunks/59d0ad1b64f8544e.js","/_next/static/chunks/8de849ca74fc071f.js","/_next/static/chunks/e62b91212ee7f8ff.js","/_next/static/chunks/f19fe44237e54646.js","/_next/static/chunks/cb0a883bafeb6805.js"],"Pre"]
f:["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}]
10:["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}]
11:["$","p",null,{"children":"A $d$-dimensional embedding space can represent $\\mathcal{O}(e^d)$ nearly-orthogonal vectors. For semantic search, we want documents with different meanings to map to different regions. Higher dimensions provide more room."}]
12:["$","h3",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"information-theoretic-view","children":[["$","a",null,{"data-card":"","href":"#information-theoretic-view","className":"peer","children":"Information-Theoretic View"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
13:["$","p",null,{"children":"Consider a corpus of $N$ documents, each with entropy $H$ bits. A $d$-dimensional float32 embedding stores $32d$ bits. For lossless encoding:"}]
14:["$","p",null,{"children":["$","code",null,{"children":"32d \\geq N \\cdot H"}]}]
15:["$","p",null,{"children":"For a corpus of 1 billion documents with 100 bits of effective information each, we need:"}]
16:["$","p",null,{"children":["$","code",null,{"children":"d \\geq \\frac\\{10^9 \\cdot 100\\}\\{32\\} \\approx 3 \\times 10^9"}]}]
17:["$","p",null,{"children":"Clearly, embeddings are lossy compression. But higher dimensions reduce the loss."}]
18:["$","h3",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"empirical-observations","children":[["$","a",null,{"data-card":"","href":"#empirical-observations","className":"peer","children":"Empirical Observations"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
19:["$","p",null,{"children":"Retrieval quality on our internal benchmarks plateaued around:"}]
1a:["$","ul",null,{"children":["\n",["$","li",null,{"children":"768 dimensions: 71% recall@10"}],"\n",["$","li",null,{"children":"1536 dimensions: 78% recall@10"}],"\n",["$","li",null,{"children":"3072 dimensions: 83% recall@10"}],"\n",["$","li",null,{"children":"7680 dimensions: 87% recall@10"}],"\n"]}]
1b:["$","p",null,{"children":"Diminishing returns set in, but gains persist well beyond conventional wisdom."}]
1c:["$","h2",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"training-high-dimensional-embeddings","children":[["$","a",null,{"data-card":"","href":"#training-high-dimensional-embeddings","className":"peer","children":"Training High-Dimensional Embeddings"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
1d:["$","h3",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"architecture","children":[["$","a",null,{"data-card":"","href":"#architecture","className":"peer","children":"Architecture"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
1e:["$","p",null,{"children":"We use a standard transformer encoder with a projection head:"}]
1f:["$","$L4e",null,{"className":"shiki shiki-themes github-light github-dark","style":{"--shiki-light":"#24292e","--shiki-dark":"#e1e4e8","--shiki-light-bg":"#fff","--shiki-dark-bg":"#24292e"},"tabIndex":"0","icon":"<svg viewBox=\"0 0 24 24\"><path d=\"M 6,1 C 4.354992,1 3,2.354992 3,4 v 16 c 0,1.645008 1.354992,3 3,3 h 12 c 1.645008,0 3,-1.354992 3,-3 V 8 7 A 1.0001,1.0001 0 0 0 20.707031,6.2929687 l -5,-5 A 1.0001,1.0001 0 0 0 15,1 h -1 z m 0,2 h 7 v 3 c 0,1.645008 1.354992,3 3,3 h 3 v 11 c 0,0.564129 -0.435871,1 -1,1 H 6 C 5.4358712,21 5,20.564129 5,20 V 4 C 5,3.4358712 5.4358712,3 6,3 Z M 15,3.4140625 18.585937,7 H 16 C 15.435871,7 15,6.5641288 15,6 Z\" fill=\"currentColor\" /></svg>","children":["$","$L4f",null,{"children":["$","code",null,{"children":[["$","span",null,{"className":"line","children":["$","span",null,{}]}],"\n",["$","span",null,{"className":"line","children":["$","span",null,{"children":" Input --> Transformer(L=12, H=768) --> Pool --> Linear(768, 7680) --> Normalize"}]}],"\n",["$","span",null,{"className":"line","children":["$","span",null,{"children":" "}]}]]}]}]}]
20:["$","p",null,{"children":"The projection head maps from the transformer’s hidden dimension to the embedding space. This decouples representation capacity from compute requirements."}]
21:["$","h3",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"contrastive-learning","children":[["$","a",null,{"data-card":"","href":"#contrastive-learning","className":"peer","children":"Contrastive Learning"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
22:["$","p",null,{"children":"We train with InfoNCE loss over batches of (query, positive, negatives):"}]
23:["$","p",null,{"children":["$","code",null,{"children":"\\mathcal\\{L\\} = -\\log \\frac\\{\\exp(q \\cdot p^+ / \\tau)\\}\\{\\sum_\\{i\\} \\exp(q \\cdot p_i / \\tau)\\}"}]}]
24:["$","p",null,{"children":"With temperature $\\tau = 0.01$ for high-dimensional spaces (lower than typical)."}]
25:["$","h3",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"hard-negative-mining","children":[["$","a",null,{"data-card":"","href":"#hard-negative-mining","className":"peer","children":"Hard Negative Mining"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
26:["$","p",null,{"children":"High-dimensional spaces require harder negatives to provide gradient signal. Our mining strategy:"}]
27:["$","ol",null,{"children":["\n",["$","li",null,{"children":"Retrieve top-100 candidates via approximate nearest neighbor"}],"\n",["$","li",null,{"children":"Filter to exclude true positives"}],"\n",["$","li",null,{"children":"Sample negatives weighted by similarity (harder = more likely)"}],"\n"]}]
28:["$","p",null,{"children":"This curriculum focuses training on the decision boundary."}]
29:["$","h2",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"practical-considerations","children":[["$","a",null,{"data-card":"","href":"#practical-considerations","className":"peer","children":"Practical Considerations"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
2a:["$","h3",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"storage","children":[["$","a",null,{"data-card":"","href":"#storage","className":"peer","children":"Storage"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
2b:["$","p",null,{"children":"7680-dimensional float32 embeddings require 30KB per vector. For 1 billion documents:"}]
2c:["$","p",null,{"children":["$","code",null,{"children":"10^9 \\times 30\\text\\{KB\\} = 30\\text\\{TB\\}"}]}]
2d:["$","p",null,{"children":"This is substantial but manageable with modern storage."}]
2e:["$","h3",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"quantization","children":[["$","a",null,{"data-card":"","href":"#quantization","className":"peer","children":"Quantization"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
2f:["$","p",null,{"children":"We can reduce storage through quantization:"}]
30:["$","div",null,{"className":"relative overflow-auto prose-no-margin my-6","children":["$","table",null,{"children":[["$","thead",null,{"children":["$","tr",null,{"children":[["$","th",null,{"children":"Precision"}],["$","th",null,{"children":"Bytes/Vector"}],["$","th",null,{"children":"Recall@10"}]]}]}],["$","tbody",null,{"children":[["$","tr",null,{"children":[["$","td",null,{"children":"float32"}],["$","td",null,{"children":"30,720"}],["$","td",null,{"children":"87.3%"}]]}],["$","tr",null,{"children":[["$","td",null,{"children":"float16"}],["$","td",null,{"children":"15,360"}],["$","td",null,{"children":"87.1%"}]]}],["$","tr",null,{"children":[["$","td",null,{"children":"int8"}],["$","td",null,{"children":"7,680"}],["$","td",null,{"children":"85.9%"}]]}],["$","tr",null,{"children":[["$","td",null,{"children":"binary"}],["$","td",null,{"children":"960"}],["$","td",null,{"children":"78.2%"}]]}]]}]]}]}]
31:["$","p",null,{"children":"int8 quantization provides 4x compression with minimal quality loss."}]
32:["$","h3",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"approximate-search","children":[["$","a",null,{"data-card":"","href":"#approximate-search","className":"peer","children":"Approximate Search"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
33:["$","p",null,{"children":"Exact nearest neighbor search in 7680 dimensions is expensive. We use hierarchical navigable small world (HNSW) graphs:"}]
34:["$","div",null,{"className":"relative overflow-auto prose-no-margin my-6","children":["$","table",null,{"children":[["$","thead",null,{"children":["$","tr",null,{"children":[["$","th",null,{"children":"Dimensions"}],["$","th",null,{"children":"Build Time"}],["$","th",null,{"children":"Query Time"}],["$","th",null,{"children":"Recall@10"}]]}]}],["$","tbody",null,{"children":[["$","tr",null,{"children":[["$","td",null,{"children":"768"}],["$","td",null,{"children":"1x"}],["$","td",null,{"children":"1x"}],["$","td",null,{"children":"99.2%"}]]}],["$","tr",null,{"children":[["$","td",null,{"children":"7680"}],["$","td",null,{"children":"3.2x"}],["$","td",null,{"children":"2.8x"}],["$","td",null,{"children":"98.7%"}]]}]]}]]}]}]
35:["$","p",null,{"children":"The overhead is sublinear in dimensionality due to efficient distance computations."}]
36:["$","h2",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"benchmark-results","children":[["$","a",null,{"data-card":"","href":"#benchmark-results","className":"peer","children":"Benchmark Results"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
37:["$","h3",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"ms-marco-passage-retrieval","children":[["$","a",null,{"data-card":"","href":"#ms-marco-passage-retrieval","className":"peer","children":"MS MARCO Passage Retrieval"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
38:["$","div",null,{"className":"relative overflow-auto prose-no-margin my-6","children":["$","table",null,{"children":[["$","thead",null,{"children":["$","tr",null,{"children":[["$","th",null,{"children":"Model"}],["$","th",null,{"children":"Dimensions"}],["$","th",null,{"children":"MRR@10"}],["$","th",null,{"children":"Recall@100"}]]}]}],["$","tbody",null,{"children":[["$","tr",null,{"children":[["$","td",null,{"children":"BM25"}],["$","td",null,{"children":"-"}],["$","td",null,{"children":"18.4"}],["$","td",null,{"children":"66.5"}]]}],["$","tr",null,{"children":[["$","td",null,{"children":"DPR"}],["$","td",null,{"children":"768"}],["$","td",null,{"children":"31.1"}],["$","td",null,{"children":"82.4"}]]}],["$","tr",null,{"children":[["$","td",null,{"children":"Contriever"}],["$","td",null,{"children":"768"}],["$","td",null,{"children":"32.8"}],["$","td",null,{"children":"84.1"}]]}],["$","tr",null,{"children":[["$","td",null,{"children":"Zen-Embed"}],["$","td",null,{"children":"7680"}],["$","td",null,{"children":"38.6"}],["$","td",null,{"children":"91.3"}]]}]]}]]}]}]
39:["$","h3",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"natural-questions","children":[["$","a",null,{"data-card":"","href":"#natural-questions","className":"peer","children":"Natural Questions"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
3a:["$","div",null,{"className":"relative overflow-auto prose-no-margin my-6","children":["$","table",null,{"children":[["$","thead",null,{"children":["$","tr",null,{"children":[["$","th",null,{"children":"Model"}],["$","th",null,{"children":"Dimensions"}],["$","th",null,{"children":"Top-20 Acc"}],["$","th",null,{"children":"Top-100 Acc"}]]}]}],["$","tbody",null,{"children":[["$","tr",null,{"children":[["$","td",null,{"children":"DPR"}],["$","td",null,{"children":"768"}],["$","td",null,{"children":"78.4"}],["$","td",null,{"children":"85.4"}]]}],["$","tr",null,{"children":[["$","td",null,{"children":"Contriever"}],["$","td",null,{"children":"768"}],["$","td",null,{"children":"81.3"}],["$","td",null,{"children":"88.1"}]]}],["$","tr",null,{"children":[["$","td",null,{"children":"Zen-Embed"}],["$","td",null,{"children":"7680"}],["$","td",null,{"children":"86.7"}],["$","td",null,{"children":"93.2"}]]}]]}]]}]}]
3b:["$","p",null,{"children":"High-dimensional embeddings provide substantial gains on retrieval benchmarks."}]
3c:["$","h2",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"analysis","children":[["$","a",null,{"data-card":"","href":"#analysis","className":"peer","children":"Analysis"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
3d:["$","h3",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"what-do-extra-dimensions-encode","children":[["$","a",null,{"data-card":"","href":"#what-do-extra-dimensions-encode","className":"peer","children":"What Do Extra Dimensions Encode?"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
3e:["$","p",null,{"children":"We analyze the learned embedding space through probing tasks:"}]
3f:["$","div",null,{"className":"relative overflow-auto prose-no-margin my-6","children":["$","table",null,{"children":[["$","thead",null,{"children":["$","tr",null,{"children":[["$","th",null,{"children":"Property"}],["$","th",null,{"children":"768d Probe Acc"}],["$","th",null,{"children":"7680d Probe Acc"}]]}]}],["$","tbody",null,{"children":[["$","tr",null,{"children":[["$","td",null,{"children":"Topic"}],["$","td",null,{"children":"84.2%"}],["$","td",null,{"children":"86.1%"}]]}],["$","tr",null,{"children":[["$","td",null,{"children":"Sentiment"}],["$","td",null,{"children":"91.3%"}],["$","td",null,{"children":"92.8%"}]]}],["$","tr",null,{"children":[["$","td",null,{"children":"Entity"}],["$","td",null,{"children":"67.4%"}],["$","td",null,{"children":"78.9%"}]]}],["$","tr",null,{"children":[["$","td",null,{"children":"Relation"}],["$","td",null,{"children":"52.1%"}],["$","td",null,{"children":"71.3%"}]]}]]}]]}]}]
40:["$","p",null,{"children":"The largest gains are in entity and relation encoding—fine-grained semantic properties that require more capacity."}]
41:["$","h3",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"nearest-neighbor-analysis","children":[["$","a",null,{"data-card":"","href":"#nearest-neighbor-analysis","className":"peer","children":"Nearest Neighbor Analysis"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
42:["$","p",null,{"children":"For the query “What causes inflation?”, nearest neighbors at different dimensions:"}]
43:["$","p",null,{"children":["$","strong",null,{"children":"768 dimensions:"}]}]
44:["$","ol",null,{"children":["\n",["$","li",null,{"children":"What is inflation? (similar query)"}],"\n",["$","li",null,{"children":"How does inflation work? (similar query)"}],"\n",["$","li",null,{"children":"Inflation rates by country (tangential)"}],"\n"]}]
45:["$","p",null,{"children":["$","strong",null,{"children":"7680 dimensions:"}]}]
46:["$","ol",null,{"children":["\n",["$","li",null,{"children":"Inflation is caused by… (direct answer)"}],"\n",["$","li",null,{"children":"The primary drivers of inflation include… (direct answer)"}],"\n",["$","li",null,{"children":"Central bank policies affect inflation through… (relevant detail)"}],"\n"]}]
47:["$","p",null,{"children":"Higher dimensions better distinguish queries from answers."}]
48:["$","h2",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"recommendations","children":[["$","a",null,{"data-card":"","href":"#recommendations","className":"peer","children":"Recommendations"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
49:["$","p",null,{"children":"Based on our experiments:"}]
4a:["$","ol",null,{"children":["\n",["$","li",null,{"children":[["$","strong",null,{"children":"Try 3072+ dimensions"}]," if retrieval quality matters and storage is available"]}],"\n",["$","li",null,{"children":[["$","strong",null,{"children":"Use int8 quantization"}]," for production deployments"]}],"\n",["$","li",null,{"children":[["$","strong",null,{"children":"Invest in hard negative mining"}]," to realize the benefits of capacity"]}],"\n",["$","li",null,{"children":[["$","strong",null,{"children":"Benchmark on your data"}]," —gains vary by domain"]}],"\n"]}]
4b:["$","p",null,{"children":"Conventional embedding sizes are conventions, not laws. Question them."}]
4c:["$","hr",null,{}]
4d:["$","p",null,{"children":["$","em",null,{"children":"Technical details in “Scaling Embedding Dimensions for Semantic Retrieval” (2022). Model weights available on Hugging Face."}]}]
b:[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","meta","2",{"name":"theme-color","media":"(prefers-color-scheme: dark)","content":"#0A0A0A"}],["$","meta","3",{"name":"theme-color","media":"(prefers-color-scheme: light)","content":"#fff"}]]
9:null
d:[["$","title","0",{"children":"Embedding Spaces at 7680 Dimensions — Zen LM Blog | Zen LM"}],["$","meta","1",{"name":"description","content":"Exploring high-dimensional embedding spaces for semantic search and retrieval."}],["$","meta","2",{"property":"og:title","content":"Zen LM - Open Foundation Models"}],["$","meta","3",{"property":"og:description","content":"Zen AI model family by Hanzo AI. 14 frontier models from 4B to 1T+ parameters for code, reasoning, vision, and multimodal tasks."}],["$","meta","4",{"property":"og:url","content":"https://zenlm.org"}],["$","meta","5",{"property":"og:site_name","content":"Zen LM"}],["$","meta","6",{"property":"og:type","content":"website"}],["$","meta","7",{"name":"twitter:card","content":"summary_large_image"}],["$","meta","8",{"name":"twitter:site","content":"@zenlmorg"}],["$","meta","9",{"name":"twitter:creator","content":"@zenlmorg"}],["$","meta","10",{"name":"twitter:title","content":"Zen LM - Open Foundation Models"}],["$","meta","11",{"name":"twitter:description","content":"Zen AI model family by Hanzo AI. 14 frontier models from 4B to 1T+ parameters for code, reasoning, vision, and multimodal tasks."}]]