mirror of
https://github.com/zenlm/zen-blog.git
synced 2026-11-03 14:11:45 +00:00
62 lines
30 KiB
Plaintext
62 lines
30 KiB
Plaintext
1:"$Sreact.fragment"
|
||
2:I[106,["/_next/static/chunks/59d0ad1b64f8544e.js"],"RootProvider"]
|
||
3:I[53113,["/_next/static/chunks/4d80e004cf4896dd.js","/_next/static/chunks/350ee4303b732916.js"],"default"]
|
||
4:I[73211,["/_next/static/chunks/4d80e004cf4896dd.js","/_next/static/chunks/350ee4303b732916.js"],"default"]
|
||
5:I[10086,["/_next/static/chunks/59d0ad1b64f8544e.js","/_next/static/chunks/8de849ca74fc071f.js","/_next/static/chunks/e62b91212ee7f8ff.js","/_next/static/chunks/f19fe44237e54646.js","/_next/static/chunks/cb0a883bafeb6805.js"],""]
|
||
7:I[89923,["/_next/static/chunks/4d80e004cf4896dd.js","/_next/static/chunks/350ee4303b732916.js"],"OutletBoundary"]
|
||
8:"$Sreact.suspense"
|
||
a:I[89923,["/_next/static/chunks/4d80e004cf4896dd.js","/_next/static/chunks/350ee4303b732916.js"],"ViewportBoundary"]
|
||
c:I[89923,["/_next/static/chunks/4d80e004cf4896dd.js","/_next/static/chunks/350ee4303b732916.js"],"MetadataBoundary"]
|
||
e:I[6998,["/_next/static/chunks/4d80e004cf4896dd.js","/_next/static/chunks/350ee4303b732916.js"],"default"]
|
||
:HL["/_next/static/chunks/f2332aac77592f9d.css","style"]
|
||
0:{"P":null,"b":"i-dnJM_MIpJSOCQWNJVMq","c":["","blog","7680-dim-embeddings"],"q":"","i":false,"f":[[["",{"children":["blog",{"children":[["slug","7680-dim-embeddings","d"],{"children":["__PAGE__",{}]}]}]},"$undefined","$undefined",true],[["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/_next/static/chunks/f2332aac77592f9d.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","script","script-0",{"src":"/_next/static/chunks/59d0ad1b64f8544e.js","async":true,"nonce":"$undefined"}]],["$","html",null,{"lang":"en","className":"dark","suppressHydrationWarning":true,"children":["$","body",null,{"className":"antialiased","children":["$","$L2",null,{"children":["$","$L3",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L4",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","main",null,{"className":"flex min-h-screen flex-col items-center justify-center px-4 text-center","children":[["$","div",null,{"className":"mb-8 opacity-20","children":["$","svg",null,{"width":"120","height":"120","viewBox":"0 0 120 120","fill":"none","aria-hidden":"true","children":["$","circle",null,{"cx":"60","cy":"60","r":"50","stroke":"currentColor","strokeWidth":"3","strokeLinecap":"round","strokeDasharray":"280 40"}]}]}],["$","p",null,{"className":"text-xs font-mono uppercase tracking-[0.3em] text-fd-muted-foreground mb-4","children":"404"}],["$","h1",null,{"className":"text-3xl font-semibold mb-3","children":"Page not found"}],["$","p",null,{"className":"text-fd-muted-foreground max-w-sm mb-10","children":"This page doesn't exist, or it may have moved. Try the documentation or head home."}],["$","div",null,{"className":"flex flex-wrap gap-3 justify-center","children":[["$","$L5",null,{"href":"/","className":"inline-flex items-center gap-2 rounded-lg bg-fd-primary text-fd-primary-foreground px-4 py-2 text-sm font-medium hover:opacity-90 transition","children":"Go home"}],["$","$L5",null,{"href":"/docs","className":"inline-flex items-center gap-2 rounded-lg border border-fd-border bg-fd-background px-4 py-2 text-sm font-medium hover:bg-fd-muted transition","children":"Documentation"}],["$","$L5",null,{"href":"/docs/models","className":"inline-flex items-center gap-2 rounded-lg border border-fd-border bg-fd-background px-4 py-2 text-sm font-medium hover:bg-fd-muted transition","children":"Browse models"}]]}],["$","p",null,{"className":"mt-16 text-xs font-mono text-fd-muted-foreground opacity-50","children":"zenlm.org"}]]}],[]],"forbidden":"$undefined","unauthorized":"$undefined"}]}]}]}]]}],{"children":[["$","$1","c",{"children":[null,["$","$L3",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L4",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","forbidden":"$undefined","unauthorized":"$undefined"}]]}],{"children":[["$","$1","c",{"children":[null,["$","$L3",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L4",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","forbidden":"$undefined","unauthorized":"$undefined"}]]}],{"children":[["$","$1","c",{"children":["$L6",[["$","script","script-0",{"src":"/_next/static/chunks/8de849ca74fc071f.js","async":true,"nonce":"$undefined"}],["$","script","script-1",{"src":"/_next/static/chunks/e62b91212ee7f8ff.js","async":true,"nonce":"$undefined"}],["$","script","script-2",{"src":"/_next/static/chunks/f19fe44237e54646.js","async":true,"nonce":"$undefined"}],["$","script","script-3",{"src":"/_next/static/chunks/cb0a883bafeb6805.js","async":true,"nonce":"$undefined"}]],["$","$L7",null,{"children":["$","$8",null,{"name":"Next.MetadataOutlet","children":"$@9"}]}]]}],{},null,false,false]},null,false,false]},null,false,false]},null,false,false],["$","$1","h",{"children":[null,["$","$La",null,{"children":"$Lb"}],["$","div",null,{"hidden":true,"children":["$","$Lc",null,{"children":["$","$8",null,{"name":"Next.Metadata","children":"$Ld"}]}]}],null]}],false]],"m":"$undefined","G":["$e",[]],"S":true}
|
||
6:["$","main",null,{"className":"mx-auto w-full max-w-2xl px-4 py-16","children":[["$","$L5",null,{"href":"/blog","className":"inline-flex items-center gap-1.5 text-sm text-fd-muted-foreground hover:text-fd-foreground mb-10 transition-colors","children":"← Back to Blog"}],["$","div",null,{"className":"mb-8","children":[["$","time",null,{"className":"text-xs font-mono text-fd-muted-foreground uppercase tracking-wider","children":"December 4, 2022"}],["$","h1",null,{"className":"text-3xl font-bold mt-2 mb-3","children":"7680-Dimensional Embeddings: More Dimensions, Better Retrieval"}],["$","p",null,{"className":"text-fd-muted-foreground text-lg mb-4","children":"Why we trained embedding models with 7680 dimensions and what we learned about the relationship between dimensionality and retrieval quality."}],["$","div",null,{"className":"flex items-center gap-3 pt-4 border-t border-fd-border","children":[["$","span",null,{"className":"text-sm text-fd-muted-foreground","children":["By ","Zach Kelling"]}],["$","div",null,{"className":"flex gap-1.5 ml-auto","children":[["$","span","Research",{"className":"text-[10px] font-mono uppercase tracking-wider bg-fd-muted text-fd-muted-foreground px-1.5 py-0.5 rounded","children":"Research"}],["$","span","Embeddings",{"className":"text-[10px] font-mono uppercase tracking-wider bg-fd-muted text-fd-muted-foreground px-1.5 py-0.5 rounded","children":"Embeddings"}],["$","span","Retrieval",{"className":"text-[10px] font-mono uppercase tracking-wider bg-fd-muted text-fd-muted-foreground px-1.5 py-0.5 rounded","children":"Retrieval"}]]}]]}]]}],["$","div",null,{"className":"prose dark:prose-invert max-w-none","children":[["$","p",null,{"children":"Embedding dimensions have standardized around powers of two: 768, 1536, occasionally 4096. We asked a simple question: what happens if we go bigger? The answer surprised us."}],"\n",["$","h2",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"background-why-dimensions-matter","children":[["$","a",null,{"data-card":"","href":"#background-why-dimensions-matter","className":"peer","children":"Background: Why Dimensions Matter"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}],"\n",["$","p",null,{"children":"Text embeddings map variable-length sequences to fixed-dimensional vectors. These vectors enable semantic similarity search, clustering, and retrieval. The dimension count determines the vector space’s capacity."}],"\n",["$","p",null,{"children":"Lower dimensions mean:"}],"\n",["$","ul",null,{"children":["\n",["$","li",null,{"children":"Smaller storage requirements"}],"\n",["$","li",null,{"children":"Faster similarity computations"}],"\n",["$","li",null,{"children":"Potential information loss"}],"\n"]}],"\n",["$","p",null,{"children":"Higher dimensions mean:"}],"\n",["$","ul",null,{"children":["\n",["$","li",null,{"children":"More expressive capacity"}],"\n",["$","li",null,{"children":"Larger memory footprint"}],"\n",["$","li",null,{"children":"Computational overhead"}],"\n"]}],"\n",["$","p",null,{"children":"The conventional wisdom holds that returns diminish quickly past 1024-2048 dimensions. Our experiments challenge this."}],"\n",["$","h2",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"experimental-setup","children":[["$","a",null,{"data-card":"","href":"#experimental-setup","className":"peer","children":"Experimental Setup"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":["$Lf","$L10","$undefined"]}]]}],"\n","$L11","\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n","$L12","\n","$L13","\n","$L14","\n","$L15","\n","$L16","\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n","$L17","\n","$L18","\n","$L19","\n","$L1a","\n","$L1b","\n","$L1c","\n","$L1d","\n","$L1e","\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n","$L1f","\n","$L20","\n","$L21","\n","$L22","\n","$L23","\n","$L24","\n","$L25","\n","$L26","\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n","$L27","\n","$L28","\n","$L29","\n","$L2a","\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n","$L2b","\n","$L2c","\n","$L2d","\n","$L2e","\n","$L2f","\n","$L30","\n","$L31","\n","$L32","\n","$L33","\n","$L34","\n","$L35","\n","$L36","\n","$L37","\n","$L38","\n","$L39","\n","$L3a","\n","$L3b"]}]]}]
|
||
f:["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}]
|
||
10:["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}]
|
||
11:["$","p",null,{"children":"We trained a series of embedding models with identical architectures except for output dimension:"}]
|
||
12:["$","div",null,{"className":"relative overflow-auto prose-no-margin my-6","children":["$","table",null,{"children":[["$","thead",null,{"children":["$","tr",null,{"children":[["$","th",null,{"children":"Model"}],["$","th",null,{"children":"Dimensions"}],["$","th",null,{"children":"Parameters"}]]}]}],["$","tbody",null,{"children":[["$","tr",null,{"children":[["$","td",null,{"children":"Zen-Embed-S"}],["$","td",null,{"children":"768"}],["$","td",null,{"children":"110M"}]]}],["$","tr",null,{"children":[["$","td",null,{"children":"Zen-Embed-M"}],["$","td",null,{"children":"1536"}],["$","td",null,{"children":"125M"}]]}],["$","tr",null,{"children":[["$","td",null,{"children":"Zen-Embed-L"}],["$","td",null,{"children":"3072"}],["$","td",null,{"children":"155M"}]]}],["$","tr",null,{"children":[["$","td",null,{"children":"Zen-Embed-XL"}],["$","td",null,{"children":"7680"}],["$","td",null,{"children":"230M"}]]}]]}]]}]}]
|
||
13:["$","p",null,{"children":"Training data: 1.2B text pairs with contrastive learning objective."}]
|
||
14:["$","h2",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"results","children":[["$","a",null,{"data-card":"","href":"#results","className":"peer","children":"Results"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
|
||
15:["$","h3",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"retrieval-benchmarks","children":[["$","a",null,{"data-card":"","href":"#retrieval-benchmarks","className":"peer","children":"Retrieval Benchmarks"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
|
||
16:["$","p",null,{"children":"BEIR (Benchmarking IR) results across 15 datasets:"}]
|
||
17:["$","div",null,{"className":"relative overflow-auto prose-no-margin my-6","children":["$","table",null,{"children":[["$","thead",null,{"children":["$","tr",null,{"children":[["$","th",null,{"children":"Model"}],["$","th",null,{"children":"NDCG@10"}],["$","th",null,{"children":"Recall@100"}],["$","th",null,{"children":"MRR"}]]}]}],["$","tbody",null,{"children":[["$","tr",null,{"children":[["$","td",null,{"children":"Zen-Embed-S"}],["$","td",null,{"children":"48.2"}],["$","td",null,{"children":"71.3"}],["$","td",null,{"children":"45.1"}]]}],["$","tr",null,{"children":[["$","td",null,{"children":"Zen-Embed-M"}],["$","td",null,{"children":"51.7"}],["$","td",null,{"children":"75.8"}],["$","td",null,{"children":"48.9"}]]}],["$","tr",null,{"children":[["$","td",null,{"children":"Zen-Embed-L"}],["$","td",null,{"children":"54.1"}],["$","td",null,{"children":"79.2"}],["$","td",null,{"children":"52.3"}]]}],["$","tr",null,{"children":[["$","td",null,{"children":"Zen-Embed-XL"}],["$","td",null,{"children":"57.3"}],["$","td",null,{"children":"83.6"}],["$","td",null,{"children":"55.8"}]]}]]}]]}]}]
|
||
18:["$","p",null,{"children":"The improvements continue well past conventional dimension counts."}]
|
||
19:["$","h3",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"scaling-analysis","children":[["$","a",null,{"data-card":"","href":"#scaling-analysis","className":"peer","children":"Scaling Analysis"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
|
||
1a:["$","p",null,{"children":"Plotting performance against log(dimensions) reveals near-linear scaling:"}]
|
||
1b:["$","p",null,{"children":["$","code",null,{"children":"\\text\\{NDCG@10\\} \\approx 0.12 \\cdot \\log_2(d) + 37.4"}]}]
|
||
1c:["$","p",null,{"children":"This suggests embedding capacity remains a bottleneck even at high dimensions."}]
|
||
1d:["$","h3",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"per-domain-breakdown","children":[["$","a",null,{"data-card":"","href":"#per-domain-breakdown","className":"peer","children":"Per-Domain Breakdown"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
|
||
1e:["$","p",null,{"children":"The benefits are not uniform across domains:"}]
|
||
1f:["$","div",null,{"className":"relative overflow-auto prose-no-margin my-6","children":["$","table",null,{"children":[["$","thead",null,{"children":["$","tr",null,{"children":[["$","th",null,{"children":"Domain"}],["$","th",null,{"children":"768d"}],["$","th",null,{"children":"7680d"}],["$","th",null,{"children":"Improvement"}]]}]}],["$","tbody",null,{"children":[["$","tr",null,{"children":[["$","td",null,{"children":"Scientific"}],["$","td",null,{"children":"42.1"}],["$","td",null,{"children":"54.7"}],["$","td",null,{"children":"+30%"}]]}],["$","tr",null,{"children":[["$","td",null,{"children":"Legal"}],["$","td",null,{"children":"38.9"}],["$","td",null,{"children":"51.2"}],["$","td",null,{"children":"+32%"}]]}],["$","tr",null,{"children":[["$","td",null,{"children":"Conversational"}],["$","td",null,{"children":"52.3"}],["$","td",null,{"children":"55.1"}],["$","td",null,{"children":"+5%"}]]}],["$","tr",null,{"children":[["$","td",null,{"children":"News"}],["$","td",null,{"children":"49.8"}],["$","td",null,{"children":"53.4"}],["$","td",null,{"children":"+7%"}]]}]]}]]}]}]
|
||
20:["$","p",null,{"children":"Technical and specialized domains benefit most. Everyday conversational content sees smaller gains."}]
|
||
21:["$","h3",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"interpretability","children":[["$","a",null,{"data-card":"","href":"#interpretability","className":"peer","children":"Interpretability"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
|
||
22:["$","p",null,{"children":"Higher dimensions don’t just improve metrics; they enable finer distinctions. Analysis of the 7680d space shows:"}]
|
||
23:["$","ul",null,{"children":["\n",["$","li",null,{"children":[["$","strong",null,{"children":"Cleaner clusters"}]," : Topic boundaries are sharper"]}],"\n",["$","li",null,{"children":[["$","strong",null,{"children":"Preserved nuance"}]," : Similar but distinct concepts remain separable"]}],"\n",["$","li",null,{"children":[["$","strong",null,{"children":"Hierarchical structure"}]," : Taxonomic relationships emerge naturally"]}],"\n"]}]
|
||
24:["$","h2",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"the-efficiency-question","children":[["$","a",null,{"data-card":"","href":"#the-efficiency-question","className":"peer","children":"The Efficiency Question"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
|
||
25:["$","p",null,{"children":"7680 dimensions cost more to store and search. Is it worth it?"}]
|
||
26:["$","h3",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"storage","children":[["$","a",null,{"data-card":"","href":"#storage","className":"peer","children":"Storage"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
|
||
27:["$","div",null,{"className":"relative overflow-auto prose-no-margin my-6","children":["$","table",null,{"children":[["$","thead",null,{"children":["$","tr",null,{"children":[["$","th",null,{"children":"Dimensions"}],["$","th",null,{"children":"Bytes per Vector"}],["$","th",null,{"children":"1M Vectors"}]]}]}],["$","tbody",null,{"children":[["$","tr",null,{"children":[["$","td",null,{"children":"768"}],["$","td",null,{"children":"3,072"}],["$","td",null,{"children":"2.9 GB"}]]}],["$","tr",null,{"children":[["$","td",null,{"children":"7680"}],["$","td",null,{"children":"30,720"}],["$","td",null,{"children":"29.3 GB"}]]}]]}]]}]}]
|
||
28:["$","p",null,{"children":"10x storage for higher dimensions. Significant but manageable with modern hardware."}]
|
||
29:["$","h3",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"search-latency","children":[["$","a",null,{"data-card":"","href":"#search-latency","className":"peer","children":"Search Latency"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
|
||
2a:["$","p",null,{"children":"Exact search scales linearly with dimensions. But approximate methods (HNSW, IVF) show sublinear scaling:"}]
|
||
2b:["$","div",null,{"className":"relative overflow-auto prose-no-margin my-6","children":["$","table",null,{"children":[["$","thead",null,{"children":["$","tr",null,{"children":[["$","th",null,{"children":"Dimensions"}],["$","th",null,{"children":"Exact (ms)"}],["$","th",null,{"children":"HNSW (ms)"}],["$","th",null,{"children":"IVF-PQ (ms)"}]]}]}],["$","tbody",null,{"children":[["$","tr",null,{"children":[["$","td",null,{"children":"768"}],["$","td",null,{"children":"12.3"}],["$","td",null,{"children":"0.8"}],["$","td",null,{"children":"0.3"}]]}],["$","tr",null,{"children":[["$","td",null,{"children":"7680"}],["$","td",null,{"children":"118.7"}],["$","td",null,{"children":"2.1"}],["$","td",null,{"children":"0.7"}]]}]]}]]}]}]
|
||
2c:["$","p",null,{"children":"With appropriate indexing, 7680d search remains practical."}]
|
||
2d:["$","h3",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"compression","children":[["$","a",null,{"data-card":"","href":"#compression","className":"peer","children":"Compression"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
|
||
2e:["$","p",null,{"children":"Quantization recovers much of the efficiency loss:"}]
|
||
2f:["$","ul",null,{"children":["\n",["$","li",null,{"children":[["$","strong",null,{"children":"INT8"}]," : 4x compression, <1% quality loss"]}],"\n",["$","li",null,{"children":[["$","strong",null,{"children":"Binary"}]," : 32x compression, 5% quality loss"]}],"\n",["$","li",null,{"children":[["$","strong",null,{"children":"Product Quantization"}]," : 16x compression, 2% quality loss"]}],"\n"]}]
|
||
30:["$","h2",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"practical-recommendations","children":[["$","a",null,{"data-card":"","href":"#practical-recommendations","className":"peer","children":"Practical Recommendations"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
|
||
31:["$","p",null,{"children":"Based on our experiments:"}]
|
||
32:["$","ol",null,{"children":["\n",["$","li",null,{"children":[["$","strong",null,{"children":"If retrieval quality matters most"}]," : Use 7680d with HNSW indexing"]}],"\n",["$","li",null,{"children":[["$","strong",null,{"children":"If storage is constrained"}]," : Use 7680d with INT8 quantization (still beats 768d float32)"]}],"\n",["$","li",null,{"children":[["$","strong",null,{"children":"For conversational applications"}]," : 1536d is sufficient"]}],"\n",["$","li",null,{"children":[["$","strong",null,{"children":"For technical/specialized domains"}]," : Higher dimensions provide outsized benefits"]}],"\n"]}]
|
||
33:["$","h2",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"release","children":[["$","a",null,{"data-card":"","href":"#release","className":"peer","children":"Release"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
|
||
34:["$","p",null,{"children":"We’re releasing the Zen-Embed family:"}]
|
||
35:["$","ul",null,{"children":["\n",["$","li",null,{"children":[["$","strong",null,{"children":"Zen-Embed-S"}]," (768d): Free, MIT license"]}],"\n",["$","li",null,{"children":[["$","strong",null,{"children":"Zen-Embed-M"}]," (1536d): Free, MIT license"]}],"\n",["$","li",null,{"children":[["$","strong",null,{"children":"Zen-Embed-L"}]," (3072d): Free, MIT license"]}],"\n",["$","li",null,{"children":[["$","strong",null,{"children":"Zen-Embed-XL"}]," (7680d): Free, MIT license"]}],"\n"]}]
|
||
36:["$","p",null,{"children":"All models available on Hugging Face: huggingface.co/zoo-labs"}]
|
||
37:["$","h2",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"what-this-means","children":[["$","a",null,{"data-card":"","href":"#what-this-means","className":"peer","children":"What This Means"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
|
||
38:["$","p",null,{"children":"The embedding dimension race isn’t over. There’s room to improve retrieval quality by increasing capacity. As hardware improves and indexing methods advance, higher-dimensional embeddings become increasingly practical."}]
|
||
39:["$","p",null,{"children":"More dimensions, better retrieval. Sometimes the simple approach works."}]
|
||
3a:["$","hr",null,{}]
|
||
3b:["$","p",null,{"children":["$","em",null,{"children":"Zach Kelling is a co-founder of Zoo Labs Foundation."}]}]
|
||
b:[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","meta","2",{"name":"theme-color","media":"(prefers-color-scheme: dark)","content":"#0A0A0A"}],["$","meta","3",{"name":"theme-color","media":"(prefers-color-scheme: light)","content":"#fff"}]]
|
||
9:null
|
||
d:[["$","title","0",{"children":"7680-Dimensional Embeddings: More Dimensions, Better Retrieval — Zen LM Blog | Zen LM"}],["$","meta","1",{"name":"description","content":"Why we trained embedding models with 7680 dimensions and what we learned about the relationship between dimensionality and retrieval quality."}],["$","meta","2",{"property":"og:title","content":"Zen LM - Open Foundation Models"}],["$","meta","3",{"property":"og:description","content":"Zen AI model family by Hanzo AI. 14 frontier models from 4B to 1T+ parameters for code, reasoning, vision, and multimodal tasks."}],["$","meta","4",{"property":"og:url","content":"https://zenlm.org"}],["$","meta","5",{"property":"og:site_name","content":"Zen LM"}],["$","meta","6",{"property":"og:type","content":"website"}],["$","meta","7",{"name":"twitter:card","content":"summary_large_image"}],["$","meta","8",{"name":"twitter:site","content":"@zenlmorg"}],["$","meta","9",{"name":"twitter:creator","content":"@zenlmorg"}],["$","meta","10",{"name":"twitter:title","content":"Zen LM - Open Foundation Models"}],["$","meta","11",{"name":"twitter:description","content":"Zen AI model family by Hanzo AI. 14 frontier models from 4B to 1T+ parameters for code, reasoning, vision, and multimodal tasks."}]]
|