mirror of
https://github.com/zenlm/zen-blog.git
synced 2026-11-03 14:11:45 +00:00
82 lines
37 KiB
Plaintext
82 lines
37 KiB
Plaintext
1:"$Sreact.fragment"
|
||
2:I[106,["/_next/static/chunks/59d0ad1b64f8544e.js"],"RootProvider"]
|
||
3:I[53113,["/_next/static/chunks/4d80e004cf4896dd.js","/_next/static/chunks/350ee4303b732916.js"],"default"]
|
||
4:I[73211,["/_next/static/chunks/4d80e004cf4896dd.js","/_next/static/chunks/350ee4303b732916.js"],"default"]
|
||
5:I[10086,["/_next/static/chunks/59d0ad1b64f8544e.js","/_next/static/chunks/8de849ca74fc071f.js","/_next/static/chunks/e62b91212ee7f8ff.js","/_next/static/chunks/f19fe44237e54646.js","/_next/static/chunks/cb0a883bafeb6805.js"],""]
|
||
7:I[89923,["/_next/static/chunks/4d80e004cf4896dd.js","/_next/static/chunks/350ee4303b732916.js"],"OutletBoundary"]
|
||
8:"$Sreact.suspense"
|
||
a:I[89923,["/_next/static/chunks/4d80e004cf4896dd.js","/_next/static/chunks/350ee4303b732916.js"],"ViewportBoundary"]
|
||
c:I[89923,["/_next/static/chunks/4d80e004cf4896dd.js","/_next/static/chunks/350ee4303b732916.js"],"MetadataBoundary"]
|
||
e:I[6998,["/_next/static/chunks/4d80e004cf4896dd.js","/_next/static/chunks/350ee4303b732916.js"],"default"]
|
||
:HL["/_next/static/chunks/f2332aac77592f9d.css","style"]
|
||
0:{"P":null,"b":"i-dnJM_MIpJSOCQWNJVMq","c":["","blog","embedding-spaces-7680-dimensions"],"q":"","i":false,"f":[[["",{"children":["blog",{"children":[["slug","embedding-spaces-7680-dimensions","d"],{"children":["__PAGE__",{}]}]}]},"$undefined","$undefined",true],[["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/_next/static/chunks/f2332aac77592f9d.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","script","script-0",{"src":"/_next/static/chunks/59d0ad1b64f8544e.js","async":true,"nonce":"$undefined"}]],["$","html",null,{"lang":"en","className":"dark","suppressHydrationWarning":true,"children":["$","body",null,{"className":"antialiased","children":["$","$L2",null,{"children":["$","$L3",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L4",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","main",null,{"className":"flex min-h-screen flex-col items-center justify-center px-4 text-center","children":[["$","div",null,{"className":"mb-8 opacity-20","children":["$","svg",null,{"width":"120","height":"120","viewBox":"0 0 120 120","fill":"none","aria-hidden":"true","children":["$","circle",null,{"cx":"60","cy":"60","r":"50","stroke":"currentColor","strokeWidth":"3","strokeLinecap":"round","strokeDasharray":"280 40"}]}]}],["$","p",null,{"className":"text-xs font-mono uppercase tracking-[0.3em] text-fd-muted-foreground mb-4","children":"404"}],["$","h1",null,{"className":"text-3xl font-semibold mb-3","children":"Page not found"}],["$","p",null,{"className":"text-fd-muted-foreground max-w-sm mb-10","children":"This page doesn't exist, or it may have moved. Try the documentation or head home."}],["$","div",null,{"className":"flex flex-wrap gap-3 justify-center","children":[["$","$L5",null,{"href":"/","className":"inline-flex items-center gap-2 rounded-lg bg-fd-primary text-fd-primary-foreground px-4 py-2 text-sm font-medium hover:opacity-90 transition","children":"Go home"}],["$","$L5",null,{"href":"/docs","className":"inline-flex items-center gap-2 rounded-lg border border-fd-border bg-fd-background px-4 py-2 text-sm font-medium hover:bg-fd-muted transition","children":"Documentation"}],["$","$L5",null,{"href":"/docs/models","className":"inline-flex items-center gap-2 rounded-lg border border-fd-border bg-fd-background px-4 py-2 text-sm font-medium hover:bg-fd-muted transition","children":"Browse models"}]]}],["$","p",null,{"className":"mt-16 text-xs font-mono text-fd-muted-foreground opacity-50","children":"zenlm.org"}]]}],[]],"forbidden":"$undefined","unauthorized":"$undefined"}]}]}]}]]}],{"children":[["$","$1","c",{"children":[null,["$","$L3",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L4",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","forbidden":"$undefined","unauthorized":"$undefined"}]]}],{"children":[["$","$1","c",{"children":[null,["$","$L3",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L4",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","forbidden":"$undefined","unauthorized":"$undefined"}]]}],{"children":[["$","$1","c",{"children":["$L6",[["$","script","script-0",{"src":"/_next/static/chunks/8de849ca74fc071f.js","async":true,"nonce":"$undefined"}],["$","script","script-1",{"src":"/_next/static/chunks/e62b91212ee7f8ff.js","async":true,"nonce":"$undefined"}],["$","script","script-2",{"src":"/_next/static/chunks/f19fe44237e54646.js","async":true,"nonce":"$undefined"}],["$","script","script-3",{"src":"/_next/static/chunks/cb0a883bafeb6805.js","async":true,"nonce":"$undefined"}]],["$","$L7",null,{"children":["$","$8",null,{"name":"Next.MetadataOutlet","children":"$@9"}]}]]}],{},null,false,false]},null,false,false]},null,false,false]},null,false,false],["$","$1","h",{"children":[null,["$","$La",null,{"children":"$Lb"}],["$","div",null,{"hidden":true,"children":["$","$Lc",null,{"children":["$","$8",null,{"name":"Next.Metadata","children":"$Ld"}]}]}],null]}],false]],"m":"$undefined","G":["$e",[]],"S":true}
|
||
6:["$","main",null,{"className":"mx-auto w-full max-w-2xl px-4 py-16","children":[["$","$L5",null,{"href":"/blog","className":"inline-flex items-center gap-1.5 text-sm text-fd-muted-foreground hover:text-fd-foreground mb-10 transition-colors","children":"← Back to Blog"}],["$","div",null,{"className":"mb-8","children":[["$","time",null,{"className":"text-xs font-mono text-fd-muted-foreground uppercase tracking-wider","children":"December 4, 2022"}],["$","h1",null,{"className":"text-3xl font-bold mt-2 mb-3","children":"Embedding Spaces at 7680 Dimensions"}],["$","p",null,{"className":"text-fd-muted-foreground text-lg mb-4","children":"Exploring high-dimensional embedding spaces for semantic search and retrieval."}],["$","div",null,{"className":"flex items-center gap-3 pt-4 border-t border-fd-border","children":[["$","span",null,{"className":"text-sm text-fd-muted-foreground","children":["By ","Zach Kelling"]}],["$","div",null,{"className":"flex gap-1.5 ml-auto","children":[["$","span","Research",{"className":"text-[10px] font-mono uppercase tracking-wider bg-fd-muted text-fd-muted-foreground px-1.5 py-0.5 rounded","children":"Research"}],["$","span","Embeddings",{"className":"text-[10px] font-mono uppercase tracking-wider bg-fd-muted text-fd-muted-foreground px-1.5 py-0.5 rounded","children":"Embeddings"}],["$","span","Retrieval",{"className":"text-[10px] font-mono uppercase tracking-wider bg-fd-muted text-fd-muted-foreground px-1.5 py-0.5 rounded","children":"Retrieval"}]]}]]}]]}],["$","div",null,{"className":"prose dark:prose-invert max-w-none","children":[["$","h1",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"the-dimension-question","children":[["$","a",null,{"data-card":"","href":"#the-dimension-question","className":"peer","children":"The Dimension Question"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}],"\n",["$","p",null,{"children":"How many dimensions does a text embedding need?"}],"\n",["$","p",null,{"children":"The field has settled on conventions: 768 for BERT-scale models, 1536 for OpenAI’s ada-002, 4096 for some recent models. But these choices reflect architectural constraints, not fundamental requirements."}],"\n",["$","p",null,{"children":"We investigate what happens when we scale embedding dimensions to 7680—ten times the BERT baseline."}],"\n",["$","h2",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"why-higher-dimensions","children":[["$","a",null,{"data-card":"","href":"#why-higher-dimensions","className":"peer","children":"Why Higher Dimensions?"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}],"\n",["$","h3",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"capacity-arguments","children":[["$","a",null,{"data-card":"","href":"#capacity-arguments","className":"peer","children":"Capacity Arguments"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":["$Lf","$L10","$undefined"]}]]}],"\n","$L11","\n","$L12","\n","$L13","\n","$L14","\n","$L15","\n","$L16","\n","$L17","\n","$L18","\n","$L19","\n","$L1a","\n","$L1b","\n","$L1c","\n","$L1d","\n","$L1e","\n","$L1f","\n","$L20","\n","$L21","\n","$L22","\n","$L23","\n","$L24","\n","$L25","\n","$L26","\n","$L27","\n","$L28","\n","$L29","\n","$L2a","\n","$L2b","\n","$L2c","\n","$L2d","\n","$L2e","\n","$L2f","\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n","$L30","\n","$L31","\n","$L32","\n","$L33","\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n","$L34","\n","$L35","\n","$L36","\n","$L37","\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n","$L38","\n","$L39","\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n","$L3a","\n","$L3b","\n","$L3c","\n","$L3d","\n","$L3e","\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n","$L3f","\n","$L40","\n","$L41","\n","$L42","\n","$L43","\n","$L44","\n","$L45","\n","$L46","\n","$L47","\n","$L48","\n","$L49","\n","$L4a","\n","$L4b","\n","$L4c","\n","$L4d"]}]]}]
|
||
4e:I[51504,["/_next/static/chunks/59d0ad1b64f8544e.js","/_next/static/chunks/8de849ca74fc071f.js","/_next/static/chunks/e62b91212ee7f8ff.js","/_next/static/chunks/f19fe44237e54646.js","/_next/static/chunks/cb0a883bafeb6805.js"],"CodeBlock"]
|
||
4f:I[51504,["/_next/static/chunks/59d0ad1b64f8544e.js","/_next/static/chunks/8de849ca74fc071f.js","/_next/static/chunks/e62b91212ee7f8ff.js","/_next/static/chunks/f19fe44237e54646.js","/_next/static/chunks/cb0a883bafeb6805.js"],"Pre"]
|
||
f:["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}]
|
||
10:["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}]
|
||
11:["$","p",null,{"children":"A $d$-dimensional embedding space can represent $\\mathcal{O}(e^d)$ nearly-orthogonal vectors. For semantic search, we want documents with different meanings to map to different regions. Higher dimensions provide more room."}]
|
||
12:["$","h3",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"information-theoretic-view","children":[["$","a",null,{"data-card":"","href":"#information-theoretic-view","className":"peer","children":"Information-Theoretic View"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
|
||
13:["$","p",null,{"children":"Consider a corpus of $N$ documents, each with entropy $H$ bits. A $d$-dimensional float32 embedding stores $32d$ bits. For lossless encoding:"}]
|
||
14:["$","p",null,{"children":["$","code",null,{"children":"32d \\geq N \\cdot H"}]}]
|
||
15:["$","p",null,{"children":"For a corpus of 1 billion documents with 100 bits of effective information each, we need:"}]
|
||
16:["$","p",null,{"children":["$","code",null,{"children":"d \\geq \\frac\\{10^9 \\cdot 100\\}\\{32\\} \\approx 3 \\times 10^9"}]}]
|
||
17:["$","p",null,{"children":"Clearly, embeddings are lossy compression. But higher dimensions reduce the loss."}]
|
||
18:["$","h3",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"empirical-observations","children":[["$","a",null,{"data-card":"","href":"#empirical-observations","className":"peer","children":"Empirical Observations"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
|
||
19:["$","p",null,{"children":"Retrieval quality on our internal benchmarks plateaued around:"}]
|
||
1a:["$","ul",null,{"children":["\n",["$","li",null,{"children":"768 dimensions: 71% recall@10"}],"\n",["$","li",null,{"children":"1536 dimensions: 78% recall@10"}],"\n",["$","li",null,{"children":"3072 dimensions: 83% recall@10"}],"\n",["$","li",null,{"children":"7680 dimensions: 87% recall@10"}],"\n"]}]
|
||
1b:["$","p",null,{"children":"Diminishing returns set in, but gains persist well beyond conventional wisdom."}]
|
||
1c:["$","h2",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"training-high-dimensional-embeddings","children":[["$","a",null,{"data-card":"","href":"#training-high-dimensional-embeddings","className":"peer","children":"Training High-Dimensional Embeddings"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
|
||
1d:["$","h3",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"architecture","children":[["$","a",null,{"data-card":"","href":"#architecture","className":"peer","children":"Architecture"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
|
||
1e:["$","p",null,{"children":"We use a standard transformer encoder with a projection head:"}]
|
||
1f:["$","$L4e",null,{"className":"shiki shiki-themes github-light github-dark","style":{"--shiki-light":"#24292e","--shiki-dark":"#e1e4e8","--shiki-light-bg":"#fff","--shiki-dark-bg":"#24292e"},"tabIndex":"0","icon":"<svg viewBox=\"0 0 24 24\"><path d=\"M 6,1 C 4.354992,1 3,2.354992 3,4 v 16 c 0,1.645008 1.354992,3 3,3 h 12 c 1.645008,0 3,-1.354992 3,-3 V 8 7 A 1.0001,1.0001 0 0 0 20.707031,6.2929687 l -5,-5 A 1.0001,1.0001 0 0 0 15,1 h -1 z m 0,2 h 7 v 3 c 0,1.645008 1.354992,3 3,3 h 3 v 11 c 0,0.564129 -0.435871,1 -1,1 H 6 C 5.4358712,21 5,20.564129 5,20 V 4 C 5,3.4358712 5.4358712,3 6,3 Z M 15,3.4140625 18.585937,7 H 16 C 15.435871,7 15,6.5641288 15,6 Z\" fill=\"currentColor\" /></svg>","children":["$","$L4f",null,{"children":["$","code",null,{"children":[["$","span",null,{"className":"line","children":["$","span",null,{}]}],"\n",["$","span",null,{"className":"line","children":["$","span",null,{"children":" Input --> Transformer(L=12, H=768) --> Pool --> Linear(768, 7680) --> Normalize"}]}],"\n",["$","span",null,{"className":"line","children":["$","span",null,{"children":" "}]}]]}]}]}]
|
||
20:["$","p",null,{"children":"The projection head maps from the transformer’s hidden dimension to the embedding space. This decouples representation capacity from compute requirements."}]
|
||
21:["$","h3",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"contrastive-learning","children":[["$","a",null,{"data-card":"","href":"#contrastive-learning","className":"peer","children":"Contrastive Learning"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
|
||
22:["$","p",null,{"children":"We train with InfoNCE loss over batches of (query, positive, negatives):"}]
|
||
23:["$","p",null,{"children":["$","code",null,{"children":"\\mathcal\\{L\\} = -\\log \\frac\\{\\exp(q \\cdot p^+ / \\tau)\\}\\{\\sum_\\{i\\} \\exp(q \\cdot p_i / \\tau)\\}"}]}]
|
||
24:["$","p",null,{"children":"With temperature $\\tau = 0.01$ for high-dimensional spaces (lower than typical)."}]
|
||
25:["$","h3",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"hard-negative-mining","children":[["$","a",null,{"data-card":"","href":"#hard-negative-mining","className":"peer","children":"Hard Negative Mining"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
|
||
26:["$","p",null,{"children":"High-dimensional spaces require harder negatives to provide gradient signal. Our mining strategy:"}]
|
||
27:["$","ol",null,{"children":["\n",["$","li",null,{"children":"Retrieve top-100 candidates via approximate nearest neighbor"}],"\n",["$","li",null,{"children":"Filter to exclude true positives"}],"\n",["$","li",null,{"children":"Sample negatives weighted by similarity (harder = more likely)"}],"\n"]}]
|
||
28:["$","p",null,{"children":"This curriculum focuses training on the decision boundary."}]
|
||
29:["$","h2",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"practical-considerations","children":[["$","a",null,{"data-card":"","href":"#practical-considerations","className":"peer","children":"Practical Considerations"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
|
||
2a:["$","h3",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"storage","children":[["$","a",null,{"data-card":"","href":"#storage","className":"peer","children":"Storage"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
|
||
2b:["$","p",null,{"children":"7680-dimensional float32 embeddings require 30KB per vector. For 1 billion documents:"}]
|
||
2c:["$","p",null,{"children":["$","code",null,{"children":"10^9 \\times 30\\text\\{KB\\} = 30\\text\\{TB\\}"}]}]
|
||
2d:["$","p",null,{"children":"This is substantial but manageable with modern storage."}]
|
||
2e:["$","h3",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"quantization","children":[["$","a",null,{"data-card":"","href":"#quantization","className":"peer","children":"Quantization"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
|
||
2f:["$","p",null,{"children":"We can reduce storage through quantization:"}]
|
||
30:["$","div",null,{"className":"relative overflow-auto prose-no-margin my-6","children":["$","table",null,{"children":[["$","thead",null,{"children":["$","tr",null,{"children":[["$","th",null,{"children":"Precision"}],["$","th",null,{"children":"Bytes/Vector"}],["$","th",null,{"children":"Recall@10"}]]}]}],["$","tbody",null,{"children":[["$","tr",null,{"children":[["$","td",null,{"children":"float32"}],["$","td",null,{"children":"30,720"}],["$","td",null,{"children":"87.3%"}]]}],["$","tr",null,{"children":[["$","td",null,{"children":"float16"}],["$","td",null,{"children":"15,360"}],["$","td",null,{"children":"87.1%"}]]}],["$","tr",null,{"children":[["$","td",null,{"children":"int8"}],["$","td",null,{"children":"7,680"}],["$","td",null,{"children":"85.9%"}]]}],["$","tr",null,{"children":[["$","td",null,{"children":"binary"}],["$","td",null,{"children":"960"}],["$","td",null,{"children":"78.2%"}]]}]]}]]}]}]
|
||
31:["$","p",null,{"children":"int8 quantization provides 4x compression with minimal quality loss."}]
|
||
32:["$","h3",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"approximate-search","children":[["$","a",null,{"data-card":"","href":"#approximate-search","className":"peer","children":"Approximate Search"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
|
||
33:["$","p",null,{"children":"Exact nearest neighbor search in 7680 dimensions is expensive. We use hierarchical navigable small world (HNSW) graphs:"}]
|
||
34:["$","div",null,{"className":"relative overflow-auto prose-no-margin my-6","children":["$","table",null,{"children":[["$","thead",null,{"children":["$","tr",null,{"children":[["$","th",null,{"children":"Dimensions"}],["$","th",null,{"children":"Build Time"}],["$","th",null,{"children":"Query Time"}],["$","th",null,{"children":"Recall@10"}]]}]}],["$","tbody",null,{"children":[["$","tr",null,{"children":[["$","td",null,{"children":"768"}],["$","td",null,{"children":"1x"}],["$","td",null,{"children":"1x"}],["$","td",null,{"children":"99.2%"}]]}],["$","tr",null,{"children":[["$","td",null,{"children":"7680"}],["$","td",null,{"children":"3.2x"}],["$","td",null,{"children":"2.8x"}],["$","td",null,{"children":"98.7%"}]]}]]}]]}]}]
|
||
35:["$","p",null,{"children":"The overhead is sublinear in dimensionality due to efficient distance computations."}]
|
||
36:["$","h2",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"benchmark-results","children":[["$","a",null,{"data-card":"","href":"#benchmark-results","className":"peer","children":"Benchmark Results"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
|
||
37:["$","h3",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"ms-marco-passage-retrieval","children":[["$","a",null,{"data-card":"","href":"#ms-marco-passage-retrieval","className":"peer","children":"MS MARCO Passage Retrieval"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
|
||
38:["$","div",null,{"className":"relative overflow-auto prose-no-margin my-6","children":["$","table",null,{"children":[["$","thead",null,{"children":["$","tr",null,{"children":[["$","th",null,{"children":"Model"}],["$","th",null,{"children":"Dimensions"}],["$","th",null,{"children":"MRR@10"}],["$","th",null,{"children":"Recall@100"}]]}]}],["$","tbody",null,{"children":[["$","tr",null,{"children":[["$","td",null,{"children":"BM25"}],["$","td",null,{"children":"-"}],["$","td",null,{"children":"18.4"}],["$","td",null,{"children":"66.5"}]]}],["$","tr",null,{"children":[["$","td",null,{"children":"DPR"}],["$","td",null,{"children":"768"}],["$","td",null,{"children":"31.1"}],["$","td",null,{"children":"82.4"}]]}],["$","tr",null,{"children":[["$","td",null,{"children":"Contriever"}],["$","td",null,{"children":"768"}],["$","td",null,{"children":"32.8"}],["$","td",null,{"children":"84.1"}]]}],["$","tr",null,{"children":[["$","td",null,{"children":"Zen-Embed"}],["$","td",null,{"children":"7680"}],["$","td",null,{"children":"38.6"}],["$","td",null,{"children":"91.3"}]]}]]}]]}]}]
|
||
39:["$","h3",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"natural-questions","children":[["$","a",null,{"data-card":"","href":"#natural-questions","className":"peer","children":"Natural Questions"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
|
||
3a:["$","div",null,{"className":"relative overflow-auto prose-no-margin my-6","children":["$","table",null,{"children":[["$","thead",null,{"children":["$","tr",null,{"children":[["$","th",null,{"children":"Model"}],["$","th",null,{"children":"Dimensions"}],["$","th",null,{"children":"Top-20 Acc"}],["$","th",null,{"children":"Top-100 Acc"}]]}]}],["$","tbody",null,{"children":[["$","tr",null,{"children":[["$","td",null,{"children":"DPR"}],["$","td",null,{"children":"768"}],["$","td",null,{"children":"78.4"}],["$","td",null,{"children":"85.4"}]]}],["$","tr",null,{"children":[["$","td",null,{"children":"Contriever"}],["$","td",null,{"children":"768"}],["$","td",null,{"children":"81.3"}],["$","td",null,{"children":"88.1"}]]}],["$","tr",null,{"children":[["$","td",null,{"children":"Zen-Embed"}],["$","td",null,{"children":"7680"}],["$","td",null,{"children":"86.7"}],["$","td",null,{"children":"93.2"}]]}]]}]]}]}]
|
||
3b:["$","p",null,{"children":"High-dimensional embeddings provide substantial gains on retrieval benchmarks."}]
|
||
3c:["$","h2",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"analysis","children":[["$","a",null,{"data-card":"","href":"#analysis","className":"peer","children":"Analysis"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
|
||
3d:["$","h3",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"what-do-extra-dimensions-encode","children":[["$","a",null,{"data-card":"","href":"#what-do-extra-dimensions-encode","className":"peer","children":"What Do Extra Dimensions Encode?"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
|
||
3e:["$","p",null,{"children":"We analyze the learned embedding space through probing tasks:"}]
|
||
3f:["$","div",null,{"className":"relative overflow-auto prose-no-margin my-6","children":["$","table",null,{"children":[["$","thead",null,{"children":["$","tr",null,{"children":[["$","th",null,{"children":"Property"}],["$","th",null,{"children":"768d Probe Acc"}],["$","th",null,{"children":"7680d Probe Acc"}]]}]}],["$","tbody",null,{"children":[["$","tr",null,{"children":[["$","td",null,{"children":"Topic"}],["$","td",null,{"children":"84.2%"}],["$","td",null,{"children":"86.1%"}]]}],["$","tr",null,{"children":[["$","td",null,{"children":"Sentiment"}],["$","td",null,{"children":"91.3%"}],["$","td",null,{"children":"92.8%"}]]}],["$","tr",null,{"children":[["$","td",null,{"children":"Entity"}],["$","td",null,{"children":"67.4%"}],["$","td",null,{"children":"78.9%"}]]}],["$","tr",null,{"children":[["$","td",null,{"children":"Relation"}],["$","td",null,{"children":"52.1%"}],["$","td",null,{"children":"71.3%"}]]}]]}]]}]}]
|
||
40:["$","p",null,{"children":"The largest gains are in entity and relation encoding—fine-grained semantic properties that require more capacity."}]
|
||
41:["$","h3",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"nearest-neighbor-analysis","children":[["$","a",null,{"data-card":"","href":"#nearest-neighbor-analysis","className":"peer","children":"Nearest Neighbor Analysis"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
|
||
42:["$","p",null,{"children":"For the query “What causes inflation?”, nearest neighbors at different dimensions:"}]
|
||
43:["$","p",null,{"children":["$","strong",null,{"children":"768 dimensions:"}]}]
|
||
44:["$","ol",null,{"children":["\n",["$","li",null,{"children":"What is inflation? (similar query)"}],"\n",["$","li",null,{"children":"How does inflation work? (similar query)"}],"\n",["$","li",null,{"children":"Inflation rates by country (tangential)"}],"\n"]}]
|
||
45:["$","p",null,{"children":["$","strong",null,{"children":"7680 dimensions:"}]}]
|
||
46:["$","ol",null,{"children":["\n",["$","li",null,{"children":"Inflation is caused by… (direct answer)"}],"\n",["$","li",null,{"children":"The primary drivers of inflation include… (direct answer)"}],"\n",["$","li",null,{"children":"Central bank policies affect inflation through… (relevant detail)"}],"\n"]}]
|
||
47:["$","p",null,{"children":"Higher dimensions better distinguish queries from answers."}]
|
||
48:["$","h2",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"recommendations","children":[["$","a",null,{"data-card":"","href":"#recommendations","className":"peer","children":"Recommendations"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
|
||
49:["$","p",null,{"children":"Based on our experiments:"}]
|
||
4a:["$","ol",null,{"children":["\n",["$","li",null,{"children":[["$","strong",null,{"children":"Try 3072+ dimensions"}]," if retrieval quality matters and storage is available"]}],"\n",["$","li",null,{"children":[["$","strong",null,{"children":"Use int8 quantization"}]," for production deployments"]}],"\n",["$","li",null,{"children":[["$","strong",null,{"children":"Invest in hard negative mining"}]," to realize the benefits of capacity"]}],"\n",["$","li",null,{"children":[["$","strong",null,{"children":"Benchmark on your data"}]," —gains vary by domain"]}],"\n"]}]
|
||
4b:["$","p",null,{"children":"Conventional embedding sizes are conventions, not laws. Question them."}]
|
||
4c:["$","hr",null,{}]
|
||
4d:["$","p",null,{"children":["$","em",null,{"children":"Technical details in “Scaling Embedding Dimensions for Semantic Retrieval” (2022). Model weights available on Hugging Face."}]}]
|
||
b:[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","meta","2",{"name":"theme-color","media":"(prefers-color-scheme: dark)","content":"#0A0A0A"}],["$","meta","3",{"name":"theme-color","media":"(prefers-color-scheme: light)","content":"#fff"}]]
|
||
9:null
|
||
d:[["$","title","0",{"children":"Embedding Spaces at 7680 Dimensions — Zen LM Blog | Zen LM"}],["$","meta","1",{"name":"description","content":"Exploring high-dimensional embedding spaces for semantic search and retrieval."}],["$","meta","2",{"property":"og:title","content":"Zen LM - Open Foundation Models"}],["$","meta","3",{"property":"og:description","content":"Zen AI model family by Hanzo AI. 14 frontier models from 4B to 1T+ parameters for code, reasoning, vision, and multimodal tasks."}],["$","meta","4",{"property":"og:url","content":"https://zenlm.org"}],["$","meta","5",{"property":"og:site_name","content":"Zen LM"}],["$","meta","6",{"property":"og:type","content":"website"}],["$","meta","7",{"name":"twitter:card","content":"summary_large_image"}],["$","meta","8",{"name":"twitter:site","content":"@zenlmorg"}],["$","meta","9",{"name":"twitter:creator","content":"@zenlmorg"}],["$","meta","10",{"name":"twitter:title","content":"Zen LM - Open Foundation Models"}],["$","meta","11",{"name":"twitter:description","content":"Zen AI model family by Hanzo AI. 14 frontier models from 4B to 1T+ parameters for code, reasoning, vision, and multimodal tasks."}]]
|