mirror of
https://github.com/zenlm/zen-blog.git
synced 2026-11-03 14:11:45 +00:00
77 lines
31 KiB
Plaintext
77 lines
31 KiB
Plaintext
1:"$Sreact.fragment"
|
||
2:I[10086,["/_next/static/chunks/59d0ad1b64f8544e.js","/_next/static/chunks/8de849ca74fc071f.js","/_next/static/chunks/e62b91212ee7f8ff.js","/_next/static/chunks/f19fe44237e54646.js","/_next/static/chunks/cb0a883bafeb6805.js"],""]
|
||
47:I[51504,["/_next/static/chunks/59d0ad1b64f8544e.js","/_next/static/chunks/8de849ca74fc071f.js","/_next/static/chunks/e62b91212ee7f8ff.js","/_next/static/chunks/f19fe44237e54646.js","/_next/static/chunks/cb0a883bafeb6805.js"],"CodeBlock"]
|
||
48:I[51504,["/_next/static/chunks/59d0ad1b64f8544e.js","/_next/static/chunks/8de849ca74fc071f.js","/_next/static/chunks/e62b91212ee7f8ff.js","/_next/static/chunks/f19fe44237e54646.js","/_next/static/chunks/cb0a883bafeb6805.js"],"Pre"]
|
||
49:I[89923,["/_next/static/chunks/4d80e004cf4896dd.js","/_next/static/chunks/350ee4303b732916.js"],"OutletBoundary"]
|
||
4a:"$Sreact.suspense"
|
||
0:{"buildId":"i-dnJM_MIpJSOCQWNJVMq","rsc":["$","$1","c",{"children":[["$","main",null,{"className":"mx-auto w-full max-w-2xl px-4 py-16","children":[["$","$L2",null,{"href":"/blog","className":"inline-flex items-center gap-1.5 text-sm text-fd-muted-foreground hover:text-fd-foreground mb-10 transition-colors","children":"← Back to Blog"}],["$","div",null,{"className":"mb-8","children":[["$","time",null,{"className":"text-xs font-mono text-fd-muted-foreground uppercase tracking-wider","children":"December 4, 2022"}],["$","h1",null,{"className":"text-3xl font-bold mt-2 mb-3","children":"Embedding Spaces at 7680 Dimensions"}],["$","p",null,{"className":"text-fd-muted-foreground text-lg mb-4","children":"Exploring high-dimensional embedding spaces for semantic search and retrieval."}],["$","div",null,{"className":"flex items-center gap-3 pt-4 border-t border-fd-border","children":[["$","span",null,{"className":"text-sm text-fd-muted-foreground","children":["By ","Zach Kelling"]}],["$","div",null,{"className":"flex gap-1.5 ml-auto","children":[["$","span","Research",{"className":"text-[10px] font-mono uppercase tracking-wider bg-fd-muted text-fd-muted-foreground px-1.5 py-0.5 rounded","children":"Research"}],["$","span","Embeddings",{"className":"text-[10px] font-mono uppercase tracking-wider bg-fd-muted text-fd-muted-foreground px-1.5 py-0.5 rounded","children":"Embeddings"}],["$","span","Retrieval",{"className":"text-[10px] font-mono uppercase tracking-wider bg-fd-muted text-fd-muted-foreground px-1.5 py-0.5 rounded","children":"Retrieval"}]]}]]}]]}],["$","div",null,{"className":"prose dark:prose-invert max-w-none","children":[["$","h1",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"the-dimension-question","children":[["$","a",null,{"data-card":"","href":"#the-dimension-question","className":"peer","children":"The Dimension Question"}],["$","svg",null,{"xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}],"\n",["$","p",null,{"children":"How many dimensions does a text embedding need?"}],"\n",["$","p",null,{"children":"The field has settled on conventions: 768 for BERT-scale models, 1536 for OpenAI’s ada-002, 4096 for some recent models. But these choices reflect architectural constraints, not fundamental requirements."}],"\n",["$","p",null,{"children":"We investigate what happens when we scale embedding dimensions to 7680—ten times the BERT baseline."}],"\n",["$","h2",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"why-higher-dimensions","children":[["$","a",null,{"data-card":"","href":"#why-higher-dimensions","className":"peer","children":"Why Higher Dimensions?"}],["$","svg",null,{"xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}],"\n",["$","h3",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"capacity-arguments","children":[["$","a",null,{"data-card":"","href":"#capacity-arguments","className":"peer","children":"Capacity Arguments"}],["$","svg",null,{"xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":["$L3","$L4","$undefined"]}]]}],"\n","$L5","\n","$L6","\n","$L7","\n","$L8","\n","$L9","\n","$La","\n","$Lb","\n","$Lc","\n","$Ld","\n","$Le","\n","$Lf","\n","$L10","\n","$L11","\n","$L12","\n","$L13","\n","$L14","\n","$L15","\n","$L16","\n","$L17","\n","$L18","\n","$L19","\n","$L1a","\n","$L1b","\n","$L1c","\n","$L1d","\n","$L1e","\n","$L1f","\n","$L20","\n","$L21","\n","$L22","\n","$L23","\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n","$L24","\n","$L25","\n","$L26","\n","$L27","\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n","$L28","\n","$L29","\n","$L2a","\n","$L2b","\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n","$L2c","\n","$L2d","\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n","$L2e","\n","$L2f","\n","$L30","\n","$L31","\n","$L32","\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n","$L33","\n","$L34","\n","$L35","\n","$L36","\n","$L37","\n","$L38","\n","$L39","\n","$L3a","\n","$L3b","\n","$L3c","\n","$L3d","\n","$L3e","\n","$L3f","\n","$L40","\n","$L41"]}]]}],["$L42","$L43","$L44","$L45"],"$L46"]}],"loading":null,"isPartial":false}
|
||
3:["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}]
|
||
4:["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}]
|
||
5:["$","p",null,{"children":"A $d$-dimensional embedding space can represent $\\mathcal{O}(e^d)$ nearly-orthogonal vectors. For semantic search, we want documents with different meanings to map to different regions. Higher dimensions provide more room."}]
|
||
6:["$","h3",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"information-theoretic-view","children":[["$","a",null,{"data-card":"","href":"#information-theoretic-view","className":"peer","children":"Information-Theoretic View"}],["$","svg",null,{"xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
|
||
7:["$","p",null,{"children":"Consider a corpus of $N$ documents, each with entropy $H$ bits. A $d$-dimensional float32 embedding stores $32d$ bits. For lossless encoding:"}]
|
||
8:["$","p",null,{"children":["$","code",null,{"children":"32d \\geq N \\cdot H"}]}]
|
||
9:["$","p",null,{"children":"For a corpus of 1 billion documents with 100 bits of effective information each, we need:"}]
|
||
a:["$","p",null,{"children":["$","code",null,{"children":"d \\geq \\frac\\{10^9 \\cdot 100\\}\\{32\\} \\approx 3 \\times 10^9"}]}]
|
||
b:["$","p",null,{"children":"Clearly, embeddings are lossy compression. But higher dimensions reduce the loss."}]
|
||
c:["$","h3",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"empirical-observations","children":[["$","a",null,{"data-card":"","href":"#empirical-observations","className":"peer","children":"Empirical Observations"}],["$","svg",null,{"xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
|
||
d:["$","p",null,{"children":"Retrieval quality on our internal benchmarks plateaued around:"}]
|
||
e:["$","ul",null,{"children":["\n",["$","li",null,{"children":"768 dimensions: 71% recall@10"}],"\n",["$","li",null,{"children":"1536 dimensions: 78% recall@10"}],"\n",["$","li",null,{"children":"3072 dimensions: 83% recall@10"}],"\n",["$","li",null,{"children":"7680 dimensions: 87% recall@10"}],"\n"]}]
|
||
f:["$","p",null,{"children":"Diminishing returns set in, but gains persist well beyond conventional wisdom."}]
|
||
10:["$","h2",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"training-high-dimensional-embeddings","children":[["$","a",null,{"data-card":"","href":"#training-high-dimensional-embeddings","className":"peer","children":"Training High-Dimensional Embeddings"}],["$","svg",null,{"xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
|
||
11:["$","h3",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"architecture","children":[["$","a",null,{"data-card":"","href":"#architecture","className":"peer","children":"Architecture"}],["$","svg",null,{"xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
|
||
12:["$","p",null,{"children":"We use a standard transformer encoder with a projection head:"}]
|
||
13:["$","$L47",null,{"className":"shiki shiki-themes github-light github-dark","style":{"--shiki-light":"#24292e","--shiki-dark":"#e1e4e8","--shiki-light-bg":"#fff","--shiki-dark-bg":"#24292e"},"tabIndex":"0","icon":"<svg viewBox=\"0 0 24 24\"><path d=\"M 6,1 C 4.354992,1 3,2.354992 3,4 v 16 c 0,1.645008 1.354992,3 3,3 h 12 c 1.645008,0 3,-1.354992 3,-3 V 8 7 A 1.0001,1.0001 0 0 0 20.707031,6.2929687 l -5,-5 A 1.0001,1.0001 0 0 0 15,1 h -1 z m 0,2 h 7 v 3 c 0,1.645008 1.354992,3 3,3 h 3 v 11 c 0,0.564129 -0.435871,1 -1,1 H 6 C 5.4358712,21 5,20.564129 5,20 V 4 C 5,3.4358712 5.4358712,3 6,3 Z M 15,3.4140625 18.585937,7 H 16 C 15.435871,7 15,6.5641288 15,6 Z\" fill=\"currentColor\" /></svg>","children":["$","$L48",null,{"children":["$","code",null,{"children":[["$","span",null,{"className":"line","children":["$","span",null,{}]}],"\n",["$","span",null,{"className":"line","children":["$","span",null,{"children":" Input --> Transformer(L=12, H=768) --> Pool --> Linear(768, 7680) --> Normalize"}]}],"\n",["$","span",null,{"className":"line","children":["$","span",null,{"children":" "}]}]]}]}]}]
|
||
14:["$","p",null,{"children":"The projection head maps from the transformer’s hidden dimension to the embedding space. This decouples representation capacity from compute requirements."}]
|
||
15:["$","h3",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"contrastive-learning","children":[["$","a",null,{"data-card":"","href":"#contrastive-learning","className":"peer","children":"Contrastive Learning"}],["$","svg",null,{"xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
|
||
16:["$","p",null,{"children":"We train with InfoNCE loss over batches of (query, positive, negatives):"}]
|
||
17:["$","p",null,{"children":["$","code",null,{"children":"\\mathcal\\{L\\} = -\\log \\frac\\{\\exp(q \\cdot p^+ / \\tau)\\}\\{\\sum_\\{i\\} \\exp(q \\cdot p_i / \\tau)\\}"}]}]
|
||
18:["$","p",null,{"children":"With temperature $\\tau = 0.01$ for high-dimensional spaces (lower than typical)."}]
|
||
19:["$","h3",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"hard-negative-mining","children":[["$","a",null,{"data-card":"","href":"#hard-negative-mining","className":"peer","children":"Hard Negative Mining"}],["$","svg",null,{"xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
|
||
1a:["$","p",null,{"children":"High-dimensional spaces require harder negatives to provide gradient signal. Our mining strategy:"}]
|
||
1b:["$","ol",null,{"children":["\n",["$","li",null,{"children":"Retrieve top-100 candidates via approximate nearest neighbor"}],"\n",["$","li",null,{"children":"Filter to exclude true positives"}],"\n",["$","li",null,{"children":"Sample negatives weighted by similarity (harder = more likely)"}],"\n"]}]
|
||
1c:["$","p",null,{"children":"This curriculum focuses training on the decision boundary."}]
|
||
1d:["$","h2",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"practical-considerations","children":[["$","a",null,{"data-card":"","href":"#practical-considerations","className":"peer","children":"Practical Considerations"}],["$","svg",null,{"xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
|
||
1e:["$","h3",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"storage","children":[["$","a",null,{"data-card":"","href":"#storage","className":"peer","children":"Storage"}],["$","svg",null,{"xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
|
||
1f:["$","p",null,{"children":"7680-dimensional float32 embeddings require 30KB per vector. For 1 billion documents:"}]
|
||
20:["$","p",null,{"children":["$","code",null,{"children":"10^9 \\times 30\\text\\{KB\\} = 30\\text\\{TB\\}"}]}]
|
||
21:["$","p",null,{"children":"This is substantial but manageable with modern storage."}]
|
||
22:["$","h3",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"quantization","children":[["$","a",null,{"data-card":"","href":"#quantization","className":"peer","children":"Quantization"}],["$","svg",null,{"xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
|
||
23:["$","p",null,{"children":"We can reduce storage through quantization:"}]
|
||
24:["$","div",null,{"className":"relative overflow-auto prose-no-margin my-6","children":["$","table",null,{"children":[["$","thead",null,{"children":["$","tr",null,{"children":[["$","th",null,{"children":"Precision"}],["$","th",null,{"children":"Bytes/Vector"}],["$","th",null,{"children":"Recall@10"}]]}]}],["$","tbody",null,{"children":[["$","tr",null,{"children":[["$","td",null,{"children":"float32"}],["$","td",null,{"children":"30,720"}],["$","td",null,{"children":"87.3%"}]]}],["$","tr",null,{"children":[["$","td",null,{"children":"float16"}],["$","td",null,{"children":"15,360"}],["$","td",null,{"children":"87.1%"}]]}],["$","tr",null,{"children":[["$","td",null,{"children":"int8"}],["$","td",null,{"children":"7,680"}],["$","td",null,{"children":"85.9%"}]]}],["$","tr",null,{"children":[["$","td",null,{"children":"binary"}],["$","td",null,{"children":"960"}],["$","td",null,{"children":"78.2%"}]]}]]}]]}]}]
|
||
25:["$","p",null,{"children":"int8 quantization provides 4x compression with minimal quality loss."}]
|
||
26:["$","h3",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"approximate-search","children":[["$","a",null,{"data-card":"","href":"#approximate-search","className":"peer","children":"Approximate Search"}],["$","svg",null,{"xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
|
||
27:["$","p",null,{"children":"Exact nearest neighbor search in 7680 dimensions is expensive. We use hierarchical navigable small world (HNSW) graphs:"}]
|
||
28:["$","div",null,{"className":"relative overflow-auto prose-no-margin my-6","children":["$","table",null,{"children":[["$","thead",null,{"children":["$","tr",null,{"children":[["$","th",null,{"children":"Dimensions"}],["$","th",null,{"children":"Build Time"}],["$","th",null,{"children":"Query Time"}],["$","th",null,{"children":"Recall@10"}]]}]}],["$","tbody",null,{"children":[["$","tr",null,{"children":[["$","td",null,{"children":"768"}],["$","td",null,{"children":"1x"}],["$","td",null,{"children":"1x"}],["$","td",null,{"children":"99.2%"}]]}],["$","tr",null,{"children":[["$","td",null,{"children":"7680"}],["$","td",null,{"children":"3.2x"}],["$","td",null,{"children":"2.8x"}],["$","td",null,{"children":"98.7%"}]]}]]}]]}]}]
|
||
29:["$","p",null,{"children":"The overhead is sublinear in dimensionality due to efficient distance computations."}]
|
||
2a:["$","h2",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"benchmark-results","children":[["$","a",null,{"data-card":"","href":"#benchmark-results","className":"peer","children":"Benchmark Results"}],["$","svg",null,{"xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
|
||
2b:["$","h3",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"ms-marco-passage-retrieval","children":[["$","a",null,{"data-card":"","href":"#ms-marco-passage-retrieval","className":"peer","children":"MS MARCO Passage Retrieval"}],["$","svg",null,{"xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
|
||
2c:["$","div",null,{"className":"relative overflow-auto prose-no-margin my-6","children":["$","table",null,{"children":[["$","thead",null,{"children":["$","tr",null,{"children":[["$","th",null,{"children":"Model"}],["$","th",null,{"children":"Dimensions"}],["$","th",null,{"children":"MRR@10"}],["$","th",null,{"children":"Recall@100"}]]}]}],["$","tbody",null,{"children":[["$","tr",null,{"children":[["$","td",null,{"children":"BM25"}],["$","td",null,{"children":"-"}],["$","td",null,{"children":"18.4"}],["$","td",null,{"children":"66.5"}]]}],["$","tr",null,{"children":[["$","td",null,{"children":"DPR"}],["$","td",null,{"children":"768"}],["$","td",null,{"children":"31.1"}],["$","td",null,{"children":"82.4"}]]}],["$","tr",null,{"children":[["$","td",null,{"children":"Contriever"}],["$","td",null,{"children":"768"}],["$","td",null,{"children":"32.8"}],["$","td",null,{"children":"84.1"}]]}],["$","tr",null,{"children":[["$","td",null,{"children":"Zen-Embed"}],["$","td",null,{"children":"7680"}],["$","td",null,{"children":"38.6"}],["$","td",null,{"children":"91.3"}]]}]]}]]}]}]
|
||
2d:["$","h3",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"natural-questions","children":[["$","a",null,{"data-card":"","href":"#natural-questions","className":"peer","children":"Natural Questions"}],["$","svg",null,{"xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
|
||
2e:["$","div",null,{"className":"relative overflow-auto prose-no-margin my-6","children":["$","table",null,{"children":[["$","thead",null,{"children":["$","tr",null,{"children":[["$","th",null,{"children":"Model"}],["$","th",null,{"children":"Dimensions"}],["$","th",null,{"children":"Top-20 Acc"}],["$","th",null,{"children":"Top-100 Acc"}]]}]}],["$","tbody",null,{"children":[["$","tr",null,{"children":[["$","td",null,{"children":"DPR"}],["$","td",null,{"children":"768"}],["$","td",null,{"children":"78.4"}],["$","td",null,{"children":"85.4"}]]}],["$","tr",null,{"children":[["$","td",null,{"children":"Contriever"}],["$","td",null,{"children":"768"}],["$","td",null,{"children":"81.3"}],["$","td",null,{"children":"88.1"}]]}],["$","tr",null,{"children":[["$","td",null,{"children":"Zen-Embed"}],["$","td",null,{"children":"7680"}],["$","td",null,{"children":"86.7"}],["$","td",null,{"children":"93.2"}]]}]]}]]}]}]
|
||
2f:["$","p",null,{"children":"High-dimensional embeddings provide substantial gains on retrieval benchmarks."}]
|
||
30:["$","h2",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"analysis","children":[["$","a",null,{"data-card":"","href":"#analysis","className":"peer","children":"Analysis"}],["$","svg",null,{"xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
|
||
31:["$","h3",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"what-do-extra-dimensions-encode","children":[["$","a",null,{"data-card":"","href":"#what-do-extra-dimensions-encode","className":"peer","children":"What Do Extra Dimensions Encode?"}],["$","svg",null,{"xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
|
||
32:["$","p",null,{"children":"We analyze the learned embedding space through probing tasks:"}]
|
||
33:["$","div",null,{"className":"relative overflow-auto prose-no-margin my-6","children":["$","table",null,{"children":[["$","thead",null,{"children":["$","tr",null,{"children":[["$","th",null,{"children":"Property"}],["$","th",null,{"children":"768d Probe Acc"}],["$","th",null,{"children":"7680d Probe Acc"}]]}]}],["$","tbody",null,{"children":[["$","tr",null,{"children":[["$","td",null,{"children":"Topic"}],["$","td",null,{"children":"84.2%"}],["$","td",null,{"children":"86.1%"}]]}],["$","tr",null,{"children":[["$","td",null,{"children":"Sentiment"}],["$","td",null,{"children":"91.3%"}],["$","td",null,{"children":"92.8%"}]]}],["$","tr",null,{"children":[["$","td",null,{"children":"Entity"}],["$","td",null,{"children":"67.4%"}],["$","td",null,{"children":"78.9%"}]]}],["$","tr",null,{"children":[["$","td",null,{"children":"Relation"}],["$","td",null,{"children":"52.1%"}],["$","td",null,{"children":"71.3%"}]]}]]}]]}]}]
|
||
34:["$","p",null,{"children":"The largest gains are in entity and relation encoding—fine-grained semantic properties that require more capacity."}]
|
||
35:["$","h3",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"nearest-neighbor-analysis","children":[["$","a",null,{"data-card":"","href":"#nearest-neighbor-analysis","className":"peer","children":"Nearest Neighbor Analysis"}],["$","svg",null,{"xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
|
||
36:["$","p",null,{"children":"For the query “What causes inflation?”, nearest neighbors at different dimensions:"}]
|
||
37:["$","p",null,{"children":["$","strong",null,{"children":"768 dimensions:"}]}]
|
||
38:["$","ol",null,{"children":["\n",["$","li",null,{"children":"What is inflation? (similar query)"}],"\n",["$","li",null,{"children":"How does inflation work? (similar query)"}],"\n",["$","li",null,{"children":"Inflation rates by country (tangential)"}],"\n"]}]
|
||
39:["$","p",null,{"children":["$","strong",null,{"children":"7680 dimensions:"}]}]
|
||
3a:["$","ol",null,{"children":["\n",["$","li",null,{"children":"Inflation is caused by… (direct answer)"}],"\n",["$","li",null,{"children":"The primary drivers of inflation include… (direct answer)"}],"\n",["$","li",null,{"children":"Central bank policies affect inflation through… (relevant detail)"}],"\n"]}]
|
||
3b:["$","p",null,{"children":"Higher dimensions better distinguish queries from answers."}]
|
||
3c:["$","h2",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"recommendations","children":[["$","a",null,{"data-card":"","href":"#recommendations","className":"peer","children":"Recommendations"}],["$","svg",null,{"xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
|
||
3d:["$","p",null,{"children":"Based on our experiments:"}]
|
||
3e:["$","ol",null,{"children":["\n",["$","li",null,{"children":[["$","strong",null,{"children":"Try 3072+ dimensions"}]," if retrieval quality matters and storage is available"]}],"\n",["$","li",null,{"children":[["$","strong",null,{"children":"Use int8 quantization"}]," for production deployments"]}],"\n",["$","li",null,{"children":[["$","strong",null,{"children":"Invest in hard negative mining"}]," to realize the benefits of capacity"]}],"\n",["$","li",null,{"children":[["$","strong",null,{"children":"Benchmark on your data"}]," —gains vary by domain"]}],"\n"]}]
|
||
3f:["$","p",null,{"children":"Conventional embedding sizes are conventions, not laws. Question them."}]
|
||
40:["$","hr",null,{}]
|
||
41:["$","p",null,{"children":["$","em",null,{"children":"Technical details in “Scaling Embedding Dimensions for Semantic Retrieval” (2022). Model weights available on Hugging Face."}]}]
|
||
42:["$","script","script-0",{"src":"/_next/static/chunks/8de849ca74fc071f.js","async":true}]
|
||
43:["$","script","script-1",{"src":"/_next/static/chunks/e62b91212ee7f8ff.js","async":true}]
|
||
44:["$","script","script-2",{"src":"/_next/static/chunks/f19fe44237e54646.js","async":true}]
|
||
45:["$","script","script-3",{"src":"/_next/static/chunks/cb0a883bafeb6805.js","async":true}]
|
||
46:["$","$L49",null,{"children":["$","$4a",null,{"name":"Next.MetadataOutlet","children":"$@4b"}]}]
|
||
4b:null
|