Files

40 lines
20 KiB
Plaintext
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
1:"$Sreact.fragment"
2:I[106,["/_next/static/chunks/59d0ad1b64f8544e.js"],"RootProvider"]
3:I[53113,["/_next/static/chunks/4d80e004cf4896dd.js","/_next/static/chunks/350ee4303b732916.js"],"default"]
4:I[73211,["/_next/static/chunks/4d80e004cf4896dd.js","/_next/static/chunks/350ee4303b732916.js"],"default"]
5:I[10086,["/_next/static/chunks/59d0ad1b64f8544e.js","/_next/static/chunks/8de849ca74fc071f.js","/_next/static/chunks/e62b91212ee7f8ff.js","/_next/static/chunks/f19fe44237e54646.js","/_next/static/chunks/cb0a883bafeb6805.js"],""]
7:I[89923,["/_next/static/chunks/4d80e004cf4896dd.js","/_next/static/chunks/350ee4303b732916.js"],"OutletBoundary"]
8:"$Sreact.suspense"
a:I[89923,["/_next/static/chunks/4d80e004cf4896dd.js","/_next/static/chunks/350ee4303b732916.js"],"ViewportBoundary"]
c:I[89923,["/_next/static/chunks/4d80e004cf4896dd.js","/_next/static/chunks/350ee4303b732916.js"],"MetadataBoundary"]
e:I[6998,["/_next/static/chunks/4d80e004cf4896dd.js","/_next/static/chunks/350ee4303b732916.js"],"default"]
:HL["/_next/static/chunks/f2332aac77592f9d.css","style"]
0:{"P":null,"b":"i-dnJM_MIpJSOCQWNJVMq","c":["","blog","training-llms-collective-intelligence"],"q":"","i":false,"f":[[["",{"children":["blog",{"children":[["slug","training-llms-collective-intelligence","d"],{"children":["__PAGE__",{}]}]}]},"$undefined","$undefined",true],[["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/_next/static/chunks/f2332aac77592f9d.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","script","script-0",{"src":"/_next/static/chunks/59d0ad1b64f8544e.js","async":true,"nonce":"$undefined"}]],["$","html",null,{"lang":"en","className":"dark","suppressHydrationWarning":true,"children":["$","body",null,{"className":"antialiased","children":["$","$L2",null,{"children":["$","$L3",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L4",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","main",null,{"className":"flex min-h-screen flex-col items-center justify-center px-4 text-center","children":[["$","div",null,{"className":"mb-8 opacity-20","children":["$","svg",null,{"width":"120","height":"120","viewBox":"0 0 120 120","fill":"none","aria-hidden":"true","children":["$","circle",null,{"cx":"60","cy":"60","r":"50","stroke":"currentColor","strokeWidth":"3","strokeLinecap":"round","strokeDasharray":"280 40"}]}]}],["$","p",null,{"className":"text-xs font-mono uppercase tracking-[0.3em] text-fd-muted-foreground mb-4","children":"404"}],["$","h1",null,{"className":"text-3xl font-semibold mb-3","children":"Page not found"}],["$","p",null,{"className":"text-fd-muted-foreground max-w-sm mb-10","children":"This page doesn't exist, or it may have moved. Try the documentation or head home."}],["$","div",null,{"className":"flex flex-wrap gap-3 justify-center","children":[["$","$L5",null,{"href":"/","className":"inline-flex items-center gap-2 rounded-lg bg-fd-primary text-fd-primary-foreground px-4 py-2 text-sm font-medium hover:opacity-90 transition","children":"Go home"}],["$","$L5",null,{"href":"/docs","className":"inline-flex items-center gap-2 rounded-lg border border-fd-border bg-fd-background px-4 py-2 text-sm font-medium hover:bg-fd-muted transition","children":"Documentation"}],["$","$L5",null,{"href":"/docs/models","className":"inline-flex items-center gap-2 rounded-lg border border-fd-border bg-fd-background px-4 py-2 text-sm font-medium hover:bg-fd-muted transition","children":"Browse models"}]]}],["$","p",null,{"className":"mt-16 text-xs font-mono text-fd-muted-foreground opacity-50","children":"zenlm.org"}]]}],[]],"forbidden":"$undefined","unauthorized":"$undefined"}]}]}]}]]}],{"children":[["$","$1","c",{"children":[null,["$","$L3",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L4",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","forbidden":"$undefined","unauthorized":"$undefined"}]]}],{"children":[["$","$1","c",{"children":[null,["$","$L3",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L4",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","forbidden":"$undefined","unauthorized":"$undefined"}]]}],{"children":[["$","$1","c",{"children":["$L6",[["$","script","script-0",{"src":"/_next/static/chunks/8de849ca74fc071f.js","async":true,"nonce":"$undefined"}],["$","script","script-1",{"src":"/_next/static/chunks/e62b91212ee7f8ff.js","async":true,"nonce":"$undefined"}],["$","script","script-2",{"src":"/_next/static/chunks/f19fe44237e54646.js","async":true,"nonce":"$undefined"}],["$","script","script-3",{"src":"/_next/static/chunks/cb0a883bafeb6805.js","async":true,"nonce":"$undefined"}]],["$","$L7",null,{"children":["$","$8",null,{"name":"Next.MetadataOutlet","children":"$@9"}]}]]}],{},null,false,false]},null,false,false]},null,false,false]},null,false,false],["$","$1","h",{"children":[null,["$","$La",null,{"children":"$Lb"}],["$","div",null,{"hidden":true,"children":["$","$Lc",null,{"children":["$","$8",null,{"name":"Next.Metadata","children":"$Ld"}]}]}],null]}],false]],"m":"$undefined","G":["$e",[]],"S":true}
6:["$","main",null,{"className":"mx-auto w-full max-w-2xl px-4 py-16","children":[["$","$L5",null,{"href":"/blog","className":"inline-flex items-center gap-1.5 text-sm text-fd-muted-foreground hover:text-fd-foreground mb-10 transition-colors","children":"← Back to Blog"}],["$","div",null,{"className":"mb-8","children":[["$","time",null,{"className":"text-xs font-mono text-fd-muted-foreground uppercase tracking-wider","children":"July 21, 2021"}],["$","h1",null,{"className":"text-3xl font-bold mt-2 mb-3","children":"Training LLMs on Collective Intelligence"}],["$","p",null,{"className":"text-fd-muted-foreground text-lg mb-4","children":"How we're approaching training data curation to capture humanity's collective intelligence."}],["$","div",null,{"className":"flex items-center gap-3 pt-4 border-t border-fd-border","children":[["$","span",null,{"className":"text-sm text-fd-muted-foreground","children":["By ","Zach Kelling"]}],["$","div",null,{"className":"flex gap-1.5 ml-auto","children":[["$","span","Research",{"className":"text-[10px] font-mono uppercase tracking-wider bg-fd-muted text-fd-muted-foreground px-1.5 py-0.5 rounded","children":"Research"}],["$","span","Training",{"className":"text-[10px] font-mono uppercase tracking-wider bg-fd-muted text-fd-muted-foreground px-1.5 py-0.5 rounded","children":"Training"}]]}]]}]]}],["$","div",null,{"className":"prose dark:prose-invert max-w-none","children":[["$","p",null,{"children":"Language models are trained on text. That text represents the accumulated knowledge, reasoning, and creativity of countless individuals. Yet the curation process that selects training data receives surprisingly little attention."}],"\n",["$","h2",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"the-data-problem","children":[["$","a",null,{"data-card":"","href":"#the-data-problem","className":"peer","children":"The Data Problem"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}],"\n",["$","p",null,{"children":"Most large language models are trained on web scrapes filtered by simple heuristics. This approach has several issues:"}],"\n",["$","ol",null,{"children":["\n",["$","li",null,{"children":[["$","strong",null,{"children":"Quality variance"}]," : Web content ranges from expert research to spam"]}],"\n",["$","li",null,{"children":[["$","strong",null,{"children":"Hidden biases"}]," : Filtering decisions embed value judgments"]}],"\n",["$","li",null,{"children":[["$","strong",null,{"children":"Provenance opacity"}]," : It’s unclear what’s included or excluded"]}],"\n",["$","li",null,{"children":[["$","strong",null,{"children":"Legal ambiguity"}]," : Copyright and consent questions remain unresolved"]}],"\n"]}],"\n",["$","h2",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"our-approach-transparent-curation","children":[["$","a",null,{"data-card":"","href":"#our-approach-transparent-curation","className":"peer","children":"Our Approach: Transparent Curation"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}],"\n",["$","p",null,{"children":"At Zen, we’re taking a different path. Our data pipeline operates on three principles."}],"\n","$Lf","\n","$L10","\n","$L11","\n","$L12","\n","$L13","\n","$L14","\n","$L15","\n","$L16","\n","$L17","\n","$L18","\n","$L19","\n","$L1a","\n","$L1b","\n","$L1c","\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n","$L1d","\n","$L1e","\n","$L1f","\n","$L20","\n","$L21","\n","$L22","\n","$L23"]}]]}]
24:I[51504,["/_next/static/chunks/59d0ad1b64f8544e.js","/_next/static/chunks/8de849ca74fc071f.js","/_next/static/chunks/e62b91212ee7f8ff.js","/_next/static/chunks/f19fe44237e54646.js","/_next/static/chunks/cb0a883bafeb6805.js"],"CodeBlock"]
25:I[51504,["/_next/static/chunks/59d0ad1b64f8544e.js","/_next/static/chunks/8de849ca74fc071f.js","/_next/static/chunks/e62b91212ee7f8ff.js","/_next/static/chunks/f19fe44237e54646.js","/_next/static/chunks/cb0a883bafeb6805.js"],"Pre"]
f:["$","h3",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"explicit-criteria","children":[["$","a",null,{"data-card":"","href":"#explicit-criteria","className":"peer","children":"Explicit Criteria"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
10:["$","p",null,{"children":"Every filtering decision has documented rationale. When we exclude content, we record why. When we weight certain sources higher, we explain the reasoning. This creates an auditable trail."}]
11:["$","h3",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"community-input","children":[["$","a",null,{"data-card":"","href":"#community-input","className":"peer","children":"Community Input"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
12:["$","p",null,{"children":"Data curation involves value judgments. What counts as “quality”? Which perspectives matter? These aren’t purely technical questions. We’re building mechanisms for community input into curation criteria."}]
13:["$","h3",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"provenance-tracking","children":[["$","a",null,{"data-card":"","href":"#provenance-tracking","className":"peer","children":"Provenance Tracking"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
14:["$","p",null,{"children":"Each training example links to its source with metadata about:"}]
15:["$","ul",null,{"children":["\n",["$","li",null,{"children":"Original publication context"}],"\n",["$","li",null,{"children":"Author information (when available)"}],"\n",["$","li",null,{"children":"Licensing terms"}],"\n",["$","li",null,{"children":"Processing steps applied"}],"\n"]}]
16:["$","h2",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"technical-implementation","children":[["$","a",null,{"data-card":"","href":"#technical-implementation","className":"peer","children":"Technical Implementation"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
17:["$","p",null,{"children":"We’ve developed a pipeline that processes documents through several stages:"}]
18:["$","$L24",null,{"className":"shiki shiki-themes github-light github-dark","style":{"--shiki-light":"#24292e","--shiki-dark":"#e1e4e8","--shiki-light-bg":"#fff","--shiki-dark-bg":"#24292e"},"tabIndex":"0","icon":"<svg viewBox=\"0 0 24 24\"><path d=\"M 6,1 C 4.354992,1 3,2.354992 3,4 v 16 c 0,1.645008 1.354992,3 3,3 h 12 c 1.645008,0 3,-1.354992 3,-3 V 8 7 A 1.0001,1.0001 0 0 0 20.707031,6.2929687 l -5,-5 A 1.0001,1.0001 0 0 0 15,1 h -1 z m 0,2 h 7 v 3 c 0,1.645008 1.354992,3 3,3 h 3 v 11 c 0,0.564129 -0.435871,1 -1,1 H 6 C 5.4358712,21 5,20.564129 5,20 V 4 C 5,3.4358712 5.4358712,3 6,3 Z M 15,3.4140625 18.585937,7 H 16 C 15.435871,7 15,6.5641288 15,6 Z\" fill=\"currentColor\" /></svg>","children":["$","$L25",null,{"children":["$","code",null,{"children":[["$","span",null,{"className":"line","children":["$","span",null,{}]}],"\n",["$","span",null,{"className":"line","children":["$","span",null,{"children":" Source -> Extraction -> Deduplication -> Quality Scoring -> Attribution -> Storage"}]}],"\n",["$","span",null,{"className":"line","children":["$","span",null,{"children":" "}]}]]}]}]}]
19:["$","p",null,{"children":"The quality scoring model itself is trained on human judgments, with explicit criteria:"}]
1a:["$","ul",null,{"children":["\n",["$","li",null,{"children":"Factual accuracy (where verifiable)"}],"\n",["$","li",null,{"children":"Reasoning coherence"}],"\n",["$","li",null,{"children":"Writing clarity"}],"\n",["$","li",null,{"children":"Information density"}],"\n"]}]
1b:["$","h2",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"early-results","children":[["$","a",null,{"data-card":"","href":"#early-results","className":"peer","children":"Early Results"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
1c:["$","p",null,{"children":"Our initial corpus contains 2.3 trillion tokens with full provenance tracking. Early experiments suggest that careful curation can match larger but noisier datasets:"}]
1d:["$","div",null,{"className":"relative overflow-auto prose-no-margin my-6","children":["$","table",null,{"children":[["$","thead",null,{"children":["$","tr",null,{"children":[["$","th",null,{"children":"Corpus"}],["$","th",null,{"children":"Tokens"}],["$","th",null,{"children":"Benchmark Score"}]]}]}],["$","tbody",null,{"children":[["$","tr",null,{"children":[["$","td",null,{"children":"Web-raw"}],["$","td",null,{"children":"5T"}],["$","td",null,{"children":"72.3"}]]}],["$","tr",null,{"children":[["$","td",null,{"children":"Web-filtered"}],["$","td",null,{"children":"3T"}],["$","td",null,{"children":"74.1"}]]}],["$","tr",null,{"children":[["$","td",null,{"children":"Zen-curated"}],["$","td",null,{"children":"2.3T"}],["$","td",null,{"children":"75.8"}]]}]]}]]}]}]
1e:["$","p",null,{"children":"The numbers are preliminary, but the direction is clear: quality over quantity."}]
1f:["$","h2",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"what-this-means","children":[["$","a",null,{"data-card":"","href":"#what-this-means","className":"peer","children":"What This Means"}],["$","svg",null,{"ref":"$undefined","xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
20:["$","p",null,{"children":"Training on collective intelligence isn’t just about scraping more data. It’s about respecting the sources, maintaining transparency, and involving the community in decisions that shape AI behavior."}]
21:["$","p",null,{"children":"We’ll publish our full curation methodology and tooling in the coming months."}]
22:["$","hr",null,{}]
23:["$","p",null,{"children":["$","em",null,{"children":"Zach Kelling is a co-founder of Zoo Labs Foundation."}]}]
b:[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","meta","2",{"name":"theme-color","media":"(prefers-color-scheme: dark)","content":"#0A0A0A"}],["$","meta","3",{"name":"theme-color","media":"(prefers-color-scheme: light)","content":"#fff"}]]
9:null
d:[["$","title","0",{"children":"Training LLMs on Collective Intelligence — Zen LM Blog | Zen LM"}],["$","meta","1",{"name":"description","content":"How we're approaching training data curation to capture humanity's collective intelligence."}],["$","meta","2",{"property":"og:title","content":"Zen LM - Open Foundation Models"}],["$","meta","3",{"property":"og:description","content":"Zen AI model family by Hanzo AI. 14 frontier models from 4B to 1T+ parameters for code, reasoning, vision, and multimodal tasks."}],["$","meta","4",{"property":"og:url","content":"https://zenlm.org"}],["$","meta","5",{"property":"og:site_name","content":"Zen LM"}],["$","meta","6",{"property":"og:type","content":"website"}],["$","meta","7",{"name":"twitter:card","content":"summary_large_image"}],["$","meta","8",{"name":"twitter:site","content":"@zenlmorg"}],["$","meta","9",{"name":"twitter:creator","content":"@zenlmorg"}],["$","meta","10",{"name":"twitter:title","content":"Zen LM - Open Foundation Models"}],["$","meta","11",{"name":"twitter:description","content":"Zen AI model family by Hanzo AI. 14 frontier models from 4B to 1T+ parameters for code, reasoning, vision, and multimodal tasks."}]]