mirror of
https://github.com/zenlm/zen-blog.git
synced 2026-11-03 14:11:45 +00:00
36 lines
14 KiB
Plaintext
36 lines
14 KiB
Plaintext
1:"$Sreact.fragment"
|
||
2:I[10086,["/_next/static/chunks/59d0ad1b64f8544e.js","/_next/static/chunks/8de849ca74fc071f.js","/_next/static/chunks/e62b91212ee7f8ff.js","/_next/static/chunks/f19fe44237e54646.js","/_next/static/chunks/cb0a883bafeb6805.js"],""]
|
||
1e:I[51504,["/_next/static/chunks/59d0ad1b64f8544e.js","/_next/static/chunks/8de849ca74fc071f.js","/_next/static/chunks/e62b91212ee7f8ff.js","/_next/static/chunks/f19fe44237e54646.js","/_next/static/chunks/cb0a883bafeb6805.js"],"CodeBlock"]
|
||
1f:I[51504,["/_next/static/chunks/59d0ad1b64f8544e.js","/_next/static/chunks/8de849ca74fc071f.js","/_next/static/chunks/e62b91212ee7f8ff.js","/_next/static/chunks/f19fe44237e54646.js","/_next/static/chunks/cb0a883bafeb6805.js"],"Pre"]
|
||
20:I[89923,["/_next/static/chunks/4d80e004cf4896dd.js","/_next/static/chunks/350ee4303b732916.js"],"OutletBoundary"]
|
||
21:"$Sreact.suspense"
|
||
0:{"buildId":"i-dnJM_MIpJSOCQWNJVMq","rsc":["$","$1","c",{"children":[["$","main",null,{"className":"mx-auto w-full max-w-2xl px-4 py-16","children":[["$","$L2",null,{"href":"/blog","className":"inline-flex items-center gap-1.5 text-sm text-fd-muted-foreground hover:text-fd-foreground mb-10 transition-colors","children":"← Back to Blog"}],["$","div",null,{"className":"mb-8","children":[["$","time",null,{"className":"text-xs font-mono text-fd-muted-foreground uppercase tracking-wider","children":"July 21, 2021"}],["$","h1",null,{"className":"text-3xl font-bold mt-2 mb-3","children":"Training LLMs on Collective Intelligence"}],["$","p",null,{"className":"text-fd-muted-foreground text-lg mb-4","children":"How we're approaching training data curation to capture humanity's collective intelligence."}],["$","div",null,{"className":"flex items-center gap-3 pt-4 border-t border-fd-border","children":[["$","span",null,{"className":"text-sm text-fd-muted-foreground","children":["By ","Zach Kelling"]}],["$","div",null,{"className":"flex gap-1.5 ml-auto","children":[["$","span","Research",{"className":"text-[10px] font-mono uppercase tracking-wider bg-fd-muted text-fd-muted-foreground px-1.5 py-0.5 rounded","children":"Research"}],["$","span","Training",{"className":"text-[10px] font-mono uppercase tracking-wider bg-fd-muted text-fd-muted-foreground px-1.5 py-0.5 rounded","children":"Training"}]]}]]}]]}],["$","div",null,{"className":"prose dark:prose-invert max-w-none","children":[["$","p",null,{"children":"Language models are trained on text. That text represents the accumulated knowledge, reasoning, and creativity of countless individuals. Yet the curation process that selects training data receives surprisingly little attention."}],"\n",["$","h2",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"the-data-problem","children":[["$","a",null,{"data-card":"","href":"#the-data-problem","className":"peer","children":"The Data Problem"}],["$","svg",null,{"xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}],"\n",["$","p",null,{"children":"Most large language models are trained on web scrapes filtered by simple heuristics. This approach has several issues:"}],"\n",["$","ol",null,{"children":["\n",["$","li",null,{"children":[["$","strong",null,{"children":"Quality variance"}]," : Web content ranges from expert research to spam"]}],"\n",["$","li",null,{"children":[["$","strong",null,{"children":"Hidden biases"}]," : Filtering decisions embed value judgments"]}],"\n",["$","li",null,{"children":[["$","strong",null,{"children":"Provenance opacity"}]," : It’s unclear what’s included or excluded"]}],"\n",["$","li",null,{"children":[["$","strong",null,{"children":"Legal ambiguity"}]," : Copyright and consent questions remain unresolved"]}],"\n"]}],"\n",["$","h2",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"our-approach-transparent-curation","children":[["$","a",null,{"data-card":"","href":"#our-approach-transparent-curation","className":"peer","children":"Our Approach: Transparent Curation"}],["$","svg",null,{"xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}],"\n","$L3","\n","$L4","\n","$L5","\n","$L6","\n","$L7","\n","$L8","\n","$L9","\n","$La","\n","$Lb","\n","$Lc","\n","$Ld","\n","$Le","\n","$Lf","\n","$L10","\n","$L11","\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n","$L12","\n","$L13","\n","$L14","\n","$L15","\n","$L16","\n","$L17","\n","$L18"]}]]}],["$L19","$L1a","$L1b","$L1c"],"$L1d"]}],"loading":null,"isPartial":false}
|
||
3:["$","p",null,{"children":"At Zen, we’re taking a different path. Our data pipeline operates on three principles."}]
|
||
4:["$","h3",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"explicit-criteria","children":[["$","a",null,{"data-card":"","href":"#explicit-criteria","className":"peer","children":"Explicit Criteria"}],["$","svg",null,{"xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
|
||
5:["$","p",null,{"children":"Every filtering decision has documented rationale. When we exclude content, we record why. When we weight certain sources higher, we explain the reasoning. This creates an auditable trail."}]
|
||
6:["$","h3",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"community-input","children":[["$","a",null,{"data-card":"","href":"#community-input","className":"peer","children":"Community Input"}],["$","svg",null,{"xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
|
||
7:["$","p",null,{"children":"Data curation involves value judgments. What counts as “quality”? Which perspectives matter? These aren’t purely technical questions. We’re building mechanisms for community input into curation criteria."}]
|
||
8:["$","h3",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"provenance-tracking","children":[["$","a",null,{"data-card":"","href":"#provenance-tracking","className":"peer","children":"Provenance Tracking"}],["$","svg",null,{"xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
|
||
9:["$","p",null,{"children":"Each training example links to its source with metadata about:"}]
|
||
a:["$","ul",null,{"children":["\n",["$","li",null,{"children":"Original publication context"}],"\n",["$","li",null,{"children":"Author information (when available)"}],"\n",["$","li",null,{"children":"Licensing terms"}],"\n",["$","li",null,{"children":"Processing steps applied"}],"\n"]}]
|
||
b:["$","h2",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"technical-implementation","children":[["$","a",null,{"data-card":"","href":"#technical-implementation","className":"peer","children":"Technical Implementation"}],["$","svg",null,{"xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
|
||
c:["$","p",null,{"children":"We’ve developed a pipeline that processes documents through several stages:"}]
|
||
d:["$","$L1e",null,{"className":"shiki shiki-themes github-light github-dark","style":{"--shiki-light":"#24292e","--shiki-dark":"#e1e4e8","--shiki-light-bg":"#fff","--shiki-dark-bg":"#24292e"},"tabIndex":"0","icon":"<svg viewBox=\"0 0 24 24\"><path d=\"M 6,1 C 4.354992,1 3,2.354992 3,4 v 16 c 0,1.645008 1.354992,3 3,3 h 12 c 1.645008,0 3,-1.354992 3,-3 V 8 7 A 1.0001,1.0001 0 0 0 20.707031,6.2929687 l -5,-5 A 1.0001,1.0001 0 0 0 15,1 h -1 z m 0,2 h 7 v 3 c 0,1.645008 1.354992,3 3,3 h 3 v 11 c 0,0.564129 -0.435871,1 -1,1 H 6 C 5.4358712,21 5,20.564129 5,20 V 4 C 5,3.4358712 5.4358712,3 6,3 Z M 15,3.4140625 18.585937,7 H 16 C 15.435871,7 15,6.5641288 15,6 Z\" fill=\"currentColor\" /></svg>","children":["$","$L1f",null,{"children":["$","code",null,{"children":[["$","span",null,{"className":"line","children":["$","span",null,{}]}],"\n",["$","span",null,{"className":"line","children":["$","span",null,{"children":" Source -> Extraction -> Deduplication -> Quality Scoring -> Attribution -> Storage"}]}],"\n",["$","span",null,{"className":"line","children":["$","span",null,{"children":" "}]}]]}]}]}]
|
||
e:["$","p",null,{"children":"The quality scoring model itself is trained on human judgments, with explicit criteria:"}]
|
||
f:["$","ul",null,{"children":["\n",["$","li",null,{"children":"Factual accuracy (where verifiable)"}],"\n",["$","li",null,{"children":"Reasoning coherence"}],"\n",["$","li",null,{"children":"Writing clarity"}],"\n",["$","li",null,{"children":"Information density"}],"\n"]}]
|
||
10:["$","h2",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"early-results","children":[["$","a",null,{"data-card":"","href":"#early-results","className":"peer","children":"Early Results"}],["$","svg",null,{"xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
|
||
11:["$","p",null,{"children":"Our initial corpus contains 2.3 trillion tokens with full provenance tracking. Early experiments suggest that careful curation can match larger but noisier datasets:"}]
|
||
12:["$","div",null,{"className":"relative overflow-auto prose-no-margin my-6","children":["$","table",null,{"children":[["$","thead",null,{"children":["$","tr",null,{"children":[["$","th",null,{"children":"Corpus"}],["$","th",null,{"children":"Tokens"}],["$","th",null,{"children":"Benchmark Score"}]]}]}],["$","tbody",null,{"children":[["$","tr",null,{"children":[["$","td",null,{"children":"Web-raw"}],["$","td",null,{"children":"5T"}],["$","td",null,{"children":"72.3"}]]}],["$","tr",null,{"children":[["$","td",null,{"children":"Web-filtered"}],["$","td",null,{"children":"3T"}],["$","td",null,{"children":"74.1"}]]}],["$","tr",null,{"children":[["$","td",null,{"children":"Zen-curated"}],["$","td",null,{"children":"2.3T"}],["$","td",null,{"children":"75.8"}]]}]]}]]}]}]
|
||
13:["$","p",null,{"children":"The numbers are preliminary, but the direction is clear: quality over quantity."}]
|
||
14:["$","h2",null,{"className":"flex scroll-m-28 flex-row items-center gap-2","id":"what-this-means","children":[["$","a",null,{"data-card":"","href":"#what-this-means","className":"peer","children":"What This Means"}],["$","svg",null,{"xmlns":"http://www.w3.org/2000/svg","width":24,"height":24,"viewBox":"0 0 24 24","fill":"none","stroke":"currentColor","strokeWidth":2,"strokeLinecap":"round","strokeLinejoin":"round","className":"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100","aria-hidden":true,"children":[["$","path","1cjeqo",{"d":"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"}],["$","path","19qd67",{"d":"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"}],"$undefined"]}]]}]
|
||
15:["$","p",null,{"children":"Training on collective intelligence isn’t just about scraping more data. It’s about respecting the sources, maintaining transparency, and involving the community in decisions that shape AI behavior."}]
|
||
16:["$","p",null,{"children":"We’ll publish our full curation methodology and tooling in the coming months."}]
|
||
17:["$","hr",null,{}]
|
||
18:["$","p",null,{"children":["$","em",null,{"children":"Zach Kelling is a co-founder of Zoo Labs Foundation."}]}]
|
||
19:["$","script","script-0",{"src":"/_next/static/chunks/8de849ca74fc071f.js","async":true}]
|
||
1a:["$","script","script-1",{"src":"/_next/static/chunks/e62b91212ee7f8ff.js","async":true}]
|
||
1b:["$","script","script-2",{"src":"/_next/static/chunks/f19fe44237e54646.js","async":true}]
|
||
1c:["$","script","script-3",{"src":"/_next/static/chunks/cb0a883bafeb6805.js","async":true}]
|
||
1d:["$","$L20",null,{"children":["$","$21",null,{"name":"Next.MetadataOutlet","children":"$@22"}]}]
|
||
22:null
|