mirror of
https://github.com/zenlm/zen.git
synced 2026-11-17 11:36:07 +00:00
At 390 pixels the blog's tables, the datasets tables and the model card specs ran past the right edge, and body's overflow-x clip cut them off rather than letting them scroll. Tables now scroll in their own box, and a spec column shrinks and wraps instead of pushing its neighbour out of the card.
90 lines
35 KiB
HTML
90 lines
35 KiB
HTML
<!DOCTYPE html><!--_BoRG_r6Z3v9udL0lI0J_--><html lang="en"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="stylesheet" href="/_next/static/css/1d1f6bc532e5f43f.css" data-precedence="next"/><link rel="stylesheet" href="/_next/static/css/eb87e4f7aea490c6.css" data-precedence="next"/><link rel="preload" as="script" fetchPriority="low" href="/_next/static/chunks/webpack-f43ad49c79db3299.js"/><script src="/_next/static/chunks/4bd1b696-c023c6e3521b1417.js" async=""></script><script src="/_next/static/chunks/255-566af14aff71e752.js" async=""></script><script src="/_next/static/chunks/main-app-7eaab9f523b3767f.js" async=""></script><script src="/_next/static/chunks/619-ba102abea3e3d0e4.js" async=""></script><script src="/_next/static/chunks/app/layout-425ca81883167c16.js" async=""></script><script src="/_next/static/chunks/app/blog/%5Bslug%5D/page-38f8c2a205f98d97.js" async=""></script><script src="/assets/js/main.js" async=""></script><link rel="icon" type="image/svg+xml" href="/favicon.svg"/><link rel="alternate icon" href="/favicon.png"/><title>Federated Learning for Open AI — Zen Blog</title><meta name="description" content="How federated learning enables collaborative model training while preserving data privacy."/><meta name="keywords" content="AI, LLM, Agentic AI, Code Generation, Zen Coder, Multimodal, Open Source, Machine Learning"/><script src="/_next/static/chunks/polyfills-42372ed130431b0a.js" noModule=""></script></head><body><div hidden=""><!--$--><!--/$--></div><header><nav class="navbar"><div class="container"><div class="logo"><a href="/"><svg class="enso logo-img" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 100 100" width="34" height="34" fill="none" role="img" aria-label="Zen ensō"><defs><linearGradient id="_R_adb_" x1="12%" y1="0%" x2="88%" y2="100%"><stop offset="0%" stop-color="#ffffff"></stop><stop offset="50%" stop-color="#d4d4d8"></stop><stop offset="100%" stop-color="#7c7c82"></stop></linearGradient></defs><path class="enso-stroke" pathLength="1" d="M58 13 C38 9 17 22 13 44 C9 68 26 89 50 90 C74 91 91 74 89 50 C88 41 87 35 84 30" stroke="url(#_R_adb_)" stroke-width="7" stroke-linecap="round" stroke-linejoin="round"></path></svg><div class="logo-text"><h1>Zen LM</h1></div></a></div><div class="header-right"><div class="nav-links"><a class="" href="/">Home</a><div class="nav-item"><a class="" href="/models/">Models<svg class="nav-caret" width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M6 9l6 6 6-6"></path></svg></a><div class="nav-dropdown"><div class="nav-dropdown-inner"><a class="dd-item" href="/models/#zen6"><div class="dd-item-text"><span class="dd-item-label">Zen 6<span class="dd-tag">Available</span></span><span class="dd-item-desc">27B dense — text, images and video, 1M context with YaRN</span></div><svg class="dd-arrow" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M5 12h14M13 6l6 6-6 6"></path></svg></a><a class="dd-item" href="/models/#zen6"><div class="dd-item-text"><span class="dd-item-label">Zen 6 Flash<span class="dd-tag">Available</span></span><span class="dd-item-desc">The ternary Zen 6 — 5.95 GB, fits an Apple Silicon laptop</span></div><svg class="dd-arrow" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M5 12h14M13 6l6 6-6 6"></path></svg></a><a class="dd-item" href="https://hanzo.ai/research-access" target="_blank" rel="noopener noreferrer"><div class="dd-item-text"><span class="dd-item-label">Zen 7<span class="dd-tag">Research preview</span></span><span class="dd-item-desc">No weights yet — request access</span></div><svg class="dd-arrow" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M5 12h14M13 6l6 6-6 6"></path></svg></a><a class="dd-item" href="/models/#zen5"><div class="dd-item-text"><span class="dd-item-label">Earlier generations</span><span class="dd-item-desc">Zen 5, Zen 4 and Zen 3</span></div><svg class="dd-arrow" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M5 12h14M13 6l6 6-6 6"></path></svg></a><a class="dd-item" href="/models/#catalog"><div class="dd-item-text"><span class="dd-item-label">All models</span><span class="dd-item-desc">Browse the full catalog</span></div><svg class="dd-arrow" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M5 12h14M13 6l6 6-6 6"></path></svg></a></div></div></div><div class="nav-item"><a class="" href="/datasets/">Datasets<svg class="nav-caret" width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M6 9l6 6 6-6"></path></svg></a><div class="nav-dropdown"><div class="nav-dropdown-inner"><a class="dd-item" href="/datasets/"><div class="dd-item-text"><span class="dd-item-label">Zen Agentic Dataset<span class="dd-tag">New</span></span><span class="dd-item-desc">10B+ tokens of real tool use and reasoning</span></div><svg class="dd-arrow" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M5 12h14M13 6l6 6-6 6"></path></svg></a><a class="dd-item" href="/datasets/"><div class="dd-item-text"><span class="dd-item-label">Training Data</span><span class="dd-item-desc">Curation, sources, and methodology</span></div><svg class="dd-arrow" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M5 12h14M13 6l6 6-6 6"></path></svg></a><a class="dd-item" href="https://huggingface.co/datasets/hanzoai/zen-agentic-dataset" target="_blank" rel="noopener noreferrer"><div class="dd-item-text"><span class="dd-item-label">On HuggingFace</span><span class="dd-item-desc">Download and license the datasets</span></div><svg class="dd-arrow" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M5 12h14M13 6l6 6-6 6"></path></svg></a></div></div></div><div class="nav-item"><a class="" href="/research/">Research<svg class="nav-caret" width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M6 9l6 6 6-6"></path></svg></a><div class="nav-dropdown"><div class="nav-dropdown-inner"><a class="dd-item" href="/research/"><div class="dd-item-text"><span class="dd-item-label">Research Papers</span><span class="dd-item-desc">Technical reports across the Zen family</span></div><svg class="dd-arrow" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M5 12h14M13 6l6 6-6 6"></path></svg></a><a class="dd-item" href="https://papers.zenlm.org" target="_blank" rel="noopener noreferrer"><div class="dd-item-text"><span class="dd-item-label">Papers Archive</span><span class="dd-item-desc">Full paper archive — papers.zenlm.org</span></div><svg class="dd-arrow" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M5 12h14M13 6l6 6-6 6"></path></svg></a><a class="dd-item" href="/blog/"><div class="dd-item-text"><span class="dd-item-label">Blog</span><span class="dd-item-desc">Research notes, deep dives, and releases</span></div><svg class="dd-arrow" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M5 12h14M13 6l6 6-6 6"></path></svg></a></div></div></div><a class="active" href="/blog/">Blog</a></div><div class="logo-links"><a href="https://github.com/zenlm" target="_blank" rel="noopener noreferrer" class="icon-link" title="GitHub"><svg width="24" height="24" viewBox="0 0 24 24" fill="currentColor"><path d="M12 0c-6.626 0-12 5.373-12 12 0 5.302 3.438 9.8 8.207 11.387.599.111.793-.261.793-.577v-2.234c-3.338.726-4.033-1.416-4.033-1.416-.546-1.387-1.333-1.756-1.333-1.756-1.089-.745.083-.729.083-.729 1.205.084 1.839 1.237 1.839 1.237 1.07 1.834 2.807 1.304 3.492.997.107-.775.418-1.305.762-1.604-2.665-.305-5.467-1.334-5.467-5.931 0-1.311.469-2.381 1.236-3.221-.124-.303-.535-1.524.117-3.176 0 0 1.008-.322 3.301 1.23.957-.266 1.983-.399 3.003-.404 1.02.005 2.047.138 3.006.404 2.291-1.552 3.297-1.23 3.297-1.23.653 1.653.242 2.874.118 3.176.77.84 1.235 1.911 1.235 3.221 0 4.609-2.807 5.624-5.479 5.921.43.372.823 1.102.823 2.222v3.293c0 .319.192.694.801.576 4.765-1.589 8.199-6.086 8.199-11.386 0-6.627-5.373-12-12-12z"></path></svg></a><a href="https://huggingface.co/zenlm" target="_blank" rel="noopener noreferrer" class="icon-link" title="HuggingFace"><svg width="28" height="28" viewBox="0 0 32 32" fill="currentColor" aria-hidden="true"><circle cx="16" cy="16" r="13" fill="#d4d4d8"></circle><circle cx="11" cy="14" r="2" fill="#18181b"></circle><circle cx="21" cy="14" r="2" fill="#18181b"></circle><path d="M11 19 Q16 24 21 19" stroke="#18181b" stroke-width="1.6" fill="none" stroke-linecap="round"></path></svg></a></div><div class="nav-item try-item"><a href="https://hanzo.chat/?model=zen6" target="_blank" rel="noopener noreferrer" class="btn-try">Try Zen<svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M5 12h14M13 6l6 6-6 6"></path></svg></a><div class="nav-dropdown nav-dropdown-right"><div class="nav-dropdown-inner"><a class="dd-item" href="https://hanzo.chat/?model=zen6" target="_blank" rel="noopener noreferrer"><div class="dd-item-text"><span class="dd-item-label">Zen 6<span class="dd-tag">Available</span></span><span class="dd-item-desc">Chat with Zen 6</span></div><svg class="dd-arrow" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M5 12h14M13 6l6 6-6 6"></path></svg></a><a class="dd-item" href="https://hanzo.chat/?model=zen6-flash" target="_blank" rel="noopener noreferrer"><div class="dd-item-text"><span class="dd-item-label">Zen 6 Flash<span class="dd-tag">Available</span></span><span class="dd-item-desc">Chat with the ternary Zen 6</span></div><svg class="dd-arrow" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M5 12h14M13 6l6 6-6 6"></path></svg></a><a class="dd-item" href="https://api.hanzo.ai" target="_blank" rel="noopener noreferrer"><div class="dd-item-text"><span class="dd-item-label">Zen API</span><span class="dd-item-desc">OpenAI- & Anthropic-compatible. One key.</span></div><svg class="dd-arrow" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M5 12h14M13 6l6 6-6 6"></path></svg></a><a class="dd-item" href="https://hanzo.chat" target="_blank" rel="noopener noreferrer"><div class="dd-item-text"><span class="dd-item-label">Zen Chat</span><span class="dd-item-desc">Open the full chat experience</span></div><svg class="dd-arrow" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M5 12h14M13 6l6 6-6 6"></path></svg></a></div></div></div></div></div></nav></header><main><article class="blog-article"><a class="blog-back" href="/blog/">← Blog</a><div class="blog-post-meta">May 8, 2022<!-- --> <!-- -->·<!-- --> <!-- -->3<!-- --> min read</div><h1 class="blog-post-title">Federated Learning for Open AI</h1><p class="blog-post-lede">How federated learning enables collaborative model training while preserving data privacy.</p><div class="blog-prose"><p>Training large language models requires vast amounts of data. That data often contains sensitive information. Federated learning offers a path to train on distributed, private data without centralizing it.</p>
|
|
<h2 id="the-centralization-problem">The Centralization Problem</h2>
|
|
<p>Traditional ML training follows a simple pattern: collect data, aggregate it centrally, train models. This creates problems:</p>
|
|
<ul>
|
|
<li><strong>Privacy risk</strong>: Sensitive data leaves user control</li>
|
|
<li><strong>Legal barriers</strong>: Regulations prevent data movement across jurisdictions</li>
|
|
<li><strong>Trust requirements</strong>: Data holders must trust the training party</li>
|
|
<li><strong>Single points of failure</strong>: Central aggregation creates vulnerabilities</li>
|
|
</ul>
|
|
<h2 id="federated-learning-basics">Federated Learning Basics</h2>
|
|
<p>Federated learning inverts the pattern. Instead of bringing data to the model, we bring the model to the data.</p>
|
|
<pre><code> +-----------+
|
|
| Central |
|
|
| Server |
|
|
+-----+-----+
|
|
|
|
|
+--------------+--------------+
|
|
| | |
|
|
+-----v-----+ +-----v-----+ +-----v-----+
|
|
| Client 1 | | Client 2 | | Client N |
|
|
| (Data A) | | (Data B) | | (Data N) |
|
|
+-----------+ +-----------+ +-----------+
|
|
</code></pre>
|
|
<ol>
|
|
<li>Central server distributes model weights</li>
|
|
<li>Clients train locally on their data</li>
|
|
<li>Clients send gradient updates (not data) back</li>
|
|
<li>Server aggregates updates into improved model</li>
|
|
<li>Repeat</li>
|
|
</ol>
|
|
<p>Data never leaves client devices. Only model updates travel.</p>
|
|
<h2 id="challenges-at-scale">Challenges at Scale</h2>
|
|
<p>Federated learning for LLMs faces unique challenges:</p>
|
|
<h3 id="communication-costs">Communication Costs</h3>
|
|
<p>Model gradients are large. With billions of parameters, naive federation is impractical. We address this through:</p>
|
|
<ul>
|
|
<li><strong>Gradient compression</strong>: Sparsification and quantization reduce bandwidth by 100-1000x</li>
|
|
<li><strong>Asynchronous updates</strong>: Clients contribute when convenient, not in synchronized rounds</li>
|
|
<li><strong>Hierarchical aggregation</strong>: Regional aggregators reduce central server load</li>
|
|
</ul>
|
|
<h3 id="heterogeneous-compute">Heterogeneous Compute</h3>
|
|
<p>Participants have varied hardware. A phone differs from a workstation differs from a server. Our approach:</p>
|
|
<ul>
|
|
<li><strong>Adaptive batch sizes</strong>: Smaller devices process smaller batches</li>
|
|
<li><strong>Model sharding</strong>: Large models split across capable participants</li>
|
|
<li><strong>Contribution weighting</strong>: Update importance scales with compute contributed</li>
|
|
</ul>
|
|
<h3 id="data-heterogeneity">Data Heterogeneity</h3>
|
|
<p>Different participants have different data distributions. This creates convergence challenges. Solutions:</p>
|
|
<ul>
|
|
<li><strong>Personalization layers</strong>: Some parameters remain local</li>
|
|
<li><strong>Clustered federation</strong>: Similar participants form training groups</li>
|
|
<li><strong>Importance sampling</strong>: Under-represented distributions get higher weight</li>
|
|
</ul>
|
|
<h2 id="privacy-enhancements">Privacy Enhancements</h2>
|
|
<p>Basic federation protects raw data but gradients can leak information. We add:</p>
|
|
<h3 id="differential-privacy">Differential Privacy</h3>
|
|
<p>Noise added to gradients provides mathematical privacy guarantees. Each participant's contribution becomes statistically indistinguishable.</p>
|
|
<h3 id="secure-aggregation">Secure Aggregation</h3>
|
|
<p>Cryptographic protocols ensure the server only sees aggregated updates, not individual contributions. Even a compromised server learns nothing about specific participants.</p>
|
|
<h3 id="trusted-execution">Trusted Execution</h3>
|
|
<p>Hardware enclaves (SGX, TrustZone) provide additional isolation. Computation occurs in protected memory regions.</p>
|
|
<h2 id="zen-federation-protocol">Zen Federation Protocol</h2>
|
|
<p>We've developed a federation protocol specifically for language model training:</p>
|
|
<ol>
|
|
<li><strong>Enrollment</strong>: Participants register compute capacity and data characteristics</li>
|
|
<li><strong>Matching</strong>: Coordinator assigns participants to training cohorts</li>
|
|
<li><strong>Distribution</strong>: Model shards route to appropriate participants</li>
|
|
<li><strong>Training</strong>: Local training with privacy-preserving gradient computation</li>
|
|
<li><strong>Aggregation</strong>: Secure combination of participant updates</li>
|
|
<li><strong>Verification</strong>: Cryptographic proofs of correct computation</li>
|
|
</ol>
|
|
<p>Early benchmarks show we achieve 85% of centralized training efficiency while maintaining strong privacy guarantees.</p>
|
|
<h2 id="join-the-network">Join the Network</h2>
|
|
<p>We're opening the Zen federation network to participants. Contribute compute, contribute data (privately), contribute to open AI.</p>
|
|
<p>Requirements:</p>
|
|
<ul>
|
|
<li>Minimum 16GB RAM</li>
|
|
<li>Stable internet connection</li>
|
|
<li>Willingness to run our client software</li>
|
|
</ul>
|
|
<p>In return, participants receive:</p>
|
|
<ul>
|
|
<li>Governance tokens proportional to contribution</li>
|
|
<li>Early access to trained models</li>
|
|
<li>Recognition in model cards</li>
|
|
</ul>
|
|
<p>Details at zen.ai/federate.</p>
|
|
<hr>
|
|
<p><em>Zach Kelling is a co-founder of Zoo Labs Foundation.</em></p></div></article></main><!--$--><!--/$--><footer><div class="container"><div class="footer-content"><div class="footer-section"><h4>Zen LM</h4><p>Open models from Zoo Labs Foundation, a 501(c)(3) non-profit, for agentic coding on your own machine and marketing work. Served on api.hanzo.ai.</p></div><div class="footer-section"><h4>Zen 6</h4><ul><li><a href="/models/#zen6">Zen 6</a></li><li><a href="/models/#zen6">Zen 6 Flash</a></li><li><a href="https://hanzo.ai/research-access" target="_blank" rel="noopener noreferrer">Zen 7 research preview</a></li></ul></div><div class="footer-section"><h4>Earlier Generations</h4><ul><li><a href="/models/#zen5">Zen 5</a></li><li><a href="/models/#zen4">Zen 4</a></li><li><a href="/models/#zen3">Zen 3</a></li></ul></div><div class="footer-section"><h4>Resources</h4><ul><li><a href="/datasets/">Training Data</a></li><li><a href="https://huggingface.co/zenlm" target="_blank" rel="noopener noreferrer">HuggingFace</a></li><li><a href="https://github.com/zenlm" target="_blank" rel="noopener noreferrer">GitHub</a></li><li><a href="/research/">Research Papers</a></li><li><a href="https://api.hanzo.ai" target="_blank" rel="noopener noreferrer">Zen API</a></li></ul></div></div><div class="footer-bottom"><p>© <!-- -->2026<!-- --> Zoo Labs Foundation, a 501(c)(3) non-profit. Zen LM open models.</p></div></div></footer><script src="/_next/static/chunks/webpack-f43ad49c79db3299.js" id="_R_" async=""></script><script>(self.__next_f=self.__next_f||[]).push([0])</script><script>self.__next_f.push([1,"1:\"$Sreact.fragment\"\n2:I[6529,[\"619\",\"static/chunks/619-ba102abea3e3d0e4.js\",\"177\",\"static/chunks/app/layout-425ca81883167c16.js\"],\"default\"]\n3:I[9766,[],\"\"]\n4:I[8924,[],\"\"]\n5:I[2619,[\"619\",\"static/chunks/619-ba102abea3e3d0e4.js\",\"953\",\"static/chunks/app/blog/%5Bslug%5D/page-38f8c2a205f98d97.js\"],\"\"]\n6:I[7405,[\"619\",\"static/chunks/619-ba102abea3e3d0e4.js\",\"177\",\"static/chunks/app/layout-425ca81883167c16.js\"],\"default\"]\na:I[7150,[],\"\"]\n:HL[\"/_next/static/css/1d1f6bc532e5f43f.css\",\"style\"]\n:HL[\"/_next/static/css/eb87e4f7aea490c6.css\",\"style\"]\n"])</script><script>self.__next_f.push([1,"0:{\"P\":null,\"b\":\"-BoRG_r6Z3v9udL0lI0J_\",\"p\":\"\",\"c\":[\"\",\"blog\",\"federated-learning\",\"\"],\"i\":false,\"f\":[[[\"\",{\"children\":[\"blog\",{\"children\":[[\"slug\",\"federated-learning\",\"d\"],{\"children\":[\"__PAGE__\",{}]}]}]},\"$undefined\",\"$undefined\",true],[\"\",[\"$\",\"$1\",\"c\",{\"children\":[[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/css/1d1f6bc532e5f43f.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\",\"nonce\":\"$undefined\"}]],[\"$\",\"html\",null,{\"lang\":\"en\",\"children\":[[\"$\",\"head\",null,{\"children\":[[\"$\",\"link\",null,{\"rel\":\"icon\",\"type\":\"image/svg+xml\",\"href\":\"/favicon.svg\"}],[\"$\",\"link\",null,{\"rel\":\"alternate icon\",\"href\":\"/favicon.png\"}]]}],[\"$\",\"body\",null,{\"children\":[[\"$\",\"$L2\",null,{}],[\"$\",\"$L3\",null,{\"parallelRouterKey\":\"children\",\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L4\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":[[[\"$\",\"title\",null,{\"children\":\"404: This page could not be found.\"}],[\"$\",\"div\",null,{\"style\":{\"fontFamily\":\"system-ui,\\\"Segoe UI\\\",Roboto,Helvetica,Arial,sans-serif,\\\"Apple Color Emoji\\\",\\\"Segoe UI Emoji\\\"\",\"height\":\"100vh\",\"textAlign\":\"center\",\"display\":\"flex\",\"flexDirection\":\"column\",\"alignItems\":\"center\",\"justifyContent\":\"center\"},\"children\":[\"$\",\"div\",null,{\"children\":[[\"$\",\"style\",null,{\"dangerouslySetInnerHTML\":{\"__html\":\"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}\"}}],[\"$\",\"h1\",null,{\"className\":\"next-error-h1\",\"style\":{\"display\":\"inline-block\",\"margin\":\"0 20px 0 0\",\"padding\":\"0 23px 0 0\",\"fontSize\":24,\"fontWeight\":500,\"verticalAlign\":\"top\",\"lineHeight\":\"49px\"},\"children\":404}],[\"$\",\"div\",null,{\"style\":{\"display\":\"inline-block\"},\"children\":[\"$\",\"h2\",null,{\"style\":{\"fontSize\":14,\"fontWeight\":400,\"lineHeight\":\"49px\",\"margin\":0},\"children\":\"This page could not be found.\"}]}]]}]}]],[]],\"forbidden\":\"$undefined\",\"unauthorized\":\"$undefined\"}],[\"$\",\"footer\",null,{\"children\":[\"$\",\"div\",null,{\"className\":\"container\",\"children\":[[\"$\",\"div\",null,{\"className\":\"footer-content\",\"children\":[[\"$\",\"div\",null,{\"className\":\"footer-section\",\"children\":[[\"$\",\"h4\",null,{\"children\":\"Zen LM\"}],[\"$\",\"p\",null,{\"children\":\"Open models from Zoo Labs Foundation, a 501(c)(3) non-profit, for agentic coding on your own machine and marketing work. Served on api.hanzo.ai.\"}]]}],[\"$\",\"div\",null,{\"className\":\"footer-section\",\"children\":[[\"$\",\"h4\",null,{\"children\":\"Zen 6\"}],[\"$\",\"ul\",null,{\"children\":[[\"$\",\"li\",null,{\"children\":[\"$\",\"$L5\",null,{\"href\":\"/models#zen6\",\"children\":\"Zen 6\"}]}],[\"$\",\"li\",null,{\"children\":[\"$\",\"$L5\",null,{\"href\":\"/models#zen6\",\"children\":\"Zen 6 Flash\"}]}],[\"$\",\"li\",null,{\"children\":[\"$\",\"a\",null,{\"href\":\"https://hanzo.ai/research-access\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"children\":\"Zen 7 research preview\"}]}]]}]]}],[\"$\",\"div\",null,{\"className\":\"footer-section\",\"children\":[[\"$\",\"h4\",null,{\"children\":\"Earlier Generations\"}],[\"$\",\"ul\",null,{\"children\":[[\"$\",\"li\",null,{\"children\":[\"$\",\"$L5\",null,{\"href\":\"/models#zen5\",\"children\":\"Zen 5\"}]}],[\"$\",\"li\",null,{\"children\":[\"$\",\"$L5\",null,{\"href\":\"/models#zen4\",\"children\":\"Zen 4\"}]}],[\"$\",\"li\",null,{\"children\":[\"$\",\"$L5\",null,{\"href\":\"/models#zen3\",\"children\":\"Zen 3\"}]}]]}]]}],[\"$\",\"div\",null,{\"className\":\"footer-section\",\"children\":[[\"$\",\"h4\",null,{\"children\":\"Resources\"}],[\"$\",\"ul\",null,{\"children\":[[\"$\",\"li\",null,{\"children\":[\"$\",\"$L5\",null,{\"href\":\"/datasets\",\"children\":\"Training Data\"}]}],[\"$\",\"li\",null,{\"children\":[\"$\",\"a\",null,{\"href\":\"https://huggingface.co/zenlm\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"children\":\"HuggingFace\"}]}],[\"$\",\"li\",null,{\"children\":[\"$\",\"a\",null,{\"href\":\"https://github.com/zenlm\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"children\":\"GitHub\"}]}],[\"$\",\"li\",null,{\"children\":[\"$\",\"$L5\",null,{\"href\":\"/research\",\"children\":\"Research Papers\"}]}],[\"$\",\"li\",null,{\"children\":[\"$\",\"a\",null,{\"href\":\"https://api.hanzo.ai\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"children\":\"Zen API\"}]}]]}]]}]]}],[\"$\",\"div\",null,{\"className\":\"footer-bottom\",\"children\":[\"$\",\"p\",null,{\"children\":[\"© \",2026,\" Zoo Labs Foundation, a 501(c)(3) non-profit. Zen LM open models.\"]}]}]]}]}],[\"$\",\"$L6\",null,{}],[\"$\",\"script\",null,{\"src\":\"/assets/js/main.js\",\"async\":true}]]}]]}]]}],{\"children\":[\"blog\",[\"$\",\"$1\",\"c\",{\"children\":[null,[\"$\",\"$L3\",null,{\"parallelRouterKey\":\"children\",\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L4\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":\"$undefined\",\"forbidden\":\"$undefined\",\"unauthorized\":\"$undefined\"}]]}],{\"children\":[[\"slug\",\"federated-learning\",\"d\"],[\"$\",\"$1\",\"c\",{\"children\":[null,[\"$\",\"$L3\",null,{\"parallelRouterKey\":\"children\",\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L4\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":\"$undefined\",\"forbidden\":\"$undefined\",\"unauthorized\":\"$undefined\"}]]}],{\"children\":[\"__PAGE__\",[\"$\",\"$1\",\"c\",{\"children\":[\"$L7\",[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/css/eb87e4f7aea490c6.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\",\"nonce\":\"$undefined\"}]],\"$L8\"]}],{},null,false]},null,false]},null,false]},null,false],\"$L9\",false]],\"m\":\"$undefined\",\"G\":[\"$a\",[]],\"s\":false,\"S\":true}\n"])</script><script>self.__next_f.push([1,"b:I[4431,[],\"OutletBoundary\"]\nd:I[5278,[],\"AsyncMetadataOutlet\"]\nf:I[4431,[],\"ViewportBoundary\"]\n11:I[4431,[],\"MetadataBoundary\"]\n12:\"$Sreact.suspense\"\n8:[\"$\",\"$Lb\",null,{\"children\":[\"$Lc\",[\"$\",\"$Ld\",null,{\"promise\":\"$@e\"}]]}]\n9:[\"$\",\"$1\",\"h\",{\"children\":[null,[[\"$\",\"$Lf\",null,{\"children\":\"$L10\"}],null],[\"$\",\"$L11\",null,{\"children\":[\"$\",\"div\",null,{\"hidden\":true,\"children\":[\"$\",\"$12\",null,{\"fallback\":null,\"children\":\"$L13\"}]}]}]]}]\n"])</script><script>self.__next_f.push([1,"14:T153b,"])</script><script>self.__next_f.push([1,"\u003cp\u003eTraining large language models requires vast amounts of data. That data often contains sensitive information. Federated learning offers a path to train on distributed, private data without centralizing it.\u003c/p\u003e\n\u003ch2 id=\"the-centralization-problem\"\u003eThe Centralization Problem\u003c/h2\u003e\n\u003cp\u003eTraditional ML training follows a simple pattern: collect data, aggregate it centrally, train models. This creates problems:\u003c/p\u003e\n\u003cul\u003e\n\u003cli\u003e\u003cstrong\u003ePrivacy risk\u003c/strong\u003e: Sensitive data leaves user control\u003c/li\u003e\n\u003cli\u003e\u003cstrong\u003eLegal barriers\u003c/strong\u003e: Regulations prevent data movement across jurisdictions\u003c/li\u003e\n\u003cli\u003e\u003cstrong\u003eTrust requirements\u003c/strong\u003e: Data holders must trust the training party\u003c/li\u003e\n\u003cli\u003e\u003cstrong\u003eSingle points of failure\u003c/strong\u003e: Central aggregation creates vulnerabilities\u003c/li\u003e\n\u003c/ul\u003e\n\u003ch2 id=\"federated-learning-basics\"\u003eFederated Learning Basics\u003c/h2\u003e\n\u003cp\u003eFederated learning inverts the pattern. Instead of bringing data to the model, we bring the model to the data.\u003c/p\u003e\n\u003cpre\u003e\u003ccode\u003e +-----------+\n | Central |\n | Server |\n +-----+-----+\n |\n +--------------+--------------+\n | | |\n +-----v-----+ +-----v-----+ +-----v-----+\n | Client 1 | | Client 2 | | Client N |\n | (Data A) | | (Data B) | | (Data N) |\n +-----------+ +-----------+ +-----------+\n\u003c/code\u003e\u003c/pre\u003e\n\u003col\u003e\n\u003cli\u003eCentral server distributes model weights\u003c/li\u003e\n\u003cli\u003eClients train locally on their data\u003c/li\u003e\n\u003cli\u003eClients send gradient updates (not data) back\u003c/li\u003e\n\u003cli\u003eServer aggregates updates into improved model\u003c/li\u003e\n\u003cli\u003eRepeat\u003c/li\u003e\n\u003c/ol\u003e\n\u003cp\u003eData never leaves client devices. Only model updates travel.\u003c/p\u003e\n\u003ch2 id=\"challenges-at-scale\"\u003eChallenges at Scale\u003c/h2\u003e\n\u003cp\u003eFederated learning for LLMs faces unique challenges:\u003c/p\u003e\n\u003ch3 id=\"communication-costs\"\u003eCommunication Costs\u003c/h3\u003e\n\u003cp\u003eModel gradients are large. With billions of parameters, naive federation is impractical. We address this through:\u003c/p\u003e\n\u003cul\u003e\n\u003cli\u003e\u003cstrong\u003eGradient compression\u003c/strong\u003e: Sparsification and quantization reduce bandwidth by 100-1000x\u003c/li\u003e\n\u003cli\u003e\u003cstrong\u003eAsynchronous updates\u003c/strong\u003e: Clients contribute when convenient, not in synchronized rounds\u003c/li\u003e\n\u003cli\u003e\u003cstrong\u003eHierarchical aggregation\u003c/strong\u003e: Regional aggregators reduce central server load\u003c/li\u003e\n\u003c/ul\u003e\n\u003ch3 id=\"heterogeneous-compute\"\u003eHeterogeneous Compute\u003c/h3\u003e\n\u003cp\u003eParticipants have varied hardware. A phone differs from a workstation differs from a server. Our approach:\u003c/p\u003e\n\u003cul\u003e\n\u003cli\u003e\u003cstrong\u003eAdaptive batch sizes\u003c/strong\u003e: Smaller devices process smaller batches\u003c/li\u003e\n\u003cli\u003e\u003cstrong\u003eModel sharding\u003c/strong\u003e: Large models split across capable participants\u003c/li\u003e\n\u003cli\u003e\u003cstrong\u003eContribution weighting\u003c/strong\u003e: Update importance scales with compute contributed\u003c/li\u003e\n\u003c/ul\u003e\n\u003ch3 id=\"data-heterogeneity\"\u003eData Heterogeneity\u003c/h3\u003e\n\u003cp\u003eDifferent participants have different data distributions. This creates convergence challenges. Solutions:\u003c/p\u003e\n\u003cul\u003e\n\u003cli\u003e\u003cstrong\u003ePersonalization layers\u003c/strong\u003e: Some parameters remain local\u003c/li\u003e\n\u003cli\u003e\u003cstrong\u003eClustered federation\u003c/strong\u003e: Similar participants form training groups\u003c/li\u003e\n\u003cli\u003e\u003cstrong\u003eImportance sampling\u003c/strong\u003e: Under-represented distributions get higher weight\u003c/li\u003e\n\u003c/ul\u003e\n\u003ch2 id=\"privacy-enhancements\"\u003ePrivacy Enhancements\u003c/h2\u003e\n\u003cp\u003eBasic federation protects raw data but gradients can leak information. We add:\u003c/p\u003e\n\u003ch3 id=\"differential-privacy\"\u003eDifferential Privacy\u003c/h3\u003e\n\u003cp\u003eNoise added to gradients provides mathematical privacy guarantees. Each participant's contribution becomes statistically indistinguishable.\u003c/p\u003e\n\u003ch3 id=\"secure-aggregation\"\u003eSecure Aggregation\u003c/h3\u003e\n\u003cp\u003eCryptographic protocols ensure the server only sees aggregated updates, not individual contributions. Even a compromised server learns nothing about specific participants.\u003c/p\u003e\n\u003ch3 id=\"trusted-execution\"\u003eTrusted Execution\u003c/h3\u003e\n\u003cp\u003eHardware enclaves (SGX, TrustZone) provide additional isolation. Computation occurs in protected memory regions.\u003c/p\u003e\n\u003ch2 id=\"zen-federation-protocol\"\u003eZen Federation Protocol\u003c/h2\u003e\n\u003cp\u003eWe've developed a federation protocol specifically for language model training:\u003c/p\u003e\n\u003col\u003e\n\u003cli\u003e\u003cstrong\u003eEnrollment\u003c/strong\u003e: Participants register compute capacity and data characteristics\u003c/li\u003e\n\u003cli\u003e\u003cstrong\u003eMatching\u003c/strong\u003e: Coordinator assigns participants to training cohorts\u003c/li\u003e\n\u003cli\u003e\u003cstrong\u003eDistribution\u003c/strong\u003e: Model shards route to appropriate participants\u003c/li\u003e\n\u003cli\u003e\u003cstrong\u003eTraining\u003c/strong\u003e: Local training with privacy-preserving gradient computation\u003c/li\u003e\n\u003cli\u003e\u003cstrong\u003eAggregation\u003c/strong\u003e: Secure combination of participant updates\u003c/li\u003e\n\u003cli\u003e\u003cstrong\u003eVerification\u003c/strong\u003e: Cryptographic proofs of correct computation\u003c/li\u003e\n\u003c/ol\u003e\n\u003cp\u003eEarly benchmarks show we achieve 85% of centralized training efficiency while maintaining strong privacy guarantees.\u003c/p\u003e\n\u003ch2 id=\"join-the-network\"\u003eJoin the Network\u003c/h2\u003e\n\u003cp\u003eWe're opening the Zen federation network to participants. Contribute compute, contribute data (privately), contribute to open AI.\u003c/p\u003e\n\u003cp\u003eRequirements:\u003c/p\u003e\n\u003cul\u003e\n\u003cli\u003eMinimum 16GB RAM\u003c/li\u003e\n\u003cli\u003eStable internet connection\u003c/li\u003e\n\u003cli\u003eWillingness to run our client software\u003c/li\u003e\n\u003c/ul\u003e\n\u003cp\u003eIn return, participants receive:\u003c/p\u003e\n\u003cul\u003e\n\u003cli\u003eGovernance tokens proportional to contribution\u003c/li\u003e\n\u003cli\u003eEarly access to trained models\u003c/li\u003e\n\u003cli\u003eRecognition in model cards\u003c/li\u003e\n\u003c/ul\u003e\n\u003cp\u003eDetails at zen.ai/federate.\u003c/p\u003e\n\u003chr\u003e\n\u003cp\u003e\u003cem\u003eZach Kelling is a co-founder of Zoo Labs Foundation.\u003c/em\u003e\u003c/p\u003e"])</script><script>self.__next_f.push([1,"7:[\"$\",\"main\",null,{\"children\":[\"$\",\"article\",null,{\"className\":\"blog-article\",\"data-upstream\":\"$undefined\",\"children\":[[\"$\",\"$L5\",null,{\"className\":\"blog-back\",\"href\":\"/blog\",\"children\":\"← Blog\"}],[\"$\",\"div\",null,{\"className\":\"blog-post-meta\",\"children\":[\"May 8, 2022\",\" \",\"·\",\" \",3,\" min read\"]}],[\"$\",\"h1\",null,{\"className\":\"blog-post-title\",\"children\":\"Federated Learning for Open AI\"}],[\"$\",\"p\",null,{\"className\":\"blog-post-lede\",\"children\":\"How federated learning enables collaborative model training while preserving data privacy.\"}],[\"$\",\"div\",null,{\"className\":\"blog-prose\",\"dangerouslySetInnerHTML\":{\"__html\":\"$14\"}}]]}]}]\n"])</script><script>self.__next_f.push([1,"10:[[\"$\",\"meta\",\"0\",{\"charSet\":\"utf-8\"}],[\"$\",\"meta\",\"1\",{\"name\":\"viewport\",\"content\":\"width=device-width, initial-scale=1\"}]]\nc:null\n"])</script><script>self.__next_f.push([1,"e:{\"metadata\":[[\"$\",\"title\",\"0\",{\"children\":\"Federated Learning for Open AI — Zen Blog\"}],[\"$\",\"meta\",\"1\",{\"name\":\"description\",\"content\":\"How federated learning enables collaborative model training while preserving data privacy.\"}],[\"$\",\"meta\",\"2\",{\"name\":\"keywords\",\"content\":\"AI, LLM, Agentic AI, Code Generation, Zen Coder, Multimodal, Open Source, Machine Learning\"}]],\"error\":null,\"digest\":\"$undefined\"}\n"])</script><script>self.__next_f.push([1,"13:\"$e:metadata\"\n"])</script></body></html> |