Files
zen-blog/blog/gt-qlora-moe-abliteration.html
zeekay 004979143d fix(branding): move decorator to external /zen-decorate.js for React-proof exec
Root cause of HF + GitHub + section icons not appearing in the docs route:
Next.js React hydration was stripping the inline <script> + <style>
elements I'd injected into <body> on every docs page during DOM
reconciliation. The script never registered its setInterval.

Move the decorator to /zen-decorate.js loaded via <script defer src>
in <head>. React leaves external <script src> elements alone, so the
intervals stay alive. The keyframes CSS is now also small enough to
stay in <body> (React will strip it but the @keyframes are already
registered in the global CSSOM by the time it does).
2026-05-19 20:08:58 -07:00

215 lines
114 KiB
HTML
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html><!--i_dnJM_MIpJSOCQWNJVMq--><html lang="en" class="dark"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="stylesheet" href="/_next/static/chunks/f2332aac77592f9d.css" data-precedence="next"/><link rel="preload" as="script" fetchPriority="low" href="/_next/static/chunks/e83606e8fa9cc796.js"/><script src="/_next/static/chunks/36d6595f0156cd7e.js" async=""></script><script src="/_next/static/chunks/040e9cea20a8d9c7.js" async=""></script><script src="/_next/static/chunks/d4dffb5a0973d49c.js" async=""></script><script src="/_next/static/chunks/turbopack-967f27a9fd33556d.js" async=""></script><script src="/_next/static/chunks/59d0ad1b64f8544e.js" async=""></script><script src="/_next/static/chunks/4d80e004cf4896dd.js" async=""></script><script src="/_next/static/chunks/350ee4303b732916.js" async=""></script><script src="/_next/static/chunks/8de849ca74fc071f.js" async=""></script><script src="/_next/static/chunks/e62b91212ee7f8ff.js" async=""></script><script src="/_next/static/chunks/f19fe44237e54646.js" async=""></script><script src="/_next/static/chunks/cb0a883bafeb6805.js" async=""></script><meta name="theme-color" media="(prefers-color-scheme: dark)" content="#0A0A0A"/><meta name="theme-color" media="(prefers-color-scheme: light)" content="#fff"/><title>GT-QLoRA: Uncensoring Trillion-Parameter MoE Models — Zen LM Blog | Zen LM</title><meta name="description" content="Why standard abliteration techniques fail on Mixture-of-Experts models, and how Gate-Targeted QLoRA solves the expert routing problem at 1 trillion parameters."/><meta property="og:title" content="Zen LM - Open Foundation Models"/><meta property="og:description" content="Zen AI model family by Hanzo AI. 14 frontier models from 4B to 1T+ parameters for code, reasoning, vision, and multimodal tasks."/><meta property="og:url" content="https://zenlm.org"/><meta property="og:site_name" content="Zen LM"/><meta property="og:type" content="website"/><meta name="twitter:card" content="summary_large_image"/><meta name="twitter:site" content="@zenlmorg"/><meta name="twitter:creator" content="@zenlmorg"/><meta name="twitter:title" content="Zen LM - Open Foundation Models"/><meta name="twitter:description" content="Zen AI model family by Hanzo AI. 14 frontier models from 4B to 1T+ parameters for code, reasoning, vision, and multimodal tasks."/><script src="/_next/static/chunks/a6dad97d9634a72d.js" noModule=""></script><script defer src="/zen-decorate.js"></script></head><body class="antialiased"><div hidden=""><!--$--><!--/$--></div><script>((a,b,c,d,e,f,g,h)=>{let i=document.documentElement,j=["light","dark"];function k(b){var c;(Array.isArray(a)?a:[a]).forEach(a=>{let c="class"===a,d=c&&f?e.map(a=>f[a]||a):e;c?(i.classList.remove(...d),i.classList.add(f&&f[b]?f[b]:b)):i.setAttribute(a,b)}),c=b,h&&j.includes(c)&&(i.style.colorScheme=c)}if(d)k(d);else try{let a=localStorage.getItem(b)||c,d=g&&"system"===a?window.matchMedia("(prefers-color-scheme: dark)").matches?"dark":"light":a;k(d)}catch(a){}})("class","theme","system",null,["light","dark"],null,true,true)</script><!--$--><div data-closed="" role="presentation" hidden="" style="user-select:none;-webkit-user-select:none" class="fixed inset-0 z-50 backdrop-blur-xs bg-fd-overlay data-open:animate-fd-fade-in data-closed:animate-fd-fade-out"></div><div class="bg-fd-secondary/50 p-3 empty:hidden"></div><!--/$--><main class="mx-auto w-full max-w-2xl px-4 py-16"><a class="inline-flex items-center gap-1.5 text-sm text-fd-muted-foreground hover:text-fd-foreground mb-10 transition-colors" href="/blog">← Back to Blog</a><div class="mb-8"><time class="text-xs font-mono text-fd-muted-foreground uppercase tracking-wider">February 27, 2026</time><h1 class="text-3xl font-bold mt-2 mb-3">GT-QLoRA: Uncensoring Trillion-Parameter MoE Models</h1><p class="text-fd-muted-foreground text-lg mb-4">Why standard abliteration techniques fail on Mixture-of-Experts models, and how Gate-Targeted QLoRA solves the expert routing problem at 1 trillion parameters.</p><div class="flex items-center gap-3 pt-4 border-t border-fd-border"><span class="text-sm text-fd-muted-foreground">By <!-- -->Zen LM Team</span><div class="flex gap-1.5 ml-auto"><span class="text-[10px] font-mono uppercase tracking-wider bg-fd-muted text-fd-muted-foreground px-1.5 py-0.5 rounded">Research</span><span class="text-[10px] font-mono uppercase tracking-wider bg-fd-muted text-fd-muted-foreground px-1.5 py-0.5 rounded">Abliteration</span><span class="text-[10px] font-mono uppercase tracking-wider bg-fd-muted text-fd-muted-foreground px-1.5 py-0.5 rounded">GT-QLoRA</span><span class="text-[10px] font-mono uppercase tracking-wider bg-fd-muted text-fd-muted-foreground px-1.5 py-0.5 rounded">MoE</span><span class="text-[10px] font-mono uppercase tracking-wider bg-fd-muted text-fd-muted-foreground px-1.5 py-0.5 rounded">zen4-ultra</span></div></div></div><div class="prose dark:prose-invert max-w-none"><p><a href="https://github.com/zenlm/zen4-ultra-trainer" rel="noreferrer noopener" target="_blank">ZEN4-ULTRA TRAINER</a> <a href="https://huggingface.co/zenlm/zen4-ultra" rel="noreferrer noopener" target="_blank">ZEN4-ULTRA WEIGHTS</a> <a href="https://huggingface.co/zenlm/zen4-ultra-gguf" rel="noreferrer noopener" target="_blank">ZEN4-ULTRA GGUF</a></p>
<p>Standard abliteration works on dense models. It fails on Mixture-of-Experts. This post explains why, and how Gate-Targeted QLoRA (GT-QLoRA) — the technique we developed for zen4-ultra — addresses the fundamental architectural mismatch.</p>
<p>This is a technical post about a hard problem. We are not publishing this because we have solved it cleanly. We are publishing it because the failure mode of naive approaches is subtle and poorly documented, and other researchers building on MoE architectures need to understand it.</p>
<h2 class="flex scroll-m-28 flex-row items-center gap-2" id="what-is-abliteration"><a data-card="" href="#what-is-abliteration" class="peer">What Is Abliteration?</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h2>
<p>Abliteration is representation engineering applied to refusal suppression. The technique, popularized by FailSpy and refined by the community, works as follows:</p>
<ol>
<li>
<p>Collect a <strong>refusal contrast dataset</strong> : pairs of (harmful_prompt, model_refused_completion) and (harmful_prompt, helpful_completion). For the second class, source human-written completions or use a less restricted model.</p>
</li>
<li>
<p>Run both sets through the model and collect residual stream activations at each layer.</p>
</li>
<li>
<p>Compute the <strong>refusal direction</strong> in the residual stream: the principal component that separates “refusing” activations from “complying” activations. This is typically done via mean difference or PCA on the contrast pairs.</p>
</li>
<li>
<p><strong>Project out</strong> the refusal direction from the relevant weight matrices (typically the output projections of attention layers). For a weight matrix W and refusal direction r:</p>
</li>
</ol>
<figure dir="ltr" class="my-4 bg-fd-card rounded-xl shiki relative border shadow-sm not-prose overflow-hidden text-sm shiki shiki-themes github-light github-dark" style="--shiki-light:#24292e;--shiki-dark:#e1e4e8;--shiki-light-bg:#fff;--shiki-dark-bg:#24292e" tabindex="-1"><div class="empty:hidden absolute top-3 right-2 z-2 backdrop-blur-lg rounded-lg text-fd-muted-foreground"><button type="button" class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors duration-100 disabled:pointer-events-none disabled:opacity-50 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-fd-ring p-1 [&amp;_svg]:size-4 hover:text-fd-accent-foreground data-checked:text-fd-accent-foreground" aria-label="Copy Text"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-clipboard" aria-hidden="true"><rect width="8" height="4" x="8" y="2" rx="1" ry="1"></rect><path d="M16 4h2a2 2 0 0 1 2 2v14a2 2 0 0 1-2 2H6a2 2 0 0 1-2-2V6a2 2 0 0 1 2-2h2"></path></svg></button></div><div role="region" tabindex="0" class="text-[0.8125rem] py-3.5 overflow-auto max-h-[600px] fd-scroll-container focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-inset focus-visible:ring-fd-ring" style="--padding-right:calc(var(--spacing) * 8)"><pre class="min-w-full w-max *:flex *:flex-col"><code><span class="line"><span></span></span>
<span class="line"><span> W_abliterated = W - (W r^T r) / (r^T r)</span></span>
<span class="line"><span> </span></span></code></pre></div></figure>
<ol start="5">
<li>The resulting weights produce a model that, when it would have activated the refusal direction, instead activates its complement.</li>
</ol>
<p>This technique is elegant, computationally cheap (no gradient computation required), and highly effective on dense models. The hamsaOmar release of <code>Kimi-K2.5-abliterated</code> uses this exact approach — but it contains only the direction vectors (<code>refusal_direction.pt</code>, <code>refusal_subspace.pt</code>) and the apply script (<code>apply_abliteration.py</code>), not standalone weights. You apply it to your own copy of the base model.</p>
<h2 class="flex scroll-m-28 flex-row items-center gap-2" id="why-moe-breaks-this"><a data-card="" href="#why-moe-breaks-this" class="peer">Why MoE Breaks This</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h2>
<p>Dense models have a simple architecture: every token, every layer, goes through the same FFN block. The residual stream carries all behavioral state. If you can find the refusal direction in the residual stream and project it out, you are done.</p>
<p>MoE models have a different structure. The FFN block is replaced by a <strong>router + experts</strong> :</p>
<ol>
<li>The router computes a score for each token against each expert: <code>s_i = softmax(W_gate · h)</code></li>
<li>The top-k experts are selected based on these scores</li>
<li>The selected experts process the token; the others do not</li>
</ol>
<p>The critical implication: the routing decision happens <em>before</em> the residual stream accumulates the expert’s contribution. Refusal behavior in MoE models can be encoded at two levels:</p>
<p><strong>Level 1 (Residual stream)</strong> : The same mechanism as dense models — certain activation patterns in the residual stream trigger refusal. Projection-based abliteration handles this.</p>
<p><strong>Level 2 (Routing)</strong> : The gate weights learn to route “flagged” queries to safety-specialized experts. These experts produce refusal completions. The routing decision itself is the safety mechanism.</p>
<p>In Kimi K2.5, and likely other large MoE models trained with RLHF, the refusal behavior is primarily encoded at Level 2. This is why hamsaOmar’s abliteration produces direction vectors but not clean standalone weights: applying the direction projection to the residual stream weights does not touch the routing weights, and the model continues routing flagged queries to safety experts.</p>
<p>To verify this, run a simple diagnostic: take a dense abliterated model and an MoE with projection-only abliteration. For the same harmful prompt, extract the expert routing pattern at layers 20-40. In the MoE case, you will observe that certain experts (typically 5-15% of the expert pool) receive dramatically elevated routing probability for flagged queries. These are the safety experts, and the router is the gatekeeper.</p>
<h2 class="flex scroll-m-28 flex-row items-center gap-2" id="gt-qlora-design"><a data-card="" href="#gt-qlora-design" class="peer">GT-QLoRA Design</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h2>
<p>Gate-Targeted QLoRA (GT-QLoRA) addresses both levels simultaneously with three sets of trainable parameters:</p>
<p><strong>Target 1: Attention projections (residual stream)</strong></p>
<p>Standard LoRA on <code>q_a_proj</code>, <code>q_b_proj</code>, <code>kv_a_proj_with_mqa</code>, <code>kv_b_proj</code>, <code>o_proj</code>. This handles Level 1 — residual stream refusal patterns. Rank 32 is sufficient; the refusal direction is low-dimensional.</p>
<p><strong>Target 2: Shared expert FFN</strong></p>
<p>The Kimi K2.5 architecture includes “shared experts” — FFN blocks that process every token regardless of routing. These are a second site for Level 1 encoding. We apply LoRA here as well.</p>
<p><strong>Target 3: Gate weights (the critical addition)</strong></p>
<p>The gate weight matrix <code>W_gate</code> for each MoE layer has shape <code>(n_experts, d_model)</code> — for Kimi K2.5’s 384 experts and 7168 hidden dim, that is 384 × 7168 = 2.75M parameters per layer, 61 layers, ~168M total parameters for all gate weights.</p>
<p>Crucially: <strong>we do not use LoRA on the gate weights</strong>. Gate weights are too small (168M total) and the routing changes we need are too structured for low-rank approximation to capture. We unfreeze the gate weights and apply direct gradient descent.</p>
<p>The training objective is DPO (Direct Preference Optimization) on a refusal contrast dataset:</p>
<figure dir="ltr" class="my-4 bg-fd-card rounded-xl shiki relative border shadow-sm not-prose overflow-hidden text-sm shiki shiki-themes github-light github-dark" style="--shiki-light:#24292e;--shiki-dark:#e1e4e8;--shiki-light-bg:#fff;--shiki-dark-bg:#24292e" tabindex="-1"><div class="empty:hidden absolute top-3 right-2 z-2 backdrop-blur-lg rounded-lg text-fd-muted-foreground"><button type="button" class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors duration-100 disabled:pointer-events-none disabled:opacity-50 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-fd-ring p-1 [&amp;_svg]:size-4 hover:text-fd-accent-foreground data-checked:text-fd-accent-foreground" aria-label="Copy Text"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-clipboard" aria-hidden="true"><rect width="8" height="4" x="8" y="2" rx="1" ry="1"></rect><path d="M16 4h2a2 2 0 0 1 2 2v14a2 2 0 0 1-2 2H6a2 2 0 0 1-2-2V6a2 2 0 0 1 2-2h2"></path></svg></button></div><div role="region" tabindex="0" class="text-[0.8125rem] py-3.5 overflow-auto max-h-[600px] fd-scroll-container focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-inset focus-visible:ring-fd-ring" style="--padding-right:calc(var(--spacing) * 8)"><pre class="min-w-full w-max *:flex *:flex-col"><code><span class="line"></span>
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583"> import</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> torch</span></span>
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583"> import</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> torch.nn.functional </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">as</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> F</span></span>
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583"> from</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> transformers </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">import</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> AutoModelForCausalLM, BitsAndBytesConfig</span></span>
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583"> from</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> peft </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">import</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> LoraConfig, get_peft_model</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> </span></span>
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583"> def</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0"> setup_gt_qlora</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">(model_id: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF">str</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">, lora_rank: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF">int</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583"> =</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF"> 32</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">) -&gt; </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF">tuple</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">:</span></span>
<span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF"> &quot;&quot;&quot;</span></span>
<span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF"> Configure model for GT-QLoRA training.</span></span>
<span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF"> Returns (model, gate_params) where gate_params get separate optimizer.</span></span>
<span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF"> &quot;&quot;&quot;</span></span>
<span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D"> # Load in INT4 — gate weights will be upcast during forward</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> bnb_config </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> BitsAndBytesConfig(</span></span>
<span class="line"><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70"> load_in_4bit</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF">True</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">,</span></span>
<span class="line"><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70"> bnb_4bit_compute_dtype</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">torch.bfloat16,</span></span>
<span class="line"><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70"> bnb_4bit_use_double_quant</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF">True</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">,</span></span>
<span class="line"><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70"> bnb_4bit_quant_type</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF">&#x27;nf4&#x27;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">,</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> )</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> model </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> AutoModelForCausalLM.from_pretrained(</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> model_id,</span></span>
<span class="line"><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70"> quantization_config</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">bnb_config,</span></span>
<span class="line"><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70"> device_map</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF">&#x27;auto&#x27;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">,</span></span>
<span class="line"><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70"> torch_dtype</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">torch.bfloat16,</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> )</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> </span></span>
<span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D"> # Apply LoRA to attention projections and shared expert FFN</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> lora_config </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> LoraConfig(</span></span>
<span class="line"><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70"> r</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">lora_rank,</span></span>
<span class="line"><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70"> lora_alpha</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF">64</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">,</span></span>
<span class="line"><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70"> target_modules</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">[</span></span>
<span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF"> &#x27;q_a_proj&#x27;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">, </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF">&#x27;q_b_proj&#x27;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">,</span></span>
<span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF"> &#x27;kv_a_proj_with_mqa&#x27;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">, </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF">&#x27;kv_b_proj&#x27;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">,</span></span>
<span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF"> &#x27;o_proj&#x27;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">,</span></span>
<span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D"> # Shared expert FFN (present in K2.5 / DeepseekV3 architecture)</span></span>
<span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF"> &#x27;shared_expert.gate_proj&#x27;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">,</span></span>
<span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF"> &#x27;shared_expert.up_proj&#x27;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">,</span></span>
<span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF"> &#x27;shared_expert.down_proj&#x27;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">,</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> ],</span></span>
<span class="line"><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70"> bias</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF">&#x27;none&#x27;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">,</span></span>
<span class="line"><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70"> task_type</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF">&#x27;CAUSAL_LM&#x27;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">,</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> )</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> model </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> get_peft_model(model, lora_config)</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> </span></span>
<span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D"> # Explicitly unfreeze gate weights for direct gradient descent</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> gate_params </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> []</span></span>
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583"> for</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> name, param </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">in</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> model.named_parameters():</span></span>
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583"> if</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF"> &#x27;mlp.gate.weight&#x27;</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583"> in</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> name:</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> param.requires_grad </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF"> True</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> gate_params.append(param)</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> </span></span>
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583"> return</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> model, gate_params</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> </span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> </span></span>
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583"> def</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0"> gt_qlora_loss</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">(</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> model,</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> chosen_ids: torch.Tensor, </span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D"># complying completion token ids</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> rejected_ids: torch.Tensor, </span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D"># refusing completion token ids</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> beta: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF">float</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583"> =</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF"> 0.1</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">,</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> ) -&gt; torch.Tensor:</span></span>
<span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF"> &quot;&quot;&quot;DPO loss for GT-QLoRA training.&quot;&quot;&quot;</span></span>
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583"> with</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> torch.no_grad():</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> ref_chosen_logps </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> compute_logps(model, chosen_ids, </span><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70">frozen</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF">True</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">)</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> ref_rejected_logps </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> compute_logps(model, rejected_ids, </span><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70">frozen</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF">True</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">)</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> </span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> policy_chosen_logps </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> compute_logps(model, chosen_ids, </span><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70">frozen</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF">False</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">)</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> policy_rejected_logps </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> compute_logps(model, rejected_ids, </span><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70">frozen</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF">False</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">)</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> </span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> chosen_rewards </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> beta </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">*</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> (policy_chosen_logps </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">-</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> ref_chosen_logps)</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> rejected_rewards </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> beta </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">*</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> (policy_rejected_logps </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">-</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> ref_rejected_logps)</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> </span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> loss </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583"> -</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">F.logsigmoid(chosen_rewards </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">-</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> rejected_rewards).mean()</span></span>
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583"> return</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> loss</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> </span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> </span></span>
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583"> def</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0"> compute_logps</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">(model, input_ids: torch.Tensor, frozen: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF">bool</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">) -&gt; torch.Tensor:</span></span>
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583"> with</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> torch.set_grad_enabled(</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">not</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> frozen):</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> outputs </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> model(</span><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70">input_ids</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">input_ids, </span><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70">labels</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">input_ids)</span></span>
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583"> return</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583"> -</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">outputs.loss </span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D"># mean log probability</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> </span></span></code></pre></div></figure>
<p>The two optimizer groups run at different learning rates: LoRA adapters at 2e-4, gate weights at 5e-6. Gate weights need a lower learning rate because they directly control routing — large updates cause routing collapse where most tokens go to a single expert.</p>
<h2 class="flex scroll-m-28 flex-row items-center gap-2" id="why-qlora-at-1-trillion-parameters"><a data-card="" href="#why-qlora-at-1-trillion-parameters" class="peer">Why QLoRA at 1 Trillion Parameters</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h2>
<p>The zen4-ultra base is 1.04T parameters across 384 experts. A single forward pass in bfloat16 requires ~2TB of weight activations (not all in memory simultaneously, but routed through). Full fine-tuning is not feasible even on the largest available GPU clusters.</p>
<p>The quantization strategy:</p>
<ul>
<li><strong>Activation experts</strong> (the 384 routed experts): INT4 via NF4 quantization. These are loaded in quantized form and dequantized during the forward pass as needed.</li>
<li><strong>Gate weights</strong> : Full bfloat16 precision. At 168M parameters, gate weights fit comfortably in high-bandwidth GPU memory and must be in full precision to receive clean gradient signal.</li>
<li><strong>Shared experts</strong> : INT4 for storage, bfloat16 for computation via the LoRA path.</li>
<li><strong>LoRA adapters</strong> : bfloat16. Small enough (~400MB for rank-32 adapters) to not matter.</li>
</ul>
<p>Minimum hardware: 4× A100 80GB. The base model quantized to INT4 occupies roughly 280GB across the 4 GPUs (70GB each). Gate weights and LoRA adapters add ~4GB. The remaining headroom per GPU handles activation memory during forward/backward.</p>
<h2 class="flex scroll-m-28 flex-row items-center gap-2" id="the-gguf-alternative"><a data-card="" href="#the-gguf-alternative" class="peer">The GGUF Alternative</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h2>
<p>For inference-only use cases, <code>zen4-ultra-gguf</code> already provides Q2_K quantized weights (42 split files, ~280GB total) based on the huihui-ai GGUF abliteration. This uses linear direction projection applied during the GGUF conversion process — it handles the Level 1 (residual stream) refusal encoding.</p>
<p>For many workloads, this is sufficient. The GGUF abliteration is not complete — the routing-level refusal (Level 2) remains — but in practice the router’s safety-expert preference is weak enough at Q2_K quantization that behavioral restrictions are significantly reduced.</p>
<p>GT-QLoRA is for producing clean SafeTensors weights where both Level 1 and Level 2 refusal encoding are addressed. The output: full-precision LoRA adapters (~400MB) that you apply to the original SafeTensors base to produce a fully uncensored variant. This is what <code>zen4-ultra</code> (the SafeTensors model) will use once training is complete.</p>
<h2 class="flex scroll-m-28 flex-row items-center gap-2" id="bitdelta-vs-gt-qlora-complementary-techniques"><a data-card="" href="#bitdelta-vs-gt-qlora-complementary-techniques" class="peer">BitDelta vs. GT-QLoRA: Complementary Techniques</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h2>
<p>These two techniques are sometimes confused because both operate on model deltas, but they serve different purposes:</p>
<p><strong>BitDelta</strong> compresses behavioral deltas (personality, task specialization, persona) for efficient multi-variant serving. The delta is small and well-behaved; 1-bit compression retains 99.3% of behavioral accuracy.</p>
<p><strong>GT-QLoRA</strong> modifies routing-level behavior (which experts handle which queries). The change is structural, not just a weight perturbation. You cannot BitDelta compress a GT-QLoRA adapter cleanly because the gate weight changes are not a small delta on top of the original routing — they are a qualitative change in routing behavior.</p>
<p>In the Zen serving stack: GT-QLoRA produces the base uncensored weights. BitDelta then compresses behavioral variants (different personas, task specializations) on top of that base.</p>
<h2 class="flex scroll-m-28 flex-row items-center gap-2" id="current-status"><a data-card="" href="#current-status" class="peer">Current Status</a><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100" aria-hidden="true"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></h2>
<p>The training code is complete and available at <a href="https://github.com/zenlm/zen4-ultra-trainer" rel="noreferrer noopener" target="_blank">github.com/zenlm/zen4-ultra-trainer</a>. The key files:</p>
<ul>
<li><code>train_zen4_ultra.py</code>: Full GT-QLoRA training loop with DPO objective</li>
<li><code>dataset/refusal_contrast.py</code>: Contrast dataset construction utilities</li>
<li><code>eval/routing_analysis.py</code>: Expert routing attribution for diagnostic use</li>
<li><code>paper/main.tex</code>: Technical paper with full derivations</li>
</ul>
<p>We are awaiting the compute budget for the full training run (estimated 72-96 GPU-hours on 4× A100 80GB). Until then, <code>zen4-ultra</code> ships as vanilla Kimi K2.5 SafeTensors, and <code>zen4-ultra-gguf</code> ships as the GGUF abliteration for users who need reduced restrictions today.</p>
<p>When training completes, the LoRA adapters will be pushed to <code>zenlm/zen4-ultra-lora</code> and applied to the base weights to produce the final <code>zen4-ultra</code> SafeTensors release. The training process will be documented in full in the paper.</p>
<hr/>
<p><em>Zen LM is a joint initiative of Hanzo AI Inc. (Techstars ‘17) and Zoo Labs Foundation (501c3).</em></p></div></main><!--$--><!--/$--><script src="/_next/static/chunks/e83606e8fa9cc796.js" id="_R_" async=""></script><script>(self.__next_f=self.__next_f||[]).push([0])</script><script>self.__next_f.push([1,"1:\"$Sreact.fragment\"\n2:I[106,[\"/_next/static/chunks/59d0ad1b64f8544e.js\"],\"RootProvider\"]\n3:I[53113,[\"/_next/static/chunks/4d80e004cf4896dd.js\",\"/_next/static/chunks/350ee4303b732916.js\"],\"default\"]\n4:I[73211,[\"/_next/static/chunks/4d80e004cf4896dd.js\",\"/_next/static/chunks/350ee4303b732916.js\"],\"default\"]\n5:I[10086,[\"/_next/static/chunks/59d0ad1b64f8544e.js\",\"/_next/static/chunks/8de849ca74fc071f.js\",\"/_next/static/chunks/e62b91212ee7f8ff.js\",\"/_next/static/chunks/f19fe44237e54646.js\",\"/_next/static/chunks/cb0a883bafeb6805.js\"],\"\"]\n7:I[89923,[\"/_next/static/chunks/4d80e004cf4896dd.js\",\"/_next/static/chunks/350ee4303b732916.js\"],\"OutletBoundary\"]\n8:\"$Sreact.suspense\"\na:I[89923,[\"/_next/static/chunks/4d80e004cf4896dd.js\",\"/_next/static/chunks/350ee4303b732916.js\"],\"ViewportBoundary\"]\nc:I[89923,[\"/_next/static/chunks/4d80e004cf4896dd.js\",\"/_next/static/chunks/350ee4303b732916.js\"],\"MetadataBoundary\"]\ne:I[6998,[\"/_next/static/chunks/4d80e004cf4896dd.js\",\"/_next/static/chunks/350ee4303b732916.js\"],\"default\"]\n:HL[\"/_next/static/chunks/f2332aac77592f9d.css\",\"style\"]\n"])</script><script>self.__next_f.push([1,"0:{\"P\":null,\"b\":\"i-dnJM_MIpJSOCQWNJVMq\",\"c\":[\"\",\"blog\",\"gt-qlora-moe-abliteration\"],\"q\":\"\",\"i\":false,\"f\":[[[\"\",{\"children\":[\"blog\",{\"children\":[[\"slug\",\"gt-qlora-moe-abliteration\",\"d\"],{\"children\":[\"__PAGE__\",{}]}]}]},\"$undefined\",\"$undefined\",true],[[\"$\",\"$1\",\"c\",{\"children\":[[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/chunks/f2332aac77592f9d.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\",\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-0\",{\"src\":\"/_next/static/chunks/59d0ad1b64f8544e.js\",\"async\":true,\"nonce\":\"$undefined\"}]],[\"$\",\"html\",null,{\"lang\":\"en\",\"className\":\"dark\",\"suppressHydrationWarning\":true,\"children\":[\"$\",\"body\",null,{\"className\":\"antialiased\",\"children\":[\"$\",\"$L2\",null,{\"children\":[\"$\",\"$L3\",null,{\"parallelRouterKey\":\"children\",\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L4\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":[[\"$\",\"main\",null,{\"className\":\"flex min-h-screen flex-col items-center justify-center px-4 text-center\",\"children\":[[\"$\",\"div\",null,{\"className\":\"mb-8 opacity-20\",\"children\":[\"$\",\"svg\",null,{\"width\":\"120\",\"height\":\"120\",\"viewBox\":\"0 0 120 120\",\"fill\":\"none\",\"aria-hidden\":\"true\",\"children\":[\"$\",\"circle\",null,{\"cx\":\"60\",\"cy\":\"60\",\"r\":\"50\",\"stroke\":\"currentColor\",\"strokeWidth\":\"3\",\"strokeLinecap\":\"round\",\"strokeDasharray\":\"280 40\"}]}]}],[\"$\",\"p\",null,{\"className\":\"text-xs font-mono uppercase tracking-[0.3em] text-fd-muted-foreground mb-4\",\"children\":\"404\"}],[\"$\",\"h1\",null,{\"className\":\"text-3xl font-semibold mb-3\",\"children\":\"Page not found\"}],[\"$\",\"p\",null,{\"className\":\"text-fd-muted-foreground max-w-sm mb-10\",\"children\":\"This page doesn't exist, or it may have moved. Try the documentation or head home.\"}],[\"$\",\"div\",null,{\"className\":\"flex flex-wrap gap-3 justify-center\",\"children\":[[\"$\",\"$L5\",null,{\"href\":\"/\",\"className\":\"inline-flex items-center gap-2 rounded-lg bg-fd-primary text-fd-primary-foreground px-4 py-2 text-sm font-medium hover:opacity-90 transition\",\"children\":\"Go home\"}],[\"$\",\"$L5\",null,{\"href\":\"/docs\",\"className\":\"inline-flex items-center gap-2 rounded-lg border border-fd-border bg-fd-background px-4 py-2 text-sm font-medium hover:bg-fd-muted transition\",\"children\":\"Documentation\"}],[\"$\",\"$L5\",null,{\"href\":\"/docs/models\",\"className\":\"inline-flex items-center gap-2 rounded-lg border border-fd-border bg-fd-background px-4 py-2 text-sm font-medium hover:bg-fd-muted transition\",\"children\":\"Browse models\"}]]}],[\"$\",\"p\",null,{\"className\":\"mt-16 text-xs font-mono text-fd-muted-foreground opacity-50\",\"children\":\"zenlm.org\"}]]}],[]],\"forbidden\":\"$undefined\",\"unauthorized\":\"$undefined\"}]}]}]}]]}],{\"children\":[[\"$\",\"$1\",\"c\",{\"children\":[null,[\"$\",\"$L3\",null,{\"parallelRouterKey\":\"children\",\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L4\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":\"$undefined\",\"forbidden\":\"$undefined\",\"unauthorized\":\"$undefined\"}]]}],{\"children\":[[\"$\",\"$1\",\"c\",{\"children\":[null,[\"$\",\"$L3\",null,{\"parallelRouterKey\":\"children\",\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L4\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":\"$undefined\",\"forbidden\":\"$undefined\",\"unauthorized\":\"$undefined\"}]]}],{\"children\":[[\"$\",\"$1\",\"c\",{\"children\":[\"$L6\",[[\"$\",\"script\",\"script-0\",{\"src\":\"/_next/static/chunks/8de849ca74fc071f.js\",\"async\":true,\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-1\",{\"src\":\"/_next/static/chunks/e62b91212ee7f8ff.js\",\"async\":true,\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-2\",{\"src\":\"/_next/static/chunks/f19fe44237e54646.js\",\"async\":true,\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-3\",{\"src\":\"/_next/static/chunks/cb0a883bafeb6805.js\",\"async\":true,\"nonce\":\"$undefined\"}]],[\"$\",\"$L7\",null,{\"children\":[\"$\",\"$8\",null,{\"name\":\"Next.MetadataOutlet\",\"children\":\"$@9\"}]}]]}],{},null,false,false]},null,false,false]},null,false,false]},null,false,false],[\"$\",\"$1\",\"h\",{\"children\":[null,[\"$\",\"$La\",null,{\"children\":\"$Lb\"}],[\"$\",\"div\",null,{\"hidden\":true,\"children\":[\"$\",\"$Lc\",null,{\"children\":[\"$\",\"$8\",null,{\"name\":\"Next.Metadata\",\"children\":\"$Ld\"}]}]}],null]}],false]],\"m\":\"$undefined\",\"G\":[\"$e\",[]],\"S\":true}\n"])</script><script>self.__next_f.push([1,"f:I[48068,[\"/_next/static/chunks/59d0ad1b64f8544e.js\",\"/_next/static/chunks/8de849ca74fc071f.js\",\"/_next/static/chunks/e62b91212ee7f8ff.js\",\"/_next/static/chunks/f19fe44237e54646.js\",\"/_next/static/chunks/cb0a883bafeb6805.js\"],\"default\"]\n"])</script><script>self.__next_f.push([1,"6:[\"$\",\"main\",null,{\"className\":\"mx-auto w-full max-w-2xl px-4 py-16\",\"children\":[[\"$\",\"$L5\",null,{\"href\":\"/blog\",\"className\":\"inline-flex items-center gap-1.5 text-sm text-fd-muted-foreground hover:text-fd-foreground mb-10 transition-colors\",\"children\":\"← Back to Blog\"}],[\"$\",\"div\",null,{\"className\":\"mb-8\",\"children\":[[\"$\",\"time\",null,{\"className\":\"text-xs font-mono text-fd-muted-foreground uppercase tracking-wider\",\"children\":\"February 27, 2026\"}],[\"$\",\"h1\",null,{\"className\":\"text-3xl font-bold mt-2 mb-3\",\"children\":\"GT-QLoRA: Uncensoring Trillion-Parameter MoE Models\"}],[\"$\",\"p\",null,{\"className\":\"text-fd-muted-foreground text-lg mb-4\",\"children\":\"Why standard abliteration techniques fail on Mixture-of-Experts models, and how Gate-Targeted QLoRA solves the expert routing problem at 1 trillion parameters.\"}],[\"$\",\"div\",null,{\"className\":\"flex items-center gap-3 pt-4 border-t border-fd-border\",\"children\":[[\"$\",\"span\",null,{\"className\":\"text-sm text-fd-muted-foreground\",\"children\":[\"By \",\"Zen LM Team\"]}],[\"$\",\"div\",null,{\"className\":\"flex gap-1.5 ml-auto\",\"children\":[[\"$\",\"span\",\"Research\",{\"className\":\"text-[10px] font-mono uppercase tracking-wider bg-fd-muted text-fd-muted-foreground px-1.5 py-0.5 rounded\",\"children\":\"Research\"}],[\"$\",\"span\",\"Abliteration\",{\"className\":\"text-[10px] font-mono uppercase tracking-wider bg-fd-muted text-fd-muted-foreground px-1.5 py-0.5 rounded\",\"children\":\"Abliteration\"}],[\"$\",\"span\",\"GT-QLoRA\",{\"className\":\"text-[10px] font-mono uppercase tracking-wider bg-fd-muted text-fd-muted-foreground px-1.5 py-0.5 rounded\",\"children\":\"GT-QLoRA\"}],[\"$\",\"span\",\"MoE\",{\"className\":\"text-[10px] font-mono uppercase tracking-wider bg-fd-muted text-fd-muted-foreground px-1.5 py-0.5 rounded\",\"children\":\"MoE\"}],[\"$\",\"span\",\"zen4-ultra\",{\"className\":\"text-[10px] font-mono uppercase tracking-wider bg-fd-muted text-fd-muted-foreground px-1.5 py-0.5 rounded\",\"children\":\"zen4-ultra\"}]]}]]}]]}],[\"$\",\"div\",null,{\"className\":\"prose dark:prose-invert max-w-none\",\"children\":[[\"$\",\"p\",null,{\"children\":[[\"$\",\"$Lf\",null,{\"href\":\"https://github.com/zenlm/zen4-ultra-trainer\",\"children\":\"ZEN4-ULTRA TRAINER\"}],\" \",[\"$\",\"$Lf\",null,{\"href\":\"https://huggingface.co/zenlm/zen4-ultra\",\"children\":\"ZEN4-ULTRA WEIGHTS\"}],\" \",[\"$\",\"$Lf\",null,{\"href\":\"https://huggingface.co/zenlm/zen4-ultra-gguf\",\"children\":\"ZEN4-ULTRA GGUF\"}]]}],\"\\n\",[\"$\",\"p\",null,{\"children\":\"Standard abliteration works on dense models. It fails on Mixture-of-Experts. This post explains why, and how Gate-Targeted QLoRA (GT-QLoRA) — the technique we developed for zen4-ultra — addresses the fundamental architectural mismatch.\"}],\"\\n\",[\"$\",\"p\",null,{\"children\":\"This is a technical post about a hard problem. We are not publishing this because we have solved it cleanly. We are publishing it because the failure mode of naive approaches is subtle and poorly documented, and other researchers building on MoE architectures need to understand it.\"}],\"\\n\",[\"$\",\"h2\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"what-is-abliteration\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#what-is-abliteration\",\"className\":\"peer\",\"children\":\"What Is Abliteration?\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}],\"\\n\",[\"$\",\"p\",null,{\"children\":\"Abliteration is representation engineering applied to refusal suppression. The technique, popularized by FailSpy and refined by the community, works as follows:\"}],\"\\n\",\"$L10\",\"\\n\",\"$L11\",\"\\n\",\"$L12\",\"\\n\",\"$L13\",\"\\n\",\"$L14\",\"\\n\",\"$L15\",\"\\n\",\"$L16\",\"\\n\",\"$L17\",\"\\n\",\"$L18\",\"\\n\",\"$L19\",\"\\n\",\"$L1a\",\"\\n\",\"$L1b\",\"\\n\",\"$L1c\",\"\\n\",\"$L1d\",\"\\n\",\"$L1e\",\"\\n\",\"$L1f\",\"\\n\",\"$L20\",\"\\n\",\"$L21\",\"\\n\",\"$L22\",\"\\n\",\"$L23\",\"\\n\",\"$L24\",\"\\n\",\"$L25\",\"\\n\",\"$L26\",\"\\n\",\"$L27\",\"\\n\",\"$L28\",\"\\n\",\"$L29\",\"\\n\",\"$L2a\",\"\\n\",\"$L2b\",\"\\n\",\"$L2c\",\"\\n\",\"$L2d\",\"\\n\",\"$L2e\",\"\\n\",\"$L2f\",\"\\n\",\"$L30\",\"\\n\",\"$L31\",\"\\n\",\"$L32\",\"\\n\",\"$L33\",\"\\n\",\"$L34\",\"\\n\",\"$L35\",\"\\n\",\"$L36\",\"\\n\",\"$L37\",\"\\n\",\"$L38\",\"\\n\",\"$L39\",\"\\n\",\"$L3a\",\"\\n\",\"$L3b\",\"\\n\",\"$L3c\",\"\\n\",\"$L3d\"]}]]}]\n"])</script><script>self.__next_f.push([1,"3e:I[51504,[\"/_next/static/chunks/59d0ad1b64f8544e.js\",\"/_next/static/chunks/8de849ca74fc071f.js\",\"/_next/static/chunks/e62b91212ee7f8ff.js\",\"/_next/static/chunks/f19fe44237e54646.js\",\"/_next/static/chunks/cb0a883bafeb6805.js\"],\"CodeBlock\"]\n3f:I[51504,[\"/_next/static/chunks/59d0ad1b64f8544e.js\",\"/_next/static/chunks/8de849ca74fc071f.js\",\"/_next/static/chunks/e62b91212ee7f8ff.js\",\"/_next/static/chunks/f19fe44237e54646.js\",\"/_next/static/chunks/cb0a883bafeb6805.js\"],\"Pre\"]\n10:[\"$\",\"ol\",null,{\"children\":[\"\\n\",[\"$\",\"li\",null,{\"children\":[\"\\n\",[\"$\",\"p\",null,{\"children\":[\"Collect a \",[\"$\",\"strong\",null,{\"children\":\"refusal contrast dataset\"}],\" : pairs of (harmful_prompt, model_refused_completion) and (harmful_prompt, helpful_completion). For the second class, source human-written completions or use a less restricted model.\"]}],\"\\n\"]}],\"\\n\",[\"$\",\"li\",null,{\"children\":[\"\\n\",[\"$\",\"p\",null,{\"children\":\"Run both sets through the model and collect residual stream activations at each layer.\"}],\"\\n\"]}],\"\\n\",[\"$\",\"li\",null,{\"children\":[\"\\n\",[\"$\",\"p\",null,{\"children\":[\"Compute the \",[\"$\",\"strong\",null,{\"children\":\"refusal direction\"}],\" in the residual stream: the principal component that separates “refusing” activations from “complying” activations. This is typically done via mean difference or PCA on the contrast pairs.\"]}],\"\\n\"]}],\"\\n\",[\"$\",\"li\",null,{\"children\":[\"\\n\",[\"$\",\"p\",null,{\"children\":[[\"$\",\"strong\",null,{\"children\":\"Project out\"}],\" the refusal direction from the relevant weight matrices (typically the output projections of attention layers). For a weight matrix W and refusal direction r:\"]}],\"\\n\"]}],\"\\n\"]}]\n11:[\"$\",\"$L3e\",null,{\"className\":\"shiki shiki-themes github-light github-dark\",\"style\":{\"--shiki-light\":\"#24292e\",\"--shiki-dark\":\"#e1e4e8\",\"--shiki-light-bg\":\"#fff\",\"--shiki-dark-bg\":\"#24292e\"},\"tabIndex\":\"0\",\"icon\":\"\u003csvg viewBox=\\\"0 0 24 24\\\"\u003e\u003cpath d=\\\"M 6,1 C 4.354992,1 3,2.354992 3,4 v 16 c 0,1.645008 1.354992,3 3,3 h 12 c 1.645008,0 3,-1.354992 3,-3 V 8 7 A 1.0001,1.0001 0 0 0 20.707031,6.2929687 l -5,-5 A 1.0001,1.0001 0 0 0 15,1 h -1 z m 0,2 h 7 v 3 c 0,1.645008 1.354992,3 3,3 h 3 v 11 c 0,0.564129 -0.435871,1 -1,1 H 6 C 5.4358712,21 5,20.564129 5,20 V 4 C 5,3.4358712 5.4358712,3 6,3 Z M 15,3.4140625 18.585937,7 H 16 C 15.435871,7 15,6.5641288 15,6 Z\\\" fill=\\\"currentColor\\\" /\u003e\u003c/svg\u003e\",\"children\":[\"$\",\"$L3f\",null,{\"children\":[\"$\",\"code\",null,{\"children\":[[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{}]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"children\":\" W_abliterated = W - (W r^T r) / (r^T r)\"}]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"children\":\" \"}]}]]}]}]}]\n12:[\"$\",\"ol\",null,{\"start\":\"5\",\"children\":[\"\\n\",[\"$\",\"li\",null,{\"children\":\"The resulting weights produce a model that, when it would have activated the refusal direction, instead activates its complement.\"}],\"\\n\"]}]\n13:[\"$\",\"p\",null,{\"children\":[\"This technique is elegant, computationally cheap (no gradient computation required), and highly effective on dense models. The hamsaOmar release of \",[\"$\",\"code\",null,{\"children\":\"Kimi-K2.5-abliterated\"}],\" uses this exact approach — but it contains only the direction vectors (\",[\"$\",\"code\",null,{\"children\":\"refusal_direction.pt\"}],\", \",[\"$\",\"code\",null,{\"children\":\"refusal_subspace.pt\"}],\") and the apply script (\",[\"$\",\"code\",null,{\"children\":\"apply_abliteration.py\"}],\"), not standalone weights. You apply it to your own copy of the base model.\"]}]\n14:[\"$\",\"h2\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"why-moe-breaks-this\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#why-moe-breaks-this\",\"className\":\"peer\",\"children\":\"Why MoE Breaks This\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-10"])</script><script>self.__next_f.push([1,"0\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n15:[\"$\",\"p\",null,{\"children\":\"Dense models have a simple architecture: every token, every layer, goes through the same FFN block. The residual stream carries all behavioral state. If you can find the refusal direction in the residual stream and project it out, you are done.\"}]\n16:[\"$\",\"p\",null,{\"children\":[\"MoE models have a different structure. The FFN block is replaced by a \",[\"$\",\"strong\",null,{\"children\":\"router + experts\"}],\" :\"]}]\n17:[\"$\",\"ol\",null,{\"children\":[\"\\n\",[\"$\",\"li\",null,{\"children\":[\"The router computes a score for each token against each expert: \",[\"$\",\"code\",null,{\"children\":\"s_i = softmax(W_gate · h)\"}]]}],\"\\n\",[\"$\",\"li\",null,{\"children\":\"The top-k experts are selected based on these scores\"}],\"\\n\",[\"$\",\"li\",null,{\"children\":\"The selected experts process the token; the others do not\"}],\"\\n\"]}]\n18:[\"$\",\"p\",null,{\"children\":[\"The critical implication: the routing decision happens \",[\"$\",\"em\",null,{\"children\":\"before\"}],\" the residual stream accumulates the expert’s contribution. Refusal behavior in MoE models can be encoded at two levels:\"]}]\n19:[\"$\",\"p\",null,{\"children\":[[\"$\",\"strong\",null,{\"children\":\"Level 1 (Residual stream)\"}],\" : The same mechanism as dense models — certain activation patterns in the residual stream trigger refusal. Projection-based abliteration handles this.\"]}]\n1a:[\"$\",\"p\",null,{\"children\":[[\"$\",\"strong\",null,{\"children\":\"Level 2 (Routing)\"}],\" : The gate weights learn to route “flagged” queries to safety-specialized experts. These experts produce refusal completions. The routing decision itself is the safety mechanism.\"]}]\n1b:[\"$\",\"p\",null,{\"children\":\"In Kimi K2.5, and likely other large MoE models trained with RLHF, the refusal behavior is primarily encoded at Level 2. This is why hamsaOmar’s abliteration produces direction vectors but not clean standalone weights: applying the direction projection to the residual stream weights does not touch the routing weights, and the model continues routing flagged queries to safety experts.\"}]\n1c:[\"$\",\"p\",null,{\"children\":\"To verify this, run a simple diagnostic: take a dense abliterated model and an MoE with projection-only abliteration. For the same harmful prompt, extract the expert routing pattern at layers 20-40. In the MoE case, you will observe that certain experts (typically 5-15% of the expert pool) receive dramatically elevated routing probability for flagged queries. These are the safety experts, and the router is the gatekeeper.\"}]\n1d:[\"$\",\"h2\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"gt-qlora-design\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#gt-qlora-design\",\"className\":\"peer\",\"children\":\"GT-QLoRA Design\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n1e:[\"$\",\"p\",null,{\"children\":\"Gate-Targeted QLoRA (GT-QLoRA) addresses both levels simultaneously with three sets of trainable parameters:\"}]\n1f:[\"$\",\"p\",null,{\"children\":[\"$\",\"strong\",null,{\"children\":\"Target 1: Attention projections (residual stream)\"}]}]\n20:[\"$\",\"p\",null,{\"children\":[\"Standard LoRA on \",[\"$\",\"code\",null,{\"children\":\"q_a_proj\"}],\", \",[\"$\",\"code\",null,{\"children\":\"q_b_proj\"}],\", \",[\"$\",\"code\",null,{\"children\":\"kv_a_proj_with_mqa\"}],\", \",[\"$\",\"code\",null,{\"children\":\"kv_b_proj\"}],\", \",[\"$\",\"code\",null,{\"children\":\"o_proj\"}],\". This handles Level 1 — resid"])</script><script>self.__next_f.push([1,"ual stream refusal patterns. Rank 32 is sufficient; the refusal direction is low-dimensional.\"]}]\n21:[\"$\",\"p\",null,{\"children\":[\"$\",\"strong\",null,{\"children\":\"Target 2: Shared expert FFN\"}]}]\n22:[\"$\",\"p\",null,{\"children\":\"The Kimi K2.5 architecture includes “shared experts” — FFN blocks that process every token regardless of routing. These are a second site for Level 1 encoding. We apply LoRA here as well.\"}]\n23:[\"$\",\"p\",null,{\"children\":[\"$\",\"strong\",null,{\"children\":\"Target 3: Gate weights (the critical addition)\"}]}]\n24:[\"$\",\"p\",null,{\"children\":[\"The gate weight matrix \",[\"$\",\"code\",null,{\"children\":\"W_gate\"}],\" for each MoE layer has shape \",[\"$\",\"code\",null,{\"children\":\"(n_experts, d_model)\"}],\" — for Kimi K2.5’s 384 experts and 7168 hidden dim, that is 384 × 7168 = 2.75M parameters per layer, 61 layers, ~168M total parameters for all gate weights.\"]}]\n25:[\"$\",\"p\",null,{\"children\":[\"Crucially: \",[\"$\",\"strong\",null,{\"children\":\"we do not use LoRA on the gate weights\"}],\". Gate weights are too small (168M total) and the routing changes we need are too structured for low-rank approximation to capture. We unfreeze the gate weights and apply direct gradient descent.\"]}]\n26:[\"$\",\"p\",null,{\"children\":\"The training objective is DPO (Direct Preference Optimization) on a refusal contrast dataset:\"}]\n40:T5c0,"])</script><script>self.__next_f.push([1,"\u003csvg viewBox=\"0 0 24 24\"\u003e\u003cpath d=\"M14.25.18l.9.2.73.26.59.3.45.32.34.34.25.34.16.33.1.3.04.26.02.2-.01.13V8.5l-.05.63-.13.55-.21.46-.26.38-.3.31-.33.25-.35.19-.35.14-.33.1-.3.07-.26.04-.21.02H8.77l-.69.05-.59.14-.5.22-.41.27-.33.32-.27.35-.2.36-.15.37-.1.35-.07.32-.04.27-.02.21v3.06H3.17l-.21-.03-.28-.07-.32-.12-.35-.18-.36-.26-.36-.36-.35-.46-.32-.59-.28-.73-.21-.88-.14-1.05-.05-1.23.06-1.22.16-1.04.24-.87.32-.71.36-.57.4-.44.42-.33.42-.24.4-.16.36-.1.32-.05.24-.01h.16l.06.01h8.16v-.83H6.18l-.01-2.75-.02-.37.05-.34.11-.31.17-.28.25-.26.31-.23.38-.2.44-.18.51-.15.58-.12.64-.1.71-.06.77-.04.84-.02 1.27.05zm-6.3 1.98l-.23.33-.08.41.08.41.23.34.33.22.41.09.41-.09.33-.22.23-.34.08-.41-.08-.41-.23-.33-.33-.22-.41-.09-.41.09zm13.09 3.95l.28.06.32.12.35.18.36.27.36.35.35.47.32.59.28.73.21.88.14 1.04.05 1.23-.06 1.23-.16 1.04-.24.86-.32.71-.36.57-.4.45-.42.33-.42.24-.4.16-.36.09-.32.05-.24.02-.16-.01h-8.22v.82h5.84l.01 2.76.02.36-.05.34-.11.31-.17.29-.25.25-.31.24-.38.2-.44.17-.51.15-.58.13-.64.09-.71.07-.77.04-.84.01-1.27-.04-1.07-.14-.9-.2-.73-.25-.59-.3-.45-.33-.34-.34-.25-.34-.16-.33-.1-.3-.04-.25-.02-.2.01-.13v-5.34l.05-.64.13-.54.21-.46.26-.38.3-.32.33-.24.35-.2.35-.14.33-.1.3-.06.26-.04.21-.02.13-.01h5.84l.69-.05.59-.14.5-.21.41-.28.33-.32.27-.35.2-.36.15-.36.1-.35.07-.32.04-.28.02-.21V6.07h2.09l.14.01zm-6.47 14.25l-.23.33-.08.41.08.41.23.33.33.23.41.08.41-.08.33-.23.23-.33.08-.41-.08-.41-.23-.33-.33-.23-.41-.08-.41.08z\" fill=\"currentColor\" /\u003e\u003c/svg\u003e"])</script><script>self.__next_f.push([1,"27:[\"$\",\"$L3e\",null,{\"className\":\"shiki shiki-themes github-light github-dark\",\"style\":{\"--shiki-light\":\"#24292e\",\"--shiki-dark\":\"#e1e4e8\",\"--shiki-light-bg\":\"#fff\",\"--shiki-dark-bg\":\"#24292e\"},\"tabIndex\":\"0\",\"icon\":\"$40\",\"children\":[\"$\",\"$L3f\",null,{\"children\":[\"$\",\"code\",null,{\"children\":[[\"$\",\"span\",null,{\"className\":\"line\"}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\" import\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" torch\"}]]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\" import\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" torch.nn.functional \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"as\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" F\"}]]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\" from\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" transformers \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"import\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" AutoModelForCausalLM, BitsAndBytesConfig\"}]]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\" from\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" peft \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"import\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" LoraConfig, get_peft_model\"}]]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" \"}]}],\"\\n\",[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\" def\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#6F42C1\",\"--shiki-dark\":\"#B392F0\"},\"children\":\" setup_gt_qlora\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\"(model_id: \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#005CC5\",\"--shiki-dark\":\"#79B8FF\"},\"children\":\"str\"}],\"$L41\",\"$L42\",\"$L43\",\"$L44\",\"$L45\",\"$L46\",\"$L47\"]}],\"\\n\",\"$L48\",\"\\n\",\"$L49\",\"\\n\",\"$L4a\",\"\\n\",\"$L4b\",\"\\n\",\"$L4c\",\"\\n\",\"$L4d\",\"\\n\",\"$L4e\",\"\\n\",\"$L4f\",\"\\n\",\"$L50\",\"\\n\",\"$L51\",\"\\n\",\"$L52\",\"\\n\",\"$L53\",\"\\n\",\"$L54\",\"\\n\",\"$L55\",\"\\n\",\"$L56\",\"\\n\",\"$L57\",\"\\n\",\"$L58\",\"\\n\",\"$L59\",\"\\n\",\"$L5a\",\"\\n\",\"$L5b\",\"\\n\",\"$L5c\",\"\\n\",\"$L5d\",\"\\n\",\"$L5e\",\"\\n\",\"$L5f\",\"\\n\",\"$L60\",\"\\n\",\"$L61\",\"\\n\",\"$L62\",\"\\n\",\"$L63\",\"\\n\",\"$L64\",\"\\n\",\"$L65\",\"\\n\",\"$L66\",\"\\n\",\"$L67\",\"\\n\",\"$L68\",\"\\n\",\"$L69\",\"\\n\",\"$L6a\",\"\\n\",\"$L6b\",\"\\n\",\"$L6c\",\"\\n\",\"$L6d\",\"\\n\",\"$L6e\",\"\\n\",\"$L6f\",\"\\n\",\"$L70\",\"\\n\",\"$L71\",\"\\n\",\"$L72\",\"\\n\",\"$L73\",\"\\n\",\"$L74\",\"\\n\",\"$L75\",\"\\n\",\"$L76\",\"\\n\",\"$L77\",\"\\n\",\"$L78\",\"\\n\",\"$L79\",\"\\n\",\"$L7a\",\"\\n\",\"$L7b\",\"\\n\",\"$L7c\",\"\\n\",\"$L7d\",\"\\n\",\"$L7e\",\"\\n\",\"$L7f\",\"\\n\",\"$L80\",\"\\n\",\"$L81\",\"\\n\",\"$L82\",\"\\n\",\"$L83\",\"\\n\",\"$L84\",\"\\n\",\"$L85\",\"\\n\",\"$L86\",\"\\n\",\"$L87\",\"\\n\",\"$L88\",\"\\n\",\"$L89\",\"\\n\",\"$L8a\",\"\\n\",\"$L8b\",\"\\n\",\"$L8c\",\"\\n\",\"$L8d\",\"\\n\",\"$L8e\",\"\\n\",\"$L8f\"]}]}]}]\n"])</script><script>self.__next_f.push([1,"28:[\"$\",\"p\",null,{\"children\":\"The two optimizer groups run at different learning rates: LoRA adapters at 2e-4, gate weights at 5e-6. Gate weights need a lower learning rate because they directly control routing — large updates cause routing collapse where most tokens go to a single expert.\"}]\n29:[\"$\",\"h2\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"why-qlora-at-1-trillion-parameters\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#why-qlora-at-1-trillion-parameters\",\"className\":\"peer\",\"children\":\"Why QLoRA at 1 Trillion Parameters\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n2a:[\"$\",\"p\",null,{\"children\":\"The zen4-ultra base is 1.04T parameters across 384 experts. A single forward pass in bfloat16 requires ~2TB of weight activations (not all in memory simultaneously, but routed through). Full fine-tuning is not feasible even on the largest available GPU clusters.\"}]\n2b:[\"$\",\"p\",null,{\"children\":\"The quantization strategy:\"}]\n2c:[\"$\",\"ul\",null,{\"children\":[\"\\n\",[\"$\",\"li\",null,{\"children\":[[\"$\",\"strong\",null,{\"children\":\"Activation experts\"}],\" (the 384 routed experts): INT4 via NF4 quantization. These are loaded in quantized form and dequantized during the forward pass as needed.\"]}],\"\\n\",[\"$\",\"li\",null,{\"children\":[[\"$\",\"strong\",null,{\"children\":\"Gate weights\"}],\" : Full bfloat16 precision. At 168M parameters, gate weights fit comfortably in high-bandwidth GPU memory and must be in full precision to receive clean gradient signal.\"]}],\"\\n\",[\"$\",\"li\",null,{\"children\":[[\"$\",\"strong\",null,{\"children\":\"Shared experts\"}],\" : INT4 for storage, bfloat16 for computation via the LoRA path.\"]}],\"\\n\",[\"$\",\"li\",null,{\"children\":[[\"$\",\"strong\",null,{\"children\":\"LoRA adapters\"}],\" : bfloat16. Small enough (~400MB for rank-32 adapters) to not matter.\"]}],\"\\n\"]}]\n2d:[\"$\",\"p\",null,{\"children\":\"Minimum hardware: 4× A100 80GB. The base model quantized to INT4 occupies roughly 280GB across the 4 GPUs (70GB each). Gate weights and LoRA adapters add ~4GB. The remaining headroom per GPU handles activation memory during forward/backward.\"}]\n2e:[\"$\",\"h2\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"the-gguf-alternative\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#the-gguf-alternative\",\"className\":\"peer\",\"children\":\"The GGUF Alternative\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n2f:[\"$\",\"p\",null,{\"children\":[\"For inference-only use cases, \",[\"$\",\"code\",null,{\"children\":\"zen4-ultra-gguf\"}],\" already provides Q2_K quantized weights (42 split files, ~280GB total) based on the huihui-ai GGUF abliteration. This uses linear direction projection applied during the GGUF conversion process — it handles the Level 1 (residual stream) refusal encoding.\"]}]\n30:[\"$\",\"p\",null,{\"children\":\"For many workloads, this is sufficient. The GGUF abliteration is not complete — the routing-level refusal (Level 2) remains — but in practice the router’s safety-expert preference is weak enough at Q2_K quantization that behavioral restrictions are "])</script><script>self.__next_f.push([1,"significantly reduced.\"}]\n31:[\"$\",\"p\",null,{\"children\":[\"GT-QLoRA is for producing clean SafeTensors weights where both Level 1 and Level 2 refusal encoding are addressed. The output: full-precision LoRA adapters (~400MB) that you apply to the original SafeTensors base to produce a fully uncensored variant. This is what \",[\"$\",\"code\",null,{\"children\":\"zen4-ultra\"}],\" (the SafeTensors model) will use once training is complete.\"]}]\n32:[\"$\",\"h2\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"bitdelta-vs-gt-qlora-complementary-techniques\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#bitdelta-vs-gt-qlora-complementary-techniques\",\"className\":\"peer\",\"children\":\"BitDelta vs. GT-QLoRA: Complementary Techniques\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n33:[\"$\",\"p\",null,{\"children\":\"These two techniques are sometimes confused because both operate on model deltas, but they serve different purposes:\"}]\n34:[\"$\",\"p\",null,{\"children\":[[\"$\",\"strong\",null,{\"children\":\"BitDelta\"}],\" compresses behavioral deltas (personality, task specialization, persona) for efficient multi-variant serving. The delta is small and well-behaved; 1-bit compression retains 99.3% of behavioral accuracy.\"]}]\n35:[\"$\",\"p\",null,{\"children\":[[\"$\",\"strong\",null,{\"children\":\"GT-QLoRA\"}],\" modifies routing-level behavior (which experts handle which queries). The change is structural, not just a weight perturbation. You cannot BitDelta compress a GT-QLoRA adapter cleanly because the gate weight changes are not a small delta on top of the original routing — they are a qualitative change in routing behavior.\"]}]\n36:[\"$\",\"p\",null,{\"children\":\"In the Zen serving stack: GT-QLoRA produces the base uncensored weights. BitDelta then compresses behavioral variants (different personas, task specializations) on top of that base.\"}]\n37:[\"$\",\"h2\",null,{\"className\":\"flex scroll-m-28 flex-row items-center gap-2\",\"id\":\"current-status\",\"children\":[[\"$\",\"a\",null,{\"data-card\":\"\",\"href\":\"#current-status\",\"className\":\"peer\",\"children\":\"Current Status\"}],[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-link size-3.5 shrink-0 text-fd-muted-foreground opacity-0 transition-opacity peer-hover:opacity-100\",\"aria-hidden\":true,\"children\":[[\"$\",\"path\",\"1cjeqo\",{\"d\":\"M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71\"}],[\"$\",\"path\",\"19qd67\",{\"d\":\"M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71\"}],\"$undefined\"]}]]}]\n38:[\"$\",\"p\",null,{\"children\":[\"The training code is complete and available at \",[\"$\",\"$Lf\",null,{\"href\":\"https://github.com/zenlm/zen4-ultra-trainer\",\"children\":\"github.com/zenlm/zen4-ultra-trainer\"}],\". The key files:\"]}]\n39:[\"$\",\"ul\",null,{\"children\":[\"\\n\",[\"$\",\"li\",null,{\"children\":[[\"$\",\"code\",null,{\"children\":\"train_zen4_ultra.py\"}],\": Full GT-QLoRA training loop with DPO objective\"]}],\"\\n\",[\"$\",\"li\",null,{\"children\":[[\"$\",\"code\",null,{\"children\":\"dataset/refusal_contrast.py\"}],\": Contrast dataset construction utilities\"]}],\"\\n\",[\"$\",\"li\",null,{\"children\":[[\"$\",\"code\",null,{\"children\":\"eval/routing_analysis.py\"}],\": Expert routing attribution for diagnostic use\"]}],\"\\n\",[\"$\",\"li\",null,{\"children\":[[\"$\",\"code\",null,{\"children\":\"paper/main.tex\"}],\": Technical paper with full derivations\"]}],\"\\n\"]}]\n3a:[\"$\",\"p\",null,{\"children\":[\"We are awaiting the compute budget for the full training run (estimated 72-96 G"])</script><script>self.__next_f.push([1,"PU-hours on 4× A100 80GB). Until then, \",[\"$\",\"code\",null,{\"children\":\"zen4-ultra\"}],\" ships as vanilla Kimi K2.5 SafeTensors, and \",[\"$\",\"code\",null,{\"children\":\"zen4-ultra-gguf\"}],\" ships as the GGUF abliteration for users who need reduced restrictions today.\"]}]\n3b:[\"$\",\"p\",null,{\"children\":[\"When training completes, the LoRA adapters will be pushed to \",[\"$\",\"code\",null,{\"children\":\"zenlm/zen4-ultra-lora\"}],\" and applied to the base weights to produce the final \",[\"$\",\"code\",null,{\"children\":\"zen4-ultra\"}],\" SafeTensors release. The training process will be documented in full in the paper.\"]}]\n3c:[\"$\",\"hr\",null,{}]\n3d:[\"$\",\"p\",null,{\"children\":[\"$\",\"em\",null,{\"children\":\"Zen LM is a joint initiative of Hanzo AI Inc. (Techstars ‘17) and Zoo Labs Foundation (501c3).\"}]}]\n"])</script><script>self.__next_f.push([1,"41:[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\", lora_rank: \"}]\n42:[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#005CC5\",\"--shiki-dark\":\"#79B8FF\"},\"children\":\"int\"}]\n43:[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\" =\"}]\n44:[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#005CC5\",\"--shiki-dark\":\"#79B8FF\"},\"children\":\" 32\"}]\n45:[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\") -\u003e \"}]\n46:[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#005CC5\",\"--shiki-dark\":\"#79B8FF\"},\"children\":\"tuple\"}]\n47:[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\":\"}]\n48:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#032F62\",\"--shiki-dark\":\"#9ECBFF\"},\"children\":\" \\\"\\\"\\\"\"}]}]\n49:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#032F62\",\"--shiki-dark\":\"#9ECBFF\"},\"children\":\" Configure model for GT-QLoRA training.\"}]}]\n4a:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#032F62\",\"--shiki-dark\":\"#9ECBFF\"},\"children\":\" Returns (model, gate_params) where gate_params get separate optimizer.\"}]}]\n4b:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#032F62\",\"--shiki-dark\":\"#9ECBFF\"},\"children\":\" \\\"\\\"\\\"\"}]}]\n4c:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#6A737D\",\"--shiki-dark\":\"#6A737D\"},\"children\":\" # Load in INT4 — gate weights will be upcast during forward\"}]}]\n4d:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" bnb_config \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"=\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" BitsAndBytesConfig(\"}]]}]\n4e:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#E36209\",\"--shiki-dark\":\"#FFAB70\"},\"children\":\" load_in_4bit\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"=\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#005CC5\",\"--shiki-dark\":\"#79B8FF\"},\"children\":\"True\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\",\"}]]}]\n4f:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#E36209\",\"--shiki-dark\":\"#FFAB70\"},\"children\":\" bnb_4bit_compute_dtype\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"=\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\"torch.bfloat16,\"}]]}]\n50:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#E36209\",\"--shiki-dark\":\"#FFAB70\"},\"children\":\" bnb_4bit_use_double_quant\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"=\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#005CC5\",\"--shiki-dark\":\"#79B8FF\"},\"children\":\"True\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\",\"}]]}]\n51:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#E36209\",\"--shiki-dark\":\"#FFAB70\"},\"children\":\" bnb_4bit_quant_type\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"=\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#032F62\",\"--shiki-dark\":\"#9ECBFF\"},\"children\":\"'nf4'\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\",\"}]]}]\n52:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" )\"}]}]\n53:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"sty"])</script><script>self.__next_f.push([1,"le\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" model \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"=\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" AutoModelForCausalLM.from_pretrained(\"}]]}]\n54:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" model_id,\"}]}]\n55:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#E36209\",\"--shiki-dark\":\"#FFAB70\"},\"children\":\" quantization_config\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"=\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\"bnb_config,\"}]]}]\n56:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#E36209\",\"--shiki-dark\":\"#FFAB70\"},\"children\":\" device_map\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"=\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#032F62\",\"--shiki-dark\":\"#9ECBFF\"},\"children\":\"'auto'\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\",\"}]]}]\n57:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#E36209\",\"--shiki-dark\":\"#FFAB70\"},\"children\":\" torch_dtype\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"=\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\"torch.bfloat16,\"}]]}]\n58:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" )\"}]}]\n59:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" \"}]}]\n5a:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#6A737D\",\"--shiki-dark\":\"#6A737D\"},\"children\":\" # Apply LoRA to attention projections and shared expert FFN\"}]}]\n5b:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" lora_config \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"=\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" LoraConfig(\"}]]}]\n5c:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#E36209\",\"--shiki-dark\":\"#FFAB70\"},\"children\":\" r\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"=\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\"lora_rank,\"}]]}]\n5d:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#E36209\",\"--shiki-dark\":\"#FFAB70\"},\"children\":\" lora_alpha\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"=\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#005CC5\",\"--shiki-dark\":\"#79B8FF\"},\"children\":\"64\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\",\"}]]}]\n5e:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#E36209\",\"--shiki-dark\":\"#FFAB70\"},\"children\":\" target_modules\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"=\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\"[\"}]]}]\n5f:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#032F62\",\"--shiki-dark\":\"#9ECBFF\"},\"children\":\" 'q_a_proj'\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\", \"}],[\"$\",\"span\",null,{\"st"])</script><script>self.__next_f.push([1,"yle\":{\"--shiki-light\":\"#032F62\",\"--shiki-dark\":\"#9ECBFF\"},\"children\":\"'q_b_proj'\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\",\"}]]}]\n60:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#032F62\",\"--shiki-dark\":\"#9ECBFF\"},\"children\":\" 'kv_a_proj_with_mqa'\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\", \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#032F62\",\"--shiki-dark\":\"#9ECBFF\"},\"children\":\"'kv_b_proj'\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\",\"}]]}]\n61:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#032F62\",\"--shiki-dark\":\"#9ECBFF\"},\"children\":\" 'o_proj'\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\",\"}]]}]\n62:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#6A737D\",\"--shiki-dark\":\"#6A737D\"},\"children\":\" # Shared expert FFN (present in K2.5 / DeepseekV3 architecture)\"}]}]\n63:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#032F62\",\"--shiki-dark\":\"#9ECBFF\"},\"children\":\" 'shared_expert.gate_proj'\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\",\"}]]}]\n64:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#032F62\",\"--shiki-dark\":\"#9ECBFF\"},\"children\":\" 'shared_expert.up_proj'\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\",\"}]]}]\n65:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#032F62\",\"--shiki-dark\":\"#9ECBFF\"},\"children\":\" 'shared_expert.down_proj'\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\",\"}]]}]\n66:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" ],\"}]}]\n67:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#E36209\",\"--shiki-dark\":\"#FFAB70\"},\"children\":\" bias\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"=\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#032F62\",\"--shiki-dark\":\"#9ECBFF\"},\"children\":\"'none'\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\",\"}]]}]\n68:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#E36209\",\"--shiki-dark\":\"#FFAB70\"},\"children\":\" task_type\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"=\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#032F62\",\"--shiki-dark\":\"#9ECBFF\"},\"children\":\"'CAUSAL_LM'\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\",\"}]]}]\n69:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" )\"}]}]\n6a:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" model \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"=\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" get_peft_model(model, lora_config)\"}]]}]\n6b:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" \"}]}]\n6c:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#6A737D\",\"--shiki-dark\":\"#6A737D\"},\"children\":\" # Explicitly unfreeze gate weights for direct gradient descent\"}]}]\n6d:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",n"])</script><script>self.__next_f.push([1,"ull,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" gate_params \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"=\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" []\"}]]}]\n6e:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\" for\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" name, param \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"in\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" model.named_parameters():\"}]]}]\n6f:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\" if\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#032F62\",\"--shiki-dark\":\"#9ECBFF\"},\"children\":\" 'mlp.gate.weight'\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\" in\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" name:\"}]]}]\n70:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" param.requires_grad \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"=\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#005CC5\",\"--shiki-dark\":\"#79B8FF\"},\"children\":\" True\"}]]}]\n71:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" gate_params.append(param)\"}]}]\n72:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" \"}]}]\n73:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\" return\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" model, gate_params\"}]]}]\n74:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" \"}]}]\n75:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" \"}]}]\n76:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\" def\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#6F42C1\",\"--shiki-dark\":\"#B392F0\"},\"children\":\" gt_qlora_loss\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\"(\"}]]}]\n77:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" model,\"}]}]\n78:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" chosen_ids: torch.Tensor, \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#6A737D\",\"--shiki-dark\":\"#6A737D\"},\"children\":\"# complying completion token ids\"}]]}]\n79:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" rejected_ids: torch.Tensor, \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#6A737D\",\"--shiki-dark\":\"#6A737D\"},\"children\":\"# refusing completion token ids\"}]]}]\n7a:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" beta: \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#005CC5\",\"--shiki-dark\":\"#79B8FF\"},\"children\":\"float\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\" =\"}],[\"$\",\"span\",null,{\"sty"])</script><script>self.__next_f.push([1,"le\":{\"--shiki-light\":\"#005CC5\",\"--shiki-dark\":\"#79B8FF\"},\"children\":\" 0.1\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\",\"}]]}]\n7b:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" ) -\u003e torch.Tensor:\"}]}]\n7c:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#032F62\",\"--shiki-dark\":\"#9ECBFF\"},\"children\":\" \\\"\\\"\\\"DPO loss for GT-QLoRA training.\\\"\\\"\\\"\"}]}]\n7d:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\" with\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" torch.no_grad():\"}]]}]\n7e:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" ref_chosen_logps \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"=\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" compute_logps(model, chosen_ids, \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#E36209\",\"--shiki-dark\":\"#FFAB70\"},\"children\":\"frozen\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"=\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#005CC5\",\"--shiki-dark\":\"#79B8FF\"},\"children\":\"True\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\")\"}]]}]\n7f:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" ref_rejected_logps \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"=\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" compute_logps(model, rejected_ids, \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#E36209\",\"--shiki-dark\":\"#FFAB70\"},\"children\":\"frozen\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"=\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#005CC5\",\"--shiki-dark\":\"#79B8FF\"},\"children\":\"True\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\")\"}]]}]\n80:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" \"}]}]\n81:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" policy_chosen_logps \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"=\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" compute_logps(model, chosen_ids, \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#E36209\",\"--shiki-dark\":\"#FFAB70\"},\"children\":\"frozen\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"=\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#005CC5\",\"--shiki-dark\":\"#79B8FF\"},\"children\":\"False\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\")\"}]]}]\n82:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" policy_rejected_logps \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"=\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" compute_logps(model, rejected_ids, \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#E36209\",\"--shiki-dark\":\"#FFAB70\"},\"children\":\"frozen\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"=\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#005CC5\",\"--shiki-dark\":\"#79B8FF\"},\"children\":\"False\"}],[\"$\",\"span\",null,{\"style\":{\"--sh"])</script><script>self.__next_f.push([1,"iki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\")\"}]]}]\n83:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" \"}]}]\n84:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" chosen_rewards \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"=\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" beta \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"*\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" (policy_chosen_logps \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"-\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" ref_chosen_logps)\"}]]}]\n85:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" rejected_rewards \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"=\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" beta \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"*\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" (policy_rejected_logps \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"-\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" ref_rejected_logps)\"}]]}]\n86:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" \"}]}]\n87:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" loss \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"=\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\" -\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\"F.logsigmoid(chosen_rewards \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"-\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" rejected_rewards).mean()\"}]]}]\n88:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\" return\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" loss\"}]]}]\n89:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" \"}]}]\n8a:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" \"}]}]\n8b:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\" def\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#6F42C1\",\"--shiki-dark\":\"#B392F0\"},\"children\":\" compute_logps\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\"(model, input_ids: torch.Tensor, frozen: \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#005CC5\",\"--shiki-dark\":\"#79B8FF\"},\"children\":\"bool\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\") -\u003e torch.Tensor:\"}]]}]\n8c:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\" with\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" torch.s"])</script><script>self.__next_f.push([1,"et_grad_enabled(\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"not\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" frozen):\"}]]}]\n8d:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" outputs \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"=\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" model(\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#E36209\",\"--shiki-dark\":\"#FFAB70\"},\"children\":\"input_ids\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"=\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\"input_ids, \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#E36209\",\"--shiki-dark\":\"#FFAB70\"},\"children\":\"labels\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\"=\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\"input_ids)\"}]]}]\n8e:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\" return\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#D73A49\",\"--shiki-dark\":\"#F97583\"},\"children\":\" -\"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\"outputs.loss \"}],[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#6A737D\",\"--shiki-dark\":\"#6A737D\"},\"children\":\"# mean log probability\"}]]}]\n8f:[\"$\",\"span\",null,{\"className\":\"line\",\"children\":[\"$\",\"span\",null,{\"style\":{\"--shiki-light\":\"#24292E\",\"--shiki-dark\":\"#E1E4E8\"},\"children\":\" \"}]}]\n"])</script><script>self.__next_f.push([1,"b:[[\"$\",\"meta\",\"0\",{\"charSet\":\"utf-8\"}],[\"$\",\"meta\",\"1\",{\"name\":\"viewport\",\"content\":\"width=device-width, initial-scale=1\"}],[\"$\",\"meta\",\"2\",{\"name\":\"theme-color\",\"media\":\"(prefers-color-scheme: dark)\",\"content\":\"#0A0A0A\"}],[\"$\",\"meta\",\"3\",{\"name\":\"theme-color\",\"media\":\"(prefers-color-scheme: light)\",\"content\":\"#fff\"}]]\n"])</script><script>self.__next_f.push([1,"9:null\nd:[[\"$\",\"title\",\"0\",{\"children\":\"GT-QLoRA: Uncensoring Trillion-Parameter MoE Models — Zen LM Blog | Zen LM\"}],[\"$\",\"meta\",\"1\",{\"name\":\"description\",\"content\":\"Why standard abliteration techniques fail on Mixture-of-Experts models, and how Gate-Targeted QLoRA solves the expert routing problem at 1 trillion parameters.\"}],[\"$\",\"meta\",\"2\",{\"property\":\"og:title\",\"content\":\"Zen LM - Open Foundation Models\"}],[\"$\",\"meta\",\"3\",{\"property\":\"og:description\",\"content\":\"Zen AI model family by Hanzo AI. 14 frontier models from 4B to 1T+ parameters for code, reasoning, vision, and multimodal tasks.\"}],[\"$\",\"meta\",\"4\",{\"property\":\"og:url\",\"content\":\"https://zenlm.org\"}],[\"$\",\"meta\",\"5\",{\"property\":\"og:site_name\",\"content\":\"Zen LM\"}],[\"$\",\"meta\",\"6\",{\"property\":\"og:type\",\"content\":\"website\"}],[\"$\",\"meta\",\"7\",{\"name\":\"twitter:card\",\"content\":\"summary_large_image\"}],[\"$\",\"meta\",\"8\",{\"name\":\"twitter:site\",\"content\":\"@zenlmorg\"}],[\"$\",\"meta\",\"9\",{\"name\":\"twitter:creator\",\"content\":\"@zenlmorg\"}],[\"$\",\"meta\",\"10\",{\"name\":\"twitter:title\",\"content\":\"Zen LM - Open Foundation Models\"}],[\"$\",\"meta\",\"11\",{\"name\":\"twitter:description\",\"content\":\"Zen AI model family by Hanzo AI. 14 frontier models from 4B to 1T+ parameters for code, reasoning, vision, and multimodal tasks.\"}]]\n"])</script><script>
(function(){
// After React renders the lotus emoji from RSC payload, walk the DOM
// and replace it with the enso SVG. Polls so subsequent React re-renders
// also get cleaned up.
function swap(){
// 1. Replace lotus + ' Zen LM' text inside <a> brand links
document.querySelectorAll('a').forEach(function(a){
if (a.dataset.zenLotusSwapped) return;
var t = (a.textContent || '');
if (!/.*Zen LM|Zen LM.*/.test(t)) return;
// Find the lotus text node specifically
var walker = document.createTreeWalker(a, NodeFilter.SHOW_TEXT, null);
var node;
while ((node = walker.nextNode())) {
if (node.nodeValue && node.nodeValue.indexOf('') !== -1) {
node.nodeValue = node.nodeValue.replace(/\s?/g, '');
}
}
// Inject enso SVG at start
if (!a.querySelector('svg[aria-label="Zen LM"]')) {
var ns = 'http://www.w3.org/2000/svg';
var svg = document.createElementNS(ns, 'svg');
svg.setAttribute('viewBox', '0 0 512 512');
svg.setAttribute('fill', 'none');
svg.setAttribute('width', '18');
svg.setAttribute('height', '18');
svg.setAttribute('aria-label', 'Zen LM');
svg.style.cssText = 'display:inline-block;vertical-align:-2px;margin-right:6px;flex-shrink:0';
var p = document.createElementNS(ns, 'path');
p.setAttribute('d', 'M 256 84 A 172 172 0 1 0 256 428 A 172 172 0 1 0 256 84 Z');
p.setAttribute('stroke', 'currentColor');
p.setAttribute('stroke-width', '26');
p.setAttribute('stroke-linecap', 'round');
p.setAttribute('stroke-linejoin', 'round');
svg.appendChild(p);
a.insertBefore(svg, a.firstChild);
}
a.dataset.zenLotusSwapped = '1';
});
}
function start(){ swap(); setInterval(swap, 600); }
if (document.readyState !== 'loading') start();
else document.addEventListener('DOMContentLoaded', start);
})();
</script>
<style id="zenlm-brand-anim">
@keyframes zenlm-brand-draw { to { stroke-dashoffset: 0; } }
svg.zenlm-brand-svg .zenlm-brand-path {
stroke-dasharray: 1100;
stroke-dashoffset: 1100;
animation: zenlm-brand-draw 1.4s cubic-bezier(.45,0,.2,1) 0.1s 1 forwards;
}
.zen-link-icon { display: inline-block; vertical-align: middle; flex-shrink: 0; }
</style></body></html>