mirror of
https://github.com/hanzoai/skills.git
synced 2026-11-19 00:47:36 +00:00
hanzoskills.com/llms.txt now opens with the install line, the API reference index and the company index, so an agent that lands on any one of the three can find the rest. robots.txt claimed 537 documents while llms.txt counts 920. The number is gone from robots.txt; llms.txt derives it. public/ is regenerated from scripts/site.py, which it had fallen behind: the copy-prompt menu and agent bar were in the generator and not in the committed pages. tests/site.mjs passes against the result. Co-authored-by: Hanzo Dev <[email protected]>
389 lines
23 KiB
HTML
389 lines
23 KiB
HTML
<!doctype html>
|
|
<html lang="en"><head>
|
|
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
<title>hanzo-sho</title>
|
|
<meta name="color-scheme" content="dark">
|
|
<style>
|
|
/* Committed to one visual world: a dark instrument panel. No light variant —
|
|
the dot field and the hairline rules only read on black, and a naive
|
|
inversion would wash both out. Every colour is painted explicitly so the page
|
|
never borrows a host background. */
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Pixel Square';src:url('/fonts/ZenPixel-Square.woff2') format('woff2');font-style:normal;font-display:swap}
|
|
|
|
:root{
|
|
--ink:#f4f4f2; --soft:#9c9c98; --faint:#6a6a66; --ground:#000; --panel:#0a0a0a;
|
|
--line:#232323; --line-hi:#3a3a3a; --accent:#f4f4f2;
|
|
--font-zen-sans:'Zen',-apple-system,BlinkMacSystemFont,"Segoe UI",Helvetica,Arial,sans-serif;
|
|
--font-zen-mono:'Zen Mono',ui-monospace,"SF Mono",SFMono-Regular,Menlo,Consolas,monospace;
|
|
--font-zen-pixel-square:'Zen Pixel Square',monospace;
|
|
--mono:var(--font-zen-mono);
|
|
--sans:var(--font-zen-sans);
|
|
/* Controls are pills and surfaces are soft, as on hanzo.ai. */
|
|
--r:999px; --r-soft:14px;
|
|
}
|
|
*{box-sizing:border-box}
|
|
html{-webkit-text-size-adjust:100%}
|
|
body{margin:0;background:var(--ground);color:var(--ink);font-family:var(--sans);
|
|
font-size:16px;line-height:1.65;-webkit-font-smoothing:antialiased}
|
|
.wrap{max-width:60rem;margin:0 auto;padding:0 1.75rem}
|
|
|
|
/* The dot field. A halftone made of one repeating radial-gradient, faded out
|
|
along the diagonal by a mask so it reads as a wave rather than wallpaper. */
|
|
/* No overflow clip here: the dot field is inset to the box already, and a clip
|
|
would cut off the copy-prompt menu. */
|
|
.hero{position:relative;border-bottom:1px solid var(--line)}
|
|
.hero::before{content:"";position:absolute;inset:0;
|
|
background-image:radial-gradient(var(--line-hi) 1px, transparent 1px);
|
|
background-size:7px 7px;
|
|
-webkit-mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
pointer-events:none}
|
|
.hero .wrap{position:relative;padding-top:5.5rem;padding-bottom:4rem}
|
|
|
|
nav{border-bottom:1px solid var(--line)}
|
|
nav .in{max-width:60rem;margin:0 auto;padding:1.1rem 1.75rem;display:flex;
|
|
justify-content:space-between;align-items:center;gap:1rem}
|
|
.brand{font-family:var(--mono);font-size:.85rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--ink);text-decoration:none;
|
|
display:inline-flex;align-items:center;gap:.6rem}
|
|
.brand i{font-style:normal;color:var(--faint)}
|
|
.mark{width:1.15em;height:1.15em;display:block;flex:0 0 auto}
|
|
nav a.up{font-family:var(--mono);font-size:.72rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--soft);text-decoration:none}
|
|
nav a.up:hover{color:var(--ink)}
|
|
|
|
h1{font-family:var(--mono);font-size:clamp(2rem,5.4vw,3.4rem);line-height:1.12;
|
|
letter-spacing:-.02em;margin:0 0 1.1rem;font-weight:500;text-wrap:balance}
|
|
h1 em{font-style:normal;color:var(--faint)}
|
|
.lede{color:var(--soft);font-size:1.05rem;max-width:52ch;margin:0 0 2.25rem}
|
|
|
|
.btns{display:flex;flex-wrap:wrap;gap:.7rem;margin:0 0 .5rem;align-items:center}
|
|
.btn{display:inline-flex;align-items:center;gap:.5rem;font-family:var(--mono);
|
|
font-size:.72rem;letter-spacing:.14em;text-transform:uppercase;
|
|
padding:.8rem 1.4rem;border:1px solid var(--line-hi);border-radius:var(--r);
|
|
color:var(--ink);text-decoration:none;background:transparent;cursor:pointer;
|
|
line-height:1.2;font-weight:400}
|
|
.btn:hover{background:#141414;text-decoration:none;border-color:var(--soft)}
|
|
.btn.solid{background:var(--ink);color:#000;border-color:var(--ink)}
|
|
.btn.solid:hover{background:#d8d8d4}
|
|
.btn .chev{width:.7em;height:.7em;flex:0 0 auto;transition:transform .15s ease}
|
|
.btn[aria-expanded="true"] .chev{transform:rotate(180deg)}
|
|
|
|
/* Copy-prompt selector. Each row copies the prompt for its own target. */
|
|
.copy{position:relative;display:inline-flex}
|
|
.menu{position:absolute;top:calc(100% + .5rem);left:0;z-index:20;min-width:21rem;
|
|
background:var(--panel);border:1px solid var(--line-hi);border-radius:var(--r-soft);
|
|
padding:.4rem;display:flex;flex-direction:column;
|
|
box-shadow:0 12px 32px rgba(0,0,0,.75)}
|
|
.menu[hidden]{display:none}
|
|
.mi{display:flex;align-items:center;gap:.7rem;width:100%;text-align:left;
|
|
padding:.62rem .8rem;border:0;border-radius:calc(var(--r-soft) - .35rem);
|
|
background:transparent;color:var(--ink);font-family:var(--mono);font-size:.76rem;
|
|
letter-spacing:.04em;text-decoration:none;cursor:pointer;line-height:1.3;
|
|
white-space:nowrap}
|
|
.mi:hover,.mi:focus-visible{background:#181818;outline:none}
|
|
.mi .mark{width:1.05rem;height:1.05rem;color:var(--soft)}
|
|
.mi:hover .mark{color:var(--ink)}
|
|
.mi u{margin-left:auto;color:var(--faint);text-decoration:none;font-size:.68rem;
|
|
letter-spacing:.1em;text-transform:uppercase}
|
|
|
|
/* Works-with bar: one scrolling row, name and mark per agent. */
|
|
.strip{border-bottom:1px solid var(--line);background:var(--panel)}
|
|
.strip .in{max-width:60rem;margin:0 auto;padding:.9rem 1.75rem;display:flex;
|
|
align-items:center;gap:1.25rem;font-family:var(--mono);font-size:.72rem;
|
|
letter-spacing:.1em;text-transform:uppercase;color:var(--faint)}
|
|
.strip b{color:var(--soft);font-weight:400;flex:0 0 auto;
|
|
border-right:1px solid var(--line);padding-right:1.25rem;align-self:stretch;
|
|
display:flex;align-items:center}
|
|
.bar{display:flex;align-items:center;gap:.5rem;flex:1 1 auto;min-width:0;
|
|
overflow-x:auto;scrollbar-width:none;scroll-snap-type:x proximity;
|
|
-webkit-overflow-scrolling:touch;
|
|
-webkit-mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent);
|
|
mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent)}
|
|
.bar::-webkit-scrollbar{display:none}
|
|
.agent{display:inline-flex;align-items:center;gap:.45rem;flex:0 0 auto;
|
|
padding:.4rem .85rem;border:1px solid var(--line);border-radius:var(--r);
|
|
color:var(--soft);white-space:nowrap;scroll-snap-align:start;
|
|
text-decoration:none;background:var(--ground)}
|
|
.agent:hover{color:var(--ink);border-color:var(--line-hi);text-decoration:none}
|
|
.agent .mark{width:.95rem;height:.95rem}
|
|
|
|
h2{font-family:var(--mono);font-size:1.1rem;letter-spacing:-.01em;margin:2.75rem 0 .75rem;font-weight:500}
|
|
h3{font-size:1rem;margin:2rem 0 .5rem;font-weight:600}
|
|
h4,h5,h6{font-size:.94rem;margin:1.5rem 0 .5rem;font-weight:600}
|
|
p{margin:0 0 1rem;max-width:70ch}
|
|
ul,ol{margin:0 0 1rem;padding-left:1.35rem;max-width:70ch}
|
|
li{margin:.22rem 0}
|
|
a{color:var(--ink);text-decoration:underline;text-underline-offset:3px;
|
|
text-decoration-color:var(--line-hi)}
|
|
a:hover{text-decoration-color:var(--ink)}
|
|
a:focus-visible{outline:1px solid var(--ink);outline-offset:3px}
|
|
code{font-family:var(--mono);font-size:.85em;background:var(--panel);
|
|
border:1px solid var(--line);border-radius:5px;padding:.1em .36em}
|
|
pre{background:var(--panel);border:1px solid var(--line);padding:1rem 1.1rem;
|
|
border-radius:var(--r-soft);overflow-x:auto;margin:0 0 1.25rem}
|
|
pre code{background:none;border:0;padding:0;font-size:.82rem;line-height:1.7}
|
|
blockquote{margin:0 0 1rem;padding-left:1rem;border-left:1px solid var(--line-hi);color:var(--soft)}
|
|
hr{border:0;border-top:1px solid var(--line);margin:2.5rem 0}
|
|
table{border-collapse:collapse;width:100%;margin:0 0 1.25rem;font-size:.92rem;
|
|
display:block;overflow-x:auto}
|
|
th,td{border:1px solid var(--line);padding:.5rem .7rem;text-align:left}
|
|
|
|
.curl{display:block;background:var(--panel);border:1px solid var(--line);
|
|
border-radius:var(--r-soft);padding:.95rem 1.1rem;font-family:var(--mono);
|
|
font-size:.8rem;color:var(--soft);overflow-x:auto;margin:0 0 2.5rem;
|
|
white-space:pre;line-height:1.9}
|
|
.curl b{color:var(--ink);font-weight:400}
|
|
.curl i{font-style:normal;color:var(--faint)}
|
|
|
|
/* index */
|
|
#q{width:100%;padding:.85rem 1.2rem;font-family:var(--mono);font-size:.85rem;color:var(--ink);
|
|
background:var(--panel);border:1px solid var(--line);border-radius:var(--r);margin:0 0 .6rem}
|
|
#q::placeholder{color:var(--faint);text-transform:uppercase;letter-spacing:.1em;font-size:.72rem}
|
|
#q:focus{outline:none;border-color:var(--soft)}
|
|
#count{color:var(--faint);font-family:var(--mono);font-size:.72rem;letter-spacing:.1em;
|
|
text-transform:uppercase;margin:0 0 2rem}
|
|
.cat{margin:2.5rem 0 .5rem;font-family:var(--mono);font-size:.7rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);border-bottom:1px solid var(--line);
|
|
padding-bottom:.45rem}
|
|
.row{display:block;padding:.6rem .9rem;margin:0 -.9rem;border-bottom:1px solid var(--line);
|
|
border-radius:var(--r-soft);text-decoration:none}
|
|
.row:hover{background:#0a0a0a}
|
|
.nm{font-family:var(--mono);font-size:.84rem;color:var(--ink);display:block}
|
|
.ds{color:var(--soft);font-size:.9rem;display:block;margin-top:.12rem}
|
|
footer{margin-top:4rem;padding:2rem 0 4rem;border-top:1px solid var(--line);
|
|
color:var(--faint);font-size:.85rem}
|
|
footer a{color:var(--soft)}
|
|
.fmap{display:grid;gap:1.75rem;grid-template-columns:repeat(auto-fit,minmax(11rem,1fr));
|
|
margin:0 0 2rem;font-size:.85rem}
|
|
.fmap b{display:block;font-family:var(--mono);font-size:.68rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);font-weight:400;margin:0 0 .6rem;
|
|
border-bottom:1px solid var(--line);padding-bottom:.4rem}
|
|
.fmap a{display:inline-block;color:var(--ink);text-decoration:none;margin:.3rem .35rem 0 0}
|
|
.fmap a:hover{text-decoration:underline}
|
|
.fmap div{color:var(--faint);line-height:1.5}
|
|
.fend{border-top:1px solid var(--line);padding-top:1.25rem;font-size:.82rem}
|
|
|
|
@media (max-width:640px){
|
|
.hero .wrap{padding-top:3.25rem;padding-bottom:2.75rem}
|
|
.btns{gap:.5rem}
|
|
.btn{padding:.75rem 1.15rem;font-size:.68rem}
|
|
.copy,.copy .btn{width:100%;justify-content:center}
|
|
.menu{left:0;right:0;min-width:0}
|
|
.strip .in{flex-direction:column;align-items:flex-start;gap:.6rem;padding:.9rem 1.75rem}
|
|
.strip b{border-right:0;padding-right:0}
|
|
.bar{width:100%}
|
|
}
|
|
</style>
|
|
<link rel="alternate" type="text/markdown" href="/hanzo-sho.md">
|
|
|
|
<link rel="icon" href="/favicon.svg">
|
|
<link rel="canonical" href="https://hanzoskills.com/hanzo-sho">
|
|
</head>
|
|
<!-- Reading this with a machine? Every page here is also plain markdown.
|
|
This one: https://hanzoskills.com/hanzo-sho.md
|
|
Index: https://hanzoskills.com/llms.txt
|
|
Corpus: https://hanzoskills.com/all.md
|
|
Append .md to any path for the markdown twin. -->
|
|
<body>
|
|
<nav><div class="in">
|
|
<a class="brand" href="/"><svg class="mark" viewBox="0 0 67 67" fill="currentColor" aria-hidden="true"><path d="M22.21 67V44.6369H0V67H22.21Z"/><path d="M66.7038 22.3184H22.2534L0.0878906 44.6367H44.4634L66.7038 22.3184Z"/><path d="M22.21 0H0V22.3184H22.21V0Z"/><path d="M66.7198 0H44.5098V22.3184H66.7198V0Z"/><path d="M66.7198 67V44.6369H44.5098V67H66.7198Z"/></svg> HANZO<i>SKILLS</i></a>
|
|
<a class="up" href="/">All skills</a>
|
|
</div></nav>
|
|
|
|
<div class="wrap">
|
|
<h1>hanzo-sho</h1><p class="lede">Sho is a text diffusion engine that generates text using masked diffusion rather than autoregressive token prediction.</p><h1>Hanzo Sho - Text Diffusion Engine</h1>
|
|
<p><strong>Category</strong>: Hanzo Ecosystem <strong>Related Skills</strong>: <code>hanzo/hanzo-kensho.md</code>, <code>hanzo/hanzo-ml.md</code>, <code>hanzo/hanzo-engine.md</code></p>
|
|
<h2>Overview</h2>
|
|
<p>Sho is a <strong>text diffusion engine</strong> that generates text using masked diffusion rather than autoregressive token prediction. The core model (Genjo, 8B parameters) uses a Transformer Encoder with bidirectional attention and an iterative masked denoising process to produce coherent text. This approach allows parallel token prediction with progressive refinement, producing high-quality output competitive with autoregressive models on standard benchmarks.</p>
|
|
<p>Sho serves as the foundation for integration with the Enso diffusion Mixture of Experts (MoE) architecture, targeting 16B+ parameter models with expert specialization.</p>
|
|
<h3>Key Innovation</h3>
|
|
<p>Unlike autoregressive models (LLaMA) that generate one token at a time left-to-right, Sho:</p>
|
|
<ul><li>Predicts <strong>all masked tokens simultaneously</strong> at each step</li><li>Uses <strong>iterative denoising</strong>: selectively unmasks the most confident predictions</li><li>Employs a <strong>varying masking ratio</strong> (0 to 1) during training as an upper bound on negative log-likelihood</li><li>Supports <strong>classifier-free guidance</strong> for improved benchmark performance</li><li>Uses <strong>semi-autoregressive block generation</strong> for variable-length output</li></ul>
|
|
<h2>When to use</h2>
|
|
<ul><li>Research into non-autoregressive text generation</li><li>Applications where parallel token prediction is advantageous</li><li>Benchmarking diffusion-based language models against autoregressive baselines</li><li>Exploring hybrid diffusion + MoE architectures</li></ul>
|
|
<h2>Hard requirements</h2>
|
|
<ol><li><strong>Python 3.8+</strong> with PyTorch</li><li><strong>transformers==4.38.2</strong> (specific version required for model loading)</li><li><strong>GPU recommended</strong> for inference (bfloat16 support)</li><li><strong>lm-evaluation-harness</strong> for benchmarking</li></ol>
|
|
<h2>Quick reference</h2>
|
|
<p>| Item | Value | |------|-------| | Repo | <code>github.com/hanzoai/sho</code> | | Branch | <code>main</code> | | Language | Python | | Parameters | 8B | | Architecture | Transformer Encoder (bidirectional) | | Mask Token ID | 126336 | | Generate | <code>python generate.py</code> | | Chat | <code>python chat.py</code> | | Evaluate | <code>python eval_llada.py</code> | | Gradio Demo | <code>python app.py</code> | | License | See repo |</p>
|
|
<h2>Model Variants</h2>
|
|
<p>| Variant | Purpose | HuggingFace | |---------|---------|-------------| | Genjo-8B-Base | Foundation model | <code>GSAI-ML/Genjo-8B-Base</code> | | Genjo-8B-Instruct | Chat/instruction following | <code>GSAI-ML/Genjo-8B-Instruct</code> |</p>
|
|
<h2>Architecture</h2>
|
|
<pre><code>┌──────────────────────────────────────────┐
|
|
│ Sho Diffusion Process │
|
|
├──────────────────────────────────────────┤
|
|
│ │
|
|
│ Input: [prompt tokens] [MASK MASK ...] │
|
|
│ │
|
|
│ For each denoising step: │
|
|
│ 1. Transformer Encoder (bidirectional)│
|
|
│ - Full self-attention (no causal) │
|
|
│ - Same params as decoder (8B) │
|
|
│ │
|
|
│ 2. Predict all masked positions │
|
|
│ - Gumbel noise sampling │
|
|
│ - Optional CFG guidance │
|
|
│ │
|
|
│ 3. Confidence-based unmasking │
|
|
│ - Score: softmax probability │
|
|
│ - Unmask top-k most confident │
|
|
│ - Or random selection │
|
|
│ │
|
|
│ 4. Re-mask remaining positions │
|
|
│ - Linear schedule across steps │
|
|
│ │
|
|
│ Output: fully unmasked response │
|
|
└──────────────────────────────────────────┘</code></pre>
|
|
<h3>Comparison with Autoregressive Models</h3>
|
|
<p>| Feature | Sho (Diffusion) | Autoregressive (LLaMA) | |---------|-----------------|------------------------| | Architecture | Transformer Encoder | Transformer Decoder | | Attention | Bidirectional | Unidirectional (causal) | | Training | Masked diffusion, varying ratio | Next-token prediction | | Generation | Parallel predict + iterative denoise | Sequential token-by-token | | KV-Cache | Not applicable | Yes (faster inference) | | In-context Learning | Yes | Yes |</p>
|
|
<h3>Generation Algorithm</h3>
|
|
<p>The generation function implements block-based semi-autoregressive diffusion:</p>
|
|
<ol><li><strong>Initialize</strong> fully masked response sequence</li><li><strong>Divide</strong> into blocks of <code>block_length</code> tokens</li><li><strong>Per block</strong>, run <code>steps</code> denoising iterations:</li></ol>
|
|
<ul><li>Forward pass through encoder to get logits for all positions</li><li>Apply Gumbel noise (temperature-controlled) for sampling</li><li>Apply classifier-free guidance if <code>cfg_scale > 0</code></li><li>Select tokens to unmask based on confidence (<code>low_confidence</code>) or randomly</li><li>Number of tokens unmasked per step is pre-computed for uniform distribution</li></ul>
|
|
<p>Key parameters:</p>
|
|
<ul><li><code>steps</code>: Total denoising steps (default 128)</li><li><code>gen_length</code>: Maximum generated tokens (default 128)</li><li><code>block_length</code>: Semi-autoregressive block size (default 128)</li><li><code>temperature</code>: Sampling temperature (0 = greedy)</li><li><code>cfg_scale</code>: Classifier-free guidance strength (0 = disabled)</li><li><code>remasking</code>: Strategy - <code>low_confidence</code> or <code>random</code></li></ul>
|
|
<h2>One-file quickstart</h2>
|
|
<h3>Text Generation</h3>
|
|
<pre><code class="lang-python">import torch
|
|
from transformers import AutoModel, AutoTokenizer
|
|
from generate import generate
|
|
|
|
device = "cuda"
|
|
tokenizer = AutoTokenizer.from_pretrained(
|
|
"GSAI-ML/Genjo-8B-Base", trust_remote_code=True
|
|
)
|
|
model = AutoModel.from_pretrained(
|
|
"GSAI-ML/Genjo-8B-Base",
|
|
trust_remote_code=True,
|
|
torch_dtype=torch.bfloat16,
|
|
).to(device).eval()
|
|
|
|
prompt = "The history of artificial intelligence begins with"
|
|
input_ids = torch.tensor(
|
|
tokenizer(prompt)["input_ids"], device=device
|
|
).unsqueeze(0)
|
|
|
|
output = generate(
|
|
model, input_ids,
|
|
steps=128,
|
|
gen_length=128,
|
|
block_length=32,
|
|
temperature=0.0,
|
|
cfg_scale=0.0,
|
|
remasking="low_confidence",
|
|
)
|
|
|
|
result = tokenizer.batch_decode(
|
|
output[:, input_ids.shape[1]:],
|
|
skip_special_tokens=True,
|
|
)[0]
|
|
print(result)</code></pre>
|
|
<h3>Chat with Instruct Model</h3>
|
|
<pre><code class="lang-python">import torch
|
|
from transformers import AutoModel, AutoTokenizer
|
|
from generate import generate
|
|
|
|
device = "cuda"
|
|
tokenizer = AutoTokenizer.from_pretrained(
|
|
"GSAI-ML/Genjo-8B-Instruct", trust_remote_code=True
|
|
)
|
|
model = AutoModel.from_pretrained(
|
|
"GSAI-ML/Genjo-8B-Instruct",
|
|
trust_remote_code=True,
|
|
torch_dtype=torch.bfloat16,
|
|
).to(device).eval()
|
|
|
|
messages = [{"role": "user", "content": "Explain diffusion models in 3 sentences."}]
|
|
chat_input = tokenizer.apply_chat_template(
|
|
messages, add_generation_prompt=True, tokenize=False
|
|
)
|
|
input_ids = torch.tensor(
|
|
tokenizer(chat_input)["input_ids"], device=device
|
|
).unsqueeze(0)
|
|
|
|
output = generate(
|
|
model, input_ids,
|
|
steps=128, gen_length=256, block_length=64,
|
|
)
|
|
print(tokenizer.batch_decode(
|
|
output[:, input_ids.shape[1]:], skip_special_tokens=True
|
|
)[0])</code></pre>
|
|
<h2>Training</h2>
|
|
<h3>Pre-training (Core Code)</h3>
|
|
<pre><code class="lang-python">def forward_process(input_ids, eps=1e-3):
|
|
b, l = input_ids.shape
|
|
t = torch.rand(b, device=input_ids.device)
|
|
p_mask = (1 - eps) * t + eps
|
|
p_mask = p_mask[:, None].repeat(1, l)
|
|
masked_indices = torch.rand((b, l), device=input_ids.device) < p_mask
|
|
noisy_batch = torch.where(masked_indices, 126336, input_ids)
|
|
return noisy_batch, masked_indices, p_mask
|
|
|
|
# Training step
|
|
noisy_batch, masked_indices, p_mask = forward_process(input_ids)
|
|
logits = model(input_ids=noisy_batch).logits
|
|
token_loss = F.cross_entropy(
|
|
logits[masked_indices], input_ids[masked_indices], reduction="none"
|
|
) / p_mask[masked_indices]
|
|
loss = token_loss.sum() / (input_ids.shape[0] * input_ids.shape[1])</code></pre>
|
|
<h3>SFT (Supervised Fine-Tuning)</h3>
|
|
<p>Same as pre-training but noise is only applied to the response portion; the prompt remains unmasked.</p>
|
|
<h2>Evaluation</h2>
|
|
<p>Integration with <code>lm-evaluation-harness</code> via the <code>GenjoEvalHarness</code> class:</p>
|
|
<pre><code class="lang-bash"># Conditional likelihood estimation
|
|
accelerate launch eval_llada.py \
|
|
--tasks gpqa_main_n_shot --num_fewshot 5 --model genjo_dist
|
|
|
|
# Conditional generation
|
|
accelerate launch eval_llada.py \
|
|
--tasks bbh --model genjo_dist \
|
|
--model_args gen_length=1024,steps=1024,block_length=1024</code></pre>
|
|
<p>Evaluated on: BBH, GSM8K, Math, HumanEval, MBPP.</p>
|
|
<h2>Project Structure</h2>
|
|
<pre><code>sho/
|
|
├── generate.py # Core generation: add_gumbel_noise, generate()
|
|
├── get_log_likelihood.py # Monte Carlo log-likelihood estimation
|
|
├── eval_llada.py # lm-evaluation-harness integration
|
|
├── eval_llada.sh # Evaluation benchmark scripts
|
|
├── chat.py # Terminal chat interface (Instruct model)
|
|
├── app.py # Gradio web demo with denoising visualization
|
|
├── integration.py # Integration utilities
|
|
├── demo_integration.py # Demo integration
|
|
├── GUIDELINES.md # Architecture and training details
|
|
├── EVAL.md # Evaluation documentation
|
|
├── LLM.md # Detailed project documentation
|
|
├── visualization/
|
|
│ ├── generate.py # Visualization generation
|
|
│ ├── html_to_png.py # Export visualizations
|
|
│ └── visualization_paper.py
|
|
└── imgs/ # Benchmark comparison charts</code></pre>
|
|
<h2>Future Directions</h2>
|
|
<ul><li><strong>Enso MoE integration</strong>: Scale to 16B+ with expert specialization via DiT-MoE</li><li><strong>Semi-autoregressive sampling</strong>: Reduce fixed context length limitations</li><li><strong>Consistency distillation</strong>: Fewer denoising steps without quality loss</li><li><strong>DeepSpeed training</strong>: Efficient large-scale training</li></ul>
|
|
<h2>Related Skills</h2>
|
|
<ul><li><code>hanzo/hanzo-kensho.md</code> - Image diffusion (same diffusion paradigm, different modality)</li><li><code>hanzo/hanzo-ml.md</code> - Rust ML framework</li><li><code>hanzo/hanzo-engine.md</code> - Model serving</li><li><code>hanzo/hanzo-jin.md</code> - Visual self-supervised learning</li></ul>
|
|
<hr>
|
|
<p><strong>Last Updated</strong>: 2026-03-13 <strong>Category</strong>: Hanzo Ecosystem <strong>Related</strong>: text-diffusion, language-model, masked-diffusion, non-autoregressive, generation <strong>Prerequisites</strong>: Python, PyTorch, transformers, diffusion model concepts</p>
|
|
<footer>
|
|
<div class="fmap">
|
|
<div><b>Build</b>
|
|
<a href="https://hanzo.app">hanzo.app</a> app builder
|
|
<a href="https://hanzo.codes">hanzo.codes</a> code with an agent
|
|
<a href="https://hanzoskills.com">hanzoskills.com</a> what an agent reads</div>
|
|
<div><b>Run</b>
|
|
<a href="https://console.hanzo.ai">console.hanzo.ai</a> cloud console
|
|
<a href="https://api.hanzo.ai">api.hanzo.ai</a> the API</div>
|
|
<div><b>Account</b>
|
|
<a href="https://hanzo.id">hanzo.id</a> sign in
|
|
<a href="https://pay.hanzo.ai">pay.hanzo.ai</a> top up
|
|
<a href="https://billing.hanzo.ai">billing.hanzo.ai</a> invoices and usage</div>
|
|
<div><b>Company</b>
|
|
<a href="https://hanzo.ai">hanzo.ai</a> canonical
|
|
<a href="https://hanzo.ventures">hanzo.ventures</a> ventures</div>
|
|
</div>
|
|
<div class="fend"><a href="/hanzo-sho.md">This page as plain text</a> · <a href="/llms.txt">Index for agents</a></div>
|
|
</footer>
|
|
</div></body></html> |