mirror of
https://github.com/hanzoai/skills.git
synced 2026-11-19 00:47:36 +00:00
hanzoskills.com/llms.txt now opens with the install line, the API reference index and the company index, so an agent that lands on any one of the three can find the rest. robots.txt claimed 537 documents while llms.txt counts 920. The number is gone from robots.txt; llms.txt derives it. public/ is regenerated from scripts/site.py, which it had fallen behind: the copy-prompt menu and agent bar were in the generator and not in the committed pages. tests/site.mjs passes against the result. Co-authored-by: Hanzo Dev <[email protected]>
768 lines
53 KiB
HTML
768 lines
53 KiB
HTML
<!doctype html>
|
|
<html lang="en"><head>
|
|
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
<title>hip-0019</title>
|
|
<meta name="color-scheme" content="dark">
|
|
<style>
|
|
/* Committed to one visual world: a dark instrument panel. No light variant —
|
|
the dot field and the hairline rules only read on black, and a naive
|
|
inversion would wash both out. Every colour is painted explicitly so the page
|
|
never borrows a host background. */
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Pixel Square';src:url('/fonts/ZenPixel-Square.woff2') format('woff2');font-style:normal;font-display:swap}
|
|
|
|
:root{
|
|
--ink:#f4f4f2; --soft:#9c9c98; --faint:#6a6a66; --ground:#000; --panel:#0a0a0a;
|
|
--line:#232323; --line-hi:#3a3a3a; --accent:#f4f4f2;
|
|
--font-zen-sans:'Zen',-apple-system,BlinkMacSystemFont,"Segoe UI",Helvetica,Arial,sans-serif;
|
|
--font-zen-mono:'Zen Mono',ui-monospace,"SF Mono",SFMono-Regular,Menlo,Consolas,monospace;
|
|
--font-zen-pixel-square:'Zen Pixel Square',monospace;
|
|
--mono:var(--font-zen-mono);
|
|
--sans:var(--font-zen-sans);
|
|
/* Controls are pills and surfaces are soft, as on hanzo.ai. */
|
|
--r:999px; --r-soft:14px;
|
|
}
|
|
*{box-sizing:border-box}
|
|
html{-webkit-text-size-adjust:100%}
|
|
body{margin:0;background:var(--ground);color:var(--ink);font-family:var(--sans);
|
|
font-size:16px;line-height:1.65;-webkit-font-smoothing:antialiased}
|
|
.wrap{max-width:60rem;margin:0 auto;padding:0 1.75rem}
|
|
|
|
/* The dot field. A halftone made of one repeating radial-gradient, faded out
|
|
along the diagonal by a mask so it reads as a wave rather than wallpaper. */
|
|
/* No overflow clip here: the dot field is inset to the box already, and a clip
|
|
would cut off the copy-prompt menu. */
|
|
.hero{position:relative;border-bottom:1px solid var(--line)}
|
|
.hero::before{content:"";position:absolute;inset:0;
|
|
background-image:radial-gradient(var(--line-hi) 1px, transparent 1px);
|
|
background-size:7px 7px;
|
|
-webkit-mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
pointer-events:none}
|
|
.hero .wrap{position:relative;padding-top:5.5rem;padding-bottom:4rem}
|
|
|
|
nav{border-bottom:1px solid var(--line)}
|
|
nav .in{max-width:60rem;margin:0 auto;padding:1.1rem 1.75rem;display:flex;
|
|
justify-content:space-between;align-items:center;gap:1rem}
|
|
.brand{font-family:var(--mono);font-size:.85rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--ink);text-decoration:none;
|
|
display:inline-flex;align-items:center;gap:.6rem}
|
|
.brand i{font-style:normal;color:var(--faint)}
|
|
.mark{width:1.15em;height:1.15em;display:block;flex:0 0 auto}
|
|
nav a.up{font-family:var(--mono);font-size:.72rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--soft);text-decoration:none}
|
|
nav a.up:hover{color:var(--ink)}
|
|
|
|
h1{font-family:var(--mono);font-size:clamp(2rem,5.4vw,3.4rem);line-height:1.12;
|
|
letter-spacing:-.02em;margin:0 0 1.1rem;font-weight:500;text-wrap:balance}
|
|
h1 em{font-style:normal;color:var(--faint)}
|
|
.lede{color:var(--soft);font-size:1.05rem;max-width:52ch;margin:0 0 2.25rem}
|
|
|
|
.btns{display:flex;flex-wrap:wrap;gap:.7rem;margin:0 0 .5rem;align-items:center}
|
|
.btn{display:inline-flex;align-items:center;gap:.5rem;font-family:var(--mono);
|
|
font-size:.72rem;letter-spacing:.14em;text-transform:uppercase;
|
|
padding:.8rem 1.4rem;border:1px solid var(--line-hi);border-radius:var(--r);
|
|
color:var(--ink);text-decoration:none;background:transparent;cursor:pointer;
|
|
line-height:1.2;font-weight:400}
|
|
.btn:hover{background:#141414;text-decoration:none;border-color:var(--soft)}
|
|
.btn.solid{background:var(--ink);color:#000;border-color:var(--ink)}
|
|
.btn.solid:hover{background:#d8d8d4}
|
|
.btn .chev{width:.7em;height:.7em;flex:0 0 auto;transition:transform .15s ease}
|
|
.btn[aria-expanded="true"] .chev{transform:rotate(180deg)}
|
|
|
|
/* Copy-prompt selector. Each row copies the prompt for its own target. */
|
|
.copy{position:relative;display:inline-flex}
|
|
.menu{position:absolute;top:calc(100% + .5rem);left:0;z-index:20;min-width:21rem;
|
|
background:var(--panel);border:1px solid var(--line-hi);border-radius:var(--r-soft);
|
|
padding:.4rem;display:flex;flex-direction:column;
|
|
box-shadow:0 12px 32px rgba(0,0,0,.75)}
|
|
.menu[hidden]{display:none}
|
|
.mi{display:flex;align-items:center;gap:.7rem;width:100%;text-align:left;
|
|
padding:.62rem .8rem;border:0;border-radius:calc(var(--r-soft) - .35rem);
|
|
background:transparent;color:var(--ink);font-family:var(--mono);font-size:.76rem;
|
|
letter-spacing:.04em;text-decoration:none;cursor:pointer;line-height:1.3;
|
|
white-space:nowrap}
|
|
.mi:hover,.mi:focus-visible{background:#181818;outline:none}
|
|
.mi .mark{width:1.05rem;height:1.05rem;color:var(--soft)}
|
|
.mi:hover .mark{color:var(--ink)}
|
|
.mi u{margin-left:auto;color:var(--faint);text-decoration:none;font-size:.68rem;
|
|
letter-spacing:.1em;text-transform:uppercase}
|
|
|
|
/* Works-with bar: one scrolling row, name and mark per agent. */
|
|
.strip{border-bottom:1px solid var(--line);background:var(--panel)}
|
|
.strip .in{max-width:60rem;margin:0 auto;padding:.9rem 1.75rem;display:flex;
|
|
align-items:center;gap:1.25rem;font-family:var(--mono);font-size:.72rem;
|
|
letter-spacing:.1em;text-transform:uppercase;color:var(--faint)}
|
|
.strip b{color:var(--soft);font-weight:400;flex:0 0 auto;
|
|
border-right:1px solid var(--line);padding-right:1.25rem;align-self:stretch;
|
|
display:flex;align-items:center}
|
|
.bar{display:flex;align-items:center;gap:.5rem;flex:1 1 auto;min-width:0;
|
|
overflow-x:auto;scrollbar-width:none;scroll-snap-type:x proximity;
|
|
-webkit-overflow-scrolling:touch;
|
|
-webkit-mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent);
|
|
mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent)}
|
|
.bar::-webkit-scrollbar{display:none}
|
|
.agent{display:inline-flex;align-items:center;gap:.45rem;flex:0 0 auto;
|
|
padding:.4rem .85rem;border:1px solid var(--line);border-radius:var(--r);
|
|
color:var(--soft);white-space:nowrap;scroll-snap-align:start;
|
|
text-decoration:none;background:var(--ground)}
|
|
.agent:hover{color:var(--ink);border-color:var(--line-hi);text-decoration:none}
|
|
.agent .mark{width:.95rem;height:.95rem}
|
|
|
|
h2{font-family:var(--mono);font-size:1.1rem;letter-spacing:-.01em;margin:2.75rem 0 .75rem;font-weight:500}
|
|
h3{font-size:1rem;margin:2rem 0 .5rem;font-weight:600}
|
|
h4,h5,h6{font-size:.94rem;margin:1.5rem 0 .5rem;font-weight:600}
|
|
p{margin:0 0 1rem;max-width:70ch}
|
|
ul,ol{margin:0 0 1rem;padding-left:1.35rem;max-width:70ch}
|
|
li{margin:.22rem 0}
|
|
a{color:var(--ink);text-decoration:underline;text-underline-offset:3px;
|
|
text-decoration-color:var(--line-hi)}
|
|
a:hover{text-decoration-color:var(--ink)}
|
|
a:focus-visible{outline:1px solid var(--ink);outline-offset:3px}
|
|
code{font-family:var(--mono);font-size:.85em;background:var(--panel);
|
|
border:1px solid var(--line);border-radius:5px;padding:.1em .36em}
|
|
pre{background:var(--panel);border:1px solid var(--line);padding:1rem 1.1rem;
|
|
border-radius:var(--r-soft);overflow-x:auto;margin:0 0 1.25rem}
|
|
pre code{background:none;border:0;padding:0;font-size:.82rem;line-height:1.7}
|
|
blockquote{margin:0 0 1rem;padding-left:1rem;border-left:1px solid var(--line-hi);color:var(--soft)}
|
|
hr{border:0;border-top:1px solid var(--line);margin:2.5rem 0}
|
|
table{border-collapse:collapse;width:100%;margin:0 0 1.25rem;font-size:.92rem;
|
|
display:block;overflow-x:auto}
|
|
th,td{border:1px solid var(--line);padding:.5rem .7rem;text-align:left}
|
|
|
|
.curl{display:block;background:var(--panel);border:1px solid var(--line);
|
|
border-radius:var(--r-soft);padding:.95rem 1.1rem;font-family:var(--mono);
|
|
font-size:.8rem;color:var(--soft);overflow-x:auto;margin:0 0 2.5rem;
|
|
white-space:pre;line-height:1.9}
|
|
.curl b{color:var(--ink);font-weight:400}
|
|
.curl i{font-style:normal;color:var(--faint)}
|
|
|
|
/* index */
|
|
#q{width:100%;padding:.85rem 1.2rem;font-family:var(--mono);font-size:.85rem;color:var(--ink);
|
|
background:var(--panel);border:1px solid var(--line);border-radius:var(--r);margin:0 0 .6rem}
|
|
#q::placeholder{color:var(--faint);text-transform:uppercase;letter-spacing:.1em;font-size:.72rem}
|
|
#q:focus{outline:none;border-color:var(--soft)}
|
|
#count{color:var(--faint);font-family:var(--mono);font-size:.72rem;letter-spacing:.1em;
|
|
text-transform:uppercase;margin:0 0 2rem}
|
|
.cat{margin:2.5rem 0 .5rem;font-family:var(--mono);font-size:.7rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);border-bottom:1px solid var(--line);
|
|
padding-bottom:.45rem}
|
|
.row{display:block;padding:.6rem .9rem;margin:0 -.9rem;border-bottom:1px solid var(--line);
|
|
border-radius:var(--r-soft);text-decoration:none}
|
|
.row:hover{background:#0a0a0a}
|
|
.nm{font-family:var(--mono);font-size:.84rem;color:var(--ink);display:block}
|
|
.ds{color:var(--soft);font-size:.9rem;display:block;margin-top:.12rem}
|
|
footer{margin-top:4rem;padding:2rem 0 4rem;border-top:1px solid var(--line);
|
|
color:var(--faint);font-size:.85rem}
|
|
footer a{color:var(--soft)}
|
|
.fmap{display:grid;gap:1.75rem;grid-template-columns:repeat(auto-fit,minmax(11rem,1fr));
|
|
margin:0 0 2rem;font-size:.85rem}
|
|
.fmap b{display:block;font-family:var(--mono);font-size:.68rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);font-weight:400;margin:0 0 .6rem;
|
|
border-bottom:1px solid var(--line);padding-bottom:.4rem}
|
|
.fmap a{display:inline-block;color:var(--ink);text-decoration:none;margin:.3rem .35rem 0 0}
|
|
.fmap a:hover{text-decoration:underline}
|
|
.fmap div{color:var(--faint);line-height:1.5}
|
|
.fend{border-top:1px solid var(--line);padding-top:1.25rem;font-size:.82rem}
|
|
|
|
@media (max-width:640px){
|
|
.hero .wrap{padding-top:3.25rem;padding-bottom:2.75rem}
|
|
.btns{gap:.5rem}
|
|
.btn{padding:.75rem 1.15rem;font-size:.68rem}
|
|
.copy,.copy .btn{width:100%;justify-content:center}
|
|
.menu{left:0;right:0;min-width:0}
|
|
.strip .in{flex-direction:column;align-items:flex-start;gap:.6rem;padding:.9rem 1.75rem}
|
|
.strip b{border-right:0;padding-right:0}
|
|
.bar{width:100%}
|
|
}
|
|
</style>
|
|
<link rel="alternate" type="text/markdown" href="/hip-0019.md">
|
|
|
|
<link rel="icon" href="/favicon.svg">
|
|
<link rel="canonical" href="https://hanzoskills.com/hip-0019">
|
|
</head>
|
|
<!-- Reading this with a machine? Every page here is also plain markdown.
|
|
This one: https://hanzoskills.com/hip-0019.md
|
|
Index: https://hanzoskills.com/llms.txt
|
|
Corpus: https://hanzoskills.com/all.md
|
|
Append .md to any path for the markdown twin. -->
|
|
<body>
|
|
<nav><div class="in">
|
|
<a class="brand" href="/"><svg class="mark" viewBox="0 0 67 67" fill="currentColor" aria-hidden="true"><path d="M22.21 67V44.6369H0V67H22.21Z"/><path d="M66.7038 22.3184H22.2534L0.0878906 44.6367H44.4634L66.7038 22.3184Z"/><path d="M22.21 0H0V22.3184H22.21V0Z"/><path d="M66.7198 0H44.5098V22.3184H66.7198V0Z"/><path d="M66.7198 67V44.6369H44.5098V67H66.7198Z"/></svg> HANZO<i>SKILLS</i></a>
|
|
<a class="up" href="/">All skills</a>
|
|
</div></nav>
|
|
|
|
<div class="wrap">
|
|
<h1>hip-0019</h1><p class="lede">HIP-19: Tensor Operations Standard. Status Final. Hanzo architectural specification.</p><h1>HIP-0019: Tensor Operations Standard</h1>
|
|
<h2>Abstract</h2>
|
|
<p>This proposal defines the tensor operations standard for all ML computations in the Hanzo ecosystem. It specifies the data types, device backends, operation primitives, model formats, quantization strategies, memory management, custom operations, WebAssembly compilation pipeline, and API surface that every inference workload MUST use. The reference implementation is Hanzo Candle, a fork of HuggingFace's Candle Rust ML framework extended with Apple Silicon Metal support, custom quantization kernels, Hamiltonian dynamics operations, and integration with the Hanzo model serving pipeline.</p>
|
|
<p><strong>Repository</strong>: <a href="https://github.com/hanzoai/ml">github.com/hanzoai/ml</a> <strong>Crates</strong>: <code>hanzo-ml</code> (tensor core), <code>hanzo-nn</code>, <code>hanzo-transformers</code>, <code>hanzo-metal-kernels</code> -- the fork publishes under the <code>hanzo-<em></code> prefix; the <code>candle-</em></code> names used below follow the upstream layout. <strong>NPM Package</strong>: none yet -- the WASM build described below is proposed, not published.</p>
|
|
<h2>Motivation</h2>
|
|
<p>ML inference in production has a set of concrete problems that existing tooling does not solve together:</p>
|
|
<ol><li><strong>Runtime bloat</strong>: PyTorch requires a 2GB+ runtime. For a service that loads a 4-bit quantized 7B model into 4GB of RAM, the framework itself should not double the memory footprint. Inference servers should be small, fast to start, and free of dynamic library dependency chains.</li><li><strong>No single-binary deployment</strong>: Python-based inference requires a virtualenv, pip dependencies, CUDA toolkit installation, and careful version pinning. Deploying to a new machine means reproducing this environment exactly. A compiled binary that statically links its dependencies eliminates this class of failure.</li><li><strong>GIL contention</strong>: Python's Global Interpreter Lock serializes CPU-bound work across threads. For inference servers handling concurrent requests, this means either multiprocessing (with memory duplication) or yielding to C extensions (with complex FFI). Rust has no GIL.</li><li><strong>Memory safety at the hardware boundary</strong>: CUDA and Metal kernels operate on raw pointers. Buffer overflows, use-after-free, and data races in kernel launch code cause silent corruption or segfaults. Rust's ownership model catches these at compile time.</li><li><strong>No edge/browser deployment path</strong>: PyTorch cannot compile to WebAssembly. ONNX Runtime has a WASM build but it is large and limited. Candle compiles to WASM natively, enabling in-browser inference with no server round-trip.</li><li><strong>Fragmented hardware support</strong>: Most frameworks treat CUDA as first-class and everything else as an afterthought. Apple Silicon (Metal) and WebAssembly are second-class or unsupported. We need uniform performance across CPU, CUDA, Metal, and WASM.</li><li><strong>No custom operation pathway for scientific computing</strong>: Production ML frameworks are optimized for standard neural network operations. Applications that require physics-informed layers -- such as Hamiltonian dynamics for the HMM protocol (HIP-0008) or active inference for HLLM (HIP-0002) -- have no clean extension mechanism in PyTorch or ONNX. Candle's trait-based backend system makes custom operations first-class.</li></ol>
|
|
<p>Hanzo Candle addresses all seven problems with a single codebase.</p>
|
|
<h2>Design Philosophy</h2>
|
|
<p>This section explains the reasoning behind every major architectural decision. These are not arbitrary choices -- each follows from a specific constraint.</p>
|
|
<h3>Why Not Just Use PyTorch or ONNX Runtime</h3>
|
|
<p>This deserves explicit comparison because the question comes up frequently.</p>
|
|
<p><strong>PyTorch</strong> is the correct choice when:</p>
|
|
<ul><li>You are training a model</li><li>You are prototyping a new architecture</li><li>You need the full operator set (2000+ ops)</li><li>Your team only knows Python</li></ul>
|
|
<p>PyTorch is the wrong choice when:</p>
|
|
<ul><li>Your inference server must start in <1 second (PyTorch import alone takes 3-5 seconds)</li><li>You need single-binary deployment (PyTorch requires CUDA toolkit, cuDNN, NCCL)</li><li>You want to run in a browser or on edge devices (no WASM support)</li><li>Memory efficiency matters (PyTorch's allocator fragments under long-running workloads)</li><li>You serve on Apple Silicon (PyTorch MPS backend is incomplete and slower than Metal)</li></ul>
|
|
<p><strong>ONNX Runtime</strong> is the correct choice when:</p>
|
|
<ul><li>Your model exports cleanly to ONNX</li><li>You need the widest hardware support (CPU, CUDA, TensorRT, DirectML, CoreML, NNAPI)</li><li>You do not need custom attention kernels or KV cache management</li></ul>
|
|
<p>ONNX Runtime is the wrong choice when:</p>
|
|
<ul><li>Your model uses dynamic control flow (if/else in forward pass)</li><li>You need custom quantization (GPTQ, AWQ) beyond ONNX's built-in INT8</li><li>You want WASM deployment without a 20MB+ runtime</li><li>You need to modify the inference loop (e.g., speculative decoding, guided generation)</li></ul>
|
|
<p>Candle fills the gap: <strong>a minimal, embeddable ML runtime that compiles to native code or WASM, supports the model formats and quantization strategies that matter for LLM/diffusion inference, and gives the developer full control over the inference loop</strong>.</p>
|
|
<h2>Specification</h2>
|
|
<h3>Architecture</h3>
|
|
<pre><code>Application Layer
|
|
├── Jin (HIP-0003) - Multimodal inference
|
|
├── Node (HIP-0020) - Blockchain inference
|
|
├── Browser Client - Edge ML via WASM
|
|
└── LLM Gateway (HIP-0004) - Model serving
|
|
|
|
|
v
|
|
Candle High-Level API
|
|
├── candle-transformers - Pre-built model architectures
|
|
├── candle-nn - Neural network layers
|
|
└── candle-hamiltonian - Physics-informed operations
|
|
|
|
|
v
|
|
Candle Core
|
|
├── Tensor - N-dimensional array
|
|
├── Op - Operation enum (for autograd)
|
|
├── Backend trait - Device abstraction
|
|
└── Storage - Typed memory buffer
|
|
|
|
|
v
|
|
Backend Implementations
|
|
├── CPU (BLAS/LAPACK: OpenBLAS, MKL, Accelerate)
|
|
├── CUDA (cuBLAS, cuDNN, custom kernels)
|
|
├── Metal (MPS, custom compute shaders)
|
|
└── WASM (Rust stdlib, SIMD128 where available)</code></pre>
|
|
<h3>Tensor Types</h3>
|
|
<p>All tensor operations MUST support the following data types:</p>
|
|
<p>| DType | Size | Use Case | Backend Support | |-------|------|----------|-----------------| | <code>f32</code> | 4 bytes | Default precision, training, CPU inference | CPU, CUDA, Metal, WASM | | <code>f16</code> | 2 bytes | GPU inference, reduced memory | CPU (emulated), CUDA, Metal | | <code>bf16</code> | 2 bytes | Training stability, GPU inference | CPU (emulated), CUDA, Metal | | <code>f64</code> | 8 bytes | High-precision numerical computation, Hamiltonian dynamics | CPU, CUDA | | <code>i64</code> | 8 bytes | Indices, token IDs, positions | CPU, CUDA, Metal, WASM | | <code>u32</code> | 4 bytes | Indices, masks, vocabulary IDs | CPU, CUDA, Metal, WASM | | <code>u8</code> | 1 byte | Quantized weights, raw data | CPU, CUDA, Metal, WASM |</p>
|
|
<p>Type promotion rules follow NumPy conventions: <code>u8 + f32 -> f32</code>, <code>f16 + f32 -> f32</code>, <code>bf16 + f16 -> f32</code>.</p>
|
|
<h3>Device Backends</h3>
|
|
<pre><code class="lang-rust">pub enum Device {
|
|
Cpu,
|
|
Cuda(usize), // GPU ordinal
|
|
Metal(usize), // Metal device ordinal
|
|
}</code></pre>
|
|
<p>| Backend | Library | Matmul Dispatch | Notes | |---------|---------|-----------------|-------| | <strong>CPU</strong> | BLAS (OpenBLAS, MKL, Accelerate) | <code>sgemm</code>/<code>dgemm</code> | Universal fallback; vectorized with AVX2/NEON | | <strong>CUDA</strong> | cuBLAS, cuDNN, custom kernels | <code>cublasSgemm</code>, flash attention | NVIDIA GPUs; requires CUDA toolkit >= 12.0 | | <strong>Metal</strong> | Metal Performance Shaders, custom kernels | <code>MPSMatrixMultiplication</code>, custom Q4 matmul | Apple Silicon; M1/M2/M3/M4 | | <strong>WASM</strong> | Rust stdlib, SIMD128 | Manual tiling | Browser/edge; no GPU, SIMD optional |</p>
|
|
<p>Backend selection is explicit at tensor creation time. Tensors on different devices cannot be combined -- the caller must explicitly transfer with <code>tensor.to_device(&device)</code>.</p>
|
|
<h3>Memory Management</h3>
|
|
<p>Efficient memory management is critical for inference workloads where model weights may consume most available RAM. Candle uses three strategies:</p>
|
|
<p><strong>Zero-copy memory mapping</strong> for model weights. When loading safetensors files, Candle uses <code>mmap(2)</code> via the <code>memmap2</code> crate. The OS maps the file into virtual address space without copying it into a heap buffer. This means:</p>
|
|
<ul><li>Model loading is near-instant (page faults load data on demand)</li><li>Multiple processes serving the same model share physical memory pages</li><li>The OS can evict unused pages under memory pressure</li></ul>
|
|
<pre><code class="lang-rust">// Zero-copy loading: no heap allocation for weight data
|
|
let tensors = safetensors::MmapedSafetensors::new(&["model.safetensors"])?;
|
|
let weight = tensors.load("model.layers.0.attention.wq.weight", &device)?;</code></pre>
|
|
<p><strong>Arena allocation</strong> for intermediate tensors during inference. A single forward pass through a 7B-parameter model creates hundreds of intermediate tensors (attention scores, normalized activations, FFN outputs). Allocating and freeing each one individually causes memory fragmentation over millions of requests. Candle uses a per-request arena:</p>
|
|
<ul><li>Pre-allocate a contiguous buffer sized for the worst-case forward pass</li><li>Intermediate tensors are bump-allocated within the arena</li><li>The entire arena is freed at once when the request completes</li><li>No fragmentation, no GC pauses, deterministic memory usage</li></ul>
|
|
<p><strong>Ring-buffer KV cache</strong> for autoregressive generation. The key-value cache for transformer attention grows linearly with sequence length. Rather than reallocating on each token, Candle pre-allocates the full cache at the maximum sequence length and writes into it with a rotating index. This eliminates allocation during the generation loop entirely.</p>
|
|
<pre><code class="lang-rust">pub struct KvCache {
|
|
k: Tensor, // [batch, heads, max_seq, head_dim] -- pre-allocated
|
|
v: Tensor, // [batch, heads, max_seq, head_dim] -- pre-allocated
|
|
current_len: usize, // current position in the ring buffer
|
|
}</code></pre>
|
|
<h3>Core Operations</h3>
|
|
<p>Every backend MUST implement the following operation set:</p>
|
|
<h4>Element-wise Operations</h4>
|
|
<pre><code class="lang-rust">// Arithmetic
|
|
fn add(&self, rhs: &Tensor) -> Result<Tensor>;
|
|
fn sub(&self, rhs: &Tensor) -> Result<Tensor>;
|
|
fn mul(&self, rhs: &Tensor) -> Result<Tensor>;
|
|
fn div(&self, rhs: &Tensor) -> Result<Tensor>;
|
|
|
|
// Unary
|
|
fn neg(&self) -> Result<Tensor>;
|
|
fn abs(&self) -> Result<Tensor>;
|
|
fn exp(&self) -> Result<Tensor>;
|
|
fn log(&self) -> Result<Tensor>;
|
|
fn sqrt(&self) -> Result<Tensor>;
|
|
fn recip(&self) -> Result<Tensor>;
|
|
fn relu(&self) -> Result<Tensor>;
|
|
fn gelu(&self) -> Result<Tensor>;
|
|
fn silu(&self) -> Result<Tensor>;
|
|
fn tanh(&self) -> Result<Tensor>;
|
|
|
|
// Comparison
|
|
fn eq(&self, rhs: &Tensor) -> Result<Tensor>; // returns u8
|
|
fn lt(&self, rhs: &Tensor) -> Result<Tensor>;
|
|
fn gt(&self, rhs: &Tensor) -> Result<Tensor>;</code></pre>
|
|
<h4>Reduction Operations</h4>
|
|
<pre><code class="lang-rust">fn sum(&self, dims: &[usize]) -> Result<Tensor>;
|
|
fn mean(&self, dims: &[usize]) -> Result<Tensor>;
|
|
fn max(&self, dim: usize) -> Result<Tensor>;
|
|
fn min(&self, dim: usize) -> Result<Tensor>;
|
|
fn argmax(&self, dim: usize) -> Result<Tensor>;
|
|
fn argmin(&self, dim: usize) -> Result<Tensor>;</code></pre>
|
|
<h4>Linear Algebra</h4>
|
|
<pre><code class="lang-rust">fn matmul(&self, rhs: &Tensor) -> Result<Tensor>;
|
|
fn transpose(&self, dim1: usize, dim2: usize) -> Result<Tensor>;
|
|
fn contiguous(&self) -> Result<Tensor>;
|
|
fn broadcast_as(&self, shape: &[usize]) -> Result<Tensor>;</code></pre>
|
|
<h4>Neural Network Primitives</h4>
|
|
<pre><code class="lang-rust">// Attention
|
|
fn scaled_dot_product_attention(
|
|
query: &Tensor, // [batch, heads, seq_len, head_dim]
|
|
key: &Tensor, // [batch, heads, kv_len, head_dim]
|
|
value: &Tensor, // [batch, heads, kv_len, head_dim]
|
|
mask: Option<&Tensor>,
|
|
scale: f64,
|
|
) -> Result<Tensor>;
|
|
|
|
// Normalization
|
|
fn layer_norm(x: &Tensor, weight: &Tensor, bias: Option<&Tensor>, eps: f64) -> Result<Tensor>;
|
|
fn rms_norm(x: &Tensor, weight: &Tensor, eps: f64) -> Result<Tensor>;
|
|
|
|
// Positional encoding
|
|
fn rotary_embedding(x: &Tensor, cos: &Tensor, sin: &Tensor) -> Result<Tensor>;
|
|
|
|
// Activation
|
|
fn softmax(x: &Tensor, dim: usize) -> Result<Tensor>;
|
|
fn log_softmax(x: &Tensor, dim: usize) -> Result<Tensor>;
|
|
|
|
// Embedding
|
|
fn embedding(ids: &Tensor, weight: &Tensor) -> Result<Tensor>;
|
|
|
|
// Convolution
|
|
fn conv2d(
|
|
input: &Tensor, // [batch, channels, height, width]
|
|
kernel: &Tensor, // [out_ch, in_ch, kh, kw]
|
|
bias: Option<&Tensor>,
|
|
padding: usize,
|
|
stride: usize,
|
|
) -> Result<Tensor>;
|
|
|
|
// Pooling
|
|
fn avg_pool2d(input: &Tensor, kernel_size: usize, stride: usize) -> Result<Tensor>;
|
|
fn max_pool2d(input: &Tensor, kernel_size: usize, stride: usize) -> Result<Tensor>;</code></pre>
|
|
<h4>Shape Operations</h4>
|
|
<pre><code class="lang-rust">fn reshape(&self, shape: &[usize]) -> Result<Tensor>;
|
|
fn squeeze(&self, dim: usize) -> Result<Tensor>;
|
|
fn unsqueeze(&self, dim: usize) -> Result<Tensor>;
|
|
fn narrow(&self, dim: usize, start: usize, len: usize) -> Result<Tensor>;
|
|
fn chunk(&self, chunks: usize, dim: usize) -> Result<Vec<Tensor>>;
|
|
fn cat(tensors: &[&Tensor], dim: usize) -> Result<Tensor>;
|
|
fn stack(tensors: &[&Tensor], dim: usize) -> Result<Tensor>;
|
|
fn gather(&self, indices: &Tensor, dim: usize) -> Result<Tensor>;
|
|
fn scatter(&self, indices: &Tensor, src: &Tensor, dim: usize) -> Result<Tensor>;
|
|
fn index_select(&self, indices: &Tensor, dim: usize) -> Result<Tensor>;</code></pre>
|
|
<h4>Hamiltonian Dynamics Operations</h4>
|
|
<p>The HMM protocol (HIP-0008) and active inference framework (HIP-0007) require physics-informed tensor operations that preserve energy and symplectic structure. These are not available in standard ML frameworks. Candle extends its operation set with:</p>
|
|
<pre><code class="lang-rust">/// Symplectic Euler integrator: advances (q, p) by one timestep dt
|
|
/// under Hamiltonian H(q, p). Preserves the symplectic 2-form exactly.
|
|
fn symplectic_euler_step(
|
|
q: &Tensor, // generalized positions [batch, dim]
|
|
p: &Tensor, // generalized momenta [batch, dim]
|
|
grad_h_q: &Tensor, // dH/dq evaluated at (q, p)
|
|
grad_h_p: &Tensor, // dH/dp evaluated at (q, p)
|
|
dt: f64,
|
|
) -> Result<(Tensor, Tensor)>; // (q_next, p_next)
|
|
|
|
/// Leapfrog (Stormer-Verlet) integrator: second-order symplectic method.
|
|
/// Used for Hamiltonian Monte Carlo sampling in HLLM (HIP-0002).
|
|
fn leapfrog_step(
|
|
q: &Tensor,
|
|
p: &Tensor,
|
|
grad_h: impl Fn(&Tensor) -> Result<Tensor>, // gradient of H w.r.t. q
|
|
dt: f64,
|
|
n_steps: usize,
|
|
) -> Result<(Tensor, Tensor)>;
|
|
|
|
/// Compute the Hamiltonian energy H(q, p) = kinetic(p) + potential(q).
|
|
/// Used to verify energy conservation in symplectic integrators.
|
|
fn hamiltonian_energy(
|
|
q: &Tensor,
|
|
p: &Tensor,
|
|
potential: impl Fn(&Tensor) -> Result<Tensor>,
|
|
mass_matrix: Option<&Tensor>,
|
|
) -> Result<Tensor>; // scalar energy per batch element</code></pre>
|
|
<p>These operations are critical for:</p>
|
|
<ul><li><strong>HMM (HIP-0008)</strong>: Hamiltonian Market Maker uses symplectic integrators to evolve market state along energy-preserving trajectories, ensuring conservation laws that prevent arbitrage.</li><li><strong>HLLM (HIP-0002)</strong>: Hamiltonian LLMs use leapfrog integration for HMC sampling during inference, enabling Bayesian uncertainty quantification over token probabilities.</li><li><strong>Active Inference (HIP-0007)</strong>: Free energy minimization via variational inference requires Hamiltonian dynamics for efficient posterior sampling.</li></ul>
|
|
<h3>Model Formats</h3>
|
|
<p>| Format | Extension | Use Case | Quantization | Loading | |--------|-----------|----------|--------------|---------| | <strong>safetensors</strong> | <code>.safetensors</code> | Primary format for all models | No (full precision) | Memory-mapped, zero-copy | | <strong>GGUF</strong> | <code>.gguf</code> | Quantized models for CPU/Metal inference | Q4_0, Q4_K_M, Q5_K_M, Q8_0 | Streamed with on-the-fly dequant | | <strong>ONNX</strong> | <code>.onnx</code> | Import from external frameworks | INT8 (limited) | Full graph import |</p>
|
|
<p>safetensors is the REQUIRED format for model storage and distribution. GGUF is REQUIRED for quantized deployment. ONNX is OPTIONAL for import compatibility.</p>
|
|
<p><strong>Why safetensors, not pickle</strong>: Python pickle files (PyTorch's <code>.pt</code>/<code>.bin</code> format) can execute arbitrary code on load. A malicious model file can run <code>os.system("rm -rf /")</code> when loaded. safetensors is a simple binary format that contains only tensor metadata and raw bytes. It cannot execute code. For a system that loads models from external sources (HuggingFace Hub, user uploads), this is a non-negotiable security requirement.</p>
|
|
<p><strong>Why safetensors as primary, not GGUF</strong>: GGUF is excellent for quantized deployment but limited as a storage format. It encodes quantization into the file format -- you cannot store full-precision weights in GGUF without waste. safetensors stores full-precision weights that can be quantized at load time to any target precision. The workflow is: store in safetensors, quantize to GGUF at deployment time for the target hardware.</p>
|
|
<p><strong>Why not ONNX as primary</strong>: ONNX is a graph format, not a weight format. It encodes the model's computation graph alongside weights. This means ONNX files are specific to a model architecture version -- you cannot swap in new weights for the same architecture without re-exporting. safetensors stores weights independently of architecture, enabling weight-only updates.</p>
|
|
<h3>Quantization</h3>
|
|
<p>Quantization reduces model size and inference cost by representing weights in lower precision. The following quantization schemes MUST be supported:</p>
|
|
<p>| Scheme | Bits | Block Size | Quality | Speed | Memory | |--------|------|------------|---------|-------|--------| | <code>f16</code> | 16 | - | Baseline | 1x | 50% of f32 | | <code>INT8</code> | 8 | per-channel | ~99% of f16 | 1.5x | 50% of f16 | | <code>GPTQ</code> | 4 | 128 | ~97% of f16 | 2x | 25% of f16 | | <code>AWQ</code> | 4 | 128 | ~97% of f16 | 2x | 25% of f16 | | <code>Q4_K_M</code> | 4.5 (avg) | 256 (super-blocks) | ~96% of f16 | 2.5x (CPU) | 27% of f16 | | <code>Q5_K_M</code> | 5.5 (avg) | 256 (super-blocks) | ~98% of f16 | 2x (CPU) | 34% of f16 | | <code>Q8_0</code> | 8 | 32 | ~99.5% of f16 | 1.8x (CPU) | 50% of f16 |</p>
|
|
<p>GPTQ and AWQ are GPU-optimized (CUDA). Q4_K_M and Q5_K_M are CPU/Metal-optimized (GGUF format). For Apple Silicon deployment, Q4_K_M provides the best quality-per-TFLOP.</p>
|
|
<h3>Model Architectures</h3>
|
|
<p>The following architectures MUST have reference implementations in <code>candle-transformers</code>:</p>
|
|
<p>| Architecture | Type | Models | Key Operations | |--------------|------|--------|----------------| | <strong>Transformer (decoder)</strong> | Autoregressive LLM | LLaMA, Mistral, Phi, Falcon, Qwen, Zen | Causal attention, RoPE, GQA, MoE routing | | <strong>Transformer (encoder)</strong> | Embedding/classification | BERT, RoBERTa, BGE, E5 | Bidirectional attention, [CLS] pooling | | <strong>Transformer (encoder-decoder)</strong> | Seq2seq | T5, Whisper | Cross-attention, encoder KV cache | | <strong>Diffusion (UNet)</strong> | Image generation | Stable Diffusion, SDXL | Conv2d, GroupNorm, cross-attention | | <strong>Diffusion (DiT)</strong> | Image generation | Flux, SD3 | AdaLN, patchify, unpatchify | | <strong>Vision (ViT)</strong> | Image understanding | CLIP, SigLIP, DINOv2, BLIP, SAM | Patch embedding, CLS token | | <strong>Audio (Conformer)</strong> | Speech processing | Whisper, Encodec | Conv1d, relative attention, mel spectrogram |</p>
|
|
<h3>GPU Kernel Interface</h3>
|
|
<p>Custom CUDA and Metal kernels are essential for competitive performance on quantized operations and fused attention. Candle defines a kernel interface that backend-specific code must implement:</p>
|
|
<p><strong>CUDA Kernels</strong> (<code>candle-kernels/</code>):</p>
|
|
<pre><code class="lang-c">// Quantized matrix multiplication: dequantize Q4_K_M blocks on-the-fly
|
|
// during matmul, avoiding a separate dequantization pass.
|
|
__global__ void q4_k_m_matmul(
|
|
const void* __restrict__ a, // quantized weights [M, K] in Q4_K_M
|
|
const float* __restrict__ b, // activations [K, N] in f32
|
|
float* __restrict__ c, // output [M, N] in f32
|
|
int M, int K, int N
|
|
);
|
|
|
|
// Flash Attention v2: fused softmax(QK^T/sqrt(d))V with online softmax
|
|
// and tiling for O(N) memory instead of O(N^2).
|
|
__global__ void flash_attention_v2(
|
|
const half* __restrict__ Q, const half* __restrict__ K,
|
|
const half* __restrict__ V, half* __restrict__ O,
|
|
int batch, int heads, int seq_len, int head_dim
|
|
);
|
|
|
|
// Fused RMSNorm + residual add: avoids writing intermediate
|
|
// residual tensor to global memory.
|
|
__global__ void fused_rms_norm_residual(
|
|
const float* __restrict__ input,
|
|
const float* __restrict__ residual,
|
|
const float* __restrict__ weight,
|
|
float* __restrict__ output,
|
|
float eps, int hidden_size
|
|
);</code></pre>
|
|
<p><strong>Metal Kernels</strong> (<code>candle-metal-kernels/</code>):</p>
|
|
<pre><code class="lang-metal">// Custom Q4_K_M matmul for Apple Silicon: uses threadgroup memory
|
|
// for block dequantization, achieving 2-3x over naive implementation.
|
|
kernel void q4_k_m_matmul_metal(
|
|
device const void* a [[buffer(0)]],
|
|
device const float* b [[buffer(1)]],
|
|
device float* c [[buffer(2)]],
|
|
constant uint& M [[buffer(3)]],
|
|
constant uint& K [[buffer(4)]],
|
|
constant uint& N [[buffer(5)]],
|
|
uint2 gid [[thread_position_in_grid]],
|
|
uint2 lid [[thread_position_in_threadgroup]]
|
|
);</code></pre>
|
|
<p>Custom kernels are registered via the <code>CustomOp</code> trait, which allows backend-specific dispatch without modifying core tensor code:</p>
|
|
<pre><code class="lang-rust">pub trait CustomOp: Send + Sync {
|
|
fn name(&self) -> &str;
|
|
fn cpu_fwd(&self, storage: &CpuStorage, layout: &Layout) -> Result<(CpuStorage, Shape)>;
|
|
fn cuda_fwd(&self, storage: &CudaStorage, layout: &Layout) -> Result<(CudaStorage, Shape)>;
|
|
fn metal_fwd(&self, storage: &MetalStorage, layout: &Layout) -> Result<(MetalStorage, Shape)>;
|
|
}</code></pre>
|
|
<h3>WASM Compilation Pipeline</h3>
|
|
<p>The WASM pipeline compiles Candle to WebAssembly and packages it as an npm module for browser consumption. The repository carries per-model WASM examples (<code>hanzo-ml-wasm-examples/</code>), but no package is published to npm yet, so the pipeline and the package name below describe what this HIP proposes rather than something you can install today:</p>
|
|
<pre><code>Rust source (candle-core, candle-nn, candle-transformers)
|
|
|
|
|
v
|
|
wasm-pack build --target web --release
|
|
| (compiles to wasm32-unknown-unknown, runs wasm-opt)
|
|
v
|
|
pkg/
|
|
candle_wasm_bg.wasm (~2MB, gzipped ~800KB)
|
|
candle_wasm.js (JS bindings generated by wasm-bindgen)
|
|
candle_wasm.d.ts (TypeScript declarations)
|
|
|
|
|
v
|
|
npm publish @hanzoai/candle-wasm</code></pre>
|
|
<p>Browser usage:</p>
|
|
<pre><code class="lang-typescript">import init, { Model } from '@hanzoai/candle-wasm';
|
|
|
|
// Initialize WASM module
|
|
await init();
|
|
|
|
// Load a quantized model (fetched as ArrayBuffer)
|
|
const weights = await fetch('/models/phi-3-mini-q4.gguf')
|
|
.then(r => r.arrayBuffer());
|
|
|
|
const model = new Model(new Uint8Array(weights));
|
|
|
|
// Run inference
|
|
const tokens = model.encode("What is the capital of France?");
|
|
const output = model.generate(tokens, { max_tokens: 100, temperature: 0.7 });
|
|
const text = model.decode(output);</code></pre>
|
|
<p>WASM-specific constraints:</p>
|
|
<ul><li>No GPU access: All computation runs on CPU with optional SIMD128</li><li>Memory limit: Browsers limit WASM linear memory to 4GB (in practice, 2GB is safer)</li><li>No filesystem: Model weights must be fetched via HTTP and loaded from <code>ArrayBuffer</code></li><li>No threads by default: <code>SharedArrayBuffer</code> requires COOP/COEP headers; single-threaded fallback is mandatory</li></ul>
|
|
<h3>Crate Structure</h3>
|
|
<pre><code>candle/
|
|
candle-core/ # Tensor type, DType, Device, Storage, ops
|
|
src/
|
|
tensor.rs # Core Tensor struct and operations
|
|
dtype.rs # Data type definitions and conversions
|
|
device.rs # Device enum and backend dispatch
|
|
storage.rs # CpuStorage, CudaStorage, MetalStorage
|
|
op.rs # Operation enum for autograd graph
|
|
error.rs # CandleError type
|
|
layout.rs # Shape, stride, contiguity checks
|
|
backprop.rs # Backward pass (training only)
|
|
candle-nn/ # Neural network layers
|
|
src/
|
|
linear.rs # Linear (dense) layer
|
|
conv.rs # Conv1d, Conv2d
|
|
embedding.rs # Token/position embedding
|
|
layer_norm.rs # LayerNorm, RMSNorm
|
|
activation.rs # GELU, SiLU, ReLU
|
|
attention.rs # Multi-head attention, GQA, flash attention
|
|
rotary.rs # Rotary position embedding (RoPE)
|
|
var_builder.rs # Weight loading from safetensors
|
|
candle-transformers/ # Pre-built model architectures
|
|
src/models/
|
|
llama.rs # LLaMA 2/3 and derivatives
|
|
mistral.rs # Mistral, Mixtral (MoE)
|
|
phi.rs # Phi-2, Phi-3
|
|
falcon.rs # Falcon
|
|
qwen.rs # Qwen
|
|
stable_diffusion/ # SD 1.5, SDXL, SD3
|
|
whisper.rs # Whisper (speech-to-text)
|
|
clip.rs # CLIP (vision-language)
|
|
blip.rs # BLIP (image captioning)
|
|
sam.rs # SAM (segmentation)
|
|
bert.rs # BERT, RoBERTa
|
|
bge.rs # BGE (embedding)
|
|
quantized_llama.rs # GGUF quantized LLaMA
|
|
candle-hamiltonian/ # Physics-informed operations
|
|
src/
|
|
integrators.rs # Symplectic Euler, Leapfrog, Yoshida
|
|
energy.rs # Hamiltonian energy computation
|
|
hmc.rs # Hamiltonian Monte Carlo sampling
|
|
candle-metal-kernels/ # Custom Metal compute shaders
|
|
src/
|
|
quantized.metal # Q4/Q5 dequantization + matmul
|
|
attention.metal # Flash attention for Metal
|
|
layernorm.metal # Fused LayerNorm kernel
|
|
candle-kernels/ # Custom CUDA kernels
|
|
src/
|
|
quantized.cu # Q4/Q5 dequantization + matmul
|
|
flash_attn.cu # Flash attention v2
|
|
fused_ops.cu # Fused RMSNorm + residual
|
|
candle-wasm/ # WASM build target and JS bindings
|
|
src/
|
|
lib.rs # wasm-bindgen entry points
|
|
model.rs # High-level Model API for JS
|
|
pkg/ # wasm-pack output (gitignored)</code></pre>
|
|
<h3>API Surface</h3>
|
|
<p>The public API is organized into three layers:</p>
|
|
<p><strong>Layer 1: Tensor operations</strong> (<code>candle-core</code>)</p>
|
|
<pre><code class="lang-rust">use candle_core::{Device, DType, Tensor, Result};
|
|
|
|
// Create tensors
|
|
let device = Device::Metal(0);
|
|
let x = Tensor::randn(0f32, 1.0, (2, 3), &device)?;
|
|
let w = Tensor::zeros((3, 4), DType::F32, &device)?;
|
|
|
|
// Operations
|
|
let y = x.matmul(&w)?; // [2, 4]
|
|
let y = y.relu()?; // element-wise ReLU
|
|
let y = candle_nn::ops::softmax(&y, 1)?; // softmax over dim 1
|
|
|
|
// Device transfer
|
|
let y_cpu = y.to_device(&Device::Cpu)?;
|
|
let data: Vec<f32> = y_cpu.to_vec2()?;</code></pre>
|
|
<p><strong>Layer 2: Neural network layers</strong> (<code>candle-nn</code>)</p>
|
|
<pre><code class="lang-rust">use candle_nn::{Linear, LayerNorm, Module, VarBuilder};
|
|
|
|
// Load weights from safetensors
|
|
let vb = VarBuilder::from_safetensors(
|
|
&["model.safetensors"],
|
|
DType::F16,
|
|
&Device::Cuda(0),
|
|
)?;
|
|
|
|
// Build layers
|
|
let linear = Linear::new(vb.pp("fc1"), 768, 3072)?;
|
|
let norm = LayerNorm::new(vb.pp("ln"), 768, 1e-5)?;
|
|
|
|
// Forward pass
|
|
let x = norm.forward(&input)?;
|
|
let x = linear.forward(&x)?;</code></pre>
|
|
<p><strong>Layer 3: Complete models</strong> (<code>candle-transformers</code>)</p>
|
|
<pre><code class="lang-rust">use candle_transformers::models::llama::{Llama, Config};
|
|
|
|
// Load model
|
|
let config = Config::from_file("config.json")?;
|
|
let model = Llama::load(&vb, &config)?;
|
|
|
|
// Generate
|
|
let tokens = tokenizer.encode("Hello, world")?;
|
|
let input = Tensor::new(tokens.as_slice(), &device)?;
|
|
let logits = model.forward(&input, 0)?; // position offset = 0
|
|
let next_token = logits.argmax(D::Minus1)?;</code></pre>
|
|
<h2>Implementation</h2>
|
|
<h3>Repository</h3>
|
|
<p><a href="https://github.com/hanzoai/ml">github.com/hanzoai/ml</a> -- fork of <a href="https://github.com/huggingface/candle">huggingface/candle</a> with Hanzo extensions.</p>
|
|
<h3>Build</h3>
|
|
<pre><code class="lang-bash"># CPU only (all platforms)
|
|
cargo build --release
|
|
|
|
# CUDA (NVIDIA GPUs, Linux)
|
|
cargo build --release --features cuda
|
|
|
|
# Metal (Apple Silicon, macOS)
|
|
cargo build --release --features metal
|
|
|
|
# WASM (browser/edge)
|
|
wasm-pack build candle-wasm --target web --release
|
|
|
|
# All features (development)
|
|
cargo build --release --features "cuda metal"</code></pre>
|
|
<h3>Integration Points</h3>
|
|
<p>| System | Integration | Details | |--------|-------------|---------| | <strong>Zen Gateway</strong> (HIP-0039) | Edge inference | Quantized Zen models served via Candle on CPU/Metal for low-latency edge nodes | | <strong>Studio</strong> (HIP-1211) | Diffusion inference | Stable Diffusion / Flux pipelines run on Candle Metal backend for Apple Silicon users | | <strong>Jin</strong> (HIP-0003) | Multimodal backbone | Jin model architectures implemented as Candle modules in <code>candle-transformers</code> | | <strong>LLM Gateway</strong> (HIP-0004) | Local model serving | Gateway routes to local Candle inference workers for on-premise deployments | | <strong>Object Storage</strong> (HIP-0405) | Model distribution | Model weights stored in safetensors/GGUF format in Hanzo Object Storage, loaded by Candle | | <strong>MCP</strong> (HIP-0010) | Tool inference | MCP tool servers embed Candle for classification, embedding, and small-model inference | | <strong>Node</strong> (HIP-0020) | Verified inference | Blockchain nodes use Candle for deterministic inference with reproducible outputs | | <strong>HMM</strong> (HIP-0008) | Market dynamics | Hamiltonian Market Maker uses <code>candle-hamiltonian</code> for symplectic market state evolution |</p>
|
|
<h3>Execution Flow</h3>
|
|
<pre><code>User Request
|
|
|
|
|
v
|
|
Zen Gateway (HIP-0039) / LLM Gateway (HIP-0004)
|
|
|
|
|
v
|
|
Model Router (selects model + quantization)
|
|
|
|
|
v
|
|
Candle Inference Worker
|
|
|
|
|
+---> Tokenizer (HuggingFace tokenizers, Rust)
|
|
| |
|
|
| v
|
|
+---> Model Forward Pass (candle-transformers)
|
|
| |
|
|
| +---> Embedding lookup
|
|
| +---> N x Transformer Block
|
|
| | +---> RMSNorm
|
|
| | +---> Multi-Head Attention (with KV cache)
|
|
| | +---> RMSNorm
|
|
| | +---> FFN (or MoE routing)
|
|
| +---> Final LayerNorm
|
|
| +---> LM Head (matmul to vocab)
|
|
| v
|
|
+---> Sampling (temperature, top-p, top-k)
|
|
| |
|
|
| v
|
|
+---> Detokenize
|
|
|
|
|
v
|
|
Response (streamed tokens)</code></pre>
|
|
<h3>Performance Characteristics</h3>
|
|
<p>Benchmarks on Apple M3 Max (36GB unified memory), LLaMA-2 7B:</p>
|
|
<p>| Configuration | Tokens/sec | Memory | Startup | |---------------|------------|--------|---------| | PyTorch f16 (MPS) | ~25 t/s | 14GB | 8s | | llama.cpp Q4_K_M | ~55 t/s | 4.2GB | 0.3s | | Candle f16 (Metal) | ~30 t/s | 13GB | 0.1s | | Candle Q4_K_M (Metal) | ~50 t/s | 4.2GB | 0.1s | | Candle Q4_K_M (CPU, Accelerate) | ~15 t/s | 4.2GB | 0.08s |</p>
|
|
<p>Benchmarks on NVIDIA A100 (80GB), LLaMA-2 7B:</p>
|
|
<p>| Configuration | Tokens/sec | Memory | Startup | |---------------|------------|--------|---------| | vLLM f16 (CUDA) | ~120 t/s | 14GB | 15s | | Candle f16 (CUDA) | ~95 t/s | 13.5GB | 0.2s | | Candle GPTQ-4bit (CUDA) | ~140 t/s | 4GB | 0.2s | | TensorRT-LLM INT4 | ~180 t/s | 4GB | 30s |</p>
|
|
<p>Benchmarks for WASM (Chrome 120, M3 MacBook Pro), Phi-3-mini 3.8B Q4:</p>
|
|
<p>| Configuration | Tokens/sec | WASM Size | Model Size | |---------------|------------|-----------|------------| | Candle WASM (SIMD128) | ~8 t/s | 2.1MB | 2.0GB | | Candle WASM (no SIMD) | ~3 t/s | 1.9MB | 2.0GB | | ONNX Runtime WASM | ~5 t/s | 18MB | 2.2GB (ONNX) |</p>
|
|
<p>Key observations:</p>
|
|
<ul><li>Candle is not the fastest on raw throughput (TensorRT-LLM and vLLM win on GPU). Its advantage is startup time, memory efficiency, and deployment simplicity.</li><li>On Apple Silicon, Candle with custom Metal kernels approaches llama.cpp performance while offering a much cleaner Rust API.</li><li>For WASM deployment, Candle is the only viable option among production-quality frameworks. The WASM binary is 10x smaller than ONNX Runtime's WASM build.</li></ul>
|
|
<h3>Error Handling</h3>
|
|
<p>All operations return <code>candle_core::Result<T></code>, which is <code>std::result::Result<T, candle_core::Error></code>. Error variants:</p>
|
|
<pre><code class="lang-rust">pub enum Error {
|
|
// Shape errors
|
|
ShapeMismatch { expected: Shape, got: Shape, op: &'static str },
|
|
UnexpectedNumberOfDims { expected: usize, got: usize },
|
|
|
|
// Device errors
|
|
DeviceMismatch { lhs: Device, rhs: Device, op: &'static str },
|
|
NotAvailable(&'static str), // e.g., "CUDA not available"
|
|
|
|
// Type errors
|
|
DTypeMismatch { expected: DType, got: DType, op: &'static str },
|
|
UnsupportedDTypeForOp { dtype: DType, op: &'static str },
|
|
|
|
// Backend errors
|
|
Cuda(CudaError),
|
|
Metal(MetalError),
|
|
|
|
// I/O
|
|
SafetensorError(safetensors::Error),
|
|
Io(std::io::Error),
|
|
|
|
// Generic
|
|
Msg(String),
|
|
Wrapped(Box<dyn std::error::Error + Send + Sync>),
|
|
}</code></pre>
|
|
<p>Errors are precise and actionable. A <code>ShapeMismatch</code> tells you the expected shape, the actual shape, and which operation failed. No guessing.</p>
|
|
<h2>Security Considerations</h2>
|
|
<ul><li><strong>Model file validation</strong>: safetensors format is designed to be safe against arbitrary code execution (unlike pickle-based formats). GGUF files contain only tensor data and metadata. Neither format can execute code on load.</li><li><strong>Memory safety</strong>: Rust's ownership model prevents buffer overflows in tensor operations. CUDA and Metal kernels are the primary attack surface -- these are audited manually and fuzzed with random tensor shapes.</li><li><strong>WASM sandboxing</strong>: Browser-based inference runs in the WASM sandbox. The model cannot access the filesystem, network, or other browser APIs unless explicitly granted by the host application.</li><li><strong>Supply chain</strong>: The <code>candle-core</code> crate has minimal dependencies (num-traits, half, safetensors, memmap2). No transitive dependency on OpenSSL or other C libraries for the CPU backend.</li><li><strong>Quantized weight integrity</strong>: GGUF files loaded from untrusted sources MUST have their header checksums verified before tensor data is accessed. Malformed block sizes or dimension metadata could cause out-of-bounds reads in dequantization kernels.</li></ul>
|
|
<h2>Test Vectors</h2>
|
|
<p>Reference implementations MUST pass the following test cases:</p>
|
|
<pre><code class="lang-rust">#[test]
|
|
fn test_matmul_basic() {
|
|
let a = Tensor::new(&[[1f32, 2.], [3., 4.]], &Device::Cpu).unwrap();
|
|
let b = Tensor::new(&[[5f32, 6.], [7., 8.]], &Device::Cpu).unwrap();
|
|
let c = a.matmul(&b).unwrap();
|
|
assert_eq!(c.to_vec2::<f32>().unwrap(), &[[19., 22.], [43., 50.]]);
|
|
}
|
|
|
|
#[test]
|
|
fn test_softmax() {
|
|
let x = Tensor::new(&[1f32, 2., 3.], &Device::Cpu).unwrap();
|
|
let y = candle_nn::ops::softmax(&x, 0).unwrap();
|
|
let vals = y.to_vec1::<f32>().unwrap();
|
|
assert!((vals[0] - 0.0900).abs() < 1e-3);
|
|
assert!((vals[1] - 0.2447).abs() < 1e-3);
|
|
assert!((vals[2] - 0.6652).abs() < 1e-3);
|
|
}
|
|
|
|
#[test]
|
|
fn test_rope_embedding() {
|
|
// RoPE must produce position-dependent embeddings
|
|
let x = Tensor::ones((1, 1, 4, 64), DType::F32, &Device::Cpu).unwrap();
|
|
let cos = Tensor::ones((4, 64), DType::F32, &Device::Cpu).unwrap();
|
|
let sin = Tensor::zeros((4, 64), DType::F32, &Device::Cpu).unwrap();
|
|
let y = candle_nn::rotary::apply_rotary_emb(&x, &cos, &sin).unwrap();
|
|
// With cos=1, sin=0, output should equal input
|
|
let diff = (&x - &y).unwrap().abs().unwrap().sum_all().unwrap();
|
|
assert_eq!(diff.to_scalar::<f32>().unwrap(), 0.0);
|
|
}
|
|
|
|
#[test]
|
|
fn test_device_transfer() {
|
|
let x = Tensor::randn(0f32, 1.0, (2, 3), &Device::Cpu).unwrap();
|
|
// Round-trip through another device should preserve values
|
|
let x2 = x.to_device(&Device::Cpu).unwrap();
|
|
let diff = (x - x2).unwrap().abs().unwrap().sum_all().unwrap();
|
|
assert_eq!(diff.to_scalar::<f32>().unwrap(), 0.0);
|
|
}
|
|
|
|
#[test]
|
|
fn test_symplectic_energy_conservation() {
|
|
// Symplectic integrator must conserve energy to within numerical precision
|
|
let q = Tensor::new(&[1.0f64, 0.0], &Device::Cpu).unwrap();
|
|
let p = Tensor::new(&[0.0f64, 1.0], &Device::Cpu).unwrap();
|
|
let h0 = hamiltonian_energy(&q, &p, harmonic_potential, None).unwrap();
|
|
let (q1, p1) = leapfrog_step(&q, &p, harmonic_grad, 0.01, 1000).unwrap();
|
|
let h1 = hamiltonian_energy(&q1, &p1, harmonic_potential, None).unwrap();
|
|
let drift = (h1 - h0).unwrap().abs().unwrap().to_scalar::<f64>().unwrap();
|
|
assert!(drift < 1e-6, "Energy drift {} exceeds tolerance", drift);
|
|
}
|
|
|
|
#[test]
|
|
fn test_quantized_load() {
|
|
// GGUF quantized model must load and produce valid logits
|
|
let model = QuantizedLlama::load("test_model.gguf", &Device::Cpu).unwrap();
|
|
let input = Tensor::new(&[1u32, 2, 3], &Device::Cpu).unwrap();
|
|
let logits = model.forward(&input, 0).unwrap();
|
|
assert_eq!(logits.dims(), &[1, 3, VOCAB_SIZE]);
|
|
// Logits must be finite
|
|
assert!(logits.to_vec3::<f32>().unwrap().iter()
|
|
.all(|batch| batch.iter().all(|seq| seq.iter().all(|v| v.is_finite()))));
|
|
}</code></pre>
|
|
<h2>References</h2>
|
|
<ol><li><a href="./hip-0002-hamiltonian-large-language-models-hllms-specification.md">HIP-0002: Hamiltonian Large Language Models</a></li><li><a href="./hip-0003-jin-multimodal-ai-architecture.md">HIP-0003: Jin Multimodal AI Architecture</a></li><li><a href="./hip-0004-llm-gateway-unified-ai-provider-interface.md">HIP-0004: LLM Gateway</a></li><li><a href="./hip-0007-active-inference-integration-for-hamiltonian-llms.md">HIP-0007: Active Inference Integration</a></li><li><a href="./hip-0008-hmm-hanzo-market-maker-native-dex-for-ai-compute-resources.md">HIP-0008: HMM Hanzo Market Maker</a></li><li><a href="./hip-0010-model-context-protocol-mcp-integration-standards.md">HIP-0010: Model Context Protocol</a></li><li><a href="./hip-0020-blockchain-node-standard.md">HIP-0020: Blockchain Node Standard</a></li><li><a href="./hip-0405-s3-crd.md">HIP-0405: S3 CRD</a></li><li><a href="./hip-1211-ai-the-model-api.md">HIP-1211: AI — The Model API</a></li><li><a href="./hip-0039-zen-model-architecture.md">HIP-0039: Zen Model Architecture</a></li><li><a href="https://github.com/huggingface/candle">HuggingFace Candle</a> -- upstream repository</li><li><a href="https://huggingface.co/docs/safetensors/">safetensors format</a> -- model serialization standard</li><li><a href="https://github.com/ggerganov/ggml/blob/master/docs/gguf.md">GGUF format</a> -- quantized model format</li><li><a href="https://en.wikipedia.org/wiki/Leapfrog_integration">Leapfrog integration</a> -- symplectic integrator theory</li></ol>
|
|
<h2>Copyright</h2>
|
|
<p>Copyright and related rights waived via <a href="https://creativecommons.org/publicdomain/zero/1.0/">CC0</a>.</p>
|
|
<footer>
|
|
<div class="fmap">
|
|
<div><b>Build</b>
|
|
<a href="https://hanzo.app">hanzo.app</a> app builder
|
|
<a href="https://hanzo.codes">hanzo.codes</a> code with an agent
|
|
<a href="https://hanzoskills.com">hanzoskills.com</a> what an agent reads</div>
|
|
<div><b>Run</b>
|
|
<a href="https://console.hanzo.ai">console.hanzo.ai</a> cloud console
|
|
<a href="https://api.hanzo.ai">api.hanzo.ai</a> the API</div>
|
|
<div><b>Account</b>
|
|
<a href="https://hanzo.id">hanzo.id</a> sign in
|
|
<a href="https://pay.hanzo.ai">pay.hanzo.ai</a> top up
|
|
<a href="https://billing.hanzo.ai">billing.hanzo.ai</a> invoices and usage</div>
|
|
<div><b>Company</b>
|
|
<a href="https://hanzo.ai">hanzo.ai</a> canonical
|
|
<a href="https://hanzo.ventures">hanzo.ventures</a> ventures</div>
|
|
</div>
|
|
<div class="fend"><a href="/hip-0019.md">This page as plain text</a> · <a href="/llms.txt">Index for agents</a></div>
|
|
</footer>
|
|
</div></body></html> |