mirror of
https://github.com/hanzoai/skills.git
synced 2026-11-19 00:47:36 +00:00
hanzoskills.com/llms.txt now opens with the install line, the API reference index and the company index, so an agent that lands on any one of the three can find the rest. robots.txt claimed 537 documents while llms.txt counts 920. The number is gone from robots.txt; llms.txt derives it. public/ is regenerated from scripts/site.py, which it had fallen behind: the copy-prompt menu and agent bar were in the generator and not in the committed pages. tests/site.mjs passes against the result. Co-authored-by: Hanzo Dev <[email protected]>
613 lines
42 KiB
HTML
613 lines
42 KiB
HTML
<!doctype html>
|
|
<html lang="en"><head>
|
|
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
<title>hip-0043</title>
|
|
<meta name="color-scheme" content="dark">
|
|
<style>
|
|
/* Committed to one visual world: a dark instrument panel. No light variant —
|
|
the dot field and the hairline rules only read on black, and a naive
|
|
inversion would wash both out. Every colour is painted explicitly so the page
|
|
never borrows a host background. */
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Pixel Square';src:url('/fonts/ZenPixel-Square.woff2') format('woff2');font-style:normal;font-display:swap}
|
|
|
|
:root{
|
|
--ink:#f4f4f2; --soft:#9c9c98; --faint:#6a6a66; --ground:#000; --panel:#0a0a0a;
|
|
--line:#232323; --line-hi:#3a3a3a; --accent:#f4f4f2;
|
|
--font-zen-sans:'Zen',-apple-system,BlinkMacSystemFont,"Segoe UI",Helvetica,Arial,sans-serif;
|
|
--font-zen-mono:'Zen Mono',ui-monospace,"SF Mono",SFMono-Regular,Menlo,Consolas,monospace;
|
|
--font-zen-pixel-square:'Zen Pixel Square',monospace;
|
|
--mono:var(--font-zen-mono);
|
|
--sans:var(--font-zen-sans);
|
|
/* Controls are pills and surfaces are soft, as on hanzo.ai. */
|
|
--r:999px; --r-soft:14px;
|
|
}
|
|
*{box-sizing:border-box}
|
|
html{-webkit-text-size-adjust:100%}
|
|
body{margin:0;background:var(--ground);color:var(--ink);font-family:var(--sans);
|
|
font-size:16px;line-height:1.65;-webkit-font-smoothing:antialiased}
|
|
.wrap{max-width:60rem;margin:0 auto;padding:0 1.75rem}
|
|
|
|
/* The dot field. A halftone made of one repeating radial-gradient, faded out
|
|
along the diagonal by a mask so it reads as a wave rather than wallpaper. */
|
|
/* No overflow clip here: the dot field is inset to the box already, and a clip
|
|
would cut off the copy-prompt menu. */
|
|
.hero{position:relative;border-bottom:1px solid var(--line)}
|
|
.hero::before{content:"";position:absolute;inset:0;
|
|
background-image:radial-gradient(var(--line-hi) 1px, transparent 1px);
|
|
background-size:7px 7px;
|
|
-webkit-mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
pointer-events:none}
|
|
.hero .wrap{position:relative;padding-top:5.5rem;padding-bottom:4rem}
|
|
|
|
nav{border-bottom:1px solid var(--line)}
|
|
nav .in{max-width:60rem;margin:0 auto;padding:1.1rem 1.75rem;display:flex;
|
|
justify-content:space-between;align-items:center;gap:1rem}
|
|
.brand{font-family:var(--mono);font-size:.85rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--ink);text-decoration:none;
|
|
display:inline-flex;align-items:center;gap:.6rem}
|
|
.brand i{font-style:normal;color:var(--faint)}
|
|
.mark{width:1.15em;height:1.15em;display:block;flex:0 0 auto}
|
|
nav a.up{font-family:var(--mono);font-size:.72rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--soft);text-decoration:none}
|
|
nav a.up:hover{color:var(--ink)}
|
|
|
|
h1{font-family:var(--mono);font-size:clamp(2rem,5.4vw,3.4rem);line-height:1.12;
|
|
letter-spacing:-.02em;margin:0 0 1.1rem;font-weight:500;text-wrap:balance}
|
|
h1 em{font-style:normal;color:var(--faint)}
|
|
.lede{color:var(--soft);font-size:1.05rem;max-width:52ch;margin:0 0 2.25rem}
|
|
|
|
.btns{display:flex;flex-wrap:wrap;gap:.7rem;margin:0 0 .5rem;align-items:center}
|
|
.btn{display:inline-flex;align-items:center;gap:.5rem;font-family:var(--mono);
|
|
font-size:.72rem;letter-spacing:.14em;text-transform:uppercase;
|
|
padding:.8rem 1.4rem;border:1px solid var(--line-hi);border-radius:var(--r);
|
|
color:var(--ink);text-decoration:none;background:transparent;cursor:pointer;
|
|
line-height:1.2;font-weight:400}
|
|
.btn:hover{background:#141414;text-decoration:none;border-color:var(--soft)}
|
|
.btn.solid{background:var(--ink);color:#000;border-color:var(--ink)}
|
|
.btn.solid:hover{background:#d8d8d4}
|
|
.btn .chev{width:.7em;height:.7em;flex:0 0 auto;transition:transform .15s ease}
|
|
.btn[aria-expanded="true"] .chev{transform:rotate(180deg)}
|
|
|
|
/* Copy-prompt selector. Each row copies the prompt for its own target. */
|
|
.copy{position:relative;display:inline-flex}
|
|
.menu{position:absolute;top:calc(100% + .5rem);left:0;z-index:20;min-width:21rem;
|
|
background:var(--panel);border:1px solid var(--line-hi);border-radius:var(--r-soft);
|
|
padding:.4rem;display:flex;flex-direction:column;
|
|
box-shadow:0 12px 32px rgba(0,0,0,.75)}
|
|
.menu[hidden]{display:none}
|
|
.mi{display:flex;align-items:center;gap:.7rem;width:100%;text-align:left;
|
|
padding:.62rem .8rem;border:0;border-radius:calc(var(--r-soft) - .35rem);
|
|
background:transparent;color:var(--ink);font-family:var(--mono);font-size:.76rem;
|
|
letter-spacing:.04em;text-decoration:none;cursor:pointer;line-height:1.3;
|
|
white-space:nowrap}
|
|
.mi:hover,.mi:focus-visible{background:#181818;outline:none}
|
|
.mi .mark{width:1.05rem;height:1.05rem;color:var(--soft)}
|
|
.mi:hover .mark{color:var(--ink)}
|
|
.mi u{margin-left:auto;color:var(--faint);text-decoration:none;font-size:.68rem;
|
|
letter-spacing:.1em;text-transform:uppercase}
|
|
|
|
/* Works-with bar: one scrolling row, name and mark per agent. */
|
|
.strip{border-bottom:1px solid var(--line);background:var(--panel)}
|
|
.strip .in{max-width:60rem;margin:0 auto;padding:.9rem 1.75rem;display:flex;
|
|
align-items:center;gap:1.25rem;font-family:var(--mono);font-size:.72rem;
|
|
letter-spacing:.1em;text-transform:uppercase;color:var(--faint)}
|
|
.strip b{color:var(--soft);font-weight:400;flex:0 0 auto;
|
|
border-right:1px solid var(--line);padding-right:1.25rem;align-self:stretch;
|
|
display:flex;align-items:center}
|
|
.bar{display:flex;align-items:center;gap:.5rem;flex:1 1 auto;min-width:0;
|
|
overflow-x:auto;scrollbar-width:none;scroll-snap-type:x proximity;
|
|
-webkit-overflow-scrolling:touch;
|
|
-webkit-mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent);
|
|
mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent)}
|
|
.bar::-webkit-scrollbar{display:none}
|
|
.agent{display:inline-flex;align-items:center;gap:.45rem;flex:0 0 auto;
|
|
padding:.4rem .85rem;border:1px solid var(--line);border-radius:var(--r);
|
|
color:var(--soft);white-space:nowrap;scroll-snap-align:start;
|
|
text-decoration:none;background:var(--ground)}
|
|
.agent:hover{color:var(--ink);border-color:var(--line-hi);text-decoration:none}
|
|
.agent .mark{width:.95rem;height:.95rem}
|
|
|
|
h2{font-family:var(--mono);font-size:1.1rem;letter-spacing:-.01em;margin:2.75rem 0 .75rem;font-weight:500}
|
|
h3{font-size:1rem;margin:2rem 0 .5rem;font-weight:600}
|
|
h4,h5,h6{font-size:.94rem;margin:1.5rem 0 .5rem;font-weight:600}
|
|
p{margin:0 0 1rem;max-width:70ch}
|
|
ul,ol{margin:0 0 1rem;padding-left:1.35rem;max-width:70ch}
|
|
li{margin:.22rem 0}
|
|
a{color:var(--ink);text-decoration:underline;text-underline-offset:3px;
|
|
text-decoration-color:var(--line-hi)}
|
|
a:hover{text-decoration-color:var(--ink)}
|
|
a:focus-visible{outline:1px solid var(--ink);outline-offset:3px}
|
|
code{font-family:var(--mono);font-size:.85em;background:var(--panel);
|
|
border:1px solid var(--line);border-radius:5px;padding:.1em .36em}
|
|
pre{background:var(--panel);border:1px solid var(--line);padding:1rem 1.1rem;
|
|
border-radius:var(--r-soft);overflow-x:auto;margin:0 0 1.25rem}
|
|
pre code{background:none;border:0;padding:0;font-size:.82rem;line-height:1.7}
|
|
blockquote{margin:0 0 1rem;padding-left:1rem;border-left:1px solid var(--line-hi);color:var(--soft)}
|
|
hr{border:0;border-top:1px solid var(--line);margin:2.5rem 0}
|
|
table{border-collapse:collapse;width:100%;margin:0 0 1.25rem;font-size:.92rem;
|
|
display:block;overflow-x:auto}
|
|
th,td{border:1px solid var(--line);padding:.5rem .7rem;text-align:left}
|
|
|
|
.curl{display:block;background:var(--panel);border:1px solid var(--line);
|
|
border-radius:var(--r-soft);padding:.95rem 1.1rem;font-family:var(--mono);
|
|
font-size:.8rem;color:var(--soft);overflow-x:auto;margin:0 0 2.5rem;
|
|
white-space:pre;line-height:1.9}
|
|
.curl b{color:var(--ink);font-weight:400}
|
|
.curl i{font-style:normal;color:var(--faint)}
|
|
|
|
/* index */
|
|
#q{width:100%;padding:.85rem 1.2rem;font-family:var(--mono);font-size:.85rem;color:var(--ink);
|
|
background:var(--panel);border:1px solid var(--line);border-radius:var(--r);margin:0 0 .6rem}
|
|
#q::placeholder{color:var(--faint);text-transform:uppercase;letter-spacing:.1em;font-size:.72rem}
|
|
#q:focus{outline:none;border-color:var(--soft)}
|
|
#count{color:var(--faint);font-family:var(--mono);font-size:.72rem;letter-spacing:.1em;
|
|
text-transform:uppercase;margin:0 0 2rem}
|
|
.cat{margin:2.5rem 0 .5rem;font-family:var(--mono);font-size:.7rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);border-bottom:1px solid var(--line);
|
|
padding-bottom:.45rem}
|
|
.row{display:block;padding:.6rem .9rem;margin:0 -.9rem;border-bottom:1px solid var(--line);
|
|
border-radius:var(--r-soft);text-decoration:none}
|
|
.row:hover{background:#0a0a0a}
|
|
.nm{font-family:var(--mono);font-size:.84rem;color:var(--ink);display:block}
|
|
.ds{color:var(--soft);font-size:.9rem;display:block;margin-top:.12rem}
|
|
footer{margin-top:4rem;padding:2rem 0 4rem;border-top:1px solid var(--line);
|
|
color:var(--faint);font-size:.85rem}
|
|
footer a{color:var(--soft)}
|
|
.fmap{display:grid;gap:1.75rem;grid-template-columns:repeat(auto-fit,minmax(11rem,1fr));
|
|
margin:0 0 2rem;font-size:.85rem}
|
|
.fmap b{display:block;font-family:var(--mono);font-size:.68rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);font-weight:400;margin:0 0 .6rem;
|
|
border-bottom:1px solid var(--line);padding-bottom:.4rem}
|
|
.fmap a{display:inline-block;color:var(--ink);text-decoration:none;margin:.3rem .35rem 0 0}
|
|
.fmap a:hover{text-decoration:underline}
|
|
.fmap div{color:var(--faint);line-height:1.5}
|
|
.fend{border-top:1px solid var(--line);padding-top:1.25rem;font-size:.82rem}
|
|
|
|
@media (max-width:640px){
|
|
.hero .wrap{padding-top:3.25rem;padding-bottom:2.75rem}
|
|
.btns{gap:.5rem}
|
|
.btn{padding:.75rem 1.15rem;font-size:.68rem}
|
|
.copy,.copy .btn{width:100%;justify-content:center}
|
|
.menu{left:0;right:0;min-width:0}
|
|
.strip .in{flex-direction:column;align-items:flex-start;gap:.6rem;padding:.9rem 1.75rem}
|
|
.strip b{border-right:0;padding-right:0}
|
|
.bar{width:100%}
|
|
}
|
|
</style>
|
|
<link rel="alternate" type="text/markdown" href="/hip-0043.md">
|
|
|
|
<link rel="icon" href="/favicon.svg">
|
|
<link rel="canonical" href="https://hanzoskills.com/hip-0043">
|
|
</head>
|
|
<!-- Reading this with a machine? Every page here is also plain markdown.
|
|
This one: https://hanzoskills.com/hip-0043.md
|
|
Index: https://hanzoskills.com/llms.txt
|
|
Corpus: https://hanzoskills.com/all.md
|
|
Append .md to any path for the markdown twin. -->
|
|
<body>
|
|
<nav><div class="in">
|
|
<a class="brand" href="/"><svg class="mark" viewBox="0 0 67 67" fill="currentColor" aria-hidden="true"><path d="M22.21 67V44.6369H0V67H22.21Z"/><path d="M66.7038 22.3184H22.2534L0.0878906 44.6367H44.4634L66.7038 22.3184Z"/><path d="M22.21 0H0V22.3184H22.21V0Z"/><path d="M66.7198 0H44.5098V22.3184H66.7198V0Z"/><path d="M66.7198 67V44.6369H44.5098V67H66.7198Z"/></svg> HANZO<i>SKILLS</i></a>
|
|
<a class="up" href="/">All skills</a>
|
|
</div></nav>
|
|
|
|
<div class="wrap">
|
|
<h1>hip-0043</h1><p class="lede">HIP-43: Hanzo Engine — LLM Inference Engine Standard. Status Final. Hanzo architectural specification.</p><h1>HIP-0043: Hanzo Engine — LLM Inference Engine Standard</h1>
|
|
<h2>Abstract</h2>
|
|
<p>This proposal defines Hanzo Engine, the high-performance LLM inference runtime for cloud and datacenter GPU deployment. The engine serves Zen models (HIP-0039) and powers <code>llm.hanzo.ai</code> cloud inference.</p>
|
|
<p>The engine is a Rust-based inference server forked from mistral.rs (<a href="https://github.com/hanzoai/engine">github.com/hanzoai/engine</a>), extended with Hanzo-specific optimizations for PagedAttention, FlashAttention, in-situ quantization (ISQ), speculative decoding, and continuous batching. It supports <strong>60+ model architectures</strong> across CUDA, Metal, and CPU backends. It exposes an OpenAI-compatible HTTP API, a Rust SDK, a Python SDK (via PyO3), and an MCP interface, and serves as the execution backend for the LLM Gateway (HIP-0004) and Zen Gateway.</p>
|
|
<p>The engine supports GGUF, SafeTensors, AWQ, GPTQ, HQQ, FP8, and AFQ model formats. Quantization options include ISQ (in-situ), GGUF, GPTQ, AWQ, HQQ, FP8, and AFQ. It is designed for deterministic memory management, sub-millisecond scheduling latency, and zero-downtime model swaps -- properties that are difficult or impossible to achieve with Python-based serving frameworks.</p>
|
|
<p>The engine is built on the Hanzo ML framework (Candle, at <code>~/work/hanzo/ml</code>) for core tensor operations and GPU kernel dispatch.</p>
|
|
<p><strong>Repository</strong>: <a href="https://github.com/hanzoai/engine">github.com/hanzoai/engine</a> (mistral.rs fork, Rust) <strong>ML Framework</strong>: <a href="https://github.com/hanzoai/ml">github.com/hanzoai/ml</a> (Candle, Rust) <strong>Production</strong>: <code>llm.hanzo.ai</code> (cloud inference) <strong>Port</strong>: 8080 <strong>Binary</strong>: <code>hanzo-engine</code> <strong>Container</strong>: <code>hanzoai/engine:latest</code> <strong>APIs</strong>: OpenAI-compatible HTTP, Rust SDK, Python SDK (PyO3), MCP</p>
|
|
<h2>Motivation</h2>
|
|
<p>HIP-0039 defines the Zen model family and the Zen Gateway that routes requests to the optimal model size. HIP-0004 defines the LLM Gateway that proxies across 100+ providers. Neither specifies how models are actually loaded into GPU memory, how tokens are generated, or how concurrent requests share hardware resources. That is the domain of the inference engine.</p>
|
|
<p>Production model serving has concrete problems that existing tools address incompletely:</p>
|
|
<ol><li><strong>OOM kills are unacceptable in production.</strong> Python-based servers (vLLM, TGI) rely on garbage collection for memory reclamation. Under sustained load with variable-length sequences, GC pauses cause memory spikes that exceed GPU capacity. The process is killed by the OOM killer, dropping all in-flight requests. Rust's deterministic memory model eliminates this failure mode.</li></ol>
|
|
<ol><li><strong>The Python GIL serializes request scheduling.</strong> vLLM and TGI schedule requests in Python. Under high concurrency, the GIL becomes a bottleneck for the scheduling loop itself -- not the GPU compute, but the CPU-side decision of which requests to batch together. At 1000+ concurrent requests, scheduling latency in Python exceeds 5ms. The engine's Rust scheduler operates in <100us.</li></ol>
|
|
<ol><li><strong>Cold start time matters for autoscaling.</strong> Python inference servers take 10-30 seconds to initialize (import PyTorch, load CUDA drivers, compile kernels). The engine starts in <2 seconds including model loading from local disk, enabling aggressive autoscaling without pre-warming.</li></ol>
|
|
<ol><li><strong>Single-GPU deployments need maximum efficiency.</strong> vLLM is optimized for multi-GPU tensor parallelism. For single-GPU serving (the common case for zen-7b through zen-32b), its abstractions add overhead. The engine is 30-40% faster than vLLM for single-GPU deployments with equivalent model quality.</li></ol>
|
|
<ol><li><strong>Edge and on-premises deployments cannot run Python.</strong> Embedded systems, air-gapped environments, and appliance form factors require a single static binary. The engine compiles to a self-contained binary with no runtime dependencies beyond the GPU driver.</li></ol>
|
|
<ol><li><strong>Multiple quantization formats serve different deployment targets.</strong> Consumer hardware runs GGUF. Datacenter GPUs run AWQ or GPTQ. Research clusters run full-precision SafeTensors. A single engine must support all formats without separate codebases.</li></ol>
|
|
<h2>Design Philosophy</h2>
|
|
<h3>Why Rust over Python</h3>
|
|
<p>This decision is not about language preference. It follows from three production constraints:</p>
|
|
<p><strong>Constraint 1: Deterministic memory usage.</strong> A GPU has a fixed amount of VRAM. The engine must know, at compile time and at model load time, the maximum memory it will consume. Rust's ownership model guarantees that every allocation has a single owner with a known lifetime. There is no garbage collector that might defer deallocation, causing temporary memory spikes. When a request completes, its KV cache blocks are freed immediately, not "eventually."</p>
|
|
<p><strong>Constraint 2: No GIL.</strong> The inference pipeline has CPU-bound stages (tokenization, sampling, scheduling) and GPU-bound stages (attention, FFN). These must run concurrently. In Python, CPU-bound work across threads is serialized by the GIL. The standard workaround -- multiprocessing -- duplicates model weights in memory. Rust's <code>tokio</code> async runtime and <code>rayon</code> data parallelism run all stages concurrently on a single process with zero duplication.</p>
|
|
<p><strong>Constraint 3: Single-binary deployment.</strong> The engine ships as one static binary (<code>hanzo-engine</code>) that includes the HTTP server, tokenizer, model loader, CUDA/Metal kernels, and health check endpoints. No virtualenv, no pip, no dynamic library resolution. This simplifies container images (FROM scratch), reduces attack surface, and eliminates "works on my machine" deployment failures.</p>
|
|
<h2>Specification</h2>
|
|
<h3>Architecture Overview</h3>
|
|
<p>The engine implements a six-stage pipeline for each inference request:</p>
|
|
<pre><code>Request → Tokenize → Schedule → Prefill → Decode → Sample → Response
|
|
│ │ │ │ │
|
|
▼ ▼ ▼ ▼ ▼
|
|
Tokenizer Scheduler GPU GPU Sampler
|
|
(CPU) (CPU) Compute Compute (CPU)
|
|
│
|
|
▼
|
|
KV Cache Manager
|
|
(PagedAttention)</code></pre>
|
|
<p><strong>Stage 1: Tokenization.</strong> Input text is converted to token IDs using the model's byte-level BPE tokenizer. The engine uses a Rust-native tokenizer (HuggingFace <code>tokenizers</code> crate) that runs at >1M tokens/sec on a single CPU core. Tokenization runs on a dedicated thread pool to avoid blocking the GPU pipeline.</p>
|
|
<p><strong>Stage 2: Scheduling.</strong> The scheduler decides which requests to batch together for the next forward pass. It implements continuous batching: new requests are inserted into the running batch without waiting for existing requests to complete. The scheduler considers GPU memory pressure, request priority, and sequence length when forming batches.</p>
|
|
<p><strong>Stage 3: Prefill.</strong> For new sequences, the engine computes attention over all input tokens in parallel. This is the compute-intensive phase where the model "reads" the prompt. Prefill throughput is measured in tokens/sec and is typically 10-50x higher than decode throughput because it benefits from parallelism.</p>
|
|
<p><strong>Stage 4: Decode.</strong> For ongoing sequences, the engine generates one token at a time using autoregressive decoding. Each decode step runs the full model forward pass for a single token position, attending to all previous tokens via the KV cache.</p>
|
|
<p><strong>Stage 5: Sampling.</strong> The model's output logits are converted to a token selection using the configured sampling strategy (temperature, top-p, top-k, repetition penalty, etc.). Sampling runs on CPU and is negligible in cost relative to the GPU stages.</p>
|
|
<p><strong>Stage 6: Response.</strong> Generated tokens are detokenized and returned to the client. For streaming requests, each token is sent as an SSE event immediately after sampling.</p>
|
|
<h3>Model Loading</h3>
|
|
<p>The engine loads models from three sources:</p>
|
|
<pre><code class="lang-yaml">Local Filesystem:
|
|
Path: /models/{model_name}/
|
|
Formats: SafeTensors, GGUF, GPTQ, AWQ
|
|
Discovery: Scan directory for config.json + weight files
|
|
|
|
Hugging Face Hub:
|
|
Registry: huggingface.co/zenlm/*
|
|
Auth: HF_TOKEN environment variable
|
|
Cache: ~/.cache/hanzo-engine/models/
|
|
|
|
Hanzo Object Storage (HIP-0405):
|
|
Endpoint: models.hanzo.ai/zen/{model}/{version}/{format}/
|
|
Auth: HANZO_API_KEY
|
|
Cache: /var/cache/hanzo-engine/models/</code></pre>
|
|
<p>Model loading is staged: first load the configuration and tokenizer (milliseconds), then load weights shard by shard into GPU memory. The health endpoint reports <code>loading</code> with a progress percentage during this phase. After weight loading, optional ISQ can quantize FP16 weights to INT8/INT4 in-place before serving begins.</p>
|
|
<h3>Supported Model Architectures</h3>
|
|
<p>The engine supports <strong>60+ model architectures</strong> across the Zen model family and broader open-source ecosystem. Key architectures include:</p>
|
|
<p>| Architecture | Zen Models | Key Features | |-------------|------------|--------------| | LlamaForCausalLM | zen-7b, zen-14b | GQA, RoPE, SwiGLU | | MistralForCausalLM | zen-3b | Sliding window attention | | PhiForCausalLM | zen-1b | Dense, partial RoPE | | Phi3ForCausalLM | zen-600m | Dense, long-context | | MixtralForCausalLM | zen-32b, zen-72b | MoDE with top-k routing | | ZenMoEForCausalLM | zen-235b, zen-480b | Large-scale MoDE | | GemmaForCausalLM | zen-3b-code | Code-optimized variant | | Qwen2ForCausalLM | zen4-mini, zen4-pro | Dense/MoE, long-context | | Qwen2MoeForCausalLM | zen4, zen4-max | MoDE, 1T+ params | | DeepseekV2ForCausalLM | zen4-coder | MoDE, code-optimized | | CLIPModel | zen-vl, zen-omni | Vision-language | | WhisperForConditionalGeneration | zen-scribe | Audio transcription |</p>
|
|
<p>The full list of 60+ supported architectures covers text, vision, audio, code, and multimodal models across dense, MoE, and MoDE configurations.</p>
|
|
<p>New architectures are added by implementing the <code>ModelPipeline</code> trait:</p>
|
|
<pre><code class="lang-rust">pub trait ModelPipeline: Send + Sync {
|
|
fn forward(
|
|
&self,
|
|
input_ids: &Tensor,
|
|
positions: &Tensor,
|
|
kv_cache: &mut KvCache,
|
|
attention_mask: Option<&Tensor>,
|
|
) -> Result<Tensor>;
|
|
|
|
fn config(&self) -> &ModelConfig;
|
|
fn tokenizer(&self) -> &Tokenizer;
|
|
fn device(&self) -> &Device;
|
|
}</code></pre>
|
|
<h3>KV Cache and PagedAttention</h3>
|
|
<p>The engine uses PagedAttention for KV cache management, which treats KV cache as virtual memory divided into fixed-size blocks.</p>
|
|
<pre><code class="lang-yaml">PagedAttention Configuration:
|
|
Block Size: 16 tokens (default)
|
|
Block Pool: Allocated at startup based on available GPU memory
|
|
Allocation: On-demand per sequence, returned to pool on completion
|
|
Sharing: Copy-on-write for shared prefixes (system prompts)
|
|
Eviction: LRU when pool is exhausted (preempts lowest-priority sequences)
|
|
|
|
Memory Layout:
|
|
# For zen-72b with 80 layers, 64 KV heads, 128 head_dim
|
|
Block Memory: 80 layers * 2 (K+V) * 64 heads * 128 dim * 16 tokens * 2 bytes
|
|
= 80 * 2 * 64 * 128 * 16 * 2 = 41,943,040 bytes (~40 MB per block)
|
|
|
|
# With 80GB GPU, ~60GB available after model weights:
|
|
Max Blocks: 60 GB / 40 MB = ~1,500 blocks
|
|
Max Tokens: 1,500 * 16 = 24,000 tokens in-flight</code></pre>
|
|
<p><strong>Copy-on-Write Prefix Sharing.</strong> When multiple requests use the same system prompt, the engine detects the shared prefix and maps their KV cache blocks to the same physical memory. Only when the sequences diverge (user message) are new blocks allocated. For Zen Gateway deployments where all requests to a model share a system prompt, this reduces KV cache memory by 30-60%.</p>
|
|
<h3>Sampling and Decoding Strategies</h3>
|
|
<p>The engine supports configurable sampling with the following parameters:</p>
|
|
<pre><code class="lang-yaml">Sampling Parameters:
|
|
temperature: 0.0 - 2.0 (default: 1.0)
|
|
top_p: 0.0 - 1.0 (nucleus sampling, default: 1.0)
|
|
top_k: 0 - vocab_size (default: 0, disabled)
|
|
min_p: 0.0 - 1.0 (minimum probability threshold, default: 0.0)
|
|
repetition_penalty: 1.0 - 2.0 (default: 1.0)
|
|
frequency_penalty: -2.0 - 2.0 (OpenAI-compatible, default: 0.0)
|
|
presence_penalty: -2.0 - 2.0 (OpenAI-compatible, default: 0.0)
|
|
max_tokens: 1 - model_max (default: model_max)
|
|
stop: string[] (stop sequences)
|
|
seed: u64 (deterministic sampling)
|
|
|
|
Logit Processing Order:
|
|
1. Repetition penalty (multiplicative on seen tokens)
|
|
2. Frequency/presence penalty (additive)
|
|
3. Temperature scaling
|
|
4. Top-k filtering
|
|
5. Top-p (nucleus) filtering
|
|
6. Min-p filtering
|
|
7. Categorical sampling (or argmax if temperature=0)</code></pre>
|
|
<h3>Speculative Decoding</h3>
|
|
<p>Speculative decoding accelerates generation by using a small draft model to predict multiple tokens ahead, then verifying them in parallel with the target model.</p>
|
|
<pre><code class="lang-yaml">Speculative Decoding:
|
|
Draft Model: A smaller model (e.g., zen-1b as draft for zen-72b)
|
|
Speculation Length: 4-8 tokens (configurable)
|
|
Acceptance: Modified rejection sampling to match target distribution
|
|
Speedup: 2-3x on average for greedy/low-temperature generation
|
|
|
|
Configuration:
|
|
draft_model: "zenlm/zen-1b"
|
|
num_speculative_tokens: 5
|
|
|
|
Constraints:
|
|
- Draft and target must share the same tokenizer
|
|
- Speedup depends on acceptance rate (task-dependent)
|
|
- Higher temperatures reduce acceptance rate
|
|
- Not beneficial for high-temperature creative generation</code></pre>
|
|
<p>Typical acceptance rates are 70-85% for code and structured text, 50-65% for creative writing.</p>
|
|
<h3>Continuous Batching</h3>
|
|
<p>The engine forms batches dynamically, inserting new requests into running batches without synchronization barriers.</p>
|
|
<pre><code class="lang-yaml">Continuous Batching:
|
|
Max Batch Size: Configurable (default: 256 sequences)
|
|
Batch Formation: Every decode iteration
|
|
Preemption: Lower-priority sequences swapped to CPU when GPU memory is full
|
|
Priority: FIFO by default, configurable per-request
|
|
|
|
Iteration Cycle:
|
|
1. Check for new requests in the waiting queue
|
|
2. Allocate KV cache blocks for new requests
|
|
3. If memory insufficient, preempt lowest-priority running sequences
|
|
4. Run prefill for new sequences
|
|
5. Run decode for all active sequences
|
|
6. Sample output tokens
|
|
7. Check stopping conditions (max_tokens, stop sequences, EOS)
|
|
8. Release completed sequences and their KV cache blocks
|
|
9. Return to step 1</code></pre>
|
|
<p>Unlike static batching, continuous batching achieves near-optimal GPU utilization because short sequences do not hold up long ones.</p>
|
|
<h3>API Specification</h3>
|
|
<p>The engine exposes four API surfaces:</p>
|
|
<h4>1. OpenAI-Compatible HTTP API (port 8080)</h4>
|
|
<p>| Endpoint | Method | Description | |----------|--------|-------------| | <code>/v1/chat/completions</code> | POST | Chat completions (messages array, streaming via SSE) | | <code>/v1/completions</code> | POST | Legacy text completions (prompt string) | | <code>/v1/embeddings</code> | POST | Text embeddings (when embedding model is loaded) | | <code>/v1/models</code> | GET | List loaded models | | <code>/health</code> | GET | Engine health, model status, KV cache utilization | | <code>/ready</code> | GET | Readiness probe (returns 200 when models are loaded) | | <code>/alive</code> | GET | Liveness probe (returns 200 when process is healthy) | | <code>/metrics</code> | GET | Prometheus metrics (configurable port, default 9090) |</p>
|
|
<p>Request and response formats follow the OpenAI API specification exactly. Streaming responses use Server-Sent Events with <code>text/event-stream</code> content type; each token is sent as a <code>data:</code> event, terminated by <code>data: [DONE]</code>.</p>
|
|
<h4>2. Rust SDK</h4>
|
|
<p>The engine exposes a native Rust API via the <code>hanzo-engine</code> crate for direct in-process inference without HTTP overhead. This is used by Hanzo Edge (HIP-0050) and embedding pipelines.</p>
|
|
<pre><code class="lang-rust">use hanzo_engine::{Engine, EngineConfig, ChatMessage};
|
|
|
|
let engine = Engine::new(EngineConfig::from_yaml("engine.yaml")?)?;
|
|
let response = engine.chat_completion(vec![
|
|
ChatMessage::user("Explain PagedAttention in one sentence."),
|
|
]).await?;</code></pre>
|
|
<h4>3. Python SDK (PyO3)</h4>
|
|
<p>A Python binding via PyO3 enables integration with Python ML pipelines without running a separate HTTP server:</p>
|
|
<pre><code class="lang-python">from hanzo_engine import Engine
|
|
|
|
engine = Engine.from_config("engine.yaml")
|
|
response = engine.chat_completion(
|
|
messages=[{"role": "user", "content": "Hello"}],
|
|
temperature=0.7,
|
|
)</code></pre>
|
|
<h4>4. MCP Interface</h4>
|
|
<p>The engine implements the Model Context Protocol (HIP-0010) for tool-augmented inference, allowing MCP clients to invoke engine capabilities as tools.</p>
|
|
<h3>Health and Metrics</h3>
|
|
<pre><code class="lang-yaml">Metrics (all labeled by model):
|
|
Counters:
|
|
hanzo_engine_requests_total{model, status} # Total inference requests
|
|
hanzo_engine_tokens_generated_total{model} # Total output tokens
|
|
hanzo_engine_prompt_tokens_total{model} # Total input tokens
|
|
|
|
Histograms:
|
|
hanzo_engine_time_to_first_token_seconds{model} # TTFT distribution
|
|
hanzo_engine_request_duration_seconds{model} # End-to-end latency
|
|
|
|
Gauges:
|
|
hanzo_engine_tokens_per_second{model} # Current throughput
|
|
hanzo_engine_gpu_memory_used_bytes{gpu} # GPU memory usage
|
|
hanzo_engine_kv_cache_utilization{model} # KV cache block ratio
|
|
hanzo_engine_batch_size{model} # Current batch size
|
|
hanzo_engine_queue_depth{model} # Waiting requests</code></pre>
|
|
<h3>Multi-Model Serving</h3>
|
|
<p>The engine can load multiple models simultaneously, partitioning GPU memory across them:</p>
|
|
<pre><code class="lang-yaml"># engine.yaml - Multi-model configuration
|
|
models:
|
|
- name: zen-7b
|
|
source: zenlm/zen-7b
|
|
format: safetensors
|
|
quantization: isq-int8
|
|
gpu_memory_fraction: 0.3
|
|
max_batch_size: 128
|
|
|
|
- name: zen-32b
|
|
source: zenlm/zen-32b
|
|
format: gptq
|
|
quantization: gptq-int4
|
|
gpu_memory_fraction: 0.7
|
|
max_batch_size: 32
|
|
|
|
routing:
|
|
strategy: model_name # Route by "model" field in request</code></pre>
|
|
<p>When GPU memory is insufficient for all configured models, the engine supports automatic offloading:</p>
|
|
<pre><code class="lang-yaml">Offloading Strategies:
|
|
layer_split:
|
|
# Split model layers across GPU and CPU
|
|
gpu_layers: 60 # First 60 layers on GPU
|
|
cpu_layers: 20 # Remaining 20 layers on CPU
|
|
|
|
model_swap:
|
|
# Unload inactive models to CPU, load on demand
|
|
idle_timeout: 300s # Swap to CPU after 5 minutes idle
|
|
load_timeout: 10s # Maximum time to swap back to GPU
|
|
|
|
expert_offload:
|
|
# MoDE-specific: keep active experts on GPU, inactive on CPU
|
|
gpu_experts: 16 # Top-16 most-used experts on GPU
|
|
cpu_experts: 48 # Remaining experts on CPU/NVMe
|
|
prefetch: true # Predict and prefetch likely experts</code></pre>
|
|
<h3>GPU Memory Management</h3>
|
|
<p>The engine manages GPU memory through three regions:</p>
|
|
<pre><code class="lang-yaml">Memory Regions:
|
|
Model Weights:
|
|
Description: Static after loading
|
|
Management: Allocated at startup, never freed during serving
|
|
Sizing: Determined by model size and quantization
|
|
|
|
KV Cache Pool:
|
|
Description: Dynamic, managed by PagedAttention
|
|
Management: Block-level allocation and deallocation
|
|
Sizing: All remaining GPU memory after weights + scratch
|
|
|
|
Scratch Space:
|
|
Description: Temporary buffers for forward pass computation
|
|
Management: Pre-allocated, reused across iterations
|
|
Sizing: Function of max batch size and sequence length
|
|
|
|
Memory Budget Calculation:
|
|
total_gpu_memory = detect_gpu_memory()
|
|
model_weight_memory = model_size * bytes_per_param(quantization)
|
|
scratch_memory = estimate_scratch(max_batch_size, max_seq_len)
|
|
kv_cache_memory = total_gpu_memory * gpu_memory_utilization - model_weight_memory - scratch_memory
|
|
num_kv_blocks = kv_cache_memory / block_size_bytes</code></pre>
|
|
<p>The <code>gpu_memory_utilization</code> parameter (default: 0.90) controls what fraction of GPU memory the engine may use. Setting this below 1.0 leaves headroom for CUDA kernel allocations and prevents OOM under edge-case memory fragmentation.</p>
|
|
<h3>Configuration</h3>
|
|
<p>The engine is configured via a YAML file and/or environment variables:</p>
|
|
<pre><code class="lang-yaml"># /etc/hanzo-engine/engine.yaml
|
|
|
|
server:
|
|
host: 0.0.0.0
|
|
port: 8080
|
|
workers: 4 # HTTP worker threads
|
|
max_connections: 4096
|
|
request_timeout: 300s # Maximum request duration
|
|
|
|
models:
|
|
- name: zen-72b
|
|
source: zenlm/zen-72b # HF repo or local path
|
|
format: safetensors # safetensors | gguf | gptq | awq
|
|
quantization: null # null (native) | isq-int8 | isq-int4
|
|
revision: main # Git revision / branch
|
|
max_batch_size: 64
|
|
max_seq_len: 131072
|
|
|
|
speculative:
|
|
draft_model: zenlm/zen-1b
|
|
num_tokens: 5
|
|
|
|
paged_attention:
|
|
block_size: 16
|
|
enable_prefix_caching: true
|
|
max_prefix_cache_tokens: 4096
|
|
|
|
gpu:
|
|
memory_utilization: 0.90
|
|
device_ids: [0] # GPU device IDs
|
|
dtype: bfloat16 # float16 | bfloat16
|
|
|
|
logging:
|
|
level: info # trace | debug | info | warn | error
|
|
format: json # json | pretty
|
|
|
|
metrics:
|
|
enabled: true
|
|
port: 9090 # Separate metrics port
|
|
path: /metrics
|
|
|
|
health:
|
|
port: 8080 # Same as server by default
|
|
path: /health
|
|
readiness_path: /ready
|
|
liveness_path: /alive</code></pre>
|
|
<p>Environment variable overrides follow the pattern <code>HANZO_ENGINE_{SECTION}_{KEY}</code> (e.g., <code>HANZO_ENGINE_SERVER_PORT=8080</code>, <code>HANZO_ENGINE_GPU_DEVICE_IDS=0,1</code>).</p>
|
|
<h3>Streaming and Backpressure</h3>
|
|
<p>SSE streaming flushes each token immediately with no buffering. If a client cannot consume fast enough, the engine buffers up to 1000 tokens per stream before applying backpressure, pausing decode for that sequence only. Client disconnection is detected within 1 second; disconnected sequences are cancelled and their KV cache freed immediately.</p>
|
|
<h3>Integration with LLM Gateway (HIP-0004)</h3>
|
|
<p>The engine registers as a backend provider with the LLM Gateway:</p>
|
|
<pre><code class="lang-yaml"># LLM Gateway provider configuration
|
|
providers:
|
|
hanzo-engine:
|
|
type: openai_compatible
|
|
base_url: http://engine:8080/v1
|
|
api_key: internal-engine-key
|
|
models:
|
|
- zen-7b
|
|
- zen-14b
|
|
- zen-32b
|
|
- zen-72b
|
|
health_check:
|
|
url: http://engine:8080/health
|
|
interval: 10s
|
|
priority: 1 # Prefer engine over external providers
|
|
|
|
hanzo-engine-edge:
|
|
type: openai_compatible
|
|
base_url: http://edge-engine:8080/v1
|
|
models:
|
|
- zen-600m
|
|
- zen-1b
|
|
- zen-3b
|
|
priority: 2</code></pre>
|
|
<p>The LLM Gateway routes requests with <code>model: "zen-*"</code> to the engine. The Zen Gateway (HIP-0039) may sit between them to handle task-aware routing and failover.</p>
|
|
<pre><code>Client → LLM Gateway (4000) → Zen Gateway (8081) → Engine (8080) → GPU
|
|
│
|
|
└── vLLM (8000) [for multi-GPU models]</code></pre>
|
|
<h3>Deployment</h3>
|
|
<h4>Docker</h4>
|
|
<pre><code class="lang-dockerfile">FROM hanzoai/engine:latest
|
|
COPY engine.yaml /etc/hanzo-engine/engine.yaml
|
|
EXPOSE 8080 9090
|
|
ENTRYPOINT ["hanzo-engine", "--config", "/etc/hanzo-engine/engine.yaml"]</code></pre>
|
|
<pre><code class="lang-bash">docker run --gpus all -p 8080:8080 -p 9090:9090 \
|
|
-v /models:/models \
|
|
hanzoai/engine:latest \
|
|
--config /etc/hanzo-engine/engine.yaml</code></pre>
|
|
<h4>Kubernetes</h4>
|
|
<p>Standard Kubernetes Deployment with GPU resource requests (<code>nvidia.com/gpu</code>), Prometheus scrape annotations, readiness probe on <code>/ready</code>, and liveness probe on <code>/alive</code>. Model weights are stored on a PersistentVolumeClaim; engine configuration is mounted from a ConfigMap. Each model size gets its own Deployment with appropriate GPU and memory resource limits.</p>
|
|
<h2>Performance Benchmarks</h2>
|
|
<p>Benchmarks measured on a single NVIDIA A100 80GB GPU, zen-72b with INT4 quantization (AWQ):</p>
|
|
<p>| Metric | Engine (Rust) | vLLM (Python) | TGI (Rust/Python) | |--------|--------------|---------------|-------------------| | Time to First Token (TTFT) | 38ms | 52ms | 45ms | | Output Tokens/sec (single) | 42 | 31 | 35 | | Output Tokens/sec (batch=32) | 980 | 780 | 850 | | Max Concurrent Sequences | 256 | 192 | 210 | | Cold Start Time | 1.8s | 28s | 12s | | Memory Overhead (runtime) | 45 MB | 2.1 GB | 680 MB | | P99 Scheduling Latency | 85us | 4.2ms | 1.1ms | | GPU Memory Utilization | 94% | 88% | 91% |</p>
|
|
<p>Benchmarks for single-GPU serving across Zen model sizes (engine, INT8 ISQ, A100 80GB):</p>
|
|
<p>| Model | TTFT | Tokens/sec (single) | Tokens/sec (batch=32) | Max Concurrent | |-------|------|---------------------|-----------------------|----------------| | zen-7b | 12ms | 118 | 3,200 | 512 | | zen-14b | 18ms | 86 | 2,100 | 384 | | zen-32b | 28ms | 54 | 1,400 | 192 | | zen-72b (INT4) | 38ms | 42 | 980 | 256 |</p>
|
|
<h2>Security Considerations</h2>
|
|
<h3>Network Security</h3>
|
|
<p>The engine listens on port 8080 and should not be exposed directly to the internet. It is designed to sit behind the LLM Gateway or a reverse proxy that handles authentication, rate limiting, and TLS termination.</p>
|
|
<pre><code class="lang-yaml">Security Model:
|
|
Authentication: Delegated to LLM Gateway (API keys)
|
|
TLS: Terminated at load balancer or Gateway
|
|
Internal Auth: Optional bearer token for engine-to-engine communication
|
|
|
|
HANZO_ENGINE_AUTH_TOKEN: "internal-secret-token"
|
|
# If set, all requests must include: Authorization: Bearer <token></code></pre>
|
|
<h3>Input Validation</h3>
|
|
<pre><code class="lang-yaml">Input Limits:
|
|
max_prompt_tokens: 131072 # Reject prompts exceeding model context
|
|
max_output_tokens: 32768 # Cap generation length
|
|
max_stop_sequences: 16 # Limit stop sequence count
|
|
max_request_size: 10MB # HTTP body size limit
|
|
|
|
Sanitization:
|
|
- UTF-8 validation on all string inputs
|
|
- JSON schema validation on request bodies
|
|
- Numeric range checks on all sampling parameters
|
|
- Reject NaN/Inf in temperature, top_p, penalties</code></pre>
|
|
<h3>Model Isolation</h3>
|
|
<p>When serving multiple models, each model runs in an isolated memory region. A bug in one model's forward pass cannot corrupt another model's weights or KV cache. This isolation is enforced by Rust's ownership model -- each model owns its tensors, and cross-model references are impossible without explicit sharing primitives.</p>
|
|
<h2>Implementation Roadmap</h2>
|
|
<h3>Phase 4: Optimization (Q4 2026)</h3>
|
|
<ul><li>Custom CUDA kernels for Zen-specific attention patterns</li><li>FlashAttention-3 integration for Hopper GPUs (H100/H200)</li><li>FP8 native tensor core utilization on Hopper and Blackwell GPUs</li><li>Disaggregated prefill/decode for latency-sensitive workloads</li><li>Apple Silicon Metal backend for local development</li><li>Benchmark suite and regression testing infrastructure</li><li>Shared model format and quantization pipeline with Hanzo Edge (HIP-0050)</li></ul>
|
|
<pre><code class="lang-python"># Before: OpenAI direct
|
|
client = OpenAI(api_key="sk-openai-...")
|
|
|
|
# After: Hanzo Engine
|
|
client = OpenAI(
|
|
base_url="http://localhost:8080/v1",
|
|
api_key="hanzo-engine-key"
|
|
)</code></pre>
|
|
<p>Model format compatibility:</p>
|
|
<ul><li><strong>SafeTensors</strong>: Standard HuggingFace format, no modifications needed.</li><li><strong>GGUF</strong>: Standard llama.cpp format, compatible with Ollama and other GGUF consumers.</li><li><strong>GPTQ/AWQ</strong>: Standard quantized formats from AutoGPTQ and AutoAWQ.</li></ul>
|
|
<p>The engine does not modify model weights or tokenizers. A model loaded in the engine produces identical outputs (given the same seed and sampling parameters) as the same model loaded in vLLM, TGI, or llama.cpp.</p>
|
|
<h2>References</h2>
|
|
<ol><li><a href="./hip-0004-llm-gateway-unified-ai-provider-interface.md">HIP-0004: LLM Gateway - Unified AI Provider Interface</a></li><li><a href="./hip-0019-tensor-operations-standard.md">HIP-0019: Tensor Operations Standard (Candle)</a></li><li><a href="./hip-0039-zen-model-architecture.md">HIP-0039: Zen Model Architecture</a></li><li><a href="https://github.com/EricLBuehler/mistral.rs">mistral.rs: Blazingly fast LLM inference</a></li><li><a href="https://arxiv.org/abs/2309.06180">Efficient Memory Management for Large Language Model Serving with PagedAttention</a></li><li><a href="https://arxiv.org/abs/2211.17192">Fast Inference from Transformers via Speculative Decoding</a></li><li><a href="https://arxiv.org/abs/2205.14135">FlashAttention: Fast and Memory-Efficient Exact Attention</a></li><li><a href="https://arxiv.org/abs/2306.00978">AWQ: Activation-aware Weight Quantization</a></li><li><a href="https://arxiv.org/abs/2210.17323">GPTQ: Accurate Post-Training Quantization for Generative Pre-trained Transformers</a></li><li><a href="https://github.com/ggerganov/ggml">GGML/GGUF: Tensor Library for Machine Learning</a></li><li><a href="https://www.usenix.org/conference/osdi22/presentation/yu">Orca: A Distributed Serving System for Transformer-Based Models</a></li><li><a href="https://zenlm.org">Zen LM Documentation</a></li><li><a href="https://github.com/hanzoai/engine">Hanzo Engine Repository</a></li><li><a href="https://github.com/hanzoai/ml">Hanzo ML Framework (Candle)</a></li><li><a href="./hip-0050-edge-computing-standard.md">HIP-0050: Edge AI Runtime Standard</a></li><li><a href="./hip-0068-ingress-standard.md">HIP-0068: Ingress Standard</a></li></ol>
|
|
<h2>Copyright</h2>
|
|
<p>Copyright and related rights waived via <a href="https://creativecommons.org/publicdomain/zero/1.0/">CC0</a>.</p>
|
|
<footer>
|
|
<div class="fmap">
|
|
<div><b>Build</b>
|
|
<a href="https://hanzo.app">hanzo.app</a> app builder
|
|
<a href="https://hanzo.codes">hanzo.codes</a> code with an agent
|
|
<a href="https://hanzoskills.com">hanzoskills.com</a> what an agent reads</div>
|
|
<div><b>Run</b>
|
|
<a href="https://console.hanzo.ai">console.hanzo.ai</a> cloud console
|
|
<a href="https://api.hanzo.ai">api.hanzo.ai</a> the API</div>
|
|
<div><b>Account</b>
|
|
<a href="https://hanzo.id">hanzo.id</a> sign in
|
|
<a href="https://pay.hanzo.ai">pay.hanzo.ai</a> top up
|
|
<a href="https://billing.hanzo.ai">billing.hanzo.ai</a> invoices and usage</div>
|
|
<div><b>Company</b>
|
|
<a href="https://hanzo.ai">hanzo.ai</a> canonical
|
|
<a href="https://hanzo.ventures">hanzo.ventures</a> ventures</div>
|
|
</div>
|
|
<div class="fend"><a href="/hip-0043.md">This page as plain text</a> · <a href="/llms.txt">Index for agents</a></div>
|
|
</footer>
|
|
</div></body></html> |