mirror of
https://github.com/hanzoai/skills.git
synced 2026-11-19 00:47:36 +00:00
hanzoskills.com/llms.txt now opens with the install line, the API reference index and the company index, so an agent that lands on any one of the three can find the rest. robots.txt claimed 537 documents while llms.txt counts 920. The number is gone from robots.txt; llms.txt derives it. public/ is regenerated from scripts/site.py, which it had fallen behind: the copy-prompt menu and agent bar were in the generator and not in the committed pages. tests/site.mjs passes against the result. Co-authored-by: Hanzo Dev <[email protected]>
541 lines
23 KiB
HTML
541 lines
23 KiB
HTML
<!doctype html>
|
|
<html lang="en"><head>
|
|
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
<title>hip-0004</title>
|
|
<meta name="color-scheme" content="dark">
|
|
<style>
|
|
/* Committed to one visual world: a dark instrument panel. No light variant —
|
|
the dot field and the hairline rules only read on black, and a naive
|
|
inversion would wash both out. Every colour is painted explicitly so the page
|
|
never borrows a host background. */
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Pixel Square';src:url('/fonts/ZenPixel-Square.woff2') format('woff2');font-style:normal;font-display:swap}
|
|
|
|
:root{
|
|
--ink:#f4f4f2; --soft:#9c9c98; --faint:#6a6a66; --ground:#000; --panel:#0a0a0a;
|
|
--line:#232323; --line-hi:#3a3a3a; --accent:#f4f4f2;
|
|
--font-zen-sans:'Zen',-apple-system,BlinkMacSystemFont,"Segoe UI",Helvetica,Arial,sans-serif;
|
|
--font-zen-mono:'Zen Mono',ui-monospace,"SF Mono",SFMono-Regular,Menlo,Consolas,monospace;
|
|
--font-zen-pixel-square:'Zen Pixel Square',monospace;
|
|
--mono:var(--font-zen-mono);
|
|
--sans:var(--font-zen-sans);
|
|
/* Controls are pills and surfaces are soft, as on hanzo.ai. */
|
|
--r:999px; --r-soft:14px;
|
|
}
|
|
*{box-sizing:border-box}
|
|
html{-webkit-text-size-adjust:100%}
|
|
body{margin:0;background:var(--ground);color:var(--ink);font-family:var(--sans);
|
|
font-size:16px;line-height:1.65;-webkit-font-smoothing:antialiased}
|
|
.wrap{max-width:60rem;margin:0 auto;padding:0 1.75rem}
|
|
|
|
/* The dot field. A halftone made of one repeating radial-gradient, faded out
|
|
along the diagonal by a mask so it reads as a wave rather than wallpaper. */
|
|
/* No overflow clip here: the dot field is inset to the box already, and a clip
|
|
would cut off the copy-prompt menu. */
|
|
.hero{position:relative;border-bottom:1px solid var(--line)}
|
|
.hero::before{content:"";position:absolute;inset:0;
|
|
background-image:radial-gradient(var(--line-hi) 1px, transparent 1px);
|
|
background-size:7px 7px;
|
|
-webkit-mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
pointer-events:none}
|
|
.hero .wrap{position:relative;padding-top:5.5rem;padding-bottom:4rem}
|
|
|
|
nav{border-bottom:1px solid var(--line)}
|
|
nav .in{max-width:60rem;margin:0 auto;padding:1.1rem 1.75rem;display:flex;
|
|
justify-content:space-between;align-items:center;gap:1rem}
|
|
.brand{font-family:var(--mono);font-size:.85rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--ink);text-decoration:none;
|
|
display:inline-flex;align-items:center;gap:.6rem}
|
|
.brand i{font-style:normal;color:var(--faint)}
|
|
.mark{width:1.15em;height:1.15em;display:block;flex:0 0 auto}
|
|
nav a.up{font-family:var(--mono);font-size:.72rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--soft);text-decoration:none}
|
|
nav a.up:hover{color:var(--ink)}
|
|
|
|
h1{font-family:var(--mono);font-size:clamp(2rem,5.4vw,3.4rem);line-height:1.12;
|
|
letter-spacing:-.02em;margin:0 0 1.1rem;font-weight:500;text-wrap:balance}
|
|
h1 em{font-style:normal;color:var(--faint)}
|
|
.lede{color:var(--soft);font-size:1.05rem;max-width:52ch;margin:0 0 2.25rem}
|
|
|
|
.btns{display:flex;flex-wrap:wrap;gap:.7rem;margin:0 0 .5rem;align-items:center}
|
|
.btn{display:inline-flex;align-items:center;gap:.5rem;font-family:var(--mono);
|
|
font-size:.72rem;letter-spacing:.14em;text-transform:uppercase;
|
|
padding:.8rem 1.4rem;border:1px solid var(--line-hi);border-radius:var(--r);
|
|
color:var(--ink);text-decoration:none;background:transparent;cursor:pointer;
|
|
line-height:1.2;font-weight:400}
|
|
.btn:hover{background:#141414;text-decoration:none;border-color:var(--soft)}
|
|
.btn.solid{background:var(--ink);color:#000;border-color:var(--ink)}
|
|
.btn.solid:hover{background:#d8d8d4}
|
|
.btn .chev{width:.7em;height:.7em;flex:0 0 auto;transition:transform .15s ease}
|
|
.btn[aria-expanded="true"] .chev{transform:rotate(180deg)}
|
|
|
|
/* Copy-prompt selector. Each row copies the prompt for its own target. */
|
|
.copy{position:relative;display:inline-flex}
|
|
.menu{position:absolute;top:calc(100% + .5rem);left:0;z-index:20;min-width:21rem;
|
|
background:var(--panel);border:1px solid var(--line-hi);border-radius:var(--r-soft);
|
|
padding:.4rem;display:flex;flex-direction:column;
|
|
box-shadow:0 12px 32px rgba(0,0,0,.75)}
|
|
.menu[hidden]{display:none}
|
|
.mi{display:flex;align-items:center;gap:.7rem;width:100%;text-align:left;
|
|
padding:.62rem .8rem;border:0;border-radius:calc(var(--r-soft) - .35rem);
|
|
background:transparent;color:var(--ink);font-family:var(--mono);font-size:.76rem;
|
|
letter-spacing:.04em;text-decoration:none;cursor:pointer;line-height:1.3;
|
|
white-space:nowrap}
|
|
.mi:hover,.mi:focus-visible{background:#181818;outline:none}
|
|
.mi .mark{width:1.05rem;height:1.05rem;color:var(--soft)}
|
|
.mi:hover .mark{color:var(--ink)}
|
|
.mi u{margin-left:auto;color:var(--faint);text-decoration:none;font-size:.68rem;
|
|
letter-spacing:.1em;text-transform:uppercase}
|
|
|
|
/* Works-with bar: one scrolling row, name and mark per agent. */
|
|
.strip{border-bottom:1px solid var(--line);background:var(--panel)}
|
|
.strip .in{max-width:60rem;margin:0 auto;padding:.9rem 1.75rem;display:flex;
|
|
align-items:center;gap:1.25rem;font-family:var(--mono);font-size:.72rem;
|
|
letter-spacing:.1em;text-transform:uppercase;color:var(--faint)}
|
|
.strip b{color:var(--soft);font-weight:400;flex:0 0 auto;
|
|
border-right:1px solid var(--line);padding-right:1.25rem;align-self:stretch;
|
|
display:flex;align-items:center}
|
|
.bar{display:flex;align-items:center;gap:.5rem;flex:1 1 auto;min-width:0;
|
|
overflow-x:auto;scrollbar-width:none;scroll-snap-type:x proximity;
|
|
-webkit-overflow-scrolling:touch;
|
|
-webkit-mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent);
|
|
mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent)}
|
|
.bar::-webkit-scrollbar{display:none}
|
|
.agent{display:inline-flex;align-items:center;gap:.45rem;flex:0 0 auto;
|
|
padding:.4rem .85rem;border:1px solid var(--line);border-radius:var(--r);
|
|
color:var(--soft);white-space:nowrap;scroll-snap-align:start;
|
|
text-decoration:none;background:var(--ground)}
|
|
.agent:hover{color:var(--ink);border-color:var(--line-hi);text-decoration:none}
|
|
.agent .mark{width:.95rem;height:.95rem}
|
|
|
|
h2{font-family:var(--mono);font-size:1.1rem;letter-spacing:-.01em;margin:2.75rem 0 .75rem;font-weight:500}
|
|
h3{font-size:1rem;margin:2rem 0 .5rem;font-weight:600}
|
|
h4,h5,h6{font-size:.94rem;margin:1.5rem 0 .5rem;font-weight:600}
|
|
p{margin:0 0 1rem;max-width:70ch}
|
|
ul,ol{margin:0 0 1rem;padding-left:1.35rem;max-width:70ch}
|
|
li{margin:.22rem 0}
|
|
a{color:var(--ink);text-decoration:underline;text-underline-offset:3px;
|
|
text-decoration-color:var(--line-hi)}
|
|
a:hover{text-decoration-color:var(--ink)}
|
|
a:focus-visible{outline:1px solid var(--ink);outline-offset:3px}
|
|
code{font-family:var(--mono);font-size:.85em;background:var(--panel);
|
|
border:1px solid var(--line);border-radius:5px;padding:.1em .36em}
|
|
pre{background:var(--panel);border:1px solid var(--line);padding:1rem 1.1rem;
|
|
border-radius:var(--r-soft);overflow-x:auto;margin:0 0 1.25rem}
|
|
pre code{background:none;border:0;padding:0;font-size:.82rem;line-height:1.7}
|
|
blockquote{margin:0 0 1rem;padding-left:1rem;border-left:1px solid var(--line-hi);color:var(--soft)}
|
|
hr{border:0;border-top:1px solid var(--line);margin:2.5rem 0}
|
|
table{border-collapse:collapse;width:100%;margin:0 0 1.25rem;font-size:.92rem;
|
|
display:block;overflow-x:auto}
|
|
th,td{border:1px solid var(--line);padding:.5rem .7rem;text-align:left}
|
|
|
|
.curl{display:block;background:var(--panel);border:1px solid var(--line);
|
|
border-radius:var(--r-soft);padding:.95rem 1.1rem;font-family:var(--mono);
|
|
font-size:.8rem;color:var(--soft);overflow-x:auto;margin:0 0 2.5rem;
|
|
white-space:pre;line-height:1.9}
|
|
.curl b{color:var(--ink);font-weight:400}
|
|
.curl i{font-style:normal;color:var(--faint)}
|
|
|
|
/* index */
|
|
#q{width:100%;padding:.85rem 1.2rem;font-family:var(--mono);font-size:.85rem;color:var(--ink);
|
|
background:var(--panel);border:1px solid var(--line);border-radius:var(--r);margin:0 0 .6rem}
|
|
#q::placeholder{color:var(--faint);text-transform:uppercase;letter-spacing:.1em;font-size:.72rem}
|
|
#q:focus{outline:none;border-color:var(--soft)}
|
|
#count{color:var(--faint);font-family:var(--mono);font-size:.72rem;letter-spacing:.1em;
|
|
text-transform:uppercase;margin:0 0 2rem}
|
|
.cat{margin:2.5rem 0 .5rem;font-family:var(--mono);font-size:.7rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);border-bottom:1px solid var(--line);
|
|
padding-bottom:.45rem}
|
|
.row{display:block;padding:.6rem .9rem;margin:0 -.9rem;border-bottom:1px solid var(--line);
|
|
border-radius:var(--r-soft);text-decoration:none}
|
|
.row:hover{background:#0a0a0a}
|
|
.nm{font-family:var(--mono);font-size:.84rem;color:var(--ink);display:block}
|
|
.ds{color:var(--soft);font-size:.9rem;display:block;margin-top:.12rem}
|
|
footer{margin-top:4rem;padding:2rem 0 4rem;border-top:1px solid var(--line);
|
|
color:var(--faint);font-size:.85rem}
|
|
footer a{color:var(--soft)}
|
|
.fmap{display:grid;gap:1.75rem;grid-template-columns:repeat(auto-fit,minmax(11rem,1fr));
|
|
margin:0 0 2rem;font-size:.85rem}
|
|
.fmap b{display:block;font-family:var(--mono);font-size:.68rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);font-weight:400;margin:0 0 .6rem;
|
|
border-bottom:1px solid var(--line);padding-bottom:.4rem}
|
|
.fmap a{display:inline-block;color:var(--ink);text-decoration:none;margin:.3rem .35rem 0 0}
|
|
.fmap a:hover{text-decoration:underline}
|
|
.fmap div{color:var(--faint);line-height:1.5}
|
|
.fend{border-top:1px solid var(--line);padding-top:1.25rem;font-size:.82rem}
|
|
|
|
@media (max-width:640px){
|
|
.hero .wrap{padding-top:3.25rem;padding-bottom:2.75rem}
|
|
.btns{gap:.5rem}
|
|
.btn{padding:.75rem 1.15rem;font-size:.68rem}
|
|
.copy,.copy .btn{width:100%;justify-content:center}
|
|
.menu{left:0;right:0;min-width:0}
|
|
.strip .in{flex-direction:column;align-items:flex-start;gap:.6rem;padding:.9rem 1.75rem}
|
|
.strip b{border-right:0;padding-right:0}
|
|
.bar{width:100%}
|
|
}
|
|
</style>
|
|
<link rel="alternate" type="text/markdown" href="/hip-0004.md">
|
|
|
|
<link rel="icon" href="/favicon.svg">
|
|
<link rel="canonical" href="https://hanzoskills.com/hip-0004">
|
|
</head>
|
|
<!-- Reading this with a machine? Every page here is also plain markdown.
|
|
This one: https://hanzoskills.com/hip-0004.md
|
|
Index: https://hanzoskills.com/llms.txt
|
|
Corpus: https://hanzoskills.com/all.md
|
|
Append .md to any path for the markdown twin. -->
|
|
<body>
|
|
<nav><div class="in">
|
|
<a class="brand" href="/"><svg class="mark" viewBox="0 0 67 67" fill="currentColor" aria-hidden="true"><path d="M22.21 67V44.6369H0V67H22.21Z"/><path d="M66.7038 22.3184H22.2534L0.0878906 44.6367H44.4634L66.7038 22.3184Z"/><path d="M22.21 0H0V22.3184H22.21V0Z"/><path d="M66.7198 0H44.5098V22.3184H66.7198V0Z"/><path d="M66.7198 67V44.6369H44.5098V67H66.7198Z"/></svg> HANZO<i>SKILLS</i></a>
|
|
<a class="up" href="/">All skills</a>
|
|
</div></nav>
|
|
|
|
<div class="wrap">
|
|
<h1>hip-0004</h1><p class="lede">HIP-4: LLM Gateway - Unified AI Provider Interface. Status Final. Hanzo architectural specification.</p><h1>HIP-0004: LLM Gateway - Unified AI Provider Interface</h1>
|
|
<h2>Abstract</h2>
|
|
<p>This proposal defines the LLM Gateway specification, Hanzo's unified proxy for 100+ LLM providers with OpenAI-compatible API, intelligent routing, cost optimization, and enterprise features. The gateway serves as the central infrastructure for all AI operations in the Hanzo ecosystem.</p>
|
|
<p><strong>Repository</strong>: <a href="https://github.com/hanzoai/gateway">github.com/hanzoai/gateway</a> (historically <code>hanzoai/llm</code>) <strong>Port</strong>: 4000 <strong>Image</strong>: <code>oci.hanzo.ai/hanzoai/gateway</code> (HIP-0033) <strong>Upstream credentials</strong>: none in this process — see §Deployment and HIP-0143</p>
|
|
<h2>Motivation</h2>
|
|
<p>Current AI infrastructure challenges:</p>
|
|
<ol><li><strong>Provider Fragmentation</strong>: Each provider has different APIs</li><li><strong>Cost Management</strong>: No unified billing or optimization</li><li><strong>Reliability Issues</strong>: Single points of failure</li><li><strong>Feature Gaps</strong>: Providers have different capabilities</li><li><strong>Compliance Requirements</strong>: No unified audit/security layer</li></ol>
|
|
<p>The LLM Gateway provides a single, unified interface to all AI providers.</p>
|
|
<h2>Specification</h2>
|
|
<h3>Architecture Overview</h3>
|
|
<pre><code class="lang-yaml">Components:
|
|
Router:
|
|
- Provider selection
|
|
- Load balancing
|
|
- Failover handling
|
|
- Cost optimization
|
|
|
|
Proxy:
|
|
- Request transformation
|
|
- Response normalization
|
|
- Streaming support
|
|
- Error handling
|
|
|
|
Cache:
|
|
- Semantic caching
|
|
- Embedding cache
|
|
- Response cache
|
|
- TTL management
|
|
|
|
Analytics:
|
|
- Usage tracking
|
|
- Cost monitoring
|
|
- Performance metrics
|
|
- Audit logging</code></pre>
|
|
<h3>Supported Providers</h3>
|
|
<pre><code class="lang-yaml">Tier 1 (Full Support):
|
|
- OpenAI (GPT-4, GPT-3.5, DALL-E, Whisper)
|
|
- Anthropic (Claude 3.5, Claude 3)
|
|
- Google (Gemini Pro, Gemini Ultra)
|
|
- Hanzo (HLLMs, Jin models)
|
|
|
|
Tier 2 (Core Features):
|
|
- Together AI
|
|
- Anyscale
|
|
- Replicate
|
|
- Cohere
|
|
- AI21 Labs
|
|
- Hugging Face
|
|
|
|
Tier 3 (Basic Support):
|
|
- Ollama (local)
|
|
- LM Studio (local)
|
|
- Azure OpenAI
|
|
- AWS Bedrock
|
|
- Google Vertex AI
|
|
|
|
Specialized:
|
|
- ElevenLabs (voice)
|
|
- Runway (video)
|
|
- Midjourney (images)
|
|
- Stability AI (images)</code></pre>
|
|
<h3>API Specification</h3>
|
|
<h4>Configuration</h4>
|
|
<pre><code class="lang-yaml"># /app/config.yaml
|
|
server:
|
|
port: 4000
|
|
host: 0.0.0.0
|
|
|
|
routing:
|
|
default_provider: openai
|
|
fallback_providers:
|
|
- anthropic
|
|
- together
|
|
|
|
rules:
|
|
- pattern: "code.*"
|
|
provider: anthropic
|
|
- pattern: "image.*"
|
|
provider: dalle3
|
|
- pattern: "voice.*"
|
|
provider: elevenlabs
|
|
|
|
cache:
|
|
enabled: true
|
|
ttl: 3600
|
|
max_size: 10GB
|
|
|
|
providers:
|
|
openai:
|
|
models: [...]
|
|
rate_limit: 10000/min
|
|
|
|
anthropic:
|
|
models: [...]
|
|
rate_limit: 5000/min</code></pre>
|
|
<p><strong>No provider block carries a key.</strong> The gateway asks <code>egress</code> for a call and names the provider; egress holds the credential in KMS custody and returns what the upstream said (HIP-0143). A <code>provider.api_key</code> field is the design egress replaces: a key in this file is a key in the process's memory, in its config volume, and in whatever reads either.</p>
|
|
<h4>OpenAI-Compatible API</h4>
|
|
<pre><code class="lang-python"># Python client example
|
|
from openai import OpenAI
|
|
|
|
client = OpenAI(
|
|
base_url="http://localhost:4000/v1",
|
|
api_key="sk-hanzo-..."
|
|
)
|
|
|
|
# Text completion
|
|
response = client.chat.completions.create(
|
|
model="gpt-4", # or "claude-3", "gemini-pro", etc.
|
|
messages=[
|
|
{"role": "user", "content": "Hello!"}
|
|
],
|
|
temperature=0.7,
|
|
stream=True
|
|
)
|
|
|
|
# Image generation
|
|
image = client.images.generate(
|
|
model="dall-e-3",
|
|
prompt="A sunset over mountains",
|
|
size="1024x1024"
|
|
)
|
|
|
|
# Embeddings
|
|
embedding = client.embeddings.create(
|
|
model="text-embedding-3-large",
|
|
input="Sample text"
|
|
)
|
|
|
|
# Audio transcription
|
|
transcription = client.audio.transcriptions.create(
|
|
model="whisper-1",
|
|
file=audio_file
|
|
)</code></pre>
|
|
<h3>Routing Engine</h3>
|
|
<pre><code class="lang-python">class RoutingEngine:
|
|
"""
|
|
Intelligent routing based on multiple factors
|
|
"""
|
|
def select_provider(self, request):
|
|
# 1. Check request pattern rules
|
|
for rule in self.rules:
|
|
if rule.matches(request):
|
|
return rule.provider
|
|
|
|
# 2. Cost optimization
|
|
if request.optimize_for == "cost":
|
|
return self.cheapest_provider(request)
|
|
|
|
# 3. Performance optimization
|
|
if request.optimize_for == "speed":
|
|
return self.fastest_provider(request)
|
|
|
|
# 4. Capability matching
|
|
providers = self.filter_by_capability(request)
|
|
|
|
# 5. Load balancing
|
|
return self.load_balance(providers)</code></pre>
|
|
<h3>Cost Management</h3>
|
|
<pre><code class="lang-yaml">Pricing Tiers:
|
|
Economy:
|
|
- Mixtral-8x7B: $0.0002/1K tokens
|
|
- Llama-3-70B: $0.0009/1K tokens
|
|
- GPT-3.5: $0.0010/1K tokens
|
|
|
|
Standard:
|
|
- GPT-4-Turbo: $0.01/1K tokens
|
|
- Claude-3-Sonnet: $0.003/1K tokens
|
|
- Gemini-Pro: $0.00125/1K tokens
|
|
|
|
Premium:
|
|
- GPT-4: $0.03/1K tokens
|
|
- Claude-3-Opus: $0.015/1K tokens
|
|
- Gemini-Ultra: $0.02/1K tokens
|
|
|
|
Budget Controls:
|
|
- Per-user limits
|
|
- Per-project quotas
|
|
- Automatic fallback to cheaper models
|
|
- Real-time cost tracking</code></pre>
|
|
<h3>Caching Layer</h3>
|
|
<pre><code class="lang-python">class SemanticCache:
|
|
"""
|
|
Intelligent caching with semantic similarity
|
|
"""
|
|
def __init__(self):
|
|
self.embedding_model = "text-embedding-3-small"
|
|
self.similarity_threshold = 0.95
|
|
|
|
async def get(self, prompt):
|
|
# Generate embedding
|
|
embedding = await self.embed(prompt)
|
|
|
|
# Search cache
|
|
similar = self.vector_search(
|
|
embedding,
|
|
threshold=self.similarity_threshold
|
|
)
|
|
|
|
if similar:
|
|
return similar.response
|
|
|
|
return None
|
|
|
|
async def set(self, prompt, response):
|
|
embedding = await self.embed(prompt)
|
|
self.store(embedding, prompt, response)</code></pre>
|
|
<h3>Monitoring & Analytics</h3>
|
|
<pre><code class="lang-yaml">Metrics:
|
|
Performance:
|
|
- Request latency (p50, p95, p99)
|
|
- Token generation speed
|
|
- Cache hit rate
|
|
- Error rate
|
|
|
|
Usage:
|
|
- Requests per second
|
|
- Tokens per minute
|
|
- Active users
|
|
- Model distribution
|
|
|
|
Cost:
|
|
- Cost per request
|
|
- Cost per user
|
|
- Provider breakdown
|
|
- Savings from cache
|
|
|
|
Alerting:
|
|
- Rate limit approaching
|
|
- Provider outage detected
|
|
- Budget exceeded
|
|
- Anomaly detection</code></pre>
|
|
<h3>Enterprise Features</h3>
|
|
<h4>Multi-Tenancy</h4>
|
|
<pre><code class="lang-python">class TenantManager:
|
|
def authenticate(self, api_key):
|
|
tenant = self.get_tenant(api_key)
|
|
return TenantContext(
|
|
id=tenant.id,
|
|
limits=tenant.limits,
|
|
providers=tenant.allowed_providers,
|
|
models=tenant.allowed_models
|
|
)</code></pre>
|
|
<h4>Audit Logging</h4>
|
|
<pre><code class="lang-json">{
|
|
"timestamp": "2025-01-09T10:30:00Z",
|
|
"tenant_id": "tenant_123",
|
|
"user_id": "user_456",
|
|
"request": {
|
|
"model": "gpt-4",
|
|
"prompt_tokens": 150,
|
|
"max_tokens": 500
|
|
},
|
|
"response": {
|
|
"provider": "openai",
|
|
"completion_tokens": 230,
|
|
"latency_ms": 1250
|
|
},
|
|
"cost": {
|
|
"amount": 0.0038,
|
|
"currency": "USD"
|
|
}
|
|
}</code></pre>
|
|
<h4>Data Privacy</h4>
|
|
<pre><code class="lang-yaml">Privacy Controls:
|
|
- PII detection and masking
|
|
- Data retention policies
|
|
- GDPR compliance
|
|
- Encryption at rest/transit
|
|
- No logging mode</code></pre>
|
|
<h3>Deployment</h3>
|
|
<p>The gateway holds <strong>no upstream credential</strong>. It is a caller of <code>egress</code> (HIP-0143), which means the deployment carries a machine identity and nothing that spends:</p>
|
|
<pre><code class="lang-yaml">apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: gateway
|
|
spec:
|
|
replicas: 3
|
|
template:
|
|
spec:
|
|
containers:
|
|
- name: gateway
|
|
image: oci.hanzo.ai/hanzoai/gateway
|
|
env:
|
|
# what it needs to identify ITSELF — not what it spends
|
|
- name: IAM_CLIENT_ID
|
|
valueFrom: { secretKeyRef: { name: gateway-env, key: IAM_CLIENT_ID } }
|
|
- name: IAM_CLIENT_SECRET
|
|
valueFrom: { secretKeyRef: { name: gateway-env, key: IAM_CLIENT_SECRET } }
|
|
- name: EGRESS_ADDRESS
|
|
value: egress.hanzo.ai:9653</code></pre>
|
|
<p>The Secret is materialized from KMS by the KMSSecret controller at the path HIP-0136 derives (<code>hanzo/gateway/<NAME>@prod</code>); the chart carries a reference and never a value.</p>
|
|
<p><strong>The test for whether this is done is falsifiable</strong>: enumerate the pod's environment and mounted secrets and find no upstream provider key. A gateway that still holds one has not adopted egress, it has moved the key.</p>
|
|
<p>State follows HIP-0144: there is no <code>llm_gateway</code> database and no gateway-owned Postgres or Redis. What the gateway keeps — routing state, budgets, a response cache — is per-tenant at rank 1 or in the one shared KV, and usage events go to the column store, which is what event data is for.</p>
|
|
<h3>SDK Support</h3>
|
|
<pre><code class="lang-python"># Python SDK
|
|
from hanzoai import LLMGateway
|
|
|
|
gateway = LLMGateway(
|
|
api_key="sk-hanzo-...",
|
|
base_url="https://api.hanzo.ai/v1"
|
|
)
|
|
|
|
response = gateway.chat.completions.create(
|
|
model="auto", # Automatic selection
|
|
messages=[{"role": "user", "content": "Hello"}],
|
|
optimize_for="cost" # or "speed", "quality"
|
|
)</code></pre>
|
|
<pre><code class="lang-typescript">// TypeScript SDK
|
|
import { LLMGateway } from '@hanzoai/llm-gateway';
|
|
|
|
const gateway = new LLMGateway({
|
|
apiKey: 'sk-hanzo-...',
|
|
baseURL: 'https://api.hanzo.ai/v1'
|
|
});
|
|
|
|
const response = await gateway.chat.completions.create({
|
|
model: 'auto',
|
|
messages: [{ role: 'user', content: 'Hello' }],
|
|
optimizeFor: 'cost'
|
|
});</code></pre>
|
|
<ol><li><a href="https://github.com/hanzoai/llm">LLM Gateway Repository</a></li></ol>
|
|
<h2>Copyright</h2>
|
|
<p>Copyright and related rights waived via <a href="https://creativecommons.org/publicdomain/zero/1.0/">CC0</a>.</p>
|
|
<footer>
|
|
<div class="fmap">
|
|
<div><b>Build</b>
|
|
<a href="https://hanzo.app">hanzo.app</a> app builder
|
|
<a href="https://hanzo.codes">hanzo.codes</a> code with an agent
|
|
<a href="https://hanzoskills.com">hanzoskills.com</a> what an agent reads</div>
|
|
<div><b>Run</b>
|
|
<a href="https://console.hanzo.ai">console.hanzo.ai</a> cloud console
|
|
<a href="https://api.hanzo.ai">api.hanzo.ai</a> the API</div>
|
|
<div><b>Account</b>
|
|
<a href="https://hanzo.id">hanzo.id</a> sign in
|
|
<a href="https://pay.hanzo.ai">pay.hanzo.ai</a> top up
|
|
<a href="https://billing.hanzo.ai">billing.hanzo.ai</a> invoices and usage</div>
|
|
<div><b>Company</b>
|
|
<a href="https://hanzo.ai">hanzo.ai</a> canonical
|
|
<a href="https://hanzo.ventures">hanzo.ventures</a> ventures</div>
|
|
</div>
|
|
<div class="fend"><a href="/hip-0004.md">This page as plain text</a> · <a href="/llms.txt">Index for agents</a></div>
|
|
</footer>
|
|
</div></body></html> |