mirror of
https://github.com/hanzoai/skills.git
synced 2026-11-19 00:47:36 +00:00
477 lines
35 KiB
HTML
477 lines
35 KiB
HTML
<!doctype html>
|
|
<html lang="en"><head>
|
|
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
<title>hip-0028</title>
|
|
<meta name="color-scheme" content="dark">
|
|
<style>
|
|
/* Committed to one visual world: a dark instrument panel. No light variant —
|
|
the dot field and the hairline rules only read on black, and a naive
|
|
inversion would wash both out. Every colour is painted explicitly so the page
|
|
never borrows a host background. */
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Pixel Square';src:url('/fonts/ZenPixel-Square.woff2') format('woff2');font-style:normal;font-display:swap}
|
|
|
|
:root{
|
|
--ink:#f4f4f2; --soft:#9c9c98; --faint:#6a6a66; --ground:#000; --panel:#0a0a0a;
|
|
--line:#232323; --line-hi:#3a3a3a; --accent:#f4f4f2;
|
|
--font-zen-sans:'Zen',-apple-system,BlinkMacSystemFont,"Segoe UI",Helvetica,Arial,sans-serif;
|
|
--font-zen-mono:'Zen Mono',ui-monospace,"SF Mono",SFMono-Regular,Menlo,Consolas,monospace;
|
|
--font-zen-pixel-square:'Zen Pixel Square',monospace;
|
|
--mono:var(--font-zen-mono);
|
|
--sans:var(--font-zen-sans);
|
|
/* Controls are pills and surfaces are soft, as on hanzo.ai. */
|
|
--r:999px; --r-soft:14px;
|
|
}
|
|
*{box-sizing:border-box}
|
|
html{-webkit-text-size-adjust:100%}
|
|
body{margin:0;background:var(--ground);color:var(--ink);font-family:var(--sans);
|
|
font-size:16px;line-height:1.65;-webkit-font-smoothing:antialiased}
|
|
.wrap{max-width:60rem;margin:0 auto;padding:0 1.75rem}
|
|
|
|
/* The dot field. A halftone made of one repeating radial-gradient, faded out
|
|
along the diagonal by a mask so it reads as a wave rather than wallpaper. */
|
|
/* No overflow clip here: the dot field is inset to the box already, and a clip
|
|
would cut off the copy-prompt menu. */
|
|
.hero{position:relative;border-bottom:1px solid var(--line)}
|
|
.hero::before{content:"";position:absolute;inset:0;
|
|
background-image:radial-gradient(var(--line-hi) 1px, transparent 1px);
|
|
background-size:7px 7px;
|
|
-webkit-mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
pointer-events:none}
|
|
.hero .wrap{position:relative;padding-top:5.5rem;padding-bottom:4rem}
|
|
|
|
nav{border-bottom:1px solid var(--line)}
|
|
nav .in{max-width:60rem;margin:0 auto;padding:1.1rem 1.75rem;display:flex;
|
|
justify-content:space-between;align-items:center;gap:1rem}
|
|
.brand{font-family:var(--mono);font-size:.85rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--ink);text-decoration:none;
|
|
display:inline-flex;align-items:center;gap:.6rem}
|
|
.brand i{font-style:normal;color:var(--faint)}
|
|
.mark{width:1.15em;height:1.15em;display:block;flex:0 0 auto}
|
|
nav a.up{font-family:var(--mono);font-size:.72rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--soft);text-decoration:none}
|
|
nav a.up:hover{color:var(--ink)}
|
|
|
|
h1{font-family:var(--mono);font-size:clamp(2rem,5.4vw,3.4rem);line-height:1.12;
|
|
letter-spacing:-.02em;margin:0 0 1.1rem;font-weight:500;text-wrap:balance}
|
|
h1 em{font-style:normal;color:var(--faint)}
|
|
.lede{color:var(--soft);font-size:1.05rem;max-width:52ch;margin:0 0 2.25rem}
|
|
|
|
.btns{display:flex;flex-wrap:wrap;gap:.7rem;margin:0 0 .5rem;align-items:center}
|
|
.btn{display:inline-flex;align-items:center;gap:.5rem;font-family:var(--mono);
|
|
font-size:.72rem;letter-spacing:.14em;text-transform:uppercase;
|
|
padding:.8rem 1.4rem;border:1px solid var(--line-hi);border-radius:var(--r);
|
|
color:var(--ink);text-decoration:none;background:transparent;cursor:pointer;
|
|
line-height:1.2;font-weight:400}
|
|
.btn:hover{background:#141414;text-decoration:none;border-color:var(--soft)}
|
|
.btn.solid{background:var(--ink);color:#000;border-color:var(--ink)}
|
|
.btn.solid:hover{background:#d8d8d4}
|
|
.btn .chev{width:.7em;height:.7em;flex:0 0 auto;transition:transform .15s ease}
|
|
.btn[aria-expanded="true"] .chev{transform:rotate(180deg)}
|
|
|
|
/* Copy-prompt selector. Each row copies the prompt for its own target. */
|
|
.copy{position:relative;display:inline-flex}
|
|
.menu{position:absolute;top:calc(100% + .5rem);left:0;z-index:20;min-width:21rem;
|
|
background:var(--panel);border:1px solid var(--line-hi);border-radius:var(--r-soft);
|
|
padding:.4rem;display:flex;flex-direction:column;
|
|
box-shadow:0 12px 32px rgba(0,0,0,.75)}
|
|
.menu[hidden]{display:none}
|
|
.mi{display:flex;align-items:center;gap:.7rem;width:100%;text-align:left;
|
|
padding:.62rem .8rem;border:0;border-radius:calc(var(--r-soft) - .35rem);
|
|
background:transparent;color:var(--ink);font-family:var(--mono);font-size:.76rem;
|
|
letter-spacing:.04em;text-decoration:none;cursor:pointer;line-height:1.3;
|
|
white-space:nowrap}
|
|
.mi:hover,.mi:focus-visible{background:#181818;outline:none}
|
|
.mi .mark{width:1.05rem;height:1.05rem;color:var(--soft)}
|
|
.mi:hover .mark{color:var(--ink)}
|
|
.mi u{margin-left:auto;color:var(--faint);text-decoration:none;font-size:.68rem;
|
|
letter-spacing:.1em;text-transform:uppercase}
|
|
|
|
/* Works-with bar: one scrolling row, name and mark per agent. */
|
|
.strip{border-bottom:1px solid var(--line);background:var(--panel)}
|
|
.strip .in{max-width:60rem;margin:0 auto;padding:.9rem 1.75rem;display:flex;
|
|
align-items:center;gap:1.25rem;font-family:var(--mono);font-size:.72rem;
|
|
letter-spacing:.1em;text-transform:uppercase;color:var(--faint)}
|
|
.strip b{color:var(--soft);font-weight:400;flex:0 0 auto;
|
|
border-right:1px solid var(--line);padding-right:1.25rem;align-self:stretch;
|
|
display:flex;align-items:center}
|
|
.bar{display:flex;align-items:center;gap:.5rem;flex:1 1 auto;min-width:0;
|
|
overflow-x:auto;scrollbar-width:none;scroll-snap-type:x proximity;
|
|
-webkit-overflow-scrolling:touch;
|
|
-webkit-mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent);
|
|
mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent)}
|
|
.bar::-webkit-scrollbar{display:none}
|
|
.agent{display:inline-flex;align-items:center;gap:.45rem;flex:0 0 auto;
|
|
padding:.4rem .85rem;border:1px solid var(--line);border-radius:var(--r);
|
|
color:var(--soft);white-space:nowrap;scroll-snap-align:start;
|
|
text-decoration:none;background:var(--ground)}
|
|
.agent:hover{color:var(--ink);border-color:var(--line-hi);text-decoration:none}
|
|
.agent .mark{width:.95rem;height:.95rem}
|
|
|
|
h2{font-family:var(--mono);font-size:1.1rem;letter-spacing:-.01em;margin:2.75rem 0 .75rem;font-weight:500}
|
|
h3{font-size:1rem;margin:2rem 0 .5rem;font-weight:600}
|
|
h4,h5,h6{font-size:.94rem;margin:1.5rem 0 .5rem;font-weight:600}
|
|
p{margin:0 0 1rem;max-width:70ch}
|
|
ul,ol{margin:0 0 1rem;padding-left:1.35rem;max-width:70ch}
|
|
li{margin:.22rem 0}
|
|
a{color:var(--ink);text-decoration:underline;text-underline-offset:3px;
|
|
text-decoration-color:var(--line-hi)}
|
|
a:hover{text-decoration-color:var(--ink)}
|
|
a:focus-visible{outline:1px solid var(--ink);outline-offset:3px}
|
|
code{font-family:var(--mono);font-size:.85em;background:var(--panel);
|
|
border:1px solid var(--line);border-radius:5px;padding:.1em .36em}
|
|
pre{background:var(--panel);border:1px solid var(--line);padding:1rem 1.1rem;
|
|
border-radius:var(--r-soft);overflow-x:auto;margin:0 0 1.25rem}
|
|
pre code{background:none;border:0;padding:0;font-size:.82rem;line-height:1.7}
|
|
blockquote{margin:0 0 1rem;padding-left:1rem;border-left:1px solid var(--line-hi);color:var(--soft)}
|
|
hr{border:0;border-top:1px solid var(--line);margin:2.5rem 0}
|
|
table{border-collapse:collapse;width:100%;margin:0 0 1.25rem;font-size:.92rem;
|
|
display:block;overflow-x:auto}
|
|
th,td{border:1px solid var(--line);padding:.5rem .7rem;text-align:left}
|
|
|
|
.curl{display:block;background:var(--panel);border:1px solid var(--line);
|
|
border-radius:var(--r-soft);padding:.95rem 1.1rem;font-family:var(--mono);
|
|
font-size:.8rem;color:var(--soft);overflow-x:auto;margin:0 0 2.5rem;
|
|
white-space:pre;line-height:1.9}
|
|
.curl b{color:var(--ink);font-weight:400}
|
|
.curl i{font-style:normal;color:var(--faint)}
|
|
|
|
/* index */
|
|
#q{width:100%;padding:.85rem 1.2rem;font-family:var(--mono);font-size:.85rem;color:var(--ink);
|
|
background:var(--panel);border:1px solid var(--line);border-radius:var(--r);margin:0 0 .6rem}
|
|
#q::placeholder{color:var(--faint);text-transform:uppercase;letter-spacing:.1em;font-size:.72rem}
|
|
#q:focus{outline:none;border-color:var(--soft)}
|
|
#count{color:var(--faint);font-family:var(--mono);font-size:.72rem;letter-spacing:.1em;
|
|
text-transform:uppercase;margin:0 0 2rem}
|
|
.cat{margin:2.5rem 0 .5rem;font-family:var(--mono);font-size:.7rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);border-bottom:1px solid var(--line);
|
|
padding-bottom:.45rem}
|
|
.row{display:block;padding:.6rem .9rem;margin:0 -.9rem;border-bottom:1px solid var(--line);
|
|
border-radius:var(--r-soft);text-decoration:none}
|
|
.row:hover{background:#0a0a0a}
|
|
.nm{font-family:var(--mono);font-size:.84rem;color:var(--ink);display:block}
|
|
.ds{color:var(--soft);font-size:.9rem;display:block;margin-top:.12rem}
|
|
footer{margin-top:4rem;padding:2rem 0 4rem;border-top:1px solid var(--line);
|
|
color:var(--faint);font-size:.85rem}
|
|
footer a{color:var(--soft)}
|
|
.fmap{display:grid;gap:1.75rem;grid-template-columns:repeat(auto-fit,minmax(11rem,1fr));
|
|
margin:0 0 2rem;font-size:.85rem}
|
|
.fmap b{display:block;font-family:var(--mono);font-size:.68rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);font-weight:400;margin:0 0 .6rem;
|
|
border-bottom:1px solid var(--line);padding-bottom:.4rem}
|
|
.fmap a{display:inline-block;color:var(--ink);text-decoration:none;margin:.3rem .35rem 0 0}
|
|
.fmap a:hover{text-decoration:underline}
|
|
.fmap div{color:var(--faint);line-height:1.5}
|
|
.fend{border-top:1px solid var(--line);padding-top:1.25rem;font-size:.82rem}
|
|
|
|
@media (max-width:640px){
|
|
.hero .wrap{padding-top:3.25rem;padding-bottom:2.75rem}
|
|
.btns{gap:.5rem}
|
|
.btn{padding:.75rem 1.15rem;font-size:.68rem}
|
|
.copy,.copy .btn{width:100%;justify-content:center}
|
|
.menu{left:0;right:0;min-width:0}
|
|
.strip .in{flex-direction:column;align-items:flex-start;gap:.6rem;padding:.9rem 1.75rem}
|
|
.strip b{border-right:0;padding-right:0}
|
|
.bar{width:100%}
|
|
}
|
|
</style>
|
|
<link rel="alternate" type="text/markdown" href="/hip-0028.md">
|
|
|
|
<link rel="icon" href="/favicon.svg">
|
|
<link rel="canonical" href="https://hanzoskills.com/hip-0028">
|
|
</head>
|
|
<!-- Reading this with a machine? Every page here is also plain markdown.
|
|
This one: https://hanzoskills.com/hip-0028.md
|
|
Index: https://hanzoskills.com/llms.txt
|
|
Corpus: https://hanzoskills.com/all.md
|
|
Append .md to any path for the markdown twin. -->
|
|
<body>
|
|
<nav><div class="in">
|
|
<a class="brand" href="/"><svg class="mark" viewBox="0 0 67 67" fill="currentColor" aria-hidden="true"><path d="M22.21 67V44.6369H0V67H22.21Z"/><path d="M66.7038 22.3184H22.2534L0.0878906 44.6367H44.4634L66.7038 22.3184Z"/><path d="M22.21 0H0V22.3184H22.21V0Z"/><path d="M66.7198 0H44.5098V22.3184H66.7198V0Z"/><path d="M66.7198 67V44.6369H44.5098V67H66.7198Z"/></svg> HANZO<i>SKILLS</i></a>
|
|
<a class="up" href="/">All skills</a>
|
|
</div></nav>
|
|
|
|
<div class="wrap">
|
|
<h1>hip-0028</h1><p class="lede">HIP-28: Key-Value Store Standard. Status Active. Hanzo's own standard — read this before implementing against it.</p><h1>HIP-0028: Key-Value Store Standard</h1>
|
|
<h2>Abstract</h2>
|
|
<p>This proposal defines the standard for Hanzo KV, the high-performance key-value store that serves as the shared caching, session, pub/sub, and streaming backbone for all services in the Hanzo ecosystem. Hanzo KV is built on KV 8.1, the Linux Foundation fork of Redis, and is distributed as <code>ghcr.io/hanzoai/kv:latest</code>. It exposes the RESP3 wire protocol on port 6379 and is a drop-in replacement for any KV client.</p>
|
|
<p><strong>Repository</strong>: <a href="https://github.com/hanzoai/kv">github.com/hanzoai/kv</a> <strong>Port</strong>: 6379 <strong>Docker</strong>: <code>ghcr.io/hanzoai/kv:latest</code> and <code>docker.io/hanzoai/kv:latest</code> <strong>License</strong>: BSD-3-Clause</p>
|
|
<h2>Motivation</h2>
|
|
<p>Every service in the Hanzo ecosystem -- IAM, LLM Gateway, Cloud, Chat, Commerce, Bot, Analytics, Zen -- needs a fast shared store for at least one of the following:</p>
|
|
<ol><li><strong>Session and token caching</strong>: OAuth tokens, rate-limit counters, CSRF nonces</li><li><strong>Pub/Sub messaging</strong>: real-time event propagation between services</li><li><strong>Streams</strong>: append-only logs for audit trails and event sourcing</li><li><strong>Ephemeral state</strong>: inference request queues, job locks, circuit-breaker state</li><li><strong>Leaderboard/sorted-set operations</strong>: billing rank, usage tracking</li></ol>
|
|
<p>Previously, the Hanzo infrastructure relied on the Bitnami KV Helm chart deployed via <code>helm install redis bitnami/redis</code>. This worked, but introduced three problems:</p>
|
|
<ul><li><strong>Licensing risk</strong>: KV Labs changed KV to a dual-license model (RSALv2 + SSPLv1)</li></ul>
|
|
<p> in March 2024. Both licenses restrict how cloud providers and SaaS platforms can distribute KV. For an infrastructure company like Hanzo that ships managed services, this is a direct legal exposure.</p>
|
|
<ul><li><strong>Operational opacity</strong>: The Bitnami chart bundles a metrics sidecar (redis-exporter),</li></ul>
|
|
<p> init containers, and Sentinel by default. When any of these sidecars fail (e.g., the exporter cannot authenticate to a password-protected instance), the entire pod enters CrashLoopBackOff and the root cause is obscured.</p>
|
|
<ul><li><strong>Image bloat</strong>: The Bitnami KV image is ~150MB compressed. A minimal Alpine-based</li></ul>
|
|
<p> KV image is ~12MB. In a cluster with rolling updates, smaller images mean faster pulls and shorter disruption windows.</p>
|
|
<p>Hanzo KV solves all three by replacing the entire Bitnami stack with a single, purpose-built container image based on KV.</p>
|
|
<h2>Design Philosophy</h2>
|
|
<p>This section explains every major design decision and why the alternatives were rejected. Infrastructure choices compound -- a wrong call here propagates to every service that touches KV. Each heading below addresses one decision.</p>
|
|
<h3>Why Single Instance over Cluster Mode</h3>
|
|
<p>Hanzo KV currently runs as a single-instance StatefulSet with 2Gi of PVC storage and a 2Gi memory limit. This is a deliberate choice, not a shortcut.</p>
|
|
<p><strong>Scale math</strong>: Our current production dataset (sessions, rate-limit counters, cache entries across all services) occupies approximately 400MB of memory. Even with 10x growth, we stay under 4GB. A single KV instance on modern hardware can saturate a 10Gbps NIC at ~1.2 million ops/sec. Our peak observed throughput is approximately 8,000 ops/sec. We are three orders of magnitude below the single-node ceiling.</p>
|
|
<p><strong>Cluster complexity</strong>: KV Cluster (and by extension KV Cluster) introduces hash slots, cross-slot restrictions on multi-key operations, MOVED/ASK redirects, and cluster bus gossip traffic. Every KV client library must understand cluster topology. Some operations (MULTI/EXEC across slots, Lua scripts touching multiple keys on different slots) simply do not work. This complexity buys horizontal scaling we do not need.</p>
|
|
<p><strong>Failure modes</strong>: A single instance has exactly one failure mode -- the pod dies and restarts. With AOF persistence, data loss on restart is bounded to the last fsync interval (1 second by default). A cluster has N failure modes: split-brain during network partition, slot migration failures, gossip protocol desynchronization, and partial availability when a master is down and its replica has not yet been promoted.</p>
|
|
<p><strong>Vertical ceiling</strong>: Kubernetes nodes support up to 64GB of memory. We can scale the KV StatefulSet to 32GB before even considering cluster mode. When we reach that point (which would imply ~80x current load), we will revisit with a separate HIP.</p>
|
|
<h3>Why We Removed the Metrics Sidecar</h3>
|
|
<p>The Bitnami KV chart ships with a <code>redis-exporter</code> sidecar that scrapes <code>INFO</code> output and exposes Prometheus metrics on port 9121. When we migrated to Hanzo KV with password authentication, the exporter sidecar could not authenticate because it expected the password in a different environment variable format than our secret layout provided.</p>
|
|
<p>Rather than debug the exporter's authentication logic and add another secret reference, we removed the sidecar entirely. The reasoning:</p>
|
|
<ul><li>KV's built-in <code>INFO</code> command already provides all metrics (memory, connections,</li></ul>
|
|
<p> keyspace, replication, persistence) in a machine-parseable format.</p>
|
|
<ul><li>For our current scale, <code>kubectl exec</code> into the pod and running <code>kv-cli INFO</code> is</li></ul>
|
|
<p> sufficient for debugging.</p>
|
|
<ul><li>When we need continuous Prometheus metrics, we will deploy <code>oliver006/redis_exporter</code></li></ul>
|
|
<p> as a separate Deployment (not a sidecar) with its own authentication config, decoupled from the KV pod lifecycle.</p>
|
|
<p><strong>Principle</strong>: a database pod should contain exactly one process -- the database. Every sidecar is a potential crash-loop vector that takes the database down with it.</p>
|
|
<h2>Specification</h2>
|
|
<h3>Wire Protocol</h3>
|
|
<p>Hanzo KV implements RESP3 (REdis Serialization Protocol version 3) as defined by the KV protocol specification. All commands from the KV 7.2 command set are supported. Any client library that speaks RESP2 or RESP3 is compatible.</p>
|
|
<h3>Connection Parameters</h3>
|
|
<pre><code class="lang-yaml">host: localhost
|
|
port: 6379
|
|
password: <from K8s secret "redis", key "redis-password">
|
|
db: 0 # default database
|
|
protocol: resp3 # RESP3 preferred, RESP2 accepted
|
|
tls: false # intra-cluster, TLS not required</code></pre>
|
|
<h3>Client Connection String</h3>
|
|
<p>Services should construct their connection URL as:</p>
|
|
<pre><code>redis://:${REDIS_PASSWORD}@redis-master:6379/0</code></pre>
|
|
<p>Or for explicit host within the hanzo namespace:</p>
|
|
<pre><code>redis://:${REDIS_PASSWORD}@localhost:6379/0</code></pre>
|
|
<h3>Configuration Reference</h3>
|
|
<p>The production <code>kv.conf</code> (mounted from ConfigMap):</p>
|
|
<pre><code class="lang-conf"># Persistence: AOF only, no RDB snapshots
|
|
appendonly yes
|
|
save ""
|
|
|
|
# Eviction: LRU when memory limit is reached
|
|
maxmemory-policy allkeys-lru
|
|
|
|
# Safety: disable destructive bulk operations
|
|
rename-command FLUSHDB ""
|
|
rename-command FLUSHALL ""</code></pre>
|
|
<p>Additional settings applied via container command-line arguments:</p>
|
|
<pre><code>--requirepass $(REDIS_PASSWORD) # authentication
|
|
--dir /data # persistence directory
|
|
--bind 0.0.0.0 # accept connections on all interfaces
|
|
--maxmemory-policy allkeys-lru # eviction policy (also in kv.conf for safety)
|
|
--protected-mode no # allow non-loopback connections (K8s networking)</code></pre>
|
|
<h3>Health Checks</h3>
|
|
<p><strong>Readiness probe</strong> (is the instance ready to accept commands?):</p>
|
|
<pre><code class="lang-yaml">exec:
|
|
command: ["sh", "-c", "kv-cli -a \"$REDIS_PASSWORD\" ping | grep -q PONG"]
|
|
initialDelaySeconds: 5
|
|
periodSeconds: 10
|
|
failureThreshold: 3</code></pre>
|
|
<p><strong>Liveness probe</strong> (is the instance alive and not deadlocked?):</p>
|
|
<pre><code class="lang-yaml">exec:
|
|
command: ["sh", "-c", "kv-cli -a \"$REDIS_PASSWORD\" ping | grep -q PONG"]
|
|
initialDelaySeconds: 15
|
|
periodSeconds: 30
|
|
failureThreshold: 5</code></pre>
|
|
<p>The liveness probe has a longer <code>initialDelaySeconds</code> and <code>failureThreshold</code> to avoid killing a pod that is replaying a large AOF on startup.</p>
|
|
<h3>Resource Allocation</h3>
|
|
<pre><code class="lang-yaml">resources:
|
|
requests:
|
|
cpu: 100m
|
|
memory: 256Mi
|
|
limits:
|
|
cpu: 1000m
|
|
memory: 2Gi</code></pre>
|
|
<p>The memory limit (2Gi) acts as a hard ceiling. Combined with <code>allkeys-lru</code>, KV will evict the least-recently-used keys when approaching this limit rather than crashing with an OOM error.</p>
|
|
<h3>Storage</h3>
|
|
<pre><code class="lang-yaml">volumeClaimTemplates:
|
|
- metadata:
|
|
name: redis-data
|
|
spec:
|
|
accessModes: [ReadWriteOnce]
|
|
resources:
|
|
requests:
|
|
storage: 2Gi</code></pre>
|
|
<p>The PVC stores the AOF file. With our current workload, the AOF (after automatic compaction) stays under 100MB. The 2Gi allocation provides 20x headroom.</p>
|
|
<h2>Implementation</h2>
|
|
<h3>Container Image</h3>
|
|
<p>The Dockerfile is minimal by design:</p>
|
|
<pre><code class="lang-dockerfile">ARG KV_VERSION=8.1
|
|
|
|
FROM valkey/valkey:${KV_VERSION}-alpine AS base
|
|
FROM base
|
|
|
|
LABEL maintainer="[email protected]"
|
|
LABEL org.opencontainers.image.source="https://github.com/hanzoai/kv"
|
|
LABEL org.opencontainers.image.description="Hanzo KV - High-performance key-value store"
|
|
LABEL org.opencontainers.image.vendor="Hanzo AI"
|
|
|
|
# Install Hanzo KV CLI tools
|
|
# Primary names are kv-* ; legacy valkey-* names remain as symlinks
|
|
RUN cp /usr/local/bin/valkey-server /usr/local/bin/kv-server \
|
|
&& cp /usr/local/bin/valkey-cli /usr/local/bin/kv-cli \
|
|
&& ln -sf /usr/local/bin/kv-cli /usr/local/bin/kv \
|
|
&& cp /usr/local/bin/valkey-sentinel /usr/local/bin/kv-sentinel 2>/dev/null; \
|
|
cp /usr/local/bin/valkey-benchmark /usr/local/bin/kv-benchmark 2>/dev/null; \
|
|
cp /usr/local/bin/valkey-check-aof /usr/local/bin/kv-check-aof 2>/dev/null; \
|
|
cp /usr/local/bin/valkey-check-rdb /usr/local/bin/kv-check-rdb 2>/dev/null; \
|
|
true
|
|
|
|
EXPOSE 6379
|
|
|
|
HEALTHCHECK --interval=15s --timeout=3s --start-period=10s --retries=3 \
|
|
CMD kv ping | grep -q PONG || exit 1
|
|
|
|
ENTRYPOINT ["kv-server"]
|
|
CMD ["--bind", "0.0.0.0", "--dir", "/data", \
|
|
"--maxmemory-policy", "allkeys-lru", "--protected-mode", "no"]</code></pre>
|
|
<p>Key points:</p>
|
|
<ul><li><strong>Base image</strong>: <code>valkey/valkey:8.1-alpine</code> (~12MB compressed)</li><li><strong>CLI renaming</strong>: All KV binaries are copied to <code>kv-*</code> names. The original</li></ul>
|
|
<p> <code>valkey-*</code> names remain as the originals. This gives operators a clean Hanzo-branded CLI while maintaining compatibility with scripts that reference <code>valkey-cli</code>.</p>
|
|
<ul><li><strong>No custom compilation</strong>: We use the upstream KV binary as-is. Custom patches</li></ul>
|
|
<p> would create a maintenance burden and diverge from upstream security fixes.</p>
|
|
<h3>CLI Tools</h3>
|
|
<p>| Command | Description | |---------|-------------| | <code>kv</code> | Interactive CLI (symlink to kv-cli) | | <code>kv-server</code> | Start KV server | | <code>kv-cli</code> | Command-line client | | <code>kv-sentinel</code> | High-availability sentinel | | <code>kv-benchmark</code> | Performance benchmarking tool | | <code>kv-check-aof</code> | AOF file integrity checker | | <code>kv-check-rdb</code> | RDB file integrity checker |</p>
|
|
<h3>CI/CD Pipeline</h3>
|
|
<p>The deploy workflow (<code>.github/workflows/deploy.yml</code>) has two stages:</p>
|
|
<p><strong>Stage 1: Build</strong></p>
|
|
<ol><li>Checkout source from <code>github.com/hanzoai/kv</code></li><li>Authenticate to Hanzo KMS (Universal Auth) to fetch CI secrets</li><li>Build multi-arch image (<code>linux/amd64</code>, <code>linux/arm64</code>) via Docker Buildx</li><li>Push to GHCR (<code>ghcr.io/hanzoai/kv</code>) with tags: <code>latest</code>, git SHA, semver</li><li>Push to Docker Hub (<code>docker.io/hanzoai/kv</code>) as fallback (continue-on-error)</li></ol>
|
|
<p><strong>Stage 2: Deploy (main branch only)</strong></p>
|
|
<ol><li>Authenticate to Hanzo KMS for DigitalOcean API token</li><li>Configure <code>kubectl</code> for <code>the cluster</code> cluster via <code>doctl</code></li><li>Rolling update: <code>kubectl set image statefulset/redis-master kv=ghcr.io/hanzoai/kv:latest</code></li><li>Wait for rollout: <code>kubectl rollout status statefulset/redis-master --timeout=120s</code></li></ol>
|
|
<p>Trigger conditions: push to <code>main</code>, tag push (<code>v*</code>), or manual <code>workflow_dispatch</code>.</p>
|
|
<h3>K8s Manifest Structure</h3>
|
|
<p>All manifests live in <code>universe/infra/k8s/kv/</code> and are aggregated via Kustomize:</p>
|
|
<pre><code class="lang-yaml"># kustomization.yaml
|
|
apiVersion: kustomize.config.k8s.io/v1beta1
|
|
kind: Kustomization
|
|
resources:
|
|
- statefulset.yaml
|
|
- service.yaml
|
|
- secret.yaml
|
|
- configmap.yaml</code></pre>
|
|
<h3>Migration from Bitnami KV</h3>
|
|
<p>The migration from the Bitnami KV Helm chart to Hanzo KV was performed as follows:</p>
|
|
<ol><li><strong>Scale down Bitnami</strong>: <code>helm uninstall redis</code> removes the Deployment and Service but</li></ol>
|
|
<p> preserves the PVC (Helm default <code>resourcePolicy: keep</code>).</p>
|
|
<ol><li><strong>Apply Hanzo KV manifests</strong>: The StatefulSet uses the same PVC name (<code>redis-data</code>),</li></ol>
|
|
<p> same Service name (<code>redis-master</code>), and same secret name (<code>redis</code>). This means the new pod attaches to the existing PVC with all data intact.</p>
|
|
<ol><li><strong>Verify data</strong>: <code>kv-cli -a "$REDIS_PASSWORD" DBSIZE</code> confirms key count matches</li></ol>
|
|
<p> pre-migration.</p>
|
|
<ol><li><strong>Remove Helm artifacts</strong>: Clean up orphaned Helm release secrets.</li></ol>
|
|
<p>The migration is zero-downtime because the Service name and selector labels are preserved. Client connections fail for the ~30 seconds between the old pod terminating and the new pod passing its readiness probe, which is within the retry tolerance of all Hanzo services.</p>
|
|
<h3>Client SDKs</h3>
|
|
<p>| Language | Package | Install | |----------|---------|---------| | Python | <a href="https://pypi.org/project/hanzo-kv">hanzo-kv</a> | <code>pip install hanzo-kv</code> | | Go | <a href="https://github.com/hanzoai/kv-go">hanzo/kv-go</a> | <code>go get github.com/hanzoai/kv-go</code> | | Node.js | <a href="https://github.com/hanzoai/kv-client">@hanzo/kv</a> | <code>npm install @hanzo/kv</code> |</p>
|
|
<p>All three are thin wrappers around standard KV client libraries (<code>redis-py</code>, <code>go-redis</code>, <code>ioredis</code>) with Hanzo-specific defaults (connection URL construction, KMS secret resolution, structured logging). Any vanilla KV client works equally well.</p>
|
|
<h2>Security</h2>
|
|
<h3>Authentication</h3>
|
|
<p>All connections require a password. The password is stored in a K8s Secret:</p>
|
|
<pre><code class="lang-yaml">apiVersion: v1
|
|
kind: Secret
|
|
metadata:
|
|
name: redis
|
|
namespace: hanzo
|
|
type: Opaque
|
|
stringData:
|
|
redis-password: "<generated-value>"</code></pre>
|
|
<p>Services receive the password via environment variable injection from this secret. The secret name (<code>redis</code>) and key (<code>redis-password</code>) match the Bitnami convention to avoid changing every service deployment manifest.</p>
|
|
<p>In production, this secret is synced from Hanzo KMS (<code>kms.hanzo.ai</code>) via the KMS Operator. The plaintext value in the manifest is a bootstrap default that gets overwritten on first KMS sync.</p>
|
|
<h3>Network Isolation</h3>
|
|
<ul><li><strong>Service type</strong>: ClusterIP (no external exposure)</li><li><strong>No NodePort, no LoadBalancer, no Ingress</strong></li><li>Only pods within the <code>hanzo</code> namespace (or with appropriate NetworkPolicy) can reach</li></ul>
|
|
<p> port 6379</p>
|
|
<ul><li>The <code>--protected-mode no</code> flag is safe because the pod is never exposed outside the</li></ul>
|
|
<p> cluster. Protected mode is a KV safety net for instances accidentally exposed to the internet without a password; our instance has both network isolation and a password.</p>
|
|
<h3>Dangerous Command Disablement</h3>
|
|
<p>As specified in the Configuration section, <code>FLUSHDB</code> and <code>FLUSHALL</code> are renamed to empty strings (disabled). Additional commands to consider disabling in future:</p>
|
|
<ul><li><code>DEBUG</code> -- can crash the server or dump memory</li><li><code>CONFIG</code> -- can change runtime settings (e.g., disable authentication)</li><li><code>SHUTDOWN</code> -- can stop the server</li></ul>
|
|
<p>These are not currently disabled because they are useful for debugging in a cluster environment where only operators have <code>kubectl exec</code> access.</p>
|
|
<h3>TLS</h3>
|
|
<p>TLS is available in KV 8.1 but not enabled for intra-cluster communication. The reasoning:</p>
|
|
<ul><li>All traffic stays within the Kubernetes VPC, encrypted at the network layer by DigitalOcean</li><li>TLS adds ~15% latency overhead on every command due to encryption/decryption</li><li>The threat model (attacker with VPC access) implies they already have <code>kubectl</code> access</li></ul>
|
|
<p> and can read secrets directly</p>
|
|
<p>If we add external replication (e.g., cross-cluster) or expose KV outside the VPC, TLS will be enabled via <code>--tls-port 6380 --tls-cert-file --tls-key-file --tls-ca-cert-file</code>.</p>
|
|
<h3>Memory Limits</h3>
|
|
<p>The 2Gi memory limit prevents a runaway client from consuming all node memory and triggering the Linux OOM killer (which would kill the KV process and potentially other pods on the same node). With <code>allkeys-lru</code>, KV gracefully evicts cold keys instead of refusing writes or crashing.</p>
|
|
<h2>Consumers</h2>
|
|
<p>Services in the Hanzo ecosystem that connect to KV:</p>
|
|
<p>| Service | Use Case | Key Pattern | |---------|----------|-------------| | IAM (hanzo.id) | Session tokens, OAuth state | <code>iam:session:<em></code>, <code>iam:oauth:</em></code> | | LLM Gateway | Rate limiting, response cache | <code>llm:rate:<em></code>, <code>llm:cache:</em></code> | | Cloud | Job queues, inference state | <code>cloud:job:<em></code>, <code>cloud:inf:</em></code> | | Console | Session cache | <code>console:session:<em></code> | | Chat | Conversation state, pub/sub | <code>chat:conv:</em></code>, <code>chat:stream:<em></code> | | Bot | Command state, cooldowns | <code>bot:state:</em></code>, <code>bot:cd:<em></code> | | Analytics | Event buffering | <code>analytics:buf:</em></code> | | Zen | Model routing cache | <code>zen:route:<em></code> | | Commerce | Cart state, rate limits | <code>commerce:cart:</em></code> |</p>
|
|
<h3>Key Namespace Convention</h3>
|
|
<p>All keys SHOULD be prefixed with <code><service>:<category>:<id></code>. This enables:</p>
|
|
<ul><li>Per-service monitoring via <code>kv-cli --stat</code> or <code>SCAN</code> with pattern matching</li><li>Targeted eviction of one service's keys without affecting others</li><li>Clear ownership when debugging unexpected key growth</li></ul>
|
|
<h2>Monitoring</h2>
|
|
<h3>Built-in Metrics</h3>
|
|
<p>KV's <code>INFO</code> command provides comprehensive metrics without any sidecar:</p>
|
|
<pre><code class="lang-bash"># Memory usage
|
|
kv-cli -a "$REDIS_PASSWORD" INFO memory
|
|
|
|
# Client connections
|
|
kv-cli -a "$REDIS_PASSWORD" INFO clients
|
|
|
|
# Keyspace statistics
|
|
kv-cli -a "$REDIS_PASSWORD" INFO keyspace
|
|
|
|
# Persistence status
|
|
kv-cli -a "$REDIS_PASSWORD" INFO persistence
|
|
|
|
# All metrics
|
|
kv-cli -a "$REDIS_PASSWORD" INFO all</code></pre>
|
|
<h3>Key Metrics to Watch</h3>
|
|
<p>| Metric | Warning Threshold | Critical Threshold | |--------|------------------|--------------------| | <code>used_memory</code> | > 1.5Gi (75% of limit) | > 1.8Gi (90%) | | <code>connected_clients</code> | > 100 | > 500 | | <code>evicted_keys</code> | > 0 (indicates memory pressure) | > 1000/min | | <code>rejected_connections</code> | > 0 | > 10/min | | <code>aof_last_bgrewrite_status</code> | <code>err</code> | - | | <code>instantaneous_ops_per_sec</code> | > 50,000 | > 100,000 |</p>
|
|
<h3>Future: Prometheus Integration</h3>
|
|
<p>When continuous monitoring is needed, deploy <code>oliver006/redis_exporter</code> as a standalone Deployment in the <code>hanzo</code> namespace:</p>
|
|
<pre><code class="lang-yaml">apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: kv-exporter
|
|
namespace: hanzo
|
|
spec:
|
|
replicas: 1
|
|
template:
|
|
spec:
|
|
containers:
|
|
- name: exporter
|
|
image: oliver006/redis_exporter:latest
|
|
env:
|
|
- name: REDIS_ADDR
|
|
value: redis-master:6379
|
|
- name: REDIS_PASSWORD
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: redis
|
|
key: redis-password
|
|
ports:
|
|
- containerPort: 9121</code></pre>
|
|
<p>This runs as a separate pod, not a sidecar. If the exporter crashes, KV is unaffected.</p>
|
|
<h2>Reference Implementation</h2>
|
|
<p><strong>Repository</strong>: <a href="https://github.com/hanzoai/kv">github.com/hanzoai/kv</a></p>
|
|
<p><strong>Key Files</strong>:</p>
|
|
<ul><li><code>Dockerfile</code> -- Multi-arch container image based on KV 8.1 Alpine</li><li><code>.github/workflows/deploy.yml</code> -- CI/CD: build, push to GHCR/Docker Hub, deploy to K8s</li><li><code>.github/workflows/ci.yml</code> -- Upstream KV test suite</li><li><code>valkey.conf</code> -- Full reference configuration (upstream defaults)</li><li><code>sentinel.conf</code> -- Sentinel configuration for HA deployments</li></ul>
|
|
<p><strong>K8s Manifests</strong> (<code>universe/infra/k8s/kv/</code>):</p>
|
|
<ul><li><code>statefulset.yaml</code> -- StatefulSet <code>redis-master</code> with PVC and health checks</li><li><code>service.yaml</code> -- ClusterIP Service on port 6379</li><li><code>configmap.yaml</code> -- <code>kv.conf</code> (AOF, eviction policy, disabled commands)</li><li><code>secret.yaml</code> -- KV-compatible password secret</li><li><code>kustomization.yaml</code> -- Kustomize aggregation</li></ul>
|
|
<p><strong>Status</strong>: Implemented and running in production on <code>the cluster</code> (<code>24.199.76.156</code>)</p>
|
|
<h2>References</h2>
|
|
<ol><li><a href="./hip-0000-hanzo-ai-architecture-framework.md">HIP-0: Hanzo AI Architecture Framework</a></li><li><a href="./hip-0004-llm-gateway-unified-ai-provider-interface.md">HIP-4: LLM Gateway</a></li><li><a href="https://valkey.io/">KV Project</a> -- Linux Foundation fork of KV</li><li><a href="https://redis.io/blog/redis-adopts-dual-source-available-licensing/">Redis License Change Announcement</a> -- March 2024</li><li><a href="https://valkey.io/blog/valkey-8-1-0-rc1/">KV 8.1 Release Notes</a></li><li><a href="https://github.com/redis/redis-specifications/blob/master/protocol/RESP3.md">RESP3 Protocol Specification</a></li></ol>
|
|
<h2>Copyright</h2>
|
|
<p>Copyright and related rights waived via <a href="https://creativecommons.org/publicdomain/zero/1.0/">CC0</a>.</p>
|
|
<footer>
|
|
<div class="fmap">
|
|
<div><b>Build</b>
|
|
<a href="https://hanzo.app">hanzo.app</a> app builder
|
|
<a href="https://hanzo.codes">hanzo.codes</a> code with an agent
|
|
<a href="https://hanzoskills.com">hanzoskills.com</a> what an agent reads</div>
|
|
<div><b>Run</b>
|
|
<a href="https://console.hanzo.ai">console.hanzo.ai</a> cloud console
|
|
<a href="https://api.hanzo.ai">api.hanzo.ai</a> the API</div>
|
|
<div><b>Account</b>
|
|
<a href="https://hanzo.id">hanzo.id</a> sign in
|
|
<a href="https://pay.hanzo.ai">pay.hanzo.ai</a> top up
|
|
<a href="https://billing.hanzo.ai">billing.hanzo.ai</a> invoices and usage</div>
|
|
<div><b>Company</b>
|
|
<a href="https://hanzo.ai">hanzo.ai</a> canonical
|
|
<a href="https://hanzo.ventures">hanzo.ventures</a> ventures</div>
|
|
</div>
|
|
<div class="fend"><a href="/hip-0028.md">This page as plain text</a> · <a href="/llms.txt">Index for agents</a></div>
|
|
</footer>
|
|
</div>
|
|
</body></html> |