mirror of
https://github.com/hanzoai/skills.git
synced 2026-11-19 00:47:36 +00:00
820 lines
52 KiB
HTML
820 lines
52 KiB
HTML
<!doctype html>
|
|
<html lang="en"><head>
|
|
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
<title>hip-0047</title>
|
|
<meta name="color-scheme" content="dark">
|
|
<style>
|
|
/* Committed to one visual world: a dark instrument panel. No light variant —
|
|
the dot field and the hairline rules only read on black, and a naive
|
|
inversion would wash both out. Every colour is painted explicitly so the page
|
|
never borrows a host background. */
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Pixel Square';src:url('/fonts/ZenPixel-Square.woff2') format('woff2');font-style:normal;font-display:swap}
|
|
|
|
:root{
|
|
--ink:#f4f4f2; --soft:#9c9c98; --faint:#6a6a66; --ground:#000; --panel:#0a0a0a;
|
|
--line:#232323; --line-hi:#3a3a3a; --accent:#f4f4f2;
|
|
--font-zen-sans:'Zen',-apple-system,BlinkMacSystemFont,"Segoe UI",Helvetica,Arial,sans-serif;
|
|
--font-zen-mono:'Zen Mono',ui-monospace,"SF Mono",SFMono-Regular,Menlo,Consolas,monospace;
|
|
--font-zen-pixel-square:'Zen Pixel Square',monospace;
|
|
--mono:var(--font-zen-mono);
|
|
--sans:var(--font-zen-sans);
|
|
/* Controls are pills and surfaces are soft, as on hanzo.ai. */
|
|
--r:999px; --r-soft:14px;
|
|
}
|
|
*{box-sizing:border-box}
|
|
html{-webkit-text-size-adjust:100%}
|
|
body{margin:0;background:var(--ground);color:var(--ink);font-family:var(--sans);
|
|
font-size:16px;line-height:1.65;-webkit-font-smoothing:antialiased}
|
|
.wrap{max-width:60rem;margin:0 auto;padding:0 1.75rem}
|
|
|
|
/* The dot field. A halftone made of one repeating radial-gradient, faded out
|
|
along the diagonal by a mask so it reads as a wave rather than wallpaper. */
|
|
/* No overflow clip here: the dot field is inset to the box already, and a clip
|
|
would cut off the copy-prompt menu. */
|
|
.hero{position:relative;border-bottom:1px solid var(--line)}
|
|
.hero::before{content:"";position:absolute;inset:0;
|
|
background-image:radial-gradient(var(--line-hi) 1px, transparent 1px);
|
|
background-size:7px 7px;
|
|
-webkit-mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
pointer-events:none}
|
|
.hero .wrap{position:relative;padding-top:5.5rem;padding-bottom:4rem}
|
|
|
|
nav{border-bottom:1px solid var(--line)}
|
|
nav .in{max-width:60rem;margin:0 auto;padding:1.1rem 1.75rem;display:flex;
|
|
justify-content:space-between;align-items:center;gap:1rem}
|
|
.brand{font-family:var(--mono);font-size:.85rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--ink);text-decoration:none;
|
|
display:inline-flex;align-items:center;gap:.6rem}
|
|
.brand i{font-style:normal;color:var(--faint)}
|
|
.mark{width:1.15em;height:1.15em;display:block;flex:0 0 auto}
|
|
nav a.up{font-family:var(--mono);font-size:.72rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--soft);text-decoration:none}
|
|
nav a.up:hover{color:var(--ink)}
|
|
|
|
h1{font-family:var(--mono);font-size:clamp(2rem,5.4vw,3.4rem);line-height:1.12;
|
|
letter-spacing:-.02em;margin:0 0 1.1rem;font-weight:500;text-wrap:balance}
|
|
h1 em{font-style:normal;color:var(--faint)}
|
|
.lede{color:var(--soft);font-size:1.05rem;max-width:52ch;margin:0 0 2.25rem}
|
|
|
|
.btns{display:flex;flex-wrap:wrap;gap:.7rem;margin:0 0 .5rem;align-items:center}
|
|
.btn{display:inline-flex;align-items:center;gap:.5rem;font-family:var(--mono);
|
|
font-size:.72rem;letter-spacing:.14em;text-transform:uppercase;
|
|
padding:.8rem 1.4rem;border:1px solid var(--line-hi);border-radius:var(--r);
|
|
color:var(--ink);text-decoration:none;background:transparent;cursor:pointer;
|
|
line-height:1.2;font-weight:400}
|
|
.btn:hover{background:#141414;text-decoration:none;border-color:var(--soft)}
|
|
.btn.solid{background:var(--ink);color:#000;border-color:var(--ink)}
|
|
.btn.solid:hover{background:#d8d8d4}
|
|
.btn .chev{width:.7em;height:.7em;flex:0 0 auto;transition:transform .15s ease}
|
|
.btn[aria-expanded="true"] .chev{transform:rotate(180deg)}
|
|
|
|
/* Copy-prompt selector. Each row copies the prompt for its own target. */
|
|
.copy{position:relative;display:inline-flex}
|
|
.menu{position:absolute;top:calc(100% + .5rem);left:0;z-index:20;min-width:21rem;
|
|
background:var(--panel);border:1px solid var(--line-hi);border-radius:var(--r-soft);
|
|
padding:.4rem;display:flex;flex-direction:column;
|
|
box-shadow:0 12px 32px rgba(0,0,0,.75)}
|
|
.menu[hidden]{display:none}
|
|
.mi{display:flex;align-items:center;gap:.7rem;width:100%;text-align:left;
|
|
padding:.62rem .8rem;border:0;border-radius:calc(var(--r-soft) - .35rem);
|
|
background:transparent;color:var(--ink);font-family:var(--mono);font-size:.76rem;
|
|
letter-spacing:.04em;text-decoration:none;cursor:pointer;line-height:1.3;
|
|
white-space:nowrap}
|
|
.mi:hover,.mi:focus-visible{background:#181818;outline:none}
|
|
.mi .mark{width:1.05rem;height:1.05rem;color:var(--soft)}
|
|
.mi:hover .mark{color:var(--ink)}
|
|
.mi u{margin-left:auto;color:var(--faint);text-decoration:none;font-size:.68rem;
|
|
letter-spacing:.1em;text-transform:uppercase}
|
|
|
|
/* Works-with bar: one scrolling row, name and mark per agent. */
|
|
.strip{border-bottom:1px solid var(--line);background:var(--panel)}
|
|
.strip .in{max-width:60rem;margin:0 auto;padding:.9rem 1.75rem;display:flex;
|
|
align-items:center;gap:1.25rem;font-family:var(--mono);font-size:.72rem;
|
|
letter-spacing:.1em;text-transform:uppercase;color:var(--faint)}
|
|
.strip b{color:var(--soft);font-weight:400;flex:0 0 auto;
|
|
border-right:1px solid var(--line);padding-right:1.25rem;align-self:stretch;
|
|
display:flex;align-items:center}
|
|
.bar{display:flex;align-items:center;gap:.5rem;flex:1 1 auto;min-width:0;
|
|
overflow-x:auto;scrollbar-width:none;scroll-snap-type:x proximity;
|
|
-webkit-overflow-scrolling:touch;
|
|
-webkit-mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent);
|
|
mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent)}
|
|
.bar::-webkit-scrollbar{display:none}
|
|
.agent{display:inline-flex;align-items:center;gap:.45rem;flex:0 0 auto;
|
|
padding:.4rem .85rem;border:1px solid var(--line);border-radius:var(--r);
|
|
color:var(--soft);white-space:nowrap;scroll-snap-align:start;
|
|
text-decoration:none;background:var(--ground)}
|
|
.agent:hover{color:var(--ink);border-color:var(--line-hi);text-decoration:none}
|
|
.agent .mark{width:.95rem;height:.95rem}
|
|
|
|
h2{font-family:var(--mono);font-size:1.1rem;letter-spacing:-.01em;margin:2.75rem 0 .75rem;font-weight:500}
|
|
h3{font-size:1rem;margin:2rem 0 .5rem;font-weight:600}
|
|
h4,h5,h6{font-size:.94rem;margin:1.5rem 0 .5rem;font-weight:600}
|
|
p{margin:0 0 1rem;max-width:70ch}
|
|
ul,ol{margin:0 0 1rem;padding-left:1.35rem;max-width:70ch}
|
|
li{margin:.22rem 0}
|
|
a{color:var(--ink);text-decoration:underline;text-underline-offset:3px;
|
|
text-decoration-color:var(--line-hi)}
|
|
a:hover{text-decoration-color:var(--ink)}
|
|
a:focus-visible{outline:1px solid var(--ink);outline-offset:3px}
|
|
code{font-family:var(--mono);font-size:.85em;background:var(--panel);
|
|
border:1px solid var(--line);border-radius:5px;padding:.1em .36em}
|
|
pre{background:var(--panel);border:1px solid var(--line);padding:1rem 1.1rem;
|
|
border-radius:var(--r-soft);overflow-x:auto;margin:0 0 1.25rem}
|
|
pre code{background:none;border:0;padding:0;font-size:.82rem;line-height:1.7}
|
|
blockquote{margin:0 0 1rem;padding-left:1rem;border-left:1px solid var(--line-hi);color:var(--soft)}
|
|
hr{border:0;border-top:1px solid var(--line);margin:2.5rem 0}
|
|
table{border-collapse:collapse;width:100%;margin:0 0 1.25rem;font-size:.92rem;
|
|
display:block;overflow-x:auto}
|
|
th,td{border:1px solid var(--line);padding:.5rem .7rem;text-align:left}
|
|
|
|
.curl{display:block;background:var(--panel);border:1px solid var(--line);
|
|
border-radius:var(--r-soft);padding:.95rem 1.1rem;font-family:var(--mono);
|
|
font-size:.8rem;color:var(--soft);overflow-x:auto;margin:0 0 2.5rem;
|
|
white-space:pre;line-height:1.9}
|
|
.curl b{color:var(--ink);font-weight:400}
|
|
.curl i{font-style:normal;color:var(--faint)}
|
|
|
|
/* index */
|
|
#q{width:100%;padding:.85rem 1.2rem;font-family:var(--mono);font-size:.85rem;color:var(--ink);
|
|
background:var(--panel);border:1px solid var(--line);border-radius:var(--r);margin:0 0 .6rem}
|
|
#q::placeholder{color:var(--faint);text-transform:uppercase;letter-spacing:.1em;font-size:.72rem}
|
|
#q:focus{outline:none;border-color:var(--soft)}
|
|
#count{color:var(--faint);font-family:var(--mono);font-size:.72rem;letter-spacing:.1em;
|
|
text-transform:uppercase;margin:0 0 2rem}
|
|
.cat{margin:2.5rem 0 .5rem;font-family:var(--mono);font-size:.7rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);border-bottom:1px solid var(--line);
|
|
padding-bottom:.45rem}
|
|
.row{display:block;padding:.6rem .9rem;margin:0 -.9rem;border-bottom:1px solid var(--line);
|
|
border-radius:var(--r-soft);text-decoration:none}
|
|
.row:hover{background:#0a0a0a}
|
|
.nm{font-family:var(--mono);font-size:.84rem;color:var(--ink);display:block}
|
|
.ds{color:var(--soft);font-size:.9rem;display:block;margin-top:.12rem}
|
|
footer{margin-top:4rem;padding:2rem 0 4rem;border-top:1px solid var(--line);
|
|
color:var(--faint);font-size:.85rem}
|
|
footer a{color:var(--soft)}
|
|
.fmap{display:grid;gap:1.75rem;grid-template-columns:repeat(auto-fit,minmax(11rem,1fr));
|
|
margin:0 0 2rem;font-size:.85rem}
|
|
.fmap b{display:block;font-family:var(--mono);font-size:.68rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);font-weight:400;margin:0 0 .6rem;
|
|
border-bottom:1px solid var(--line);padding-bottom:.4rem}
|
|
.fmap a{display:inline-block;color:var(--ink);text-decoration:none;margin:.3rem .35rem 0 0}
|
|
.fmap a:hover{text-decoration:underline}
|
|
.fmap div{color:var(--faint);line-height:1.5}
|
|
.fend{border-top:1px solid var(--line);padding-top:1.25rem;font-size:.82rem}
|
|
|
|
@media (max-width:640px){
|
|
.hero .wrap{padding-top:3.25rem;padding-bottom:2.75rem}
|
|
.btns{gap:.5rem}
|
|
.btn{padding:.75rem 1.15rem;font-size:.68rem}
|
|
.copy,.copy .btn{width:100%;justify-content:center}
|
|
.menu{left:0;right:0;min-width:0}
|
|
.strip .in{flex-direction:column;align-items:flex-start;gap:.6rem;padding:.9rem 1.75rem}
|
|
.strip b{border-right:0;padding-right:0}
|
|
.bar{width:100%}
|
|
}
|
|
</style>
|
|
<link rel="alternate" type="text/markdown" href="/hip-0047.md">
|
|
|
|
<link rel="icon" href="/favicon.svg">
|
|
<link rel="canonical" href="https://hanzoskills.com/hip-0047">
|
|
</head>
|
|
<!-- Reading this with a machine? Every page here is also plain markdown.
|
|
This one: https://hanzoskills.com/hip-0047.md
|
|
Index: https://hanzoskills.com/llms.txt
|
|
Corpus: https://hanzoskills.com/all.md
|
|
Append .md to any path for the markdown twin. -->
|
|
<body>
|
|
<nav><div class="in">
|
|
<a class="brand" href="/"><svg class="mark" viewBox="0 0 67 67" fill="currentColor" aria-hidden="true"><path d="M22.21 67V44.6369H0V67H22.21Z"/><path d="M66.7038 22.3184H22.2534L0.0878906 44.6367H44.4634L66.7038 22.3184Z"/><path d="M22.21 0H0V22.3184H22.21V0Z"/><path d="M66.7198 0H44.5098V22.3184H66.7198V0Z"/><path d="M66.7198 67V44.6369H44.5098V67H66.7198Z"/></svg> HANZO<i>SKILLS</i></a>
|
|
<a class="up" href="/">All skills</a>
|
|
</div></nav>
|
|
|
|
<div class="wrap">
|
|
<h1>hip-0047</h1><p class="lede">HIP-47: Analytics Datastore Standard. Status Active. Hanzo's own standard — read this before implementing against it.</p><h1>HIP-0047: Analytics Datastore Standard</h1>
|
|
<h2>Abstract</h2>
|
|
<p>This proposal defines the analytics datastore standard for the Hanzo ecosystem. Hanzo Datastore, our Apache-2.0 Hanzo Datastore fork, provides columnar analytics storage, deployed as a replicated cluster on each Kubernetes Kubernetes cluster. Every Hanzo service that requires high-throughput event ingestion, OLAP queries, or time-series aggregation MUST use the cluster-local Datastore instance following this specification.</p>
|
|
<p><strong>Repository</strong>: <a href="https://github.com/hanzoai/datastore">github.com/hanzoai/datastore</a> <strong>Image</strong>: <code>ghcr.io/hanzoai/datastore:latest</code> <strong>Ports</strong>: 8123 (HTTP), 9000 (native TCP), 9440 (native TLS) <strong>Engine</strong>: Hanzo Datastore 24.x+</p>
|
|
<h2>Motivation</h2>
|
|
<p>Hanzo generates massive volumes of event data across its services. Insights (HIP-0017) ingests 10M+ events per day from product analytics, session replay, feature flags, and A/B tests. Metrics (HIP-0031) ships structured logs to long-term storage. Billing analytics tracks token usage, credit consumption, and API call volumes across every organization.</p>
|
|
<p>All of this data shares a common characteristic: it is <strong>append-only</strong>, <strong>time- stamped</strong>, and <strong>queried in aggregate</strong>. These are the exact access patterns that row-oriented databases like SQL handle poorly and columnar databases handle exceptionally well.</p>
|
|
<p>Without a dedicated analytics datastore:</p>
|
|
<ol><li><strong>SQL becomes the bottleneck.</strong> Running `SELECT COUNT(*) FROM events</li></ol>
|
|
<p> WHERE timestamp > '2026-01-01' GROUP BY user_id` on a 500M-row SQL table takes 45-120 seconds. The same query on Hanzo Datastore completes in 200-800 milliseconds. SQL scans every column of every row; Hanzo Datastore reads only the columns referenced in the query.</p>
|
|
<ol><li><strong>Storage costs explode.</strong> A single analytics event averages 1-2 KB in</li></ol>
|
|
<p> SQL (row storage, TOAST overhead, indexes). Hanzo Datastore compresses the same event to 50-100 bytes using columnar compression. At 10M events/day, SQL consumes ~15 GB/day; Hanzo Datastore consumes ~700 MB/day. Over a year, that is 5.4 TB vs 250 GB.</p>
|
|
<ol><li><strong>SaaS analytics are prohibitively expensive.</strong> BigQuery charges $5/TB</li></ol>
|
|
<p> scanned. Snowflake charges per-second compute plus storage. At our query volume (thousands of dashboard loads per day, each scanning 10-100 GB), managed analytics would cost $5,000-20,000/month. Self-hosted Hanzo Datastore runs on existing Kubernetes infrastructure for near-zero marginal cost.</p>
|
|
<ol><li><strong>No unified query layer.</strong> Without a standard, Insights uses one database,</li></ol>
|
|
<p> Metrics uses another, and billing analytics lives in a third. Cross-cutting queries (correlate LLM token usage with product engagement) become multi-system joins that are slow and fragile.</p>
|
|
<p>We need ONE analytics datastore that all event-driven workloads share, with a single schema convention, replication strategy, backup policy, and query interface.</p>
|
|
<h2>Design Philosophy</h2>
|
|
<p>This section explains the reasoning behind each major architectural decision. Understanding the <em>why</em> is as important as understanding the <em>what</em>.</p>
|
|
<h3>Why Hanzo Datastore Over SQL for Analytics</h3>
|
|
<p>PostgreSQL is the Hanzo standard for transactional data (HIP-0029). It excels at OLTP: small reads and writes, row-level locking, ACID transactions, foreign keys, and joins across normalized tables. IAM stores users, organizations, and OAuth tokens in SQL. Cloud stores projects, API keys, and configuration.</p>
|
|
<p>Analytics data has the opposite access pattern:</p>
|
|
<p>| Property | OLTP (SQL) | OLAP (Hanzo Datastore) | |----------|-------------------|-------------------| | Write pattern | Single-row inserts/updates | Batch inserts (1000+ rows) | | Read pattern | Point lookups by primary key | Full-column scans with aggregation | | Schema | Normalized, many tables | Denormalized, few wide tables | | Transactions | Required (ACID) | Not needed (append-only) | | Compression | Row-level, 1-2x | Column-level, 10-100x | | Typical query | <code>SELECT <em> FROM users WHERE id = 42</code> | <code>SELECT COUNT(</em>) FROM events WHERE ts > '2026-01-01' GROUP BY browser</code> |</p>
|
|
<p>Hanzo Datastore stores data column-by-column on disk. When a query references 3 out of 50 columns, only those 3 columns are read from disk. SQL stores data row-by-row, so it must read all 50 columns even if only 3 are needed.</p>
|
|
<p>For a table with 1 billion rows and 50 columns, a query touching 3 columns:</p>
|
|
<ul><li><strong>SQL</strong>: Reads ~1 TB from disk (all columns, all rows)</li><li><strong>Hanzo Datastore</strong>: Reads ~6 GB from disk (3 columns, compressed)</li></ul>
|
|
<p>This is not a marginal difference. It is 100-200x less I/O, which translates directly into 100-200x faster queries.</p>
|
|
<p><strong>Decision</strong>: Use Hanzo Datastore for all analytics workloads. Keep SQL for transactional data per HIP-0029.</p>
|
|
<h3>Why Hanzo Datastore Over TimescaleDB</h3>
|
|
<p>TimescaleDB is SQL with time-series extensions. It is excellent for metrics (CPU usage, request latency, disk I/O) where each data point is a small fixed-schema tuple with a timestamp and a few numeric values.</p>
|
|
<p>Analytics events are different. Each event has 20-50 fields: timestamp, user ID, session ID, event type, URL, browser, OS, device, screen dimensions, referrer, UTM parameters, custom properties (JSON), and more. Events arrive at 100-1000x the rate of metrics. A busy dashboard page load generates 10-20 analytics events; the same page generates 1-2 metric data points.</p>
|
|
<p>TimescaleDB inherits SQL's row-oriented storage. For wide tables with billions of rows, it cannot match Hanzo Datastore's columnar performance:</p>
|
|
<p>| Workload | TimescaleDB | Hanzo Datastore | |----------|-------------|------------| | Ingestion (events/sec, single node) | 50,000-100,000 | 1,000,000-5,000,000 | | Query: COUNT by day (1B rows) | 8-15 seconds | 0.1-0.3 seconds | | Compression ratio (analytics events) | 3-5x | 10-20x | | Disk usage (1B events) | 200-400 GB | 50-100 GB |</p>
|
|
<p>TimescaleDB is the right choice for infrastructure metrics (Prometheus long-term storage, IoT sensor data). Hanzo Datastore is the right choice for analytics events.</p>
|
|
<p><strong>Decision</strong>: Use Hanzo Datastore for analytics events. Use Prometheus + Grafana for infrastructure metrics per HIP-0031. TimescaleDB is not needed.</p>
|
|
<h2>Specification</h2>
|
|
<h3>Architecture Overview</h3>
|
|
<pre><code> ┌─────────────────────────────────┐
|
|
│ Hanzo Services │
|
|
│ (Insights, Metrics, Billing) │
|
|
└────────┬────────────┬───────────┘
|
|
│ │
|
|
Batch HTTP Kafka Consumer
|
|
(port 8123) (HIP-0030)
|
|
│ │
|
|
┌────────▼────────────▼───────────┐
|
|
│ Datastore Cluster │
|
|
│ ┌──────────┐ ┌──────────┐ │
|
|
│ │ Shard 1 │ │ Shard 1 │ │
|
|
│ │ Replica A│ │ Replica B│ │
|
|
│ └──────────┘ └──────────┘ │
|
|
│ ┌──────────────────────┐ │
|
|
│ │ Datastore Keeper │ │
|
|
│ │ (coordination) │ │
|
|
│ └──────────────────────┘ │
|
|
└─────────────────────────────────┘
|
|
│
|
|
┌────────▼────────┐
|
|
│ MinIO (S3) │
|
|
│ (HIP-0032) │
|
|
│ Backups │
|
|
└─────────────────┘</code></pre>
|
|
<h3>Engine: MergeTree Family</h3>
|
|
<p>All tables MUST use a MergeTree-family engine. The MergeTree engine is the foundation of Datastore's performance. It provides:</p>
|
|
<ul><li><strong>Sorted storage</strong>: Data is stored sorted by the <code>ORDER BY</code> key, enabling</li></ul>
|
|
<p> efficient range queries and binary search.</p>
|
|
<ul><li><strong>Sparse indexing</strong>: Primary index stores one entry per 8192 rows (granule),</li></ul>
|
|
<p> not per row. This keeps the index small enough to fit in memory even for billion-row tables.</p>
|
|
<ul><li><strong>Background merges</strong>: Small data parts are merged into larger ones</li></ul>
|
|
<p> asynchronously, maintaining sort order and applying TTL/compression.</p>
|
|
<p>For production deployments with replication, all tables MUST use <code>ReplicatedMergeTree</code>:</p>
|
|
<pre><code class="lang-sql">CREATE TABLE events ON CLUSTER '{cluster}'
|
|
(
|
|
-- columns defined below
|
|
)
|
|
ENGINE = ReplicatedMergeTree('/datastore/tables/{shard}/events', '{replica}')
|
|
PARTITION BY toYYYYMM(timestamp)
|
|
ORDER BY (team_id, toDate(timestamp), event, cityHash64(distinct_id), uuid)
|
|
SETTINGS index_granularity = 8192;</code></pre>
|
|
<p>The ZooKeeper path template <code>{shard}</code> and <code>{replica}</code> are resolved from the Datastore server configuration. Datastore Keeper (a built-in Raft-based coordination service, inherited from the upstream 24.x engine) replaces ZooKeeper.</p>
|
|
<h3>Schema Design</h3>
|
|
<h4>Events Table</h4>
|
|
<p>The primary table storing all analytics events. This is the table that Insights (HIP-0017) writes to and queries from.</p>
|
|
<pre><code class="lang-sql">CREATE TABLE events ON CLUSTER '{cluster}'
|
|
(
|
|
-- Identity
|
|
uuid UUID DEFAULT generateUUIDv4(),
|
|
team_id UInt64,
|
|
distinct_id String,
|
|
session_id String DEFAULT '',
|
|
|
|
-- Event
|
|
event String,
|
|
timestamp DateTime64(3, 'UTC'),
|
|
created_at DateTime64(3, 'UTC') DEFAULT now64(3),
|
|
|
|
-- Properties (stored as JSON string, queried via JSONExtract)
|
|
properties String DEFAULT '{}',
|
|
|
|
-- Denormalized property columns for high-cardinality queries
|
|
-- These are extracted from properties at ingestion time
|
|
url String DEFAULT '',
|
|
referrer String DEFAULT '',
|
|
host String DEFAULT '',
|
|
pathname String DEFAULT '',
|
|
|
|
-- Device
|
|
browser LowCardinality(String) DEFAULT '',
|
|
browser_version String DEFAULT '',
|
|
os LowCardinality(String) DEFAULT '',
|
|
os_version String DEFAULT '',
|
|
device_type LowCardinality(String) DEFAULT '',
|
|
|
|
-- Geo (resolved at ingestion from IP)
|
|
country_code LowCardinality(FixedString(2)) DEFAULT '\0\0',
|
|
city LowCardinality(String) DEFAULT '',
|
|
region LowCardinality(String) DEFAULT '',
|
|
|
|
-- Attribution
|
|
utm_source LowCardinality(String) DEFAULT '',
|
|
utm_medium LowCardinality(String) DEFAULT '',
|
|
utm_campaign LowCardinality(String) DEFAULT '',
|
|
|
|
-- LLM-specific (Hanzo extension)
|
|
model LowCardinality(String) DEFAULT '',
|
|
provider LowCardinality(String) DEFAULT '',
|
|
tokens_prompt UInt32 DEFAULT 0,
|
|
tokens_completion UInt32 DEFAULT 0,
|
|
cost_usd Decimal64(8) DEFAULT 0,
|
|
|
|
-- Billing
|
|
organization_id String DEFAULT '',
|
|
project_id String DEFAULT '',
|
|
api_key_id String DEFAULT ''
|
|
)
|
|
ENGINE = ReplicatedMergeTree(
|
|
'/datastore/tables/{shard}/events',
|
|
'{replica}'
|
|
)
|
|
PARTITION BY toYYYYMM(timestamp)
|
|
ORDER BY (team_id, toDate(timestamp), event, cityHash64(distinct_id), uuid)
|
|
SETTINGS index_granularity = 8192;</code></pre>
|
|
<p><strong>Design decisions in this schema</strong>:</p>
|
|
<ul><li><strong><code>LowCardinality(String)</code></strong> for columns with few distinct values (browser,</li></ul>
|
|
<p> OS, country). Datastore stores these as dictionary-encoded integers, reducing storage 5-10x and speeding equality comparisons.</p>
|
|
<ul><li><strong><code>FixedString(2)</code></strong> for country codes. All ISO 3166-1 alpha-2 codes are</li></ul>
|
|
<p> exactly 2 bytes. Fixed-size storage avoids length prefix overhead.</p>
|
|
<ul><li><strong><code>properties</code> as String (JSON)</strong> rather than a Map or Nested type. This</li></ul>
|
|
<p> preserves schema flexibility -- any event can carry any custom properties. Queries use <code>JSONExtractString(properties, 'key')</code> for ad-hoc access. High-frequency property keys are denormalized into dedicated columns.</p>
|
|
<ul><li><strong>ORDER BY includes <code>cityHash64(distinct_id)</code></strong> to distribute rows with</li></ul>
|
|
<p> the same team/date/event across granules, improving parallelism for queries that filter by distinct_id.</p>
|
|
<ul><li><strong>PARTITION BY <code>toYYYYMM(timestamp)</code></strong> creates monthly partitions. This</li></ul>
|
|
<p> enables efficient partition pruning (queries with date ranges skip irrelevant months) and simplified TTL management.</p>
|
|
<h4>Sessions Table</h4>
|
|
<p>Materialized from the events table for session-level analytics.</p>
|
|
<pre><code class="lang-sql">CREATE MATERIALIZED VIEW sessions_mv ON CLUSTER '{cluster}'
|
|
TO sessions
|
|
AS SELECT
|
|
team_id,
|
|
session_id,
|
|
distinct_id,
|
|
min(timestamp) AS session_start,
|
|
max(timestamp) AS session_end,
|
|
dateDiff('second', min(timestamp),
|
|
max(timestamp)) AS duration_seconds,
|
|
count() AS event_count,
|
|
countIf(event = '$pageview') AS pageview_count,
|
|
argMin(url, timestamp) AS entry_url,
|
|
argMax(url, timestamp) AS exit_url,
|
|
argMin(referrer, timestamp) AS initial_referrer,
|
|
argMin(utm_source, timestamp) AS initial_utm_source,
|
|
argMin(utm_medium, timestamp) AS initial_utm_medium,
|
|
argMin(utm_campaign, timestamp) AS initial_utm_campaign,
|
|
argMin(browser, timestamp) AS browser,
|
|
argMin(os, timestamp) AS os,
|
|
argMin(device_type, timestamp) AS device_type,
|
|
argMin(country_code, timestamp) AS country_code
|
|
FROM events
|
|
GROUP BY team_id, session_id, distinct_id;</code></pre>
|
|
<p>The <code>sessions</code> target table uses <code>ReplicatedAggregatingMergeTree</code> so that late-arriving events for the same session are merged correctly.</p>
|
|
<h4>Metrics Rollup Tables</h4>
|
|
<p>For long-term metric storage (HIP-0031), pre-aggregated rollup tables reduce query time and storage for common dashboard queries.</p>
|
|
<pre><code class="lang-sql">-- Hourly event counts by team, event type
|
|
CREATE MATERIALIZED VIEW events_hourly_mv ON CLUSTER '{cluster}'
|
|
TO events_hourly
|
|
AS SELECT
|
|
team_id,
|
|
event,
|
|
toStartOfHour(timestamp) AS hour,
|
|
count() AS count,
|
|
uniqHLL12(distinct_id) AS unique_users
|
|
FROM events
|
|
GROUP BY team_id, event, hour;
|
|
|
|
-- Daily token usage by organization, model
|
|
CREATE MATERIALIZED VIEW token_usage_daily_mv ON CLUSTER '{cluster}'
|
|
TO token_usage_daily
|
|
AS SELECT
|
|
organization_id,
|
|
model,
|
|
provider,
|
|
toDate(timestamp) AS day,
|
|
sum(tokens_prompt) AS total_prompt_tokens,
|
|
sum(tokens_completion) AS total_completion_tokens,
|
|
sum(cost_usd) AS total_cost_usd,
|
|
count() AS request_count
|
|
FROM events
|
|
WHERE event = '$llm_request'
|
|
GROUP BY organization_id, model, provider, day;</code></pre>
|
|
<p><strong>Why materialized views</strong>: Datastore materialized views are triggers that run on INSERT. They are not periodic batch jobs. When a batch of events is inserted into the <code>events</code> table, Datastore automatically updates the materialized view target tables. This means rollup tables are always up-to-date with sub-second latency.</p>
|
|
<h3>Data Ingestion</h3>
|
|
<h4>Batch HTTP Inserts</h4>
|
|
<p>The primary ingestion path for Insights (HIP-0017). The Rust capture service buffers events in memory (or Kafka, per HIP-0030) and flushes to Datastore in batches via HTTP.</p>
|
|
<pre><code>POST http://datastore:8123/?query=INSERT+INTO+events+FORMAT+JSONEachRow
|
|
Content-Type: application/json
|
|
|
|
{"uuid":"...","team_id":1,"event":"$pageview","timestamp":"2026-02-23T12:00:00.000Z","distinct_id":"user_42","properties":"{\"url\":\"/dashboard\"}"}
|
|
{"uuid":"...","team_id":1,"event":"$pageview","timestamp":"2026-02-23T12:00:01.000Z","distinct_id":"user_43","properties":"{\"url\":\"/settings\"}"}</code></pre>
|
|
<p><strong>Batch size requirements</strong>:</p>
|
|
<p>| Parameter | Value | Rationale | |-----------|-------|-----------| | Minimum batch size | 1,000 rows | Below this, per-INSERT overhead dominates | | Recommended batch size | 10,000-100,000 rows | Optimal merge behavior | | Maximum batch size | 1,000,000 rows | Beyond this, INSERT timeout risk increases | | Flush interval | 5-10 seconds | Balance latency vs batch size |</p>
|
|
<p>Services MUST NOT insert single rows per HTTP request. Single-row inserts create thousands of tiny data parts that Datastore must merge, consuming CPU and degrading query performance. The <code>too many parts</code> exception is a hard failure mode caused by excessive single-row inserts.</p>
|
|
<h4>Kafka Consumer (Streaming)</h4>
|
|
<p>For high-throughput services that cannot batch in-process, Datastore's built-in Kafka engine consumes directly from Kafka topics (HIP-0030).</p>
|
|
<pre><code class="lang-sql">CREATE TABLE events_kafka ON CLUSTER '{cluster}'
|
|
(
|
|
uuid String,
|
|
team_id UInt64,
|
|
event String,
|
|
timestamp String,
|
|
distinct_id String,
|
|
properties String
|
|
)
|
|
ENGINE = Kafka
|
|
SETTINGS
|
|
kafka_broker_list = 'kafka:9092',
|
|
kafka_topic_list = 'hanzo_events',
|
|
kafka_group_name = 'datastore_events_consumer',
|
|
kafka_format = 'JSONEachRow',
|
|
kafka_num_consumers = 4,
|
|
kafka_max_block_size = 65536;
|
|
|
|
CREATE MATERIALIZED VIEW events_kafka_mv ON CLUSTER '{cluster}'
|
|
TO events
|
|
AS SELECT
|
|
toUUID(uuid) AS uuid,
|
|
team_id,
|
|
event,
|
|
parseDateTimeBestEffort(timestamp) AS timestamp,
|
|
now64(3) AS created_at,
|
|
distinct_id,
|
|
properties
|
|
FROM events_kafka;</code></pre>
|
|
<p>This pattern decouples ingestion from query serving. Kafka absorbs traffic spikes; Datastore consumes at its own pace.</p>
|
|
<h3>Query Interface</h3>
|
|
<p>Datastore uses SQL with extensions. All Hanzo services MUST query Datastore using standard SQL via the HTTP interface or the native TCP protocol.</p>
|
|
<h4>HTTP Interface (Port 8123)</h4>
|
|
<pre><code class="lang-bash"># Simple query
|
|
curl 'http://datastore:8123/?query=SELECT+count()+FROM+events+WHERE+team_id=1'
|
|
|
|
# Parameterized query (prevents SQL injection)
|
|
curl 'http://datastore:8123/' \
|
|
--data-urlencode "param_team_id=1" \
|
|
--data-urlencode "param_start=2026-02-01" \
|
|
-d "query=SELECT count() FROM events WHERE team_id = {team_id:UInt64} AND timestamp >= {start:Date}"</code></pre>
|
|
<h4>Common Query Patterns</h4>
|
|
<p><strong>Funnel analysis</strong> (Insights):</p>
|
|
<pre><code class="lang-sql">SELECT
|
|
level,
|
|
count() AS users
|
|
FROM
|
|
(
|
|
SELECT
|
|
distinct_id,
|
|
windowFunnel(86400)(
|
|
timestamp,
|
|
event = 'signup',
|
|
event = 'onboarding_complete',
|
|
event = 'first_api_call'
|
|
) AS level
|
|
FROM events
|
|
WHERE team_id = 1
|
|
AND timestamp >= '2026-02-01'
|
|
AND timestamp < '2026-03-01'
|
|
GROUP BY distinct_id
|
|
)
|
|
GROUP BY level
|
|
ORDER BY level DESC;</code></pre>
|
|
<p><strong>Retention analysis</strong> (Insights):</p>
|
|
<pre><code class="lang-sql">SELECT
|
|
cohort_date,
|
|
period,
|
|
count(DISTINCT distinct_id) AS retained_users
|
|
FROM
|
|
(
|
|
SELECT
|
|
distinct_id,
|
|
toDate(min(timestamp)) AS cohort_date,
|
|
dateDiff('week', toDate(min(timestamp)), toDate(timestamp)) AS period
|
|
FROM events
|
|
WHERE team_id = 1
|
|
AND event IN ('$pageview', '$screen')
|
|
AND timestamp >= '2026-01-01'
|
|
GROUP BY distinct_id, toDate(timestamp)
|
|
)
|
|
GROUP BY cohort_date, period
|
|
ORDER BY cohort_date, period;</code></pre>
|
|
<p><strong>Token usage dashboard</strong> (Billing):</p>
|
|
<pre><code class="lang-sql">SELECT
|
|
organization_id,
|
|
model,
|
|
sum(total_prompt_tokens) AS prompt_tokens,
|
|
sum(total_completion_tokens) AS completion_tokens,
|
|
sum(total_cost_usd) AS total_cost
|
|
FROM token_usage_daily
|
|
WHERE day >= today() - 30
|
|
GROUP BY organization_id, model
|
|
ORDER BY total_cost DESC
|
|
LIMIT 100;</code></pre>
|
|
<h3>Partitioning Strategy</h3>
|
|
<p>All tables MUST use time-based partitioning:</p>
|
|
<p>| Table | Partition Key | Granularity | Rationale | |-------|---------------|-------------|-----------| | events | <code>toYYYYMM(timestamp)</code> | Monthly | Balances partition count vs pruning efficiency | | sessions | <code>toYYYYMM(session_start)</code> | Monthly | Aligned with events | | events_hourly | <code>toYYYYMM(hour)</code> | Monthly | Rollups are small per partition | | token_usage_daily | <code>toYYYYMM(day)</code> | Monthly | Consistent with events |</p>
|
|
<p><strong>Why monthly, not daily</strong>: Daily partitions create 365 partitions per year. With 2 replicas, that is 730 data parts to track per table. After 3 years of data, the partition count becomes unwieldy. Monthly partitions keep the count manageable (36 per year) while still enabling efficient date-range pruning.</p>
|
|
<h4>TTL (Time-to-Live)</h4>
|
|
<p>Raw event data has a default retention of 12 months. Rollup tables retain data for 36 months. TTL is enforced at the table level:</p>
|
|
<pre><code class="lang-sql">ALTER TABLE events ON CLUSTER '{cluster}'
|
|
MODIFY TTL timestamp + INTERVAL 12 MONTH DELETE;
|
|
|
|
ALTER TABLE events_hourly ON CLUSTER '{cluster}'
|
|
MODIFY TTL hour + INTERVAL 36 MONTH DELETE;
|
|
|
|
ALTER TABLE token_usage_daily ON CLUSTER '{cluster}'
|
|
MODIFY TTL day + INTERVAL 36 MONTH DELETE;</code></pre>
|
|
<p>TTL is evaluated during background merges. Expired data is dropped during merge, not immediately upon expiration. This is a lazy deletion model that avoids write amplification.</p>
|
|
<p>Organizations on enterprise plans MAY override TTL via the <code>team_settings</code> table in SQL. The ingestion service respects per-team TTL overrides when creating partitions.</p>
|
|
<h3>Compression</h3>
|
|
<p>Datastore supports per-column codec selection. The default codec is LZ4, which provides fast compression/decompression with moderate ratios. For columns that benefit from higher compression, ZSTD is used.</p>
|
|
<pre><code class="lang-sql">CREATE TABLE events
|
|
(
|
|
uuid UUID CODEC(LZ4),
|
|
team_id UInt64 CODEC(Delta, LZ4),
|
|
distinct_id String CODEC(ZSTD(3)),
|
|
session_id String CODEC(ZSTD(3)),
|
|
event String CODEC(ZSTD(1)),
|
|
timestamp DateTime64(3) CODEC(DoubleDelta, LZ4),
|
|
properties String CODEC(ZSTD(3)),
|
|
url String CODEC(ZSTD(3)),
|
|
browser LowCardinality(String) CODEC(ZSTD(1)),
|
|
os LowCardinality(String) CODEC(ZSTD(1)),
|
|
country_code LowCardinality(FixedString(2)) CODEC(LZ4),
|
|
tokens_prompt UInt32 CODEC(Delta, LZ4),
|
|
tokens_completion UInt32 CODEC(Delta, LZ4),
|
|
cost_usd Decimal64(8) CODEC(Delta, LZ4)
|
|
-- ... remaining columns
|
|
);</code></pre>
|
|
<p><strong>Codec selection rationale</strong>:</p>
|
|
<p>| Codec | Used For | Why | |-------|----------|-----| | <code>LZ4</code> | UUIDs, fixed-width types | Fast decompression, acceptable ratio | | <code>ZSTD(1)</code> | Low-cardinality strings | Better ratio than LZ4, still fast | | <code>ZSTD(3)</code> | High-cardinality strings (URLs, properties) | 2-3x better ratio than LZ4 | | <code>Delta + LZ4</code> | Monotonic integers (team_id, timestamps) | Delta encodes small differences; LZ4 compresses the deltas | | <code>DoubleDelta + LZ4</code> | Timestamps | Timestamps increment by near-constant intervals; double-delta reduces to near-zero |</p>
|
|
<p>Expected compression ratios on production data:</p>
|
|
<p>| Column | Uncompressed | Compressed | Ratio | |--------|-------------|------------|-------| | timestamp | 8 bytes/row | 0.3 bytes/row | 27x | | team_id | 8 bytes/row | 0.1 bytes/row | 80x | | properties (JSON) | 500 bytes/row avg | 40 bytes/row avg | 12x | | url | 80 bytes/row avg | 8 bytes/row avg | 10x | | <strong>Overall</strong> | <strong>~1200 bytes/row</strong> | <strong>~80 bytes/row</strong> | <strong>~15x</strong> |</p>
|
|
<h3>Replication</h3>
|
|
<p>Production deployments MUST use a 2-replica configuration. Hanzo Datastore replication is asynchronous and log-based:</p>
|
|
<ol><li>When a write arrives at Replica A, it is written to the local MergeTree</li></ol>
|
|
<p> and a log entry is created in Hanzo Datastore Keeper.</p>
|
|
<ol><li>Replica B polls the log and fetches the new data part from Replica A.</li><li>Both replicas are eventually consistent. Replication lag is typically</li></ol>
|
|
<p> under 1 second.</p>
|
|
<h4>Cluster Configuration</h4>
|
|
<pre><code class="lang-xml"><!-- /etc/hanzo-datastore-server/config.d/cluster.xml -->
|
|
<datastore>
|
|
<remote_servers>
|
|
<hanzo_cluster>
|
|
<shard>
|
|
<internal_replication>true</internal_replication>
|
|
<replica>
|
|
<host>datastore-0</host>
|
|
<port>9000</port>
|
|
</replica>
|
|
<replica>
|
|
<host>datastore-1</host>
|
|
<port>9000</port>
|
|
</replica>
|
|
</shard>
|
|
</hanzo_cluster>
|
|
</remote_servers>
|
|
|
|
<macros>
|
|
<cluster>hanzo_cluster</cluster>
|
|
<shard>01</shard>
|
|
<replica>datastore-{replica_number}</replica>
|
|
</macros>
|
|
</datastore></code></pre>
|
|
<h4>Hanzo Datastore Keeper Configuration</h4>
|
|
<p>Since Hanzo Datastore 24.x, the built-in Hanzo Datastore Keeper replaces ZooKeeper. A 3-node Keeper ensemble provides coordination for replication:</p>
|
|
<pre><code class="lang-xml"><!-- /etc/hanzo-datastore-server/config.d/keeper.xml -->
|
|
<datastore>
|
|
<keeper_server>
|
|
<tcp_port>9181</tcp_port>
|
|
<server_id>1</server_id>
|
|
<raft_configuration>
|
|
<server>
|
|
<id>1</id>
|
|
<hostname>keeper-0</hostname>
|
|
<port>9234</port>
|
|
</server>
|
|
<server>
|
|
<id>2</id>
|
|
<hostname>keeper-1</hostname>
|
|
<port>9234</port>
|
|
</server>
|
|
<server>
|
|
<id>3</id>
|
|
<hostname>keeper-2</hostname>
|
|
<port>9234</port>
|
|
</server>
|
|
</raft_configuration>
|
|
</keeper_server>
|
|
</datastore></code></pre>
|
|
<p><strong>Why 3 Keeper nodes, not 2</strong>: Raft consensus requires a majority quorum. With 2 nodes, losing 1 node means no quorum and the cluster cannot coordinate writes. With 3 nodes, the cluster tolerates 1 node failure.</p>
|
|
<h3>Backup and Recovery</h3>
|
|
<p>Backups follow the Hanzo backup standard using S3-compatible storage (MinIO, per HIP-0032).</p>
|
|
<h4>Backup Strategy</h4>
|
|
<pre><code class="lang-bash"># Full backup to S3
|
|
clickhouse-backup create --tables 'default.events,default.sessions' daily_2026_02_23
|
|
clickhouse-backup upload daily_2026_02_23
|
|
|
|
# Incremental backup (only changed parts since last full)
|
|
clickhouse-backup create_remote --diff-from-remote daily_2026_02_23 incremental_2026_02_24</code></pre>
|
|
<p>| Backup Type | Frequency | Retention | Storage | |-------------|-----------|-----------|---------| | Full | Weekly (Sunday 02:00 UTC) | 4 weeks | MinIO (HIP-0032) | | Incremental | Daily (02:00 UTC) | 7 days | MinIO (HIP-0032) | | Partition-level | On TTL drop | 30 days | MinIO (HIP-0032) |</p>
|
|
<h4>Backup Configuration</h4>
|
|
<pre><code class="lang-yaml"># /etc/clickhouse-backup/config.yml
|
|
general:
|
|
remote_storage: s3
|
|
backups_to_keep_local: 2
|
|
backups_to_keep_remote: 28
|
|
|
|
s3:
|
|
access_key: "${MINIO_ACCESS_KEY}"
|
|
secret_key: "${MINIO_SECRET_KEY}"
|
|
bucket: "clickhouse-backups"
|
|
endpoint: "http://minio:9000"
|
|
region: "us-east-1"
|
|
path: "the cluster/"
|
|
compression_format: "zstd"</code></pre>
|
|
<h4>Recovery Procedure</h4>
|
|
<pre><code class="lang-bash"># List available backups
|
|
clickhouse-backup list remote
|
|
|
|
# Download and restore
|
|
clickhouse-backup download daily_2026_02_23
|
|
clickhouse-backup restore daily_2026_02_23
|
|
|
|
# Verify row counts
|
|
hanzo-datastore-client -q "SELECT count() FROM events"</code></pre>
|
|
<p>Recovery time objective (RTO): < 1 hour for full cluster restore. Recovery point objective (RPO): < 24 hours (last daily backup).</p>
|
|
<h3>Monitoring</h3>
|
|
<p>Hanzo Datastore exposes internal metrics through system tables. Hanzo Zap (HIP-0031) scrapes these and exports them as Prometheus metrics.</p>
|
|
<h4>Key System Tables</h4>
|
|
<p>| Table | Purpose | Query Frequency | |-------|---------|-----------------| | <code>system.metrics</code> | Current server state (connections, queries, merges) | Every 15s | | <code>system.events</code> | Cumulative counters (inserts, selects, bytes read) | Every 15s | | <code>system.asynchronous_metrics</code> | Background metrics (memory, disk, replication lag) | Every 60s | | <code>system.query_log</code> | Query history with timing and resource usage | On demand | | <code>system.parts</code> | Data part inventory per table | Every 60s | | <code>system.replicas</code> | Replication status and lag | Every 30s |</p>
|
|
<h4>Critical Alerts</h4>
|
|
<p>| Alert | Condition | Severity | |-------|-----------|----------| | Too Many Parts | <code>SELECT count() FROM system.parts WHERE active AND table = 'events'</code> > 300 | Critical | | Replication Lag | <code>SELECT replica_delay FROM system.replicas WHERE table = 'events'</code> > 300 seconds | Warning | | Disk Usage | <code>SELECT free_space FROM system.disks WHERE name = 'default'</code> < 10 GB | Critical | | Query Duration | p99 query time > 30 seconds | Warning | | Insert Failures | <code>system.events</code> counter <code>FailedInsertQuery</code> increasing | Critical | | Keeper Disconnect | <code>SELECT * FROM system.zookeeper WHERE path = '/datastore'</code> fails | Critical |</p>
|
|
<h4>Grafana Dashboard</h4>
|
|
<p>The standard Grafana dashboard for Datastore includes panels for:</p>
|
|
<ol><li><strong>Ingestion rate</strong>: Rows inserted per second (from <code>system.events</code>)</li><li><strong>Query performance</strong>: p50/p95/p99 query latency (from <code>system.query_log</code>)</li><li><strong>Active parts</strong>: Part count per table (from <code>system.parts</code>)</li><li><strong>Replication lag</strong>: Seconds behind leader (from <code>system.replicas</code>)</li><li><strong>Disk usage</strong>: Used/free per disk (from <code>system.disks</code>)</li><li><strong>Memory usage</strong>: Resident memory and cache hit rates</li><li><strong>Merge throughput</strong>: Background merge rate and duration</li><li><strong>Compression ratio</strong>: Raw vs compressed bytes per table</li></ol>
|
|
<h3>Deployment</h3>
|
|
<h4>Docker (Development)</h4>
|
|
<pre><code class="lang-yaml"># compose.yml
|
|
services:
|
|
datastore:
|
|
image: ghcr.io/hanzoai/datastore:latest
|
|
ports:
|
|
- "8123:8123" # HTTP
|
|
- "9000:9000" # Native TCP
|
|
volumes:
|
|
- datastore_data:/var/lib/hanzo-datastore
|
|
- datastore_logs:/var/log/hanzo-datastore-server
|
|
environment:
|
|
DATASTORE_DB: hanzo
|
|
DATASTORE_USER: hanzo
|
|
DATASTORE_PASSWORD: "${DATASTORE_PASSWORD}"
|
|
ulimits:
|
|
nofile:
|
|
soft: 262144
|
|
hard: 262144
|
|
healthcheck:
|
|
test: ["CMD", "hanzo-datastore-client", "--query", "SELECT 1"]
|
|
interval: 10s
|
|
timeout: 5s
|
|
retries: 3
|
|
|
|
volumes:
|
|
datastore_data:
|
|
datastore_logs:</code></pre>
|
|
<h4>Kubernetes (Production)</h4>
|
|
<p>Production deployments use the Hanzo Datastore Operator for lifecycle management.</p>
|
|
<pre><code class="lang-yaml"># k8s/datastore.yaml
|
|
apiVersion: hanzo.ai/v1
|
|
kind: Datastore
|
|
metadata:
|
|
name: hanzo-datastore
|
|
namespace: hanzo
|
|
spec:
|
|
configuration:
|
|
clusters:
|
|
- name: hanzo_cluster
|
|
layout:
|
|
shardsCount: 1
|
|
replicasCount: 2
|
|
templates:
|
|
podTemplate: datastore-pod
|
|
volumeClaimTemplate: data-volume
|
|
|
|
settings:
|
|
max_concurrent_queries: 200
|
|
max_memory_usage: 10000000000 # 10 GB per query
|
|
max_bytes_before_external_group_by: 5000000000 # 5 GB
|
|
max_execution_time: 300 # 5 minutes
|
|
log_queries: 1
|
|
merge_tree/max_parts_in_total: 100000
|
|
|
|
profiles:
|
|
default/max_memory_usage: 10000000000
|
|
default/max_execution_time: 300
|
|
readonly/readonly: 1
|
|
readonly/max_execution_time: 60
|
|
|
|
users:
|
|
hanzo/password_sha256_hex: "${DATASTORE_PASSWORD_SHA256}"
|
|
hanzo/networks/ip: "127.0.0.1/8"
|
|
hanzo/profile: default
|
|
hanzo/quota: default
|
|
readonly/password_sha256_hex: "${DATASTORE_READONLY_PASSWORD_SHA256}"
|
|
readonly/networks/ip: "127.0.0.1/8"
|
|
readonly/profile: readonly
|
|
|
|
templates:
|
|
podTemplates:
|
|
- name: datastore-pod
|
|
spec:
|
|
containers:
|
|
- name: datastore
|
|
image: ghcr.io/hanzoai/datastore:latest
|
|
resources:
|
|
requests:
|
|
memory: 8Gi
|
|
cpu: "4"
|
|
limits:
|
|
memory: 16Gi
|
|
cpu: "8"
|
|
volumeClaimTemplates:
|
|
- name: data-volume
|
|
spec:
|
|
accessModes: [ReadWriteOnce]
|
|
resources:
|
|
requests:
|
|
storage: 500Gi
|
|
storageClassName: do-block-storage</code></pre>
|
|
<h4>Resource Sizing Guide</h4>
|
|
<p>| Cluster Size | Events/Day | CPU | RAM | Disk | Replicas | |-------------|------------|-----|-----|------|----------| | Small (dev) | < 1M | 2 cores | 4 GB | 50 GB | 1 | | Medium | 1M - 50M | 4-8 cores | 8-16 GB | 200-500 GB | 2 | | Large | 50M - 500M | 16-32 cores | 32-64 GB | 1-2 TB | 2 | | XL | 500M+ | 64+ cores | 128+ GB | 4+ TB | 2 (multi-shard) |</p>
|
|
<h3>Security</h3>
|
|
<h4>Authentication</h4>
|
|
<p>Hanzo Datastore MUST require authentication for all connections. Two user profiles are provisioned:</p>
|
|
<p>| User | Purpose | Permissions | |------|---------|-------------| | <code>hanzo</code> | Application writes and reads | Full DDL + DML on <code>hanzo</code> database | | <code>readonly</code> | Dashboard queries, Grafana | SELECT only, 60s query timeout |</p>
|
|
<p>Passwords are stored as SHA-256 hashes in the Hanzo Datastore users configuration. Credentials are managed through KMS (HIP-0033) and injected as Kubernetes secrets.</p>
|
|
<h4>Network</h4>
|
|
<ul><li>HTTP port (8123) and native port (9000) MUST be accessible only from within</li></ul>
|
|
<p> the Kubernetes cluster network (<code>127.0.0.1/8</code>).</p>
|
|
<ul><li>Native TLS port (9440) is used for cross-cluster replication if needed.</li><li>No ports are exposed to the public internet.</li><li>Kubernetes NetworkPolicy restricts access to pods with the label</li></ul>
|
|
<p> <code>hanzo-datastore-client: "true"</code>.</p>
|
|
<h4>Data Classification</h4>
|
|
<p>| Data Type | Sensitivity | Handling | |-----------|-------------|----------| | Event metadata (timestamps, counts) | Low | Standard retention | | User identifiers (distinct_id) | Medium | Hashed or pseudonymized | | URLs and referrers | Medium | Retained per TTL, no PII | | Properties (custom JSON) | Variable | May contain PII; TTL enforced | | LLM request content | High | NOT stored in analytics; only token counts and costs |</p>
|
|
<p>Services MUST NOT store raw LLM prompts or completions in the analytics datastore. Only aggregate metadata (model, token counts, latency, cost) is permitted.</p>
|
|
<h3>Integration Points</h3>
|
|
<h4>Insights (HIP-0017)</h4>
|
|
<p>Insights is the primary consumer of the analytics datastore. The integration flow is:</p>
|
|
<pre><code>SDK (browser/server) --> Capture Service (Rust) --> Kafka --> Hanzo Datastore
|
|
--> SQL (metadata only)</code></pre>
|
|
<p>Insights queries Datastore for:</p>
|
|
<ul><li>Event trends and breakdowns</li><li>Funnel analysis (multi-step conversion)</li><li>Retention cohorts</li><li>Session replay event sequences</li><li>Feature flag exposure tracking</li><li>A/B test statistical analysis</li></ul>
|
|
<h4>Metrics (HIP-0031)</h4>
|
|
<p>Zap (HIP-0031) ships structured logs to Datastore for long-term storage. Prometheus handles short-term metrics (15-day retention). For queries spanning weeks or months, Grafana queries Datastore via the Hanzo Datastore data source plugin.</p>
|
|
<pre><code>Zap sidecar --> Prometheus (15-day) --> remote_write --> Datastore (36-month)</code></pre>
|
|
<h4>Billing Analytics</h4>
|
|
<p>The billing pipeline tracks per-organization resource consumption:</p>
|
|
<pre><code class="lang-sql">-- Monthly invoice line items
|
|
SELECT
|
|
organization_id,
|
|
model,
|
|
sum(tokens_prompt + tokens_completion) AS total_tokens,
|
|
sum(cost_usd) AS total_cost,
|
|
count() AS api_calls
|
|
FROM events
|
|
WHERE event = '$llm_request'
|
|
AND organization_id = 'org_123'
|
|
AND timestamp >= '2026-02-01'
|
|
AND timestamp < '2026-03-01'
|
|
GROUP BY organization_id, model
|
|
ORDER BY total_cost DESC;</code></pre>
|
|
<p>This query runs against the <code>token_usage_daily</code> materialized view in production for sub-second response times.</p>
|
|
<h2>Reference Implementation</h2>
|
|
<p>The reference implementation is at <a href="https://github.com/hanzoai/datastore">github.com/hanzoai/datastore</a> and includes:</p>
|
|
<ul><li><code>Dockerfile</code>: Hanzo Datastore image with Hanzo-specific configuration</li><li><code>schemas/</code>: DDL for all tables and materialized views</li><li><code>config/</code>: Hanzo Datastore server configuration templates</li><li><code>k8s/</code>: Kubernetes manifests and Hanzo Datastore Operator CRD</li><li><code>backup/</code>: Backup scripts and CronJob manifests</li><li><code>grafana/</code>: Dashboard JSON exports</li><li><code>tests/</code>: Integration tests using <code>hanzo-datastore-client</code></li></ul>
|
|
<h2>Copyright</h2>
|
|
<p>This document is placed in the public domain.</p>
|
|
<footer>
|
|
<div class="fmap">
|
|
<div><b>Build</b>
|
|
<a href="https://hanzo.app">hanzo.app</a> app builder
|
|
<a href="https://hanzo.codes">hanzo.codes</a> code with an agent
|
|
<a href="https://hanzoskills.com">hanzoskills.com</a> what an agent reads</div>
|
|
<div><b>Run</b>
|
|
<a href="https://console.hanzo.ai">console.hanzo.ai</a> cloud console
|
|
<a href="https://api.hanzo.ai">api.hanzo.ai</a> the API</div>
|
|
<div><b>Account</b>
|
|
<a href="https://hanzo.id">hanzo.id</a> sign in
|
|
<a href="https://pay.hanzo.ai">pay.hanzo.ai</a> top up
|
|
<a href="https://billing.hanzo.ai">billing.hanzo.ai</a> invoices and usage</div>
|
|
<div><b>Company</b>
|
|
<a href="https://hanzo.ai">hanzo.ai</a> canonical
|
|
<a href="https://hanzo.ventures">hanzo.ventures</a> ventures</div>
|
|
</div>
|
|
<div class="fend"><a href="/hip-0047.md">This page as plain text</a> · <a href="/llms.txt">Index for agents</a></div>
|
|
</footer>
|
|
</div>
|
|
</body></html> |