mirror of
https://github.com/hanzoai/skills.git
synced 2026-11-19 00:47:36 +00:00
896 lines
50 KiB
HTML
896 lines
50 KiB
HTML
<!doctype html>
|
|
<html lang="en"><head>
|
|
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
<title>hip-0017</title>
|
|
<meta name="color-scheme" content="dark">
|
|
<style>
|
|
/* Committed to one visual world: a dark instrument panel. No light variant —
|
|
the dot field and the hairline rules only read on black, and a naive
|
|
inversion would wash both out. Every colour is painted explicitly so the page
|
|
never borrows a host background. */
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Pixel Square';src:url('/fonts/ZenPixel-Square.woff2') format('woff2');font-style:normal;font-display:swap}
|
|
|
|
:root{
|
|
--ink:#f4f4f2; --soft:#9c9c98; --faint:#6a6a66; --ground:#000; --panel:#0a0a0a;
|
|
--line:#232323; --line-hi:#3a3a3a; --accent:#f4f4f2;
|
|
--font-zen-sans:'Zen',-apple-system,BlinkMacSystemFont,"Segoe UI",Helvetica,Arial,sans-serif;
|
|
--font-zen-mono:'Zen Mono',ui-monospace,"SF Mono",SFMono-Regular,Menlo,Consolas,monospace;
|
|
--font-zen-pixel-square:'Zen Pixel Square',monospace;
|
|
--mono:var(--font-zen-mono);
|
|
--sans:var(--font-zen-sans);
|
|
/* Controls are pills and surfaces are soft, as on hanzo.ai. */
|
|
--r:999px; --r-soft:14px;
|
|
}
|
|
*{box-sizing:border-box}
|
|
html{-webkit-text-size-adjust:100%}
|
|
body{margin:0;background:var(--ground);color:var(--ink);font-family:var(--sans);
|
|
font-size:16px;line-height:1.65;-webkit-font-smoothing:antialiased}
|
|
.wrap{max-width:60rem;margin:0 auto;padding:0 1.75rem}
|
|
|
|
/* The dot field. A halftone made of one repeating radial-gradient, faded out
|
|
along the diagonal by a mask so it reads as a wave rather than wallpaper. */
|
|
/* No overflow clip here: the dot field is inset to the box already, and a clip
|
|
would cut off the copy-prompt menu. */
|
|
.hero{position:relative;border-bottom:1px solid var(--line)}
|
|
.hero::before{content:"";position:absolute;inset:0;
|
|
background-image:radial-gradient(var(--line-hi) 1px, transparent 1px);
|
|
background-size:7px 7px;
|
|
-webkit-mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
pointer-events:none}
|
|
.hero .wrap{position:relative;padding-top:5.5rem;padding-bottom:4rem}
|
|
|
|
nav{border-bottom:1px solid var(--line)}
|
|
nav .in{max-width:60rem;margin:0 auto;padding:1.1rem 1.75rem;display:flex;
|
|
justify-content:space-between;align-items:center;gap:1rem}
|
|
.brand{font-family:var(--mono);font-size:.85rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--ink);text-decoration:none;
|
|
display:inline-flex;align-items:center;gap:.6rem}
|
|
.brand i{font-style:normal;color:var(--faint)}
|
|
.mark{width:1.15em;height:1.15em;display:block;flex:0 0 auto}
|
|
nav a.up{font-family:var(--mono);font-size:.72rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--soft);text-decoration:none}
|
|
nav a.up:hover{color:var(--ink)}
|
|
|
|
h1{font-family:var(--mono);font-size:clamp(2rem,5.4vw,3.4rem);line-height:1.12;
|
|
letter-spacing:-.02em;margin:0 0 1.1rem;font-weight:500;text-wrap:balance}
|
|
h1 em{font-style:normal;color:var(--faint)}
|
|
.lede{color:var(--soft);font-size:1.05rem;max-width:52ch;margin:0 0 2.25rem}
|
|
|
|
.btns{display:flex;flex-wrap:wrap;gap:.7rem;margin:0 0 .5rem;align-items:center}
|
|
.btn{display:inline-flex;align-items:center;gap:.5rem;font-family:var(--mono);
|
|
font-size:.72rem;letter-spacing:.14em;text-transform:uppercase;
|
|
padding:.8rem 1.4rem;border:1px solid var(--line-hi);border-radius:var(--r);
|
|
color:var(--ink);text-decoration:none;background:transparent;cursor:pointer;
|
|
line-height:1.2;font-weight:400}
|
|
.btn:hover{background:#141414;text-decoration:none;border-color:var(--soft)}
|
|
.btn.solid{background:var(--ink);color:#000;border-color:var(--ink)}
|
|
.btn.solid:hover{background:#d8d8d4}
|
|
.btn .chev{width:.7em;height:.7em;flex:0 0 auto;transition:transform .15s ease}
|
|
.btn[aria-expanded="true"] .chev{transform:rotate(180deg)}
|
|
|
|
/* Copy-prompt selector. Each row copies the prompt for its own target. */
|
|
.copy{position:relative;display:inline-flex}
|
|
.menu{position:absolute;top:calc(100% + .5rem);left:0;z-index:20;min-width:21rem;
|
|
background:var(--panel);border:1px solid var(--line-hi);border-radius:var(--r-soft);
|
|
padding:.4rem;display:flex;flex-direction:column;
|
|
box-shadow:0 12px 32px rgba(0,0,0,.75)}
|
|
.menu[hidden]{display:none}
|
|
.mi{display:flex;align-items:center;gap:.7rem;width:100%;text-align:left;
|
|
padding:.62rem .8rem;border:0;border-radius:calc(var(--r-soft) - .35rem);
|
|
background:transparent;color:var(--ink);font-family:var(--mono);font-size:.76rem;
|
|
letter-spacing:.04em;text-decoration:none;cursor:pointer;line-height:1.3;
|
|
white-space:nowrap}
|
|
.mi:hover,.mi:focus-visible{background:#181818;outline:none}
|
|
.mi .mark{width:1.05rem;height:1.05rem;color:var(--soft)}
|
|
.mi:hover .mark{color:var(--ink)}
|
|
.mi u{margin-left:auto;color:var(--faint);text-decoration:none;font-size:.68rem;
|
|
letter-spacing:.1em;text-transform:uppercase}
|
|
|
|
/* Works-with bar: one scrolling row, name and mark per agent. */
|
|
.strip{border-bottom:1px solid var(--line);background:var(--panel)}
|
|
.strip .in{max-width:60rem;margin:0 auto;padding:.9rem 1.75rem;display:flex;
|
|
align-items:center;gap:1.25rem;font-family:var(--mono);font-size:.72rem;
|
|
letter-spacing:.1em;text-transform:uppercase;color:var(--faint)}
|
|
.strip b{color:var(--soft);font-weight:400;flex:0 0 auto;
|
|
border-right:1px solid var(--line);padding-right:1.25rem;align-self:stretch;
|
|
display:flex;align-items:center}
|
|
.bar{display:flex;align-items:center;gap:.5rem;flex:1 1 auto;min-width:0;
|
|
overflow-x:auto;scrollbar-width:none;scroll-snap-type:x proximity;
|
|
-webkit-overflow-scrolling:touch;
|
|
-webkit-mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent);
|
|
mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent)}
|
|
.bar::-webkit-scrollbar{display:none}
|
|
.agent{display:inline-flex;align-items:center;gap:.45rem;flex:0 0 auto;
|
|
padding:.4rem .85rem;border:1px solid var(--line);border-radius:var(--r);
|
|
color:var(--soft);white-space:nowrap;scroll-snap-align:start;
|
|
text-decoration:none;background:var(--ground)}
|
|
.agent:hover{color:var(--ink);border-color:var(--line-hi);text-decoration:none}
|
|
.agent .mark{width:.95rem;height:.95rem}
|
|
|
|
h2{font-family:var(--mono);font-size:1.1rem;letter-spacing:-.01em;margin:2.75rem 0 .75rem;font-weight:500}
|
|
h3{font-size:1rem;margin:2rem 0 .5rem;font-weight:600}
|
|
h4,h5,h6{font-size:.94rem;margin:1.5rem 0 .5rem;font-weight:600}
|
|
p{margin:0 0 1rem;max-width:70ch}
|
|
ul,ol{margin:0 0 1rem;padding-left:1.35rem;max-width:70ch}
|
|
li{margin:.22rem 0}
|
|
a{color:var(--ink);text-decoration:underline;text-underline-offset:3px;
|
|
text-decoration-color:var(--line-hi)}
|
|
a:hover{text-decoration-color:var(--ink)}
|
|
a:focus-visible{outline:1px solid var(--ink);outline-offset:3px}
|
|
code{font-family:var(--mono);font-size:.85em;background:var(--panel);
|
|
border:1px solid var(--line);border-radius:5px;padding:.1em .36em}
|
|
pre{background:var(--panel);border:1px solid var(--line);padding:1rem 1.1rem;
|
|
border-radius:var(--r-soft);overflow-x:auto;margin:0 0 1.25rem}
|
|
pre code{background:none;border:0;padding:0;font-size:.82rem;line-height:1.7}
|
|
blockquote{margin:0 0 1rem;padding-left:1rem;border-left:1px solid var(--line-hi);color:var(--soft)}
|
|
hr{border:0;border-top:1px solid var(--line);margin:2.5rem 0}
|
|
table{border-collapse:collapse;width:100%;margin:0 0 1.25rem;font-size:.92rem;
|
|
display:block;overflow-x:auto}
|
|
th,td{border:1px solid var(--line);padding:.5rem .7rem;text-align:left}
|
|
|
|
.curl{display:block;background:var(--panel);border:1px solid var(--line);
|
|
border-radius:var(--r-soft);padding:.95rem 1.1rem;font-family:var(--mono);
|
|
font-size:.8rem;color:var(--soft);overflow-x:auto;margin:0 0 2.5rem;
|
|
white-space:pre;line-height:1.9}
|
|
.curl b{color:var(--ink);font-weight:400}
|
|
.curl i{font-style:normal;color:var(--faint)}
|
|
|
|
/* index */
|
|
#q{width:100%;padding:.85rem 1.2rem;font-family:var(--mono);font-size:.85rem;color:var(--ink);
|
|
background:var(--panel);border:1px solid var(--line);border-radius:var(--r);margin:0 0 .6rem}
|
|
#q::placeholder{color:var(--faint);text-transform:uppercase;letter-spacing:.1em;font-size:.72rem}
|
|
#q:focus{outline:none;border-color:var(--soft)}
|
|
#count{color:var(--faint);font-family:var(--mono);font-size:.72rem;letter-spacing:.1em;
|
|
text-transform:uppercase;margin:0 0 2rem}
|
|
.cat{margin:2.5rem 0 .5rem;font-family:var(--mono);font-size:.7rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);border-bottom:1px solid var(--line);
|
|
padding-bottom:.45rem}
|
|
.row{display:block;padding:.6rem .9rem;margin:0 -.9rem;border-bottom:1px solid var(--line);
|
|
border-radius:var(--r-soft);text-decoration:none}
|
|
.row:hover{background:#0a0a0a}
|
|
.nm{font-family:var(--mono);font-size:.84rem;color:var(--ink);display:block}
|
|
.ds{color:var(--soft);font-size:.9rem;display:block;margin-top:.12rem}
|
|
footer{margin-top:4rem;padding:2rem 0 4rem;border-top:1px solid var(--line);
|
|
color:var(--faint);font-size:.85rem}
|
|
footer a{color:var(--soft)}
|
|
.fmap{display:grid;gap:1.75rem;grid-template-columns:repeat(auto-fit,minmax(11rem,1fr));
|
|
margin:0 0 2rem;font-size:.85rem}
|
|
.fmap b{display:block;font-family:var(--mono);font-size:.68rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);font-weight:400;margin:0 0 .6rem;
|
|
border-bottom:1px solid var(--line);padding-bottom:.4rem}
|
|
.fmap a{display:inline-block;color:var(--ink);text-decoration:none;margin:.3rem .35rem 0 0}
|
|
.fmap a:hover{text-decoration:underline}
|
|
.fmap div{color:var(--faint);line-height:1.5}
|
|
.fend{border-top:1px solid var(--line);padding-top:1.25rem;font-size:.82rem}
|
|
|
|
@media (max-width:640px){
|
|
.hero .wrap{padding-top:3.25rem;padding-bottom:2.75rem}
|
|
.btns{gap:.5rem}
|
|
.btn{padding:.75rem 1.15rem;font-size:.68rem}
|
|
.copy,.copy .btn{width:100%;justify-content:center}
|
|
.menu{left:0;right:0;min-width:0}
|
|
.strip .in{flex-direction:column;align-items:flex-start;gap:.6rem;padding:.9rem 1.75rem}
|
|
.strip b{border-right:0;padding-right:0}
|
|
.bar{width:100%}
|
|
}
|
|
</style>
|
|
<link rel="alternate" type="text/markdown" href="/hip-0017.md">
|
|
|
|
<link rel="icon" href="/favicon.svg">
|
|
<link rel="canonical" href="https://hanzoskills.com/hip-0017">
|
|
</head>
|
|
<!-- Reading this with a machine? Every page here is also plain markdown.
|
|
This one: https://hanzoskills.com/hip-0017.md
|
|
Index: https://hanzoskills.com/llms.txt
|
|
Corpus: https://hanzoskills.com/all.md
|
|
Append .md to any path for the markdown twin. -->
|
|
<body>
|
|
<nav><div class="in">
|
|
<a class="brand" href="/"><svg class="mark" viewBox="0 0 67 67" fill="currentColor" aria-hidden="true"><path d="M22.21 67V44.6369H0V67H22.21Z"/><path d="M66.7038 22.3184H22.2534L0.0878906 44.6367H44.4634L66.7038 22.3184Z"/><path d="M22.21 0H0V22.3184H22.21V0Z"/><path d="M66.7198 0H44.5098V22.3184H66.7198V0Z"/><path d="M66.7198 67V44.6369H44.5098V67H66.7198Z"/></svg> HANZO<i>SKILLS</i></a>
|
|
<a class="up" href="/">All skills</a>
|
|
</div></nav>
|
|
|
|
<div class="wrap">
|
|
<h1>hip-0017</h1><p class="lede">HIP-17: Analytics Event Standard. Status Draft. Hanzo's own standard — read this before implementing against it.</p><h1>HIP-0017: Analytics Event Standard</h1>
|
|
<h2>Abstract</h2>
|
|
<p>This proposal defines the analytics event standard for the Hanzo ecosystem. All product analytics, session replay, feature flags, and A/B testing MUST use Hanzo Insights as specified in this document. Hanzo Insights is a fork of PostHog, rebranded with Hanzo token prefixes (<code>ha_</code>, <code>hax_</code>, <code>has_</code>, <code>haa_</code>), integrated with Hanzo IAM (hanzo.id), and backed by a Rust capture service for high-throughput event ingestion.</p>
|
|
<p><strong>Repository</strong>: <a href="https://github.com/hanzoai/insights">github.com/hanzoai/insights</a> <strong>Production</strong>: <code>insights.hanzo.ai</code> on <code>the cluster</code> cluster (<code>24.199.76.156</code>) <strong>Capture</strong>: Rust binary at <code>/e</code>, <code>/batch</code>, <code>/capture</code> endpoints <strong>Storage</strong>: Hanzo Datastore (analytics), Kafka (buffering, HIP-0030), SQL (metadata), KV (cache)</p>
|
|
<h2>Design Philosophy</h2>
|
|
<p>This section explains the architectural decisions behind Hanzo Insights. Understanding the <em>why</em> is essential for correct integration and for evaluating future alternatives.</p>
|
|
<h3>Why Hanzo Datastore Over TimescaleDB</h3>
|
|
<p>The original HIP-17 draft specified TimescaleDB. We replaced it with Hanzo Datastore. Here is why.</p>
|
|
<p><strong>TimescaleDB</strong> is SQL with time-series extensions. It stores data row-by-row (row-oriented). It is excellent for transactional workloads where you read and write individual rows. For analytics queries that scan millions of rows and aggregate them (e.g., "count pageviews per day for the last 90 days, grouped by browser"), TimescaleDB performs roughly the same as SQL -- because the query must read every column of every matching row, even if it only needs two columns.</p>
|
|
<p><strong>Hanzo Datastore</strong> is a columnar database purpose-built for analytics. It stores data column-by-column. When a query needs only <code>timestamp</code> and <code>event</code> from a table with 50 columns, Hanzo Datastore reads only those 2 columns from disk. This alone gives a 10-25x speedup for typical analytics queries. Add vectorized execution (SIMD operations on column batches), aggressive compression (LZ4 on columns of similar data), and sparse indexing, and Hanzo Datastore delivers 10-100x faster query performance than SQL/TimescaleDB for analytical workloads.</p>
|
|
<p><strong>Concrete numbers</strong>: A query like "count distinct users per day for the last 30 days" over 100M events takes ~200ms in Hanzo Datastore and ~15-30 seconds in SQL. Funnel analysis over 50M events: ~500ms in Hanzo Datastore, timeout in SQL.</p>
|
|
<p><strong>Trade-off</strong>: Hanzo Datastore is not good for transactional workloads (single-row updates, foreign keys, ACID transactions). We use SQL for metadata (projects, users, feature flag definitions, dashboard configurations) and Hanzo Datastore exclusively for event analytics. This is exactly PostHog's architecture.</p>
|
|
<h3>How Insights Connects to the Data Pipeline</h3>
|
|
<pre><code> ┌─────────────────────────┐
|
|
│ Hanzo Datastore │
|
|
│ (Analytics Storage) │
|
|
│ │
|
|
│ events table │
|
|
│ session_replay_events │
|
|
│ person_distinct_id │
|
|
└────────────▲─────────────┘
|
|
│ consume
|
|
│
|
|
┌──────────────┐ ┌───────────────┐ ┌───┴───────────────┐
|
|
│ Browser SDK │──→│ Rust Capture │──→│ Kafka (HIP-0030) │
|
|
│ (ha_ key) │ │ /e /batch │ │ events_plugin_ │
|
|
└──────────────┘ │ /capture │ │ ingestion │
|
|
└───────────────┘ └───┬───────────────┘
|
|
┌──────────────┐ │
|
|
│ Server SDK │──→ (same path) │ consume
|
|
│ Python/Go │ ▼
|
|
└──────────────┘ ┌───────────────────┐
|
|
│ Django Workers │
|
|
┌──────────────┐ │ (plugin-server) │
|
|
│ LLM Gateway │──→ (same path) │ │
|
|
│ (HIP-4) │ │ - person mapping │
|
|
└──────────────┘ │ - property joins │
|
|
│ - plugin execution│
|
|
└───────────────────┘</code></pre>
|
|
<p><strong>Data flow</strong>:</p>
|
|
<ol><li>SDKs send events via HTTP POST to the Rust capture service</li><li>Capture validates the project API key and writes to Kafka</li><li>The Hanzo Datastore Kafka engine consumes events and materializes them into the <code>events</code> table</li><li>Django plugin-server runs asynchronous transformations (person identification, property enrichment, plugin execution)</li><li>PostHog UI queries Hanzo Datastore via the query API for dashboards, funnels, and retention</li></ol>
|
|
<h2>Specification</h2>
|
|
<h3>Event Schema</h3>
|
|
<p>All analytics events MUST conform to the following schema. This is the canonical event format that SDKs produce and Hanzo Datastore stores.</p>
|
|
<pre><code class="lang-typescript">interface AnalyticsEvent {
|
|
// Required fields
|
|
event: string // Event name: "$pageview", "$autocapture", "llm_request", etc.
|
|
distinct_id: string // User identifier (anonymous or authenticated)
|
|
timestamp: string // ISO 8601 UTC with millisecond precision
|
|
properties: EventProperties // Event-specific properties
|
|
|
|
// Set by SDK (overridable)
|
|
uuid: string // UUIDv7, globally unique event ID
|
|
sent_at: string // Client-side send timestamp (for clock drift correction)
|
|
|
|
// Set by server (never send from client)
|
|
team_id?: number // Resolved from API key
|
|
person_id?: string // Resolved via distinct_id mapping
|
|
ip?: string // Client IP (captured server-side)
|
|
now?: string // Server receive timestamp
|
|
}
|
|
|
|
interface EventProperties {
|
|
// Standard properties ($ prefix = auto-captured by SDK)
|
|
$current_url?: string // Page URL
|
|
$host?: string // Hostname
|
|
$pathname?: string // URL path
|
|
$browser?: string // Browser name
|
|
$browser_version?: string // Browser version
|
|
$os?: string // Operating system
|
|
$os_version?: string // OS version
|
|
$device_type?: string // "Desktop", "Mobile", "Tablet"
|
|
$screen_height?: number // Screen height in pixels
|
|
$screen_width?: number // Screen width in pixels
|
|
$viewport_height?: number // Viewport height
|
|
$viewport_width?: number // Viewport width
|
|
$lib?: string // SDK name ("hanzo-js", "hanzo-python", etc.)
|
|
$lib_version?: string // SDK version
|
|
$referrer?: string // HTTP referrer
|
|
$referring_domain?: string // Referrer domain
|
|
$session_id?: string // Session identifier
|
|
$window_id?: string // Tab/window identifier
|
|
$insert_id?: string // Deduplication key
|
|
|
|
// Person properties (set once or updated)
|
|
$set?: Record<string, any> // Set person properties
|
|
$set_once?: Record<string, any> // Set person properties only if not already set
|
|
$unset?: string[] // Remove person properties
|
|
|
|
// Feature flag properties
|
|
$feature_flags?: Record<string, boolean | string>
|
|
$active_feature_flags?: string[]
|
|
|
|
// Custom properties (no $ prefix)
|
|
[key: string]: any // Application-specific properties
|
|
}</code></pre>
|
|
<h3>Reserved Event Names</h3>
|
|
<p>Events with a <code>$</code> prefix are reserved for SDK auto-capture. Custom events MUST NOT use the <code>$</code> prefix.</p>
|
|
<p>| Event | Trigger | Key Properties | |-------|---------|----------------| | <code>$pageview</code> | Page load / SPA navigation | <code>$current_url</code>, <code>$pathname</code>, <code>$referrer</code> | | <code>$pageleave</code> | Page unload / navigation away | <code>$current_url</code>, <code>$prev_pageview_duration</code> | | <code>$autocapture</code> | Click, change, submit (auto) | <code>$event_type</code>, <code>$elements</code>, <code>$element_chain</code> | | <code>$identify</code> | User identification call | <code>$set</code>, <code>$set_once</code>, <code>distinct_id</code> | | <code>$groupidentify</code> | Group identification | <code>$group_type</code>, <code>$group_key</code>, <code>$group_set</code> | | <code>$create_alias</code> | Merge anonymous + identified | <code>alias</code>, <code>distinct_id</code> | | <code>$feature_flag_called</code> | Feature flag evaluated | <code>$feature_flag</code>, <code>$feature_flag_response</code> | | <code>$session_start</code> | New session detected | <code>$session_id</code> | | <code>$session_end</code> | Session timeout | <code>$session_id</code>, <code>$session_duration</code> | | <code>$snapshot</code> | Session replay DOM snapshot | Compressed snapshot data | | <code>$exception</code> | JavaScript error caught | <code>$exception_message</code>, <code>$exception_type</code>, <code>$exception_stack_trace_raw</code> | | <code>$web_vitals</code> | Core Web Vitals measurement | <code>$web_vitals_LCP_value</code>, <code>$web_vitals_FID_value</code>, <code>$web_vitals_CLS_value</code> | | <code>$survey_sent</code> | Survey displayed | <code>$survey_id</code>, <code>$survey_name</code> | | <code>$survey_responded</code> | Survey answer submitted | <code>$survey_id</code>, <code>$survey_response</code> |</p>
|
|
<h3>Hanzo-Specific Events</h3>
|
|
<p>These are custom events standardized across Hanzo services. They do NOT use the <code>$</code> prefix.</p>
|
|
<p>| Event | Source | Key Properties | |-------|--------|----------------| | <code>llm_request</code> | LLM Gateway | <code>model</code>, <code>provider</code>, <code>prompt_tokens</code>, <code>completion_tokens</code>, <code>latency_ms</code>, <code>cost_usd</code> | | <code>llm_error</code> | LLM Gateway | <code>model</code>, <code>provider</code>, <code>error_code</code>, <code>error_message</code> | | <code>credit_purchase</code> | Commerce | <code>amount_usd</code>, <code>credits_added</code>, <code>payment_method</code> | | <code>credit_consumed</code> | Cloud / Gateway | <code>credits_used</code>, <code>service</code>, <code>model</code> | | <code>agent_task_started</code> | Agent SDK | <code>agent_id</code>, <code>task_type</code>, <code>tools_available</code> | | <code>agent_task_completed</code> | Agent SDK | <code>agent_id</code>, <code>task_type</code>, <code>duration_ms</code>, <code>tool_calls</code>, <code>tokens_used</code> | | <code>deployment_created</code> | Platform | <code>project_id</code>, <code>runtime</code>, <code>region</code> | | <code>api_key_created</code> | IAM | <code>key_prefix</code>, <code>scopes</code>, <code>application</code> |</p>
|
|
<h3>Capture Endpoints</h3>
|
|
<p>The Rust capture service exposes three HTTP endpoints for event ingestion. All three accept the same payload format. The distinction is historical (PostHog compatibility) but all three MUST be supported.</p>
|
|
<h4>POST /e (Primary)</h4>
|
|
<p>Single event or batch. This is the primary endpoint used by current SDKs.</p>
|
|
<pre><code class="lang-http">POST /e HTTP/1.1
|
|
Host: insights.hanzo.ai
|
|
Content-Type: application/json
|
|
|
|
{
|
|
"api_key": "ha_abc123def456",
|
|
"event": "$pageview",
|
|
"distinct_id": "user_789",
|
|
"timestamp": "2025-01-15T10:30:00.000Z",
|
|
"properties": {
|
|
"$current_url": "https://hanzo.ai/dashboard",
|
|
"$browser": "Chrome",
|
|
"$os": "macOS"
|
|
},
|
|
"uuid": "01945b7a-8c3d-7e2f-9a1b-4c5d6e7f8a9b",
|
|
"sent_at": "2025-01-15T10:30:00.100Z"
|
|
}</code></pre>
|
|
<p>Response (always immediate, does not wait for Hanzo Datastore):</p>
|
|
<pre><code class="lang-http">HTTP/1.1 200 OK
|
|
Content-Type: application/json
|
|
|
|
{"status": 1}</code></pre>
|
|
<h4>POST /batch (Bulk)</h4>
|
|
<p>Multiple events in a single request. SDKs SHOULD use this to reduce HTTP overhead.</p>
|
|
<pre><code class="lang-http">POST /batch HTTP/1.1
|
|
Host: insights.hanzo.ai
|
|
Content-Type: application/json
|
|
|
|
{
|
|
"api_key": "ha_abc123def456",
|
|
"batch": [
|
|
{
|
|
"event": "$pageview",
|
|
"distinct_id": "user_789",
|
|
"timestamp": "2025-01-15T10:30:00.000Z",
|
|
"properties": {"$current_url": "https://hanzo.ai/dashboard"},
|
|
"uuid": "01945b7a-8c3d-7e2f-9a1b-4c5d6e7f8a9b"
|
|
},
|
|
{
|
|
"event": "llm_request",
|
|
"distinct_id": "user_789",
|
|
"timestamp": "2025-01-15T10:30:05.000Z",
|
|
"properties": {"model": "zen-72b", "prompt_tokens": 150},
|
|
"uuid": "01945b7a-9d4e-8f3g-0b2c-5d6e7f8a9b0c"
|
|
}
|
|
],
|
|
"sent_at": "2025-01-15T10:30:06.000Z"
|
|
}</code></pre>
|
|
<p>Maximum batch size: 500 events. Requests exceeding this MUST receive HTTP 400.</p>
|
|
<h4>POST /capture (Legacy)</h4>
|
|
<p>Identical behavior to <code>/e</code>. Maintained for backward compatibility with older PostHog SDKs.</p>
|
|
<h4>Endpoint Behavior</h4>
|
|
<p>All capture endpoints MUST:</p>
|
|
<ul><li>Validate the <code>api_key</code> field (must start with <code>ha_</code>)</li><li>Reject personal API keys (<code>hax_</code> prefix) with HTTP 401</li><li>Write accepted events to Kafka topic <code>events_plugin_ingestion</code></li><li>Return HTTP 200 immediately after Kafka acknowledgment</li><li>Return HTTP 400 for malformed payloads</li><li>Return HTTP 401 for invalid or missing API keys</li><li>Return HTTP 503 when Kafka is unavailable (SDKs retry with exponential backoff)</li></ul>
|
|
<p>All capture endpoints MUST NOT:</p>
|
|
<ul><li>Query Hanzo Datastore</li><li>Query SQL (except for API key validation, which is cached in KV)</li><li>Block on any downstream processing</li></ul>
|
|
<h3>Ingestion Pipeline</h3>
|
|
<p>The full ingestion pipeline from client to queryable data:</p>
|
|
<pre><code>1. Client SDK batches events locally (max 20 events or 5 seconds)
|
|
2. SDK sends POST /batch to insights.hanzo.ai
|
|
3. K8s Ingress routes /e, /batch, /capture to Rust capture service
|
|
4. Capture validates api_key (KV cache lookup, TTL 5min)
|
|
5. Capture produces events to Kafka topic: events_plugin_ingestion
|
|
6. Kafka acknowledges (acks=all, idempotent)
|
|
7. Capture returns HTTP 200 to client
|
|
--- synchronous path ends here ---
|
|
8. Hanzo Datastore Kafka engine consumes from events_plugin_ingestion
|
|
9. Events materialized into Hanzo Datastore sharded_events table
|
|
10. Plugin-server reads from Kafka for async transforms:
|
|
- Person identification (distinct_id → person_id mapping)
|
|
- Property enrichment (GeoIP, user-agent parsing)
|
|
- Plugin execution (data transformations, webhooks)
|
|
11. Events queryable in Hanzo Datastore within ~5 seconds of capture</code></pre>
|
|
<h3>Query Engine</h3>
|
|
<p>Analytics queries are executed against Hanzo Datastore via the PostHog query API. The API accepts HogQL (PostHog's SQL dialect, a subset of Hanzo Datastore SQL with guardrails).</p>
|
|
<h4>Trends Query</h4>
|
|
<pre><code class="lang-json">{
|
|
"kind": "TrendsQuery",
|
|
"series": [
|
|
{
|
|
"event": "$pageview",
|
|
"math": "total"
|
|
}
|
|
],
|
|
"interval": "day",
|
|
"dateRange": {
|
|
"date_from": "-30d"
|
|
},
|
|
"breakdownFilter": {
|
|
"breakdown": "$browser",
|
|
"breakdown_type": "event"
|
|
}
|
|
}</code></pre>
|
|
<h4>Funnel Query</h4>
|
|
<pre><code class="lang-json">{
|
|
"kind": "FunnelsQuery",
|
|
"series": [
|
|
{"event": "$pageview"},
|
|
{"event": "llm_request"},
|
|
{"event": "credit_purchase"}
|
|
],
|
|
"funnelsFilter": {
|
|
"funnelWindowInterval": 7,
|
|
"funnelWindowIntervalUnit": "day",
|
|
"funnelOrderType": "ordered"
|
|
},
|
|
"dateRange": {
|
|
"date_from": "-30d"
|
|
}
|
|
}</code></pre>
|
|
<h4>Retention Query</h4>
|
|
<pre><code class="lang-json">{
|
|
"kind": "RetentionQuery",
|
|
"retentionFilter": {
|
|
"targetEntity": {"id": "$pageview", "type": "events"},
|
|
"returningEntity": {"id": "llm_request", "type": "events"},
|
|
"retentionType": "retention_first_time",
|
|
"period": "Week",
|
|
"totalIntervals": 8
|
|
},
|
|
"dateRange": {
|
|
"date_from": "-8w"
|
|
}
|
|
}</code></pre>
|
|
<h4>Raw HogQL</h4>
|
|
<p>For advanced queries, HogQL provides direct Hanzo Datastore access with safety guardrails:</p>
|
|
<pre><code class="lang-sql">SELECT
|
|
toDate(timestamp) AS day,
|
|
count() AS events,
|
|
uniqExact(distinct_id) AS unique_users
|
|
FROM events
|
|
WHERE event = 'llm_request'
|
|
AND timestamp >= now() - INTERVAL 30 DAY
|
|
AND properties.$model = 'zen-72b'
|
|
GROUP BY day
|
|
ORDER BY day DESC</code></pre>
|
|
<h3>Session Replay</h3>
|
|
<p>Session replay captures DOM snapshots, mouse movements, clicks, scrolls, network requests, and console logs. Replay data is stored as <code>$snapshot</code> events in Hanzo Datastore.</p>
|
|
<h4>Recording Configuration</h4>
|
|
<pre><code class="lang-javascript">hanzo.init('ha_abc123', {
|
|
api_host: 'https://insights.hanzo.ai',
|
|
session_recording: {
|
|
maskAllInputs: true, // PII protection: mask all input values
|
|
maskTextContent: false, // Show text content (mask selectively with CSS)
|
|
recordCrossOriginIframes: false,
|
|
recordCanvas: false, // Canvas recording is expensive
|
|
recordNetworkRequests: true, // Capture fetch/XHR waterfall
|
|
recordConsole: true, // Capture console.log/warn/error
|
|
}
|
|
})</code></pre>
|
|
<h4>Replay Data Flow</h4>
|
|
<pre><code>Browser → rrweb (DOM snapshot library) → $snapshot events → /e endpoint
|
|
→ Kafka → Hanzo Datastore session_replay_events table
|
|
→ PostHog UI replay player (reconstructs DOM from snapshots)</code></pre>
|
|
<p>Session replay data is significantly larger than analytics events (~100KB-1MB per minute of recording). Compression (gzip on HTTP, LZ4 in Kafka, ZSTD in Hanzo Datastore) reduces storage by ~10x.</p>
|
|
<h3>Feature Flags</h3>
|
|
<p>Feature flags enable gradual rollouts, A/B tests, and kill switches. Flags are defined in the PostHog UI and evaluated either server-side or client-side.</p>
|
|
<h4>Server-Side Evaluation</h4>
|
|
<pre><code class="lang-python"># Python SDK
|
|
from hanzo import Hanzo
|
|
|
|
hanzo = Hanzo(
|
|
api_key='ha_abc123',
|
|
personal_api_key='hax_personal_xyz',
|
|
host='https://insights.hanzo.ai'
|
|
)
|
|
|
|
# Boolean flag
|
|
if hanzo.feature_enabled('new-dashboard', distinct_id='user_789'):
|
|
show_new_dashboard()
|
|
|
|
# Multivariate flag
|
|
variant = hanzo.get_feature_flag('pricing-experiment', distinct_id='user_789')
|
|
if variant == 'annual-discount':
|
|
show_annual_pricing()</code></pre>
|
|
<h4>Client-Side Evaluation</h4>
|
|
<pre><code class="lang-javascript">// JavaScript SDK
|
|
hanzo.onFeatureFlags(() => {
|
|
if (hanzo.isFeatureEnabled('new-chat-ui')) {
|
|
renderNewChatUI()
|
|
}
|
|
|
|
const variant = hanzo.getFeatureFlag('onboarding-flow')
|
|
if (variant === 'guided-tour') {
|
|
startGuidedTour()
|
|
}
|
|
})</code></pre>
|
|
<h4>Flag Evaluation Endpoint</h4>
|
|
<pre><code class="lang-http">POST /decide/?v=3 HTTP/1.1
|
|
Host: insights.hanzo.ai
|
|
Content-Type: application/json
|
|
|
|
{
|
|
"api_key": "ha_abc123",
|
|
"distinct_id": "user_789",
|
|
"person_properties": {"plan": "pro", "country": "US"}
|
|
}</code></pre>
|
|
<p>Response:</p>
|
|
<pre><code class="lang-json">{
|
|
"featureFlags": {
|
|
"new-dashboard": true,
|
|
"pricing-experiment": "annual-discount",
|
|
"beta-agent-tools": false
|
|
},
|
|
"featureFlagPayloads": {
|
|
"pricing-experiment": "{\"discount_percent\": 20}"
|
|
},
|
|
"sessionRecording": {
|
|
"endpoint": "/s/"
|
|
}
|
|
}</code></pre>
|
|
<p>The <code>/decide</code> endpoint is served by Django, NOT the Rust capture service. It requires SQL access for flag definitions and person property lookups.</p>
|
|
<h3>SDK Integration</h3>
|
|
<h4>JavaScript (Browser)</h4>
|
|
<pre><code class="lang-javascript">import { Hanzo } from 'hanzo-js'
|
|
|
|
const hanzo = new Hanzo()
|
|
hanzo.init('ha_abc123def456', {
|
|
api_host: 'https://insights.hanzo.ai',
|
|
autocapture: true, // Auto-capture clicks, form submits
|
|
capture_pageview: true, // Auto-capture $pageview on load
|
|
capture_pageleave: true, // Auto-capture $pageleave on unload
|
|
session_recording: {
|
|
maskAllInputs: true
|
|
},
|
|
bootstrap: { // Instant flag evaluation (no /decide round-trip)
|
|
featureFlags: {
|
|
'new-dashboard': true
|
|
}
|
|
}
|
|
})
|
|
|
|
// Custom event
|
|
hanzo.capture('llm_request', {
|
|
model: 'zen-72b',
|
|
prompt_tokens: 150,
|
|
completion_tokens: 230,
|
|
latency_ms: 1250
|
|
})
|
|
|
|
// Identify user (link anonymous → authenticated)
|
|
hanzo.identify('user_789', {
|
|
email: '[email protected]',
|
|
plan: 'pro',
|
|
org: 'hanzo'
|
|
})
|
|
|
|
// Group analytics (organization-level)
|
|
hanzo.group('company', 'org_hanzo', {
|
|
name: 'Hanzo AI',
|
|
plan: 'enterprise',
|
|
employees: 50
|
|
})</code></pre>
|
|
<h4>Python (Server-Side)</h4>
|
|
<pre><code class="lang-python">from hanzo import Hanzo
|
|
|
|
hanzo = Hanzo(
|
|
api_key='ha_abc123def456',
|
|
host='https://insights.hanzo.ai'
|
|
)
|
|
|
|
# Capture event
|
|
hanzo.capture(
|
|
distinct_id='user_789',
|
|
event='llm_request',
|
|
properties={
|
|
'model': 'zen-72b',
|
|
'prompt_tokens': 150,
|
|
'completion_tokens': 230,
|
|
'cost_usd': 0.0038,
|
|
'provider': 'together'
|
|
}
|
|
)
|
|
|
|
# Identify with person properties
|
|
hanzo.identify(
|
|
distinct_id='user_789',
|
|
properties={
|
|
'$set': {'plan': 'pro', 'company': 'Hanzo AI'},
|
|
'$set_once': {'first_seen': '2025-01-15'}
|
|
}
|
|
)
|
|
|
|
# Flush on shutdown (server SDKs batch events)
|
|
hanzo.flush()
|
|
hanzo.shutdown()</code></pre>
|
|
<h4>Go (Server-Side)</h4>
|
|
<pre><code class="lang-go">package main
|
|
|
|
import "github.com/hanzoai/insights-go"
|
|
|
|
func main() {
|
|
client := insights.New("ha_abc123def456")
|
|
client.Endpoint = "https://insights.hanzo.ai"
|
|
defer client.Close()
|
|
|
|
client.Enqueue(insights.Capture{
|
|
DistinctId: "user_789",
|
|
Event: "llm_request",
|
|
Properties: insights.NewProperties().
|
|
Set("model", "zen-72b").
|
|
Set("prompt_tokens", 150).
|
|
Set("completion_tokens", 230).
|
|
Set("cost_usd", 0.0038),
|
|
})
|
|
}</code></pre>
|
|
<h4>React</h4>
|
|
<pre><code class="lang-tsx">import { HanzoProvider, useFeatureFlagEnabled, useHanzo } from 'hanzo-js/react'
|
|
|
|
function App() {
|
|
return (
|
|
<HanzoProvider
|
|
apiKey="ha_abc123def456"
|
|
options={{
|
|
api_host: 'https://insights.hanzo.ai',
|
|
autocapture: true,
|
|
}}
|
|
>
|
|
<Dashboard />
|
|
</HanzoProvider>
|
|
)
|
|
}
|
|
|
|
function Dashboard() {
|
|
const hanzo = useHanzo()
|
|
const showNewUI = useFeatureFlagEnabled('new-dashboard')
|
|
|
|
const handleAction = () => {
|
|
hanzo.capture('dashboard_action', { action: 'export_csv' })
|
|
}
|
|
|
|
return showNewUI ? <NewDashboard onAction={handleAction} /> : <LegacyDashboard />
|
|
}</code></pre>
|
|
<h2>Implementation</h2>
|
|
<h3>Production Deployment</h3>
|
|
<p>Hanzo Insights runs on <code>the cluster</code> (<code>24.199.76.156</code>) with the following services:</p>
|
|
<p>| Service | Image | Replicas | CPU | Memory | Purpose | |---------|-------|----------|-----|--------|---------| | <code>insights-web</code> | <code>ghcr.io/hanzoai/insights:latest</code> | 2 | 500m | 1Gi | Django: UI, API, /decide | | <code>insights-capture</code> | <code>ghcr.io/posthog/posthog/capture:master</code> | 2 | 250m | 128Mi | Rust: /e, /batch, /capture | | <code>insights-worker</code> | <code>ghcr.io/hanzoai/insights:latest</code> | 2 | 500m | 1Gi | Celery: async tasks | | <code>insights-plugins</code> | <code>ghcr.io/hanzoai/insights:latest</code> | 1 | 500m | 512Mi | Plugin server: transforms | | <code>insights-kafka</code> | <code>bitnami/kafka:3.7</code> | 1 | 500m | 2Gi | Event buffering (HIP-0030) | | <code>insights-datastore</code> | <code>ghcr.io/hanzoai/datastore:24.1</code> | 1 | 1000m | 4Gi | Analytics storage | | <code>insights-kv</code> | <code>redis:7-alpine</code> | 1 | 100m | 256Mi | Cache, session store | | <code>insights-postgres</code> | <code>postgres:16-alpine</code> | 1 | 250m | 512Mi | Metadata (or shared pg) |</p>
|
|
<h3>Ingress Routing</h3>
|
|
<p>The K8s Ingress splits traffic between the Rust capture service and Django based on URL path:</p>
|
|
<pre><code class="lang-yaml">apiVersion: networking.k8s.io/v1
|
|
kind: Ingress
|
|
metadata:
|
|
name: insights
|
|
namespace: hanzo
|
|
annotations:
|
|
nginx.ingress.kubernetes.io/proxy-body-size: "20m"
|
|
spec:
|
|
rules:
|
|
- host: insights.hanzo.ai
|
|
http:
|
|
paths:
|
|
# Hot path: Rust capture (high throughput, low latency)
|
|
- path: /e
|
|
pathType: Exact
|
|
backend:
|
|
service:
|
|
name: insights-capture
|
|
port:
|
|
number: 3000
|
|
- path: /batch
|
|
pathType: Exact
|
|
backend:
|
|
service:
|
|
name: insights-capture
|
|
port:
|
|
number: 3000
|
|
- path: /capture
|
|
pathType: Exact
|
|
backend:
|
|
service:
|
|
name: insights-capture
|
|
port:
|
|
number: 3000
|
|
# Everything else: Django (UI, API, /decide, /api)
|
|
- path: /
|
|
pathType: Prefix
|
|
backend:
|
|
service:
|
|
name: insights-web
|
|
port:
|
|
number: 8000</code></pre>
|
|
<p>This routing is the single most important operational detail. Getting it wrong means either:</p>
|
|
<ul><li>Sending capture traffic to Django (performance regression)</li><li>Sending UI/API traffic to Rust capture (404 errors)</li></ul>
|
|
<h3>Hanzo Datastore Schema</h3>
|
|
<p>The core events table in Hanzo Datastore:</p>
|
|
<pre><code class="lang-sql">CREATE TABLE IF NOT EXISTS sharded_events
|
|
(
|
|
uuid UUID,
|
|
event String,
|
|
properties String, -- JSON string
|
|
timestamp DateTime64(6, 'UTC'),
|
|
team_id Int64,
|
|
distinct_id String,
|
|
elements_chain String,
|
|
created_at DateTime64(6, 'UTC'),
|
|
person_id UUID,
|
|
person_created_at DateTime64(3, 'UTC'),
|
|
person_properties String, -- JSON string
|
|
group0_properties String,
|
|
group1_properties String,
|
|
group2_properties String,
|
|
group3_properties String,
|
|
group4_properties String,
|
|
$group_0 String,
|
|
$group_1 String,
|
|
$group_2 String,
|
|
$group_3 String,
|
|
$group_4 String,
|
|
$session_id String,
|
|
$window_id String,
|
|
_timestamp DateTime,
|
|
_offset UInt64
|
|
)
|
|
ENGINE = ReplicatedReplacingMergeTree('/datastore/tables/{shard}/events', '{replica}', _timestamp)
|
|
PARTITION BY toYYYYMM(timestamp)
|
|
ORDER BY (team_id, toDate(timestamp), event, cityHash64(distinct_id), cityHash64(uuid))
|
|
SAMPLE BY cityHash64(distinct_id)</code></pre>
|
|
<p>Key design decisions:</p>
|
|
<ul><li><strong>PARTITION BY month</strong>: Enables efficient partition pruning for date-range queries</li><li><strong>ORDER BY (team_id, date, event, ...)</strong>: Team isolation is the primary filter, then date, then event type</li><li><strong>SAMPLE BY distinct_id hash</strong>: Enables approximate queries on large datasets (e.g., "~5% sample of pageviews")</li><li><strong>ReplacingMergeTree</strong>: Deduplicates events by UUID on merge (handles Kafka redelivery)</li></ul>
|
|
<h3>Monitoring</h3>
|
|
<pre><code class="lang-yaml"># Key metrics to monitor
|
|
alerts:
|
|
- name: CaptureLatencyHigh
|
|
expr: histogram_quantile(0.99, capture_request_duration_seconds_bucket) > 0.5
|
|
for: 5m
|
|
severity: warning
|
|
annotations:
|
|
summary: "Capture p99 latency exceeds 500ms"
|
|
|
|
- name: CaptureErrorRateHigh
|
|
expr: rate(capture_requests_total{status="error"}[5m]) / rate(capture_requests_total[5m]) > 0.01
|
|
for: 5m
|
|
severity: critical
|
|
annotations:
|
|
summary: "Capture error rate exceeds 1%"
|
|
|
|
- name: ClickHouseQuerySlow
|
|
expr: histogram_quantile(0.95, clickhouse_query_duration_seconds_bucket) > 10
|
|
for: 10m
|
|
severity: warning
|
|
annotations:
|
|
summary: "Hanzo Datastore p95 query latency exceeds 10s"
|
|
|
|
- name: KafkaConsumerLagInsights
|
|
expr: kafka_consumer_group_lag{group="analytics-ingest"} > 100000
|
|
for: 5m
|
|
severity: warning
|
|
annotations:
|
|
summary: "Insights Hanzo Datastore consumer lag exceeds 100K events"
|
|
|
|
- name: EventIngestionDelay
|
|
expr: (max(insights_events_last_seen_at) - max(insights_events_last_ingested_at)) > 30
|
|
for: 5m
|
|
severity: critical
|
|
annotations:
|
|
summary: "Events taking >30s from capture to Hanzo Datastore"</code></pre>
|
|
<h2>Security</h2>
|
|
<h3>API Key Types and Scopes</h3>
|
|
<p>| Prefix | Type | Usage | Capture Allowed | API Allowed | |--------|------|-------|-----------------|-------------| | <code>ha_</code> | Project API Key | Client SDKs (browser, mobile) | Yes | No | | <code>hax_</code> | Personal API Key | Server-side API access | No | Yes | | <code>has_</code> | Session Recording | Internal (SDK ↔ replay) | No | No | | <code>haa_</code> | Feature Flag | Internal (SDK ↔ /decide) | No | No |</p>
|
|
<p><strong>Critical rule</strong>: The capture endpoints (<code>/e</code>, <code>/batch</code>, <code>/capture</code>) MUST reject <code>hax_</code> personal API keys. Personal keys have full API access (read events, modify projects, delete data). If a personal key leaks in client-side JavaScript, an attacker could read all analytics data. Project keys (<code>ha_</code>) can only write events -- they cannot read anything.</p>
|
|
<h3>PII Anonymization</h3>
|
|
<p>Event properties MUST NOT contain raw PII unless explicitly opted in per property. The following anonymization rules apply:</p>
|
|
<ol><li><strong>IP addresses</strong>: Captured server-side by the Rust capture service. Stored for GeoIP resolution, then replaced with <code>$geoip_city_name</code>, <code>$geoip_country_code</code>, etc. Raw IP is NOT stored in Hanzo Datastore by default.</li><li><strong>Email addresses</strong>: MUST be sent only via <code>$set</code> on <code>$identify</code> events (person properties), never in regular event properties.</li><li><strong>Session replay</strong>: <code>maskAllInputs: true</code> is the default. Input values are replaced with <code>***</code> in replay. CSS class <code>ha-no-capture</code> excludes any DOM element from recording.</li><li><strong>URLs</strong>: Query parameters may contain tokens or PII. SDKs SHOULD strip sensitive query parameters (configurable via <code>sanitize_properties</code>).</li></ol>
|
|
<h3>Data Retention</h3>
|
|
<p>| Data Type | Default Retention | Configurable | Storage | |-----------|------------------|--------------|---------| | Analytics events | 365 days | Yes, per team | Hanzo Datastore | | Session replay | 30 days | Yes, per team | Hanzo Datastore | | Person profiles | Indefinite | Deletable | Hanzo Datastore + SQL | | Feature flag definitions | Indefinite | N/A | SQL | | Kafka events | 7 days | Per topic (HIP-0030) | Kafka |</p>
|
|
<p>Retention enforcement is via Hanzo Datastore TTL:</p>
|
|
<pre><code class="lang-sql">ALTER TABLE sharded_events
|
|
MODIFY TTL timestamp + INTERVAL 365 DAY;
|
|
|
|
ALTER TABLE session_replay_events
|
|
MODIFY TTL timestamp + INTERVAL 30 DAY;</code></pre>
|
|
<h3>GDPR Compliance</h3>
|
|
<p>Hanzo Insights supports the right to erasure (GDPR Article 17):</p>
|
|
<ol><li><strong>Delete person data</strong>: API endpoint <code>DELETE /api/person/{distinct_id}</code> removes all person properties and optionally all events associated with that <code>distinct_id</code>.</li><li><strong>Anonymize events</strong>: Alternative to deletion -- replace <code>distinct_id</code> with a random hash, strip person properties, keep aggregate event data for analytics.</li><li><strong>Data export</strong>: API endpoint <code>GET /api/person/{distinct_id}/events</code> returns all events for a given user in JSON format.</li><li><strong>Consent management</strong>: SDKs support <code>opt_out_capturing()</code> which stops all event collection and deletes the local anonymous ID.</li></ol>
|
|
<pre><code class="lang-javascript">// Respect user consent
|
|
if (!userConsentedToAnalytics()) {
|
|
hanzo.opt_out_capturing() // No events sent, no cookies set
|
|
}
|
|
|
|
// Later, if consent is given
|
|
hanzo.opt_in_capturing()</code></pre>
|
|
<h3>Token Validation</h3>
|
|
<p>The Rust capture service validates API keys on every request:</p>
|
|
<ol><li>Check KV cache for key → team_id mapping (TTL 5 minutes)</li><li>On cache miss, query SQL <code>posthog_team</code> table</li><li>Reject if key prefix is not <code>ha_</code> (capture only accepts project keys)</li><li>Reject if key is not found or team is disabled</li><li>Cache the result in KV</li></ol>
|
|
<p>This validation adds < 1ms latency on cache hit and < 10ms on cache miss.</p>
|
|
<h3>Network Security</h3>
|
|
<pre><code class="lang-yaml">apiVersion: networking.k8s.io/v1
|
|
kind: NetworkPolicy
|
|
metadata:
|
|
name: insights-capture-policy
|
|
namespace: hanzo
|
|
spec:
|
|
podSelector:
|
|
matchLabels:
|
|
app: insights-capture
|
|
ingress:
|
|
- from:
|
|
- namespaceSelector:
|
|
matchLabels:
|
|
kubernetes.io/metadata.name: ingress-nginx
|
|
ports:
|
|
- port: 3000
|
|
protocol: TCP
|
|
egress:
|
|
- to:
|
|
- podSelector:
|
|
matchLabels:
|
|
app: insights-kafka
|
|
ports:
|
|
- port: 9092
|
|
- to:
|
|
- podSelector:
|
|
matchLabels:
|
|
app: insights-kv
|
|
ports:
|
|
- port: 6379</code></pre>
|
|
<p>The capture service can only talk to Kafka (write events) and KV (validate keys). It has no access to Hanzo Datastore, SQL, or the internet.</p>
|
|
<h2>Operational Runbook</h2>
|
|
<h3>Verify Event Ingestion</h3>
|
|
<pre><code class="lang-bash"># Send a test event
|
|
curl -X POST https://insights.hanzo.ai/e \
|
|
-H 'Content-Type: application/json' \
|
|
-d '{
|
|
"api_key": "ha_abc123",
|
|
"event": "test_event",
|
|
"distinct_id": "test_user",
|
|
"properties": {"test": true}
|
|
}'
|
|
|
|
# Check Kafka consumer lag
|
|
kubectl exec insights-kafka-0 -- \
|
|
kafka-consumer-groups.sh --bootstrap-server localhost:9092 \
|
|
--group analytics-ingest --describe
|
|
|
|
# Query Hanzo Datastore directly
|
|
kubectl exec insights-datastore-0 -- \
|
|
hanzo-datastore-client --query "
|
|
SELECT event, count()
|
|
FROM events
|
|
WHERE timestamp > now() - INTERVAL 1 HOUR
|
|
GROUP BY event
|
|
ORDER BY count() DESC
|
|
LIMIT 10
|
|
"</code></pre>
|
|
<h3>Debug Missing Events</h3>
|
|
<p>If events are captured but not appearing in the UI:</p>
|
|
<ol><li><strong>Check capture returned 200</strong>: SDK network tab or server logs</li><li><strong>Check Kafka</strong>: Consumer lag for <code>analytics-ingest</code> group -- if lag is growing, Hanzo Datastore consumer is stuck</li><li><strong>Check Hanzo Datastore</strong>: Query <code>events</code> table directly -- if events are there, the issue is in the PostHog query layer</li><li><strong>Check team_id</strong>: Verify the API key maps to the correct team in SQL</li></ol>
|
|
<h3>Hanzo Datastore Maintenance</h3>
|
|
<pre><code class="lang-bash"># Check table sizes
|
|
hanzo-datastore-client --query "
|
|
SELECT table, formatReadableSize(sum(bytes_on_disk)) AS size
|
|
FROM system.parts
|
|
WHERE active
|
|
GROUP BY table
|
|
ORDER BY sum(bytes_on_disk) DESC
|
|
"
|
|
|
|
# Force merge (reduces part count, improves query performance)
|
|
OPTIMIZE TABLE sharded_events FINAL;
|
|
|
|
# Check slow queries
|
|
SELECT query, elapsed, read_rows, formatReadableSize(read_bytes)
|
|
FROM system.query_log
|
|
WHERE type = 'QueryFinish'
|
|
AND elapsed > 5
|
|
ORDER BY elapsed DESC
|
|
LIMIT 10;</code></pre>
|
|
<h2>References</h2>
|
|
<ol><li><a href="./hip-0004-llm-gateway-unified-ai-provider-interface.md">HIP-4: LLM Gateway</a></li><li><a href="./hip-0005-post-quantum-security-for-ai-infrastructure.md">HIP-5: Secrets Management (KMS)</a></li><li><a href="./hip-0030-event-streaming-standard.md">HIP-30: Event Streaming Standard (Kafka)</a></li><li><a href="https://posthog.com/docs">PostHog Documentation</a></li><li><a href="https://clickhouse.com/docs">ClickHouse Documentation</a></li><li><a href="https://github.com/PostHog/posthog/tree/master/rust/capture">PostHog Rust Capture Service</a></li><li><a href="https://www.rrweb.io/">rrweb: DOM Recording Library</a></li><li><a href="https://github.com/hanzoai/insights">Hanzo Insights Repository</a></li></ol>
|
|
<h2>Copyright</h2>
|
|
<p>Copyright and related rights waived via <a href="https://creativecommons.org/publicdomain/zero/1.0/">CC0</a>.</p>
|
|
<footer>
|
|
<div class="fmap">
|
|
<div><b>Build</b>
|
|
<a href="https://hanzo.app">hanzo.app</a> app builder
|
|
<a href="https://hanzo.codes">hanzo.codes</a> code with an agent
|
|
<a href="https://hanzoskills.com">hanzoskills.com</a> what an agent reads</div>
|
|
<div><b>Run</b>
|
|
<a href="https://console.hanzo.ai">console.hanzo.ai</a> cloud console
|
|
<a href="https://api.hanzo.ai">api.hanzo.ai</a> the API</div>
|
|
<div><b>Account</b>
|
|
<a href="https://hanzo.id">hanzo.id</a> sign in
|
|
<a href="https://pay.hanzo.ai">pay.hanzo.ai</a> top up
|
|
<a href="https://billing.hanzo.ai">billing.hanzo.ai</a> invoices and usage</div>
|
|
<div><b>Company</b>
|
|
<a href="https://hanzo.ai">hanzo.ai</a> canonical
|
|
<a href="https://hanzo.ventures">hanzo.ventures</a> ventures</div>
|
|
</div>
|
|
<div class="fend"><a href="/hip-0017.md">This page as plain text</a> · <a href="/llms.txt">Index for agents</a></div>
|
|
</footer>
|
|
</div>
|
|
</body></html> |