Files
skills/public/hip-0012.html

732 lines
35 KiB
HTML

<!doctype html>
<html lang="en"><head>
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
<title>hip-0012</title>
<meta name="color-scheme" content="dark">
<style>
/* Committed to one visual world: a dark instrument panel. No light variant —
the dot field and the hairline rules only read on black, and a naive
inversion would wash both out. Every colour is painted explicitly so the page
never borrows a host background. */
@font-face{font-family:'Zen';src:url('/fonts/Zen-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
@font-face{font-family:'Zen';src:url('/fonts/Zen-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
@font-face{font-family:'Zen Pixel Square';src:url('/fonts/ZenPixel-Square.woff2') format('woff2');font-style:normal;font-display:swap}
:root{
--ink:#f4f4f2; --soft:#9c9c98; --faint:#6a6a66; --ground:#000; --panel:#0a0a0a;
--line:#232323; --line-hi:#3a3a3a; --accent:#f4f4f2;
--font-zen-sans:'Zen',-apple-system,BlinkMacSystemFont,"Segoe UI",Helvetica,Arial,sans-serif;
--font-zen-mono:'Zen Mono',ui-monospace,"SF Mono",SFMono-Regular,Menlo,Consolas,monospace;
--font-zen-pixel-square:'Zen Pixel Square',monospace;
--mono:var(--font-zen-mono);
--sans:var(--font-zen-sans);
/* Controls are pills and surfaces are soft, as on hanzo.ai. */
--r:999px; --r-soft:14px;
}
*{box-sizing:border-box}
html{-webkit-text-size-adjust:100%}
body{margin:0;background:var(--ground);color:var(--ink);font-family:var(--sans);
font-size:16px;line-height:1.65;-webkit-font-smoothing:antialiased}
.wrap{max-width:60rem;margin:0 auto;padding:0 1.75rem}
/* The dot field. A halftone made of one repeating radial-gradient, faded out
along the diagonal by a mask so it reads as a wave rather than wallpaper. */
/* No overflow clip here: the dot field is inset to the box already, and a clip
would cut off the copy-prompt menu. */
.hero{position:relative;border-bottom:1px solid var(--line)}
.hero::before{content:"";position:absolute;inset:0;
background-image:radial-gradient(var(--line-hi) 1px, transparent 1px);
background-size:7px 7px;
-webkit-mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
pointer-events:none}
.hero .wrap{position:relative;padding-top:5.5rem;padding-bottom:4rem}
nav{border-bottom:1px solid var(--line)}
nav .in{max-width:60rem;margin:0 auto;padding:1.1rem 1.75rem;display:flex;
justify-content:space-between;align-items:center;gap:1rem}
.brand{font-family:var(--mono);font-size:.85rem;letter-spacing:.14em;
text-transform:uppercase;color:var(--ink);text-decoration:none;
display:inline-flex;align-items:center;gap:.6rem}
.brand i{font-style:normal;color:var(--faint)}
.mark{width:1.15em;height:1.15em;display:block;flex:0 0 auto}
nav a.up{font-family:var(--mono);font-size:.72rem;letter-spacing:.14em;
text-transform:uppercase;color:var(--soft);text-decoration:none}
nav a.up:hover{color:var(--ink)}
h1{font-family:var(--mono);font-size:clamp(2rem,5.4vw,3.4rem);line-height:1.12;
letter-spacing:-.02em;margin:0 0 1.1rem;font-weight:500;text-wrap:balance}
h1 em{font-style:normal;color:var(--faint)}
.lede{color:var(--soft);font-size:1.05rem;max-width:52ch;margin:0 0 2.25rem}
.btns{display:flex;flex-wrap:wrap;gap:.7rem;margin:0 0 .5rem;align-items:center}
.btn{display:inline-flex;align-items:center;gap:.5rem;font-family:var(--mono);
font-size:.72rem;letter-spacing:.14em;text-transform:uppercase;
padding:.8rem 1.4rem;border:1px solid var(--line-hi);border-radius:var(--r);
color:var(--ink);text-decoration:none;background:transparent;cursor:pointer;
line-height:1.2;font-weight:400}
.btn:hover{background:#141414;text-decoration:none;border-color:var(--soft)}
.btn.solid{background:var(--ink);color:#000;border-color:var(--ink)}
.btn.solid:hover{background:#d8d8d4}
.btn .chev{width:.7em;height:.7em;flex:0 0 auto;transition:transform .15s ease}
.btn[aria-expanded="true"] .chev{transform:rotate(180deg)}
/* Copy-prompt selector. Each row copies the prompt for its own target. */
.copy{position:relative;display:inline-flex}
.menu{position:absolute;top:calc(100% + .5rem);left:0;z-index:20;min-width:21rem;
background:var(--panel);border:1px solid var(--line-hi);border-radius:var(--r-soft);
padding:.4rem;display:flex;flex-direction:column;
box-shadow:0 12px 32px rgba(0,0,0,.75)}
.menu[hidden]{display:none}
.mi{display:flex;align-items:center;gap:.7rem;width:100%;text-align:left;
padding:.62rem .8rem;border:0;border-radius:calc(var(--r-soft) - .35rem);
background:transparent;color:var(--ink);font-family:var(--mono);font-size:.76rem;
letter-spacing:.04em;text-decoration:none;cursor:pointer;line-height:1.3;
white-space:nowrap}
.mi:hover,.mi:focus-visible{background:#181818;outline:none}
.mi .mark{width:1.05rem;height:1.05rem;color:var(--soft)}
.mi:hover .mark{color:var(--ink)}
.mi u{margin-left:auto;color:var(--faint);text-decoration:none;font-size:.68rem;
letter-spacing:.1em;text-transform:uppercase}
/* Works-with bar: one scrolling row, name and mark per agent. */
.strip{border-bottom:1px solid var(--line);background:var(--panel)}
.strip .in{max-width:60rem;margin:0 auto;padding:.9rem 1.75rem;display:flex;
align-items:center;gap:1.25rem;font-family:var(--mono);font-size:.72rem;
letter-spacing:.1em;text-transform:uppercase;color:var(--faint)}
.strip b{color:var(--soft);font-weight:400;flex:0 0 auto;
border-right:1px solid var(--line);padding-right:1.25rem;align-self:stretch;
display:flex;align-items:center}
.bar{display:flex;align-items:center;gap:.5rem;flex:1 1 auto;min-width:0;
overflow-x:auto;scrollbar-width:none;scroll-snap-type:x proximity;
-webkit-overflow-scrolling:touch;
-webkit-mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent);
mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent)}
.bar::-webkit-scrollbar{display:none}
.agent{display:inline-flex;align-items:center;gap:.45rem;flex:0 0 auto;
padding:.4rem .85rem;border:1px solid var(--line);border-radius:var(--r);
color:var(--soft);white-space:nowrap;scroll-snap-align:start;
text-decoration:none;background:var(--ground)}
.agent:hover{color:var(--ink);border-color:var(--line-hi);text-decoration:none}
.agent .mark{width:.95rem;height:.95rem}
h2{font-family:var(--mono);font-size:1.1rem;letter-spacing:-.01em;margin:2.75rem 0 .75rem;font-weight:500}
h3{font-size:1rem;margin:2rem 0 .5rem;font-weight:600}
h4,h5,h6{font-size:.94rem;margin:1.5rem 0 .5rem;font-weight:600}
p{margin:0 0 1rem;max-width:70ch}
ul,ol{margin:0 0 1rem;padding-left:1.35rem;max-width:70ch}
li{margin:.22rem 0}
a{color:var(--ink);text-decoration:underline;text-underline-offset:3px;
text-decoration-color:var(--line-hi)}
a:hover{text-decoration-color:var(--ink)}
a:focus-visible{outline:1px solid var(--ink);outline-offset:3px}
code{font-family:var(--mono);font-size:.85em;background:var(--panel);
border:1px solid var(--line);border-radius:5px;padding:.1em .36em}
pre{background:var(--panel);border:1px solid var(--line);padding:1rem 1.1rem;
border-radius:var(--r-soft);overflow-x:auto;margin:0 0 1.25rem}
pre code{background:none;border:0;padding:0;font-size:.82rem;line-height:1.7}
blockquote{margin:0 0 1rem;padding-left:1rem;border-left:1px solid var(--line-hi);color:var(--soft)}
hr{border:0;border-top:1px solid var(--line);margin:2.5rem 0}
table{border-collapse:collapse;width:100%;margin:0 0 1.25rem;font-size:.92rem;
display:block;overflow-x:auto}
th,td{border:1px solid var(--line);padding:.5rem .7rem;text-align:left}
.curl{display:block;background:var(--panel);border:1px solid var(--line);
border-radius:var(--r-soft);padding:.95rem 1.1rem;font-family:var(--mono);
font-size:.8rem;color:var(--soft);overflow-x:auto;margin:0 0 2.5rem;
white-space:pre;line-height:1.9}
.curl b{color:var(--ink);font-weight:400}
.curl i{font-style:normal;color:var(--faint)}
/* index */
#q{width:100%;padding:.85rem 1.2rem;font-family:var(--mono);font-size:.85rem;color:var(--ink);
background:var(--panel);border:1px solid var(--line);border-radius:var(--r);margin:0 0 .6rem}
#q::placeholder{color:var(--faint);text-transform:uppercase;letter-spacing:.1em;font-size:.72rem}
#q:focus{outline:none;border-color:var(--soft)}
#count{color:var(--faint);font-family:var(--mono);font-size:.72rem;letter-spacing:.1em;
text-transform:uppercase;margin:0 0 2rem}
.cat{margin:2.5rem 0 .5rem;font-family:var(--mono);font-size:.7rem;letter-spacing:.16em;
text-transform:uppercase;color:var(--faint);border-bottom:1px solid var(--line);
padding-bottom:.45rem}
.row{display:block;padding:.6rem .9rem;margin:0 -.9rem;border-bottom:1px solid var(--line);
border-radius:var(--r-soft);text-decoration:none}
.row:hover{background:#0a0a0a}
.nm{font-family:var(--mono);font-size:.84rem;color:var(--ink);display:block}
.ds{color:var(--soft);font-size:.9rem;display:block;margin-top:.12rem}
footer{margin-top:4rem;padding:2rem 0 4rem;border-top:1px solid var(--line);
color:var(--faint);font-size:.85rem}
footer a{color:var(--soft)}
.fmap{display:grid;gap:1.75rem;grid-template-columns:repeat(auto-fit,minmax(11rem,1fr));
margin:0 0 2rem;font-size:.85rem}
.fmap b{display:block;font-family:var(--mono);font-size:.68rem;letter-spacing:.16em;
text-transform:uppercase;color:var(--faint);font-weight:400;margin:0 0 .6rem;
border-bottom:1px solid var(--line);padding-bottom:.4rem}
.fmap a{display:inline-block;color:var(--ink);text-decoration:none;margin:.3rem .35rem 0 0}
.fmap a:hover{text-decoration:underline}
.fmap div{color:var(--faint);line-height:1.5}
.fend{border-top:1px solid var(--line);padding-top:1.25rem;font-size:.82rem}
@media (max-width:640px){
.hero .wrap{padding-top:3.25rem;padding-bottom:2.75rem}
.btns{gap:.5rem}
.btn{padding:.75rem 1.15rem;font-size:.68rem}
.copy,.copy .btn{width:100%;justify-content:center}
.menu{left:0;right:0;min-width:0}
.strip .in{flex-direction:column;align-items:flex-start;gap:.6rem;padding:.9rem 1.75rem}
.strip b{border-right:0;padding-right:0}
.bar{width:100%}
}
</style>
<link rel="alternate" type="text/markdown" href="/hip-0012.md">
<link rel="icon" href="/favicon.svg">
<link rel="canonical" href="https://hanzoskills.com/hip-0012">
</head>
<!-- Reading this with a machine? Every page here is also plain markdown.
This one: https://hanzoskills.com/hip-0012.md
Index: https://hanzoskills.com/llms.txt
Corpus: https://hanzoskills.com/all.md
Append .md to any path for the markdown twin. -->
<body>
<nav><div class="in">
<a class="brand" href="/"><svg class="mark" viewBox="0 0 67 67" fill="currentColor" aria-hidden="true"><path d="M22.21 67V44.6369H0V67H22.21Z"/><path d="M66.7038 22.3184H22.2534L0.0878906 44.6367H44.4634L66.7038 22.3184Z"/><path d="M22.21 0H0V22.3184H22.21V0Z"/><path d="M66.7198 0H44.5098V22.3184H66.7198V0Z"/><path d="M66.7198 67V44.6369H44.5098V67H66.7198Z"/></svg> HANZO<i>SKILLS</i></a>
<a class="up" href="/">All skills</a>
</div></nav>
<div class="wrap">
<h1>hip-0012</h1><p class="lede">HIP-12: Search Interface Standard. Status Draft. Hanzo&#x27;s own standard — read this before implementing against it.</p><h1>HIP-0012: Search Interface Standard</h1>
<h2>Abstract</h2>
<p>This proposal defines the search interface standard for the Hanzo ecosystem. It specifies a two-layer architecture: a high-performance Rust search engine for indexing, retrieval, and hybrid search, and a generative search UI layer that synthesizes AI answers with inline citations from multiple source types. All search functionality in the Hanzo ecosystem MUST implement this interface.</p>
<p><strong>Engine Repository</strong>: <a href="https://github.com/hanzoai/search">github.com/hanzoai/search</a> <strong>Engine Port</strong>: 7700 <strong>UI Port</strong>: 3000 <strong>Engine Runtime</strong>: Rust (Hanzo Search fork, v1.36.0) <strong>UI Runtime</strong>: Next.js + Supabase</p>
<h2>Design Philosophy</h2>
<h3>Why Generative Search UI</h3>
<p>Traditional search returns ten blue links. The user clicks through, reads, evaluates, and synthesizes an answer themselves. This model was designed in the 1990s for a web of static pages.</p>
<p>Generative search inverts this. The system retrieves relevant documents, synthesizes a coherent answer, and presents it with citations the user can verify. The user gets an answer, not a reading list. Time-to-answer drops from minutes to seconds.</p>
<p>This is the paradigm shift from Google-style search to AI-native search. The search engine becomes a retrieval backend; the LLM becomes the synthesis frontend. The UI streams the answer in real time, with numbered citations linking to source material.</p>
<h2>Specification</h2>
<h3>System Architecture</h3>
<pre><code>┌─────────────┐ ┌──────────────────┐ ┌──────────────┐
│ Browser │────▶│ Search UI │────▶│ LLM Gateway │
│ Client │◀────│ (Next.js:3000) │◀────│ (HIP-4) │
└─────────────┘ └──────────────────┘ └──────────────┘
│ │ │
┌──────┘ └──────┐ ▼
▼ ▼ ┌──────────┐
┌──────────────┐ ┌────────────┐ │ 100+ │
│ Search Engine│ │ Supabase │ │ Providers│
│ (Rust:7700) │ │ (pgvector)│ └──────────┘
└──────────────┘ └────────────┘
│ │
▼ ▼
┌──────────────┐ ┌────────────┐
│ Index Storage│ │ SQL │
│ (LMDB) │ │ (Auth/Hist)│
└──────────────┘ └────────────┘</code></pre>
<h3>Query Pipeline</h3>
<p>The generative search pipeline processes a user query through five stages:</p>
<pre><code>1. PARSE User query → intent classification + query rewriting
2. RETRIEVE Parallel retrieval from multiple source types
3. RANK Score and deduplicate results across sources
4. SYNTHESIZE LLM generates answer with inline citations
5. STREAM Streaming markdown delivered to client via SSE</code></pre>
<h4>Stage 1: Parse</h4>
<pre><code class="lang-typescript">interface ParsedQuery {
original: string; // Raw user input
rewritten: string; // LLM-rewritten for retrieval
intent: SearchIntent; // web | academic | code | news | docs
entities: string[]; // Extracted named entities
embedding: number[]; // Query embedding vector (1536-dim)
}
type SearchIntent = &quot;web&quot; | &quot;academic&quot; | &quot;code&quot; | &quot;news&quot; | &quot;docs&quot;;</code></pre>
<p>The query rewriter uses a lightweight LLM call (or rule-based heuristics for common patterns) to transform conversational queries into retrieval-optimized forms. &quot;What&#x27;s the best way to deploy a Rust binary on Kubernetes?&quot; becomes the embedding query &quot;Rust binary Kubernetes deployment&quot; and the web query &quot;deploy Rust application Kubernetes best practices&quot;.</p>
<h4>Stage 2: Retrieve</h4>
<p>Retrieval runs in parallel across configured source types:</p>
<pre><code class="lang-typescript">interface RetrievalSource {
type: &quot;engine&quot; | &quot;web&quot; | &quot;vector&quot; | &quot;knowledge_base&quot;;
weight: number; // 0.0-1.0, used in ranking
config: SourceConfig;
}
// Source configurations
interface EngineSource {
type: &quot;engine&quot;;
endpoint: string; // Hanzo Search engine (port 7700)
index: string; // Target index name
limit: number;
}
interface WebSource {
type: &quot;web&quot;;
provider: &quot;brave&quot; | &quot;google&quot; | &quot;bing&quot;;
api_key: string;
limit: number;
}
interface VectorSource {
type: &quot;vector&quot;;
connection: string; // pgvector connection string
table: string;
embedding_column: string;
similarity: &quot;cosine&quot; | &quot;l2&quot; | &quot;inner_product&quot;;
limit: number;
}
interface KnowledgeBaseSource {
type: &quot;knowledge_base&quot;;
id: string; // KB identifier
endpoint: string; // API endpoint
limit: number;
}</code></pre>
<h4>Stage 3: Rank</h4>
<p>Results from all sources are scored and merged:</p>
<pre><code class="lang-typescript">interface RankedResult {
id: string;
title: string;
url: string;
snippet: string; // Relevant text excerpt
content: string; // Full text (for LLM context)
source_type: string; // Which retrieval source
source_name: string; // Display name (e.g., &quot;Brave Search&quot;)
score: number; // Normalized 0.0-1.0
metadata: Record&lt;string, any&gt;;
}</code></pre>
<p>Scoring combines:</p>
<ul><li><strong>Relevance</strong>: BM25 or cosine similarity from the source</li><li><strong>Source weight</strong>: Configured per-source priority</li><li><strong>Freshness</strong>: Recency bonus for time-sensitive queries</li><li><strong>Authority</strong>: Domain reputation score for web results</li></ul>
<p>Deduplication uses URL normalization and content fingerprinting (SimHash) to collapse near-duplicate results.</p>
<h4>Stage 4: Synthesize</h4>
<p>The top-K ranked results are assembled into an LLM prompt:</p>
<pre><code class="lang-typescript">interface SynthesisRequest {
query: string;
results: RankedResult[]; // Top-K (default: 10)
mode: SearchIntent;
model?: string; // Override model selection
max_tokens?: number; // Default: 2048
}</code></pre>
<p>The synthesis prompt instructs the model to:</p>
<ol><li>Answer the query using ONLY information from the provided sources</li><li>Cite sources using <code>[N]</code> notation, where N maps to the source index</li><li>Structure the answer with markdown headings when appropriate</li><li>Include a brief &quot;Key takeaway&quot; at the top for quick scanning</li><li>Flag when sources conflict or information is uncertain</li></ol>
<h4>Stage 5: Stream</h4>
<p>The response streams to the client using Server-Sent Events:</p>
<pre><code class="lang-typescript">// Answer token
event: token
data: {&quot;content&quot;: &quot;Rust binaries can be deployed&quot;, &quot;index&quot;: 0}
// Citation reference (emitted when [N] appears in stream)
event: citation
data: {&quot;index&quot;: 1, &quot;source_id&quot;: &quot;src_abc&quot;, &quot;url&quot;: &quot;https://...&quot;, &quot;title&quot;: &quot;...&quot;}
// Related questions
event: related
data: {&quot;questions&quot;: [&quot;How to optimize Rust binary size?&quot;, &quot;Kubernetes pod resource limits for Rust&quot;]}
// Completion
event: done
data: {
&quot;query_id&quot;: &quot;q_789&quot;,
&quot;model&quot;: &quot;zen-32b&quot;,
&quot;tokens&quot;: {&quot;prompt&quot;: 4200, &quot;completion&quot;: 680, &quot;total&quot;: 4880},
&quot;sources_used&quot;: 6,
&quot;latency_ms&quot;: 2340
}
// Error
event: error
data: {&quot;code&quot;: &quot;synthesis_failed&quot;, &quot;message&quot;: &quot;LLM returned empty response&quot;}</code></pre>
<h3>Core Data Models</h3>
<h4>Search Request</h4>
<pre><code class="lang-typescript">interface SearchRequest {
// Required
query: string;
// Optional configuration
mode?: SearchIntent; // Default: auto-detect
sources?: string[]; // Source type filter
limit?: number; // Max sources to retrieve (default: 10)
model?: string; // LLM model override
stream?: boolean; // Default: true
language?: string; // ISO 639-1 code
// Filters
domain_filter?: string[]; // Include only these domains
domain_block?: string[]; // Exclude these domains
date_range?: {
from?: string; // ISO 8601
to?: string;
};
// Context
user_id?: string; // For personalization
session_id?: string; // For search history grouping
}</code></pre>
<h4>Search Response</h4>
<pre><code class="lang-typescript">interface SearchResponse {
// Identifiers
query_id: string;
query: string;
// Generated answer
answer: string; // Markdown with [N] citations
answer_tokens: number;
// Sources used in synthesis
sources: Source[];
// Auto-generated follow-up queries
related_questions: string[];
// Metadata
model: string;
mode: SearchIntent;
latency_ms: number;
tokens: {
prompt: number;
completion: number;
total: number;
};
}
interface Source {
index: number; // Citation number [N]
title: string;
url: string;
snippet: string;
favicon?: string;
source_type: string;
published_date?: string;
}</code></pre>
<h3>API Endpoints</h3>
<h4>Search</h4>
<pre><code class="lang-yaml"># Generative search (primary endpoint)
POST /api/search
Headers:
Authorization: Bearer &lt;token&gt;
Accept: text/event-stream # For SSE streaming
Body: SearchRequest
Response: Stream of SSE events (see Stage 5)
# Non-streaming variant
POST /api/search
Headers:
Authorization: Bearer &lt;token&gt;
Accept: application/json
Body: SearchRequest
Response: SearchResponse</code></pre>
<h4>Sources</h4>
<pre><code class="lang-yaml"># Get sources for a completed search
GET /api/search/:query_id/sources
Response:
sources: Source[]
total: number
# Get full content of a specific source
GET /api/search/:query_id/sources/:index
Response:
source: Source
full_content: string</code></pre>
<h4>Search History</h4>
<pre><code class="lang-yaml"># List recent searches
GET /api/search/history
Query:
page?: number
limit?: number (default: 20)
mode?: SearchIntent
Response:
searches: SearchHistoryEntry[]
total: number
# Delete search history
DELETE /api/search/history
Response:
success: boolean</code></pre>
<h4>Engine Proxy (Direct Hanzo Search API)</h4>
<pre><code class="lang-yaml"># Index documents (proxied to engine on port 7700)
POST /api/engine/indexes/:index/documents
Headers:
Authorization: Bearer &lt;master-key&gt;
Body: Document[]
Response:
taskUid: number
status: &quot;enqueued&quot;
# Direct search (non-generative, engine only)
POST /api/engine/indexes/:index/search
Body:
q: string
limit?: number
filter?: string
sort?: string[]
attributesToRetrieve?: string[]
Response:
hits: Document[]
query: string
processingTimeMs: number
estimatedTotalHits: number</code></pre>
<h3>Search Modes</h3>
<p>Each mode adjusts source weights, synthesis prompts, and UI layout:</p>
<p>| Mode | Primary Sources | Synthesis Style | UI Features | |------|----------------|-----------------|-------------| | <strong>web</strong> | Brave/Google + engine | Balanced summary | Source cards, images | | <strong>academic</strong> | Scholar APIs + engine | Formal with methodology | Paper cards, DOI links | | <strong>code</strong> | GitHub + engine + docs | Code examples with explanation | Syntax highlighting, copy buttons | | <strong>news</strong> | News APIs + engine | Timeline-aware, multi-perspective | Date grouping, outlet badges | | <strong>docs</strong> | Internal knowledge bases | Precise, step-by-step | Code blocks, API examples |</p>
<p>Mode is auto-detected from query content but can be overridden in the request.</p>
<h3>Engine Configuration</h3>
<p>The Rust search engine accepts configuration via CLI flags, environment variables, or a config file:</p>
<pre><code class="lang-toml"># config.toml
[server]
http_addr = &quot;0.0.0.0:7700&quot;
master_key = &quot;HANZO_SEARCH_MASTER_KEY&quot;
max_index_size = &quot;100GB&quot;
max_task_db_size = &quot;10GB&quot;
[indexing]
max_indexing_memory = &quot;2GiB&quot;
max_indexing_threads = 4
[experimental]
enable_metrics = true
enable_logs_route = true
vector_store = true # Enable hybrid search</code></pre>
<h3>Authentication and Authorization</h3>
<p>Search uses a layered auth model:</p>
<ol><li><strong>UI Layer</strong>: Supabase Auth federated through IAM (HIP-26) via OAuth</li><li><strong>Engine Layer</strong>: API key-based with tenant isolation</li><li><strong>LLM Gateway</strong>: Bearer token forwarded from UI session</li></ol>
<pre><code class="lang-typescript">interface SearchAPIKey {
uid: string;
key: string; // Hashed
name: string;
actions: SearchAction[];
indexes: string[]; // [&quot;*&quot;] for all
expires_at?: string; // ISO 8601
}
type SearchAction =
| &quot;search&quot; // Query indexes
| &quot;documents.add&quot; // Index documents
| &quot;documents.delete&quot; // Remove documents
| &quot;indexes.create&quot; // Create indexes
| &quot;indexes.delete&quot; // Delete indexes
| &quot;settings.update&quot; // Modify index settings
| &quot;stats.get&quot; // Read metrics
| &quot;*&quot;; // Admin</code></pre>
<h3>Rate Limiting</h3>
<pre><code class="lang-yaml">Tiers:
free:
searches_per_minute: 10
searches_per_day: 200
max_sources_per_query: 5
engine_documents: 10000
pro:
searches_per_minute: 60
searches_per_day: 5000
max_sources_per_query: 15
engine_documents: 1000000
enterprise:
searches_per_minute: 300
searches_per_day: unlimited
max_sources_per_query: 30
engine_documents: unlimited</code></pre>
<h3>Error Handling</h3>
<pre><code class="lang-typescript">interface SearchError {
error: {
code: SearchErrorCode;
message: string;
details?: any;
query_id?: string;
timestamp: number;
};
}
enum SearchErrorCode {
// Client errors
INVALID_QUERY = &quot;invalid_query&quot;,
QUERY_TOO_LONG = &quot;query_too_long&quot;, // &gt; 2000 chars
UNAUTHORIZED = &quot;unauthorized&quot;,
RATE_LIMITED = &quot;rate_limited&quot;,
INVALID_MODE = &quot;invalid_mode&quot;,
INDEX_NOT_FOUND = &quot;index_not_found&quot;,
// Server errors
RETRIEVAL_FAILED = &quot;retrieval_failed&quot;, // Source unavailable
SYNTHESIS_FAILED = &quot;synthesis_failed&quot;, // LLM error
ENGINE_ERROR = &quot;engine_error&quot;, // Rust engine error
GATEWAY_ERROR = &quot;gateway_error&quot;, // HIP-4 unavailable
INTERNAL_ERROR = &quot;internal_error&quot;
}</code></pre>
<h2>Implementation</h2>
<h3>Repository Structure</h3>
<pre><code>search/ # Rust search engine
├── crates/
│ ├── meilisearch/ # HTTP server (Actix)
│ ├── milli/ # Core indexing + search engine
│ ├── index-scheduler/ # Async task queue for indexing
│ ├── filter-parser/ # Query filter DSL parser
│ ├── meilisearch-auth/ # API key + tenant management
│ ├── meilisearch-types/ # Shared type definitions
│ ├── http-client/ # HTTP client utilities
│ ├── dump/ # Index snapshot/restore
│ ├── file-store/ # File-backed storage
│ ├── flatten-serde-json/ # JSON flattening for indexing
│ ├── json-depth-checker/ # JSON validation
│ ├── meili-snap/ # Snapshot testing
│ ├── meilitool/ # CLI admin tool
│ ├── openapi-generator/ # API spec generation
│ ├── benchmarks/ # Performance benchmarks
│ ├── fuzzers/ # Fuzz testing
│ ├── tracing-trace/ # Distributed tracing
│ ├── build-info/ # Build metadata
│ └── xtask/ # Build automation
├── external-crates/
│ ├── async-openai/ # LLM client for embeddings
│ └── reqwest-eventsource/ # SSE client
├── workloads/ # Test datasets
├── Cargo.toml # Workspace root
├── Dockerfile # Multi-stage build
└── config.toml # Default configuration</code></pre>
<p>The generative UI layer lives in a separate deployment (Next.js application) that connects to the engine and LLM Gateway.</p>
<h3>Deployment</h3>
<h4>Docker Compose</h4>
<pre><code class="lang-yaml"># compose.yml
services:
search-ui:
image: hanzoai/search-ui:latest
ports:
- &quot;3000:3000&quot;
environment:
- SEARCH_ENGINE_URL=http://search-engine:7700
- LLM_GATEWAY_URL=http://gateway:4000
- SUPABASE_URL=${SUPABASE_URL}
- SUPABASE_ANON_KEY=${SUPABASE_ANON_KEY}
- BRAVE_SEARCH_API_KEY=${BRAVE_SEARCH_API_KEY}
- IAM_OAUTH_CLIENT_ID=${IAM_OAUTH_CLIENT_ID}
- IAM_OAUTH_CLIENT_SECRET=${IAM_OAUTH_CLIENT_SECRET}
depends_on:
- search-engine
- gateway
search-engine:
image: hanzoai/search:latest
ports:
- &quot;7700:7700&quot;
volumes:
- search_data:/meili_data
environment:
- MEILI_MASTER_KEY=${SEARCH_MASTER_KEY}
- MEILI_ENV=production
gateway:
image: hanzoai/llm-gateway:latest
ports:
- &quot;4000:4000&quot;
environment:
- OPENAI_API_KEY=${OPENAI_API_KEY}
- ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY}
supabase-db:
image: supabase/postgres:15
ports:
- &quot;5432:5432&quot;
volumes:
- pg_data:/var/lib/postgresql/data
environment:
- POSTGRES_DB=search
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
volumes:
search_data:
pg_data:</code></pre>
<h4>Quick Start</h4>
<pre><code class="lang-bash"># Clone and start
git clone https://github.com/hanzoai/search
cd search
# Engine only (Rust)
cargo build --release
./target/release/hanzo-search --master-key=&quot;your-key&quot;
# Full stack with generative UI
cp .env.example .env
# Edit .env with API keys
docker compose up -d
# Index documents
curl -X POST &#x27;http://localhost:7700/indexes/docs/documents&#x27; \
-H &#x27;Authorization: Bearer your-key&#x27; \
-H &#x27;Content-Type: application/json&#x27; \
--data-binary @documents.json
# Generative search
curl -N &#x27;http://localhost:3000/api/search&#x27; \
-H &#x27;Accept: text/event-stream&#x27; \
-H &#x27;Content-Type: application/json&#x27; \
-d &#x27;{&quot;query&quot;: &quot;How does hybrid search work?&quot;}&#x27;</code></pre>
<h3>SDKs</h3>
<p>| Language | Package | Repository | |----------|---------|------------| | Go | <code>github.com/hanzoai/search-go</code> | <a href="https://github.com/hanzoai/search-go">hanzoai/search-go</a> | | JavaScript / TypeScript | <code>@hanzo/search</code> | <a href="https://github.com/hanzoai/search-js">hanzoai/search-js</a> | | Python | <code>hanzo-search</code> | <a href="https://github.com/hanzoai/search-python">hanzoai/search-python</a> | | Rust | <code>hanzo-search</code> | <a href="https://github.com/hanzoai/search-rust">hanzoai/search-rust</a> |</p>
<p>SDK usage for generative search:</p>
<pre><code class="lang-typescript">import { HanzoSearch } from &#x27;@hanzo/search&#x27;;
const search = new HanzoSearch({
uiUrl: &#x27;https://search.hanzo.ai&#x27;,
engineUrl: &#x27;https://search-engine.hanzo.ai&#x27;,
apiKey: &#x27;sk-hanzo-...&#x27;,
});
// Generative search with streaming
const stream = await search.query(&#x27;What is hybrid search?&#x27;, {
mode: &#x27;docs&#x27;,
stream: true,
});
for await (const event of stream) {
if (event.type === &#x27;token&#x27;) process.stdout.write(event.content);
if (event.type === &#x27;citation&#x27;) console.log(`[${event.index}] ${event.title}`);
if (event.type === &#x27;done&#x27;) console.log(`\n${event.sources_used} sources, ${event.latency_ms}ms`);
}
// Direct engine search (non-generative)
const results = await search.engine.index(&#x27;docs&#x27;).search(&#x27;hybrid search&#x27;, {
limit: 20,
filter: &#x27;category = &quot;architecture&quot;&#x27;,
});</code></pre>
<h2>Security Considerations</h2>
<ol><li><strong>Query sanitization</strong>: Strip injection attempts before forwarding to engine or LLM</li><li><strong>Source validation</strong>: Verify URLs and content from web sources before displaying</li><li><strong>API key isolation</strong>: Engine API keys scoped to specific indexes and actions</li><li><strong>PII filtering</strong>: Optionally strip personal data from queries before web search</li><li><strong>Content filtering</strong>: Flag and filter harmful content in synthesized answers</li><li><strong>Rate limiting</strong>: Per-user and per-key limits at both UI and engine layers</li><li><strong>TLS everywhere</strong>: All inter-service communication over TLS</li><li><strong>Audit logging</strong>: Log all queries with user attribution for compliance</li></ol>
<h2>Performance Targets</h2>
<p>| Metric | Target | Notes | |--------|--------|-------| | Engine query latency | &lt; 50ms p95 | Full-text + vector hybrid | | Time to first token | &lt; 500ms | Including retrieval + LLM cold start | | Full answer latency | &lt; 3s p95 | End-to-end for typical web query | | Concurrent queries | &gt; 500 | Per engine instance | | Index throughput | &gt; 10K docs/sec | Bulk indexing | | Max index size | 100GB | Per index, configurable |</p>
<h2>Testing</h2>
<pre><code class="lang-bash"># Engine unit tests (Rust)
cargo test --workspace
# Engine benchmarks
cargo bench --package benchmarks
# UI unit tests
pnpm test
# Integration tests (full pipeline)
pnpm test:integration
# Load testing
k6 run tests/load/search-pipeline.js</code></pre>
<h2>References</h2>
<ol><li><a href="./hip-0004-llm-gateway-unified-ai-provider-interface.md">HIP-4: LLM Gateway</a></li><li><a href="./hip-0010-model-context-protocol-mcp-integration-standards.md">HIP-10: Model Context Protocol</a></li><li><a href="./hip-0011-chat-interface-standard.md">HIP-11: Chat Interface Standard</a></li><li><a href="./hip-0026-identity-access-management-standard.md">HIP-26: Identity and Access Management</a></li><li><a href="https://github.com/hanzoai/search">Hanzo Search Engine Repository</a></li><li><a href="https://www.meilisearch.com/docs">Meilisearch Documentation</a></li></ol>
<h2>Copyright</h2>
<p>Copyright and related rights waived via <a href="https://creativecommons.org/publicdomain/zero/1.0/">CC0</a>.</p>
<footer>
<div class="fmap">
<div><b>Build</b>
<a href="https://hanzo.app">hanzo.app</a> app builder
<a href="https://hanzo.codes">hanzo.codes</a> code with an agent
<a href="https://hanzoskills.com">hanzoskills.com</a> what an agent reads</div>
<div><b>Run</b>
<a href="https://console.hanzo.ai">console.hanzo.ai</a> cloud console
<a href="https://api.hanzo.ai">api.hanzo.ai</a> the API</div>
<div><b>Account</b>
<a href="https://hanzo.id">hanzo.id</a> sign in
<a href="https://pay.hanzo.ai">pay.hanzo.ai</a> top up
<a href="https://billing.hanzo.ai">billing.hanzo.ai</a> invoices and usage</div>
<div><b>Company</b>
<a href="https://hanzo.ai">hanzo.ai</a> canonical
<a href="https://hanzo.ventures">hanzo.ventures</a> ventures</div>
</div>
<div class="fend"><a href="/hip-0012.md">This page as plain text</a> · <a href="/llms.txt">Index for agents</a></div>
</footer>
</div>
</body></html>