mirror of
https://github.com/hanzoai/skills.git
synced 2026-11-19 00:47:36 +00:00
732 lines
35 KiB
HTML
732 lines
35 KiB
HTML
<!doctype html>
|
|
<html lang="en"><head>
|
|
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
<title>hip-0012</title>
|
|
<meta name="color-scheme" content="dark">
|
|
<style>
|
|
/* Committed to one visual world: a dark instrument panel. No light variant —
|
|
the dot field and the hairline rules only read on black, and a naive
|
|
inversion would wash both out. Every colour is painted explicitly so the page
|
|
never borrows a host background. */
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Pixel Square';src:url('/fonts/ZenPixel-Square.woff2') format('woff2');font-style:normal;font-display:swap}
|
|
|
|
:root{
|
|
--ink:#f4f4f2; --soft:#9c9c98; --faint:#6a6a66; --ground:#000; --panel:#0a0a0a;
|
|
--line:#232323; --line-hi:#3a3a3a; --accent:#f4f4f2;
|
|
--font-zen-sans:'Zen',-apple-system,BlinkMacSystemFont,"Segoe UI",Helvetica,Arial,sans-serif;
|
|
--font-zen-mono:'Zen Mono',ui-monospace,"SF Mono",SFMono-Regular,Menlo,Consolas,monospace;
|
|
--font-zen-pixel-square:'Zen Pixel Square',monospace;
|
|
--mono:var(--font-zen-mono);
|
|
--sans:var(--font-zen-sans);
|
|
/* Controls are pills and surfaces are soft, as on hanzo.ai. */
|
|
--r:999px; --r-soft:14px;
|
|
}
|
|
*{box-sizing:border-box}
|
|
html{-webkit-text-size-adjust:100%}
|
|
body{margin:0;background:var(--ground);color:var(--ink);font-family:var(--sans);
|
|
font-size:16px;line-height:1.65;-webkit-font-smoothing:antialiased}
|
|
.wrap{max-width:60rem;margin:0 auto;padding:0 1.75rem}
|
|
|
|
/* The dot field. A halftone made of one repeating radial-gradient, faded out
|
|
along the diagonal by a mask so it reads as a wave rather than wallpaper. */
|
|
/* No overflow clip here: the dot field is inset to the box already, and a clip
|
|
would cut off the copy-prompt menu. */
|
|
.hero{position:relative;border-bottom:1px solid var(--line)}
|
|
.hero::before{content:"";position:absolute;inset:0;
|
|
background-image:radial-gradient(var(--line-hi) 1px, transparent 1px);
|
|
background-size:7px 7px;
|
|
-webkit-mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
pointer-events:none}
|
|
.hero .wrap{position:relative;padding-top:5.5rem;padding-bottom:4rem}
|
|
|
|
nav{border-bottom:1px solid var(--line)}
|
|
nav .in{max-width:60rem;margin:0 auto;padding:1.1rem 1.75rem;display:flex;
|
|
justify-content:space-between;align-items:center;gap:1rem}
|
|
.brand{font-family:var(--mono);font-size:.85rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--ink);text-decoration:none;
|
|
display:inline-flex;align-items:center;gap:.6rem}
|
|
.brand i{font-style:normal;color:var(--faint)}
|
|
.mark{width:1.15em;height:1.15em;display:block;flex:0 0 auto}
|
|
nav a.up{font-family:var(--mono);font-size:.72rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--soft);text-decoration:none}
|
|
nav a.up:hover{color:var(--ink)}
|
|
|
|
h1{font-family:var(--mono);font-size:clamp(2rem,5.4vw,3.4rem);line-height:1.12;
|
|
letter-spacing:-.02em;margin:0 0 1.1rem;font-weight:500;text-wrap:balance}
|
|
h1 em{font-style:normal;color:var(--faint)}
|
|
.lede{color:var(--soft);font-size:1.05rem;max-width:52ch;margin:0 0 2.25rem}
|
|
|
|
.btns{display:flex;flex-wrap:wrap;gap:.7rem;margin:0 0 .5rem;align-items:center}
|
|
.btn{display:inline-flex;align-items:center;gap:.5rem;font-family:var(--mono);
|
|
font-size:.72rem;letter-spacing:.14em;text-transform:uppercase;
|
|
padding:.8rem 1.4rem;border:1px solid var(--line-hi);border-radius:var(--r);
|
|
color:var(--ink);text-decoration:none;background:transparent;cursor:pointer;
|
|
line-height:1.2;font-weight:400}
|
|
.btn:hover{background:#141414;text-decoration:none;border-color:var(--soft)}
|
|
.btn.solid{background:var(--ink);color:#000;border-color:var(--ink)}
|
|
.btn.solid:hover{background:#d8d8d4}
|
|
.btn .chev{width:.7em;height:.7em;flex:0 0 auto;transition:transform .15s ease}
|
|
.btn[aria-expanded="true"] .chev{transform:rotate(180deg)}
|
|
|
|
/* Copy-prompt selector. Each row copies the prompt for its own target. */
|
|
.copy{position:relative;display:inline-flex}
|
|
.menu{position:absolute;top:calc(100% + .5rem);left:0;z-index:20;min-width:21rem;
|
|
background:var(--panel);border:1px solid var(--line-hi);border-radius:var(--r-soft);
|
|
padding:.4rem;display:flex;flex-direction:column;
|
|
box-shadow:0 12px 32px rgba(0,0,0,.75)}
|
|
.menu[hidden]{display:none}
|
|
.mi{display:flex;align-items:center;gap:.7rem;width:100%;text-align:left;
|
|
padding:.62rem .8rem;border:0;border-radius:calc(var(--r-soft) - .35rem);
|
|
background:transparent;color:var(--ink);font-family:var(--mono);font-size:.76rem;
|
|
letter-spacing:.04em;text-decoration:none;cursor:pointer;line-height:1.3;
|
|
white-space:nowrap}
|
|
.mi:hover,.mi:focus-visible{background:#181818;outline:none}
|
|
.mi .mark{width:1.05rem;height:1.05rem;color:var(--soft)}
|
|
.mi:hover .mark{color:var(--ink)}
|
|
.mi u{margin-left:auto;color:var(--faint);text-decoration:none;font-size:.68rem;
|
|
letter-spacing:.1em;text-transform:uppercase}
|
|
|
|
/* Works-with bar: one scrolling row, name and mark per agent. */
|
|
.strip{border-bottom:1px solid var(--line);background:var(--panel)}
|
|
.strip .in{max-width:60rem;margin:0 auto;padding:.9rem 1.75rem;display:flex;
|
|
align-items:center;gap:1.25rem;font-family:var(--mono);font-size:.72rem;
|
|
letter-spacing:.1em;text-transform:uppercase;color:var(--faint)}
|
|
.strip b{color:var(--soft);font-weight:400;flex:0 0 auto;
|
|
border-right:1px solid var(--line);padding-right:1.25rem;align-self:stretch;
|
|
display:flex;align-items:center}
|
|
.bar{display:flex;align-items:center;gap:.5rem;flex:1 1 auto;min-width:0;
|
|
overflow-x:auto;scrollbar-width:none;scroll-snap-type:x proximity;
|
|
-webkit-overflow-scrolling:touch;
|
|
-webkit-mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent);
|
|
mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent)}
|
|
.bar::-webkit-scrollbar{display:none}
|
|
.agent{display:inline-flex;align-items:center;gap:.45rem;flex:0 0 auto;
|
|
padding:.4rem .85rem;border:1px solid var(--line);border-radius:var(--r);
|
|
color:var(--soft);white-space:nowrap;scroll-snap-align:start;
|
|
text-decoration:none;background:var(--ground)}
|
|
.agent:hover{color:var(--ink);border-color:var(--line-hi);text-decoration:none}
|
|
.agent .mark{width:.95rem;height:.95rem}
|
|
|
|
h2{font-family:var(--mono);font-size:1.1rem;letter-spacing:-.01em;margin:2.75rem 0 .75rem;font-weight:500}
|
|
h3{font-size:1rem;margin:2rem 0 .5rem;font-weight:600}
|
|
h4,h5,h6{font-size:.94rem;margin:1.5rem 0 .5rem;font-weight:600}
|
|
p{margin:0 0 1rem;max-width:70ch}
|
|
ul,ol{margin:0 0 1rem;padding-left:1.35rem;max-width:70ch}
|
|
li{margin:.22rem 0}
|
|
a{color:var(--ink);text-decoration:underline;text-underline-offset:3px;
|
|
text-decoration-color:var(--line-hi)}
|
|
a:hover{text-decoration-color:var(--ink)}
|
|
a:focus-visible{outline:1px solid var(--ink);outline-offset:3px}
|
|
code{font-family:var(--mono);font-size:.85em;background:var(--panel);
|
|
border:1px solid var(--line);border-radius:5px;padding:.1em .36em}
|
|
pre{background:var(--panel);border:1px solid var(--line);padding:1rem 1.1rem;
|
|
border-radius:var(--r-soft);overflow-x:auto;margin:0 0 1.25rem}
|
|
pre code{background:none;border:0;padding:0;font-size:.82rem;line-height:1.7}
|
|
blockquote{margin:0 0 1rem;padding-left:1rem;border-left:1px solid var(--line-hi);color:var(--soft)}
|
|
hr{border:0;border-top:1px solid var(--line);margin:2.5rem 0}
|
|
table{border-collapse:collapse;width:100%;margin:0 0 1.25rem;font-size:.92rem;
|
|
display:block;overflow-x:auto}
|
|
th,td{border:1px solid var(--line);padding:.5rem .7rem;text-align:left}
|
|
|
|
.curl{display:block;background:var(--panel);border:1px solid var(--line);
|
|
border-radius:var(--r-soft);padding:.95rem 1.1rem;font-family:var(--mono);
|
|
font-size:.8rem;color:var(--soft);overflow-x:auto;margin:0 0 2.5rem;
|
|
white-space:pre;line-height:1.9}
|
|
.curl b{color:var(--ink);font-weight:400}
|
|
.curl i{font-style:normal;color:var(--faint)}
|
|
|
|
/* index */
|
|
#q{width:100%;padding:.85rem 1.2rem;font-family:var(--mono);font-size:.85rem;color:var(--ink);
|
|
background:var(--panel);border:1px solid var(--line);border-radius:var(--r);margin:0 0 .6rem}
|
|
#q::placeholder{color:var(--faint);text-transform:uppercase;letter-spacing:.1em;font-size:.72rem}
|
|
#q:focus{outline:none;border-color:var(--soft)}
|
|
#count{color:var(--faint);font-family:var(--mono);font-size:.72rem;letter-spacing:.1em;
|
|
text-transform:uppercase;margin:0 0 2rem}
|
|
.cat{margin:2.5rem 0 .5rem;font-family:var(--mono);font-size:.7rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);border-bottom:1px solid var(--line);
|
|
padding-bottom:.45rem}
|
|
.row{display:block;padding:.6rem .9rem;margin:0 -.9rem;border-bottom:1px solid var(--line);
|
|
border-radius:var(--r-soft);text-decoration:none}
|
|
.row:hover{background:#0a0a0a}
|
|
.nm{font-family:var(--mono);font-size:.84rem;color:var(--ink);display:block}
|
|
.ds{color:var(--soft);font-size:.9rem;display:block;margin-top:.12rem}
|
|
footer{margin-top:4rem;padding:2rem 0 4rem;border-top:1px solid var(--line);
|
|
color:var(--faint);font-size:.85rem}
|
|
footer a{color:var(--soft)}
|
|
.fmap{display:grid;gap:1.75rem;grid-template-columns:repeat(auto-fit,minmax(11rem,1fr));
|
|
margin:0 0 2rem;font-size:.85rem}
|
|
.fmap b{display:block;font-family:var(--mono);font-size:.68rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);font-weight:400;margin:0 0 .6rem;
|
|
border-bottom:1px solid var(--line);padding-bottom:.4rem}
|
|
.fmap a{display:inline-block;color:var(--ink);text-decoration:none;margin:.3rem .35rem 0 0}
|
|
.fmap a:hover{text-decoration:underline}
|
|
.fmap div{color:var(--faint);line-height:1.5}
|
|
.fend{border-top:1px solid var(--line);padding-top:1.25rem;font-size:.82rem}
|
|
|
|
@media (max-width:640px){
|
|
.hero .wrap{padding-top:3.25rem;padding-bottom:2.75rem}
|
|
.btns{gap:.5rem}
|
|
.btn{padding:.75rem 1.15rem;font-size:.68rem}
|
|
.copy,.copy .btn{width:100%;justify-content:center}
|
|
.menu{left:0;right:0;min-width:0}
|
|
.strip .in{flex-direction:column;align-items:flex-start;gap:.6rem;padding:.9rem 1.75rem}
|
|
.strip b{border-right:0;padding-right:0}
|
|
.bar{width:100%}
|
|
}
|
|
</style>
|
|
<link rel="alternate" type="text/markdown" href="/hip-0012.md">
|
|
|
|
<link rel="icon" href="/favicon.svg">
|
|
<link rel="canonical" href="https://hanzoskills.com/hip-0012">
|
|
</head>
|
|
<!-- Reading this with a machine? Every page here is also plain markdown.
|
|
This one: https://hanzoskills.com/hip-0012.md
|
|
Index: https://hanzoskills.com/llms.txt
|
|
Corpus: https://hanzoskills.com/all.md
|
|
Append .md to any path for the markdown twin. -->
|
|
<body>
|
|
<nav><div class="in">
|
|
<a class="brand" href="/"><svg class="mark" viewBox="0 0 67 67" fill="currentColor" aria-hidden="true"><path d="M22.21 67V44.6369H0V67H22.21Z"/><path d="M66.7038 22.3184H22.2534L0.0878906 44.6367H44.4634L66.7038 22.3184Z"/><path d="M22.21 0H0V22.3184H22.21V0Z"/><path d="M66.7198 0H44.5098V22.3184H66.7198V0Z"/><path d="M66.7198 67V44.6369H44.5098V67H66.7198Z"/></svg> HANZO<i>SKILLS</i></a>
|
|
<a class="up" href="/">All skills</a>
|
|
</div></nav>
|
|
|
|
<div class="wrap">
|
|
<h1>hip-0012</h1><p class="lede">HIP-12: Search Interface Standard. Status Draft. Hanzo's own standard — read this before implementing against it.</p><h1>HIP-0012: Search Interface Standard</h1>
|
|
<h2>Abstract</h2>
|
|
<p>This proposal defines the search interface standard for the Hanzo ecosystem. It specifies a two-layer architecture: a high-performance Rust search engine for indexing, retrieval, and hybrid search, and a generative search UI layer that synthesizes AI answers with inline citations from multiple source types. All search functionality in the Hanzo ecosystem MUST implement this interface.</p>
|
|
<p><strong>Engine Repository</strong>: <a href="https://github.com/hanzoai/search">github.com/hanzoai/search</a> <strong>Engine Port</strong>: 7700 <strong>UI Port</strong>: 3000 <strong>Engine Runtime</strong>: Rust (Hanzo Search fork, v1.36.0) <strong>UI Runtime</strong>: Next.js + Supabase</p>
|
|
<h2>Design Philosophy</h2>
|
|
<h3>Why Generative Search UI</h3>
|
|
<p>Traditional search returns ten blue links. The user clicks through, reads, evaluates, and synthesizes an answer themselves. This model was designed in the 1990s for a web of static pages.</p>
|
|
<p>Generative search inverts this. The system retrieves relevant documents, synthesizes a coherent answer, and presents it with citations the user can verify. The user gets an answer, not a reading list. Time-to-answer drops from minutes to seconds.</p>
|
|
<p>This is the paradigm shift from Google-style search to AI-native search. The search engine becomes a retrieval backend; the LLM becomes the synthesis frontend. The UI streams the answer in real time, with numbered citations linking to source material.</p>
|
|
<h2>Specification</h2>
|
|
<h3>System Architecture</h3>
|
|
<pre><code>┌─────────────┐ ┌──────────────────┐ ┌──────────────┐
|
|
│ Browser │────▶│ Search UI │────▶│ LLM Gateway │
|
|
│ Client │◀────│ (Next.js:3000) │◀────│ (HIP-4) │
|
|
└─────────────┘ └──────────────────┘ └──────────────┘
|
|
│ │ │
|
|
┌──────┘ └──────┐ ▼
|
|
▼ ▼ ┌──────────┐
|
|
┌──────────────┐ ┌────────────┐ │ 100+ │
|
|
│ Search Engine│ │ Supabase │ │ Providers│
|
|
│ (Rust:7700) │ │ (pgvector)│ └──────────┘
|
|
└──────────────┘ └────────────┘
|
|
│ │
|
|
▼ ▼
|
|
┌──────────────┐ ┌────────────┐
|
|
│ Index Storage│ │ SQL │
|
|
│ (LMDB) │ │ (Auth/Hist)│
|
|
└──────────────┘ └────────────┘</code></pre>
|
|
<h3>Query Pipeline</h3>
|
|
<p>The generative search pipeline processes a user query through five stages:</p>
|
|
<pre><code>1. PARSE User query → intent classification + query rewriting
|
|
2. RETRIEVE Parallel retrieval from multiple source types
|
|
3. RANK Score and deduplicate results across sources
|
|
4. SYNTHESIZE LLM generates answer with inline citations
|
|
5. STREAM Streaming markdown delivered to client via SSE</code></pre>
|
|
<h4>Stage 1: Parse</h4>
|
|
<pre><code class="lang-typescript">interface ParsedQuery {
|
|
original: string; // Raw user input
|
|
rewritten: string; // LLM-rewritten for retrieval
|
|
intent: SearchIntent; // web | academic | code | news | docs
|
|
entities: string[]; // Extracted named entities
|
|
embedding: number[]; // Query embedding vector (1536-dim)
|
|
}
|
|
|
|
type SearchIntent = "web" | "academic" | "code" | "news" | "docs";</code></pre>
|
|
<p>The query rewriter uses a lightweight LLM call (or rule-based heuristics for common patterns) to transform conversational queries into retrieval-optimized forms. "What's the best way to deploy a Rust binary on Kubernetes?" becomes the embedding query "Rust binary Kubernetes deployment" and the web query "deploy Rust application Kubernetes best practices".</p>
|
|
<h4>Stage 2: Retrieve</h4>
|
|
<p>Retrieval runs in parallel across configured source types:</p>
|
|
<pre><code class="lang-typescript">interface RetrievalSource {
|
|
type: "engine" | "web" | "vector" | "knowledge_base";
|
|
weight: number; // 0.0-1.0, used in ranking
|
|
config: SourceConfig;
|
|
}
|
|
|
|
// Source configurations
|
|
interface EngineSource {
|
|
type: "engine";
|
|
endpoint: string; // Hanzo Search engine (port 7700)
|
|
index: string; // Target index name
|
|
limit: number;
|
|
}
|
|
|
|
interface WebSource {
|
|
type: "web";
|
|
provider: "brave" | "google" | "bing";
|
|
api_key: string;
|
|
limit: number;
|
|
}
|
|
|
|
interface VectorSource {
|
|
type: "vector";
|
|
connection: string; // pgvector connection string
|
|
table: string;
|
|
embedding_column: string;
|
|
similarity: "cosine" | "l2" | "inner_product";
|
|
limit: number;
|
|
}
|
|
|
|
interface KnowledgeBaseSource {
|
|
type: "knowledge_base";
|
|
id: string; // KB identifier
|
|
endpoint: string; // API endpoint
|
|
limit: number;
|
|
}</code></pre>
|
|
<h4>Stage 3: Rank</h4>
|
|
<p>Results from all sources are scored and merged:</p>
|
|
<pre><code class="lang-typescript">interface RankedResult {
|
|
id: string;
|
|
title: string;
|
|
url: string;
|
|
snippet: string; // Relevant text excerpt
|
|
content: string; // Full text (for LLM context)
|
|
source_type: string; // Which retrieval source
|
|
source_name: string; // Display name (e.g., "Brave Search")
|
|
score: number; // Normalized 0.0-1.0
|
|
metadata: Record<string, any>;
|
|
}</code></pre>
|
|
<p>Scoring combines:</p>
|
|
<ul><li><strong>Relevance</strong>: BM25 or cosine similarity from the source</li><li><strong>Source weight</strong>: Configured per-source priority</li><li><strong>Freshness</strong>: Recency bonus for time-sensitive queries</li><li><strong>Authority</strong>: Domain reputation score for web results</li></ul>
|
|
<p>Deduplication uses URL normalization and content fingerprinting (SimHash) to collapse near-duplicate results.</p>
|
|
<h4>Stage 4: Synthesize</h4>
|
|
<p>The top-K ranked results are assembled into an LLM prompt:</p>
|
|
<pre><code class="lang-typescript">interface SynthesisRequest {
|
|
query: string;
|
|
results: RankedResult[]; // Top-K (default: 10)
|
|
mode: SearchIntent;
|
|
model?: string; // Override model selection
|
|
max_tokens?: number; // Default: 2048
|
|
}</code></pre>
|
|
<p>The synthesis prompt instructs the model to:</p>
|
|
<ol><li>Answer the query using ONLY information from the provided sources</li><li>Cite sources using <code>[N]</code> notation, where N maps to the source index</li><li>Structure the answer with markdown headings when appropriate</li><li>Include a brief "Key takeaway" at the top for quick scanning</li><li>Flag when sources conflict or information is uncertain</li></ol>
|
|
<h4>Stage 5: Stream</h4>
|
|
<p>The response streams to the client using Server-Sent Events:</p>
|
|
<pre><code class="lang-typescript">// Answer token
|
|
event: token
|
|
data: {"content": "Rust binaries can be deployed", "index": 0}
|
|
|
|
// Citation reference (emitted when [N] appears in stream)
|
|
event: citation
|
|
data: {"index": 1, "source_id": "src_abc", "url": "https://...", "title": "..."}
|
|
|
|
// Related questions
|
|
event: related
|
|
data: {"questions": ["How to optimize Rust binary size?", "Kubernetes pod resource limits for Rust"]}
|
|
|
|
// Completion
|
|
event: done
|
|
data: {
|
|
"query_id": "q_789",
|
|
"model": "zen-32b",
|
|
"tokens": {"prompt": 4200, "completion": 680, "total": 4880},
|
|
"sources_used": 6,
|
|
"latency_ms": 2340
|
|
}
|
|
|
|
// Error
|
|
event: error
|
|
data: {"code": "synthesis_failed", "message": "LLM returned empty response"}</code></pre>
|
|
<h3>Core Data Models</h3>
|
|
<h4>Search Request</h4>
|
|
<pre><code class="lang-typescript">interface SearchRequest {
|
|
// Required
|
|
query: string;
|
|
|
|
// Optional configuration
|
|
mode?: SearchIntent; // Default: auto-detect
|
|
sources?: string[]; // Source type filter
|
|
limit?: number; // Max sources to retrieve (default: 10)
|
|
model?: string; // LLM model override
|
|
stream?: boolean; // Default: true
|
|
language?: string; // ISO 639-1 code
|
|
|
|
// Filters
|
|
domain_filter?: string[]; // Include only these domains
|
|
domain_block?: string[]; // Exclude these domains
|
|
date_range?: {
|
|
from?: string; // ISO 8601
|
|
to?: string;
|
|
};
|
|
|
|
// Context
|
|
user_id?: string; // For personalization
|
|
session_id?: string; // For search history grouping
|
|
}</code></pre>
|
|
<h4>Search Response</h4>
|
|
<pre><code class="lang-typescript">interface SearchResponse {
|
|
// Identifiers
|
|
query_id: string;
|
|
query: string;
|
|
|
|
// Generated answer
|
|
answer: string; // Markdown with [N] citations
|
|
answer_tokens: number;
|
|
|
|
// Sources used in synthesis
|
|
sources: Source[];
|
|
|
|
// Auto-generated follow-up queries
|
|
related_questions: string[];
|
|
|
|
// Metadata
|
|
model: string;
|
|
mode: SearchIntent;
|
|
latency_ms: number;
|
|
tokens: {
|
|
prompt: number;
|
|
completion: number;
|
|
total: number;
|
|
};
|
|
}
|
|
|
|
interface Source {
|
|
index: number; // Citation number [N]
|
|
title: string;
|
|
url: string;
|
|
snippet: string;
|
|
favicon?: string;
|
|
source_type: string;
|
|
published_date?: string;
|
|
}</code></pre>
|
|
<h3>API Endpoints</h3>
|
|
<h4>Search</h4>
|
|
<pre><code class="lang-yaml"># Generative search (primary endpoint)
|
|
POST /api/search
|
|
Headers:
|
|
Authorization: Bearer <token>
|
|
Accept: text/event-stream # For SSE streaming
|
|
Body: SearchRequest
|
|
Response: Stream of SSE events (see Stage 5)
|
|
|
|
# Non-streaming variant
|
|
POST /api/search
|
|
Headers:
|
|
Authorization: Bearer <token>
|
|
Accept: application/json
|
|
Body: SearchRequest
|
|
Response: SearchResponse</code></pre>
|
|
<h4>Sources</h4>
|
|
<pre><code class="lang-yaml"># Get sources for a completed search
|
|
GET /api/search/:query_id/sources
|
|
Response:
|
|
sources: Source[]
|
|
total: number
|
|
|
|
# Get full content of a specific source
|
|
GET /api/search/:query_id/sources/:index
|
|
Response:
|
|
source: Source
|
|
full_content: string</code></pre>
|
|
<h4>Search History</h4>
|
|
<pre><code class="lang-yaml"># List recent searches
|
|
GET /api/search/history
|
|
Query:
|
|
page?: number
|
|
limit?: number (default: 20)
|
|
mode?: SearchIntent
|
|
Response:
|
|
searches: SearchHistoryEntry[]
|
|
total: number
|
|
|
|
# Delete search history
|
|
DELETE /api/search/history
|
|
Response:
|
|
success: boolean</code></pre>
|
|
<h4>Engine Proxy (Direct Hanzo Search API)</h4>
|
|
<pre><code class="lang-yaml"># Index documents (proxied to engine on port 7700)
|
|
POST /api/engine/indexes/:index/documents
|
|
Headers:
|
|
Authorization: Bearer <master-key>
|
|
Body: Document[]
|
|
Response:
|
|
taskUid: number
|
|
status: "enqueued"
|
|
|
|
# Direct search (non-generative, engine only)
|
|
POST /api/engine/indexes/:index/search
|
|
Body:
|
|
q: string
|
|
limit?: number
|
|
filter?: string
|
|
sort?: string[]
|
|
attributesToRetrieve?: string[]
|
|
Response:
|
|
hits: Document[]
|
|
query: string
|
|
processingTimeMs: number
|
|
estimatedTotalHits: number</code></pre>
|
|
<h3>Search Modes</h3>
|
|
<p>Each mode adjusts source weights, synthesis prompts, and UI layout:</p>
|
|
<p>| Mode | Primary Sources | Synthesis Style | UI Features | |------|----------------|-----------------|-------------| | <strong>web</strong> | Brave/Google + engine | Balanced summary | Source cards, images | | <strong>academic</strong> | Scholar APIs + engine | Formal with methodology | Paper cards, DOI links | | <strong>code</strong> | GitHub + engine + docs | Code examples with explanation | Syntax highlighting, copy buttons | | <strong>news</strong> | News APIs + engine | Timeline-aware, multi-perspective | Date grouping, outlet badges | | <strong>docs</strong> | Internal knowledge bases | Precise, step-by-step | Code blocks, API examples |</p>
|
|
<p>Mode is auto-detected from query content but can be overridden in the request.</p>
|
|
<h3>Engine Configuration</h3>
|
|
<p>The Rust search engine accepts configuration via CLI flags, environment variables, or a config file:</p>
|
|
<pre><code class="lang-toml"># config.toml
|
|
[server]
|
|
http_addr = "0.0.0.0:7700"
|
|
master_key = "HANZO_SEARCH_MASTER_KEY"
|
|
max_index_size = "100GB"
|
|
max_task_db_size = "10GB"
|
|
|
|
[indexing]
|
|
max_indexing_memory = "2GiB"
|
|
max_indexing_threads = 4
|
|
|
|
[experimental]
|
|
enable_metrics = true
|
|
enable_logs_route = true
|
|
vector_store = true # Enable hybrid search</code></pre>
|
|
<h3>Authentication and Authorization</h3>
|
|
<p>Search uses a layered auth model:</p>
|
|
<ol><li><strong>UI Layer</strong>: Supabase Auth federated through IAM (HIP-26) via OAuth</li><li><strong>Engine Layer</strong>: API key-based with tenant isolation</li><li><strong>LLM Gateway</strong>: Bearer token forwarded from UI session</li></ol>
|
|
<pre><code class="lang-typescript">interface SearchAPIKey {
|
|
uid: string;
|
|
key: string; // Hashed
|
|
name: string;
|
|
actions: SearchAction[];
|
|
indexes: string[]; // ["*"] for all
|
|
expires_at?: string; // ISO 8601
|
|
}
|
|
|
|
type SearchAction =
|
|
| "search" // Query indexes
|
|
| "documents.add" // Index documents
|
|
| "documents.delete" // Remove documents
|
|
| "indexes.create" // Create indexes
|
|
| "indexes.delete" // Delete indexes
|
|
| "settings.update" // Modify index settings
|
|
| "stats.get" // Read metrics
|
|
| "*"; // Admin</code></pre>
|
|
<h3>Rate Limiting</h3>
|
|
<pre><code class="lang-yaml">Tiers:
|
|
free:
|
|
searches_per_minute: 10
|
|
searches_per_day: 200
|
|
max_sources_per_query: 5
|
|
engine_documents: 10000
|
|
|
|
pro:
|
|
searches_per_minute: 60
|
|
searches_per_day: 5000
|
|
max_sources_per_query: 15
|
|
engine_documents: 1000000
|
|
|
|
enterprise:
|
|
searches_per_minute: 300
|
|
searches_per_day: unlimited
|
|
max_sources_per_query: 30
|
|
engine_documents: unlimited</code></pre>
|
|
<h3>Error Handling</h3>
|
|
<pre><code class="lang-typescript">interface SearchError {
|
|
error: {
|
|
code: SearchErrorCode;
|
|
message: string;
|
|
details?: any;
|
|
query_id?: string;
|
|
timestamp: number;
|
|
};
|
|
}
|
|
|
|
enum SearchErrorCode {
|
|
// Client errors
|
|
INVALID_QUERY = "invalid_query",
|
|
QUERY_TOO_LONG = "query_too_long", // > 2000 chars
|
|
UNAUTHORIZED = "unauthorized",
|
|
RATE_LIMITED = "rate_limited",
|
|
INVALID_MODE = "invalid_mode",
|
|
INDEX_NOT_FOUND = "index_not_found",
|
|
|
|
// Server errors
|
|
RETRIEVAL_FAILED = "retrieval_failed", // Source unavailable
|
|
SYNTHESIS_FAILED = "synthesis_failed", // LLM error
|
|
ENGINE_ERROR = "engine_error", // Rust engine error
|
|
GATEWAY_ERROR = "gateway_error", // HIP-4 unavailable
|
|
INTERNAL_ERROR = "internal_error"
|
|
}</code></pre>
|
|
<h2>Implementation</h2>
|
|
<h3>Repository Structure</h3>
|
|
<pre><code>search/ # Rust search engine
|
|
├── crates/
|
|
│ ├── meilisearch/ # HTTP server (Actix)
|
|
│ ├── milli/ # Core indexing + search engine
|
|
│ ├── index-scheduler/ # Async task queue for indexing
|
|
│ ├── filter-parser/ # Query filter DSL parser
|
|
│ ├── meilisearch-auth/ # API key + tenant management
|
|
│ ├── meilisearch-types/ # Shared type definitions
|
|
│ ├── http-client/ # HTTP client utilities
|
|
│ ├── dump/ # Index snapshot/restore
|
|
│ ├── file-store/ # File-backed storage
|
|
│ ├── flatten-serde-json/ # JSON flattening for indexing
|
|
│ ├── json-depth-checker/ # JSON validation
|
|
│ ├── meili-snap/ # Snapshot testing
|
|
│ ├── meilitool/ # CLI admin tool
|
|
│ ├── openapi-generator/ # API spec generation
|
|
│ ├── benchmarks/ # Performance benchmarks
|
|
│ ├── fuzzers/ # Fuzz testing
|
|
│ ├── tracing-trace/ # Distributed tracing
|
|
│ ├── build-info/ # Build metadata
|
|
│ └── xtask/ # Build automation
|
|
├── external-crates/
|
|
│ ├── async-openai/ # LLM client for embeddings
|
|
│ └── reqwest-eventsource/ # SSE client
|
|
├── workloads/ # Test datasets
|
|
├── Cargo.toml # Workspace root
|
|
├── Dockerfile # Multi-stage build
|
|
└── config.toml # Default configuration</code></pre>
|
|
<p>The generative UI layer lives in a separate deployment (Next.js application) that connects to the engine and LLM Gateway.</p>
|
|
<h3>Deployment</h3>
|
|
<h4>Docker Compose</h4>
|
|
<pre><code class="lang-yaml"># compose.yml
|
|
services:
|
|
search-ui:
|
|
image: hanzoai/search-ui:latest
|
|
ports:
|
|
- "3000:3000"
|
|
environment:
|
|
- SEARCH_ENGINE_URL=http://search-engine:7700
|
|
- LLM_GATEWAY_URL=http://gateway:4000
|
|
- SUPABASE_URL=${SUPABASE_URL}
|
|
- SUPABASE_ANON_KEY=${SUPABASE_ANON_KEY}
|
|
- BRAVE_SEARCH_API_KEY=${BRAVE_SEARCH_API_KEY}
|
|
- IAM_OAUTH_CLIENT_ID=${IAM_OAUTH_CLIENT_ID}
|
|
- IAM_OAUTH_CLIENT_SECRET=${IAM_OAUTH_CLIENT_SECRET}
|
|
depends_on:
|
|
- search-engine
|
|
- gateway
|
|
|
|
search-engine:
|
|
image: hanzoai/search:latest
|
|
ports:
|
|
- "7700:7700"
|
|
volumes:
|
|
- search_data:/meili_data
|
|
environment:
|
|
- MEILI_MASTER_KEY=${SEARCH_MASTER_KEY}
|
|
- MEILI_ENV=production
|
|
|
|
gateway:
|
|
image: hanzoai/llm-gateway:latest
|
|
ports:
|
|
- "4000:4000"
|
|
environment:
|
|
- OPENAI_API_KEY=${OPENAI_API_KEY}
|
|
- ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY}
|
|
|
|
supabase-db:
|
|
image: supabase/postgres:15
|
|
ports:
|
|
- "5432:5432"
|
|
volumes:
|
|
- pg_data:/var/lib/postgresql/data
|
|
environment:
|
|
- POSTGRES_DB=search
|
|
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
|
|
|
|
volumes:
|
|
search_data:
|
|
pg_data:</code></pre>
|
|
<h4>Quick Start</h4>
|
|
<pre><code class="lang-bash"># Clone and start
|
|
git clone https://github.com/hanzoai/search
|
|
cd search
|
|
|
|
# Engine only (Rust)
|
|
cargo build --release
|
|
./target/release/hanzo-search --master-key="your-key"
|
|
|
|
# Full stack with generative UI
|
|
cp .env.example .env
|
|
# Edit .env with API keys
|
|
docker compose up -d
|
|
|
|
# Index documents
|
|
curl -X POST 'http://localhost:7700/indexes/docs/documents' \
|
|
-H 'Authorization: Bearer your-key' \
|
|
-H 'Content-Type: application/json' \
|
|
--data-binary @documents.json
|
|
|
|
# Generative search
|
|
curl -N 'http://localhost:3000/api/search' \
|
|
-H 'Accept: text/event-stream' \
|
|
-H 'Content-Type: application/json' \
|
|
-d '{"query": "How does hybrid search work?"}'</code></pre>
|
|
<h3>SDKs</h3>
|
|
<p>| Language | Package | Repository | |----------|---------|------------| | Go | <code>github.com/hanzoai/search-go</code> | <a href="https://github.com/hanzoai/search-go">hanzoai/search-go</a> | | JavaScript / TypeScript | <code>@hanzo/search</code> | <a href="https://github.com/hanzoai/search-js">hanzoai/search-js</a> | | Python | <code>hanzo-search</code> | <a href="https://github.com/hanzoai/search-python">hanzoai/search-python</a> | | Rust | <code>hanzo-search</code> | <a href="https://github.com/hanzoai/search-rust">hanzoai/search-rust</a> |</p>
|
|
<p>SDK usage for generative search:</p>
|
|
<pre><code class="lang-typescript">import { HanzoSearch } from '@hanzo/search';
|
|
|
|
const search = new HanzoSearch({
|
|
uiUrl: 'https://search.hanzo.ai',
|
|
engineUrl: 'https://search-engine.hanzo.ai',
|
|
apiKey: 'sk-hanzo-...',
|
|
});
|
|
|
|
// Generative search with streaming
|
|
const stream = await search.query('What is hybrid search?', {
|
|
mode: 'docs',
|
|
stream: true,
|
|
});
|
|
|
|
for await (const event of stream) {
|
|
if (event.type === 'token') process.stdout.write(event.content);
|
|
if (event.type === 'citation') console.log(`[${event.index}] ${event.title}`);
|
|
if (event.type === 'done') console.log(`\n${event.sources_used} sources, ${event.latency_ms}ms`);
|
|
}
|
|
|
|
// Direct engine search (non-generative)
|
|
const results = await search.engine.index('docs').search('hybrid search', {
|
|
limit: 20,
|
|
filter: 'category = "architecture"',
|
|
});</code></pre>
|
|
<h2>Security Considerations</h2>
|
|
<ol><li><strong>Query sanitization</strong>: Strip injection attempts before forwarding to engine or LLM</li><li><strong>Source validation</strong>: Verify URLs and content from web sources before displaying</li><li><strong>API key isolation</strong>: Engine API keys scoped to specific indexes and actions</li><li><strong>PII filtering</strong>: Optionally strip personal data from queries before web search</li><li><strong>Content filtering</strong>: Flag and filter harmful content in synthesized answers</li><li><strong>Rate limiting</strong>: Per-user and per-key limits at both UI and engine layers</li><li><strong>TLS everywhere</strong>: All inter-service communication over TLS</li><li><strong>Audit logging</strong>: Log all queries with user attribution for compliance</li></ol>
|
|
<h2>Performance Targets</h2>
|
|
<p>| Metric | Target | Notes | |--------|--------|-------| | Engine query latency | < 50ms p95 | Full-text + vector hybrid | | Time to first token | < 500ms | Including retrieval + LLM cold start | | Full answer latency | < 3s p95 | End-to-end for typical web query | | Concurrent queries | > 500 | Per engine instance | | Index throughput | > 10K docs/sec | Bulk indexing | | Max index size | 100GB | Per index, configurable |</p>
|
|
<h2>Testing</h2>
|
|
<pre><code class="lang-bash"># Engine unit tests (Rust)
|
|
cargo test --workspace
|
|
|
|
# Engine benchmarks
|
|
cargo bench --package benchmarks
|
|
|
|
# UI unit tests
|
|
pnpm test
|
|
|
|
# Integration tests (full pipeline)
|
|
pnpm test:integration
|
|
|
|
# Load testing
|
|
k6 run tests/load/search-pipeline.js</code></pre>
|
|
<h2>References</h2>
|
|
<ol><li><a href="./hip-0004-llm-gateway-unified-ai-provider-interface.md">HIP-4: LLM Gateway</a></li><li><a href="./hip-0010-model-context-protocol-mcp-integration-standards.md">HIP-10: Model Context Protocol</a></li><li><a href="./hip-0011-chat-interface-standard.md">HIP-11: Chat Interface Standard</a></li><li><a href="./hip-0026-identity-access-management-standard.md">HIP-26: Identity and Access Management</a></li><li><a href="https://github.com/hanzoai/search">Hanzo Search Engine Repository</a></li><li><a href="https://www.meilisearch.com/docs">Meilisearch Documentation</a></li></ol>
|
|
<h2>Copyright</h2>
|
|
<p>Copyright and related rights waived via <a href="https://creativecommons.org/publicdomain/zero/1.0/">CC0</a>.</p>
|
|
<footer>
|
|
<div class="fmap">
|
|
<div><b>Build</b>
|
|
<a href="https://hanzo.app">hanzo.app</a> app builder
|
|
<a href="https://hanzo.codes">hanzo.codes</a> code with an agent
|
|
<a href="https://hanzoskills.com">hanzoskills.com</a> what an agent reads</div>
|
|
<div><b>Run</b>
|
|
<a href="https://console.hanzo.ai">console.hanzo.ai</a> cloud console
|
|
<a href="https://api.hanzo.ai">api.hanzo.ai</a> the API</div>
|
|
<div><b>Account</b>
|
|
<a href="https://hanzo.id">hanzo.id</a> sign in
|
|
<a href="https://pay.hanzo.ai">pay.hanzo.ai</a> top up
|
|
<a href="https://billing.hanzo.ai">billing.hanzo.ai</a> invoices and usage</div>
|
|
<div><b>Company</b>
|
|
<a href="https://hanzo.ai">hanzo.ai</a> canonical
|
|
<a href="https://hanzo.ventures">hanzo.ventures</a> ventures</div>
|
|
</div>
|
|
<div class="fend"><a href="/hip-0012.md">This page as plain text</a> · <a href="/llms.txt">Index for agents</a></div>
|
|
</footer>
|
|
</div>
|
|
</body></html> |