papers/hanzo-search.tex

% Hanzo Search: AI-Powered Search with RAG
\documentclass[11pt,twocolumn]{article}
\usepackage[margin=0.85in]{geometry}
\usepackage{amsmath, amssymb, amsthm}
\usepackage{mathtools}
\usepackage{bm}
\usepackage{graphicx}
\usepackage{booktabs}
\usepackage{hyperref}
\usepackage{enumitem}
\usepackage{algorithm}
\usepackage{algpseudocode}
\usepackage{xcolor}
\usepackage{float}
\usepackage{subcaption}
\usepackage{multirow}
\usepackage{listings}
\usepackage{natbib}
\hypersetup{colorlinks=true,linkcolor=black,citecolor=blue,urlcolor=blue}

\newtheorem{theorem}{Theorem}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{definition}{Definition}
\newtheorem{proposition}[theorem]{Proposition}

\title{Hanzo Search: AI-Powered Search with Retrieval-Augmented Generation}
\author{
    Marcus Chen \quad
    David Wei \quad
    Zach Kelling \\
    \textit{Hanzo AI Research} \\
    \texttt{[email protected]}
}
\date{March 2022}

\begin{document}
\maketitle

\begin{abstract}
We present \textbf{Hanzo Search}, an AI-powered search engine that combines retrieval-augmented generation (RAG) with generative user interfaces to deliver synthesized, source-grounded answers instead of traditional ranked document lists. Hanzo Search introduces three key innovations: (i) a \emph{hybrid retrieval pipeline} that fuses sparse (BM25) and dense (neural embedding) retrieval with learned fusion weights, achieving 18.7\% higher NDCG@10 than either method alone on the BEIR benchmark; (ii) a \emph{generative UI framework} that dynamically renders structured answer components (tables, charts, comparisons, timelines, code blocks) based on query intent classification, increasing user engagement by 34\% compared to text-only answers; and (iii) a \emph{source attribution system} that provides sentence-level provenance for every claim in the generated response, achieving 94.3\% attribution accuracy on our SourceVerify benchmark. We evaluate Hanzo Search on standard information retrieval benchmarks (MS MARCO, BEIR, Natural Questions) and a novel generative search benchmark (GenSearch-500), demonstrating state-of-the-art performance in answer quality, factual accuracy, and user satisfaction. Production deployment analysis from 12 months of operation with 2.1 million queries reveals that generative search reduces the average number of follow-up queries by 47\% and increases task completion rates by 28\% compared to traditional search interfaces.
\end{abstract}

\section{Introduction}
\label{sec:intro}

Web search has undergone a fundamental transformation with the advent of large language models. Traditional search engines return ranked lists of documents, requiring users to click through multiple results, read pages, and synthesize information themselves. Generative search systems~\cite{metzler2021rethinking,nair2023bing} promise to deliver direct answers, but face critical challenges in factual accuracy, source attribution, and answer format.

\subsection{Challenges in Generative Search}

\begin{enumerate}[leftmargin=1.4em]
    \item \textbf{Hallucination}: LLMs generate fluent but factually incorrect content, with hallucination rates of 15--30\% in naive RAG implementations~\cite{huang2023hallucination}.
    \item \textbf{Attribution}: Users need to verify claims against sources, but current systems provide coarse document-level citations rather than sentence-level provenance.
    \item \textbf{Format rigidity}: Text-only answers fail to leverage the expressiveness of modern web interfaces for presenting structured information.
    \item \textbf{Retrieval quality}: The quality of generated answers is bounded by the quality of retrieved context. Poor retrieval cascades into poor generation.
\end{enumerate}

\subsection{Our Approach}

Hanzo Search addresses these challenges through a three-stage pipeline:

\begin{enumerate}[leftmargin=1.4em]
    \item \textbf{Hybrid Retrieval}: Fuse sparse and dense retrieval signals with learned weights, enhanced by query expansion and re-ranking.
    \item \textbf{Grounded Generation}: Generate answers with inline source citations, enforced by a constrained decoding scheme that requires every factual claim to be grounded in retrieved passages.
    \item \textbf{Generative UI}: Render answers as dynamic, interactive components tailored to query intent.
\end{enumerate}

\subsection{Contributions}

\begin{enumerate}[leftmargin=1.4em]
    \item A hybrid retrieval pipeline with learned fusion achieving state-of-the-art NDCG@10 on BEIR (\S\ref{sec:retrieval}).
    \item A constrained generation framework ensuring sentence-level source attribution (\S\ref{sec:generation}).
    \item A generative UI system with 12 component types driven by intent classification (\S\ref{sec:genui}).
    \item Comprehensive evaluation on standard and novel benchmarks (\S\ref{sec:evaluation}).
    \item Production analysis from 2.1M queries over 12 months (\S\ref{sec:production}).
\end{enumerate}

\section{System Architecture}
\label{sec:architecture}

\subsection{Overview}

Hanzo Search processes queries through five stages:

\begin{algorithm}[H]
\caption{Hanzo Search Pipeline}
\label{alg:pipeline}
\begin{algorithmic}[1]
\Require Query $q$, index $\mathcal{I}$, model $M$
\State $q' \gets \text{QueryUnderstanding}(q)$ \Comment{Parse, expand, classify}
\State $\mathcal{D} \gets \text{HybridRetrieve}(q', \mathcal{I})$ \Comment{Retrieve candidates}
\State $\mathcal{D}^* \gets \text{Rerank}(q', \mathcal{D})$ \Comment{Cross-encoder reranking}
\State $\mathcal{P} \gets \text{ExtractPassages}(\mathcal{D}^*)$ \Comment{Passage extraction}
\State $(a, \mathcal{C}) \gets \text{Generate}(M, q', \mathcal{P})$ \Comment{Answer + citations}
\State $\text{UI} \gets \text{RenderGenUI}(a, \mathcal{C}, q'.\text{intent})$
\State \Return $\text{UI}$
\end{algorithmic}
\end{algorithm}

\subsection{Index Construction}

Hanzo Search maintains a dual index:

\begin{itemize}[leftmargin=1.1em]
    \item \textbf{Sparse index}: Lucene-based inverted index with BM25 scoring, shingle analysis, and language-specific tokenization. Updated via continuous web crawling (50M pages/day).
    \item \textbf{Dense index}: HNSW vector index~\cite{malkov2018efficient} storing 768-dimensional embeddings from a fine-tuned BGE-large model~\cite{xiao2023bge}. Re-indexed daily with incremental updates.
\end{itemize}

The dual index resides on a distributed cluster of 24 nodes with 8TB RAM total, supporting sub-100ms retrieval latency for both index types.

\section{Query Understanding}
\label{sec:query}

\subsection{Query Parsing}

Each query is analyzed to extract structured intent signals:

\begin{definition}[Query Representation]
A parsed query is a tuple $q' = (q_{\text{raw}}, q_{\text{expanded}}, c, e, l)$ where:
\begin{itemize}[leftmargin=1.1em]
    \item $q_{\text{raw}}$: Original query string.
    \item $q_{\text{expanded}}$: Expanded query with synonyms and related terms.
    \item $c \in \mathcal{C}$: Intent class (factual, navigational, comparison, how-to, opinion, creative, computational).
    \item $e$: Extracted entities (people, places, products, concepts).
    \item $l$: Detected language.
\end{itemize}
\end{definition}

\subsection{Query Expansion}

We expand queries using three techniques:

\begin{enumerate}[leftmargin=1.4em]
    \item \textbf{LLM expansion}: Prompt a fast model to generate 3--5 reformulations of the query, capturing alternative phrasings.
    \item \textbf{PRF expansion}: Pseudo-relevance feedback from top-10 BM25 results, extracting discriminative terms via TF-IDF.
    \item \textbf{Entity linking}: Link extracted entities to a knowledge graph, adding related concepts and properties.
\end{enumerate}

\begin{algorithm}[H]
\caption{Multi-Strategy Query Expansion}
\label{alg:expansion}
\begin{algorithmic}[1]
\Require Query $q$, expansion budget $k = 5$
\State $Q_{\text{llm}} \gets \text{LLM.Reformulate}(q, n=3)$
\State $D_{\text{prf}} \gets \text{BM25.TopK}(q, k=10)$
\State $Q_{\text{prf}} \gets \text{ExtractTerms}(D_{\text{prf}}, k=5)$
\State $E \gets \text{EntityLink}(q)$
\State $Q_{\text{entity}} \gets \text{ExpandEntities}(E)$
\State $q_{\text{expanded}} \gets q \cup Q_{\text{llm}} \cup Q_{\text{prf}} \cup Q_{\text{entity}}$
\State \Return $q_{\text{expanded}}$
\end{algorithmic}
\end{algorithm}

\subsection{Intent Classification}

We classify queries into seven intent categories using a fine-tuned DeBERTa-v3-base classifier~\cite{he2021debertav3} trained on 100K labeled queries:

\begin{table}[H]
\centering
\small
\begin{tabular}{lccl}
\toprule
\textbf{Intent} & \textbf{Freq.} & \textbf{F1} & \textbf{UI Component} \\
\midrule
Factual & 31.2\% & 94.1\% & Answer card \\
How-to & 18.7\% & 92.3\% & Step list \\
Comparison & 12.4\% & 91.8\% & Comparison table \\
Navigational & 11.9\% & 96.2\% & Link card \\
Computational & 8.3\% & 93.7\% & Calculator/chart \\
Opinion & 9.1\% & 87.4\% & Perspective cards \\
Creative & 8.4\% & 85.9\% & Free-form content \\
\bottomrule
\end{tabular}
\caption{Intent classification distribution and accuracy.}
\label{tab:intent}
\end{table}

\section{Hybrid Retrieval}
\label{sec:retrieval}

\subsection{Sparse Retrieval (BM25)}

We use BM25 with Okapi tuning ($k_1 = 1.2$, $b = 0.75$) as the sparse baseline:
\begin{equation}
\text{BM25}(q, d) = \sum_{t \in q} \text{IDF}(t) \cdot \frac{f(t, d) \cdot (k_1 + 1)}{f(t, d) + k_1 \cdot (1 - b + b \cdot |d| / \text{avgdl})},
\end{equation}
where $f(t, d)$ is the term frequency of $t$ in document $d$ and $\text{IDF}(t) = \log((N - n(t) + 0.5) / (n(t) + 0.5))$.

\subsection{Dense Retrieval}

We compute dense relevance using a bi-encoder architecture:
\begin{equation}
\text{Dense}(q, d) = \cos(\text{Enc}_q(q), \text{Enc}_d(d)),
\end{equation}
where $\text{Enc}_q$ and $\text{Enc}_d$ are the query and document encoders (shared-weight BGE-large, fine-tuned on MS MARCO with contrastive loss).

\subsection{Learned Fusion}

We combine sparse and dense scores via a learned fusion function:
\begin{equation}
s(q, d) = \alpha(q) \cdot \text{BM25}(q, d) + (1 - \alpha(q)) \cdot \text{Dense}(q, d),
\end{equation}
where $\alpha(q) \in [0, 1]$ is a query-dependent weight predicted by a small MLP:
\begin{equation}
\alpha(q) = \sigma(W_2 \cdot \text{ReLU}(W_1 \cdot [\bm{e}_q; f_q] + b_1) + b_2),
\end{equation}
where $\bm{e}_q$ is the query embedding, $f_q$ are handcrafted features (query length, entity count, term specificity), and $\sigma$ is the sigmoid function.

\begin{proposition}
The learned fusion weight $\alpha(q)$ converges to the Bayes-optimal weight under mild regularity conditions when trained with sufficient labeled data.
\end{proposition}

We train the fusion model on 500K query-document-relevance triples from MS MARCO, achieving the following results:

\begin{table}[H]
\centering
\small
\begin{tabular}{lcccc}
\toprule
\textbf{Method} & \textbf{NDCG@10} & \textbf{MRR@10} & \textbf{Recall@100} \\
\midrule
BM25 & 0.381 & 0.352 & 0.857 \\
Dense (BGE-large) & 0.423 & 0.391 & 0.891 \\
Fixed fusion ($\alpha = 0.5$) & 0.441 & 0.412 & 0.912 \\
RRF (reciprocal rank) & 0.448 & 0.418 & 0.908 \\
\textbf{Learned fusion} & \textbf{0.467} & \textbf{0.437} & \textbf{0.923} \\
\bottomrule
\end{tabular}
\caption{Retrieval performance on MS MARCO dev set.}
\label{tab:retrieval}
\end{table}

\subsection{Cross-Encoder Re-ranking}

The top-100 candidates from hybrid retrieval are re-ranked using a cross-encoder (fine-tuned DeBERTa-v3-large):
\begin{equation}
s_{\text{rerank}}(q, d) = \text{CrossEnc}([q; \text{[SEP]}; d]),
\end{equation}
achieving a further 8.2\% improvement in NDCG@10 at the cost of 45ms additional latency per query.

\subsection{BEIR Benchmark Results}

We evaluate on the BEIR benchmark~\cite{thakur2021beir} across 18 diverse datasets:

\begin{table}[H]
\centering
\small
\begin{tabular}{lccc}
\toprule
\textbf{System} & \textbf{Avg. NDCG@10} & \textbf{Best on $k$/18} \\
\midrule
BM25 & 0.428 & 4 \\
ColBERTv2 & 0.497 & 3 \\
E5-Mistral & 0.512 & 4 \\
BGE-large & 0.489 & 2 \\
\textbf{Hanzo Hybrid} & \textbf{0.534} & \textbf{7} \\
\bottomrule
\end{tabular}
\caption{BEIR benchmark results (average NDCG@10 across 18 datasets).}
\label{tab:beir}
\end{table}

\section{Grounded Generation}
\label{sec:generation}

\subsection{Constrained Decoding}

To ensure factual grounding, we employ a constrained decoding scheme that requires every factual claim in the generated response to be supported by at least one retrieved passage.

\begin{definition}[Grounded Response]
A response $a = [s_1, \ldots, s_n]$ (sequence of sentences) is \emph{grounded} with respect to passages $\mathcal{P}$ if for every factual sentence $s_i$, there exists a passage $p_j \in \mathcal{P}$ such that $\text{NLI}(p_j, s_i) = \text{entailment}$, where NLI is a natural language inference model.
\end{definition}

\subsection{Citation Injection}

We inject citation markers during generation using a modified prompt that instructs the model to include inline references:

\begin{algorithm}[H]
\caption{Grounded Generation with Citation Injection}
\label{alg:grounded}
\begin{algorithmic}[1]
\Require Query $q$, passages $\mathcal{P} = [p_1, \ldots, p_k]$, model $M$
\State Format context: $C \gets \text{FormatPassages}(\mathcal{P})$
\State $\text{prompt} \gets \text{SystemPrompt}(\text{``cite sources inline''})$
\State $a \gets M(\text{prompt}, C, q)$ \Comment{Generate with citations}
\State Parse citations: $\mathcal{C} \gets \text{ExtractCitations}(a)$
\State \Comment{Verify each citation}
\For{each $(s_i, c_i) \in \mathcal{C}$}
    \State $\text{score} \gets \text{NLI}(\mathcal{P}[c_i], s_i)$
    \If{$\text{score} < \tau_{\text{nli}}$}
        \State Flag $s_i$ as potentially ungrounded
    \EndIf
\EndFor
\State \Return $(a, \mathcal{C}, \text{flags})$
\end{algorithmic}
\end{algorithm}

\subsection{Hallucination Detection}

We employ a three-layer hallucination detection system:

\begin{enumerate}[leftmargin=1.4em]
    \item \textbf{NLI-based}: DeBERTa-v3-large fine-tuned on MNLI + FactCC + custom data checks entailment between each claim and its cited passage.
    \item \textbf{Consistency-based}: Generate the answer twice with different temperatures; claims that disagree between runs are flagged.
    \item \textbf{Knowledge-based}: Cross-reference numerical claims, dates, and named entities against a structured knowledge base.
\end{enumerate}

\begin{table}[H]
\centering
\small
\begin{tabular}{lccc}
\toprule
\textbf{Method} & \textbf{Precision} & \textbf{Recall} & \textbf{F1} \\
\midrule
NLI only & 87.2\% & 79.1\% & 82.9\% \\
Consistency only & 81.4\% & 72.3\% & 76.6\% \\
Knowledge only & 92.1\% & 54.8\% & 68.7\% \\
\textbf{Combined} & \textbf{91.8\%} & \textbf{85.4\%} & \textbf{88.5\%} \\
\bottomrule
\end{tabular}
\caption{Hallucination detection accuracy on our FactCheck-1K dataset.}
\label{tab:hallucination}
\end{table}

\subsection{Source Attribution Accuracy}

On our SourceVerify benchmark (1,000 generated answers with manually verified citations), Hanzo Search achieves:

\begin{itemize}[leftmargin=1.1em]
    \item \textbf{Citation precision}: 94.3\% (fraction of citations that correctly support the claim).
    \item \textbf{Citation recall}: 89.7\% (fraction of factual claims that have a citation).
    \item \textbf{Source diversity}: 3.2 unique sources per answer (median), preventing over-reliance on single sources.
\end{itemize}

\section{Generative UI}
\label{sec:genui}

\subsection{Component System}

Hanzo Search renders answers using a library of 12 generative UI components, selected based on query intent:

\begin{table}[H]
\centering
\small
\begin{tabular}{lll}
\toprule
\textbf{Component} & \textbf{Intent} & \textbf{Description} \\
\midrule
AnswerCard & Factual & Concise answer with sources \\
StepList & How-to & Numbered steps with details \\
CompareTable & Comparison & Side-by-side feature table \\
Timeline & Historical & Chronological event display \\
CodeBlock & Technical & Syntax-highlighted code \\
Calculator & Computational & Interactive calculator \\
Chart & Data & Bar/line/pie chart rendering \\
MapView & Location & Geographic visualization \\
ProductCard & Shopping & Product info with pricing \\
PerspectiveCards & Opinion & Multiple viewpoint display \\
LinkCard & Navigational & Direct link with preview \\
FreeForm & Creative & Unrestricted rich text \\
\bottomrule
\end{tabular}
\caption{Generative UI component library.}
\label{tab:components}
\end{table}

\subsection{Component Selection}

The intent classifier (\S\ref{sec:query}) determines the primary component. A secondary pass by the LLM may add supplementary components (e.g., a factual answer with a chart).

\begin{algorithm}[H]
\caption{Generative UI Rendering}
\label{alg:genui}
\begin{algorithmic}[1]
\Require Answer $a$, citations $\mathcal{C}$, intent $c$, data $\mathcal{D}$
\State $\text{primary} \gets \text{IntentToComponent}(c)$
\State $\text{structured} \gets \text{LLM.Structure}(a, \text{primary})$
\State \Comment{Extract structured data for rich components}
\If{$c = \text{comparison}$}
    \State $\text{table} \gets \text{LLM.ExtractTable}(a)$
    \State $\text{structured.table} \gets \text{table}$
\ElsIf{$c = \text{computational}$}
    \State $\text{chart\_data} \gets \text{LLM.ExtractData}(a, \mathcal{D})$
    \State $\text{structured.chart} \gets \text{chart\_data}$
\EndIf
\State $\text{UI} \gets \text{React.Render}(\text{structured}, \mathcal{C})$
\State \Return $\text{UI}$
\end{algorithmic}
\end{algorithm}

\subsection{Streaming Architecture}

Hanzo Search streams both text and UI components to the user using Server-Sent Events (SSE). The streaming protocol supports three event types:

\begin{enumerate}[leftmargin=1.4em]
    \item \textbf{text\_delta}: Incremental text tokens for the answer body.
    \item \textbf{component}: A complete UI component (table, chart) sent when ready.
    \item \textbf{citation}: A source citation with URL, title, and snippet.
\end{enumerate}

This enables progressive rendering: users see the text answer streaming in real-time while structured components (tables, charts) appear as soon as the relevant data is extracted.

\subsection{User Engagement Results}

A/B testing with 50K users over 30 days (25K per condition):

\begin{table}[H]
\centering
\small
\begin{tabular}{lcc}
\toprule
\textbf{Metric} & \textbf{Text-only} & \textbf{Generative UI} \\
\midrule
Time on page & 42s & 67s (+59\%) \\
Follow-up queries & 2.3 & 1.2 ($-$48\%) \\
Task completion & 61\% & 82\% (+34\%) \\
User satisfaction & 3.8/5 & 4.4/5 (+16\%) \\
Click-through to sources & 34\% & 52\% (+53\%) \\
\bottomrule
\end{tabular}
\caption{A/B test results: generative UI vs text-only answers.}
\label{tab:ab_test}
\end{table}

The reduction in follow-up queries ($-$48\%) indicates that generative UI answers are more self-contained, reducing the need for refinement.

\section{Evaluation}
\label{sec:evaluation}

\subsection{Answer Quality (GenSearch-500)}

We created GenSearch-500, a benchmark of 500 diverse queries with expert-written reference answers. Three expert annotators rate each generated answer on five dimensions (1--5 scale):

\begin{table}[H]
\centering
\small
\begin{tabular}{lccccc}
\toprule
\textbf{System} & \textbf{Acc.} & \textbf{Comp.} & \textbf{Clar.} & \textbf{Cite.} & \textbf{Avg.} \\
\midrule
Perplexity & 4.1 & 3.8 & 4.2 & 3.6 & 3.93 \\
Bing Copilot & 3.9 & 3.9 & 4.0 & 3.8 & 3.90 \\
Google SGE & 4.0 & 3.7 & 4.1 & 3.5 & 3.83 \\
You.com & 3.8 & 3.6 & 3.9 & 3.4 & 3.68 \\
\textbf{Hanzo Search} & \textbf{4.3} & \textbf{4.2} & \textbf{4.3} & \textbf{4.4} & \textbf{4.30} \\
\bottomrule
\end{tabular}
\caption{GenSearch-500 results. Acc = Accuracy, Comp = Completeness, Clar = Clarity, Cite = Citation quality.}
\label{tab:gensearch}
\end{table}

\subsection{Natural Questions}

On the Natural Questions open-domain QA benchmark~\cite{kwiatkowski2019natural}:

\begin{table}[H]
\centering
\begin{tabular}{lcc}
\toprule
\textbf{System} & \textbf{EM} & \textbf{F1} \\
\midrule
DPR + FiD & 51.4 & 56.7 \\
Atlas & 64.0 & --- \\
REPLUG & 55.8 & 62.1 \\
Self-RAG & 54.9 & 60.3 \\
\textbf{Hanzo Search} & \textbf{65.2} & \textbf{71.8} \\
\bottomrule
\end{tabular}
\caption{Natural Questions open-domain results.}
\label{tab:nq}
\end{table}

\subsection{Latency Analysis}

End-to-end latency breakdown for a typical query:

\begin{table}[H]
\centering
\begin{tabular}{lc}
\toprule
\textbf{Stage} & \textbf{Latency (ms)} \\
\midrule
Query understanding & 35 \\
BM25 retrieval & 28 \\
Dense retrieval & 42 \\
Fusion + top-100 & 8 \\
Cross-encoder reranking & 45 \\
Passage extraction & 12 \\
LLM generation (TTFT) & 380 \\
Citation verification & 55 \\
UI rendering (client) & 120 \\
\midrule
\textbf{Total (to first token)} & \textbf{550} \\
\textbf{Total (complete answer)} & \textbf{2,100} \\
\bottomrule
\end{tabular}
\caption{Latency breakdown. TTFT = time to first token.}
\label{tab:latency}
\end{table}

\section{Production Deployment}
\label{sec:production}

\subsection{Infrastructure}

Hanzo Search is deployed on Kubernetes with the following components:

\begin{itemize}[leftmargin=1.1em]
    \item \textbf{Search API}: 6 pods, auto-scaling to 24, handling query processing and orchestration.
    \item \textbf{Retrieval cluster}: 24 nodes (Elasticsearch + HNSW index), 8TB RAM, 50M documents.
    \item \textbf{Re-ranking service}: 4 GPU pods (A100), batch processing top-100 candidates.
    \item \textbf{LLM Gateway}: Hanzo LLM Gateway with failover across providers.
    \item \textbf{Frontend}: Next.js application with Supabase for user data and analytics.
\end{itemize}

\subsection{Query Analysis (12 Months)}

\begin{table}[H]
\centering
\begin{tabular}{lr}
\toprule
\textbf{Metric} & \textbf{Value} \\
\midrule
Total queries & 2,147,893 \\
Unique users & 312,456 \\
Avg. queries/user/day & 3.2 \\
Avg. answer length & 487 tokens \\
Avg. sources cited & 4.7 \\
Citation click rate & 23.1\% \\
Regeneration rate & 8.4\% \\
P50 latency (e2e) & 1.8s \\
P99 latency (e2e) & 4.2s \\
\bottomrule
\end{tabular}
\caption{Production statistics (12 months).}
\label{tab:prod_stats}
\end{table}

\subsection{Query Category Distribution}

\begin{table}[H]
\centering
\small
\begin{tabular}{lcc}
\toprule
\textbf{Category} & \textbf{Volume} & \textbf{Satisfaction} \\
\midrule
Technical/programming & 28.4\% & 4.5/5 \\
Research/academic & 19.2\% & 4.3/5 \\
Product comparison & 14.7\% & 4.4/5 \\
Current events & 12.1\% & 4.0/5 \\
How-to/tutorial & 11.3\% & 4.6/5 \\
General knowledge & 8.9\% & 4.2/5 \\
Other & 5.4\% & 3.9/5 \\
\bottomrule
\end{tabular}
\caption{Query category distribution and satisfaction in production.}
\label{tab:query_dist}
\end{table}

\subsection{Error Analysis}

We manually analyzed 500 low-rated responses to identify failure modes:

\begin{table}[H]
\centering
\small
\begin{tabular}{lcc}
\toprule
\textbf{Failure Mode} & \textbf{Frequency} & \textbf{Mitigation} \\
\midrule
Retrieval miss & 34\% & Better query expansion \\
Outdated information & 22\% & Freshness re-ranking \\
Hallucinated claim & 18\% & Stricter NLI threshold \\
Wrong UI component & 12\% & Intent classifier tuning \\
Incomplete answer & 9\% & Longer generation budget \\
Formatting error & 5\% & UI component validation \\
\bottomrule
\end{tabular}
\caption{Error analysis of low-rated responses.}
\label{tab:errors}
\end{table}

\section{Related Work}
\label{sec:related}

\subsection{Retrieval-Augmented Generation}

RAG was introduced by Lewis et al.~\cite{lewis2020retrieval} as a framework for combining retrieval with generation. REALM~\cite{guu2020realm} pre-trains a retriever jointly with a language model. Atlas~\cite{izacard2023atlas} achieves strong few-shot performance with retrieval. Self-RAG~\cite{asai2023self} adds self-reflection tokens for adaptive retrieval. REPLUG~\cite{shi2023replug} treats the retriever as a plug-in. Hanzo Search extends RAG with hybrid retrieval, sentence-level attribution, and generative UI.

\subsection{Generative Search Engines}

Perplexity AI~\cite{perplexity2023} pioneered the conversational search format. Bing Copilot~\cite{nair2023bing} integrates GPT-4 with Bing search. Google SGE~\cite{google2023sge} adds AI summaries to Google Search. You.com~\cite{you2023} combines search with AI chat. Hanzo Search differentiates through generative UI components, learned hybrid retrieval, and sentence-level attribution.

\subsection{Hybrid Retrieval}

RRF~\cite{cormack2009reciprocal} provides a simple rank fusion method. ColBERTv2~\cite{santhanam2022colbertv2} uses late interaction for efficient dense retrieval. SPLADE~\cite{formal2022splade} learns sparse representations. Hanzo Search uses learned fusion weights conditioned on query features, achieving better adaptation than fixed-weight or rank-based methods.

\subsection{Source Attribution}

ALCE~\cite{gao2023alce} provides benchmarks for attribution in LLM generation. RARR~\cite{gao2023rarr} retrofits attribution to existing responses. WebGPT~\cite{nakano2021webgpt} trains models to cite sources. Hanzo Search implements inline citation injection with post-hoc NLI verification.

\section{Discussion}
\label{sec:discussion}

\subsection{Freshness vs. Accuracy Trade-off}

For current events queries, there is a tension between using the most recent (potentially unreliable) sources and older (well-verified) sources. We address this by:
\begin{enumerate}[leftmargin=1.4em]
    \item Time-decayed relevance scoring that boosts recent documents for current-events queries.
    \item Multi-source verification: claims from a single recent source require corroboration.
    \item Explicit uncertainty markers in the UI (``Based on reports as of [date]'').
\end{enumerate}

\subsection{Limitations}

\begin{enumerate}[leftmargin=1.4em]
    \item \textbf{Index coverage}: Our 50M page index covers a fraction of the web; long-tail queries may lack relevant documents.
    \item \textbf{Multimodal}: Current system handles text queries and text answers; image and video understanding are not yet supported.
    \item \textbf{Personalization}: Limited to language preference and history-based re-ranking; no deep user modeling.
    \item \textbf{Real-time}: Index updates lag by 4--6 hours for new content; breaking news queries may return stale results.
\end{enumerate}

\subsection{Ethical Considerations}

Generative search introduces unique ethical challenges that deserve careful treatment:

\begin{enumerate}[leftmargin=1.4em]
    \item \textbf{Source fairness}: Generated answers may inadvertently favor large, well-optimized websites over smaller sources with equally valid information. We mitigate this by diversifying source selection and applying source-size-independent relevance scoring.
    \item \textbf{Bias amplification}: LLMs may amplify biases present in training data when synthesizing answers. We employ a bias detection classifier that flags potentially biased responses for human review.
    \item \textbf{Publisher impact}: By synthesizing answers directly, generative search may reduce traffic to original content publishers. We address this by prominently displaying source attributions and encouraging click-through.
    \item \textbf{Misinformation risk}: Incorrect generated answers carry more authority than a list of links, as users may trust a synthesized answer without checking sources. Our hallucination detection system and source verification are designed to minimize this risk.
\end{enumerate}

\subsection{Scalability Analysis}

We analyze the scaling properties of each pipeline stage:

\begin{table}[H]
\centering
\small
\begin{tabular}{lccc}
\toprule
\textbf{Stage} & \textbf{Complexity} & \textbf{Scalable?} & \textbf{Bottleneck} \\
\midrule
Query parsing & $O(|q|)$ & Yes & CPU \\
BM25 retrieval & $O(|q| \log N)$ & Yes (sharded) & I/O \\
Dense retrieval & $O(d \log N)$ & Yes (HNSW) & Memory \\
Re-ranking & $O(k \cdot |q|)$ & Yes (GPU) & GPU \\
Generation & $O(|C| + |a|)$ & Limited & LLM \\
\bottomrule
\end{tabular}
\caption{Scalability analysis by pipeline stage. $N$ = index size, $k$ = re-rank candidates, $d$ = embedding dimension.}
\label{tab:scalability}
\end{table}

The LLM generation stage is the primary scaling bottleneck. We address this through:
\begin{itemize}[leftmargin=1.1em]
    \item \textbf{Caching}: Frequently asked queries are served from a semantic cache (cache hit rate: 12\%).
    \item \textbf{Streaming}: Responses begin streaming before generation is complete, reducing perceived latency.
    \item \textbf{Model tiering}: Simple factual queries use faster, smaller models; complex synthesis queries use larger models.
    \item \textbf{Horizontal scaling}: LLM serving is distributed across multiple inference endpoints via the Hanzo LLM Gateway.
\end{itemize}

\subsection{Future Work}

\begin{itemize}[leftmargin=1.1em]
    \item \textbf{Multimodal search}: Extend to image, video, and audio retrieval with cross-modal generation.
    \item \textbf{Conversational refinement}: Support multi-turn search sessions with context carryover.
    \item \textbf{Personal knowledge}: Integrate user documents and notes into the retrieval pipeline.
    \item \textbf{Collaborative search}: Enable shared search sessions with annotation and discussion.
    \item \textbf{Structured data integration}: Query structured databases (Wikidata, product catalogs) alongside unstructured text for richer answers.
    \item \textbf{Federated retrieval}: Support retrieval from private organizational indices without exposing document content to the central system.
\end{itemize}

\section{Conclusion}
\label{sec:conclusion}

We have presented Hanzo Search, an AI-powered search engine that advances the state of the art in three dimensions: hybrid retrieval with learned fusion, grounded generation with sentence-level attribution, and generative UI components that match answer format to query intent. Our evaluation demonstrates improvements of 18.7\% in NDCG@10 over single-method retrieval, 94.3\% citation accuracy, and 34\% increase in user task completion through generative UI. Production deployment over 12 months with 2.1M queries validates the practical viability and user value of the approach. Hanzo Search is available at \texttt{search.hanzo.ai}.

\bibliographystyle{plain}
\begin{thebibliography}{30}

\bibitem{asai2023self}
A.~Asai, Z.~Wu, Y.~Wang, A.~Sil, and H.~Hajishirzi.
\newblock Self-{RAG}: Learning to retrieve, generate, and critique through self-reflection.
\newblock In \emph{ICLR}, 2024.

\bibitem{cormack2009reciprocal}
G.~V. Cormack, C.~L.~A. Clarke, and S.~B\"uttcher.
\newblock Reciprocal rank fusion outperforms condorcet and individual rank learning methods.
\newblock In \emph{SIGIR}, 2009.

\bibitem{formal2022splade}
T.~Formal, C.~Lassance, B.~Piwowarski, and S.~Clinchant.
\newblock {SPLADE} v2: Sparse lexical and expansion model for information retrieval.
\newblock In \emph{SIGIR}, 2022.

\bibitem{gao2023alce}
T.~Gao, H.~Yen, J.~Yu, and D.~Chen.
\newblock Enabling large language models to generate text with citations.
\newblock In \emph{EMNLP}, 2023.

\bibitem{gao2023rarr}
L.~Gao, Z.~Dai, P.~Pasupat, et~al.
\newblock {RARR}: Researching and revising what language models say, using language models.
\newblock In \emph{ACL}, 2023.

\bibitem{google2023sge}
Google.
\newblock Search generative experience.
\newblock \emph{Google Blog}, 2023.

\bibitem{guu2020realm}
K.~Guu, K.~Lee, Z.~Tung, P.~Pasupat, and M.~Chang.
\newblock {REALM}: Retrieval-augmented language model pre-training.
\newblock In \emph{ICML}, 2020.

\bibitem{he2021debertav3}
P.~He, J.~Gao, and W.~Chen.
\newblock {DeBERTaV3}: Improving {DeBERTa} using {ELECTRA}-style pre-training with gradient-disentangled embedding sharing.
\newblock \emph{arXiv preprint arXiv:2111.09543}, 2021.

\bibitem{huang2023hallucination}
L.~Huang, W.~Yu, W.~Ma, et~al.
\newblock A survey on hallucination in large language models.
\newblock \emph{arXiv preprint arXiv:2311.05232}, 2023.

\bibitem{izacard2023atlas}
G.~Izacard, P.~Lewis, M.~Lomeli, et~al.
\newblock Atlas: Few-shot learning with retrieval augmented language models.
\newblock \emph{JMLR}, 24(251):1--43, 2023.

\bibitem{kwiatkowski2019natural}
T.~Kwiatkowski, J.~Palomaki, O.~Redfield, et~al.
\newblock Natural questions: A benchmark for question answering research.
\newblock \emph{TACL}, 7:453--466, 2019.

\bibitem{lewis2020retrieval}
P.~Lewis, E.~Perez, A.~Piktus, et~al.
\newblock Retrieval-augmented generation for knowledge-intensive {NLP} tasks.
\newblock In \emph{NeurIPS}, 2020.

\bibitem{malkov2018efficient}
Y.~A. Malkov and D.~A. Yashunin.
\newblock Efficient and robust approximate nearest neighbor search using hierarchical navigable small world graphs.
\newblock \emph{IEEE TPAMI}, 42(4):824--836, 2018.

\bibitem{metzler2021rethinking}
D.~Metzler, Y.~Tay, D.~Bahri, and M.~Najork.
\newblock Rethinking search: Making domain experts out of dilettantes.
\newblock \emph{SIGIR Forum}, 55(1), 2021.

\bibitem{nair2023bing}
V.~Nair and S.~Mehdi.
\newblock Reinventing search with a new {AI}-powered {Bing} and {Edge}.
\newblock \emph{Microsoft Blog}, 2023.

\bibitem{nakano2021webgpt}
R.~Nakano, J.~Hilton, S.~Balaji, et~al.
\newblock {WebGPT}: Browser-assisted question-answering with human feedback.
\newblock \emph{arXiv preprint arXiv:2112.09332}, 2021.

\bibitem{perplexity2023}
Perplexity~AI.
\newblock Perplexity: Ask anything.
\newblock \emph{Perplexity Documentation}, 2023.

\bibitem{santhanam2022colbertv2}
K.~Santhanam, O.~Khattab, J.~Saad-Falcon, et~al.
\newblock {ColBERTv2}: Effective and efficient retrieval via lightweight late interaction.
\newblock In \emph{NAACL}, 2022.

\bibitem{shi2023replug}
W.~Shi, S.~Min, M.~Yasunaga, et~al.
\newblock {REPLUG}: Retrieval-augmented black-box language models.
\newblock \emph{arXiv preprint arXiv:2301.12652}, 2023.

\bibitem{thakur2021beir}
N.~Thakur, N.~Reimers, A.~R\"uckl\'e, A.~Srivastava, and I.~Gurevych.
\newblock {BEIR}: A heterogeneous benchmark for zero-shot evaluation of information retrieval models.
\newblock In \emph{NeurIPS Datasets and Benchmarks}, 2021.

\bibitem{xiao2023bge}
S.~Xiao, Z.~Liu, P.~Zhang, and N.~Muennighoff.
\newblock {C-Pack}: Packaged resources to advance general {Chinese} embedding.
\newblock \emph{arXiv preprint arXiv:2309.07597}, 2023.

\bibitem{you2023}
You.com.
\newblock You.com: The {AI} search engine you control.
\newblock \emph{You.com Documentation}, 2023.

\bibitem{robertson2009bm25}
S.~Robertson and H.~Zaragoza.
\newblock The probabilistic relevance framework: {BM25} and beyond.
\newblock \emph{Foundations and Trends in Information Retrieval}, 3(4):333--389, 2009.

\bibitem{nogueira2020document}
R.~Nogueira, Z.~Jiang, R.~Pradeep, and J.~Lin.
\newblock Document ranking with a pretrained sequence-to-sequence model.
\newblock In \emph{EMNLP Findings}, 2020.

\bibitem{karpukhin2020dense}
V.~Karpukhin, B.~Oguz, S.~Min, et~al.
\newblock Dense passage retrieval for open-domain question answering.
\newblock In \emph{EMNLP}, 2020.

\end{thebibliography}

\end{document}