mirror of
https://github.com/hanzoai/skills.git
synced 2026-11-19 00:47:36 +00:00
hanzoskills.com/llms.txt now opens with the install line, the API reference index and the company index, so an agent that lands on any one of the three can find the rest. robots.txt claimed 537 documents while llms.txt counts 920. The number is gone from robots.txt; llms.txt derives it. public/ is regenerated from scripts/site.py, which it had fallen behind: the copy-prompt menu and agent bar were in the generator and not in the committed pages. tests/site.mjs passes against the result. Co-authored-by: Hanzo Dev <[email protected]>
765 lines
36 KiB
HTML
765 lines
36 KiB
HTML
<!doctype html>
|
|
<html lang="en"><head>
|
|
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
<title>ir-query-understanding</title>
|
|
<meta name="color-scheme" content="dark">
|
|
<style>
|
|
/* Committed to one visual world: a dark instrument panel. No light variant —
|
|
the dot field and the hairline rules only read on black, and a naive
|
|
inversion would wash both out. Every colour is painted explicitly so the page
|
|
never borrows a host background. */
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Pixel Square';src:url('/fonts/ZenPixel-Square.woff2') format('woff2');font-style:normal;font-display:swap}
|
|
|
|
:root{
|
|
--ink:#f4f4f2; --soft:#9c9c98; --faint:#6a6a66; --ground:#000; --panel:#0a0a0a;
|
|
--line:#232323; --line-hi:#3a3a3a; --accent:#f4f4f2;
|
|
--font-zen-sans:'Zen',-apple-system,BlinkMacSystemFont,"Segoe UI",Helvetica,Arial,sans-serif;
|
|
--font-zen-mono:'Zen Mono',ui-monospace,"SF Mono",SFMono-Regular,Menlo,Consolas,monospace;
|
|
--font-zen-pixel-square:'Zen Pixel Square',monospace;
|
|
--mono:var(--font-zen-mono);
|
|
--sans:var(--font-zen-sans);
|
|
/* Controls are pills and surfaces are soft, as on hanzo.ai. */
|
|
--r:999px; --r-soft:14px;
|
|
}
|
|
*{box-sizing:border-box}
|
|
html{-webkit-text-size-adjust:100%}
|
|
body{margin:0;background:var(--ground);color:var(--ink);font-family:var(--sans);
|
|
font-size:16px;line-height:1.65;-webkit-font-smoothing:antialiased}
|
|
.wrap{max-width:60rem;margin:0 auto;padding:0 1.75rem}
|
|
|
|
/* The dot field. A halftone made of one repeating radial-gradient, faded out
|
|
along the diagonal by a mask so it reads as a wave rather than wallpaper. */
|
|
/* No overflow clip here: the dot field is inset to the box already, and a clip
|
|
would cut off the copy-prompt menu. */
|
|
.hero{position:relative;border-bottom:1px solid var(--line)}
|
|
.hero::before{content:"";position:absolute;inset:0;
|
|
background-image:radial-gradient(var(--line-hi) 1px, transparent 1px);
|
|
background-size:7px 7px;
|
|
-webkit-mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
pointer-events:none}
|
|
.hero .wrap{position:relative;padding-top:5.5rem;padding-bottom:4rem}
|
|
|
|
nav{border-bottom:1px solid var(--line)}
|
|
nav .in{max-width:60rem;margin:0 auto;padding:1.1rem 1.75rem;display:flex;
|
|
justify-content:space-between;align-items:center;gap:1rem}
|
|
.brand{font-family:var(--mono);font-size:.85rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--ink);text-decoration:none;
|
|
display:inline-flex;align-items:center;gap:.6rem}
|
|
.brand i{font-style:normal;color:var(--faint)}
|
|
.mark{width:1.15em;height:1.15em;display:block;flex:0 0 auto}
|
|
nav a.up{font-family:var(--mono);font-size:.72rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--soft);text-decoration:none}
|
|
nav a.up:hover{color:var(--ink)}
|
|
|
|
h1{font-family:var(--mono);font-size:clamp(2rem,5.4vw,3.4rem);line-height:1.12;
|
|
letter-spacing:-.02em;margin:0 0 1.1rem;font-weight:500;text-wrap:balance}
|
|
h1 em{font-style:normal;color:var(--faint)}
|
|
.lede{color:var(--soft);font-size:1.05rem;max-width:52ch;margin:0 0 2.25rem}
|
|
|
|
.btns{display:flex;flex-wrap:wrap;gap:.7rem;margin:0 0 .5rem;align-items:center}
|
|
.btn{display:inline-flex;align-items:center;gap:.5rem;font-family:var(--mono);
|
|
font-size:.72rem;letter-spacing:.14em;text-transform:uppercase;
|
|
padding:.8rem 1.4rem;border:1px solid var(--line-hi);border-radius:var(--r);
|
|
color:var(--ink);text-decoration:none;background:transparent;cursor:pointer;
|
|
line-height:1.2;font-weight:400}
|
|
.btn:hover{background:#141414;text-decoration:none;border-color:var(--soft)}
|
|
.btn.solid{background:var(--ink);color:#000;border-color:var(--ink)}
|
|
.btn.solid:hover{background:#d8d8d4}
|
|
.btn .chev{width:.7em;height:.7em;flex:0 0 auto;transition:transform .15s ease}
|
|
.btn[aria-expanded="true"] .chev{transform:rotate(180deg)}
|
|
|
|
/* Copy-prompt selector. Each row copies the prompt for its own target. */
|
|
.copy{position:relative;display:inline-flex}
|
|
.menu{position:absolute;top:calc(100% + .5rem);left:0;z-index:20;min-width:21rem;
|
|
background:var(--panel);border:1px solid var(--line-hi);border-radius:var(--r-soft);
|
|
padding:.4rem;display:flex;flex-direction:column;
|
|
box-shadow:0 12px 32px rgba(0,0,0,.75)}
|
|
.menu[hidden]{display:none}
|
|
.mi{display:flex;align-items:center;gap:.7rem;width:100%;text-align:left;
|
|
padding:.62rem .8rem;border:0;border-radius:calc(var(--r-soft) - .35rem);
|
|
background:transparent;color:var(--ink);font-family:var(--mono);font-size:.76rem;
|
|
letter-spacing:.04em;text-decoration:none;cursor:pointer;line-height:1.3;
|
|
white-space:nowrap}
|
|
.mi:hover,.mi:focus-visible{background:#181818;outline:none}
|
|
.mi .mark{width:1.05rem;height:1.05rem;color:var(--soft)}
|
|
.mi:hover .mark{color:var(--ink)}
|
|
.mi u{margin-left:auto;color:var(--faint);text-decoration:none;font-size:.68rem;
|
|
letter-spacing:.1em;text-transform:uppercase}
|
|
|
|
/* Works-with bar: one scrolling row, name and mark per agent. */
|
|
.strip{border-bottom:1px solid var(--line);background:var(--panel)}
|
|
.strip .in{max-width:60rem;margin:0 auto;padding:.9rem 1.75rem;display:flex;
|
|
align-items:center;gap:1.25rem;font-family:var(--mono);font-size:.72rem;
|
|
letter-spacing:.1em;text-transform:uppercase;color:var(--faint)}
|
|
.strip b{color:var(--soft);font-weight:400;flex:0 0 auto;
|
|
border-right:1px solid var(--line);padding-right:1.25rem;align-self:stretch;
|
|
display:flex;align-items:center}
|
|
.bar{display:flex;align-items:center;gap:.5rem;flex:1 1 auto;min-width:0;
|
|
overflow-x:auto;scrollbar-width:none;scroll-snap-type:x proximity;
|
|
-webkit-overflow-scrolling:touch;
|
|
-webkit-mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent);
|
|
mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent)}
|
|
.bar::-webkit-scrollbar{display:none}
|
|
.agent{display:inline-flex;align-items:center;gap:.45rem;flex:0 0 auto;
|
|
padding:.4rem .85rem;border:1px solid var(--line);border-radius:var(--r);
|
|
color:var(--soft);white-space:nowrap;scroll-snap-align:start;
|
|
text-decoration:none;background:var(--ground)}
|
|
.agent:hover{color:var(--ink);border-color:var(--line-hi);text-decoration:none}
|
|
.agent .mark{width:.95rem;height:.95rem}
|
|
|
|
h2{font-family:var(--mono);font-size:1.1rem;letter-spacing:-.01em;margin:2.75rem 0 .75rem;font-weight:500}
|
|
h3{font-size:1rem;margin:2rem 0 .5rem;font-weight:600}
|
|
h4,h5,h6{font-size:.94rem;margin:1.5rem 0 .5rem;font-weight:600}
|
|
p{margin:0 0 1rem;max-width:70ch}
|
|
ul,ol{margin:0 0 1rem;padding-left:1.35rem;max-width:70ch}
|
|
li{margin:.22rem 0}
|
|
a{color:var(--ink);text-decoration:underline;text-underline-offset:3px;
|
|
text-decoration-color:var(--line-hi)}
|
|
a:hover{text-decoration-color:var(--ink)}
|
|
a:focus-visible{outline:1px solid var(--ink);outline-offset:3px}
|
|
code{font-family:var(--mono);font-size:.85em;background:var(--panel);
|
|
border:1px solid var(--line);border-radius:5px;padding:.1em .36em}
|
|
pre{background:var(--panel);border:1px solid var(--line);padding:1rem 1.1rem;
|
|
border-radius:var(--r-soft);overflow-x:auto;margin:0 0 1.25rem}
|
|
pre code{background:none;border:0;padding:0;font-size:.82rem;line-height:1.7}
|
|
blockquote{margin:0 0 1rem;padding-left:1rem;border-left:1px solid var(--line-hi);color:var(--soft)}
|
|
hr{border:0;border-top:1px solid var(--line);margin:2.5rem 0}
|
|
table{border-collapse:collapse;width:100%;margin:0 0 1.25rem;font-size:.92rem;
|
|
display:block;overflow-x:auto}
|
|
th,td{border:1px solid var(--line);padding:.5rem .7rem;text-align:left}
|
|
|
|
.curl{display:block;background:var(--panel);border:1px solid var(--line);
|
|
border-radius:var(--r-soft);padding:.95rem 1.1rem;font-family:var(--mono);
|
|
font-size:.8rem;color:var(--soft);overflow-x:auto;margin:0 0 2.5rem;
|
|
white-space:pre;line-height:1.9}
|
|
.curl b{color:var(--ink);font-weight:400}
|
|
.curl i{font-style:normal;color:var(--faint)}
|
|
|
|
/* index */
|
|
#q{width:100%;padding:.85rem 1.2rem;font-family:var(--mono);font-size:.85rem;color:var(--ink);
|
|
background:var(--panel);border:1px solid var(--line);border-radius:var(--r);margin:0 0 .6rem}
|
|
#q::placeholder{color:var(--faint);text-transform:uppercase;letter-spacing:.1em;font-size:.72rem}
|
|
#q:focus{outline:none;border-color:var(--soft)}
|
|
#count{color:var(--faint);font-family:var(--mono);font-size:.72rem;letter-spacing:.1em;
|
|
text-transform:uppercase;margin:0 0 2rem}
|
|
.cat{margin:2.5rem 0 .5rem;font-family:var(--mono);font-size:.7rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);border-bottom:1px solid var(--line);
|
|
padding-bottom:.45rem}
|
|
.row{display:block;padding:.6rem .9rem;margin:0 -.9rem;border-bottom:1px solid var(--line);
|
|
border-radius:var(--r-soft);text-decoration:none}
|
|
.row:hover{background:#0a0a0a}
|
|
.nm{font-family:var(--mono);font-size:.84rem;color:var(--ink);display:block}
|
|
.ds{color:var(--soft);font-size:.9rem;display:block;margin-top:.12rem}
|
|
footer{margin-top:4rem;padding:2rem 0 4rem;border-top:1px solid var(--line);
|
|
color:var(--faint);font-size:.85rem}
|
|
footer a{color:var(--soft)}
|
|
.fmap{display:grid;gap:1.75rem;grid-template-columns:repeat(auto-fit,minmax(11rem,1fr));
|
|
margin:0 0 2rem;font-size:.85rem}
|
|
.fmap b{display:block;font-family:var(--mono);font-size:.68rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);font-weight:400;margin:0 0 .6rem;
|
|
border-bottom:1px solid var(--line);padding-bottom:.4rem}
|
|
.fmap a{display:inline-block;color:var(--ink);text-decoration:none;margin:.3rem .35rem 0 0}
|
|
.fmap a:hover{text-decoration:underline}
|
|
.fmap div{color:var(--faint);line-height:1.5}
|
|
.fend{border-top:1px solid var(--line);padding-top:1.25rem;font-size:.82rem}
|
|
|
|
@media (max-width:640px){
|
|
.hero .wrap{padding-top:3.25rem;padding-bottom:2.75rem}
|
|
.btns{gap:.5rem}
|
|
.btn{padding:.75rem 1.15rem;font-size:.68rem}
|
|
.copy,.copy .btn{width:100%;justify-content:center}
|
|
.menu{left:0;right:0;min-width:0}
|
|
.strip .in{flex-direction:column;align-items:flex-start;gap:.6rem;padding:.9rem 1.75rem}
|
|
.strip b{border-right:0;padding-right:0}
|
|
.bar{width:100%}
|
|
}
|
|
</style>
|
|
<link rel="alternate" type="text/markdown" href="/ir-query-understanding.md">
|
|
|
|
<link rel="icon" href="/favicon.svg">
|
|
<link rel="canonical" href="https://hanzoskills.com/ir-query-understanding">
|
|
</head>
|
|
<!-- Reading this with a machine? Every page here is also plain markdown.
|
|
This one: https://hanzoskills.com/ir-query-understanding.md
|
|
Index: https://hanzoskills.com/llms.txt
|
|
Corpus: https://hanzoskills.com/all.md
|
|
Append .md to any path for the markdown twin. -->
|
|
<body>
|
|
<nav><div class="in">
|
|
<a class="brand" href="/"><svg class="mark" viewBox="0 0 67 67" fill="currentColor" aria-hidden="true"><path d="M22.21 67V44.6369H0V67H22.21Z"/><path d="M66.7038 22.3184H22.2534L0.0878906 44.6367H44.4634L66.7038 22.3184Z"/><path d="M22.21 0H0V22.3184H22.21V0Z"/><path d="M66.7198 0H44.5098V22.3184H66.7198V0Z"/><path d="M66.7198 67V44.6369H44.5098V67H66.7198Z"/></svg> HANZO<i>SKILLS</i></a>
|
|
<a class="up" href="/">All skills</a>
|
|
</div></nav>
|
|
|
|
<div class="wrap">
|
|
<h1>ir-query-understanding</h1><p class="lede">Query expansion, spell correction, semantic search, query classification, entity recognition, and autocomplete</p><h1>Information Retrieval: Query Understanding</h1>
|
|
<p><strong>Scope</strong>: Query preprocessing including expansion, spell correction, semantic understanding, classification, entity recognition, and suggestions <strong>Lines</strong>: ~300 <strong>Last Updated</strong>: 2025-10-25 <strong>Format Version</strong>: 1.0 (Atomic)</p>
|
|
<hr>
|
|
<h2>When to Use This Skill</h2>
|
|
<p>Activate this skill when:</p>
|
|
<ul><li>Improving search recall through query expansion</li><li>Handling misspelled queries with spell correction</li><li>Understanding query intent (informational, navigational, transactional)</li><li>Extracting entities from queries (people, places, products)</li><li>Implementing autocomplete and query suggestions</li><li>Bridging vocabulary gap between users and documents</li><li>Supporting multilingual search queries</li><li>Personalizing query interpretation based on user context</li></ul>
|
|
<h2>Core Concepts</h2>
|
|
<h3>Concept 1: Query Expansion</h3>
|
|
<p><strong>Techniques</strong>:</p>
|
|
<p><strong>Synonym Expansion</strong>: Add related terms</p>
|
|
<ul><li>"laptop" → "laptop", "notebook", "computer"</li></ul>
|
|
<p><strong>Pseudo-Relevance Feedback</strong>: Use top results to expand</p>
|
|
<ul><li>Search → Get top 5 → Extract terms → Re-search</li></ul>
|
|
<p><strong>Word Embeddings</strong>: Add semantically similar terms</p>
|
|
<ul><li>Use word2vec, GloVe, or transformer embeddings</li></ul>
|
|
<pre><code class="lang-python">from gensim.models import Word2Vec
|
|
import numpy as np
|
|
|
|
# Load pre-trained word embeddings
|
|
# For demo, train small model (use pre-trained in production)
|
|
sentences = [
|
|
["machine", "learning", "algorithms"],
|
|
["deep", "learning", "neural", "networks"],
|
|
["supervised", "learning", "classification"],
|
|
]
|
|
model = Word2Vec(sentences, vector_size=100, window=5, min_count=1)
|
|
|
|
def expand_query_embeddings(query, model, top_k=3):
|
|
"""Expand query with semantically similar terms"""
|
|
terms = query.lower().split()
|
|
expanded = set(terms)
|
|
|
|
for term in terms:
|
|
if term in model.wv:
|
|
# Get similar words
|
|
similar = model.wv.most_similar(term, topn=top_k)
|
|
for word, score in similar:
|
|
if score > 0.7: # Threshold for similarity
|
|
expanded.add(word)
|
|
|
|
return " ".join(expanded)
|
|
|
|
# Pseudo-relevance feedback
|
|
def pseudo_relevance_feedback(query, search_fn, top_k=5, expansion_terms=3):
|
|
"""Expand query using top search results"""
|
|
# Initial search
|
|
results = search_fn(query, limit=top_k)
|
|
|
|
# Extract terms from top results
|
|
from collections import Counter
|
|
term_counts = Counter()
|
|
|
|
for doc in results:
|
|
# Simple tokenization (use proper NLP in production)
|
|
terms = doc['content'].lower().split()
|
|
term_counts.update(terms)
|
|
|
|
# Add most common terms not in original query
|
|
query_terms = set(query.lower().split())
|
|
expansion = []
|
|
|
|
for term, count in term_counts.most_common():
|
|
if term not in query_terms and len(expansion) < expansion_terms:
|
|
expansion.append(term)
|
|
|
|
expanded_query = query + " " + " ".join(expansion)
|
|
return expanded_query
|
|
|
|
# Example
|
|
original_query = "machine learning"
|
|
expanded = expand_query_embeddings(original_query, model, top_k=2)
|
|
print(f"Expanded query: {expanded}")</code></pre>
|
|
<h3>Concept 2: Spell Correction</h3>
|
|
<p><strong>Approaches</strong>:</p>
|
|
<p><strong>Edit Distance</strong>: Levenshtein distance to find close matches <strong>Context-Aware</strong>: Use language model to pick best correction <strong>Search-Based</strong>: Correct based on index terms</p>
|
|
<pre><code class="lang-python">import re
|
|
from collections import Counter
|
|
|
|
def edit_distance(s1, s2):
|
|
"""Compute Levenshtein distance"""
|
|
if len(s1) < len(s2):
|
|
return edit_distance(s2, s1)
|
|
|
|
if len(s2) == 0:
|
|
return len(s1)
|
|
|
|
previous_row = range(len(s2) + 1)
|
|
for i, c1 in enumerate(s1):
|
|
current_row = [i + 1]
|
|
for j, c2 in enumerate(s2):
|
|
insertions = previous_row[j + 1] + 1
|
|
deletions = current_row[j] + 1
|
|
substitutions = previous_row[j] + (c1 != c2)
|
|
current_row.append(min(insertions, deletions, substitutions))
|
|
previous_row = current_row
|
|
|
|
return previous_row[-1]
|
|
|
|
class SpellChecker:
|
|
def __init__(self, corpus):
|
|
"""Build dictionary from corpus"""
|
|
self.word_counts = Counter()
|
|
for doc in corpus:
|
|
words = re.findall(r'\w+', doc.lower())
|
|
self.word_counts.update(words)
|
|
|
|
def correction(self, word):
|
|
"""Find best correction for word"""
|
|
if word in self.word_counts:
|
|
return word
|
|
|
|
# Generate candidates within edit distance 1-2
|
|
candidates = self._candidates(word)
|
|
|
|
# Return most frequent candidate
|
|
return max(candidates, key=lambda w: self.word_counts.get(w, 0))
|
|
|
|
def _candidates(self, word):
|
|
"""Generate candidate corrections"""
|
|
# Known words at edit distance 0, 1, 2
|
|
return (
|
|
self._known([word]) or
|
|
self._known(self._edits1(word)) or
|
|
self._known(self._edits2(word)) or
|
|
[word]
|
|
)
|
|
|
|
def _known(self, words):
|
|
"""Filter to known words"""
|
|
return set(w for w in words if w in self.word_counts)
|
|
|
|
def _edits1(self, word):
|
|
"""All edits 1 edit away"""
|
|
letters = 'abcdefghijklmnopqrstuvwxyz'
|
|
splits = [(word[:i], word[i:]) for i in range(len(word) + 1)]
|
|
deletes = [L + R[1:] for L, R in splits if R]
|
|
transposes = [L + R[1] + R[0] + R[2:] for L, R in splits if len(R) > 1]
|
|
replaces = [L + c + R[1:] for L, R in splits if R for c in letters]
|
|
inserts = [L + c + R for L, R in splits for c in letters]
|
|
return set(deletes + transposes + replaces + inserts)
|
|
|
|
def _edits2(self, word):
|
|
"""All edits 2 edits away"""
|
|
return set(e2 for e1 in self._edits1(word) for e2 in self._edits1(e1))
|
|
|
|
# Example
|
|
corpus = ["machine learning algorithms", "deep learning networks", "supervised learning"]
|
|
speller = SpellChecker(corpus)
|
|
|
|
misspelled = "machne lerning"
|
|
corrected = " ".join([speller.correction(word) for word in misspelled.split()])
|
|
print(f"Corrected: {corrected}") # "machine learning"</code></pre>
|
|
<h3>Concept 3: Query Classification</h3>
|
|
<p><strong>Intent Types</strong>:</p>
|
|
<ul><li><strong>Informational</strong>: Seeking knowledge ("what is machine learning")</li><li><strong>Navigational</strong>: Finding specific site/page ("github login")</li><li><strong>Transactional</strong>: Ready to act ("buy macbook pro")</li></ul>
|
|
<pre><code class="lang-python">from sklearn.feature_extraction.text import TfidfVectorizer
|
|
from sklearn.linear_model import LogisticRegression
|
|
|
|
# Training data: (query, intent)
|
|
training_data = [
|
|
("what is machine learning", "informational"),
|
|
("how does neural network work", "informational"),
|
|
("github login", "navigational"),
|
|
("facebook home page", "navigational"),
|
|
("buy macbook pro", "transactional"),
|
|
("book flight to paris", "transactional"),
|
|
("machine learning tutorial", "informational"),
|
|
("amazon prime", "navigational"),
|
|
("order pizza online", "transactional"),
|
|
]
|
|
|
|
queries, intents = zip(*training_data)
|
|
|
|
# Train classifier
|
|
vectorizer = TfidfVectorizer()
|
|
X = vectorizer.fit_transform(queries)
|
|
|
|
classifier = LogisticRegression()
|
|
classifier.fit(X, intents)
|
|
|
|
def classify_query(query, vectorizer, classifier):
|
|
"""Classify query intent"""
|
|
X_query = vectorizer.transform([query])
|
|
intent = classifier.predict(X_query)[0]
|
|
probabilities = classifier.predict_proba(X_query)[0]
|
|
|
|
return {
|
|
'intent': intent,
|
|
'confidence': max(probabilities)
|
|
}
|
|
|
|
# Example
|
|
test_query = "how to learn python"
|
|
result = classify_query(test_query, vectorizer, classifier)
|
|
print(f"Intent: {result['intent']} (confidence: {result['confidence']:.2f})")</code></pre>
|
|
<h3>Concept 4: Entity Recognition</h3>
|
|
<p><strong>Extract</strong>: People, places, products, dates, prices from queries</p>
|
|
<pre><code class="lang-python">import re
|
|
|
|
class QueryEntityExtractor:
|
|
def __init__(self):
|
|
# Patterns for common entities
|
|
self.patterns = {
|
|
'price': r'\$?\d+(?:\.\d{2})?',
|
|
'date': r'\d{4}-\d{2}-\d{2}|\d{1,2}/\d{1,2}/\d{4}',
|
|
'email': r'[\w\.-]+@[\w\.-]+\.\w+',
|
|
'phone': r'\(?\d{3}\)?[-.\s]?\d{3}[-.\s]?\d{4}',
|
|
}
|
|
|
|
# Product catalog (in production, load from database)
|
|
self.products = {'macbook', 'iphone', 'ipad', 'airpods'}
|
|
self.brands = {'apple', 'google', 'microsoft', 'amazon'}
|
|
|
|
def extract(self, query):
|
|
"""Extract entities from query"""
|
|
entities = {}
|
|
|
|
# Regex-based extraction
|
|
for entity_type, pattern in self.patterns.items():
|
|
matches = re.findall(pattern, query, re.IGNORECASE)
|
|
if matches:
|
|
entities[entity_type] = matches
|
|
|
|
# Dictionary-based extraction
|
|
query_lower = query.lower()
|
|
words = set(query_lower.split())
|
|
|
|
entities['products'] = [p for p in self.products if p in query_lower]
|
|
entities['brands'] = [b for b in self.brands if b in words]
|
|
|
|
return entities
|
|
|
|
# Example
|
|
extractor = QueryEntityExtractor()
|
|
query = "buy macbook pro under $2000"
|
|
entities = extractor.extract(query)
|
|
print(f"Entities: {entities}")
|
|
# {'price': ['2000'], 'products': ['macbook'], 'brands': ['apple']}</code></pre>
|
|
<hr>
|
|
<h2>Patterns</h2>
|
|
<h3>Pattern 1: Autocomplete with Prefix Matching</h3>
|
|
<p><strong>When to use</strong>: Real-time query suggestions as user types</p>
|
|
<pre><code class="lang-python">class TrieNode:
|
|
def __init__(self):
|
|
self.children = {}
|
|
self.is_end = False
|
|
self.frequency = 0
|
|
|
|
class Autocomplete:
|
|
def __init__(self):
|
|
self.root = TrieNode()
|
|
|
|
def insert(self, query, frequency=1):
|
|
"""Insert query into trie"""
|
|
node = self.root
|
|
for char in query.lower():
|
|
if char not in node.children:
|
|
node.children[char] = TrieNode()
|
|
node = node.children[char]
|
|
|
|
node.is_end = True
|
|
node.frequency += frequency
|
|
|
|
def search(self, prefix, max_results=5):
|
|
"""Get top suggestions for prefix"""
|
|
# Navigate to prefix
|
|
node = self.root
|
|
for char in prefix.lower():
|
|
if char not in node.children:
|
|
return []
|
|
node = node.children[char]
|
|
|
|
# Collect all completions
|
|
results = []
|
|
self._collect_completions(node, prefix, results)
|
|
|
|
# Sort by frequency and return top-k
|
|
results.sort(key=lambda x: x[1], reverse=True)
|
|
return [query for query, freq in results[:max_results]]
|
|
|
|
def _collect_completions(self, node, current, results):
|
|
"""Recursively collect all completions"""
|
|
if node.is_end:
|
|
results.append((current, node.frequency))
|
|
|
|
for char, child in node.children.items():
|
|
self._collect_completions(child, current + char, results)
|
|
|
|
# Build autocomplete from query logs
|
|
ac = Autocomplete()
|
|
query_log = [
|
|
("machine learning", 100),
|
|
("machine learning tutorial", 50),
|
|
("machine learning algorithms", 30),
|
|
("machine vision", 20),
|
|
]
|
|
|
|
for query, freq in query_log:
|
|
ac.insert(query, freq)
|
|
|
|
# Suggest
|
|
suggestions = ac.search("machine l", max_results=3)
|
|
print(f"Suggestions: {suggestions}")</code></pre>
|
|
<h3>Pattern 2: Contextual Query Rewriting</h3>
|
|
<p><strong>Use case</strong>: Rewrite query based on user context or conversation</p>
|
|
<pre><code class="lang-python">def contextual_query_rewrite(query, user_context, conversation_history):
|
|
"""Rewrite query with context"""
|
|
rewritten = query
|
|
|
|
# Resolve pronouns using conversation history
|
|
if any(pronoun in query.lower() for pronoun in ['it', 'this', 'that', 'he', 'she']):
|
|
if conversation_history:
|
|
last_entity = extract_main_entity(conversation_history[-1])
|
|
rewritten = rewritten.replace('it', last_entity)
|
|
rewritten = rewritten.replace('this', last_entity)
|
|
|
|
# Add implicit location
|
|
if user_context.get('location') and 'near' not in query.lower():
|
|
if any(term in query.lower() for term in ['restaurant', 'hotel', 'store']):
|
|
rewritten += f" near {user_context['location']}"
|
|
|
|
# Add temporal context
|
|
if user_context.get('time_period'):
|
|
if any(term in query.lower() for term in ['news', 'events', 'weather']):
|
|
rewritten += f" {user_context['time_period']}"
|
|
|
|
return rewritten
|
|
|
|
# Example
|
|
context = {'location': 'San Francisco', 'time_period': 'today'}
|
|
history = ["machine learning conferences"]
|
|
|
|
query = "when is it happening"
|
|
rewritten = contextual_query_rewrite(query, context, history)
|
|
print(f"Rewritten: {rewritten}")
|
|
# "when is machine learning conferences happening"</code></pre>
|
|
<h3>Pattern 3: Multi-Language Query Understanding</h3>
|
|
<p><strong>When to use</strong>: Support queries in multiple languages</p>
|
|
<pre><code class="lang-python">from langdetect import detect
|
|
|
|
def multilingual_query_processing(query):
|
|
"""Detect language and process accordingly"""
|
|
try:
|
|
lang = detect(query)
|
|
except:
|
|
lang = 'en'
|
|
|
|
# Language-specific processing
|
|
if lang == 'en':
|
|
# English stopwords
|
|
stopwords = {'the', 'a', 'an', 'is', 'are'}
|
|
elif lang == 'es':
|
|
# Spanish stopwords
|
|
stopwords = {'el', 'la', 'los', 'las', 'es', 'son'}
|
|
elif lang == 'zh-cn':
|
|
# Chinese - different tokenization
|
|
# Use jieba or similar
|
|
stopwords = set()
|
|
else:
|
|
stopwords = set()
|
|
|
|
# Process with language-specific rules
|
|
terms = query.lower().split()
|
|
filtered_terms = [t for t in terms if t not in stopwords]
|
|
|
|
return {
|
|
'language': lang,
|
|
'processed_query': ' '.join(filtered_terms),
|
|
'original_query': query
|
|
}
|
|
|
|
# Example
|
|
query_en = "what is machine learning"
|
|
query_es = "qué es aprendizaje automático"
|
|
|
|
result_en = multilingual_query_processing(query_en)
|
|
result_es = multilingual_query_processing(query_es)</code></pre>
|
|
<h3>Pattern 4: Query Segmentation</h3>
|
|
<p><strong>Use case</strong>: Break complex queries into components</p>
|
|
<pre><code class="lang-python">import re
|
|
|
|
def segment_query(query):
|
|
"""Segment query into structured components"""
|
|
segments = {
|
|
'what': None, # What user is looking for
|
|
'where': None, # Location constraint
|
|
'when': None, # Time constraint
|
|
'how': None, # Method/manner
|
|
'attributes': [] # Additional attributes
|
|
}
|
|
|
|
query_lower = query.lower()
|
|
|
|
# Extract location
|
|
location_pattern = r'(?:in|at|near)\s+([a-z\s]+?)(?:\s|$)'
|
|
location_match = re.search(location_pattern, query_lower)
|
|
if location_match:
|
|
segments['where'] = location_match.group(1).strip()
|
|
|
|
# Extract time
|
|
time_keywords = ['today', 'tomorrow', 'this week', 'next month']
|
|
for keyword in time_keywords:
|
|
if keyword in query_lower:
|
|
segments['when'] = keyword
|
|
break
|
|
|
|
# Extract main intent
|
|
if query_lower.startswith('how'):
|
|
segments['how'] = query
|
|
else:
|
|
# Remove location and time, rest is "what"
|
|
what = query_lower
|
|
if segments['where']:
|
|
what = what.replace(f"in {segments['where']}", "")
|
|
what = what.replace(f"near {segments['where']}", "")
|
|
if segments['when']:
|
|
what = what.replace(segments['when'], "")
|
|
segments['what'] = what.strip()
|
|
|
|
return segments
|
|
|
|
# Example
|
|
query = "italian restaurants in San Francisco open today"
|
|
segments = segment_query(query)
|
|
print(f"Segments: {segments}")</code></pre>
|
|
<h3>Pattern 5: Query Relaxation</h3>
|
|
<p><strong>When to use</strong>: No results for strict query, relax constraints</p>
|
|
<pre><code class="lang-python">def relaxed_search(query, search_fn, min_results=3):
|
|
"""Progressive query relaxation if no results"""
|
|
# Try exact query first
|
|
results = search_fn(query)
|
|
|
|
if len(results) >= min_results:
|
|
return results, "exact"
|
|
|
|
# Relaxation strategies
|
|
strategies = [
|
|
('remove_quotes', lambda q: q.replace('"', '')),
|
|
('remove_stopwords', lambda q: remove_stopwords(q)),
|
|
('expand_synonyms', lambda q: expand_with_synonyms(q)),
|
|
('stem_terms', lambda q: stem_query(q)),
|
|
]
|
|
|
|
for strategy_name, strategy_fn in strategies:
|
|
relaxed_query = strategy_fn(query)
|
|
results = search_fn(relaxed_query)
|
|
|
|
if len(results) >= min_results:
|
|
return results, strategy_name
|
|
|
|
# Ultimate fallback: search individual terms (OR query)
|
|
terms = query.split()
|
|
results = search_fn(" OR ".join(terms))
|
|
|
|
return results, "individual_terms"
|
|
|
|
def remove_stopwords(query):
|
|
stopwords = {'the', 'a', 'an', 'is', 'are', 'in', 'on', 'at'}
|
|
return ' '.join([w for w in query.split() if w.lower() not in stopwords])</code></pre>
|
|
<hr>
|
|
<h2>Quick Reference</h2>
|
|
<h3>Query Expansion Techniques</h3>
|
|
<pre><code>Technique | Use Case | Pros | Cons
|
|
-------------------------|----------------------------|-------------------|-------------
|
|
Synonyms | Vocabulary mismatch | Simple, fast | Limited coverage
|
|
Pseudo-relevance feedback| High-quality corpus | Domain-adaptive | Query drift
|
|
Word embeddings | Semantic similarity | Broader coverage | May add noise
|
|
User click data | Large query logs | User-validated | Needs logs</code></pre>
|
|
<h3>Intent Classification</h3>
|
|
<pre><code>Intent Type | Indicators | Search Strategy
|
|
-----------------|----------------------------------|------------------
|
|
Informational | what, how, why, definition | Return documents, snippets
|
|
Navigational | brand name, login, homepage | Return direct link
|
|
Transactional | buy, order, download, book | Return products, CTAs</code></pre>
|
|
<h3>Key Guidelines</h3>
|
|
<pre><code>✅ DO: Spell-check queries before search
|
|
✅ DO: Expand queries to bridge vocabulary gap
|
|
✅ DO: Classify intent to customize results
|
|
✅ DO: Extract entities for structured search
|
|
✅ DO: Suggest queries as user types (autocomplete)
|
|
✅ DO: Handle multilingual queries
|
|
|
|
❌ DON'T: Over-expand queries (query drift)
|
|
❌ DON'T: Ignore user's original terms
|
|
❌ DON'T: Auto-correct without showing original
|
|
❌ DON'T: Suggest irrelevant queries
|
|
❌ DON'T: Apply same processing to all languages</code></pre>
|
|
<hr>
|
|
<h2>Anti-Patterns</h2>
|
|
<h3>Critical Violations</h3>
|
|
<pre><code class="lang-python"># ❌ NEVER: Auto-correct without showing original query
|
|
def bad_search(query):
|
|
corrected = spell_correct(query)
|
|
return search(corrected)
|
|
# User doesn't know query was changed
|
|
|
|
# ✅ CORRECT: Show correction, let user confirm
|
|
def good_search(query):
|
|
corrected = spell_correct(query)
|
|
if corrected != query:
|
|
# Return both original and corrected results
|
|
return {
|
|
'original_results': search(query),
|
|
'did_you_mean': corrected,
|
|
'corrected_results': search(corrected)
|
|
}
|
|
return {'results': search(query)}</code></pre>
|
|
<p>❌ <strong>Silent auto-correction</strong>: User confusion, wrong results ✅ <strong>Correct approach</strong>: Show "Did you mean?" with original results</p>
|
|
<h3>Common Mistakes</h3>
|
|
<pre><code class="lang-python"># ❌ Don't: Expand query too aggressively
|
|
expanded = original_query + " " + " ".join(all_synonyms)
|
|
# Query drift: "python programming" → "python programming snake coding development"
|
|
|
|
# ✅ Correct: Selective expansion with scoring
|
|
def selective_expansion(query, synonyms, threshold=0.8):
|
|
expanded = query
|
|
for term in query.split():
|
|
if term in synonyms:
|
|
# Add only high-confidence synonyms
|
|
for syn, score in synonyms[term]:
|
|
if score > threshold:
|
|
expanded += f" {syn}"
|
|
return expanded</code></pre>
|
|
<p>❌ <strong>Over-expansion</strong>: Query drift, irrelevant results ✅ <strong>Better</strong>: Use confidence thresholds, limit expansion</p>
|
|
<pre><code class="lang-python"># ❌ Don't: Ignore query structure
|
|
query = "machine learning NOT deep learning"
|
|
processed = " ".join(query.split()) # Loses NOT operator
|
|
|
|
# ✅ Correct: Preserve operators
|
|
def parse_query(query):
|
|
# Preserve boolean operators, quotes, etc.
|
|
tokens = []
|
|
operators = {'AND', 'OR', 'NOT'}
|
|
|
|
for token in query.split():
|
|
if token.upper() in operators:
|
|
tokens.append(token.upper())
|
|
else:
|
|
tokens.append(token)
|
|
|
|
return tokens</code></pre>
|
|
<p>❌ <strong>Destroying query structure</strong>: Loses user intent ✅ <strong>Better</strong>: Respect operators, quotes, special syntax</p>
|
|
<hr>
|
|
<h2>Related Skills</h2>
|
|
<ul><li><code>ir-search-fundamentals.md</code> - Use expanded queries with BM25/Elasticsearch</li><li><code>ir-vector-search.md</code> - Semantic query understanding with embeddings</li><li><code>ir-ranking-reranking.md</code> - Rank results after query understanding</li><li><code>ir-recommendation-systems.md</code> - Query-based recommendations</li><li><code>frontend-forms-validation.md</code> - Client-side query validation</li></ul>
|
|
<hr>
|
|
<p><strong>Last Updated</strong>: 2025-10-25 <strong>Format Version</strong>: 1.0 (Atomic)</p>
|
|
<footer>
|
|
<div class="fmap">
|
|
<div><b>Build</b>
|
|
<a href="https://hanzo.app">hanzo.app</a> app builder
|
|
<a href="https://hanzo.codes">hanzo.codes</a> code with an agent
|
|
<a href="https://hanzoskills.com">hanzoskills.com</a> what an agent reads</div>
|
|
<div><b>Run</b>
|
|
<a href="https://console.hanzo.ai">console.hanzo.ai</a> cloud console
|
|
<a href="https://api.hanzo.ai">api.hanzo.ai</a> the API</div>
|
|
<div><b>Account</b>
|
|
<a href="https://hanzo.id">hanzo.id</a> sign in
|
|
<a href="https://pay.hanzo.ai">pay.hanzo.ai</a> top up
|
|
<a href="https://billing.hanzo.ai">billing.hanzo.ai</a> invoices and usage</div>
|
|
<div><b>Company</b>
|
|
<a href="https://hanzo.ai">hanzo.ai</a> canonical
|
|
<a href="https://hanzo.ventures">hanzo.ventures</a> ventures</div>
|
|
</div>
|
|
<div class="fend"><a href="/ir-query-understanding.md">This page as plain text</a> · <a href="/llms.txt">Index for agents</a></div>
|
|
</footer>
|
|
</div></body></html> |