mirror of
https://github.com/hanzoai/skills.git
synced 2026-11-19 00:47:36 +00:00
hanzoskills.com/llms.txt now opens with the install line, the API reference index and the company index, so an agent that lands on any one of the three can find the rest. robots.txt claimed 537 documents while llms.txt counts 920. The number is gone from robots.txt; llms.txt derives it. public/ is regenerated from scripts/site.py, which it had fallen behind: the copy-prompt menu and agent bar were in the generator and not in the committed pages. tests/site.mjs passes against the result. Co-authored-by: Hanzo Dev <[email protected]>
688 lines
34 KiB
HTML
688 lines
34 KiB
HTML
<!doctype html>
|
|
<html lang="en"><head>
|
|
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
<title>ir-ranking-reranking</title>
|
|
<meta name="color-scheme" content="dark">
|
|
<style>
|
|
/* Committed to one visual world: a dark instrument panel. No light variant —
|
|
the dot field and the hairline rules only read on black, and a naive
|
|
inversion would wash both out. Every colour is painted explicitly so the page
|
|
never borrows a host background. */
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Pixel Square';src:url('/fonts/ZenPixel-Square.woff2') format('woff2');font-style:normal;font-display:swap}
|
|
|
|
:root{
|
|
--ink:#f4f4f2; --soft:#9c9c98; --faint:#6a6a66; --ground:#000; --panel:#0a0a0a;
|
|
--line:#232323; --line-hi:#3a3a3a; --accent:#f4f4f2;
|
|
--font-zen-sans:'Zen',-apple-system,BlinkMacSystemFont,"Segoe UI",Helvetica,Arial,sans-serif;
|
|
--font-zen-mono:'Zen Mono',ui-monospace,"SF Mono",SFMono-Regular,Menlo,Consolas,monospace;
|
|
--font-zen-pixel-square:'Zen Pixel Square',monospace;
|
|
--mono:var(--font-zen-mono);
|
|
--sans:var(--font-zen-sans);
|
|
/* Controls are pills and surfaces are soft, as on hanzo.ai. */
|
|
--r:999px; --r-soft:14px;
|
|
}
|
|
*{box-sizing:border-box}
|
|
html{-webkit-text-size-adjust:100%}
|
|
body{margin:0;background:var(--ground);color:var(--ink);font-family:var(--sans);
|
|
font-size:16px;line-height:1.65;-webkit-font-smoothing:antialiased}
|
|
.wrap{max-width:60rem;margin:0 auto;padding:0 1.75rem}
|
|
|
|
/* The dot field. A halftone made of one repeating radial-gradient, faded out
|
|
along the diagonal by a mask so it reads as a wave rather than wallpaper. */
|
|
/* No overflow clip here: the dot field is inset to the box already, and a clip
|
|
would cut off the copy-prompt menu. */
|
|
.hero{position:relative;border-bottom:1px solid var(--line)}
|
|
.hero::before{content:"";position:absolute;inset:0;
|
|
background-image:radial-gradient(var(--line-hi) 1px, transparent 1px);
|
|
background-size:7px 7px;
|
|
-webkit-mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
pointer-events:none}
|
|
.hero .wrap{position:relative;padding-top:5.5rem;padding-bottom:4rem}
|
|
|
|
nav{border-bottom:1px solid var(--line)}
|
|
nav .in{max-width:60rem;margin:0 auto;padding:1.1rem 1.75rem;display:flex;
|
|
justify-content:space-between;align-items:center;gap:1rem}
|
|
.brand{font-family:var(--mono);font-size:.85rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--ink);text-decoration:none;
|
|
display:inline-flex;align-items:center;gap:.6rem}
|
|
.brand i{font-style:normal;color:var(--faint)}
|
|
.mark{width:1.15em;height:1.15em;display:block;flex:0 0 auto}
|
|
nav a.up{font-family:var(--mono);font-size:.72rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--soft);text-decoration:none}
|
|
nav a.up:hover{color:var(--ink)}
|
|
|
|
h1{font-family:var(--mono);font-size:clamp(2rem,5.4vw,3.4rem);line-height:1.12;
|
|
letter-spacing:-.02em;margin:0 0 1.1rem;font-weight:500;text-wrap:balance}
|
|
h1 em{font-style:normal;color:var(--faint)}
|
|
.lede{color:var(--soft);font-size:1.05rem;max-width:52ch;margin:0 0 2.25rem}
|
|
|
|
.btns{display:flex;flex-wrap:wrap;gap:.7rem;margin:0 0 .5rem;align-items:center}
|
|
.btn{display:inline-flex;align-items:center;gap:.5rem;font-family:var(--mono);
|
|
font-size:.72rem;letter-spacing:.14em;text-transform:uppercase;
|
|
padding:.8rem 1.4rem;border:1px solid var(--line-hi);border-radius:var(--r);
|
|
color:var(--ink);text-decoration:none;background:transparent;cursor:pointer;
|
|
line-height:1.2;font-weight:400}
|
|
.btn:hover{background:#141414;text-decoration:none;border-color:var(--soft)}
|
|
.btn.solid{background:var(--ink);color:#000;border-color:var(--ink)}
|
|
.btn.solid:hover{background:#d8d8d4}
|
|
.btn .chev{width:.7em;height:.7em;flex:0 0 auto;transition:transform .15s ease}
|
|
.btn[aria-expanded="true"] .chev{transform:rotate(180deg)}
|
|
|
|
/* Copy-prompt selector. Each row copies the prompt for its own target. */
|
|
.copy{position:relative;display:inline-flex}
|
|
.menu{position:absolute;top:calc(100% + .5rem);left:0;z-index:20;min-width:21rem;
|
|
background:var(--panel);border:1px solid var(--line-hi);border-radius:var(--r-soft);
|
|
padding:.4rem;display:flex;flex-direction:column;
|
|
box-shadow:0 12px 32px rgba(0,0,0,.75)}
|
|
.menu[hidden]{display:none}
|
|
.mi{display:flex;align-items:center;gap:.7rem;width:100%;text-align:left;
|
|
padding:.62rem .8rem;border:0;border-radius:calc(var(--r-soft) - .35rem);
|
|
background:transparent;color:var(--ink);font-family:var(--mono);font-size:.76rem;
|
|
letter-spacing:.04em;text-decoration:none;cursor:pointer;line-height:1.3;
|
|
white-space:nowrap}
|
|
.mi:hover,.mi:focus-visible{background:#181818;outline:none}
|
|
.mi .mark{width:1.05rem;height:1.05rem;color:var(--soft)}
|
|
.mi:hover .mark{color:var(--ink)}
|
|
.mi u{margin-left:auto;color:var(--faint);text-decoration:none;font-size:.68rem;
|
|
letter-spacing:.1em;text-transform:uppercase}
|
|
|
|
/* Works-with bar: one scrolling row, name and mark per agent. */
|
|
.strip{border-bottom:1px solid var(--line);background:var(--panel)}
|
|
.strip .in{max-width:60rem;margin:0 auto;padding:.9rem 1.75rem;display:flex;
|
|
align-items:center;gap:1.25rem;font-family:var(--mono);font-size:.72rem;
|
|
letter-spacing:.1em;text-transform:uppercase;color:var(--faint)}
|
|
.strip b{color:var(--soft);font-weight:400;flex:0 0 auto;
|
|
border-right:1px solid var(--line);padding-right:1.25rem;align-self:stretch;
|
|
display:flex;align-items:center}
|
|
.bar{display:flex;align-items:center;gap:.5rem;flex:1 1 auto;min-width:0;
|
|
overflow-x:auto;scrollbar-width:none;scroll-snap-type:x proximity;
|
|
-webkit-overflow-scrolling:touch;
|
|
-webkit-mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent);
|
|
mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent)}
|
|
.bar::-webkit-scrollbar{display:none}
|
|
.agent{display:inline-flex;align-items:center;gap:.45rem;flex:0 0 auto;
|
|
padding:.4rem .85rem;border:1px solid var(--line);border-radius:var(--r);
|
|
color:var(--soft);white-space:nowrap;scroll-snap-align:start;
|
|
text-decoration:none;background:var(--ground)}
|
|
.agent:hover{color:var(--ink);border-color:var(--line-hi);text-decoration:none}
|
|
.agent .mark{width:.95rem;height:.95rem}
|
|
|
|
h2{font-family:var(--mono);font-size:1.1rem;letter-spacing:-.01em;margin:2.75rem 0 .75rem;font-weight:500}
|
|
h3{font-size:1rem;margin:2rem 0 .5rem;font-weight:600}
|
|
h4,h5,h6{font-size:.94rem;margin:1.5rem 0 .5rem;font-weight:600}
|
|
p{margin:0 0 1rem;max-width:70ch}
|
|
ul,ol{margin:0 0 1rem;padding-left:1.35rem;max-width:70ch}
|
|
li{margin:.22rem 0}
|
|
a{color:var(--ink);text-decoration:underline;text-underline-offset:3px;
|
|
text-decoration-color:var(--line-hi)}
|
|
a:hover{text-decoration-color:var(--ink)}
|
|
a:focus-visible{outline:1px solid var(--ink);outline-offset:3px}
|
|
code{font-family:var(--mono);font-size:.85em;background:var(--panel);
|
|
border:1px solid var(--line);border-radius:5px;padding:.1em .36em}
|
|
pre{background:var(--panel);border:1px solid var(--line);padding:1rem 1.1rem;
|
|
border-radius:var(--r-soft);overflow-x:auto;margin:0 0 1.25rem}
|
|
pre code{background:none;border:0;padding:0;font-size:.82rem;line-height:1.7}
|
|
blockquote{margin:0 0 1rem;padding-left:1rem;border-left:1px solid var(--line-hi);color:var(--soft)}
|
|
hr{border:0;border-top:1px solid var(--line);margin:2.5rem 0}
|
|
table{border-collapse:collapse;width:100%;margin:0 0 1.25rem;font-size:.92rem;
|
|
display:block;overflow-x:auto}
|
|
th,td{border:1px solid var(--line);padding:.5rem .7rem;text-align:left}
|
|
|
|
.curl{display:block;background:var(--panel);border:1px solid var(--line);
|
|
border-radius:var(--r-soft);padding:.95rem 1.1rem;font-family:var(--mono);
|
|
font-size:.8rem;color:var(--soft);overflow-x:auto;margin:0 0 2.5rem;
|
|
white-space:pre;line-height:1.9}
|
|
.curl b{color:var(--ink);font-weight:400}
|
|
.curl i{font-style:normal;color:var(--faint)}
|
|
|
|
/* index */
|
|
#q{width:100%;padding:.85rem 1.2rem;font-family:var(--mono);font-size:.85rem;color:var(--ink);
|
|
background:var(--panel);border:1px solid var(--line);border-radius:var(--r);margin:0 0 .6rem}
|
|
#q::placeholder{color:var(--faint);text-transform:uppercase;letter-spacing:.1em;font-size:.72rem}
|
|
#q:focus{outline:none;border-color:var(--soft)}
|
|
#count{color:var(--faint);font-family:var(--mono);font-size:.72rem;letter-spacing:.1em;
|
|
text-transform:uppercase;margin:0 0 2rem}
|
|
.cat{margin:2.5rem 0 .5rem;font-family:var(--mono);font-size:.7rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);border-bottom:1px solid var(--line);
|
|
padding-bottom:.45rem}
|
|
.row{display:block;padding:.6rem .9rem;margin:0 -.9rem;border-bottom:1px solid var(--line);
|
|
border-radius:var(--r-soft);text-decoration:none}
|
|
.row:hover{background:#0a0a0a}
|
|
.nm{font-family:var(--mono);font-size:.84rem;color:var(--ink);display:block}
|
|
.ds{color:var(--soft);font-size:.9rem;display:block;margin-top:.12rem}
|
|
footer{margin-top:4rem;padding:2rem 0 4rem;border-top:1px solid var(--line);
|
|
color:var(--faint);font-size:.85rem}
|
|
footer a{color:var(--soft)}
|
|
.fmap{display:grid;gap:1.75rem;grid-template-columns:repeat(auto-fit,minmax(11rem,1fr));
|
|
margin:0 0 2rem;font-size:.85rem}
|
|
.fmap b{display:block;font-family:var(--mono);font-size:.68rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);font-weight:400;margin:0 0 .6rem;
|
|
border-bottom:1px solid var(--line);padding-bottom:.4rem}
|
|
.fmap a{display:inline-block;color:var(--ink);text-decoration:none;margin:.3rem .35rem 0 0}
|
|
.fmap a:hover{text-decoration:underline}
|
|
.fmap div{color:var(--faint);line-height:1.5}
|
|
.fend{border-top:1px solid var(--line);padding-top:1.25rem;font-size:.82rem}
|
|
|
|
@media (max-width:640px){
|
|
.hero .wrap{padding-top:3.25rem;padding-bottom:2.75rem}
|
|
.btns{gap:.5rem}
|
|
.btn{padding:.75rem 1.15rem;font-size:.68rem}
|
|
.copy,.copy .btn{width:100%;justify-content:center}
|
|
.menu{left:0;right:0;min-width:0}
|
|
.strip .in{flex-direction:column;align-items:flex-start;gap:.6rem;padding:.9rem 1.75rem}
|
|
.strip b{border-right:0;padding-right:0}
|
|
.bar{width:100%}
|
|
}
|
|
</style>
|
|
<link rel="alternate" type="text/markdown" href="/ir-ranking-reranking.md">
|
|
|
|
<link rel="icon" href="/favicon.svg">
|
|
<link rel="canonical" href="https://hanzoskills.com/ir-ranking-reranking">
|
|
</head>
|
|
<!-- Reading this with a machine? Every page here is also plain markdown.
|
|
This one: https://hanzoskills.com/ir-ranking-reranking.md
|
|
Index: https://hanzoskills.com/llms.txt
|
|
Corpus: https://hanzoskills.com/all.md
|
|
Append .md to any path for the markdown twin. -->
|
|
<body>
|
|
<nav><div class="in">
|
|
<a class="brand" href="/"><svg class="mark" viewBox="0 0 67 67" fill="currentColor" aria-hidden="true"><path d="M22.21 67V44.6369H0V67H22.21Z"/><path d="M66.7038 22.3184H22.2534L0.0878906 44.6367H44.4634L66.7038 22.3184Z"/><path d="M22.21 0H0V22.3184H22.21V0Z"/><path d="M66.7198 0H44.5098V22.3184H66.7198V0Z"/><path d="M66.7198 67V44.6369H44.5098V67H66.7198Z"/></svg> HANZO<i>SKILLS</i></a>
|
|
<a class="up" href="/">All skills</a>
|
|
</div></nav>
|
|
|
|
<div class="wrap">
|
|
<h1>ir-ranking-reranking</h1><p class="lede">Learning to rank, cross-encoder reranking, ranking features, and evaluation metrics (nDCG, MAP, MRR)</p><h1>Information Retrieval: Ranking and Reranking</h1>
|
|
<p><strong>Scope</strong>: Learning to rank algorithms, cross-encoder reranking, ranking features, evaluation metrics, and two-stage retrieval <strong>Lines</strong>: ~320 <strong>Last Updated</strong>: 2025-10-25 <strong>Format Version</strong>: 1.0 (Atomic)</p>
|
|
<hr>
|
|
<h2>When to Use This Skill</h2>
|
|
<p>Activate this skill when:</p>
|
|
<ul><li>Improving search result quality beyond initial retrieval</li><li>Implementing two-stage retrieval (fast retrieve, accurate rerank)</li><li>Training custom ranking models with user feedback</li><li>Using cross-encoders for semantic reranking</li><li>Evaluating ranking quality with nDCG, MAP, or MRR</li><li>Combining multiple relevance signals into final ranking</li><li>Balancing retrieval speed vs ranking accuracy</li><li>Personalizing search results based on user behavior</li></ul>
|
|
<h2>Core Concepts</h2>
|
|
<h3>Concept 1: Learning to Rank (LTR)</h3>
|
|
<p><strong>Three Approaches</strong>:</p>
|
|
<p><strong>Pointwise</strong>: Predict relevance score for each document independently</p>
|
|
<ul><li>Treat as regression/classification: predict score for (query, document)</li><li>Simple but ignores relative ordering</li></ul>
|
|
<p><strong>Pairwise</strong>: Learn which document should rank higher</p>
|
|
<ul><li>Classify pairs: is doc A > doc B?</li><li>Examples: RankNet, LambdaRank</li><li>Better than pointwise for ranking tasks</li></ul>
|
|
<p><strong>Listwise</strong>: Optimize entire ranking directly</p>
|
|
<ul><li>Loss function considers full ranked list</li><li>Examples: LambdaMART, ListNet</li><li>Best performance but more complex</li></ul>
|
|
<pre><code class="lang-python">from sklearn.ensemble import GradientBoostingRegressor
|
|
import numpy as np
|
|
|
|
# Pointwise LTR (regression)
|
|
class PointwiseRanker:
|
|
def __init__(self):
|
|
self.model = GradientBoostingRegressor(n_estimators=100)
|
|
|
|
def train(self, features, relevance_scores):
|
|
"""
|
|
features: (n_samples, n_features) - query-doc features
|
|
relevance_scores: (n_samples,) - relevance labels (0-4)
|
|
"""
|
|
self.model.fit(features, relevance_scores)
|
|
|
|
def rank(self, query_features):
|
|
"""Predict scores and rank documents"""
|
|
scores = self.model.predict(query_features)
|
|
ranked_indices = np.argsort(scores)[::-1]
|
|
return ranked_indices, scores[ranked_indices]
|
|
|
|
# Example features: BM25, embeddings similarity, freshness, etc.
|
|
features = np.array([
|
|
[0.8, 0.6, 1.0], # doc1: high BM25, medium semantic, fresh
|
|
[0.3, 0.9, 0.5], # doc2: low BM25, high semantic, old
|
|
[0.9, 0.7, 0.8], # doc3: high BM25, high semantic, recent
|
|
])
|
|
relevance = np.array([3, 2, 4]) # Ground truth ratings
|
|
|
|
ranker = PointwiseRanker()
|
|
ranker.train(features, relevance)
|
|
ranked_indices, scores = ranker.rank(features)
|
|
print(f"Ranked order: {ranked_indices}") # [2, 0, 1]</code></pre>
|
|
<h3>Concept 2: Cross-Encoder Reranking</h3>
|
|
<p><strong>Bi-encoder vs Cross-encoder</strong>:</p>
|
|
<ul><li><strong>Bi-encoder</strong>: Separate encoders for query and document (fast, scalable)</li><li>Encode once, search millions of vectors</li><li>Example: sentence-transformers for vector search</li></ul>
|
|
<ul><li><strong>Cross-encoder</strong>: Joint encoding of query + document (slow, accurate)</li><li>Process query and document together</li><li>Captures interaction between terms</li><li>10-100x slower but much better quality</li></ul>
|
|
<p><strong>Two-Stage Retrieval</strong>:</p>
|
|
<ol><li><strong>Stage 1</strong>: Bi-encoder retrieves top-k candidates (k=100-1000)</li><li><strong>Stage 2</strong>: Cross-encoder reranks top-n (n=10-50)</li></ol>
|
|
<pre><code class="lang-python">from sentence_transformers import CrossEncoder
|
|
|
|
# Load cross-encoder for reranking
|
|
cross_encoder = CrossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2')
|
|
|
|
def two_stage_retrieval(query, documents, bi_encoder, cross_encoder, top_k=100, rerank_top=10):
|
|
"""
|
|
Stage 1: Fast bi-encoder retrieval
|
|
Stage 2: Accurate cross-encoder reranking
|
|
"""
|
|
# Stage 1: Retrieve top_k with bi-encoder
|
|
query_emb = bi_encoder.encode(query, convert_to_tensor=True)
|
|
doc_embs = bi_encoder.encode(documents, convert_to_tensor=True)
|
|
|
|
from sentence_transformers import util
|
|
scores = util.cos_sim(query_emb, doc_embs)[0]
|
|
top_indices = scores.argsort(descending=True)[:top_k]
|
|
|
|
# Stage 2: Rerank top_k with cross-encoder
|
|
candidates = [documents[i] for i in top_indices]
|
|
pairs = [[query, doc] for doc in candidates]
|
|
|
|
cross_scores = cross_encoder.predict(pairs)
|
|
rerank_indices = np.argsort(cross_scores)[::-1][:rerank_top]
|
|
|
|
# Map back to original indices
|
|
final_indices = [top_indices[i] for i in rerank_indices]
|
|
final_scores = [cross_scores[i] for i in rerank_indices]
|
|
|
|
return final_indices, final_scores
|
|
|
|
# Example
|
|
from sentence_transformers import SentenceTransformer
|
|
|
|
bi_encoder = SentenceTransformer('all-MiniLM-L6-v2')
|
|
query = "machine learning classification algorithms"
|
|
documents = [...] # Large corpus
|
|
|
|
top_docs, scores = two_stage_retrieval(query, documents, bi_encoder, cross_encoder)</code></pre>
|
|
<h3>Concept 3: Ranking Features</h3>
|
|
<p><strong>Categories</strong>:</p>
|
|
<p><strong>Query-Document Features</strong>:</p>
|
|
<ul><li>TF-IDF/BM25 score</li><li>Vector similarity (cosine, dot product)</li><li>Exact match signals (title, URL)</li><li>Proximity (terms close in document)</li></ul>
|
|
<p><strong>Document Features</strong>:</p>
|
|
<ul><li>PageRank, authority score</li><li>Freshness (recency)</li><li>Length (too short/long)</li><li>Click-through rate (CTR)</li><li>Engagement metrics (time on page, bounce rate)</li></ul>
|
|
<p><strong>Query Features</strong>:</p>
|
|
<ul><li>Query length</li><li>Query type (navigational, informational)</li><li>User history, personalization</li></ul>
|
|
<pre><code class="lang-python">def extract_features(query, document, metadata):
|
|
"""Extract features for ranking"""
|
|
features = {}
|
|
|
|
# Lexical features
|
|
features['bm25_score'] = compute_bm25(query, document)
|
|
features['exact_match_title'] = int(query.lower() in document['title'].lower())
|
|
features['term_overlap'] = compute_term_overlap(query, document['content'])
|
|
|
|
# Semantic features
|
|
query_emb = model.encode(query)
|
|
doc_emb = model.encode(document['content'])
|
|
features['semantic_similarity'] = cosine_similarity(query_emb, doc_emb)
|
|
|
|
# Document quality features
|
|
features['page_rank'] = metadata.get('page_rank', 0)
|
|
features['freshness_days'] = (today - metadata['publish_date']).days
|
|
features['avg_time_on_page'] = metadata.get('avg_time_on_page', 0)
|
|
features['ctr'] = metadata.get('ctr', 0)
|
|
|
|
# Document length
|
|
features['doc_length'] = len(document['content'].split())
|
|
features['title_length'] = len(document['title'].split())
|
|
|
|
# User personalization (if available)
|
|
features['user_clicked_similar'] = check_user_history(query, document)
|
|
|
|
return list(features.values())</code></pre>
|
|
<h3>Concept 4: Evaluation Metrics</h3>
|
|
<p><strong>nDCG (Normalized Discounted Cumulative Gain)</strong>:</p>
|
|
<ul><li>Considers both relevance and position</li><li>Discounts lower positions (log discount)</li><li>Range: 0 to 1 (1 = perfect ranking)</li></ul>
|
|
<p><strong>MAP (Mean Average Precision)</strong>:</p>
|
|
<ul><li>Average precision across queries</li><li>Emphasizes precision at each relevant item</li><li>Binary relevance (relevant vs not)</li></ul>
|
|
<p><strong>MRR (Mean Reciprocal Rank)</strong>:</p>
|
|
<ul><li>Inverse rank of first relevant result</li><li>Good for "find one answer" tasks</li><li>Range: 0 to 1</li></ul>
|
|
<pre><code class="lang-python">import numpy as np
|
|
|
|
def dcg_at_k(relevance_scores, k):
|
|
"""Discounted Cumulative Gain at rank k"""
|
|
relevance = np.array(relevance_scores)[:k]
|
|
gains = 2**relevance - 1
|
|
discounts = np.log2(np.arange(2, k + 2))
|
|
return np.sum(gains / discounts)
|
|
|
|
def ndcg_at_k(predicted_order, ground_truth_scores, k):
|
|
"""Normalized DCG at rank k"""
|
|
# DCG for predicted ranking
|
|
predicted_relevance = [ground_truth_scores[i] for i in predicted_order[:k]]
|
|
dcg = dcg_at_k(predicted_relevance, k)
|
|
|
|
# IDCG (ideal ranking)
|
|
ideal_relevance = sorted(ground_truth_scores, reverse=True)
|
|
idcg = dcg_at_k(ideal_relevance, k)
|
|
|
|
return dcg / idcg if idcg > 0 else 0.0
|
|
|
|
def mean_average_precision(predicted_orders, ground_truth_binary):
|
|
"""MAP across queries (binary relevance)"""
|
|
aps = []
|
|
for predicted, relevant_indices in zip(predicted_orders, ground_truth_binary):
|
|
precisions = []
|
|
num_relevant = 0
|
|
|
|
for k, doc_id in enumerate(predicted, 1):
|
|
if doc_id in relevant_indices:
|
|
num_relevant += 1
|
|
precisions.append(num_relevant / k)
|
|
|
|
ap = np.mean(precisions) if precisions else 0.0
|
|
aps.append(ap)
|
|
|
|
return np.mean(aps)
|
|
|
|
def mean_reciprocal_rank(predicted_orders, ground_truth_binary):
|
|
"""MRR across queries"""
|
|
rrs = []
|
|
for predicted, relevant_indices in zip(predicted_orders, ground_truth_binary):
|
|
for rank, doc_id in enumerate(predicted, 1):
|
|
if doc_id in relevant_indices:
|
|
rrs.append(1.0 / rank)
|
|
break
|
|
else:
|
|
rrs.append(0.0)
|
|
|
|
return np.mean(rrs)
|
|
|
|
# Example evaluation
|
|
predicted_order = [2, 0, 3, 1] # doc IDs
|
|
ground_truth_scores = [3, 1, 4, 2] # relevance scores (0-4)
|
|
|
|
ndcg = ndcg_at_k(predicted_order, ground_truth_scores, k=10)
|
|
print(f"nDCG@10: {ndcg:.4f}")</code></pre>
|
|
<hr>
|
|
<h2>Patterns</h2>
|
|
<h3>Pattern 1: Hybrid Ranking (Combine Multiple Signals)</h3>
|
|
<p><strong>When to use</strong>: Multiple relevance signals available (BM25, semantic, popularity)</p>
|
|
<pre><code class="lang-python">def hybrid_rank(query, documents, weights={'lexical': 0.3, 'semantic': 0.5, 'popularity': 0.2}):
|
|
"""Combine multiple ranking signals"""
|
|
scores = np.zeros(len(documents))
|
|
|
|
# Lexical score (BM25)
|
|
bm25_scores = np.array([compute_bm25(query, doc) for doc in documents])
|
|
bm25_normalized = bm25_scores / bm25_scores.max() if bm25_scores.max() > 0 else bm25_scores
|
|
|
|
# Semantic score (embeddings)
|
|
query_emb = model.encode(query)
|
|
doc_embs = model.encode([doc['content'] for doc in documents])
|
|
semantic_scores = [cosine_similarity(query_emb, doc_emb) for doc_emb in doc_embs]
|
|
semantic_normalized = np.array(semantic_scores)
|
|
|
|
# Popularity score
|
|
popularity_scores = np.array([doc.get('popularity', 0) for doc in documents])
|
|
popularity_normalized = popularity_scores / popularity_scores.max() if popularity_scores.max() > 0 else popularity_scores
|
|
|
|
# Weighted combination
|
|
scores = (
|
|
weights['lexical'] * bm25_normalized +
|
|
weights['semantic'] * semantic_normalized +
|
|
weights['popularity'] * popularity_normalized
|
|
)
|
|
|
|
ranked_indices = np.argsort(scores)[::-1]
|
|
return ranked_indices, scores[ranked_indices]</code></pre>
|
|
<h3>Pattern 2: LambdaMART Training with LightGBM</h3>
|
|
<p><strong>Use case</strong>: Train listwise ranking model with user feedback data</p>
|
|
<pre><code class="lang-python">import lightgbm as lgb
|
|
|
|
# Prepare data in LightGBM ranking format
|
|
# features: (n_samples, n_features)
|
|
# labels: relevance scores (0-4)
|
|
# qids: query IDs (group by query)
|
|
|
|
train_data = lgb.Dataset(
|
|
features,
|
|
label=labels,
|
|
group=query_groups # Number of docs per query
|
|
)
|
|
|
|
params = {
|
|
'objective': 'lambdarank',
|
|
'metric': 'ndcg',
|
|
'ndcg_eval_at': [1, 3, 5, 10],
|
|
'learning_rate': 0.1,
|
|
'num_leaves': 31,
|
|
'min_data_in_leaf': 50
|
|
}
|
|
|
|
ranker = lgb.train(
|
|
params,
|
|
train_data,
|
|
num_boost_round=100,
|
|
valid_sets=[valid_data],
|
|
callbacks=[lgb.early_stopping(10)]
|
|
)
|
|
|
|
# Predict and rank
|
|
predictions = ranker.predict(test_features)
|
|
ranked_indices = np.argsort(predictions)[::-1]</code></pre>
|
|
<h3>Pattern 3: Personalized Ranking</h3>
|
|
<p><strong>When to use</strong>: User history available for personalization</p>
|
|
<pre><code class="lang-python">def personalized_rank(query, documents, user_profile, base_scores):
|
|
"""Adjust ranking based on user preferences"""
|
|
personalized_scores = base_scores.copy()
|
|
|
|
for i, doc in enumerate(documents):
|
|
# Boost based on user's preferred categories
|
|
if doc['category'] in user_profile.get('preferred_categories', []):
|
|
personalized_scores[i] *= 1.2
|
|
|
|
# Boost based on user's interaction history
|
|
if doc['id'] in user_profile.get('clicked_docs', []):
|
|
personalized_scores[i] *= 1.3
|
|
|
|
# Boost based on similar documents user engaged with
|
|
similarity = compute_user_doc_similarity(user_profile, doc)
|
|
personalized_scores[i] *= (1 + similarity * 0.5)
|
|
|
|
# Penalize recently viewed
|
|
if doc['id'] in user_profile.get('recent_views', []):
|
|
personalized_scores[i] *= 0.8
|
|
|
|
ranked_indices = np.argsort(personalized_scores)[::-1]
|
|
return ranked_indices, personalized_scores[ranked_indices]</code></pre>
|
|
<h3>Pattern 4: Diversity-Aware Ranking</h3>
|
|
<p><strong>Use case</strong>: Avoid redundant results, show diverse perspectives</p>
|
|
<pre><code class="lang-python">def diverse_rerank(candidates, scores, diversity_penalty=0.3):
|
|
"""Rerank to promote diversity (MMR-style)"""
|
|
ranked = []
|
|
remaining = list(range(len(candidates)))
|
|
doc_embeddings = model.encode([c['content'] for c in candidates])
|
|
|
|
while remaining and len(ranked) < 10:
|
|
best_idx = None
|
|
best_score = -float('inf')
|
|
|
|
for i in remaining:
|
|
# Original relevance score
|
|
relevance = scores[i]
|
|
|
|
# Diversity penalty: similarity to already ranked docs
|
|
if ranked:
|
|
ranked_embeddings = doc_embeddings[ranked]
|
|
similarities = [cosine_similarity(doc_embeddings[i], emb) for emb in ranked_embeddings]
|
|
diversity_penalty_value = max(similarities)
|
|
else:
|
|
diversity_penalty_value = 0
|
|
|
|
# MMR score: balance relevance and diversity
|
|
mmr_score = relevance - diversity_penalty * diversity_penalty_value
|
|
|
|
if mmr_score > best_score:
|
|
best_score = mmr_score
|
|
best_idx = i
|
|
|
|
ranked.append(best_idx)
|
|
remaining.remove(best_idx)
|
|
|
|
return ranked</code></pre>
|
|
<h3>Pattern 5: A/B Testing Ranking Models</h3>
|
|
<p><strong>When to use</strong>: Validate ranking improvements with real users</p>
|
|
<pre><code class="lang-python">import random
|
|
|
|
def ab_test_ranker(query, documents, ranker_a, ranker_b, user_id):
|
|
"""Route user to control or test ranker"""
|
|
# Assign user to variant (deterministic based on user_id)
|
|
variant = 'b' if hash(user_id) % 2 == 0 else 'a'
|
|
|
|
if variant == 'a':
|
|
results = ranker_a.rank(query, documents)
|
|
variant_label = 'control'
|
|
else:
|
|
results = ranker_b.rank(query, documents)
|
|
variant_label = 'test'
|
|
|
|
# Log for analysis
|
|
log_experiment(
|
|
user_id=user_id,
|
|
query=query,
|
|
variant=variant_label,
|
|
results=results
|
|
)
|
|
|
|
return results
|
|
|
|
# Analyze A/B test results
|
|
def analyze_ab_test(logs):
|
|
"""Compare metrics between variants"""
|
|
control_ndcg = []
|
|
test_ndcg = []
|
|
|
|
for log in logs:
|
|
if log['variant'] == 'control':
|
|
control_ndcg.append(log['ndcg'])
|
|
else:
|
|
test_ndcg.append(log['ndcg'])
|
|
|
|
print(f"Control nDCG: {np.mean(control_ndcg):.4f}")
|
|
print(f"Test nDCG: {np.mean(test_ndcg):.4f}")
|
|
print(f"Lift: {(np.mean(test_ndcg) / np.mean(control_ndcg) - 1) * 100:.2f}%")</code></pre>
|
|
<h3>Pattern 6: Real-Time Ranking with Caching</h3>
|
|
<p><strong>When to use</strong>: Reduce latency for repeated queries</p>
|
|
<pre><code class="lang-python">from functools import lru_cache
|
|
import hashlib
|
|
|
|
class CachedRanker:
|
|
def __init__(self, ranker, cache_size=1000):
|
|
self.ranker = ranker
|
|
self.cache = {}
|
|
self.cache_size = cache_size
|
|
|
|
def _cache_key(self, query, doc_ids):
|
|
"""Generate cache key from query and document set"""
|
|
key_str = f"{query}:{sorted(doc_ids)}"
|
|
return hashlib.md5(key_str.encode()).hexdigest()
|
|
|
|
def rank(self, query, documents):
|
|
"""Rank with caching"""
|
|
doc_ids = tuple(doc['id'] for doc in documents)
|
|
cache_key = self._cache_key(query, doc_ids)
|
|
|
|
if cache_key in self.cache:
|
|
return self.cache[cache_key]
|
|
|
|
# Compute ranking
|
|
results = self.ranker.rank(query, documents)
|
|
|
|
# Cache results (LRU eviction if needed)
|
|
if len(self.cache) >= self.cache_size:
|
|
# Remove oldest entry
|
|
oldest_key = next(iter(self.cache))
|
|
del self.cache[oldest_key]
|
|
|
|
self.cache[cache_key] = results
|
|
return results</code></pre>
|
|
<hr>
|
|
<h2>Quick Reference</h2>
|
|
<h3>Ranking Approaches</h3>
|
|
<pre><code>Approach | Training Data | Complexity | Quality
|
|
--------------|----------------------------|------------|--------
|
|
Pointwise | (query, doc) → score | Low | Good
|
|
Pairwise | (query, docA, docB) → A>B | Medium | Better
|
|
Listwise | (query, docs) → ranking | High | Best
|
|
Cross-encoder | (query, doc) → score | Very High | Excellent</code></pre>
|
|
<h3>Evaluation Metrics</h3>
|
|
<pre><code>Metric | Formula | Use Case
|
|
---------|----------------------------------|----------
|
|
nDCG@k | DCG / IDCG (log discount) | Graded relevance, position matters
|
|
MAP | Mean of Precision@k | Binary relevance, all relevant items
|
|
MRR | 1 / rank of first relevant | Single answer tasks
|
|
P@k | Relevant in top k / k | Binary relevance, top-k quality</code></pre>
|
|
<h3>Key Guidelines</h3>
|
|
<pre><code>✅ DO: Use two-stage retrieval (fast retrieve, accurate rerank)
|
|
✅ DO: Combine multiple signals (lexical, semantic, popularity)
|
|
✅ DO: Evaluate with nDCG for graded relevance
|
|
✅ DO: A/B test ranking changes with real users
|
|
✅ DO: Cache ranking results for popular queries
|
|
✅ DO: Promote diversity to avoid redundant results
|
|
|
|
❌ DON'T: Use cross-encoder for initial retrieval (too slow)
|
|
❌ DON'T: Ignore position in ranking evaluation
|
|
❌ DON'T: Deploy ranking changes without A/B testing
|
|
❌ DON'T: Use only lexical or only semantic (hybrid is better)
|
|
❌ DON'T: Forget to normalize scores before combining</code></pre>
|
|
<hr>
|
|
<h2>Anti-Patterns</h2>
|
|
<h3>Critical Violations</h3>
|
|
<pre><code class="lang-python"># ❌ NEVER: Use cross-encoder for full corpus search
|
|
for doc in million_documents:
|
|
score = cross_encoder.predict([[query, doc['content']]])
|
|
# This takes hours!
|
|
|
|
# ✅ CORRECT: Two-stage retrieval
|
|
# Stage 1: Retrieve top 100 with bi-encoder (fast)
|
|
candidates = bi_encoder_retrieve(query, million_documents, top_k=100)
|
|
|
|
# Stage 2: Rerank with cross-encoder (accurate)
|
|
reranked = cross_encoder_rerank(query, candidates, top_k=10)</code></pre>
|
|
<p>❌ <strong>Cross-encoder on full corpus</strong>: Hours of compute, unacceptable latency ✅ <strong>Correct approach</strong>: Fast retrieval, then accurate reranking</p>
|
|
<h3>Common Mistakes</h3>
|
|
<pre><code class="lang-python"># ❌ Don't: Combine scores without normalization
|
|
combined_score = bm25_score + semantic_score + popularity_score
|
|
# Problem: Scales differ (BM25: 0-100, semantic: 0-1, popularity: 0-1M)
|
|
|
|
# ✅ Correct: Normalize before combining
|
|
bm25_norm = bm25_score / max_bm25
|
|
semantic_norm = semantic_score # Already 0-1
|
|
popularity_norm = popularity_score / max_popularity
|
|
|
|
combined_score = 0.4 * bm25_norm + 0.4 * semantic_norm + 0.2 * popularity_norm</code></pre>
|
|
<p>❌ <strong>Unnormalized score combination</strong>: Dominant signal overwhelms others ✅ <strong>Better</strong>: Normalize to same range, then weighted combination</p>
|
|
<pre><code class="lang-python"># ❌ Don't: Ignore query groups in LTR training
|
|
ranker.fit(features, labels) # Treats as independent samples
|
|
|
|
# ✅ Correct: Specify query groups
|
|
ranker.fit(features, labels, group=query_groups)
|
|
# Ensures ranking loss considers docs within same query</code></pre>
|
|
<p>❌ <strong>Missing query groups</strong>: Model learns wrong objective ✅ <strong>Better</strong>: Always specify query groupings in LTR</p>
|
|
<pre><code class="lang-python"># ❌ Don't: Use only top-1 accuracy
|
|
accuracy = int(predicted_order[0] == best_doc_id)
|
|
|
|
# ✅ Correct: Use ranking metrics
|
|
ndcg = ndcg_at_k(predicted_order, ground_truth_scores, k=10)
|
|
map_score = mean_average_precision([predicted_order], [relevant_docs])</code></pre>
|
|
<p>❌ <strong>Top-1 only</strong>: Ignores quality of full ranking ✅ <strong>Better</strong>: Use nDCG, MAP, or MRR for ranking evaluation</p>
|
|
<hr>
|
|
<h2>Related Skills</h2>
|
|
<ul><li><code>ir-search-fundamentals.md</code> - BM25 and lexical features for ranking</li><li><code>ir-vector-search.md</code> - Bi-encoders for initial retrieval stage</li><li><code>ir-query-understanding.md</code> - Query expansion before ranking</li><li><code>ir-recommendation-systems.md</code> - Similar ranking techniques for recommendations</li><li><code>test-unit-testing.md</code> - Testing ranking models and metrics</li></ul>
|
|
<hr>
|
|
<p><strong>Last Updated</strong>: 2025-10-25 <strong>Format Version</strong>: 1.0 (Atomic)</p>
|
|
<footer>
|
|
<div class="fmap">
|
|
<div><b>Build</b>
|
|
<a href="https://hanzo.app">hanzo.app</a> app builder
|
|
<a href="https://hanzo.codes">hanzo.codes</a> code with an agent
|
|
<a href="https://hanzoskills.com">hanzoskills.com</a> what an agent reads</div>
|
|
<div><b>Run</b>
|
|
<a href="https://console.hanzo.ai">console.hanzo.ai</a> cloud console
|
|
<a href="https://api.hanzo.ai">api.hanzo.ai</a> the API</div>
|
|
<div><b>Account</b>
|
|
<a href="https://hanzo.id">hanzo.id</a> sign in
|
|
<a href="https://pay.hanzo.ai">pay.hanzo.ai</a> top up
|
|
<a href="https://billing.hanzo.ai">billing.hanzo.ai</a> invoices and usage</div>
|
|
<div><b>Company</b>
|
|
<a href="https://hanzo.ai">hanzo.ai</a> canonical
|
|
<a href="https://hanzo.ventures">hanzo.ventures</a> ventures</div>
|
|
</div>
|
|
<div class="fend"><a href="/ir-ranking-reranking.md">This page as plain text</a> · <a href="/llms.txt">Index for agents</a></div>
|
|
</footer>
|
|
</div></body></html> |