mirror of
https://github.com/hanzoai/skills.git
synced 2026-11-19 00:47:36 +00:00
hanzoskills.com/llms.txt now opens with the install line, the API reference index and the company index, so an agent that lands on any one of the three can find the rest. robots.txt claimed 537 documents while llms.txt counts 920. The number is gone from robots.txt; llms.txt derives it. public/ is regenerated from scripts/site.py, which it had fallen behind: the copy-prompt menu and agent bar were in the generator and not in the committed pages. tests/site.mjs passes against the result. Co-authored-by: Hanzo Dev <[email protected]>
606 lines
28 KiB
HTML
606 lines
28 KiB
HTML
<!doctype html>
|
|
<html lang="en"><head>
|
|
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
<title>dspy-evaluation</title>
|
|
<meta name="color-scheme" content="dark">
|
|
<style>
|
|
/* Committed to one visual world: a dark instrument panel. No light variant —
|
|
the dot field and the hairline rules only read on black, and a naive
|
|
inversion would wash both out. Every colour is painted explicitly so the page
|
|
never borrows a host background. */
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Pixel Square';src:url('/fonts/ZenPixel-Square.woff2') format('woff2');font-style:normal;font-display:swap}
|
|
|
|
:root{
|
|
--ink:#f4f4f2; --soft:#9c9c98; --faint:#6a6a66; --ground:#000; --panel:#0a0a0a;
|
|
--line:#232323; --line-hi:#3a3a3a; --accent:#f4f4f2;
|
|
--font-zen-sans:'Zen',-apple-system,BlinkMacSystemFont,"Segoe UI",Helvetica,Arial,sans-serif;
|
|
--font-zen-mono:'Zen Mono',ui-monospace,"SF Mono",SFMono-Regular,Menlo,Consolas,monospace;
|
|
--font-zen-pixel-square:'Zen Pixel Square',monospace;
|
|
--mono:var(--font-zen-mono);
|
|
--sans:var(--font-zen-sans);
|
|
/* Controls are pills and surfaces are soft, as on hanzo.ai. */
|
|
--r:999px; --r-soft:14px;
|
|
}
|
|
*{box-sizing:border-box}
|
|
html{-webkit-text-size-adjust:100%}
|
|
body{margin:0;background:var(--ground);color:var(--ink);font-family:var(--sans);
|
|
font-size:16px;line-height:1.65;-webkit-font-smoothing:antialiased}
|
|
.wrap{max-width:60rem;margin:0 auto;padding:0 1.75rem}
|
|
|
|
/* The dot field. A halftone made of one repeating radial-gradient, faded out
|
|
along the diagonal by a mask so it reads as a wave rather than wallpaper. */
|
|
/* No overflow clip here: the dot field is inset to the box already, and a clip
|
|
would cut off the copy-prompt menu. */
|
|
.hero{position:relative;border-bottom:1px solid var(--line)}
|
|
.hero::before{content:"";position:absolute;inset:0;
|
|
background-image:radial-gradient(var(--line-hi) 1px, transparent 1px);
|
|
background-size:7px 7px;
|
|
-webkit-mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
pointer-events:none}
|
|
.hero .wrap{position:relative;padding-top:5.5rem;padding-bottom:4rem}
|
|
|
|
nav{border-bottom:1px solid var(--line)}
|
|
nav .in{max-width:60rem;margin:0 auto;padding:1.1rem 1.75rem;display:flex;
|
|
justify-content:space-between;align-items:center;gap:1rem}
|
|
.brand{font-family:var(--mono);font-size:.85rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--ink);text-decoration:none;
|
|
display:inline-flex;align-items:center;gap:.6rem}
|
|
.brand i{font-style:normal;color:var(--faint)}
|
|
.mark{width:1.15em;height:1.15em;display:block;flex:0 0 auto}
|
|
nav a.up{font-family:var(--mono);font-size:.72rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--soft);text-decoration:none}
|
|
nav a.up:hover{color:var(--ink)}
|
|
|
|
h1{font-family:var(--mono);font-size:clamp(2rem,5.4vw,3.4rem);line-height:1.12;
|
|
letter-spacing:-.02em;margin:0 0 1.1rem;font-weight:500;text-wrap:balance}
|
|
h1 em{font-style:normal;color:var(--faint)}
|
|
.lede{color:var(--soft);font-size:1.05rem;max-width:52ch;margin:0 0 2.25rem}
|
|
|
|
.btns{display:flex;flex-wrap:wrap;gap:.7rem;margin:0 0 .5rem;align-items:center}
|
|
.btn{display:inline-flex;align-items:center;gap:.5rem;font-family:var(--mono);
|
|
font-size:.72rem;letter-spacing:.14em;text-transform:uppercase;
|
|
padding:.8rem 1.4rem;border:1px solid var(--line-hi);border-radius:var(--r);
|
|
color:var(--ink);text-decoration:none;background:transparent;cursor:pointer;
|
|
line-height:1.2;font-weight:400}
|
|
.btn:hover{background:#141414;text-decoration:none;border-color:var(--soft)}
|
|
.btn.solid{background:var(--ink);color:#000;border-color:var(--ink)}
|
|
.btn.solid:hover{background:#d8d8d4}
|
|
.btn .chev{width:.7em;height:.7em;flex:0 0 auto;transition:transform .15s ease}
|
|
.btn[aria-expanded="true"] .chev{transform:rotate(180deg)}
|
|
|
|
/* Copy-prompt selector. Each row copies the prompt for its own target. */
|
|
.copy{position:relative;display:inline-flex}
|
|
.menu{position:absolute;top:calc(100% + .5rem);left:0;z-index:20;min-width:21rem;
|
|
background:var(--panel);border:1px solid var(--line-hi);border-radius:var(--r-soft);
|
|
padding:.4rem;display:flex;flex-direction:column;
|
|
box-shadow:0 12px 32px rgba(0,0,0,.75)}
|
|
.menu[hidden]{display:none}
|
|
.mi{display:flex;align-items:center;gap:.7rem;width:100%;text-align:left;
|
|
padding:.62rem .8rem;border:0;border-radius:calc(var(--r-soft) - .35rem);
|
|
background:transparent;color:var(--ink);font-family:var(--mono);font-size:.76rem;
|
|
letter-spacing:.04em;text-decoration:none;cursor:pointer;line-height:1.3;
|
|
white-space:nowrap}
|
|
.mi:hover,.mi:focus-visible{background:#181818;outline:none}
|
|
.mi .mark{width:1.05rem;height:1.05rem;color:var(--soft)}
|
|
.mi:hover .mark{color:var(--ink)}
|
|
.mi u{margin-left:auto;color:var(--faint);text-decoration:none;font-size:.68rem;
|
|
letter-spacing:.1em;text-transform:uppercase}
|
|
|
|
/* Works-with bar: one scrolling row, name and mark per agent. */
|
|
.strip{border-bottom:1px solid var(--line);background:var(--panel)}
|
|
.strip .in{max-width:60rem;margin:0 auto;padding:.9rem 1.75rem;display:flex;
|
|
align-items:center;gap:1.25rem;font-family:var(--mono);font-size:.72rem;
|
|
letter-spacing:.1em;text-transform:uppercase;color:var(--faint)}
|
|
.strip b{color:var(--soft);font-weight:400;flex:0 0 auto;
|
|
border-right:1px solid var(--line);padding-right:1.25rem;align-self:stretch;
|
|
display:flex;align-items:center}
|
|
.bar{display:flex;align-items:center;gap:.5rem;flex:1 1 auto;min-width:0;
|
|
overflow-x:auto;scrollbar-width:none;scroll-snap-type:x proximity;
|
|
-webkit-overflow-scrolling:touch;
|
|
-webkit-mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent);
|
|
mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent)}
|
|
.bar::-webkit-scrollbar{display:none}
|
|
.agent{display:inline-flex;align-items:center;gap:.45rem;flex:0 0 auto;
|
|
padding:.4rem .85rem;border:1px solid var(--line);border-radius:var(--r);
|
|
color:var(--soft);white-space:nowrap;scroll-snap-align:start;
|
|
text-decoration:none;background:var(--ground)}
|
|
.agent:hover{color:var(--ink);border-color:var(--line-hi);text-decoration:none}
|
|
.agent .mark{width:.95rem;height:.95rem}
|
|
|
|
h2{font-family:var(--mono);font-size:1.1rem;letter-spacing:-.01em;margin:2.75rem 0 .75rem;font-weight:500}
|
|
h3{font-size:1rem;margin:2rem 0 .5rem;font-weight:600}
|
|
h4,h5,h6{font-size:.94rem;margin:1.5rem 0 .5rem;font-weight:600}
|
|
p{margin:0 0 1rem;max-width:70ch}
|
|
ul,ol{margin:0 0 1rem;padding-left:1.35rem;max-width:70ch}
|
|
li{margin:.22rem 0}
|
|
a{color:var(--ink);text-decoration:underline;text-underline-offset:3px;
|
|
text-decoration-color:var(--line-hi)}
|
|
a:hover{text-decoration-color:var(--ink)}
|
|
a:focus-visible{outline:1px solid var(--ink);outline-offset:3px}
|
|
code{font-family:var(--mono);font-size:.85em;background:var(--panel);
|
|
border:1px solid var(--line);border-radius:5px;padding:.1em .36em}
|
|
pre{background:var(--panel);border:1px solid var(--line);padding:1rem 1.1rem;
|
|
border-radius:var(--r-soft);overflow-x:auto;margin:0 0 1.25rem}
|
|
pre code{background:none;border:0;padding:0;font-size:.82rem;line-height:1.7}
|
|
blockquote{margin:0 0 1rem;padding-left:1rem;border-left:1px solid var(--line-hi);color:var(--soft)}
|
|
hr{border:0;border-top:1px solid var(--line);margin:2.5rem 0}
|
|
table{border-collapse:collapse;width:100%;margin:0 0 1.25rem;font-size:.92rem;
|
|
display:block;overflow-x:auto}
|
|
th,td{border:1px solid var(--line);padding:.5rem .7rem;text-align:left}
|
|
|
|
.curl{display:block;background:var(--panel);border:1px solid var(--line);
|
|
border-radius:var(--r-soft);padding:.95rem 1.1rem;font-family:var(--mono);
|
|
font-size:.8rem;color:var(--soft);overflow-x:auto;margin:0 0 2.5rem;
|
|
white-space:pre;line-height:1.9}
|
|
.curl b{color:var(--ink);font-weight:400}
|
|
.curl i{font-style:normal;color:var(--faint)}
|
|
|
|
/* index */
|
|
#q{width:100%;padding:.85rem 1.2rem;font-family:var(--mono);font-size:.85rem;color:var(--ink);
|
|
background:var(--panel);border:1px solid var(--line);border-radius:var(--r);margin:0 0 .6rem}
|
|
#q::placeholder{color:var(--faint);text-transform:uppercase;letter-spacing:.1em;font-size:.72rem}
|
|
#q:focus{outline:none;border-color:var(--soft)}
|
|
#count{color:var(--faint);font-family:var(--mono);font-size:.72rem;letter-spacing:.1em;
|
|
text-transform:uppercase;margin:0 0 2rem}
|
|
.cat{margin:2.5rem 0 .5rem;font-family:var(--mono);font-size:.7rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);border-bottom:1px solid var(--line);
|
|
padding-bottom:.45rem}
|
|
.row{display:block;padding:.6rem .9rem;margin:0 -.9rem;border-bottom:1px solid var(--line);
|
|
border-radius:var(--r-soft);text-decoration:none}
|
|
.row:hover{background:#0a0a0a}
|
|
.nm{font-family:var(--mono);font-size:.84rem;color:var(--ink);display:block}
|
|
.ds{color:var(--soft);font-size:.9rem;display:block;margin-top:.12rem}
|
|
footer{margin-top:4rem;padding:2rem 0 4rem;border-top:1px solid var(--line);
|
|
color:var(--faint);font-size:.85rem}
|
|
footer a{color:var(--soft)}
|
|
.fmap{display:grid;gap:1.75rem;grid-template-columns:repeat(auto-fit,minmax(11rem,1fr));
|
|
margin:0 0 2rem;font-size:.85rem}
|
|
.fmap b{display:block;font-family:var(--mono);font-size:.68rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);font-weight:400;margin:0 0 .6rem;
|
|
border-bottom:1px solid var(--line);padding-bottom:.4rem}
|
|
.fmap a{display:inline-block;color:var(--ink);text-decoration:none;margin:.3rem .35rem 0 0}
|
|
.fmap a:hover{text-decoration:underline}
|
|
.fmap div{color:var(--faint);line-height:1.5}
|
|
.fend{border-top:1px solid var(--line);padding-top:1.25rem;font-size:.82rem}
|
|
|
|
@media (max-width:640px){
|
|
.hero .wrap{padding-top:3.25rem;padding-bottom:2.75rem}
|
|
.btns{gap:.5rem}
|
|
.btn{padding:.75rem 1.15rem;font-size:.68rem}
|
|
.copy,.copy .btn{width:100%;justify-content:center}
|
|
.menu{left:0;right:0;min-width:0}
|
|
.strip .in{flex-direction:column;align-items:flex-start;gap:.6rem;padding:.9rem 1.75rem}
|
|
.strip b{border-right:0;padding-right:0}
|
|
.bar{width:100%}
|
|
}
|
|
</style>
|
|
<link rel="alternate" type="text/markdown" href="/dspy-evaluation.md">
|
|
|
|
<link rel="icon" href="/favicon.svg">
|
|
<link rel="canonical" href="https://hanzoskills.com/dspy-evaluation">
|
|
</head>
|
|
<!-- Reading this with a machine? Every page here is also plain markdown.
|
|
This one: https://hanzoskills.com/dspy-evaluation.md
|
|
Index: https://hanzoskills.com/llms.txt
|
|
Corpus: https://hanzoskills.com/all.md
|
|
Append .md to any path for the markdown twin. -->
|
|
<body>
|
|
<nav><div class="in">
|
|
<a class="brand" href="/"><svg class="mark" viewBox="0 0 67 67" fill="currentColor" aria-hidden="true"><path d="M22.21 67V44.6369H0V67H22.21Z"/><path d="M66.7038 22.3184H22.2534L0.0878906 44.6367H44.4634L66.7038 22.3184Z"/><path d="M22.21 0H0V22.3184H22.21V0Z"/><path d="M66.7198 0H44.5098V22.3184H66.7198V0Z"/><path d="M66.7198 67V44.6369H44.5098V67H66.7198Z"/></svg> HANZO<i>SKILLS</i></a>
|
|
<a class="up" href="/">All skills</a>
|
|
</div></nav>
|
|
|
|
<div class="wrap">
|
|
<h1>dspy-evaluation</h1><p class="lede">Evaluating DSPy programs with metrics, Evaluate class, and performance analysis</p><h1>DSPy Evaluation</h1>
|
|
<p><strong>Scope</strong>: Metrics, Evaluate class, performance measurement, A/B testing, debugging <strong>Lines</strong>: ~380 <strong>Last Updated</strong>: 2025-10-25</p>
|
|
<h2>When to Use This Skill</h2>
|
|
<p>Activate this skill when:</p>
|
|
<ul><li>Measuring DSPy program accuracy and performance</li><li>Comparing different modules or optimizations</li><li>Setting up automated evaluation pipelines</li><li>Debugging module failures</li><li>Running A/B tests on prompt variations</li><li>Validating program quality before production</li></ul>
|
|
<h2>Core Concepts</h2>
|
|
<h3>Evaluation in DSPy</h3>
|
|
<p><strong>Purpose</strong>: Measure how well your program performs</p>
|
|
<ul><li><strong>Quantitative</strong>: Accuracy, F1, BLEU scores</li><li><strong>Qualitative</strong>: Inspect specific predictions</li><li><strong>Comparative</strong>: Before vs after optimization</li><li><strong>Continuous</strong>: Monitor production performance</li></ul>
|
|
<p><strong>Key insight</strong>: Good evaluation drives good optimization</p>
|
|
<h3>Metrics</h3>
|
|
<p><strong>Definition</strong>: Functions that score predictions</p>
|
|
<p><strong>Types</strong>:</p>
|
|
<ol><li><strong>Binary</strong>: True/False (correct/incorrect)</li><li><strong>Continuous</strong>: 0.0 to 1.0 (partial credit)</li><li><strong>Multi-metric</strong>: Combine multiple factors</li></ol>
|
|
<p><strong>Requirements</strong>:</p>
|
|
<pre><code class="lang-python">def metric(example, pred, trace=None) -> bool | float:
|
|
"""
|
|
Args:
|
|
example: Ground truth example with .answer, etc.
|
|
pred: Prediction object with .answer, .reasoning, etc.
|
|
trace: Optional execution trace
|
|
|
|
Returns:
|
|
bool: True if correct, False otherwise
|
|
OR
|
|
float: Score between 0.0 and 1.0
|
|
"""
|
|
pass</code></pre>
|
|
<h3>Evaluate Class</h3>
|
|
<p><strong>Purpose</strong>: Run systematic evaluation on datasets</p>
|
|
<p><strong>Features</strong>:</p>
|
|
<ul><li>Parallel execution (multi-threading)</li><li>Progress tracking</li><li>Error handling</li><li>Statistical aggregation</li><li>Display modes (table, summary)</li></ul>
|
|
<hr>
|
|
<h2>Patterns</h2>
|
|
<h3>Pattern 1: Simple Accuracy Metric</h3>
|
|
<pre><code class="lang-python">import dspy
|
|
|
|
# Binary metric: exact match
|
|
def exact_match(example, pred, trace=None):
|
|
"""Return True if prediction exactly matches answer."""
|
|
return example.answer.lower().strip() == pred.answer.lower().strip()
|
|
|
|
# Usage
|
|
program = dspy.Predict("question -> answer")
|
|
|
|
# Manual evaluation
|
|
result = program(question="What is 2+2?")
|
|
example = dspy.Example(question="What is 2+2?", answer="4")
|
|
|
|
is_correct = exact_match(example, result)
|
|
print(f"Correct: {is_correct}")</code></pre>
|
|
<p><strong>When to use</strong>:</p>
|
|
<ul><li>Classification tasks</li><li>QA with definite answers</li><li>Quick validation</li></ul>
|
|
<h3>Pattern 2: Partial Credit Metric</h3>
|
|
<pre><code class="lang-python">import dspy
|
|
|
|
def fuzzy_match(example, pred, trace=None):
|
|
"""Return score based on answer overlap."""
|
|
answer = pred.answer.lower()
|
|
gold = example.answer.lower()
|
|
|
|
# Exact match
|
|
if answer == gold:
|
|
return 1.0
|
|
|
|
# Partial match: gold answer appears in prediction
|
|
if gold in answer:
|
|
return 0.7
|
|
|
|
# Weak match: any word overlap
|
|
answer_words = set(answer.split())
|
|
gold_words = set(gold.split())
|
|
overlap = len(answer_words & gold_words)
|
|
|
|
if overlap > 0:
|
|
return 0.3 * (overlap / len(gold_words))
|
|
|
|
return 0.0
|
|
|
|
# Use with optimizer
|
|
optimizer = dspy.BootstrapFewShot(metric=fuzzy_match, metric_threshold=0.7)</code></pre>
|
|
<p><strong>When to use</strong>:</p>
|
|
<ul><li>Open-ended text generation</li><li>Summarization</li><li>Tasks where partial correctness matters</li></ul>
|
|
<h3>Pattern 3: Using Evaluate Class</h3>
|
|
<pre><code class="lang-python">import dspy
|
|
from dspy.evaluate import Evaluate
|
|
|
|
# Define program
|
|
program = dspy.ChainOfThought("question -> answer")
|
|
|
|
# Prepare test set
|
|
testset = [
|
|
dspy.Example(question="What is 2+2?", answer="4").with_inputs("question"),
|
|
dspy.Example(question="Capital of France?", answer="Paris").with_inputs("question"),
|
|
dspy.Example(question="Who wrote Hamlet?", answer="Shakespeare").with_inputs("question"),
|
|
]
|
|
|
|
# Define metric
|
|
def accuracy(example, pred, trace=None):
|
|
return example.answer.lower() in pred.answer.lower()
|
|
|
|
# Create evaluator
|
|
evaluator = Evaluate(
|
|
devset=testset,
|
|
metric=accuracy,
|
|
num_threads=4, # Parallel evaluation
|
|
display_progress=True,
|
|
display_table=5, # Show first 5 results
|
|
)
|
|
|
|
# Run evaluation
|
|
score = evaluator(program)
|
|
print(f"Accuracy: {score:.1%}")</code></pre>
|
|
<p><strong>Output example</strong>:</p>
|
|
<pre><code>Question | Predicted Answer | Correct
|
|
----------------------------|------------------|--------
|
|
What is 2+2? | 4 | ✓
|
|
Capital of France? | Paris | ✓
|
|
Who wrote Hamlet? | William Shakespeare | ✓
|
|
|
|
Accuracy: 100.0%</code></pre>
|
|
<h3>Pattern 4: Multi-Metric Evaluation</h3>
|
|
<pre><code class="lang-python">import dspy
|
|
from dspy.evaluate import Evaluate
|
|
|
|
# Multiple metrics for comprehensive evaluation
|
|
def accuracy(example, pred, trace=None):
|
|
return example.answer.lower() in pred.answer.lower()
|
|
|
|
def has_reasoning(example, pred, trace=None):
|
|
"""Check if reasoning is provided and non-trivial."""
|
|
if not hasattr(pred, 'reasoning'):
|
|
return 0.0
|
|
return 1.0 if len(pred.reasoning.split()) > 5 else 0.0
|
|
|
|
def confidence_calibration(example, pred, trace=None):
|
|
"""Check if confidence matches correctness."""
|
|
is_correct = accuracy(example, pred)
|
|
if hasattr(pred, 'confidence'):
|
|
conf = float(pred.confidence) if pred.confidence else 0.5
|
|
# High confidence + correct = good
|
|
# Low confidence + incorrect = good
|
|
# Mismatch = bad
|
|
if is_correct and conf > 0.7:
|
|
return 1.0
|
|
if not is_correct and conf < 0.3:
|
|
return 1.0
|
|
return 0.5
|
|
return 0.5
|
|
|
|
# Evaluate each metric separately
|
|
eval_accuracy = Evaluate(devset=testset, metric=accuracy)
|
|
eval_reasoning = Evaluate(devset=testset, metric=has_reasoning)
|
|
eval_calibration = Evaluate(devset=testset, metric=confidence_calibration)
|
|
|
|
# Run all evaluations
|
|
acc_score = eval_accuracy(program)
|
|
reasoning_score = eval_reasoning(program)
|
|
calib_score = eval_calibration(program)
|
|
|
|
print(f"Accuracy: {acc_score:.1%}")
|
|
print(f"Reasoning Quality: {reasoning_score:.1%}")
|
|
print(f"Calibration: {calib_score:.1%}")
|
|
|
|
# Composite score
|
|
overall = 0.6 * acc_score + 0.2 * reasoning_score + 0.2 * calib_score
|
|
print(f"Overall Score: {overall:.1%}")</code></pre>
|
|
<p><strong>When to use</strong>:</p>
|
|
<ul><li>Production systems with multiple quality factors</li><li>Need comprehensive quality assessment</li><li>Balancing multiple objectives</li></ul>
|
|
<h3>Pattern 5: A/B Testing Programs</h3>
|
|
<pre><code class="lang-python">import dspy
|
|
from dspy.evaluate import Evaluate
|
|
|
|
# Program A: Simple Predict
|
|
program_a = dspy.Predict("question -> answer")
|
|
|
|
# Program B: ChainOfThought
|
|
program_b = dspy.ChainOfThought("question -> answer")
|
|
|
|
# Prepare test set
|
|
testset = [...]
|
|
|
|
# Define metric
|
|
def accuracy(example, pred, trace=None):
|
|
return example.answer.lower() in pred.answer.lower()
|
|
|
|
# Create evaluator
|
|
evaluator = Evaluate(devset=testset, metric=accuracy, num_threads=4)
|
|
|
|
# Evaluate both programs
|
|
score_a = evaluator(program_a)
|
|
score_b = evaluator(program_b)
|
|
|
|
# Compare results
|
|
print(f"Program A (Predict): {score_a:.1%}")
|
|
print(f"Program B (ChainOfThought): {score_b:.1%}")
|
|
print(f"Winner: {'B' if score_b > score_a else 'A'}")
|
|
print(f"Improvement: {abs(score_b - score_a):.1%}")</code></pre>
|
|
<p><strong>When to use</strong>:</p>
|
|
<ul><li>Comparing module types (Predict vs ChainOfThought)</li><li>Comparing optimizations (before vs after)</li><li>Choosing between LMs (GPT-4 vs Claude)</li></ul>
|
|
<h3>Pattern 6: Error Analysis</h3>
|
|
<pre><code class="lang-python">import dspy
|
|
from dspy.evaluate import Evaluate
|
|
|
|
program = dspy.ChainOfThought("question -> answer")
|
|
testset = [...]
|
|
|
|
# Track errors for analysis
|
|
errors = []
|
|
|
|
def accuracy_with_logging(example, pred, trace=None):
|
|
"""Metric that logs errors for analysis."""
|
|
correct = example.answer.lower() in pred.answer.lower()
|
|
|
|
if not correct:
|
|
errors.append({
|
|
'question': example.question,
|
|
'expected': example.answer,
|
|
'predicted': pred.answer,
|
|
'reasoning': getattr(pred, 'reasoning', 'N/A'),
|
|
})
|
|
|
|
return correct
|
|
|
|
# Run evaluation
|
|
evaluator = Evaluate(devset=testset, metric=accuracy_with_logging)
|
|
score = evaluator(program)
|
|
|
|
# Analyze errors
|
|
print(f"\nAccuracy: {score:.1%}")
|
|
print(f"\nErrors ({len(errors)}):")
|
|
for i, err in enumerate(errors[:5], 1): # Show first 5 errors
|
|
print(f"\n{i}. Question: {err['question']}")
|
|
print(f" Expected: {err['expected']}")
|
|
print(f" Predicted: {err['predicted']}")
|
|
print(f" Reasoning: {err['reasoning']}")</code></pre>
|
|
<p><strong>Benefits</strong>:</p>
|
|
<ul><li>Identify common failure patterns</li><li>Guide optimization efforts</li><li>Improve training data</li></ul>
|
|
<h3>Pattern 7: LM-as-Judge Metric</h3>
|
|
<pre><code class="lang-python">import dspy
|
|
|
|
# Use LM to evaluate predictions
|
|
judge = dspy.ChainOfThought("question, answer, predicted_answer -> score: float, explanation")
|
|
|
|
def llm_judge_metric(example, pred, trace=None):
|
|
"""Use language model to evaluate answer quality."""
|
|
result = judge(
|
|
question=example.question,
|
|
answer=example.answer,
|
|
predicted_answer=pred.answer
|
|
)
|
|
|
|
try:
|
|
score = float(result.score)
|
|
return max(0.0, min(1.0, score)) # Clamp to [0, 1]
|
|
except:
|
|
# Fallback to simple match if LM fails
|
|
return 1.0 if example.answer.lower() in pred.answer.lower() else 0.0
|
|
|
|
# Use LM-as-judge for nuanced evaluation
|
|
evaluator = Evaluate(devset=testset, metric=llm_judge_metric, num_threads=1) # Sequential for LM calls
|
|
score = evaluator(program)</code></pre>
|
|
<p><strong>When to use</strong>:</p>
|
|
<ul><li>Open-ended generation tasks</li><li>Nuanced quality assessment</li><li>When exact match is too strict</li></ul>
|
|
<p><strong>Caution</strong>:</p>
|
|
<ul><li>Slower (requires LM calls)</li><li>More expensive</li><li>LM judge can be wrong</li></ul>
|
|
<h3>Pattern 8: Production Monitoring</h3>
|
|
<pre><code class="lang-python">import dspy
|
|
from dspy.evaluate import Evaluate
|
|
import time
|
|
|
|
# Production program
|
|
program = dspy.ChainOfThought("question -> answer")
|
|
|
|
# Metrics for monitoring
|
|
def response_quality(example, pred, trace=None):
|
|
"""Monitor prediction quality."""
|
|
return example.answer.lower() in pred.answer.lower()
|
|
|
|
def response_time(example, pred, trace=None):
|
|
"""Monitor latency (tracked separately)."""
|
|
# This is illustrative; actual timing would be external
|
|
return 1.0 # Placeholder
|
|
|
|
# Sample production traffic for evaluation
|
|
sample_size = 100
|
|
production_samples = [...] # Collect from production logs
|
|
|
|
# Periodic evaluation
|
|
evaluator = Evaluate(
|
|
devset=production_samples,
|
|
metric=response_quality,
|
|
num_threads=4,
|
|
)
|
|
|
|
# Run evaluation
|
|
score = evaluator(program)
|
|
|
|
# Alert if quality drops
|
|
if score < 0.85:
|
|
print(f"⚠️ Quality degradation detected: {score:.1%}")
|
|
# Send alert, trigger investigation
|
|
else:
|
|
print(f"✓ Quality maintained: {score:.1%}")</code></pre>
|
|
<p><strong>When to use</strong>:</p>
|
|
<ul><li>Production monitoring</li><li>Detecting model drift</li><li>Continuous quality assurance</li></ul>
|
|
<hr>
|
|
<h2>Quick Reference</h2>
|
|
<h3>Metric Function Template</h3>
|
|
<pre><code class="lang-python">def my_metric(example, pred, trace=None):
|
|
"""
|
|
Evaluate prediction quality.
|
|
|
|
Args:
|
|
example: dspy.Example with ground truth
|
|
pred: Prediction object from module
|
|
trace: Optional execution trace
|
|
|
|
Returns:
|
|
bool: True/False for binary metrics
|
|
float: 0.0-1.0 for continuous metrics
|
|
"""
|
|
# Implement evaluation logic
|
|
return score</code></pre>
|
|
<h3>Common Metrics</h3>
|
|
<pre><code class="lang-python"># Exact match
|
|
def exact_match(ex, pred, trace=None):
|
|
return ex.answer == pred.answer
|
|
|
|
# Contains
|
|
def contains(ex, pred, trace=None):
|
|
return ex.answer.lower() in pred.answer.lower()
|
|
|
|
# Fuzzy match
|
|
def fuzzy(ex, pred, trace=None):
|
|
from difflib import SequenceMatcher
|
|
ratio = SequenceMatcher(None, ex.answer, pred.answer).ratio()
|
|
return ratio
|
|
|
|
# F1 score
|
|
def f1_score(ex, pred, trace=None):
|
|
ex_words = set(ex.answer.lower().split())
|
|
pred_words = set(pred.answer.lower().split())
|
|
overlap = len(ex_words & pred_words)
|
|
precision = overlap / len(pred_words) if pred_words else 0
|
|
recall = overlap / len(ex_words) if ex_words else 0
|
|
return 2 * precision * recall / (precision + recall) if (precision + recall) > 0 else 0</code></pre>
|
|
<h3>Evaluation Checklist</h3>
|
|
<pre><code>✅ DO: Use separate test set (never seen during training)
|
|
✅ DO: Evaluate baseline before optimization
|
|
✅ DO: Use multiple metrics for comprehensive view
|
|
✅ DO: Run error analysis on failures
|
|
✅ DO: Report confidence intervals if dataset is small
|
|
|
|
❌ DON'T: Evaluate on training data (overfitting!)
|
|
❌ DON'T: Rely on single metric
|
|
❌ DON'T: Ignore errors (analyze them!)
|
|
❌ DON'T: Skip baseline comparison</code></pre>
|
|
<hr>
|
|
<h2>Anti-Patterns</h2>
|
|
<p>❌ <strong>Evaluating on training data</strong>: Overfitting, invalid results</p>
|
|
<pre><code class="lang-python"># Bad
|
|
evaluator = Evaluate(devset=trainset, metric=accuracy) # WRONG!</code></pre>
|
|
<p>✅ Use held-out test set:</p>
|
|
<pre><code class="lang-python"># Good
|
|
evaluator = Evaluate(devset=testset, metric=accuracy)</code></pre>
|
|
<p>❌ <strong>Single metric only</strong>: Misses important quality factors</p>
|
|
<pre><code class="lang-python"># Bad - only accuracy
|
|
score = evaluate(program, accuracy)</code></pre>
|
|
<p>✅ Use multiple metrics:</p>
|
|
<pre><code class="lang-python"># Good - comprehensive evaluation
|
|
acc = evaluate(program, accuracy)
|
|
f1 = evaluate(program, f1_score)
|
|
reasoning = evaluate(program, has_reasoning)</code></pre>
|
|
<p>❌ <strong>No error analysis</strong>: Don't learn from failures</p>
|
|
<pre><code class="lang-python"># Bad - just get score, don't investigate
|
|
score = evaluator(program)
|
|
print(score) # Done, but learned nothing</code></pre>
|
|
<p>✅ Analyze errors:</p>
|
|
<pre><code class="lang-python"># Good - investigate failures
|
|
score = evaluator(program)
|
|
errors = [ex for ex in testset if not metric(ex, program(ex.question))]
|
|
analyze_common_patterns(errors)</code></pre>
|
|
<hr>
|
|
<h2>Related Skills</h2>
|
|
<ul><li><code>dspy-optimizers.md</code> - Using metrics for optimization</li><li><code>dspy-modules.md</code> - Programs to evaluate</li><li><code>dspy-assertions.md</code> - Runtime validation</li><li><code>llm-dataset-preparation.md</code> - Creating evaluation datasets</li></ul>
|
|
<hr>
|
|
<p><strong>Last Updated</strong>: 2025-10-25 <strong>Format Version</strong>: 1.0 (Atomic)</p>
|
|
<footer>
|
|
<div class="fmap">
|
|
<div><b>Build</b>
|
|
<a href="https://hanzo.app">hanzo.app</a> app builder
|
|
<a href="https://hanzo.codes">hanzo.codes</a> code with an agent
|
|
<a href="https://hanzoskills.com">hanzoskills.com</a> what an agent reads</div>
|
|
<div><b>Run</b>
|
|
<a href="https://console.hanzo.ai">console.hanzo.ai</a> cloud console
|
|
<a href="https://api.hanzo.ai">api.hanzo.ai</a> the API</div>
|
|
<div><b>Account</b>
|
|
<a href="https://hanzo.id">hanzo.id</a> sign in
|
|
<a href="https://pay.hanzo.ai">pay.hanzo.ai</a> top up
|
|
<a href="https://billing.hanzo.ai">billing.hanzo.ai</a> invoices and usage</div>
|
|
<div><b>Company</b>
|
|
<a href="https://hanzo.ai">hanzo.ai</a> canonical
|
|
<a href="https://hanzo.ventures">hanzo.ventures</a> ventures</div>
|
|
</div>
|
|
<div class="fend"><a href="/dspy-evaluation.md">This page as plain text</a> · <a href="/llms.txt">Index for agents</a></div>
|
|
</footer>
|
|
</div></body></html> |