mirror of
https://github.com/hanzoai/skills.git
synced 2026-11-19 00:47:36 +00:00
hanzoskills.com/llms.txt now opens with the install line, the API reference index and the company index, so an agent that lands on any one of the three can find the rest. robots.txt claimed 537 documents while llms.txt counts 920. The number is gone from robots.txt; llms.txt derives it. public/ is regenerated from scripts/site.py, which it had fallen behind: the copy-prompt menu and agent bar were in the generator and not in the committed pages. tests/site.mjs passes against the result. Co-authored-by: Hanzo Dev <[email protected]>
604 lines
29 KiB
HTML
604 lines
29 KiB
HTML
<!doctype html>
|
|
<html lang="en"><head>
|
|
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
<title>dspy-optimizers</title>
|
|
<meta name="color-scheme" content="dark">
|
|
<style>
|
|
/* Committed to one visual world: a dark instrument panel. No light variant —
|
|
the dot field and the hairline rules only read on black, and a naive
|
|
inversion would wash both out. Every colour is painted explicitly so the page
|
|
never borrows a host background. */
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Pixel Square';src:url('/fonts/ZenPixel-Square.woff2') format('woff2');font-style:normal;font-display:swap}
|
|
|
|
:root{
|
|
--ink:#f4f4f2; --soft:#9c9c98; --faint:#6a6a66; --ground:#000; --panel:#0a0a0a;
|
|
--line:#232323; --line-hi:#3a3a3a; --accent:#f4f4f2;
|
|
--font-zen-sans:'Zen',-apple-system,BlinkMacSystemFont,"Segoe UI",Helvetica,Arial,sans-serif;
|
|
--font-zen-mono:'Zen Mono',ui-monospace,"SF Mono",SFMono-Regular,Menlo,Consolas,monospace;
|
|
--font-zen-pixel-square:'Zen Pixel Square',monospace;
|
|
--mono:var(--font-zen-mono);
|
|
--sans:var(--font-zen-sans);
|
|
/* Controls are pills and surfaces are soft, as on hanzo.ai. */
|
|
--r:999px; --r-soft:14px;
|
|
}
|
|
*{box-sizing:border-box}
|
|
html{-webkit-text-size-adjust:100%}
|
|
body{margin:0;background:var(--ground);color:var(--ink);font-family:var(--sans);
|
|
font-size:16px;line-height:1.65;-webkit-font-smoothing:antialiased}
|
|
.wrap{max-width:60rem;margin:0 auto;padding:0 1.75rem}
|
|
|
|
/* The dot field. A halftone made of one repeating radial-gradient, faded out
|
|
along the diagonal by a mask so it reads as a wave rather than wallpaper. */
|
|
/* No overflow clip here: the dot field is inset to the box already, and a clip
|
|
would cut off the copy-prompt menu. */
|
|
.hero{position:relative;border-bottom:1px solid var(--line)}
|
|
.hero::before{content:"";position:absolute;inset:0;
|
|
background-image:radial-gradient(var(--line-hi) 1px, transparent 1px);
|
|
background-size:7px 7px;
|
|
-webkit-mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
pointer-events:none}
|
|
.hero .wrap{position:relative;padding-top:5.5rem;padding-bottom:4rem}
|
|
|
|
nav{border-bottom:1px solid var(--line)}
|
|
nav .in{max-width:60rem;margin:0 auto;padding:1.1rem 1.75rem;display:flex;
|
|
justify-content:space-between;align-items:center;gap:1rem}
|
|
.brand{font-family:var(--mono);font-size:.85rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--ink);text-decoration:none;
|
|
display:inline-flex;align-items:center;gap:.6rem}
|
|
.brand i{font-style:normal;color:var(--faint)}
|
|
.mark{width:1.15em;height:1.15em;display:block;flex:0 0 auto}
|
|
nav a.up{font-family:var(--mono);font-size:.72rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--soft);text-decoration:none}
|
|
nav a.up:hover{color:var(--ink)}
|
|
|
|
h1{font-family:var(--mono);font-size:clamp(2rem,5.4vw,3.4rem);line-height:1.12;
|
|
letter-spacing:-.02em;margin:0 0 1.1rem;font-weight:500;text-wrap:balance}
|
|
h1 em{font-style:normal;color:var(--faint)}
|
|
.lede{color:var(--soft);font-size:1.05rem;max-width:52ch;margin:0 0 2.25rem}
|
|
|
|
.btns{display:flex;flex-wrap:wrap;gap:.7rem;margin:0 0 .5rem;align-items:center}
|
|
.btn{display:inline-flex;align-items:center;gap:.5rem;font-family:var(--mono);
|
|
font-size:.72rem;letter-spacing:.14em;text-transform:uppercase;
|
|
padding:.8rem 1.4rem;border:1px solid var(--line-hi);border-radius:var(--r);
|
|
color:var(--ink);text-decoration:none;background:transparent;cursor:pointer;
|
|
line-height:1.2;font-weight:400}
|
|
.btn:hover{background:#141414;text-decoration:none;border-color:var(--soft)}
|
|
.btn.solid{background:var(--ink);color:#000;border-color:var(--ink)}
|
|
.btn.solid:hover{background:#d8d8d4}
|
|
.btn .chev{width:.7em;height:.7em;flex:0 0 auto;transition:transform .15s ease}
|
|
.btn[aria-expanded="true"] .chev{transform:rotate(180deg)}
|
|
|
|
/* Copy-prompt selector. Each row copies the prompt for its own target. */
|
|
.copy{position:relative;display:inline-flex}
|
|
.menu{position:absolute;top:calc(100% + .5rem);left:0;z-index:20;min-width:21rem;
|
|
background:var(--panel);border:1px solid var(--line-hi);border-radius:var(--r-soft);
|
|
padding:.4rem;display:flex;flex-direction:column;
|
|
box-shadow:0 12px 32px rgba(0,0,0,.75)}
|
|
.menu[hidden]{display:none}
|
|
.mi{display:flex;align-items:center;gap:.7rem;width:100%;text-align:left;
|
|
padding:.62rem .8rem;border:0;border-radius:calc(var(--r-soft) - .35rem);
|
|
background:transparent;color:var(--ink);font-family:var(--mono);font-size:.76rem;
|
|
letter-spacing:.04em;text-decoration:none;cursor:pointer;line-height:1.3;
|
|
white-space:nowrap}
|
|
.mi:hover,.mi:focus-visible{background:#181818;outline:none}
|
|
.mi .mark{width:1.05rem;height:1.05rem;color:var(--soft)}
|
|
.mi:hover .mark{color:var(--ink)}
|
|
.mi u{margin-left:auto;color:var(--faint);text-decoration:none;font-size:.68rem;
|
|
letter-spacing:.1em;text-transform:uppercase}
|
|
|
|
/* Works-with bar: one scrolling row, name and mark per agent. */
|
|
.strip{border-bottom:1px solid var(--line);background:var(--panel)}
|
|
.strip .in{max-width:60rem;margin:0 auto;padding:.9rem 1.75rem;display:flex;
|
|
align-items:center;gap:1.25rem;font-family:var(--mono);font-size:.72rem;
|
|
letter-spacing:.1em;text-transform:uppercase;color:var(--faint)}
|
|
.strip b{color:var(--soft);font-weight:400;flex:0 0 auto;
|
|
border-right:1px solid var(--line);padding-right:1.25rem;align-self:stretch;
|
|
display:flex;align-items:center}
|
|
.bar{display:flex;align-items:center;gap:.5rem;flex:1 1 auto;min-width:0;
|
|
overflow-x:auto;scrollbar-width:none;scroll-snap-type:x proximity;
|
|
-webkit-overflow-scrolling:touch;
|
|
-webkit-mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent);
|
|
mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent)}
|
|
.bar::-webkit-scrollbar{display:none}
|
|
.agent{display:inline-flex;align-items:center;gap:.45rem;flex:0 0 auto;
|
|
padding:.4rem .85rem;border:1px solid var(--line);border-radius:var(--r);
|
|
color:var(--soft);white-space:nowrap;scroll-snap-align:start;
|
|
text-decoration:none;background:var(--ground)}
|
|
.agent:hover{color:var(--ink);border-color:var(--line-hi);text-decoration:none}
|
|
.agent .mark{width:.95rem;height:.95rem}
|
|
|
|
h2{font-family:var(--mono);font-size:1.1rem;letter-spacing:-.01em;margin:2.75rem 0 .75rem;font-weight:500}
|
|
h3{font-size:1rem;margin:2rem 0 .5rem;font-weight:600}
|
|
h4,h5,h6{font-size:.94rem;margin:1.5rem 0 .5rem;font-weight:600}
|
|
p{margin:0 0 1rem;max-width:70ch}
|
|
ul,ol{margin:0 0 1rem;padding-left:1.35rem;max-width:70ch}
|
|
li{margin:.22rem 0}
|
|
a{color:var(--ink);text-decoration:underline;text-underline-offset:3px;
|
|
text-decoration-color:var(--line-hi)}
|
|
a:hover{text-decoration-color:var(--ink)}
|
|
a:focus-visible{outline:1px solid var(--ink);outline-offset:3px}
|
|
code{font-family:var(--mono);font-size:.85em;background:var(--panel);
|
|
border:1px solid var(--line);border-radius:5px;padding:.1em .36em}
|
|
pre{background:var(--panel);border:1px solid var(--line);padding:1rem 1.1rem;
|
|
border-radius:var(--r-soft);overflow-x:auto;margin:0 0 1.25rem}
|
|
pre code{background:none;border:0;padding:0;font-size:.82rem;line-height:1.7}
|
|
blockquote{margin:0 0 1rem;padding-left:1rem;border-left:1px solid var(--line-hi);color:var(--soft)}
|
|
hr{border:0;border-top:1px solid var(--line);margin:2.5rem 0}
|
|
table{border-collapse:collapse;width:100%;margin:0 0 1.25rem;font-size:.92rem;
|
|
display:block;overflow-x:auto}
|
|
th,td{border:1px solid var(--line);padding:.5rem .7rem;text-align:left}
|
|
|
|
.curl{display:block;background:var(--panel);border:1px solid var(--line);
|
|
border-radius:var(--r-soft);padding:.95rem 1.1rem;font-family:var(--mono);
|
|
font-size:.8rem;color:var(--soft);overflow-x:auto;margin:0 0 2.5rem;
|
|
white-space:pre;line-height:1.9}
|
|
.curl b{color:var(--ink);font-weight:400}
|
|
.curl i{font-style:normal;color:var(--faint)}
|
|
|
|
/* index */
|
|
#q{width:100%;padding:.85rem 1.2rem;font-family:var(--mono);font-size:.85rem;color:var(--ink);
|
|
background:var(--panel);border:1px solid var(--line);border-radius:var(--r);margin:0 0 .6rem}
|
|
#q::placeholder{color:var(--faint);text-transform:uppercase;letter-spacing:.1em;font-size:.72rem}
|
|
#q:focus{outline:none;border-color:var(--soft)}
|
|
#count{color:var(--faint);font-family:var(--mono);font-size:.72rem;letter-spacing:.1em;
|
|
text-transform:uppercase;margin:0 0 2rem}
|
|
.cat{margin:2.5rem 0 .5rem;font-family:var(--mono);font-size:.7rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);border-bottom:1px solid var(--line);
|
|
padding-bottom:.45rem}
|
|
.row{display:block;padding:.6rem .9rem;margin:0 -.9rem;border-bottom:1px solid var(--line);
|
|
border-radius:var(--r-soft);text-decoration:none}
|
|
.row:hover{background:#0a0a0a}
|
|
.nm{font-family:var(--mono);font-size:.84rem;color:var(--ink);display:block}
|
|
.ds{color:var(--soft);font-size:.9rem;display:block;margin-top:.12rem}
|
|
footer{margin-top:4rem;padding:2rem 0 4rem;border-top:1px solid var(--line);
|
|
color:var(--faint);font-size:.85rem}
|
|
footer a{color:var(--soft)}
|
|
.fmap{display:grid;gap:1.75rem;grid-template-columns:repeat(auto-fit,minmax(11rem,1fr));
|
|
margin:0 0 2rem;font-size:.85rem}
|
|
.fmap b{display:block;font-family:var(--mono);font-size:.68rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);font-weight:400;margin:0 0 .6rem;
|
|
border-bottom:1px solid var(--line);padding-bottom:.4rem}
|
|
.fmap a{display:inline-block;color:var(--ink);text-decoration:none;margin:.3rem .35rem 0 0}
|
|
.fmap a:hover{text-decoration:underline}
|
|
.fmap div{color:var(--faint);line-height:1.5}
|
|
.fend{border-top:1px solid var(--line);padding-top:1.25rem;font-size:.82rem}
|
|
|
|
@media (max-width:640px){
|
|
.hero .wrap{padding-top:3.25rem;padding-bottom:2.75rem}
|
|
.btns{gap:.5rem}
|
|
.btn{padding:.75rem 1.15rem;font-size:.68rem}
|
|
.copy,.copy .btn{width:100%;justify-content:center}
|
|
.menu{left:0;right:0;min-width:0}
|
|
.strip .in{flex-direction:column;align-items:flex-start;gap:.6rem;padding:.9rem 1.75rem}
|
|
.strip b{border-right:0;padding-right:0}
|
|
.bar{width:100%}
|
|
}
|
|
</style>
|
|
<link rel="alternate" type="text/markdown" href="/dspy-optimizers.md">
|
|
|
|
<link rel="icon" href="/favicon.svg">
|
|
<link rel="canonical" href="https://hanzoskills.com/dspy-optimizers">
|
|
</head>
|
|
<!-- Reading this with a machine? Every page here is also plain markdown.
|
|
This one: https://hanzoskills.com/dspy-optimizers.md
|
|
Index: https://hanzoskills.com/llms.txt
|
|
Corpus: https://hanzoskills.com/all.md
|
|
Append .md to any path for the markdown twin. -->
|
|
<body>
|
|
<nav><div class="in">
|
|
<a class="brand" href="/"><svg class="mark" viewBox="0 0 67 67" fill="currentColor" aria-hidden="true"><path d="M22.21 67V44.6369H0V67H22.21Z"/><path d="M66.7038 22.3184H22.2534L0.0878906 44.6367H44.4634L66.7038 22.3184Z"/><path d="M22.21 0H0V22.3184H22.21V0Z"/><path d="M66.7198 0H44.5098V22.3184H66.7198V0Z"/><path d="M66.7198 67V44.6369H44.5098V67H66.7198Z"/></svg> HANZO<i>SKILLS</i></a>
|
|
<a class="up" href="/">All skills</a>
|
|
</div></nav>
|
|
|
|
<div class="wrap">
|
|
<h1>dspy-optimizers</h1><p class="lede">Optimizing DSPy programs with teleprompters, BootstrapFewShot, MIPROv2, and compilation</p><h1>DSPy Optimizers</h1>
|
|
<p><strong>Scope</strong>: Teleprompters, BootstrapFewShot, MIPROv2, COPRO, compilation, metrics <strong>Lines</strong>: ~440 <strong>Last Updated</strong>: 2025-10-25</p>
|
|
<h2>When to Use This Skill</h2>
|
|
<p>Activate this skill when:</p>
|
|
<ul><li>Improving DSPy program accuracy with automatic optimization</li><li>Compiling modules with training examples</li><li>Generating optimized few-shot demonstrations</li><li>Tuning prompts and LM weights programmatically</li><li>Bootstrapping training data from small seed sets</li><li>Evaluating and iterating on DSPy programs</li></ul>
|
|
<h2>Core Concepts</h2>
|
|
<h3>What are Optimizers (Teleprompters)?</h3>
|
|
<p><strong>Definition</strong>: Optimizers automatically improve DSPy programs by finding better prompts, demonstrations, and parameters</p>
|
|
<p><strong>Original term</strong>: "Teleprompters" (legacy name, now called optimizers)</p>
|
|
<p><strong>Purpose</strong>:</p>
|
|
<ul><li><strong>Auto-tune</strong>: Find optimal prompts without manual engineering</li><li><strong>Few-shot learning</strong>: Generate effective demonstrations</li><li><strong>Weight tuning</strong>: Fine-tune model weights (if supported)</li><li><strong>Metric optimization</strong>: Maximize custom metrics</li></ul>
|
|
<p><strong>Key insight</strong>: Instead of manually tweaking prompts, define a metric and let DSPy optimize</p>
|
|
<h3>Optimization Process</h3>
|
|
<p><strong>Compilation workflow</strong>:</p>
|
|
<ol><li><strong>Define metric</strong>: Function that scores predictions</li><li><strong>Prepare dataset</strong>: Training/validation examples</li><li><strong>Choose optimizer</strong>: BootstrapFewShot, MIPROv2, etc.</li><li><strong>Compile</strong>: <code>optimizer.compile(module, trainset=data, metric=metric)</code></li><li><strong>Evaluate</strong>: Test on held-out data</li></ol>
|
|
<h3>Common Optimizers</h3>
|
|
<p><strong>BootstrapFewShot</strong>: Generate few-shot examples</p>
|
|
<ul><li>Simplest optimizer</li><li>Creates demonstrations from successful predictions</li><li>Works with any module</li><li>Fast, effective</li></ul>
|
|
<p><strong>MIPROv2</strong>: Multi-prompt instruction optimization</p>
|
|
<ul><li>Optimizes instructions AND demonstrations</li><li>More sophisticated than BootstrapFewShot</li><li>Better for complex tasks</li><li>Slower but higher quality</li></ul>
|
|
<p><strong>COPRO</strong>: Coordinate prompt optimization</p>
|
|
<ul><li>Optimizes prompts across multiple modules</li><li>Good for multi-stage pipelines</li><li>Considers module interactions</li></ul>
|
|
<p><strong>BootstrapFinetune</strong>: Generate data for fine-tuning</p>
|
|
<ul><li>Creates training data for model fine-tuning</li><li>Requires fine-tunable model</li><li>Best accuracy but most expensive</li></ul>
|
|
<hr>
|
|
<h2>Patterns</h2>
|
|
<h3>Pattern 1: Basic BootstrapFewShot</h3>
|
|
<pre><code class="lang-python">import dspy
|
|
|
|
# Configure LM
|
|
lm = dspy.LM("openai/gpt-4o-mini")
|
|
dspy.configure(lm=lm)
|
|
|
|
# Define module
|
|
qa = dspy.ChainOfThought("question -> answer")
|
|
|
|
# Prepare training data
|
|
trainset = [
|
|
dspy.Example(question="What is the capital of France?", answer="Paris").with_inputs("question"),
|
|
dspy.Example(question="What is 2+2?", answer="4").with_inputs("question"),
|
|
dspy.Example(question="Who wrote Hamlet?", answer="William Shakespeare").with_inputs("question"),
|
|
]
|
|
|
|
# Define metric (validation function)
|
|
def validate_answer(example, pred, trace=None):
|
|
"""Return True if prediction is correct."""
|
|
return example.answer.lower() == pred.answer.lower()
|
|
|
|
# Create optimizer
|
|
optimizer = dspy.BootstrapFewShot(metric=validate_answer)
|
|
|
|
# Compile (optimize) the module
|
|
compiled_qa = optimizer.compile(
|
|
student=qa,
|
|
trainset=trainset,
|
|
)
|
|
|
|
# Use optimized module
|
|
result = compiled_qa(question="What is the capital of Germany?")
|
|
print(result.answer)</code></pre>
|
|
<p><strong>What happens</strong>:</p>
|
|
<ul><li>Runs module on training examples</li><li>Keeps successful predictions as demonstrations</li><li>Optimized module uses these as few-shot examples</li><li>Better accuracy on similar tasks</li></ul>
|
|
<h3>Pattern 2: MIPROv2 Optimization</h3>
|
|
<pre><code class="lang-python">import dspy
|
|
|
|
# More sophisticated optimization
|
|
qa = dspy.ChainOfThought("question, context -> answer")
|
|
|
|
trainset = [
|
|
dspy.Example(
|
|
question="What is DSPy?",
|
|
context="DSPy is a framework for programming language models.",
|
|
answer="A framework for programming language models"
|
|
).with_inputs("question", "context"),
|
|
# ... more examples
|
|
]
|
|
|
|
devset = [
|
|
# Validation examples (separate from training)
|
|
]
|
|
|
|
# Define metric with partial credit
|
|
def qa_metric(example, pred, trace=None):
|
|
"""Score prediction quality (0.0 to 1.0)."""
|
|
answer = pred.answer.lower()
|
|
gold = example.answer.lower()
|
|
|
|
# Exact match
|
|
if answer == gold:
|
|
return 1.0
|
|
|
|
# Partial match (contains key words)
|
|
if gold in answer or answer in gold:
|
|
return 0.5
|
|
|
|
return 0.0
|
|
|
|
# MIPROv2 optimizer
|
|
optimizer = dspy.MIPROv2(
|
|
metric=qa_metric,
|
|
auto="light", # or "medium", "heavy" for more optimization
|
|
num_candidates=10, # Number of prompt variations to try
|
|
)
|
|
|
|
# Compile with training and validation sets
|
|
compiled_qa = optimizer.compile(
|
|
student=qa,
|
|
trainset=trainset,
|
|
valset=devset, # Used for selecting best prompts
|
|
max_bootstrapped_demos=4, # Max few-shot examples
|
|
max_labeled_demos=2, # Max examples from trainset
|
|
)
|
|
|
|
# Optimized module now has better prompts and demonstrations
|
|
result = compiled_qa(question="What is DSPy?", context="...")</code></pre>
|
|
<p><strong>Benefits over BootstrapFewShot</strong>:</p>
|
|
<ul><li>Optimizes instruction text, not just examples</li><li>Tries multiple prompt variations</li><li>Uses validation set to pick best prompts</li><li>Higher quality but slower</li></ul>
|
|
<h3>Pattern 3: Custom Metric Functions</h3>
|
|
<pre><code class="lang-python">import dspy
|
|
|
|
# Metric with detailed scoring
|
|
def complex_metric(example, pred, trace=None):
|
|
"""Multi-factor metric for comprehensive evaluation."""
|
|
score = 0.0
|
|
|
|
# Factor 1: Answer correctness (0.5 weight)
|
|
if example.answer.lower() in pred.answer.lower():
|
|
score += 0.5
|
|
|
|
# Factor 2: Confidence (0.2 weight)
|
|
if hasattr(pred, 'confidence'):
|
|
try:
|
|
conf = float(pred.confidence)
|
|
if 0.7 <= conf <= 1.0:
|
|
score += 0.2
|
|
except:
|
|
pass
|
|
|
|
# Factor 3: Reasoning quality (0.3 weight)
|
|
if hasattr(pred, 'reasoning'):
|
|
if len(pred.reasoning.split()) > 10: # Detailed reasoning
|
|
score += 0.15
|
|
if "because" in pred.reasoning.lower(): # Causal reasoning
|
|
score += 0.15
|
|
|
|
return score
|
|
|
|
# Use metric with optimizer
|
|
optimizer = dspy.BootstrapFewShot(
|
|
metric=complex_metric,
|
|
metric_threshold=0.7, # Only keep examples scoring >= 0.7
|
|
)</code></pre>
|
|
<p><strong>When to use</strong>:</p>
|
|
<ul><li>Multi-factor quality assessment</li><li>Domain-specific requirements</li><li>Nuanced evaluation beyond exact match</li></ul>
|
|
<h3>Pattern 4: Optimizing Multi-Module Pipelines</h3>
|
|
<pre><code class="lang-python">import dspy
|
|
|
|
class RAGPipeline(dspy.Module):
|
|
"""Retrieval-Augmented Generation pipeline."""
|
|
|
|
def __init__(self):
|
|
super().__init__()
|
|
self.retrieve = dspy.Retrieve(k=3)
|
|
self.generate = dspy.ChainOfThought("context, question -> answer")
|
|
|
|
def forward(self, question):
|
|
passages = self.retrieve(question).passages
|
|
context = "\n".join(passages)
|
|
return self.generate(context=context, question=question)
|
|
|
|
# Prepare data
|
|
trainset = [
|
|
dspy.Example(
|
|
question="What is DSPy?",
|
|
answer="A framework for programming language models"
|
|
).with_inputs("question"),
|
|
]
|
|
|
|
# Metric for RAG
|
|
def rag_metric(example, pred, trace=None):
|
|
# Check if answer is in prediction
|
|
return example.answer.lower() in pred.answer.lower()
|
|
|
|
# COPRO optimizer (coordinates across modules)
|
|
optimizer = dspy.COPRO(
|
|
metric=rag_metric,
|
|
breadth=10, # Number of prompt variations per module
|
|
depth=3, # Optimization iterations
|
|
)
|
|
|
|
# Compile entire pipeline
|
|
pipeline = RAGPipeline()
|
|
compiled_pipeline = optimizer.compile(
|
|
student=pipeline,
|
|
trainset=trainset,
|
|
)</code></pre>
|
|
<p><strong>When to use</strong>:</p>
|
|
<ul><li>Multi-stage pipelines (RAG, agents, etc.)</li><li>Need to optimize module interactions</li><li>Complex workflows</li></ul>
|
|
<h3>Pattern 5: BootstrapFinetune (Model Fine-tuning)</h3>
|
|
<pre><code class="lang-python">import dspy
|
|
|
|
# Requires fine-tunable model
|
|
teacher = dspy.ChainOfThought("question -> answer")
|
|
|
|
# Large training dataset
|
|
trainset = [
|
|
# Hundreds or thousands of examples
|
|
]
|
|
|
|
# Generate fine-tuning data
|
|
optimizer = dspy.BootstrapFinetune(
|
|
metric=lambda ex, pred, trace: ex.answer == pred.answer,
|
|
)
|
|
|
|
# Creates fine-tuning dataset
|
|
compiled_teacher = optimizer.compile(
|
|
student=teacher,
|
|
trainset=trainset,
|
|
target="openai/gpt-4o-mini-finetuned", # Your fine-tuned model
|
|
)
|
|
|
|
# Now use fine-tuned model
|
|
finetuned_lm = dspy.LM("openai/gpt-4o-mini-finetuned")
|
|
dspy.configure(lm=finetuned_lm)</code></pre>
|
|
<p><strong>When to use</strong>:</p>
|
|
<ul><li>Have large training dataset (1000+ examples)</li><li>Need maximum accuracy</li><li>Can afford fine-tuning cost</li><li>Task is well-defined and stable</li></ul>
|
|
<h3>Pattern 6: Evaluation-Driven Optimization</h3>
|
|
<pre><code class="lang-python">import dspy
|
|
from dspy.evaluate import Evaluate
|
|
|
|
# Define program
|
|
program = dspy.ChainOfThought("question -> answer")
|
|
|
|
# Prepare datasets
|
|
trainset = [...] # Training examples
|
|
valset = [...] # Validation examples
|
|
testset = [...] # Test examples (never use for optimization!)
|
|
|
|
# Define metric
|
|
def accuracy(example, pred, trace=None):
|
|
return example.answer.lower() == pred.answer.lower()
|
|
|
|
# Baseline evaluation
|
|
evaluator = Evaluate(
|
|
devset=testset,
|
|
metric=accuracy,
|
|
num_threads=4, # Parallel evaluation
|
|
)
|
|
|
|
baseline_score = evaluator(program)
|
|
print(f"Baseline accuracy: {baseline_score:.2%}")
|
|
|
|
# Optimize
|
|
optimizer = dspy.MIPROv2(metric=accuracy, auto="medium")
|
|
optimized = optimizer.compile(
|
|
student=program,
|
|
trainset=trainset,
|
|
valset=valset,
|
|
)
|
|
|
|
# Evaluate optimized version
|
|
optimized_score = evaluator(optimized)
|
|
print(f"Optimized accuracy: {optimized_score:.2%}")
|
|
print(f"Improvement: {(optimized_score - baseline_score):.2%}")</code></pre>
|
|
<p><strong>Best practice workflow</strong>:</p>
|
|
<ol><li>Baseline evaluation</li><li>Optimize on train/val sets</li><li>Final evaluation on test set</li><li>Compare before/after</li></ol>
|
|
<h3>Pattern 7: Saving and Loading Compiled Programs</h3>
|
|
<pre><code class="lang-python">import dspy
|
|
|
|
# Compile program
|
|
optimizer = dspy.MIPROv2(metric=my_metric)
|
|
compiled = optimizer.compile(student=program, trainset=trainset)
|
|
|
|
# Save compiled program
|
|
compiled.save("optimized_qa.json")
|
|
|
|
# Later, load compiled program
|
|
loaded_program = dspy.ChainOfThought("question -> answer")
|
|
loaded_program.load("optimized_qa.json")
|
|
|
|
# Use loaded program (has optimized prompts/demos)
|
|
result = loaded_program(question="What is DSPy?")</code></pre>
|
|
<p><strong>When to use</strong>:</p>
|
|
<ul><li>Production deployment</li><li>Avoid re-optimizing every run</li><li>Share optimized programs with team</li><li>Version control for prompts</li></ul>
|
|
<h3>Pattern 8: Iterative Optimization</h3>
|
|
<pre><code class="lang-python">import dspy
|
|
|
|
program = dspy.ChainOfThought("question -> answer")
|
|
|
|
# Start with simple optimizer
|
|
optimizer1 = dspy.BootstrapFewShot(metric=my_metric)
|
|
v1 = optimizer1.compile(student=program, trainset=trainset)
|
|
score_v1 = evaluate(v1)
|
|
|
|
# If not good enough, try more sophisticated optimizer
|
|
if score_v1 < 0.8:
|
|
optimizer2 = dspy.MIPROv2(metric=my_metric, auto="medium")
|
|
v2 = optimizer2.compile(student=v1, trainset=trainset, valset=valset)
|
|
score_v2 = evaluate(v2)
|
|
|
|
# If still not good enough, use heavy optimization
|
|
if score_v2 < 0.9:
|
|
optimizer3 = dspy.MIPROv2(metric=my_metric, auto="heavy")
|
|
v3 = optimizer3.compile(student=v2, trainset=trainset, valset=valset)
|
|
score_v3 = evaluate(v3)
|
|
|
|
# Use best version
|
|
best = max([(score_v1, v1), (score_v2, v2), (score_v3, v3)], key=lambda x: x[0])
|
|
print(f"Best score: {best[0]:.2%}")
|
|
final_program = best[1]</code></pre>
|
|
<p><strong>Strategy</strong>: Progressive optimization</p>
|
|
<ul><li>Start fast and simple</li><li>Increase sophistication if needed</li><li>Compare results at each stage</li></ul>
|
|
<hr>
|
|
<h2>Quick Reference</h2>
|
|
<h3>Optimizer Comparison</h3>
|
|
<p>| Optimizer | Speed | Quality | Use Case | |-----------|-------|---------|----------| | BootstrapFewShot | Fast | Good | Simple tasks, quick iteration | | MIPROv2 | Medium | Better | Complex tasks, production | | COPRO | Slow | Best (pipelines) | Multi-module workflows | | BootstrapFinetune | Slowest | Best (single module) | Large datasets, max accuracy |</p>
|
|
<h3>Optimization Workflow</h3>
|
|
<pre><code>1. Define metric function
|
|
2. Prepare trainset (and valset)
|
|
3. Choose optimizer
|
|
4. Compile: optimizer.compile(student, trainset)
|
|
5. Evaluate on testset
|
|
6. Save optimized program</code></pre>
|
|
<h3>Metric Function Template</h3>
|
|
<pre><code class="lang-python">def my_metric(example, pred, trace=None):
|
|
"""
|
|
Args:
|
|
example: Ground truth example
|
|
pred: Model prediction
|
|
trace: Execution trace (optional)
|
|
|
|
Returns:
|
|
float: Score between 0.0 and 1.0
|
|
OR
|
|
bool: True if correct, False otherwise
|
|
"""
|
|
# Implement scoring logic
|
|
return score</code></pre>
|
|
<h3>Best Practices</h3>
|
|
<pre><code>✅ DO: Start with BootstrapFewShot for quick iteration
|
|
✅ DO: Use separate train/val/test sets
|
|
✅ DO: Define clear, measurable metrics
|
|
✅ DO: Save optimized programs for production
|
|
✅ DO: Evaluate baseline before optimizing
|
|
|
|
❌ DON'T: Optimize on test set (data leakage!)
|
|
❌ DON'T: Use tiny training sets (<10 examples)
|
|
❌ DON'T: Ignore metric design (affects optimization)
|
|
❌ DON'T: Re-optimize in production (pre-compile)</code></pre>
|
|
<hr>
|
|
<h2>Anti-Patterns</h2>
|
|
<p>❌ <strong>Optimizing on test set</strong>: Data leakage, inflated scores</p>
|
|
<pre><code class="lang-python"># Bad
|
|
optimizer.compile(student=program, trainset=testset) # WRONG!</code></pre>
|
|
<p>✅ Use separate train/val/test splits:</p>
|
|
<pre><code class="lang-python"># Good
|
|
compiled = optimizer.compile(student=program, trainset=trainset, valset=valset)
|
|
final_score = evaluate(compiled, testset) # Honest evaluation</code></pre>
|
|
<p>❌ <strong>Vague metrics</strong>: Optimizer can't improve what it can't measure</p>
|
|
<pre><code class="lang-python"># Bad
|
|
def bad_metric(example, pred, trace=None):
|
|
return True # Always returns True, useless!</code></pre>
|
|
<p>✅ Define clear, discriminative metrics:</p>
|
|
<pre><code class="lang-python"># Good
|
|
def good_metric(example, pred, trace=None):
|
|
return example.answer.lower() == pred.answer.lower()</code></pre>
|
|
<p>❌ <strong>Not evaluating baseline</strong>: Don't know if optimization helped</p>
|
|
<pre><code class="lang-python"># Bad - skip straight to optimization
|
|
optimized = optimizer.compile(...) # Did it improve? Unknown!</code></pre>
|
|
<p>✅ Always evaluate baseline first:</p>
|
|
<pre><code class="lang-python"># Good
|
|
baseline_score = evaluate(program, testset)
|
|
optimized = optimizer.compile(program, trainset)
|
|
optimized_score = evaluate(optimized, testset)
|
|
print(f"Improvement: {optimized_score - baseline_score}")</code></pre>
|
|
<p>❌ <strong>Tiny training sets</strong>: Not enough signal for optimization</p>
|
|
<pre><code class="lang-python"># Bad
|
|
trainset = [example1, example2] # Only 2 examples</code></pre>
|
|
<p>✅ Use adequate training data:</p>
|
|
<pre><code class="lang-python"># Good
|
|
trainset = [...] # 50+ examples for BootstrapFewShot, 500+ for MIPROv2</code></pre>
|
|
<hr>
|
|
<h2>Related Skills</h2>
|
|
<ul><li><code>dspy-evaluation.md</code> - Evaluating DSPy programs with metrics</li><li><code>dspy-modules.md</code> - Modules to optimize</li><li><code>dspy-signatures.md</code> - Signatures affected by optimization</li><li><code>llm-dataset-preparation.md</code> - Preparing training datasets</li></ul>
|
|
<hr>
|
|
<p><strong>Last Updated</strong>: 2025-10-25 <strong>Format Version</strong>: 1.0 (Atomic)</p>
|
|
<footer>
|
|
<div class="fmap">
|
|
<div><b>Build</b>
|
|
<a href="https://hanzo.app">hanzo.app</a> app builder
|
|
<a href="https://hanzo.codes">hanzo.codes</a> code with an agent
|
|
<a href="https://hanzoskills.com">hanzoskills.com</a> what an agent reads</div>
|
|
<div><b>Run</b>
|
|
<a href="https://console.hanzo.ai">console.hanzo.ai</a> cloud console
|
|
<a href="https://api.hanzo.ai">api.hanzo.ai</a> the API</div>
|
|
<div><b>Account</b>
|
|
<a href="https://hanzo.id">hanzo.id</a> sign in
|
|
<a href="https://pay.hanzo.ai">pay.hanzo.ai</a> top up
|
|
<a href="https://billing.hanzo.ai">billing.hanzo.ai</a> invoices and usage</div>
|
|
<div><b>Company</b>
|
|
<a href="https://hanzo.ai">hanzo.ai</a> canonical
|
|
<a href="https://hanzo.ventures">hanzo.ventures</a> ventures</div>
|
|
</div>
|
|
<div class="fend"><a href="/dspy-optimizers.md">This page as plain text</a> · <a href="/llms.txt">Index for agents</a></div>
|
|
</footer>
|
|
</div></body></html> |