mirror of
https://github.com/hanzoai/skills.git
synced 2026-11-19 00:47:36 +00:00
hanzoskills.com/llms.txt now opens with the install line, the API reference index and the company index, so an agent that lands on any one of the three can find the rest. robots.txt claimed 537 documents while llms.txt counts 920. The number is gone from robots.txt; llms.txt derives it. public/ is regenerated from scripts/site.py, which it had fallen behind: the copy-prompt menu and agent bar were in the generator and not in the committed pages. tests/site.mjs passes against the result. Co-authored-by: Hanzo Dev <[email protected]>
686 lines
34 KiB
HTML
686 lines
34 KiB
HTML
<!doctype html>
|
|
<html lang="en"><head>
|
|
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
<title>dspy-setup</title>
|
|
<meta name="color-scheme" content="dark">
|
|
<style>
|
|
/* Committed to one visual world: a dark instrument panel. No light variant —
|
|
the dot field and the hairline rules only read on black, and a naive
|
|
inversion would wash both out. Every colour is painted explicitly so the page
|
|
never borrows a host background. */
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Pixel Square';src:url('/fonts/ZenPixel-Square.woff2') format('woff2');font-style:normal;font-display:swap}
|
|
|
|
:root{
|
|
--ink:#f4f4f2; --soft:#9c9c98; --faint:#6a6a66; --ground:#000; --panel:#0a0a0a;
|
|
--line:#232323; --line-hi:#3a3a3a; --accent:#f4f4f2;
|
|
--font-zen-sans:'Zen',-apple-system,BlinkMacSystemFont,"Segoe UI",Helvetica,Arial,sans-serif;
|
|
--font-zen-mono:'Zen Mono',ui-monospace,"SF Mono",SFMono-Regular,Menlo,Consolas,monospace;
|
|
--font-zen-pixel-square:'Zen Pixel Square',monospace;
|
|
--mono:var(--font-zen-mono);
|
|
--sans:var(--font-zen-sans);
|
|
/* Controls are pills and surfaces are soft, as on hanzo.ai. */
|
|
--r:999px; --r-soft:14px;
|
|
}
|
|
*{box-sizing:border-box}
|
|
html{-webkit-text-size-adjust:100%}
|
|
body{margin:0;background:var(--ground);color:var(--ink);font-family:var(--sans);
|
|
font-size:16px;line-height:1.65;-webkit-font-smoothing:antialiased}
|
|
.wrap{max-width:60rem;margin:0 auto;padding:0 1.75rem}
|
|
|
|
/* The dot field. A halftone made of one repeating radial-gradient, faded out
|
|
along the diagonal by a mask so it reads as a wave rather than wallpaper. */
|
|
/* No overflow clip here: the dot field is inset to the box already, and a clip
|
|
would cut off the copy-prompt menu. */
|
|
.hero{position:relative;border-bottom:1px solid var(--line)}
|
|
.hero::before{content:"";position:absolute;inset:0;
|
|
background-image:radial-gradient(var(--line-hi) 1px, transparent 1px);
|
|
background-size:7px 7px;
|
|
-webkit-mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
pointer-events:none}
|
|
.hero .wrap{position:relative;padding-top:5.5rem;padding-bottom:4rem}
|
|
|
|
nav{border-bottom:1px solid var(--line)}
|
|
nav .in{max-width:60rem;margin:0 auto;padding:1.1rem 1.75rem;display:flex;
|
|
justify-content:space-between;align-items:center;gap:1rem}
|
|
.brand{font-family:var(--mono);font-size:.85rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--ink);text-decoration:none;
|
|
display:inline-flex;align-items:center;gap:.6rem}
|
|
.brand i{font-style:normal;color:var(--faint)}
|
|
.mark{width:1.15em;height:1.15em;display:block;flex:0 0 auto}
|
|
nav a.up{font-family:var(--mono);font-size:.72rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--soft);text-decoration:none}
|
|
nav a.up:hover{color:var(--ink)}
|
|
|
|
h1{font-family:var(--mono);font-size:clamp(2rem,5.4vw,3.4rem);line-height:1.12;
|
|
letter-spacing:-.02em;margin:0 0 1.1rem;font-weight:500;text-wrap:balance}
|
|
h1 em{font-style:normal;color:var(--faint)}
|
|
.lede{color:var(--soft);font-size:1.05rem;max-width:52ch;margin:0 0 2.25rem}
|
|
|
|
.btns{display:flex;flex-wrap:wrap;gap:.7rem;margin:0 0 .5rem;align-items:center}
|
|
.btn{display:inline-flex;align-items:center;gap:.5rem;font-family:var(--mono);
|
|
font-size:.72rem;letter-spacing:.14em;text-transform:uppercase;
|
|
padding:.8rem 1.4rem;border:1px solid var(--line-hi);border-radius:var(--r);
|
|
color:var(--ink);text-decoration:none;background:transparent;cursor:pointer;
|
|
line-height:1.2;font-weight:400}
|
|
.btn:hover{background:#141414;text-decoration:none;border-color:var(--soft)}
|
|
.btn.solid{background:var(--ink);color:#000;border-color:var(--ink)}
|
|
.btn.solid:hover{background:#d8d8d4}
|
|
.btn .chev{width:.7em;height:.7em;flex:0 0 auto;transition:transform .15s ease}
|
|
.btn[aria-expanded="true"] .chev{transform:rotate(180deg)}
|
|
|
|
/* Copy-prompt selector. Each row copies the prompt for its own target. */
|
|
.copy{position:relative;display:inline-flex}
|
|
.menu{position:absolute;top:calc(100% + .5rem);left:0;z-index:20;min-width:21rem;
|
|
background:var(--panel);border:1px solid var(--line-hi);border-radius:var(--r-soft);
|
|
padding:.4rem;display:flex;flex-direction:column;
|
|
box-shadow:0 12px 32px rgba(0,0,0,.75)}
|
|
.menu[hidden]{display:none}
|
|
.mi{display:flex;align-items:center;gap:.7rem;width:100%;text-align:left;
|
|
padding:.62rem .8rem;border:0;border-radius:calc(var(--r-soft) - .35rem);
|
|
background:transparent;color:var(--ink);font-family:var(--mono);font-size:.76rem;
|
|
letter-spacing:.04em;text-decoration:none;cursor:pointer;line-height:1.3;
|
|
white-space:nowrap}
|
|
.mi:hover,.mi:focus-visible{background:#181818;outline:none}
|
|
.mi .mark{width:1.05rem;height:1.05rem;color:var(--soft)}
|
|
.mi:hover .mark{color:var(--ink)}
|
|
.mi u{margin-left:auto;color:var(--faint);text-decoration:none;font-size:.68rem;
|
|
letter-spacing:.1em;text-transform:uppercase}
|
|
|
|
/* Works-with bar: one scrolling row, name and mark per agent. */
|
|
.strip{border-bottom:1px solid var(--line);background:var(--panel)}
|
|
.strip .in{max-width:60rem;margin:0 auto;padding:.9rem 1.75rem;display:flex;
|
|
align-items:center;gap:1.25rem;font-family:var(--mono);font-size:.72rem;
|
|
letter-spacing:.1em;text-transform:uppercase;color:var(--faint)}
|
|
.strip b{color:var(--soft);font-weight:400;flex:0 0 auto;
|
|
border-right:1px solid var(--line);padding-right:1.25rem;align-self:stretch;
|
|
display:flex;align-items:center}
|
|
.bar{display:flex;align-items:center;gap:.5rem;flex:1 1 auto;min-width:0;
|
|
overflow-x:auto;scrollbar-width:none;scroll-snap-type:x proximity;
|
|
-webkit-overflow-scrolling:touch;
|
|
-webkit-mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent);
|
|
mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent)}
|
|
.bar::-webkit-scrollbar{display:none}
|
|
.agent{display:inline-flex;align-items:center;gap:.45rem;flex:0 0 auto;
|
|
padding:.4rem .85rem;border:1px solid var(--line);border-radius:var(--r);
|
|
color:var(--soft);white-space:nowrap;scroll-snap-align:start;
|
|
text-decoration:none;background:var(--ground)}
|
|
.agent:hover{color:var(--ink);border-color:var(--line-hi);text-decoration:none}
|
|
.agent .mark{width:.95rem;height:.95rem}
|
|
|
|
h2{font-family:var(--mono);font-size:1.1rem;letter-spacing:-.01em;margin:2.75rem 0 .75rem;font-weight:500}
|
|
h3{font-size:1rem;margin:2rem 0 .5rem;font-weight:600}
|
|
h4,h5,h6{font-size:.94rem;margin:1.5rem 0 .5rem;font-weight:600}
|
|
p{margin:0 0 1rem;max-width:70ch}
|
|
ul,ol{margin:0 0 1rem;padding-left:1.35rem;max-width:70ch}
|
|
li{margin:.22rem 0}
|
|
a{color:var(--ink);text-decoration:underline;text-underline-offset:3px;
|
|
text-decoration-color:var(--line-hi)}
|
|
a:hover{text-decoration-color:var(--ink)}
|
|
a:focus-visible{outline:1px solid var(--ink);outline-offset:3px}
|
|
code{font-family:var(--mono);font-size:.85em;background:var(--panel);
|
|
border:1px solid var(--line);border-radius:5px;padding:.1em .36em}
|
|
pre{background:var(--panel);border:1px solid var(--line);padding:1rem 1.1rem;
|
|
border-radius:var(--r-soft);overflow-x:auto;margin:0 0 1.25rem}
|
|
pre code{background:none;border:0;padding:0;font-size:.82rem;line-height:1.7}
|
|
blockquote{margin:0 0 1rem;padding-left:1rem;border-left:1px solid var(--line-hi);color:var(--soft)}
|
|
hr{border:0;border-top:1px solid var(--line);margin:2.5rem 0}
|
|
table{border-collapse:collapse;width:100%;margin:0 0 1.25rem;font-size:.92rem;
|
|
display:block;overflow-x:auto}
|
|
th,td{border:1px solid var(--line);padding:.5rem .7rem;text-align:left}
|
|
|
|
.curl{display:block;background:var(--panel);border:1px solid var(--line);
|
|
border-radius:var(--r-soft);padding:.95rem 1.1rem;font-family:var(--mono);
|
|
font-size:.8rem;color:var(--soft);overflow-x:auto;margin:0 0 2.5rem;
|
|
white-space:pre;line-height:1.9}
|
|
.curl b{color:var(--ink);font-weight:400}
|
|
.curl i{font-style:normal;color:var(--faint)}
|
|
|
|
/* index */
|
|
#q{width:100%;padding:.85rem 1.2rem;font-family:var(--mono);font-size:.85rem;color:var(--ink);
|
|
background:var(--panel);border:1px solid var(--line);border-radius:var(--r);margin:0 0 .6rem}
|
|
#q::placeholder{color:var(--faint);text-transform:uppercase;letter-spacing:.1em;font-size:.72rem}
|
|
#q:focus{outline:none;border-color:var(--soft)}
|
|
#count{color:var(--faint);font-family:var(--mono);font-size:.72rem;letter-spacing:.1em;
|
|
text-transform:uppercase;margin:0 0 2rem}
|
|
.cat{margin:2.5rem 0 .5rem;font-family:var(--mono);font-size:.7rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);border-bottom:1px solid var(--line);
|
|
padding-bottom:.45rem}
|
|
.row{display:block;padding:.6rem .9rem;margin:0 -.9rem;border-bottom:1px solid var(--line);
|
|
border-radius:var(--r-soft);text-decoration:none}
|
|
.row:hover{background:#0a0a0a}
|
|
.nm{font-family:var(--mono);font-size:.84rem;color:var(--ink);display:block}
|
|
.ds{color:var(--soft);font-size:.9rem;display:block;margin-top:.12rem}
|
|
footer{margin-top:4rem;padding:2rem 0 4rem;border-top:1px solid var(--line);
|
|
color:var(--faint);font-size:.85rem}
|
|
footer a{color:var(--soft)}
|
|
.fmap{display:grid;gap:1.75rem;grid-template-columns:repeat(auto-fit,minmax(11rem,1fr));
|
|
margin:0 0 2rem;font-size:.85rem}
|
|
.fmap b{display:block;font-family:var(--mono);font-size:.68rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);font-weight:400;margin:0 0 .6rem;
|
|
border-bottom:1px solid var(--line);padding-bottom:.4rem}
|
|
.fmap a{display:inline-block;color:var(--ink);text-decoration:none;margin:.3rem .35rem 0 0}
|
|
.fmap a:hover{text-decoration:underline}
|
|
.fmap div{color:var(--faint);line-height:1.5}
|
|
.fend{border-top:1px solid var(--line);padding-top:1.25rem;font-size:.82rem}
|
|
|
|
@media (max-width:640px){
|
|
.hero .wrap{padding-top:3.25rem;padding-bottom:2.75rem}
|
|
.btns{gap:.5rem}
|
|
.btn{padding:.75rem 1.15rem;font-size:.68rem}
|
|
.copy,.copy .btn{width:100%;justify-content:center}
|
|
.menu{left:0;right:0;min-width:0}
|
|
.strip .in{flex-direction:column;align-items:flex-start;gap:.6rem;padding:.9rem 1.75rem}
|
|
.strip b{border-right:0;padding-right:0}
|
|
.bar{width:100%}
|
|
}
|
|
</style>
|
|
<link rel="alternate" type="text/markdown" href="/dspy-setup.md">
|
|
|
|
<link rel="icon" href="/favicon.svg">
|
|
<link rel="canonical" href="https://hanzoskills.com/dspy-setup">
|
|
</head>
|
|
<!-- Reading this with a machine? Every page here is also plain markdown.
|
|
This one: https://hanzoskills.com/dspy-setup.md
|
|
Index: https://hanzoskills.com/llms.txt
|
|
Corpus: https://hanzoskills.com/all.md
|
|
Append .md to any path for the markdown twin. -->
|
|
<body>
|
|
<nav><div class="in">
|
|
<a class="brand" href="/"><svg class="mark" viewBox="0 0 67 67" fill="currentColor" aria-hidden="true"><path d="M22.21 67V44.6369H0V67H22.21Z"/><path d="M66.7038 22.3184H22.2534L0.0878906 44.6367H44.4634L66.7038 22.3184Z"/><path d="M22.21 0H0V22.3184H22.21V0Z"/><path d="M66.7198 0H44.5098V22.3184H66.7198V0Z"/><path d="M66.7198 67V44.6369H44.5098V67H66.7198Z"/></svg> HANZO<i>SKILLS</i></a>
|
|
<a class="up" href="/">All skills</a>
|
|
</div></nav>
|
|
|
|
<div class="wrap">
|
|
<h1>dspy-setup</h1><p class="lede">Installation, configuration, and language model setup for DSPy framework including Modal and HuggingFace</p><h1>DSPy Setup</h1>
|
|
<p><strong>Scope</strong>: Installation, LM configuration, environment setup, Modal/HuggingFace integration <strong>Lines</strong>: ~420 <strong>Last Updated</strong>: 2025-10-25</p>
|
|
<h2>When to Use This Skill</h2>
|
|
<p>Activate this skill when:</p>
|
|
<ul><li>Starting a new DSPy project from scratch</li><li>Configuring language models for DSPy programs</li><li>Setting up Modal-hosted models for DSPy</li><li>Integrating HuggingFace models (Inference API or self-hosted)</li><li>Troubleshooting DSPy installation issues</li><li>Switching between different LM providers (OpenAI, Anthropic, Modal, HF)</li><li>Understanding DSPy's core architecture and initialization</li></ul>
|
|
<h2>Core Concepts</h2>
|
|
<h3>DSPy Programming Model</h3>
|
|
<p><strong>Philosophy</strong>: Programming, not prompting</p>
|
|
<ul><li>Write <strong>declarative Python code</strong> instead of brittle prompt strings</li><li>Define task structure with <strong>signatures</strong> (input/output specs)</li><li>Compose reusable <strong>modules</strong> (Predict, ChainOfThought, ReAct)</li><li>Let DSPy <strong>optimize</strong> prompts and weights automatically</li></ul>
|
|
<p><strong>Key insight</strong>: DSPy separates "what" (signatures) from "how" (modules) from "optimization" (teleprompters)</p>
|
|
<h3>Installation and Dependencies</h3>
|
|
<p><strong>Core package</strong>:</p>
|
|
<pre><code class="lang-bash"># Latest stable release
|
|
pip install dspy-ai
|
|
|
|
# Or with uv (recommended)
|
|
uv add dspy-ai
|
|
|
|
# Development/latest version
|
|
pip install git+https://github.com/stanfordnlp/dspy.git</code></pre>
|
|
<p><strong>Optional dependencies</strong>:</p>
|
|
<pre><code class="lang-bash"># For specific LM providers
|
|
uv add openai anthropic cohere litellm
|
|
|
|
# For HuggingFace
|
|
uv add huggingface-hub transformers
|
|
|
|
# For Modal integration
|
|
uv add modal
|
|
|
|
# For RAG and vector search
|
|
uv add chromadb weaviate-client qdrant-client
|
|
|
|
# For evaluation and metrics
|
|
uv add datasets evaluate</code></pre>
|
|
<h3>Language Model Configuration</h3>
|
|
<p><strong>LM object</strong>: Central to DSPy workflow</p>
|
|
<ul><li><strong>Configure once</strong>, use everywhere</li><li>Supports multiple providers (OpenAI, Anthropic, Modal, HuggingFace, local)</li><li>Handles API keys, rate limits, caching</li></ul>
|
|
<p><strong>Three configuration patterns</strong>:</p>
|
|
<ol><li><strong>Global default</strong>: <code>dspy.configure(lm=lm)</code> - all modules use this LM</li><li><strong>Module-specific</strong>: <code>predictor = dspy.Predict(signature, lm=custom_lm)</code></li><li><strong>Context manager</strong>: Temporarily override LM for specific operations</li></ol>
|
|
<hr>
|
|
<h2>Patterns</h2>
|
|
<h3>Pattern 1: Basic Setup (OpenAI)</h3>
|
|
<pre><code class="lang-python">import dspy
|
|
|
|
# Configure OpenAI LM
|
|
lm = dspy.LM(
|
|
model="openai/gpt-4o-mini",
|
|
api_key="your-api-key-here", # or set OPENAI_API_KEY env var
|
|
max_tokens=1000,
|
|
temperature=0.0,
|
|
)
|
|
|
|
# Set as global default
|
|
dspy.configure(lm=lm)
|
|
|
|
# Test configuration
|
|
response = lm("Hello, DSPy!")
|
|
print(response) # Should return LM response</code></pre>
|
|
<p><strong>When to use</strong>:</p>
|
|
<ul><li>Quick prototyping with OpenAI models</li><li>Production apps with GPT-4 or GPT-3.5</li><li>Cost-effective development with gpt-4o-mini</li></ul>
|
|
<h3>Pattern 2: Anthropic Claude Setup</h3>
|
|
<pre><code class="lang-python">import dspy
|
|
import os
|
|
|
|
# Configure Claude
|
|
lm = dspy.LM(
|
|
model="anthropic/claude-3-5-sonnet-20241022",
|
|
api_key=os.getenv("ANTHROPIC_API_KEY"),
|
|
max_tokens=4096,
|
|
temperature=0.0,
|
|
)
|
|
|
|
dspy.configure(lm=lm)
|
|
|
|
# Claude-specific features
|
|
lm_with_caching = dspy.LM(
|
|
model="anthropic/claude-3-5-sonnet-20241022",
|
|
api_key=os.getenv("ANTHROPIC_API_KEY"),
|
|
max_tokens=4096,
|
|
# Claude supports prompt caching for long contexts
|
|
cache_prompt=True,
|
|
)</code></pre>
|
|
<p><strong>When to use</strong>:</p>
|
|
<ul><li>Long context windows (200K tokens)</li><li>Advanced reasoning tasks</li><li>Cost optimization with prompt caching</li></ul>
|
|
<h3>Pattern 3: Modal-Hosted Models</h3>
|
|
<pre><code class="lang-python">import dspy
|
|
import modal
|
|
|
|
# Deploy model on Modal with GPU
|
|
app = modal.App("dspy-llm")
|
|
|
|
@app.function(
|
|
image=modal.Image.debian_slim().pip_install("vllm", "transformers"),
|
|
gpu="L40S", # Cost-effective GPU
|
|
timeout=600,
|
|
)
|
|
@modal.web_endpoint(method="POST")
|
|
def inference(request: dict):
|
|
"""Modal endpoint for model inference"""
|
|
from vllm import LLM, SamplingParams
|
|
|
|
# Load model once (cached across requests)
|
|
llm = LLM(model="meta-llama/Meta-Llama-3.1-8B-Instruct")
|
|
|
|
prompt = request.get("prompt")
|
|
params = SamplingParams(
|
|
temperature=request.get("temperature", 0.0),
|
|
max_tokens=request.get("max_tokens", 1000),
|
|
)
|
|
|
|
output = llm.generate(prompt, params)
|
|
return {"response": output[0].outputs[0].text}
|
|
|
|
# Use Modal endpoint in DSPy
|
|
# After deploying: modal deploy dspy_modal.py
|
|
|
|
import requests
|
|
|
|
class ModalLM:
|
|
"""Custom LM wrapper for Modal endpoints"""
|
|
def __init__(self, endpoint_url, max_tokens=1000, temperature=0.0):
|
|
self.endpoint_url = endpoint_url
|
|
self.max_tokens = max_tokens
|
|
self.temperature = temperature
|
|
|
|
def __call__(self, prompt, **kwargs):
|
|
response = requests.post(
|
|
self.endpoint_url,
|
|
json={
|
|
"prompt": prompt,
|
|
"max_tokens": kwargs.get("max_tokens", self.max_tokens),
|
|
"temperature": kwargs.get("temperature", self.temperature),
|
|
}
|
|
)
|
|
return response.json()["response"]
|
|
|
|
# Configure DSPy with Modal endpoint
|
|
modal_lm = ModalLM(
|
|
endpoint_url="https://your-app--inference.modal.run",
|
|
max_tokens=1000,
|
|
)
|
|
|
|
# Note: For full DSPy integration, wrap in dspy.LM-compatible class</code></pre>
|
|
<p><strong>When to use</strong>:</p>
|
|
<ul><li>GPU-accelerated inference without infrastructure management</li><li>Cost-effective deployment (pay only for usage)</li><li>Custom fine-tuned models</li><li>High-throughput batch processing</li></ul>
|
|
<p><strong>Benefits</strong>:</p>
|
|
<ul><li>Serverless GPU inference (L40S, H100, A100)</li><li>Automatic scaling</li><li>Cold start optimization with Modal's caching</li></ul>
|
|
<h3>Pattern 4: HuggingFace Inference API</h3>
|
|
<pre><code class="lang-python">import dspy
|
|
import os
|
|
|
|
# Use HuggingFace Inference API
|
|
lm = dspy.LM(
|
|
model="huggingface/meta-llama/Meta-Llama-3.1-8B-Instruct",
|
|
api_key=os.getenv("HUGGINGFACE_API_KEY"),
|
|
api_base="https://api-inference.huggingface.co/models",
|
|
max_tokens=1000,
|
|
)
|
|
|
|
dspy.configure(lm=lm)
|
|
|
|
# Alternative: Use specific HuggingFace endpoint
|
|
lm = dspy.LM(
|
|
model="huggingface/mistralai/Mistral-7B-Instruct-v0.2",
|
|
api_key=os.getenv("HUGGINGFACE_API_KEY"),
|
|
max_tokens=1000,
|
|
)</code></pre>
|
|
<p><strong>When to use</strong>:</p>
|
|
<ul><li>Quick access to HuggingFace hosted models</li><li>Testing different open-source models</li><li>Prototyping without infrastructure setup</li></ul>
|
|
<h3>Pattern 5: Self-Hosted HuggingFace (vLLM/TGI)</h3>
|
|
<pre><code class="lang-python">import dspy
|
|
|
|
# Connect to self-hosted vLLM or Text Generation Inference server
|
|
# Assumes you have vLLM running: vllm serve meta-llama/Meta-Llama-3.1-8B-Instruct
|
|
|
|
lm = dspy.LM(
|
|
model="openai/meta-llama/Meta-Llama-3.1-8B-Instruct", # vLLM uses OpenAI-compatible API
|
|
api_base="http://localhost:8000/v1", # vLLM default endpoint
|
|
api_key="EMPTY", # vLLM doesn't require auth by default
|
|
max_tokens=2000,
|
|
)
|
|
|
|
dspy.configure(lm=lm)
|
|
|
|
# For Text Generation Inference (TGI)
|
|
lm_tgi = dspy.LM(
|
|
model="tgi/meta-llama/Meta-Llama-3.1-8B-Instruct",
|
|
api_base="http://localhost:8080", # TGI default port
|
|
max_tokens=2000,
|
|
)</code></pre>
|
|
<p><strong>When to use</strong>:</p>
|
|
<ul><li>Running models on your own infrastructure</li><li>Maximum control over model configuration</li><li>High-throughput production deployments</li></ul>
|
|
<h3>Pattern 6: Modal + HuggingFace Integration</h3>
|
|
<pre><code class="lang-python">import modal
|
|
import dspy
|
|
|
|
# Deploy HuggingFace model on Modal with optimized serving
|
|
app = modal.App("dspy-hf-modal")
|
|
|
|
@app.function(
|
|
image=modal.Image.debian_slim()
|
|
.pip_install("vllm", "transformers", "torch"),
|
|
gpu="L40S",
|
|
timeout=600,
|
|
# Use Modal volumes for model caching
|
|
volumes={"/models": modal.Volume.from_name("hf-models")},
|
|
)
|
|
@modal.asgi_app()
|
|
def serve():
|
|
"""Serve HuggingFace model via vLLM on Modal"""
|
|
from vllm import AsyncLLMEngine, AsyncEngineArgs, SamplingParams
|
|
from vllm.entrypoints.openai.api_server import build_app
|
|
|
|
# Configure vLLM engine
|
|
engine_args = AsyncEngineArgs(
|
|
model="meta-llama/Meta-Llama-3.1-8B-Instruct",
|
|
download_dir="/models", # Use Modal volume
|
|
tensor_parallel_size=1,
|
|
)
|
|
|
|
# Build OpenAI-compatible API
|
|
return build_app(engine_args)
|
|
|
|
# After deployment, use in DSPy
|
|
lm = dspy.LM(
|
|
model="openai/meta-llama/Meta-Llama-3.1-8B-Instruct",
|
|
api_base="https://your-app--serve.modal.run/v1",
|
|
api_key="EMPTY",
|
|
max_tokens=2000,
|
|
)
|
|
|
|
dspy.configure(lm=lm)</code></pre>
|
|
<p><strong>When to use</strong>:</p>
|
|
<ul><li>Best of both worlds: HuggingFace models + Modal's serverless GPUs</li><li>Production deployments with autoscaling</li><li>Cost optimization (only pay when using)</li></ul>
|
|
<p><strong>Benefits</strong>:</p>
|
|
<ul><li>No infrastructure management</li><li>Automatic GPU scaling</li><li>Model caching with Modal volumes</li><li>OpenAI-compatible API</li></ul>
|
|
<h3>Pattern 7: Local Models (Ollama)</h3>
|
|
<pre><code class="lang-python">import dspy
|
|
|
|
# Run local model via Ollama
|
|
# First: ollama pull llama3.2
|
|
lm = dspy.LM(
|
|
model="ollama/llama3.2",
|
|
api_base="http://localhost:11434", # Ollama default port
|
|
max_tokens=2000,
|
|
)
|
|
|
|
dspy.configure(lm=lm)
|
|
|
|
# No API key needed for local models
|
|
# Great for development and privacy-sensitive applications</code></pre>
|
|
<p><strong>When to use</strong>:</p>
|
|
<ul><li>Offline development</li><li>Privacy requirements (data stays local)</li><li>Cost-free experimentation</li><li>Custom fine-tuned models</li></ul>
|
|
<h3>Pattern 7.5: Hanzo SDK (Unified Multi-Provider with Auto-Routing)</h3>
|
|
<pre><code class="lang-python">import dspy
|
|
from hanzo import Hanzo
|
|
|
|
# Initialize Hanzo SDK with hybrid mode (local + cloud)
|
|
hanzo = Hanzo(
|
|
inference_mode='hybrid', # Local first, cloud fallback
|
|
auto_route=True, # Automatic model selection
|
|
cost_optimize=True, # Optimize for cost
|
|
|
|
# Optional: Provider credentials (falls back to env vars)
|
|
node_url='http://localhost:8080', # Hanzo Node for local
|
|
openai_api_key='sk-...',
|
|
anthropic_api_key='sk-ant-...'
|
|
)
|
|
|
|
# Create DSPy-compatible LM wrapper
|
|
class HanzoLM(dspy.LM):
|
|
def __init__(self, hanzo_client, model=None):
|
|
self.hanzo = hanzo_client
|
|
self.model = model
|
|
super().__init__(model=model or "auto")
|
|
|
|
def __call__(self, prompt, **kwargs):
|
|
response = self.hanzo.chat.completions.create(
|
|
messages=[{'role': 'user', 'content': prompt}],
|
|
model=self.model if self.model else None, # Auto-route if no model
|
|
**kwargs
|
|
)
|
|
return response.choices[0].message.content
|
|
|
|
# Configure DSPy with Hanzo
|
|
lm = HanzoLM(hanzo, model='llama-3-8b') # Or model=None for auto-routing
|
|
dspy.configure(lm=lm)
|
|
|
|
# Alternative: Use Hanzo for specific operations
|
|
local_lm = HanzoLM(Hanzo(inference_mode='local'), model='llama-3-8b')
|
|
cloud_lm = HanzoLM(Hanzo(inference_mode='cloud'), model='gpt-4')
|
|
|
|
# Privacy-sensitive operations use local
|
|
dspy.configure(lm=local_lm)
|
|
sensitive_result = predictor(private_data)
|
|
|
|
# Complex operations use cloud
|
|
with dspy.context(lm=cloud_lm):
|
|
complex_result = predictor(complex_query)</code></pre>
|
|
<p><strong>When to use</strong>:</p>
|
|
<ul><li><strong>Multi-provider flexibility</strong>: Switch between local, OpenAI, Anthropic automatically</li><li><strong>Cost optimization</strong>: Automatic routing to cheapest suitable model</li><li><strong>Privacy requirements</strong>: Local-first with selective cloud fallback</li><li><strong>Production reliability</strong>: Automatic failover and retry logic</li></ul>
|
|
<p><strong>Benefits over direct provider integration</strong>:</p>
|
|
<ul><li><strong>Automatic routing</strong>: SDK analyzes query complexity and routes to best model</li><li><strong>Cost tracking</strong>: Built-in cost monitoring and budget limits</li><li><strong>Caching</strong>: Semantic caching reduces redundant inference</li><li><strong>Privacy-first</strong>: Tries local Hanzo Node before cloud providers</li><li><strong>Unified interface</strong>: One API for all providers (local + cloud)</li><li><strong>Resilience</strong>: Automatic retries, circuit breaking, fallbacks</li></ul>
|
|
<p><strong>Comparison: Manual DSPy LM vs Hanzo SDK</strong>:</p>
|
|
<p>| Feature | Manual DSPy LM | Hanzo SDK + DSPy | |---------|----------------|------------------| | Provider setup | Configure each provider separately | One unified client | | Model routing | Manual selection per query | Automatic based on complexity | | Local inference | Requires separate Ollama/vLLM setup | Built-in via Hanzo Node | | Cost tracking | Custom implementation | Built-in | | Caching | Manual implementation | Automatic semantic caching | | Failover | Custom retry logic | Automatic with fallbacks | | Privacy | Separate local/cloud paths | Local-first automatic routing | | Lines of code | 50+ per provider | 10-15 total |</p>
|
|
<p><strong>Example: DSPy Optimizer with Hanzo</strong>:</p>
|
|
<pre><code class="lang-python">import dspy
|
|
from hanzo import Hanzo
|
|
from dspy.teleprompt import BootstrapFewShot
|
|
|
|
# Initialize Hanzo for optimization (use cheaper models)
|
|
hanzo_optimizer = Hanzo(
|
|
inference_mode='hybrid',
|
|
routing_strategy='cost_optimized',
|
|
preferred_models=['llama-3-8b', 'gpt-3.5-turbo']
|
|
)
|
|
|
|
lm_optimizer = HanzoLM(hanzo_optimizer)
|
|
|
|
# Initialize Hanzo for production (use best models)
|
|
hanzo_prod = Hanzo(
|
|
inference_mode='hybrid',
|
|
routing_strategy='quality_focused',
|
|
preferred_models=['gpt-4', 'claude-3-opus']
|
|
)
|
|
|
|
lm_prod = HanzoLM(hanzo_prod)
|
|
|
|
# Define DSPy program
|
|
class QA(dspy.Module):
|
|
def __init__(self):
|
|
super().__init__()
|
|
self.generate_answer = dspy.ChainOfThought("question -> answer")
|
|
|
|
def forward(self, question):
|
|
return self.generate_answer(question=question)
|
|
|
|
# Optimize with cheap models via Hanzo
|
|
dspy.configure(lm=lm_optimizer)
|
|
optimizer = BootstrapFewShot(metric=qa_metric)
|
|
optimized_qa = optimizer.compile(QA(), trainset=train_data)
|
|
|
|
# Deploy with best models via Hanzo
|
|
dspy.configure(lm=lm_prod)
|
|
result = optimized_qa(question="What is quantum computing?")
|
|
|
|
# Hanzo automatically:
|
|
# - Used llama-3-8b for optimization (fast, cheap)
|
|
# - Uses gpt-4 for production (high quality)
|
|
# - Tracked costs for both phases
|
|
# - Cached repeated prompts during optimization</code></pre>
|
|
<p><strong>See Also</strong>:</p>
|
|
<ul><li><strong>python-sdk.md</strong> - Complete Hanzo SDK documentation</li><li><strong>hanzo-node.md</strong> - Local inference setup</li><li><strong>llm-model-routing.md</strong> - Automatic routing strategies</li></ul>
|
|
<h3>Pattern 8: Multi-Model Setup</h3>
|
|
<pre><code class="lang-python">import dspy
|
|
|
|
# Different models for different purposes
|
|
fast_lm = dspy.LM("openai/gpt-4o-mini", max_tokens=500)
|
|
smart_lm = dspy.LM("openai/gpt-4o", max_tokens=2000)
|
|
modal_lm = dspy.LM(
|
|
"openai/meta-llama/Meta-Llama-3.1-8B-Instruct",
|
|
api_base="https://your-app.modal.run/v1",
|
|
api_key="EMPTY",
|
|
)
|
|
|
|
# Set default for most operations
|
|
dspy.configure(lm=fast_lm)
|
|
|
|
# Use smart model for complex reasoning
|
|
class ComplexReasoner(dspy.Module):
|
|
def __init__(self):
|
|
super().__init__()
|
|
# Override with smarter model
|
|
self.predictor = dspy.ChainOfThought("question -> answer", lm=smart_lm)
|
|
|
|
def forward(self, question):
|
|
return self.predictor(question=question)
|
|
|
|
# Use Modal model for high-volume, cost-sensitive tasks
|
|
high_volume_predictor = dspy.Predict("text -> category", lm=modal_lm)</code></pre>
|
|
<p><strong>Benefits</strong>:</p>
|
|
<ul><li>Cost optimization (use expensive models only when needed)</li><li>Performance optimization (fast models for simple tasks)</li><li>Flexibility (switch models without code changes)</li></ul>
|
|
<hr>
|
|
<h2>Quick Reference</h2>
|
|
<h3>Installation Commands</h3>
|
|
<pre><code class="lang-bash"># Standard installation
|
|
pip install dspy-ai
|
|
|
|
# With uv (recommended)
|
|
uv add dspy-ai
|
|
|
|
# With Modal and HuggingFace support
|
|
uv add dspy-ai modal huggingface-hub transformers
|
|
|
|
# Optional dependencies
|
|
uv add openai anthropic cohere litellm chromadb weaviate-client</code></pre>
|
|
<h3>LM Configuration Patterns</h3>
|
|
<p>| Provider | Model String | API Base | Use Case | |----------|--------------|----------|----------| | OpenAI | <code>openai/gpt-4o-mini</code> | Default | Fast, cost-effective | | OpenAI | <code>openai/gpt-4o</code> | Default | Advanced reasoning | | Anthropic | <code>anthropic/claude-3-5-sonnet-20241022</code> | Default | Long context, caching | | Modal | <code>openai/model-name</code> | Custom endpoint | Serverless GPU | | HuggingFace | <code>huggingface/model-name</code> | HF Inference API | Hosted open models | | vLLM | <code>openai/model-name</code> | <code>http://localhost:8000/v1</code> | Self-hosted | | Ollama | <code>ollama/llama3.2</code> | <code>http://localhost:11434</code> | Local, privacy |</p>
|
|
<h3>Modal Deployment Quick Start</h3>
|
|
<pre><code class="lang-bash"># Install Modal
|
|
uv add modal
|
|
|
|
# Set up Modal account
|
|
modal setup
|
|
|
|
# Deploy model endpoint
|
|
modal deploy dspy_modal.py
|
|
|
|
# Get endpoint URL
|
|
modal app list</code></pre>
|
|
<h3>Configuration Checklist</h3>
|
|
<pre><code>✅ DO: Store API keys in environment variables
|
|
✅ DO: Set reasonable max_tokens limits
|
|
✅ DO: Use temperature=0.0 for deterministic outputs
|
|
✅ DO: Configure global LM with dspy.configure()
|
|
✅ DO: Test LM configuration before building modules
|
|
✅ DO: Use Modal volumes for model caching
|
|
✅ DO: Use L40S GPU for cost-effective Modal deployments
|
|
|
|
❌ DON'T: Hard-code API keys in source code
|
|
❌ DON'T: Use very high temperatures (>0.7) for structured tasks
|
|
❌ DON'T: Forget to set max_tokens (can cause unexpected costs)
|
|
❌ DON'T: Skip error handling for API calls
|
|
❌ DON'T: Leave Modal dev apps running (use modal app stop)</code></pre>
|
|
<hr>
|
|
<h2>Anti-Patterns</h2>
|
|
<p>❌ <strong>Hard-coded API keys</strong>: Security risk, not shareable</p>
|
|
<pre><code class="lang-python"># Bad
|
|
lm = dspy.LM("openai/gpt-4o-mini", api_key="sk-1234567890")</code></pre>
|
|
<p>✅ Use environment variables:</p>
|
|
<pre><code class="lang-python"># Good
|
|
import os
|
|
lm = dspy.LM("openai/gpt-4o-mini", api_key=os.getenv("OPENAI_API_KEY"))</code></pre>
|
|
<p>❌ <strong>No max_tokens limit</strong>: Unexpected costs, slow responses</p>
|
|
<pre><code class="lang-python"># Bad
|
|
lm = dspy.LM("openai/gpt-4o") # No limit</code></pre>
|
|
<p>✅ Set reasonable limits:</p>
|
|
<pre><code class="lang-python"># Good
|
|
lm = dspy.LM("openai/gpt-4o", max_tokens=2000)</code></pre>
|
|
<p>❌ <strong>Forgetting dspy.configure()</strong>: Modules won't know which LM to use</p>
|
|
<pre><code class="lang-python"># Bad
|
|
lm = dspy.LM("openai/gpt-4o-mini")
|
|
predictor = dspy.Predict("text -> label") # No LM configured!</code></pre>
|
|
<p>✅ Configure globally or per-module:</p>
|
|
<pre><code class="lang-python"># Good
|
|
lm = dspy.LM("openai/gpt-4o-mini")
|
|
dspy.configure(lm=lm)
|
|
predictor = dspy.Predict("text -> label") # Uses configured LM</code></pre>
|
|
<p>❌ <strong>Not using Modal volumes for model caching</strong>: Slow cold starts</p>
|
|
<pre><code class="lang-python"># Bad - Downloads model on every cold start
|
|
@app.function(image=image, gpu="L40S")
|
|
def serve():
|
|
llm = LLM(model="meta-llama/Meta-Llama-3.1-70B") # Re-downloads every time</code></pre>
|
|
<p>✅ Use Modal volumes:</p>
|
|
<pre><code class="lang-python"># Good - Cache model in volume
|
|
@app.function(
|
|
image=image,
|
|
gpu="L40S",
|
|
volumes={"/models": modal.Volume.from_name("hf-models")},
|
|
)
|
|
def serve():
|
|
llm = LLM(model="meta-llama/Meta-Llama-3.1-70B", download_dir="/models")</code></pre>
|
|
<hr>
|
|
<h2>Related Skills</h2>
|
|
<ul><li><code>dspy-signatures.md</code> - Next step: Define input/output signatures</li><li><code>dspy-modules.md</code> - Building composable prediction modules</li><li><code>dspy-optimizers.md</code> - Optimizing prompts with teleprompters</li><li><code>modal-functions-basics.md</code> - Modal deployment fundamentals</li><li><code>modal-gpu-workloads.md</code> - GPU selection and optimization for Modal</li><li><code>modal-web-endpoints.md</code> - Creating Modal web endpoints for inference</li><li><code>llm-dataset-preparation.md</code> - Preparing training data for optimization</li></ul>
|
|
<hr>
|
|
<p><strong>Last Updated</strong>: 2025-10-25 <strong>Format Version</strong>: 1.0 (Atomic)</p>
|
|
<footer>
|
|
<div class="fmap">
|
|
<div><b>Build</b>
|
|
<a href="https://hanzo.app">hanzo.app</a> app builder
|
|
<a href="https://hanzo.codes">hanzo.codes</a> code with an agent
|
|
<a href="https://hanzoskills.com">hanzoskills.com</a> what an agent reads</div>
|
|
<div><b>Run</b>
|
|
<a href="https://console.hanzo.ai">console.hanzo.ai</a> cloud console
|
|
<a href="https://api.hanzo.ai">api.hanzo.ai</a> the API</div>
|
|
<div><b>Account</b>
|
|
<a href="https://hanzo.id">hanzo.id</a> sign in
|
|
<a href="https://pay.hanzo.ai">pay.hanzo.ai</a> top up
|
|
<a href="https://billing.hanzo.ai">billing.hanzo.ai</a> invoices and usage</div>
|
|
<div><b>Company</b>
|
|
<a href="https://hanzo.ai">hanzo.ai</a> canonical
|
|
<a href="https://hanzo.ventures">hanzo.ventures</a> ventures</div>
|
|
</div>
|
|
<div class="fend"><a href="/dspy-setup.md">This page as plain text</a> · <a href="/llms.txt">Index for agents</a></div>
|
|
</footer>
|
|
</div></body></html> |