mirror of
https://github.com/hanzoai/skills.git
synced 2026-11-19 00:47:36 +00:00
hanzoskills.com/llms.txt now opens with the install line, the API reference index and the company index, so an agent that lands on any one of the three can find the rest. robots.txt claimed 537 documents while llms.txt counts 920. The number is gone from robots.txt; llms.txt derives it. public/ is regenerated from scripts/site.py, which it had fallen behind: the copy-prompt menu and agent bar were in the generator and not in the committed pages. tests/site.mjs passes against the result. Co-authored-by: Hanzo Dev <[email protected]>
985 lines
44 KiB
HTML
985 lines
44 KiB
HTML
<!doctype html>
|
|
<html lang="en"><head>
|
|
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
<title>ml-llm-evaluation-frameworks</title>
|
|
<meta name="color-scheme" content="dark">
|
|
<style>
|
|
/* Committed to one visual world: a dark instrument panel. No light variant —
|
|
the dot field and the hairline rules only read on black, and a naive
|
|
inversion would wash both out. Every colour is painted explicitly so the page
|
|
never borrows a host background. */
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Pixel Square';src:url('/fonts/ZenPixel-Square.woff2') format('woff2');font-style:normal;font-display:swap}
|
|
|
|
:root{
|
|
--ink:#f4f4f2; --soft:#9c9c98; --faint:#6a6a66; --ground:#000; --panel:#0a0a0a;
|
|
--line:#232323; --line-hi:#3a3a3a; --accent:#f4f4f2;
|
|
--font-zen-sans:'Zen',-apple-system,BlinkMacSystemFont,"Segoe UI",Helvetica,Arial,sans-serif;
|
|
--font-zen-mono:'Zen Mono',ui-monospace,"SF Mono",SFMono-Regular,Menlo,Consolas,monospace;
|
|
--font-zen-pixel-square:'Zen Pixel Square',monospace;
|
|
--mono:var(--font-zen-mono);
|
|
--sans:var(--font-zen-sans);
|
|
/* Controls are pills and surfaces are soft, as on hanzo.ai. */
|
|
--r:999px; --r-soft:14px;
|
|
}
|
|
*{box-sizing:border-box}
|
|
html{-webkit-text-size-adjust:100%}
|
|
body{margin:0;background:var(--ground);color:var(--ink);font-family:var(--sans);
|
|
font-size:16px;line-height:1.65;-webkit-font-smoothing:antialiased}
|
|
.wrap{max-width:60rem;margin:0 auto;padding:0 1.75rem}
|
|
|
|
/* The dot field. A halftone made of one repeating radial-gradient, faded out
|
|
along the diagonal by a mask so it reads as a wave rather than wallpaper. */
|
|
/* No overflow clip here: the dot field is inset to the box already, and a clip
|
|
would cut off the copy-prompt menu. */
|
|
.hero{position:relative;border-bottom:1px solid var(--line)}
|
|
.hero::before{content:"";position:absolute;inset:0;
|
|
background-image:radial-gradient(var(--line-hi) 1px, transparent 1px);
|
|
background-size:7px 7px;
|
|
-webkit-mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
pointer-events:none}
|
|
.hero .wrap{position:relative;padding-top:5.5rem;padding-bottom:4rem}
|
|
|
|
nav{border-bottom:1px solid var(--line)}
|
|
nav .in{max-width:60rem;margin:0 auto;padding:1.1rem 1.75rem;display:flex;
|
|
justify-content:space-between;align-items:center;gap:1rem}
|
|
.brand{font-family:var(--mono);font-size:.85rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--ink);text-decoration:none;
|
|
display:inline-flex;align-items:center;gap:.6rem}
|
|
.brand i{font-style:normal;color:var(--faint)}
|
|
.mark{width:1.15em;height:1.15em;display:block;flex:0 0 auto}
|
|
nav a.up{font-family:var(--mono);font-size:.72rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--soft);text-decoration:none}
|
|
nav a.up:hover{color:var(--ink)}
|
|
|
|
h1{font-family:var(--mono);font-size:clamp(2rem,5.4vw,3.4rem);line-height:1.12;
|
|
letter-spacing:-.02em;margin:0 0 1.1rem;font-weight:500;text-wrap:balance}
|
|
h1 em{font-style:normal;color:var(--faint)}
|
|
.lede{color:var(--soft);font-size:1.05rem;max-width:52ch;margin:0 0 2.25rem}
|
|
|
|
.btns{display:flex;flex-wrap:wrap;gap:.7rem;margin:0 0 .5rem;align-items:center}
|
|
.btn{display:inline-flex;align-items:center;gap:.5rem;font-family:var(--mono);
|
|
font-size:.72rem;letter-spacing:.14em;text-transform:uppercase;
|
|
padding:.8rem 1.4rem;border:1px solid var(--line-hi);border-radius:var(--r);
|
|
color:var(--ink);text-decoration:none;background:transparent;cursor:pointer;
|
|
line-height:1.2;font-weight:400}
|
|
.btn:hover{background:#141414;text-decoration:none;border-color:var(--soft)}
|
|
.btn.solid{background:var(--ink);color:#000;border-color:var(--ink)}
|
|
.btn.solid:hover{background:#d8d8d4}
|
|
.btn .chev{width:.7em;height:.7em;flex:0 0 auto;transition:transform .15s ease}
|
|
.btn[aria-expanded="true"] .chev{transform:rotate(180deg)}
|
|
|
|
/* Copy-prompt selector. Each row copies the prompt for its own target. */
|
|
.copy{position:relative;display:inline-flex}
|
|
.menu{position:absolute;top:calc(100% + .5rem);left:0;z-index:20;min-width:21rem;
|
|
background:var(--panel);border:1px solid var(--line-hi);border-radius:var(--r-soft);
|
|
padding:.4rem;display:flex;flex-direction:column;
|
|
box-shadow:0 12px 32px rgba(0,0,0,.75)}
|
|
.menu[hidden]{display:none}
|
|
.mi{display:flex;align-items:center;gap:.7rem;width:100%;text-align:left;
|
|
padding:.62rem .8rem;border:0;border-radius:calc(var(--r-soft) - .35rem);
|
|
background:transparent;color:var(--ink);font-family:var(--mono);font-size:.76rem;
|
|
letter-spacing:.04em;text-decoration:none;cursor:pointer;line-height:1.3;
|
|
white-space:nowrap}
|
|
.mi:hover,.mi:focus-visible{background:#181818;outline:none}
|
|
.mi .mark{width:1.05rem;height:1.05rem;color:var(--soft)}
|
|
.mi:hover .mark{color:var(--ink)}
|
|
.mi u{margin-left:auto;color:var(--faint);text-decoration:none;font-size:.68rem;
|
|
letter-spacing:.1em;text-transform:uppercase}
|
|
|
|
/* Works-with bar: one scrolling row, name and mark per agent. */
|
|
.strip{border-bottom:1px solid var(--line);background:var(--panel)}
|
|
.strip .in{max-width:60rem;margin:0 auto;padding:.9rem 1.75rem;display:flex;
|
|
align-items:center;gap:1.25rem;font-family:var(--mono);font-size:.72rem;
|
|
letter-spacing:.1em;text-transform:uppercase;color:var(--faint)}
|
|
.strip b{color:var(--soft);font-weight:400;flex:0 0 auto;
|
|
border-right:1px solid var(--line);padding-right:1.25rem;align-self:stretch;
|
|
display:flex;align-items:center}
|
|
.bar{display:flex;align-items:center;gap:.5rem;flex:1 1 auto;min-width:0;
|
|
overflow-x:auto;scrollbar-width:none;scroll-snap-type:x proximity;
|
|
-webkit-overflow-scrolling:touch;
|
|
-webkit-mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent);
|
|
mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent)}
|
|
.bar::-webkit-scrollbar{display:none}
|
|
.agent{display:inline-flex;align-items:center;gap:.45rem;flex:0 0 auto;
|
|
padding:.4rem .85rem;border:1px solid var(--line);border-radius:var(--r);
|
|
color:var(--soft);white-space:nowrap;scroll-snap-align:start;
|
|
text-decoration:none;background:var(--ground)}
|
|
.agent:hover{color:var(--ink);border-color:var(--line-hi);text-decoration:none}
|
|
.agent .mark{width:.95rem;height:.95rem}
|
|
|
|
h2{font-family:var(--mono);font-size:1.1rem;letter-spacing:-.01em;margin:2.75rem 0 .75rem;font-weight:500}
|
|
h3{font-size:1rem;margin:2rem 0 .5rem;font-weight:600}
|
|
h4,h5,h6{font-size:.94rem;margin:1.5rem 0 .5rem;font-weight:600}
|
|
p{margin:0 0 1rem;max-width:70ch}
|
|
ul,ol{margin:0 0 1rem;padding-left:1.35rem;max-width:70ch}
|
|
li{margin:.22rem 0}
|
|
a{color:var(--ink);text-decoration:underline;text-underline-offset:3px;
|
|
text-decoration-color:var(--line-hi)}
|
|
a:hover{text-decoration-color:var(--ink)}
|
|
a:focus-visible{outline:1px solid var(--ink);outline-offset:3px}
|
|
code{font-family:var(--mono);font-size:.85em;background:var(--panel);
|
|
border:1px solid var(--line);border-radius:5px;padding:.1em .36em}
|
|
pre{background:var(--panel);border:1px solid var(--line);padding:1rem 1.1rem;
|
|
border-radius:var(--r-soft);overflow-x:auto;margin:0 0 1.25rem}
|
|
pre code{background:none;border:0;padding:0;font-size:.82rem;line-height:1.7}
|
|
blockquote{margin:0 0 1rem;padding-left:1rem;border-left:1px solid var(--line-hi);color:var(--soft)}
|
|
hr{border:0;border-top:1px solid var(--line);margin:2.5rem 0}
|
|
table{border-collapse:collapse;width:100%;margin:0 0 1.25rem;font-size:.92rem;
|
|
display:block;overflow-x:auto}
|
|
th,td{border:1px solid var(--line);padding:.5rem .7rem;text-align:left}
|
|
|
|
.curl{display:block;background:var(--panel);border:1px solid var(--line);
|
|
border-radius:var(--r-soft);padding:.95rem 1.1rem;font-family:var(--mono);
|
|
font-size:.8rem;color:var(--soft);overflow-x:auto;margin:0 0 2.5rem;
|
|
white-space:pre;line-height:1.9}
|
|
.curl b{color:var(--ink);font-weight:400}
|
|
.curl i{font-style:normal;color:var(--faint)}
|
|
|
|
/* index */
|
|
#q{width:100%;padding:.85rem 1.2rem;font-family:var(--mono);font-size:.85rem;color:var(--ink);
|
|
background:var(--panel);border:1px solid var(--line);border-radius:var(--r);margin:0 0 .6rem}
|
|
#q::placeholder{color:var(--faint);text-transform:uppercase;letter-spacing:.1em;font-size:.72rem}
|
|
#q:focus{outline:none;border-color:var(--soft)}
|
|
#count{color:var(--faint);font-family:var(--mono);font-size:.72rem;letter-spacing:.1em;
|
|
text-transform:uppercase;margin:0 0 2rem}
|
|
.cat{margin:2.5rem 0 .5rem;font-family:var(--mono);font-size:.7rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);border-bottom:1px solid var(--line);
|
|
padding-bottom:.45rem}
|
|
.row{display:block;padding:.6rem .9rem;margin:0 -.9rem;border-bottom:1px solid var(--line);
|
|
border-radius:var(--r-soft);text-decoration:none}
|
|
.row:hover{background:#0a0a0a}
|
|
.nm{font-family:var(--mono);font-size:.84rem;color:var(--ink);display:block}
|
|
.ds{color:var(--soft);font-size:.9rem;display:block;margin-top:.12rem}
|
|
footer{margin-top:4rem;padding:2rem 0 4rem;border-top:1px solid var(--line);
|
|
color:var(--faint);font-size:.85rem}
|
|
footer a{color:var(--soft)}
|
|
.fmap{display:grid;gap:1.75rem;grid-template-columns:repeat(auto-fit,minmax(11rem,1fr));
|
|
margin:0 0 2rem;font-size:.85rem}
|
|
.fmap b{display:block;font-family:var(--mono);font-size:.68rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);font-weight:400;margin:0 0 .6rem;
|
|
border-bottom:1px solid var(--line);padding-bottom:.4rem}
|
|
.fmap a{display:inline-block;color:var(--ink);text-decoration:none;margin:.3rem .35rem 0 0}
|
|
.fmap a:hover{text-decoration:underline}
|
|
.fmap div{color:var(--faint);line-height:1.5}
|
|
.fend{border-top:1px solid var(--line);padding-top:1.25rem;font-size:.82rem}
|
|
|
|
@media (max-width:640px){
|
|
.hero .wrap{padding-top:3.25rem;padding-bottom:2.75rem}
|
|
.btns{gap:.5rem}
|
|
.btn{padding:.75rem 1.15rem;font-size:.68rem}
|
|
.copy,.copy .btn{width:100%;justify-content:center}
|
|
.menu{left:0;right:0;min-width:0}
|
|
.strip .in{flex-direction:column;align-items:flex-start;gap:.6rem;padding:.9rem 1.75rem}
|
|
.strip b{border-right:0;padding-right:0}
|
|
.bar{width:100%}
|
|
}
|
|
</style>
|
|
<link rel="alternate" type="text/markdown" href="/ml-llm-evaluation-frameworks.md">
|
|
|
|
<link rel="icon" href="/favicon.svg">
|
|
<link rel="canonical" href="https://hanzoskills.com/ml-llm-evaluation-frameworks">
|
|
</head>
|
|
<!-- Reading this with a machine? Every page here is also plain markdown.
|
|
This one: https://hanzoskills.com/ml-llm-evaluation-frameworks.md
|
|
Index: https://hanzoskills.com/llms.txt
|
|
Corpus: https://hanzoskills.com/all.md
|
|
Append .md to any path for the markdown twin. -->
|
|
<body>
|
|
<nav><div class="in">
|
|
<a class="brand" href="/"><svg class="mark" viewBox="0 0 67 67" fill="currentColor" aria-hidden="true"><path d="M22.21 67V44.6369H0V67H22.21Z"/><path d="M66.7038 22.3184H22.2534L0.0878906 44.6367H44.4634L66.7038 22.3184Z"/><path d="M22.21 0H0V22.3184H22.21V0Z"/><path d="M66.7198 0H44.5098V22.3184H66.7198V0Z"/><path d="M66.7198 67V44.6369H44.5098V67H66.7198Z"/></svg> HANZO<i>SKILLS</i></a>
|
|
<a class="up" href="/">All skills</a>
|
|
</div></nav>
|
|
|
|
<div class="wrap">
|
|
<h1>ml-llm-evaluation-frameworks</h1><p class="lede">Production-grade LLM evaluation using Arize Phoenix (OpenTelemetry tracing, self-hosted evals), Braintrust (86x faster search), LangSmith, and Langfuse with cost tracking and observability integration</p><h1>LLM Evaluation Frameworks</h1>
|
|
<p>Last Updated: 2025-10-26</p>
|
|
<h2>When to Use This Skill</h2>
|
|
<p>Use LLM evaluation frameworks when:</p>
|
|
<ul><li><strong>Production monitoring</strong>: Tracking LLM performance in deployed applications</li><li><strong>Continuous evaluation</strong>: Running automated evals on every model or prompt change</li><li><strong>Cost optimization</strong>: Monitoring token usage and inference costs across projects</li><li><strong>Tracing and debugging</strong>: Understanding LLM behavior through distributed tracing</li><li><strong>A/B testing</strong>: Comparing prompt variations or model versions</li><li><strong>Dataset management</strong>: Organizing test cases and golden datasets</li><li><strong>Team collaboration</strong>: Sharing evaluation results and datasets across teams</li><li><strong>Compliance</strong>: Auditing LLM outputs for safety, bias, or regulatory requirements</li></ul>
|
|
<p><strong>Anti-pattern</strong>: Using only offline benchmarks without production evaluation. Always monitor real-world performance.</p>
|
|
<h2>Core Concepts</h2>
|
|
<h3>Framework Comparison (2024-2025)</h3>
|
|
<p>| Feature | Arize Phoenix | Braintrust | LangSmith | Langfuse | |---------|---------------|------------|-----------|----------| | <strong>Self-Hosted</strong> | ✅ Yes | ❌ Cloud only | ❌ Cloud only | ✅ Yes | | <strong>Open Source</strong> | ✅ Full | ⚠️ Partial | ❌ No | ✅ Full | | <strong>Tracing Protocol</strong> | OpenTelemetry | Proprietary | Proprietary | OpenInference | | <strong>Framework Support</strong> | All (agnostic) | All | LangChain focus | All | | <strong>Search Speed</strong> | Fast | 86x faster | Standard | Fast | | <strong>Built-in Evals</strong> | ✅ Extensive | ✅ Yes | ✅ Yes | ⚠️ Limited | | <strong>Cost Tracking</strong> | ✅ Yes | ✅ Yes | ✅ Yes | ✅ Yes | | <strong>Pricing</strong> | Free (OSS) | Paid tiers | Paid tiers | Free tier + paid |</p>
|
|
<h3>Arize Phoenix: Deep Dive</h3>
|
|
<p><strong>Why Phoenix?</strong></p>
|
|
<ul><li><strong>Framework-agnostic</strong>: Works with any LLM framework (LangChain, LlamaIndex, DSPy, raw OpenAI)</li><li><strong>OpenTelemetry standard</strong>: Industry-standard tracing protocol</li><li><strong>Self-hostable</strong>: Full control over data privacy and infrastructure</li><li><strong>Production-ready evals</strong>: Built-in library of evaluation metrics</li><li><strong>Real-time monitoring</strong>: Live dashboards for production systems</li></ul>
|
|
<p><strong>Architecture</strong>:</p>
|
|
<pre><code>Your Application
|
|
↓
|
|
OpenTelemetry Instrumentation
|
|
↓
|
|
Phoenix Collector (OTLP endpoint)
|
|
↓
|
|
Phoenix UI (React dashboard)</code></pre>
|
|
<p><strong>Key Components</strong>:</p>
|
|
<ol><li><strong>Tracing</strong>: OpenTelemetry-based distributed tracing for LLM calls</li><li><strong>Evaluations</strong>: Pre-built and custom eval functions</li><li><strong>Datasets</strong>: Test case management and versioning</li><li><strong>Experiments</strong>: Compare models, prompts, or configurations</li><li><strong>Embeddings</strong>: Visualize and cluster embeddings for debugging</li></ol>
|
|
<h3>Braintrust</h3>
|
|
<p><strong>Strengths</strong>:</p>
|
|
<ul><li><strong>Search performance</strong>: 86x faster search compared to competitors (2024 benchmark)</li><li><strong>API-first design</strong>: Programmatic access to all features</li><li><strong>Prompt playground</strong>: Interactive prompt engineering with version control</li><li><strong>Scoring functions</strong>: Flexible evaluation metric system</li></ul>
|
|
<p><strong>Best for</strong>: Teams prioritizing search performance and API-driven workflows</p>
|
|
<h3>LangSmith</h3>
|
|
<p><strong>Strengths</strong>:</p>
|
|
<ul><li><strong>LangChain integration</strong>: Native support for LangChain applications</li><li><strong>Human feedback</strong>: Built-in annotation and feedback collection</li><li><strong>Dataset sharing</strong>: Public datasets and community benchmarks</li></ul>
|
|
<p><strong>Best for</strong>: LangChain-heavy applications with human-in-the-loop workflows</p>
|
|
<h3>Langfuse</h3>
|
|
<p><strong>Strengths</strong>:</p>
|
|
<ul><li><strong>Open source + cloud</strong>: Self-host or use managed service</li><li><strong>Cost analytics</strong>: Detailed token usage and cost breakdowns</li><li><strong>Prompt management</strong>: Version control for prompts with A/B testing</li></ul>
|
|
<p><strong>Best for</strong>: Cost-conscious teams wanting open-source flexibility</p>
|
|
<h2>Implementation Patterns</h2>
|
|
<h3>Pattern 1: Arize Phoenix Setup and Tracing</h3>
|
|
<p><strong>When to use</strong>: Starting with Phoenix for framework-agnostic evaluation</p>
|
|
<pre><code class="lang-bash"># Installation
|
|
pip install arize-phoenix openinference-instrumentation-openai
|
|
|
|
# Start Phoenix server (local)
|
|
python -m phoenix.server.main serve
|
|
|
|
# Or use Docker
|
|
docker run -p 6006:6006 arizephoenix/phoenix:latest</code></pre>
|
|
<p><strong>Basic OpenAI tracing</strong>:</p>
|
|
<pre><code class="lang-python">from phoenix.otel import register
|
|
from openinference.instrumentation.openai import OpenAIInstrumentor
|
|
from openai import OpenAI
|
|
|
|
# Configure Phoenix endpoint
|
|
tracer_provider = register(
|
|
project_name="my-llm-app",
|
|
endpoint="http://localhost:6006/v1/traces",
|
|
)
|
|
|
|
# Auto-instrument OpenAI
|
|
OpenAIInstrumentor().instrument(tracer_provider=tracer_provider)
|
|
|
|
# Use OpenAI normally - traces sent automatically
|
|
client = OpenAI()
|
|
response = client.chat.completions.create(
|
|
model="gpt-4-turbo-preview",
|
|
messages=[
|
|
{"role": "system", "content": "You are a helpful assistant."},
|
|
{"role": "user", "content": "What is the capital of France?"},
|
|
],
|
|
)
|
|
|
|
print(response.choices[0].message.content)
|
|
# Phoenix UI now shows this trace at http://localhost:6006</code></pre>
|
|
<p><strong>LangChain tracing</strong>:</p>
|
|
<pre><code class="lang-python">from phoenix.otel import register
|
|
from openinference.instrumentation.langchain import LangChainInstrumentor
|
|
from langchain_openai import ChatOpenAI
|
|
from langchain.prompts import ChatPromptTemplate
|
|
from langchain.schema.output_parser import StrOutputParser
|
|
|
|
# Setup Phoenix
|
|
tracer_provider = register(project_name="langchain-app")
|
|
|
|
# Auto-instrument LangChain
|
|
LangChainInstrumentor().instrument(tracer_provider=tracer_provider)
|
|
|
|
# Build LangChain pipeline
|
|
prompt = ChatPromptTemplate.from_messages([
|
|
("system", "You are a helpful assistant."),
|
|
("user", "{input}"),
|
|
])
|
|
|
|
llm = ChatOpenAI(model="gpt-4-turbo-preview")
|
|
chain = prompt | llm | StrOutputParser()
|
|
|
|
# Run - automatically traced
|
|
result = chain.invoke({"input": "What is the capital of France?"})
|
|
print(result)</code></pre>
|
|
<p><strong>LlamaIndex tracing</strong>:</p>
|
|
<pre><code class="lang-python">from phoenix.otel import register
|
|
from openinference.instrumentation.llama_index import LlamaIndexInstrumentor
|
|
from llama_index.core import VectorStoreIndex, SimpleDirectoryReader
|
|
|
|
# Setup Phoenix
|
|
tracer_provider = register(project_name="llamaindex-app")
|
|
|
|
# Auto-instrument LlamaIndex
|
|
LlamaIndexInstrumentor().instrument(tracer_provider=tracer_provider)
|
|
|
|
# Build RAG system
|
|
documents = SimpleDirectoryReader("data").load_data()
|
|
index = VectorStoreIndex.from_documents(documents)
|
|
|
|
# Query - automatically traced
|
|
query_engine = index.as_query_engine()
|
|
response = query_engine.query("What is the main topic?")
|
|
print(response)</code></pre>
|
|
<h3>Pattern 2: Phoenix Evaluations</h3>
|
|
<p><strong>When to use</strong>: Running automated evaluations on traces</p>
|
|
<pre><code class="lang-python">import phoenix as px
|
|
from phoenix.evals import (
|
|
HallucinationEvaluator,
|
|
RelevanceEvaluator,
|
|
ToxicityEvaluator,
|
|
OpenAIModel,
|
|
)
|
|
|
|
# Launch Phoenix and get session
|
|
session = px.launch_app()
|
|
|
|
# Get traces from Phoenix
|
|
traces_df = px.Client().get_trace_dataset()
|
|
|
|
# Configure evaluator model
|
|
eval_model = OpenAIModel(model="gpt-4-turbo-preview")
|
|
|
|
# Run hallucination evaluation
|
|
hallucination_eval = HallucinationEvaluator(eval_model)
|
|
hallucination_results = hallucination_eval.evaluate(
|
|
dataframe=traces_df,
|
|
query_column="input",
|
|
response_column="output",
|
|
reference_column="retrieved_context",
|
|
)
|
|
|
|
# Run relevance evaluation
|
|
relevance_eval = RelevanceEvaluator(eval_model)
|
|
relevance_results = relevance_eval.evaluate(
|
|
dataframe=traces_df,
|
|
query_column="input",
|
|
document_column="retrieved_context",
|
|
)
|
|
|
|
# Run toxicity evaluation
|
|
toxicity_eval = ToxicityEvaluator(eval_model)
|
|
toxicity_results = toxicity_eval.evaluate(
|
|
dataframe=traces_df,
|
|
text_column="output",
|
|
)
|
|
|
|
# Combine results
|
|
traces_df["hallucination_score"] = hallucination_results["label"]
|
|
traces_df["relevance_score"] = relevance_results["label"]
|
|
traces_df["toxicity_score"] = toxicity_results["label"]
|
|
|
|
# Upload results back to Phoenix
|
|
px.Client().log_evaluations(traces_df)
|
|
|
|
# View in Phoenix UI
|
|
print(f"View results at {session.url}")</code></pre>
|
|
<p><strong>Custom Phoenix evaluators</strong>:</p>
|
|
<pre><code class="lang-python">from phoenix.evals import LLMEvaluator, OpenAIModel
|
|
from phoenix.evals.models import EvalCriteria
|
|
|
|
# Define custom evaluation criteria
|
|
code_quality_template = """
|
|
You are evaluating code quality. Rate the following code on:
|
|
1. Correctness
|
|
2. Efficiency
|
|
3. Readability
|
|
|
|
Code:
|
|
{code}
|
|
|
|
Provide a score from 1-5 where:
|
|
1 = Poor
|
|
3 = Acceptable
|
|
5 = Excellent
|
|
"""
|
|
|
|
# Create custom evaluator
|
|
code_quality_eval = LLMEvaluator(
|
|
model=OpenAIModel(model="gpt-4-turbo-preview"),
|
|
template=code_quality_template,
|
|
rails=["1", "2", "3", "4", "5"], # Valid outputs
|
|
)
|
|
|
|
# Run evaluation
|
|
code_df = traces_df[traces_df["task_type"] == "code_generation"]
|
|
code_quality_results = code_quality_eval.evaluate(
|
|
dataframe=code_df,
|
|
code_column="output",
|
|
)
|
|
|
|
# Analyze results
|
|
print(f"Average code quality: {code_quality_results['score'].mean():.2f}")
|
|
print(f"Low quality outputs: {(code_quality_results['score'] <= 2).sum()}")</code></pre>
|
|
<h3>Pattern 3: Phoenix Datasets and Experiments</h3>
|
|
<p><strong>When to use</strong>: Systematic evaluation across model versions or prompts</p>
|
|
<pre><code class="lang-python">import phoenix as px
|
|
from phoenix.experiments import run_experiment
|
|
from openai import OpenAI
|
|
|
|
# Create dataset
|
|
dataset = px.Client().create_dataset(
|
|
dataset_name="customer-support-qa",
|
|
description="Customer support question answering",
|
|
)
|
|
|
|
# Add examples
|
|
examples = [
|
|
{
|
|
"input": "How do I reset my password?",
|
|
"expected_output": "Visit the account settings page and click 'Reset Password'.",
|
|
},
|
|
{
|
|
"input": "What's your refund policy?",
|
|
"expected_output": "We offer 30-day money-back guarantee on all products.",
|
|
},
|
|
# ... more examples
|
|
]
|
|
|
|
for example in examples:
|
|
dataset.add_example(**example)
|
|
|
|
# Define task function
|
|
def qa_task(input_text: str, model: str = "gpt-4-turbo-preview") -> str:
|
|
"""Generate answer using specified model."""
|
|
client = OpenAI()
|
|
response = client.chat.completions.create(
|
|
model=model,
|
|
messages=[
|
|
{"role": "system", "content": "You are a customer support assistant."},
|
|
{"role": "user", "content": input_text},
|
|
],
|
|
)
|
|
return response.choices[0].message.content
|
|
|
|
# Run experiment with GPT-4
|
|
experiment_gpt4 = run_experiment(
|
|
dataset=dataset,
|
|
task=lambda example: qa_task(example["input"], model="gpt-4-turbo-preview"),
|
|
experiment_name="gpt4-customer-support",
|
|
)
|
|
|
|
# Run experiment with GPT-3.5
|
|
experiment_gpt35 = run_experiment(
|
|
dataset=dataset,
|
|
task=lambda example: qa_task(example["input"], model="gpt-3.5-turbo"),
|
|
experiment_name="gpt35-customer-support",
|
|
)
|
|
|
|
# Compare results
|
|
from phoenix.evals import RelevanceEvaluator
|
|
|
|
eval_model = OpenAIModel(model="gpt-4-turbo-preview")
|
|
relevance_eval = RelevanceEvaluator(eval_model)
|
|
|
|
# Evaluate both experiments
|
|
gpt4_scores = relevance_eval.evaluate(experiment_gpt4.results)
|
|
gpt35_scores = relevance_eval.evaluate(experiment_gpt35.results)
|
|
|
|
print(f"GPT-4 avg relevance: {gpt4_scores['score'].mean():.2f}")
|
|
print(f"GPT-3.5 avg relevance: {gpt35_scores['score'].mean():.2f}")
|
|
|
|
# View detailed comparison in Phoenix UI</code></pre>
|
|
<h3>Pattern 4: Braintrust Integration</h3>
|
|
<p><strong>When to use</strong>: Leveraging fast search and prompt playground</p>
|
|
<pre><code class="lang-python">import braintrust
|
|
from braintrust import init_logger
|
|
|
|
# Initialize Braintrust
|
|
project = braintrust.init(
|
|
project="customer-support-qa",
|
|
api_key="your-api-key",
|
|
)
|
|
|
|
# Log experiment
|
|
experiment = project.experiment(
|
|
name="gpt4-vs-gpt35",
|
|
dataset="customer-support-v1",
|
|
)
|
|
|
|
# Define evaluation function
|
|
def evaluate_response(input_text, expected, actual):
|
|
"""Custom scoring function."""
|
|
# Exact match
|
|
exact_match = 1.0 if actual.strip() == expected.strip() else 0.0
|
|
|
|
# Use Braintrust's built-in scorers
|
|
from braintrust.scorer import Factuality
|
|
|
|
factuality = Factuality()
|
|
fact_score = factuality(output=actual, expected=expected)
|
|
|
|
return {
|
|
"exact_match": exact_match,
|
|
"factuality": fact_score,
|
|
}
|
|
|
|
# Run evaluation
|
|
for example in dataset:
|
|
# Generate response
|
|
response = qa_task(example["input"])
|
|
|
|
# Log to Braintrust
|
|
experiment.log(
|
|
input=example["input"],
|
|
output=response,
|
|
expected=example["expected_output"],
|
|
scores=evaluate_response(
|
|
example["input"],
|
|
example["expected_output"],
|
|
response,
|
|
),
|
|
metadata={
|
|
"model": "gpt-4-turbo-preview",
|
|
"timestamp": datetime.now().isoformat(),
|
|
},
|
|
)
|
|
|
|
# Get experiment summary
|
|
summary = experiment.summarize()
|
|
print(f"Average factuality: {summary['factuality']:.2f}")
|
|
print(f"Exact match rate: {summary['exact_match']:.2f}")
|
|
|
|
# Fast search (Braintrust's 86x advantage)
|
|
search_results = project.search(
|
|
query="password reset",
|
|
limit=10,
|
|
filter={"scores.factuality": {"$gte": 0.8}},
|
|
)
|
|
|
|
for result in search_results:
|
|
print(f"Input: {result['input']}")
|
|
print(f"Output: {result['output']}")
|
|
print(f"Factuality: {result['scores']['factuality']}")</code></pre>
|
|
<h3>Pattern 5: Cost Tracking Across Frameworks</h3>
|
|
<p><strong>When to use</strong>: Monitoring and optimizing LLM costs</p>
|
|
<pre><code class="lang-python">import phoenix as px
|
|
from openai import OpenAI
|
|
from datetime import datetime, timedelta
|
|
|
|
# Setup Phoenix with cost tracking
|
|
tracer_provider = px.otel.register(
|
|
project_name="cost-tracking",
|
|
endpoint="http://localhost:6006/v1/traces",
|
|
)
|
|
|
|
# Track costs in spans
|
|
from opentelemetry import trace
|
|
|
|
def track_llm_call(model: str, input_tokens: int, output_tokens: int):
|
|
"""Add cost metadata to current span."""
|
|
# Pricing as of 2025 (per 1M tokens)
|
|
PRICING = {
|
|
"gpt-4-turbo-preview": {"input": 10.00, "output": 30.00},
|
|
"gpt-3.5-turbo": {"input": 0.50, "output": 1.50},
|
|
"claude-3-opus": {"input": 15.00, "output": 75.00},
|
|
"claude-3-sonnet": {"input": 3.00, "output": 15.00},
|
|
}
|
|
|
|
pricing = PRICING.get(model, {"input": 0, "output": 0})
|
|
|
|
input_cost = (input_tokens / 1_000_000) * pricing["input"]
|
|
output_cost = (output_tokens / 1_000_000) * pricing["output"]
|
|
total_cost = input_cost + output_cost
|
|
|
|
# Add to current span
|
|
span = trace.get_current_span()
|
|
span.set_attribute("llm.token_count.prompt", input_tokens)
|
|
span.set_attribute("llm.token_count.completion", output_tokens)
|
|
span.set_attribute("llm.cost.input", input_cost)
|
|
span.set_attribute("llm.cost.output", output_cost)
|
|
span.set_attribute("llm.cost.total", total_cost)
|
|
|
|
return total_cost
|
|
|
|
# Use with OpenAI
|
|
client = OpenAI()
|
|
|
|
def generate_with_cost_tracking(prompt: str, model: str = "gpt-4-turbo-preview"):
|
|
"""Generate response and track costs."""
|
|
response = client.chat.completions.create(
|
|
model=model,
|
|
messages=[{"role": "user", "content": prompt}],
|
|
)
|
|
|
|
# Extract token counts
|
|
usage = response.usage
|
|
cost = track_llm_call(model, usage.prompt_tokens, usage.completion_tokens)
|
|
|
|
print(f"Cost: ${cost:.4f}")
|
|
|
|
return response.choices[0].message.content
|
|
|
|
# Generate cost reports from Phoenix
|
|
px_client = px.Client()
|
|
|
|
# Get traces from last 24 hours
|
|
end_time = datetime.now()
|
|
start_time = end_time - timedelta(days=1)
|
|
|
|
traces = px_client.get_trace_dataset(
|
|
start_time=start_time,
|
|
end_time=end_time,
|
|
)
|
|
|
|
# Calculate total costs
|
|
total_cost = traces["llm.cost.total"].sum()
|
|
cost_by_model = traces.groupby("llm.model_name")["llm.cost.total"].sum()
|
|
|
|
print(f"Total cost (24h): ${total_cost:.2f}")
|
|
print("\nCost by model:")
|
|
for model, cost in cost_by_model.items():
|
|
print(f" {model}: ${cost:.2f}")
|
|
|
|
# Identify expensive queries
|
|
expensive_queries = traces.nlargest(10, "llm.cost.total")
|
|
print("\nMost expensive queries:")
|
|
for _, row in expensive_queries.iterrows():
|
|
print(f" Cost: ${row['llm.cost.total']:.4f} | Input: {row['input'][:50]}...")</code></pre>
|
|
<h2>Code Examples</h2>
|
|
<h3>Example 1: Multi-Framework Evaluation Pipeline</h3>
|
|
<pre><code class="lang-python">from typing import List, Dict, Any
|
|
import phoenix as px
|
|
from phoenix.evals import HallucinationEvaluator, OpenAIModel
|
|
from langfuse import Langfuse
|
|
from openai import OpenAI
|
|
import json
|
|
|
|
class UnifiedEvaluationPipeline:
|
|
"""Evaluation pipeline supporting multiple frameworks."""
|
|
|
|
def __init__(
|
|
self,
|
|
phoenix_endpoint: str = "http://localhost:6006",
|
|
langfuse_public_key: str = None,
|
|
langfuse_secret_key: str = None,
|
|
):
|
|
# Setup Phoenix
|
|
self.px_client = px.Client(endpoint=phoenix_endpoint)
|
|
|
|
# Setup Langfuse (optional)
|
|
self.langfuse = None
|
|
if langfuse_public_key and langfuse_secret_key:
|
|
self.langfuse = Langfuse(
|
|
public_key=langfuse_public_key,
|
|
secret_key=langfuse_secret_key,
|
|
)
|
|
|
|
# Setup OpenAI
|
|
self.openai_client = OpenAI()
|
|
|
|
# Setup evaluators
|
|
self.eval_model = OpenAIModel(model="gpt-4-turbo-preview")
|
|
self.hallucination_eval = HallucinationEvaluator(self.eval_model)
|
|
|
|
def evaluate_dataset(
|
|
self,
|
|
dataset: List[Dict[str, Any]],
|
|
model: str = "gpt-4-turbo-preview",
|
|
system_prompt: str = "You are a helpful assistant.",
|
|
) -> Dict[str, Any]:
|
|
"""
|
|
Evaluate dataset across all configured frameworks.
|
|
|
|
Args:
|
|
dataset: List of {input, expected_output, context} dicts
|
|
model: OpenAI model name
|
|
system_prompt: System prompt for generation
|
|
|
|
Returns:
|
|
Evaluation results with framework-specific metrics
|
|
"""
|
|
results = {
|
|
"model": model,
|
|
"total_examples": len(dataset),
|
|
"phoenix_results": [],
|
|
"langfuse_results": [],
|
|
"summary": {},
|
|
}
|
|
|
|
for idx, example in enumerate(dataset):
|
|
# Generate response
|
|
response = self.openai_client.chat.completions.create(
|
|
model=model,
|
|
messages=[
|
|
{"role": "system", "content": system_prompt},
|
|
{"role": "user", "content": example["input"]},
|
|
],
|
|
)
|
|
|
|
output = response.choices[0].message.content
|
|
usage = response.usage
|
|
|
|
# Log to Phoenix
|
|
phoenix_result = self._log_to_phoenix(
|
|
input_text=example["input"],
|
|
output=output,
|
|
expected=example.get("expected_output"),
|
|
context=example.get("context"),
|
|
usage=usage,
|
|
model=model,
|
|
)
|
|
results["phoenix_results"].append(phoenix_result)
|
|
|
|
# Log to Langfuse (if configured)
|
|
if self.langfuse:
|
|
langfuse_result = self._log_to_langfuse(
|
|
input_text=example["input"],
|
|
output=output,
|
|
expected=example.get("expected_output"),
|
|
usage=usage,
|
|
model=model,
|
|
)
|
|
results["langfuse_results"].append(langfuse_result)
|
|
|
|
# Progress
|
|
if (idx + 1) % 10 == 0:
|
|
print(f"Processed {idx + 1}/{len(dataset)} examples")
|
|
|
|
# Calculate summary metrics
|
|
results["summary"] = self._calculate_summary(results)
|
|
|
|
return results
|
|
|
|
def _log_to_phoenix(self, input_text, output, expected, context, usage, model):
|
|
"""Log result to Phoenix and run evaluations."""
|
|
# Create trace manually (if not using auto-instrumentation)
|
|
trace_data = {
|
|
"input": input_text,
|
|
"output": output,
|
|
"expected": expected,
|
|
"context": context,
|
|
"model": model,
|
|
"prompt_tokens": usage.prompt_tokens,
|
|
"completion_tokens": usage.completion_tokens,
|
|
}
|
|
|
|
# Run hallucination eval if context provided
|
|
hallucination_score = None
|
|
if context:
|
|
eval_result = self.hallucination_eval.evaluate_single(
|
|
query=input_text,
|
|
response=output,
|
|
reference=context,
|
|
)
|
|
hallucination_score = eval_result["label"]
|
|
|
|
trace_data["hallucination_score"] = hallucination_score
|
|
|
|
return trace_data
|
|
|
|
def _log_to_langfuse(self, input_text, output, expected, usage, model):
|
|
"""Log result to Langfuse."""
|
|
generation = self.langfuse.generation(
|
|
name="qa-generation",
|
|
model=model,
|
|
input=input_text,
|
|
output=output,
|
|
metadata={"expected": expected},
|
|
usage={
|
|
"input": usage.prompt_tokens,
|
|
"output": usage.completion_tokens,
|
|
"total": usage.total_tokens,
|
|
},
|
|
)
|
|
|
|
return {
|
|
"trace_id": generation.trace_id,
|
|
"generation_id": generation.id,
|
|
}
|
|
|
|
def _calculate_summary(self, results):
|
|
"""Calculate aggregate metrics."""
|
|
phoenix_results = results["phoenix_results"]
|
|
|
|
# Hallucination rate
|
|
hallucination_scores = [
|
|
r["hallucination_score"]
|
|
for r in phoenix_results
|
|
if r["hallucination_score"] is not None
|
|
]
|
|
hallucination_rate = (
|
|
sum(1 for s in hallucination_scores if s == "hallucinated")
|
|
/ len(hallucination_scores)
|
|
if hallucination_scores
|
|
else 0
|
|
)
|
|
|
|
# Token statistics
|
|
total_prompt_tokens = sum(r["prompt_tokens"] for r in phoenix_results)
|
|
total_completion_tokens = sum(r["completion_tokens"] for r in phoenix_results)
|
|
|
|
return {
|
|
"hallucination_rate": hallucination_rate,
|
|
"total_prompt_tokens": total_prompt_tokens,
|
|
"total_completion_tokens": total_completion_tokens,
|
|
"avg_prompt_tokens": total_prompt_tokens / len(phoenix_results),
|
|
"avg_completion_tokens": total_completion_tokens / len(phoenix_results),
|
|
}
|
|
|
|
def export_results(self, results, output_path: str):
|
|
"""Export results to JSON."""
|
|
with open(output_path, "w") as f:
|
|
json.dump(results, f, indent=2)
|
|
|
|
print(f"Results exported to {output_path}")
|
|
|
|
# Usage
|
|
pipeline = UnifiedEvaluationPipeline(
|
|
phoenix_endpoint="http://localhost:6006",
|
|
langfuse_public_key="pk-xxx",
|
|
langfuse_secret_key="sk-xxx",
|
|
)
|
|
|
|
dataset = [
|
|
{
|
|
"input": "What is the capital of France?",
|
|
"expected_output": "Paris",
|
|
"context": "France is a country in Europe. Its capital city is Paris.",
|
|
},
|
|
# ... more examples
|
|
]
|
|
|
|
results = pipeline.evaluate_dataset(
|
|
dataset=dataset,
|
|
model="gpt-4-turbo-preview",
|
|
system_prompt="You are a geography expert.",
|
|
)
|
|
|
|
print(f"Hallucination rate: {results['summary']['hallucination_rate']:.2%}")
|
|
print(f"Avg tokens per query: {results['summary']['avg_prompt_tokens']:.0f}")
|
|
|
|
pipeline.export_results(results, "evaluation_results.json")</code></pre>
|
|
<h3>Example 2: Phoenix Self-Hosted Production Setup</h3>
|
|
<pre><code class="lang-python"># docker-compose.yml for production Phoenix deployment
|
|
"""
|
|
version: '3.8'
|
|
|
|
services:
|
|
phoenix:
|
|
image: arizephoenix/phoenix:latest
|
|
ports:
|
|
- "6006:6006"
|
|
environment:
|
|
- PHOENIX_SQL_DATABASE_URL=postgresql://user:pass@postgres:5432/phoenix
|
|
- PHOENIX_WORKING_DIR=/data
|
|
volumes:
|
|
- phoenix-data:/data
|
|
depends_on:
|
|
- postgres
|
|
restart: unless-stopped
|
|
|
|
postgres:
|
|
image: postgres:15
|
|
environment:
|
|
- POSTGRES_USER=user
|
|
- POSTGRES_PASSWORD=pass
|
|
- POSTGRES_DB=phoenix
|
|
volumes:
|
|
- postgres-data:/var/lib/postgresql/data
|
|
restart: unless-stopped
|
|
|
|
volumes:
|
|
phoenix-data:
|
|
postgres-data:
|
|
"""
|
|
|
|
# Application code with production Phoenix
|
|
from phoenix.otel import register
|
|
from openinference.instrumentation.openai import OpenAIInstrumentor
|
|
import os
|
|
|
|
# Configure production Phoenix endpoint
|
|
PHOENIX_ENDPOINT = os.getenv("PHOENIX_ENDPOINT", "http://phoenix:6006/v1/traces")
|
|
|
|
tracer_provider = register(
|
|
project_name="production-app",
|
|
endpoint=PHOENIX_ENDPOINT,
|
|
)
|
|
|
|
# Auto-instrument
|
|
OpenAIInstrumentor().instrument(tracer_provider=tracer_provider)
|
|
|
|
# Your application code runs as normal
|
|
# All traces automatically sent to self-hosted Phoenix</code></pre>
|
|
<h2>Anti-Patterns</h2>
|
|
<h3>Anti-Pattern 1: Framework Lock-In</h3>
|
|
<p><strong>Wrong</strong>: Building everything for one framework</p>
|
|
<pre><code class="lang-python"># BAD: Tightly coupled to LangSmith
|
|
from langsmith import Client
|
|
|
|
def evaluate_model():
|
|
client = Client() # Only works with LangSmith
|
|
# ... LangSmith-specific code</code></pre>
|
|
<p><strong>Right</strong>: Use OpenTelemetry for portability</p>
|
|
<pre><code class="lang-python"># GOOD: Framework-agnostic tracing
|
|
from phoenix.otel import register
|
|
from opentelemetry import trace
|
|
|
|
tracer_provider = register(project_name="my-app")
|
|
tracer = trace.get_tracer(__name__)
|
|
|
|
# Works with Phoenix, or any OTLP-compatible backend
|
|
with tracer.start_as_current_span("llm-call") as span:
|
|
span.set_attribute("llm.model", "gpt-4")
|
|
# ... your code</code></pre>
|
|
<h3>Anti-Pattern 2: Not Tracking Costs</h3>
|
|
<p><strong>Wrong</strong>: Ignoring cost implications</p>
|
|
<pre><code class="lang-python"># BAD: No cost tracking
|
|
for prompt in large_dataset:
|
|
expensive_model(prompt) # Could cost $$$</code></pre>
|
|
<p><strong>Right</strong>: Monitor costs in real-time</p>
|
|
<pre><code class="lang-python"># GOOD: Track and alert on costs
|
|
cost_tracker = CostTracker(budget_limit=100.00)
|
|
|
|
for prompt in large_dataset:
|
|
if cost_tracker.remaining_budget < 10.00:
|
|
# Switch to cheaper model or stop
|
|
model = "gpt-3.5-turbo"
|
|
|
|
response = model(prompt)
|
|
cost_tracker.log_cost(response.usage)</code></pre>
|
|
<h3>Anti-Pattern 3: Manual Instrumentation Everywhere</h3>
|
|
<p><strong>Wrong</strong>: Manually logging every LLM call</p>
|
|
<pre><code class="lang-python"># BAD: Manual tracing
|
|
def call_llm(prompt):
|
|
start = time.time()
|
|
response = openai.chat.completions.create(...)
|
|
duration = time.time() - start
|
|
|
|
log_to_framework({
|
|
"prompt": prompt,
|
|
"response": response,
|
|
"duration": duration,
|
|
}) # Tedious and error-prone</code></pre>
|
|
<p><strong>Right</strong>: Use auto-instrumentation</p>
|
|
<pre><code class="lang-python"># GOOD: Auto-instrumentation
|
|
from openinference.instrumentation.openai import OpenAIInstrumentor
|
|
|
|
OpenAIInstrumentor().instrument()
|
|
|
|
# All OpenAI calls automatically traced
|
|
response = openai.chat.completions.create(...)
|
|
# No manual logging needed!</code></pre>
|
|
<h2>Related Skills</h2>
|
|
<ul><li><code>llm-benchmarks-evaluation.md</code>: Standard benchmarks (MMLU, HumanEval) for offline evaluation</li><li><code>llm-as-judge.md</code>: Using LLMs to evaluate LLM outputs with Prometheus and G-Eval</li><li><code>rag-evaluation-metrics.md</code>: RAGAS metrics for RAG systems with framework integration</li><li><code>custom-llm-evaluation.md</code>: Domain-specific evaluation metrics and continuous evaluation</li><li><code>dspy-evaluation.md</code>: DSPy-specific evaluation patterns and metric functions</li></ul>
|
|
<h2>Summary</h2>
|
|
<p>LLM evaluation frameworks provide production-grade monitoring and evaluation:</p>
|
|
<p><strong>Key Takeaways</strong>:</p>
|
|
<ol><li><strong>Arize Phoenix</strong>: Best for self-hosted, framework-agnostic tracing with OpenTelemetry</li><li><strong>Braintrust</strong>: Best for fast search (86x) and API-driven workflows</li><li><strong>LangSmith</strong>: Best for LangChain applications with human feedback</li><li><strong>Langfuse</strong>: Best for open-source + cost tracking</li></ol>
|
|
<p><strong>Phoenix Deep Dive</strong>:</p>
|
|
<ul><li>Auto-instrumentation for OpenAI, LangChain, LlamaIndex, DSPy</li><li>Built-in evaluators: Hallucination, Relevance, Toxicity, Q&A correctness</li><li>Dataset and experiment management for systematic evaluation</li><li>Self-hostable with PostgreSQL backend for production</li><li>Real-time dashboards and embedding visualization</li></ul>
|
|
<p><strong>Best Practices</strong>:</p>
|
|
<ul><li>Use OpenTelemetry for portability across frameworks</li><li>Track costs per model, user, or project</li><li>Auto-instrument instead of manual logging</li><li>Run continuous evals on production traffic</li><li>Self-host for data privacy and control (Phoenix, Langfuse)</li></ul>
|
|
<p><strong>When to combine with other skills</strong>:</p>
|
|
<ul><li>Use <code>llm-benchmarks-evaluation.md</code> for offline capability testing</li><li>Use <code>llm-as-judge.md</code> when automated metrics don't capture quality</li><li>Use <code>rag-evaluation-metrics.md</code> for RAG-specific metrics (RAGAS)</li><li>Use <code>custom-llm-evaluation.md</code> for domain-specific or safety evaluations</li></ul>
|
|
<footer>
|
|
<div class="fmap">
|
|
<div><b>Build</b>
|
|
<a href="https://hanzo.app">hanzo.app</a> app builder
|
|
<a href="https://hanzo.codes">hanzo.codes</a> code with an agent
|
|
<a href="https://hanzoskills.com">hanzoskills.com</a> what an agent reads</div>
|
|
<div><b>Run</b>
|
|
<a href="https://console.hanzo.ai">console.hanzo.ai</a> cloud console
|
|
<a href="https://api.hanzo.ai">api.hanzo.ai</a> the API</div>
|
|
<div><b>Account</b>
|
|
<a href="https://hanzo.id">hanzo.id</a> sign in
|
|
<a href="https://pay.hanzo.ai">pay.hanzo.ai</a> top up
|
|
<a href="https://billing.hanzo.ai">billing.hanzo.ai</a> invoices and usage</div>
|
|
<div><b>Company</b>
|
|
<a href="https://hanzo.ai">hanzo.ai</a> canonical
|
|
<a href="https://hanzo.ventures">hanzo.ventures</a> ventures</div>
|
|
</div>
|
|
<div class="fend"><a href="/ml-llm-evaluation-frameworks.md">This page as plain text</a> · <a href="/llms.txt">Index for agents</a></div>
|
|
</footer>
|
|
</div></body></html> |