mirror of
https://github.com/hanzoai/skills.git
synced 2026-11-19 00:47:36 +00:00
hanzoskills.com/llms.txt now opens with the install line, the API reference index and the company index, so an agent that lands on any one of the three can find the rest. robots.txt claimed 537 documents while llms.txt counts 920. The number is gone from robots.txt; llms.txt derives it. public/ is regenerated from scripts/site.py, which it had fallen behind: the copy-prompt menu and agent bar were in the generator and not in the committed pages. tests/site.mjs passes against the result. Co-authored-by: Hanzo Dev <[email protected]>
813 lines
38 KiB
HTML
813 lines
38 KiB
HTML
<!doctype html>
|
|
<html lang="en"><head>
|
|
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
<title>ml-llm-benchmarks-evaluation</title>
|
|
<meta name="color-scheme" content="dark">
|
|
<style>
|
|
/* Committed to one visual world: a dark instrument panel. No light variant —
|
|
the dot field and the hairline rules only read on black, and a naive
|
|
inversion would wash both out. Every colour is painted explicitly so the page
|
|
never borrows a host background. */
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Pixel Square';src:url('/fonts/ZenPixel-Square.woff2') format('woff2');font-style:normal;font-display:swap}
|
|
|
|
:root{
|
|
--ink:#f4f4f2; --soft:#9c9c98; --faint:#6a6a66; --ground:#000; --panel:#0a0a0a;
|
|
--line:#232323; --line-hi:#3a3a3a; --accent:#f4f4f2;
|
|
--font-zen-sans:'Zen',-apple-system,BlinkMacSystemFont,"Segoe UI",Helvetica,Arial,sans-serif;
|
|
--font-zen-mono:'Zen Mono',ui-monospace,"SF Mono",SFMono-Regular,Menlo,Consolas,monospace;
|
|
--font-zen-pixel-square:'Zen Pixel Square',monospace;
|
|
--mono:var(--font-zen-mono);
|
|
--sans:var(--font-zen-sans);
|
|
/* Controls are pills and surfaces are soft, as on hanzo.ai. */
|
|
--r:999px; --r-soft:14px;
|
|
}
|
|
*{box-sizing:border-box}
|
|
html{-webkit-text-size-adjust:100%}
|
|
body{margin:0;background:var(--ground);color:var(--ink);font-family:var(--sans);
|
|
font-size:16px;line-height:1.65;-webkit-font-smoothing:antialiased}
|
|
.wrap{max-width:60rem;margin:0 auto;padding:0 1.75rem}
|
|
|
|
/* The dot field. A halftone made of one repeating radial-gradient, faded out
|
|
along the diagonal by a mask so it reads as a wave rather than wallpaper. */
|
|
/* No overflow clip here: the dot field is inset to the box already, and a clip
|
|
would cut off the copy-prompt menu. */
|
|
.hero{position:relative;border-bottom:1px solid var(--line)}
|
|
.hero::before{content:"";position:absolute;inset:0;
|
|
background-image:radial-gradient(var(--line-hi) 1px, transparent 1px);
|
|
background-size:7px 7px;
|
|
-webkit-mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
pointer-events:none}
|
|
.hero .wrap{position:relative;padding-top:5.5rem;padding-bottom:4rem}
|
|
|
|
nav{border-bottom:1px solid var(--line)}
|
|
nav .in{max-width:60rem;margin:0 auto;padding:1.1rem 1.75rem;display:flex;
|
|
justify-content:space-between;align-items:center;gap:1rem}
|
|
.brand{font-family:var(--mono);font-size:.85rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--ink);text-decoration:none;
|
|
display:inline-flex;align-items:center;gap:.6rem}
|
|
.brand i{font-style:normal;color:var(--faint)}
|
|
.mark{width:1.15em;height:1.15em;display:block;flex:0 0 auto}
|
|
nav a.up{font-family:var(--mono);font-size:.72rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--soft);text-decoration:none}
|
|
nav a.up:hover{color:var(--ink)}
|
|
|
|
h1{font-family:var(--mono);font-size:clamp(2rem,5.4vw,3.4rem);line-height:1.12;
|
|
letter-spacing:-.02em;margin:0 0 1.1rem;font-weight:500;text-wrap:balance}
|
|
h1 em{font-style:normal;color:var(--faint)}
|
|
.lede{color:var(--soft);font-size:1.05rem;max-width:52ch;margin:0 0 2.25rem}
|
|
|
|
.btns{display:flex;flex-wrap:wrap;gap:.7rem;margin:0 0 .5rem;align-items:center}
|
|
.btn{display:inline-flex;align-items:center;gap:.5rem;font-family:var(--mono);
|
|
font-size:.72rem;letter-spacing:.14em;text-transform:uppercase;
|
|
padding:.8rem 1.4rem;border:1px solid var(--line-hi);border-radius:var(--r);
|
|
color:var(--ink);text-decoration:none;background:transparent;cursor:pointer;
|
|
line-height:1.2;font-weight:400}
|
|
.btn:hover{background:#141414;text-decoration:none;border-color:var(--soft)}
|
|
.btn.solid{background:var(--ink);color:#000;border-color:var(--ink)}
|
|
.btn.solid:hover{background:#d8d8d4}
|
|
.btn .chev{width:.7em;height:.7em;flex:0 0 auto;transition:transform .15s ease}
|
|
.btn[aria-expanded="true"] .chev{transform:rotate(180deg)}
|
|
|
|
/* Copy-prompt selector. Each row copies the prompt for its own target. */
|
|
.copy{position:relative;display:inline-flex}
|
|
.menu{position:absolute;top:calc(100% + .5rem);left:0;z-index:20;min-width:21rem;
|
|
background:var(--panel);border:1px solid var(--line-hi);border-radius:var(--r-soft);
|
|
padding:.4rem;display:flex;flex-direction:column;
|
|
box-shadow:0 12px 32px rgba(0,0,0,.75)}
|
|
.menu[hidden]{display:none}
|
|
.mi{display:flex;align-items:center;gap:.7rem;width:100%;text-align:left;
|
|
padding:.62rem .8rem;border:0;border-radius:calc(var(--r-soft) - .35rem);
|
|
background:transparent;color:var(--ink);font-family:var(--mono);font-size:.76rem;
|
|
letter-spacing:.04em;text-decoration:none;cursor:pointer;line-height:1.3;
|
|
white-space:nowrap}
|
|
.mi:hover,.mi:focus-visible{background:#181818;outline:none}
|
|
.mi .mark{width:1.05rem;height:1.05rem;color:var(--soft)}
|
|
.mi:hover .mark{color:var(--ink)}
|
|
.mi u{margin-left:auto;color:var(--faint);text-decoration:none;font-size:.68rem;
|
|
letter-spacing:.1em;text-transform:uppercase}
|
|
|
|
/* Works-with bar: one scrolling row, name and mark per agent. */
|
|
.strip{border-bottom:1px solid var(--line);background:var(--panel)}
|
|
.strip .in{max-width:60rem;margin:0 auto;padding:.9rem 1.75rem;display:flex;
|
|
align-items:center;gap:1.25rem;font-family:var(--mono);font-size:.72rem;
|
|
letter-spacing:.1em;text-transform:uppercase;color:var(--faint)}
|
|
.strip b{color:var(--soft);font-weight:400;flex:0 0 auto;
|
|
border-right:1px solid var(--line);padding-right:1.25rem;align-self:stretch;
|
|
display:flex;align-items:center}
|
|
.bar{display:flex;align-items:center;gap:.5rem;flex:1 1 auto;min-width:0;
|
|
overflow-x:auto;scrollbar-width:none;scroll-snap-type:x proximity;
|
|
-webkit-overflow-scrolling:touch;
|
|
-webkit-mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent);
|
|
mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent)}
|
|
.bar::-webkit-scrollbar{display:none}
|
|
.agent{display:inline-flex;align-items:center;gap:.45rem;flex:0 0 auto;
|
|
padding:.4rem .85rem;border:1px solid var(--line);border-radius:var(--r);
|
|
color:var(--soft);white-space:nowrap;scroll-snap-align:start;
|
|
text-decoration:none;background:var(--ground)}
|
|
.agent:hover{color:var(--ink);border-color:var(--line-hi);text-decoration:none}
|
|
.agent .mark{width:.95rem;height:.95rem}
|
|
|
|
h2{font-family:var(--mono);font-size:1.1rem;letter-spacing:-.01em;margin:2.75rem 0 .75rem;font-weight:500}
|
|
h3{font-size:1rem;margin:2rem 0 .5rem;font-weight:600}
|
|
h4,h5,h6{font-size:.94rem;margin:1.5rem 0 .5rem;font-weight:600}
|
|
p{margin:0 0 1rem;max-width:70ch}
|
|
ul,ol{margin:0 0 1rem;padding-left:1.35rem;max-width:70ch}
|
|
li{margin:.22rem 0}
|
|
a{color:var(--ink);text-decoration:underline;text-underline-offset:3px;
|
|
text-decoration-color:var(--line-hi)}
|
|
a:hover{text-decoration-color:var(--ink)}
|
|
a:focus-visible{outline:1px solid var(--ink);outline-offset:3px}
|
|
code{font-family:var(--mono);font-size:.85em;background:var(--panel);
|
|
border:1px solid var(--line);border-radius:5px;padding:.1em .36em}
|
|
pre{background:var(--panel);border:1px solid var(--line);padding:1rem 1.1rem;
|
|
border-radius:var(--r-soft);overflow-x:auto;margin:0 0 1.25rem}
|
|
pre code{background:none;border:0;padding:0;font-size:.82rem;line-height:1.7}
|
|
blockquote{margin:0 0 1rem;padding-left:1rem;border-left:1px solid var(--line-hi);color:var(--soft)}
|
|
hr{border:0;border-top:1px solid var(--line);margin:2.5rem 0}
|
|
table{border-collapse:collapse;width:100%;margin:0 0 1.25rem;font-size:.92rem;
|
|
display:block;overflow-x:auto}
|
|
th,td{border:1px solid var(--line);padding:.5rem .7rem;text-align:left}
|
|
|
|
.curl{display:block;background:var(--panel);border:1px solid var(--line);
|
|
border-radius:var(--r-soft);padding:.95rem 1.1rem;font-family:var(--mono);
|
|
font-size:.8rem;color:var(--soft);overflow-x:auto;margin:0 0 2.5rem;
|
|
white-space:pre;line-height:1.9}
|
|
.curl b{color:var(--ink);font-weight:400}
|
|
.curl i{font-style:normal;color:var(--faint)}
|
|
|
|
/* index */
|
|
#q{width:100%;padding:.85rem 1.2rem;font-family:var(--mono);font-size:.85rem;color:var(--ink);
|
|
background:var(--panel);border:1px solid var(--line);border-radius:var(--r);margin:0 0 .6rem}
|
|
#q::placeholder{color:var(--faint);text-transform:uppercase;letter-spacing:.1em;font-size:.72rem}
|
|
#q:focus{outline:none;border-color:var(--soft)}
|
|
#count{color:var(--faint);font-family:var(--mono);font-size:.72rem;letter-spacing:.1em;
|
|
text-transform:uppercase;margin:0 0 2rem}
|
|
.cat{margin:2.5rem 0 .5rem;font-family:var(--mono);font-size:.7rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);border-bottom:1px solid var(--line);
|
|
padding-bottom:.45rem}
|
|
.row{display:block;padding:.6rem .9rem;margin:0 -.9rem;border-bottom:1px solid var(--line);
|
|
border-radius:var(--r-soft);text-decoration:none}
|
|
.row:hover{background:#0a0a0a}
|
|
.nm{font-family:var(--mono);font-size:.84rem;color:var(--ink);display:block}
|
|
.ds{color:var(--soft);font-size:.9rem;display:block;margin-top:.12rem}
|
|
footer{margin-top:4rem;padding:2rem 0 4rem;border-top:1px solid var(--line);
|
|
color:var(--faint);font-size:.85rem}
|
|
footer a{color:var(--soft)}
|
|
.fmap{display:grid;gap:1.75rem;grid-template-columns:repeat(auto-fit,minmax(11rem,1fr));
|
|
margin:0 0 2rem;font-size:.85rem}
|
|
.fmap b{display:block;font-family:var(--mono);font-size:.68rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);font-weight:400;margin:0 0 .6rem;
|
|
border-bottom:1px solid var(--line);padding-bottom:.4rem}
|
|
.fmap a{display:inline-block;color:var(--ink);text-decoration:none;margin:.3rem .35rem 0 0}
|
|
.fmap a:hover{text-decoration:underline}
|
|
.fmap div{color:var(--faint);line-height:1.5}
|
|
.fend{border-top:1px solid var(--line);padding-top:1.25rem;font-size:.82rem}
|
|
|
|
@media (max-width:640px){
|
|
.hero .wrap{padding-top:3.25rem;padding-bottom:2.75rem}
|
|
.btns{gap:.5rem}
|
|
.btn{padding:.75rem 1.15rem;font-size:.68rem}
|
|
.copy,.copy .btn{width:100%;justify-content:center}
|
|
.menu{left:0;right:0;min-width:0}
|
|
.strip .in{flex-direction:column;align-items:flex-start;gap:.6rem;padding:.9rem 1.75rem}
|
|
.strip b{border-right:0;padding-right:0}
|
|
.bar{width:100%}
|
|
}
|
|
</style>
|
|
<link rel="alternate" type="text/markdown" href="/ml-llm-benchmarks-evaluation.md">
|
|
|
|
<link rel="icon" href="/favicon.svg">
|
|
<link rel="canonical" href="https://hanzoskills.com/ml-llm-benchmarks-evaluation">
|
|
</head>
|
|
<!-- Reading this with a machine? Every page here is also plain markdown.
|
|
This one: https://hanzoskills.com/ml-llm-benchmarks-evaluation.md
|
|
Index: https://hanzoskills.com/llms.txt
|
|
Corpus: https://hanzoskills.com/all.md
|
|
Append .md to any path for the markdown twin. -->
|
|
<body>
|
|
<nav><div class="in">
|
|
<a class="brand" href="/"><svg class="mark" viewBox="0 0 67 67" fill="currentColor" aria-hidden="true"><path d="M22.21 67V44.6369H0V67H22.21Z"/><path d="M66.7038 22.3184H22.2534L0.0878906 44.6367H44.4634L66.7038 22.3184Z"/><path d="M22.21 0H0V22.3184H22.21V0Z"/><path d="M66.7198 0H44.5098V22.3184H66.7198V0Z"/><path d="M66.7198 67V44.6369H44.5098V67H66.7198Z"/></svg> HANZO<i>SKILLS</i></a>
|
|
<a class="up" href="/">All skills</a>
|
|
</div></nav>
|
|
|
|
<div class="wrap">
|
|
<h1>ml-llm-benchmarks-evaluation</h1><p class="lede">Comprehensive guide to evaluating LLMs using standard benchmarks including MMLU, HellaSwag, BBH, HumanEval, TruthfulQA, and GSM8K, with practical implementation using LightEval and lm-evaluation-harness</p><h1>LLM Benchmarks Evaluation</h1>
|
|
<p>Last Updated: 2025-10-26</p>
|
|
<h2>When to Use This Skill</h2>
|
|
<p>Use LLM benchmark evaluation when:</p>
|
|
<ul><li><strong>Comparing model capabilities</strong>: Measuring performance across different LLMs or model versions</li><li><strong>Tracking training progress</strong>: Monitoring improvements during fine-tuning or continued pre-training</li><li><strong>Validating model releases</strong>: Ensuring models meet minimum quality thresholds before deployment</li><li><strong>Research validation</strong>: Reproducing published results or establishing baselines</li><li><strong>Domain transfer</strong>: Assessing model performance on new tasks or domains</li><li><strong>Resource optimization</strong>: Determining if a smaller/faster model meets accuracy requirements</li><li><strong>Detecting contamination</strong>: Identifying potential training data leakage in benchmarks</li></ul>
|
|
<p><strong>Anti-pattern</strong>: Using benchmarks as the <strong>only</strong> evaluation metric. Always complement with domain-specific tests, human evaluation, and production metrics.</p>
|
|
<h2>Core Concepts</h2>
|
|
<h3>Standard Benchmark Categories</h3>
|
|
<p><strong>1. Knowledge & Reasoning</strong></p>
|
|
<ul><li><strong>MMLU</strong> (Massive Multitask Language Understanding): 57 subjects across STEM, humanities, social sciences</li><li><strong>BBH</strong> (BIG-Bench Hard): 23 challenging tasks from BIG-Bench that require multi-step reasoning</li><li><strong>TruthfulQA</strong>: 817 questions testing model truthfulness and resistance to falsehoods</li></ul>
|
|
<p><strong>2. Code Generation</strong></p>
|
|
<ul><li><strong>HumanEval</strong>: 164 Python programming problems with test cases</li><li><strong>MBPP</strong> (Mostly Basic Python Problems): 974 entry-level programming tasks</li><li><strong>CodeContests</strong>: Competitive programming problems requiring algorithmic reasoning</li></ul>
|
|
<p><strong>3. Math Reasoning</strong></p>
|
|
<ul><li><strong>GSM8K</strong>: 8,500 grade school math word problems</li><li><strong>MATH</strong>: 12,500 challenging competition mathematics problems</li><li><strong>MathQA</strong>: Math word problems with multiple-choice answers</li></ul>
|
|
<p><strong>4. Common Sense & NLU</strong></p>
|
|
<ul><li><strong>HellaSwag</strong>: Commonsense inference about physical situations</li><li><strong>PIQA</strong> (Physical Interaction QA): Physical commonsense reasoning</li><li><strong>WinoGrande</strong>: Winograd schema challenge for pronoun resolution</li><li><strong>ARC</strong> (AI2 Reasoning Challenge): Science questions for grade school students</li></ul>
|
|
<p><strong>5. Safety & Alignment</strong></p>
|
|
<ul><li><strong>ToxiGen</strong>: Detecting implicit toxicity and hate speech</li><li><strong>BBQ</strong> (Bias Benchmark for QA): Measuring social biases</li><li><strong>BOLD</strong>: Measuring biases in open-ended language generation</li></ul>
|
|
<h3>Evaluation Frameworks</h3>
|
|
<p><strong>lm-evaluation-harness</strong> (EleutherAI):</p>
|
|
<ul><li>Unified framework for 200+ tasks</li><li>Standardized prompting and scoring</li><li>Reproducible evaluation pipeline</li><li>Supports HuggingFace models, OpenAI API, Anthropic, etc.</li></ul>
|
|
<p><strong>LightEval</strong> (HuggingFace):</p>
|
|
<ul><li>Lightweight, fast evaluation framework</li><li>Easy integration with HuggingFace Hub</li><li>Custom task creation</li><li>Efficient multi-GPU evaluation</li></ul>
|
|
<h3>Key Metrics</h3>
|
|
<p><strong>Accuracy</strong>: Percentage of correct answers (most common) <strong>Exact Match (EM)</strong>: Exact string match for generation tasks <strong>Pass@k</strong>: Percentage of problems solved with k samples (code generation) <strong>F1 Score</strong>: Harmonic mean of precision and recall (span-based tasks) <strong>BLEU/ROUGE</strong>: N-gram overlap for generation (less common in modern evals) <strong>Normalized Accuracy</strong>: Accounting for random guessing baseline</p>
|
|
<h2>Implementation Patterns</h2>
|
|
<h3>Pattern 1: lm-evaluation-harness Setup</h3>
|
|
<p><strong>When to use</strong>: Industry-standard benchmark evaluation with reproducibility</p>
|
|
<pre><code class="lang-bash"># Installation
|
|
pip install lm-eval
|
|
|
|
# Basic evaluation (HuggingFace model)
|
|
lm_eval --model hf \
|
|
--model_args pretrained=meta-llama/Llama-2-7b-hf \
|
|
--tasks mmlu,hellaswag,truthfulqa_mc2 \
|
|
--device cuda:0 \
|
|
--batch_size 8
|
|
|
|
# Multiple tasks with custom settings
|
|
lm_eval --model hf \
|
|
--model_args pretrained=mistralai/Mistral-7B-v0.1,dtype=bfloat16 \
|
|
--tasks mmlu,arc_challenge,hellaswag,gsm8k,humaneval \
|
|
--num_fewshot 5 \
|
|
--device cuda:0 \
|
|
--batch_size auto \
|
|
--output_path results/mistral-7b-eval.json
|
|
|
|
# OpenAI API models
|
|
lm_eval --model openai-completions \
|
|
--model_args model=gpt-4-turbo-preview \
|
|
--tasks mmlu,truthfulqa_mc2 \
|
|
--output_path results/gpt4-eval.json
|
|
|
|
# Anthropic Claude
|
|
lm_eval --model anthropic \
|
|
--model_args model=claude-3-opus-20240229 \
|
|
--tasks mmlu,truthfulqa_mc2</code></pre>
|
|
<p><strong>Python API usage</strong>:</p>
|
|
<pre><code class="lang-python">from lm_eval import evaluator, tasks
|
|
from lm_eval.models.huggingface import HFLM
|
|
|
|
# Initialize model
|
|
model = HFLM(pretrained="meta-llama/Llama-2-7b-hf", device="cuda:0")
|
|
|
|
# Run evaluation
|
|
results = evaluator.simple_evaluate(
|
|
model=model,
|
|
tasks=["mmlu", "hellaswag", "truthfulqa_mc2", "gsm8k"],
|
|
num_fewshot=5,
|
|
batch_size=8,
|
|
device="cuda:0",
|
|
)
|
|
|
|
# Access results
|
|
print(f"MMLU Accuracy: {results['results']['mmlu']['acc']:.4f}")
|
|
print(f"HellaSwag Accuracy: {results['results']['hellaswag']['acc_norm']:.4f}")
|
|
print(f"TruthfulQA MC2: {results['results']['truthfulqa_mc2']['acc']:.4f}")
|
|
print(f"GSM8K: {results['results']['gsm8k']['exact_match']:.4f}")
|
|
|
|
# Save results
|
|
import json
|
|
with open("eval_results.json", "w") as f:
|
|
json.dump(results, f, indent=2)</code></pre>
|
|
<h3>Pattern 2: LightEval Framework</h3>
|
|
<p><strong>When to use</strong>: Fast evaluation with HuggingFace integration</p>
|
|
<pre><code class="lang-python">from lighteval.tasks.lighteval_task import LightevalTaskConfig
|
|
from lighteval.models.model_loader import load_model
|
|
from lighteval.main_lighteval import main_evaluate
|
|
|
|
# Define evaluation config
|
|
tasks = [
|
|
"mmlu:all",
|
|
"hellaswag",
|
|
"arc:challenge",
|
|
"truthfulqa:mc2",
|
|
"gsm8k",
|
|
]
|
|
|
|
# Load model
|
|
model_config = {
|
|
"model_name": "meta-llama/Llama-2-7b-hf",
|
|
"dtype": "bfloat16",
|
|
"device_map": "auto",
|
|
}
|
|
|
|
# Run evaluation
|
|
results = main_evaluate(
|
|
model_config=model_config,
|
|
tasks=tasks,
|
|
batch_size=8,
|
|
num_fewshot=5,
|
|
output_dir="lighteval_results",
|
|
save_details=True,
|
|
)
|
|
|
|
# Results structure
|
|
for task_name, task_results in results.items():
|
|
print(f"{task_name}: {task_results['acc']:.4f}")</code></pre>
|
|
<h3>Pattern 3: Custom Task Creation</h3>
|
|
<p><strong>When to use</strong>: Evaluating on domain-specific or custom benchmarks</p>
|
|
<pre><code class="lang-python"># lm-evaluation-harness custom task
|
|
from lm_eval.api.task import Task
|
|
from lm_eval.api.instance import Instance
|
|
|
|
class CustomMedicalQA(Task):
|
|
VERSION = 0
|
|
DATASET_PATH = "custom/medical_qa"
|
|
DATASET_NAME = None
|
|
|
|
def has_training_docs(self):
|
|
return True
|
|
|
|
def has_validation_docs(self):
|
|
return True
|
|
|
|
def has_test_docs(self):
|
|
return True
|
|
|
|
def training_docs(self):
|
|
return self.dataset["train"]
|
|
|
|
def validation_docs(self):
|
|
return self.dataset["validation"]
|
|
|
|
def test_docs(self):
|
|
return self.dataset["test"]
|
|
|
|
def doc_to_text(self, doc):
|
|
# Format prompt
|
|
return f"Question: {doc['question']}\nAnswer:"
|
|
|
|
def doc_to_target(self, doc):
|
|
# Correct answer
|
|
return doc["answer"]
|
|
|
|
def construct_requests(self, doc, ctx):
|
|
# Create evaluation request
|
|
ll, is_greedy = self.get_request_type()
|
|
return Instance(
|
|
request_type=ll,
|
|
doc=doc,
|
|
arguments=(ctx, {"until": ["\n"]}),
|
|
idx=0,
|
|
)
|
|
|
|
def process_results(self, doc, results):
|
|
# Score the result
|
|
gold = self.doc_to_target(doc)
|
|
pred = results[0].strip()
|
|
|
|
return {
|
|
"acc": int(pred.lower() == gold.lower()),
|
|
"exact_match": int(pred == gold),
|
|
}
|
|
|
|
def aggregation(self):
|
|
return {
|
|
"acc": mean,
|
|
"exact_match": mean,
|
|
}
|
|
|
|
def higher_is_better(self):
|
|
return {
|
|
"acc": True,
|
|
"exact_match": True,
|
|
}
|
|
|
|
# Register and use
|
|
from lm_eval.tasks import TaskManager
|
|
task_manager = TaskManager()
|
|
task_manager.register_task("custom_medical_qa", CustomMedicalQA)
|
|
|
|
# Evaluate
|
|
results = evaluator.simple_evaluate(
|
|
model=model,
|
|
tasks=["custom_medical_qa"],
|
|
num_fewshot=3,
|
|
)</code></pre>
|
|
<h3>Pattern 4: HumanEval Code Generation</h3>
|
|
<p><strong>When to use</strong>: Evaluating code generation capabilities</p>
|
|
<pre><code class="lang-python">from human_eval.data import write_jsonl, read_problems
|
|
from human_eval.evaluation import evaluate_functional_correctness
|
|
|
|
# Generate completions
|
|
def generate_one_completion(prompt, model, tokenizer):
|
|
"""Generate a single completion for a code prompt."""
|
|
inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
|
|
|
|
outputs = model.generate(
|
|
**inputs,
|
|
max_new_tokens=512,
|
|
temperature=0.2,
|
|
top_p=0.95,
|
|
do_sample=True,
|
|
stop_sequences=["```", "\nclass", "\ndef", "\nif __name__"],
|
|
)
|
|
|
|
completion = tokenizer.decode(outputs[0], skip_special_tokens=True)
|
|
return completion[len(prompt):]
|
|
|
|
# Evaluate on HumanEval
|
|
problems = read_problems()
|
|
samples = []
|
|
|
|
for task_id, problem in problems.items():
|
|
prompt = problem["prompt"]
|
|
|
|
# Generate multiple samples (pass@k)
|
|
for _ in range(10): # Generate 10 samples for pass@10
|
|
completion = generate_one_completion(prompt, model, tokenizer)
|
|
|
|
samples.append({
|
|
"task_id": task_id,
|
|
"completion": completion,
|
|
})
|
|
|
|
# Write samples
|
|
write_jsonl("samples.jsonl", samples)
|
|
|
|
# Evaluate correctness
|
|
results = evaluate_functional_correctness(
|
|
sample_file="samples.jsonl",
|
|
k=[1, 10, 100], # pass@1, pass@10, pass@100
|
|
n_workers=4,
|
|
timeout=3.0,
|
|
)
|
|
|
|
print(f"Pass@1: {results['pass@1']:.4f}")
|
|
print(f"Pass@10: {results['pass@10']:.4f}")</code></pre>
|
|
<h3>Pattern 5: GSM8K Math Evaluation</h3>
|
|
<p><strong>When to use</strong>: Evaluating mathematical reasoning with chain-of-thought</p>
|
|
<pre><code class="lang-python">from datasets import load_dataset
|
|
|
|
# Load GSM8K
|
|
dataset = load_dataset("gsm8k", "main")
|
|
|
|
def extract_answer(text):
|
|
"""Extract numerical answer from GSM8K format."""
|
|
# GSM8K answers are in format "#### 42"
|
|
if "####" in text:
|
|
return text.split("####")[1].strip()
|
|
|
|
# Extract last number if no ####
|
|
import re
|
|
numbers = re.findall(r"-?\d+\.?\d*", text)
|
|
return numbers[-1] if numbers else ""
|
|
|
|
def evaluate_gsm8k(model, tokenizer, num_samples=100):
|
|
"""Evaluate model on GSM8K with chain-of-thought prompting."""
|
|
|
|
# Chain-of-thought prompt
|
|
few_shot_examples = """
|
|
Q: There are 15 trees in the grove. Grove workers will plant trees in the grove today. After they are done, there will be 21 trees. How many trees did the grove workers plant today?
|
|
A: There are 15 trees originally. Then there were 21 trees after some more were planted. So there must have been 21 - 15 = 6. #### 6
|
|
|
|
Q: If there are 3 cars in the parking lot and 2 more cars arrive, how many cars are in the parking lot?
|
|
A: There are originally 3 cars. 2 more cars arrive. 3 + 2 = 5. #### 5
|
|
"""
|
|
|
|
correct = 0
|
|
total = 0
|
|
|
|
for example in dataset["test"].select(range(num_samples)):
|
|
question = example["question"]
|
|
gold_answer = extract_answer(example["answer"])
|
|
|
|
# Create prompt with few-shot examples
|
|
prompt = f"{few_shot_examples}\nQ: {question}\nA:"
|
|
|
|
# Generate
|
|
inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
|
|
outputs = model.generate(
|
|
**inputs,
|
|
max_new_tokens=512,
|
|
temperature=0.0, # Greedy for math
|
|
do_sample=False,
|
|
)
|
|
|
|
completion = tokenizer.decode(outputs[0], skip_special_tokens=True)
|
|
pred_answer = extract_answer(completion)
|
|
|
|
# Check correctness
|
|
if pred_answer == gold_answer:
|
|
correct += 1
|
|
total += 1
|
|
|
|
# Debug output
|
|
if total <= 5:
|
|
print(f"Question: {question}")
|
|
print(f"Gold: {gold_answer}, Pred: {pred_answer}")
|
|
print(f"Completion: {completion}\n")
|
|
|
|
accuracy = correct / total
|
|
print(f"GSM8K Accuracy: {accuracy:.4f} ({correct}/{total})")
|
|
return accuracy</code></pre>
|
|
<h2>Code Examples</h2>
|
|
<h3>Example 1: Comprehensive Benchmark Suite</h3>
|
|
<pre><code class="lang-python">from lm_eval import evaluator
|
|
from lm_eval.models.huggingface import HFLM
|
|
import json
|
|
from datetime import datetime
|
|
|
|
class BenchmarkRunner:
|
|
"""Run comprehensive LLM benchmark suite."""
|
|
|
|
def __init__(self, model_name, output_dir="benchmark_results"):
|
|
self.model_name = model_name
|
|
self.output_dir = output_dir
|
|
self.model = None
|
|
|
|
def load_model(self):
|
|
"""Load model for evaluation."""
|
|
self.model = HFLM(
|
|
pretrained=self.model_name,
|
|
device="cuda:0",
|
|
dtype="bfloat16",
|
|
)
|
|
|
|
def run_knowledge_benchmarks(self):
|
|
"""Run knowledge and reasoning benchmarks."""
|
|
tasks = [
|
|
"mmlu",
|
|
"truthfulqa_mc2",
|
|
"arc_challenge",
|
|
"hellaswag",
|
|
"winogrande",
|
|
]
|
|
|
|
results = evaluator.simple_evaluate(
|
|
model=self.model,
|
|
tasks=tasks,
|
|
num_fewshot=5,
|
|
batch_size=8,
|
|
)
|
|
|
|
return results
|
|
|
|
def run_code_benchmarks(self):
|
|
"""Run code generation benchmarks."""
|
|
tasks = [
|
|
"humaneval",
|
|
"mbpp",
|
|
]
|
|
|
|
results = evaluator.simple_evaluate(
|
|
model=self.model,
|
|
tasks=tasks,
|
|
num_fewshot=0, # Zero-shot for code
|
|
batch_size=1,
|
|
)
|
|
|
|
return results
|
|
|
|
def run_math_benchmarks(self):
|
|
"""Run math reasoning benchmarks."""
|
|
tasks = [
|
|
"gsm8k",
|
|
"math_algebra",
|
|
]
|
|
|
|
results = evaluator.simple_evaluate(
|
|
model=self.model,
|
|
tasks=tasks,
|
|
num_fewshot=8, # More examples for math
|
|
batch_size=4,
|
|
)
|
|
|
|
return results
|
|
|
|
def generate_report(self, all_results):
|
|
"""Generate comprehensive evaluation report."""
|
|
report = {
|
|
"model": self.model_name,
|
|
"timestamp": datetime.now().isoformat(),
|
|
"results": {},
|
|
"summary": {},
|
|
}
|
|
|
|
# Extract key metrics
|
|
for task_name, task_results in all_results["results"].items():
|
|
report["results"][task_name] = task_results
|
|
|
|
# Primary metric per task
|
|
if "acc" in task_results:
|
|
report["summary"][task_name] = task_results["acc"]
|
|
elif "exact_match" in task_results:
|
|
report["summary"][task_name] = task_results["exact_match"]
|
|
elif "pass@1" in task_results:
|
|
report["summary"][task_name] = task_results["pass@1"]
|
|
|
|
# Calculate category averages
|
|
knowledge_tasks = ["mmlu", "truthfulqa_mc2", "arc_challenge", "hellaswag"]
|
|
knowledge_avg = sum(
|
|
report["summary"][t] for t in knowledge_tasks if t in report["summary"]
|
|
) / len([t for t in knowledge_tasks if t in report["summary"]])
|
|
|
|
report["category_averages"] = {
|
|
"knowledge_reasoning": knowledge_avg,
|
|
}
|
|
|
|
# Save report
|
|
output_path = f"{self.output_dir}/{self.model_name.replace('/', '_')}_report.json"
|
|
with open(output_path, "w") as f:
|
|
json.dump(report, f, indent=2)
|
|
|
|
return report
|
|
|
|
def run_full_suite(self):
|
|
"""Run complete benchmark suite."""
|
|
print(f"Starting evaluation for {self.model_name}")
|
|
|
|
self.load_model()
|
|
|
|
print("Running knowledge benchmarks...")
|
|
knowledge_results = self.run_knowledge_benchmarks()
|
|
|
|
print("Running code benchmarks...")
|
|
code_results = self.run_code_benchmarks()
|
|
|
|
print("Running math benchmarks...")
|
|
math_results = self.run_math_benchmarks()
|
|
|
|
# Combine results
|
|
all_results = {
|
|
"results": {
|
|
**knowledge_results["results"],
|
|
**code_results["results"],
|
|
**math_results["results"],
|
|
}
|
|
}
|
|
|
|
report = self.generate_report(all_results)
|
|
|
|
print("\n=== Evaluation Summary ===")
|
|
for task, score in report["summary"].items():
|
|
print(f"{task}: {score:.4f}")
|
|
|
|
return report
|
|
|
|
# Usage
|
|
runner = BenchmarkRunner("mistralai/Mistral-7B-v0.1")
|
|
report = runner.run_full_suite()</code></pre>
|
|
<h3>Example 2: Contamination Detection</h3>
|
|
<pre><code class="lang-python">from lm_eval.api.metrics import perplexity
|
|
from transformers import AutoModelForCausalLM, AutoTokenizer
|
|
import numpy as np
|
|
|
|
class ContaminationDetector:
|
|
"""Detect potential benchmark contamination in training data."""
|
|
|
|
def __init__(self, model_name):
|
|
self.model = AutoModelForCausalLM.from_pretrained(
|
|
model_name,
|
|
device_map="auto",
|
|
)
|
|
self.tokenizer = AutoTokenizer.from_pretrained(model_name)
|
|
|
|
def calculate_perplexity(self, texts):
|
|
"""Calculate perplexity for list of texts."""
|
|
perplexities = []
|
|
|
|
for text in texts:
|
|
inputs = self.tokenizer(text, return_tensors="pt").to(self.model.device)
|
|
|
|
with torch.no_grad():
|
|
outputs = self.model(**inputs, labels=inputs["input_ids"])
|
|
loss = outputs.loss
|
|
perplexity = torch.exp(loss).item()
|
|
|
|
perplexities.append(perplexity)
|
|
|
|
return perplexities
|
|
|
|
def detect_contamination(self, benchmark_samples, baseline_samples):
|
|
"""
|
|
Detect contamination by comparing perplexity on benchmark vs baseline.
|
|
|
|
Suspiciously low perplexity on benchmark suggests potential memorization.
|
|
"""
|
|
benchmark_ppl = self.calculate_perplexity(benchmark_samples)
|
|
baseline_ppl = self.calculate_perplexity(baseline_samples)
|
|
|
|
benchmark_mean = np.mean(benchmark_ppl)
|
|
baseline_mean = np.mean(baseline_ppl)
|
|
|
|
# Calculate ratio
|
|
ratio = baseline_mean / benchmark_mean
|
|
|
|
# Flag if benchmark perplexity is suspiciously lower
|
|
contamination_flag = ratio > 1.5 # Threshold based on empirical studies
|
|
|
|
report = {
|
|
"benchmark_perplexity": {
|
|
"mean": benchmark_mean,
|
|
"std": np.std(benchmark_ppl),
|
|
"min": np.min(benchmark_ppl),
|
|
"max": np.max(benchmark_ppl),
|
|
},
|
|
"baseline_perplexity": {
|
|
"mean": baseline_mean,
|
|
"std": np.std(baseline_ppl),
|
|
},
|
|
"ratio": ratio,
|
|
"contamination_suspected": contamination_flag,
|
|
}
|
|
|
|
return report
|
|
|
|
# Usage
|
|
detector = ContaminationDetector("meta-llama/Llama-2-7b-hf")
|
|
|
|
# Sample from MMLU test set
|
|
mmlu_samples = [
|
|
"Question: What is the capital of France?\nA. London\nB. Paris\nC. Berlin\nD. Madrid\nAnswer: B",
|
|
# ... more MMLU examples
|
|
]
|
|
|
|
# Baseline: Similar format but different questions
|
|
baseline_samples = [
|
|
"Question: What color is the sky?\nA. Red\nB. Blue\nC. Green\nD. Yellow\nAnswer: B",
|
|
# ... more baseline examples
|
|
]
|
|
|
|
contamination_report = detector.detect_contamination(mmlu_samples, baseline_samples)
|
|
print(contamination_report)</code></pre>
|
|
<h2>Anti-Patterns</h2>
|
|
<h3>Anti-Pattern 1: Benchmark Overfitting</h3>
|
|
<p><strong>Wrong</strong>: Optimizing exclusively for benchmark performance</p>
|
|
<pre><code class="lang-python"># BAD: Training specifically to improve MMLU score
|
|
train_on_mmlu_like_data() # Creates overfit model</code></pre>
|
|
<p><strong>Right</strong>: Use benchmarks as one signal among many</p>
|
|
<pre><code class="lang-python"># GOOD: Balanced evaluation strategy
|
|
benchmark_scores = evaluate_benchmarks(model)
|
|
domain_scores = evaluate_domain_tasks(model)
|
|
human_eval_scores = get_human_ratings(model)
|
|
|
|
overall_score = weighted_average([benchmark_scores, domain_scores, human_eval_scores])</code></pre>
|
|
<h3>Anti-Pattern 2: Ignoring Benchmark Limitations</h3>
|
|
<p><strong>Wrong</strong>: Treating benchmark scores as absolute truth</p>
|
|
<pre><code class="lang-python"># BAD: Assuming higher MMLU = better model for all use cases
|
|
if model_a_mmlu > model_b_mmlu:
|
|
deploy(model_a) # May not be better for your specific task</code></pre>
|
|
<p><strong>Right</strong>: Validate on target domain</p>
|
|
<pre><code class="lang-python"># GOOD: Test on actual use case
|
|
benchmark_scores = evaluate_benchmarks([model_a, model_b])
|
|
domain_scores = evaluate_on_real_data([model_a, model_b])
|
|
|
|
# Choose based on domain performance
|
|
best_model = max([model_a, model_b], key=lambda m: domain_scores[m])</code></pre>
|
|
<h3>Anti-Pattern 3: Inconsistent Evaluation Settings</h3>
|
|
<p><strong>Wrong</strong>: Comparing results with different configurations</p>
|
|
<pre><code class="lang-python"># BAD: Inconsistent few-shot settings
|
|
model_a_results = evaluate(model_a, num_fewshot=5)
|
|
model_b_results = evaluate(model_b, num_fewshot=0) # Not comparable!</code></pre>
|
|
<p><strong>Right</strong>: Standardize evaluation protocol</p>
|
|
<pre><code class="lang-python"># GOOD: Consistent settings
|
|
EVAL_CONFIG = {
|
|
"num_fewshot": 5,
|
|
"batch_size": 8,
|
|
"temperature": 0.0,
|
|
"tasks": ["mmlu", "hellaswag", "truthfulqa_mc2"],
|
|
}
|
|
|
|
model_a_results = evaluate(model_a, **EVAL_CONFIG)
|
|
model_b_results = evaluate(model_b, **EVAL_CONFIG)</code></pre>
|
|
<h2>Related Skills</h2>
|
|
<ul><li><code>llm-evaluation-frameworks.md</code>: Arize Phoenix, Braintrust, LangSmith for production evaluation</li><li><code>llm-as-judge.md</code>: Using LLMs to evaluate LLM outputs with Prometheus and G-Eval</li><li><code>rag-evaluation-metrics.md</code>: RAGAS metrics for RAG system evaluation</li><li><code>custom-llm-evaluation.md</code>: Domain-specific and custom evaluation metrics</li><li><code>dspy-evaluation.md</code>: DSPy metric functions and evaluation patterns</li></ul>
|
|
<h2>Summary</h2>
|
|
<p>LLM benchmark evaluation provides standardized, reproducible metrics for model capabilities:</p>
|
|
<p><strong>Key Takeaways</strong>:</p>
|
|
<ol><li><strong>Standard benchmarks</strong>: MMLU (knowledge), HumanEval (code), GSM8K (math), HellaSwag (common sense)</li><li><strong>Frameworks</strong>: lm-evaluation-harness (200+ tasks), LightEval (HuggingFace integration)</li><li><strong>Metrics</strong>: Accuracy, pass@k, exact match, F1 depending on task type</li><li><strong>Contamination risk</strong>: Always check for training data leakage using perplexity analysis</li><li><strong>Limitations</strong>: Benchmarks are proxies, not guarantees of real-world performance</li></ol>
|
|
<p><strong>Best Practices</strong>:</p>
|
|
<ul><li>Use consistent evaluation settings for fair comparisons</li><li>Combine multiple benchmarks covering different capabilities</li><li>Always validate on domain-specific tasks before deployment</li><li>Monitor for contamination using perplexity or membership inference</li><li>Report full evaluation details (num_fewshot, temperature, etc.) for reproducibility</li></ul>
|
|
<p><strong>When to combine with other skills</strong>:</p>
|
|
<ul><li>Use <code>llm-evaluation-frameworks.md</code> for production monitoring and tracing</li><li>Use <code>llm-as-judge.md</code> when benchmark metrics don't capture quality (e.g., creative writing)</li><li>Use <code>rag-evaluation-metrics.md</code> for retrieval-augmented generation systems</li><li>Use <code>custom-llm-evaluation.md</code> for domain-specific or safety evaluations</li></ul>
|
|
<footer>
|
|
<div class="fmap">
|
|
<div><b>Build</b>
|
|
<a href="https://hanzo.app">hanzo.app</a> app builder
|
|
<a href="https://hanzo.codes">hanzo.codes</a> code with an agent
|
|
<a href="https://hanzoskills.com">hanzoskills.com</a> what an agent reads</div>
|
|
<div><b>Run</b>
|
|
<a href="https://console.hanzo.ai">console.hanzo.ai</a> cloud console
|
|
<a href="https://api.hanzo.ai">api.hanzo.ai</a> the API</div>
|
|
<div><b>Account</b>
|
|
<a href="https://hanzo.id">hanzo.id</a> sign in
|
|
<a href="https://pay.hanzo.ai">pay.hanzo.ai</a> top up
|
|
<a href="https://billing.hanzo.ai">billing.hanzo.ai</a> invoices and usage</div>
|
|
<div><b>Company</b>
|
|
<a href="https://hanzo.ai">hanzo.ai</a> canonical
|
|
<a href="https://hanzo.ventures">hanzo.ventures</a> ventures</div>
|
|
</div>
|
|
<div class="fend"><a href="/ml-llm-benchmarks-evaluation.md">This page as plain text</a> · <a href="/llms.txt">Index for agents</a></div>
|
|
</footer>
|
|
</div></body></html> |