mirror of
https://github.com/hanzoai/skills.git
synced 2026-11-19 00:47:36 +00:00
hanzoskills.com/llms.txt now opens with the install line, the API reference index and the company index, so an agent that lands on any one of the three can find the rest. robots.txt claimed 537 documents while llms.txt counts 920. The number is gone from robots.txt; llms.txt derives it. public/ is regenerated from scripts/site.py, which it had fallen behind: the copy-prompt menu and agent bar were in the generator and not in the committed pages. tests/site.mjs passes against the result. Co-authored-by: Hanzo Dev <[email protected]>
629 lines
30 KiB
HTML
629 lines
30 KiB
HTML
<!doctype html>
|
|
<html lang="en"><head>
|
|
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
<title>ml-huggingface-transformers</title>
|
|
<meta name="color-scheme" content="dark">
|
|
<style>
|
|
/* Committed to one visual world: a dark instrument panel. No light variant —
|
|
the dot field and the hairline rules only read on black, and a naive
|
|
inversion would wash both out. Every colour is painted explicitly so the page
|
|
never borrows a host background. */
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Pixel Square';src:url('/fonts/ZenPixel-Square.woff2') format('woff2');font-style:normal;font-display:swap}
|
|
|
|
:root{
|
|
--ink:#f4f4f2; --soft:#9c9c98; --faint:#6a6a66; --ground:#000; --panel:#0a0a0a;
|
|
--line:#232323; --line-hi:#3a3a3a; --accent:#f4f4f2;
|
|
--font-zen-sans:'Zen',-apple-system,BlinkMacSystemFont,"Segoe UI",Helvetica,Arial,sans-serif;
|
|
--font-zen-mono:'Zen Mono',ui-monospace,"SF Mono",SFMono-Regular,Menlo,Consolas,monospace;
|
|
--font-zen-pixel-square:'Zen Pixel Square',monospace;
|
|
--mono:var(--font-zen-mono);
|
|
--sans:var(--font-zen-sans);
|
|
/* Controls are pills and surfaces are soft, as on hanzo.ai. */
|
|
--r:999px; --r-soft:14px;
|
|
}
|
|
*{box-sizing:border-box}
|
|
html{-webkit-text-size-adjust:100%}
|
|
body{margin:0;background:var(--ground);color:var(--ink);font-family:var(--sans);
|
|
font-size:16px;line-height:1.65;-webkit-font-smoothing:antialiased}
|
|
.wrap{max-width:60rem;margin:0 auto;padding:0 1.75rem}
|
|
|
|
/* The dot field. A halftone made of one repeating radial-gradient, faded out
|
|
along the diagonal by a mask so it reads as a wave rather than wallpaper. */
|
|
/* No overflow clip here: the dot field is inset to the box already, and a clip
|
|
would cut off the copy-prompt menu. */
|
|
.hero{position:relative;border-bottom:1px solid var(--line)}
|
|
.hero::before{content:"";position:absolute;inset:0;
|
|
background-image:radial-gradient(var(--line-hi) 1px, transparent 1px);
|
|
background-size:7px 7px;
|
|
-webkit-mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
pointer-events:none}
|
|
.hero .wrap{position:relative;padding-top:5.5rem;padding-bottom:4rem}
|
|
|
|
nav{border-bottom:1px solid var(--line)}
|
|
nav .in{max-width:60rem;margin:0 auto;padding:1.1rem 1.75rem;display:flex;
|
|
justify-content:space-between;align-items:center;gap:1rem}
|
|
.brand{font-family:var(--mono);font-size:.85rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--ink);text-decoration:none;
|
|
display:inline-flex;align-items:center;gap:.6rem}
|
|
.brand i{font-style:normal;color:var(--faint)}
|
|
.mark{width:1.15em;height:1.15em;display:block;flex:0 0 auto}
|
|
nav a.up{font-family:var(--mono);font-size:.72rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--soft);text-decoration:none}
|
|
nav a.up:hover{color:var(--ink)}
|
|
|
|
h1{font-family:var(--mono);font-size:clamp(2rem,5.4vw,3.4rem);line-height:1.12;
|
|
letter-spacing:-.02em;margin:0 0 1.1rem;font-weight:500;text-wrap:balance}
|
|
h1 em{font-style:normal;color:var(--faint)}
|
|
.lede{color:var(--soft);font-size:1.05rem;max-width:52ch;margin:0 0 2.25rem}
|
|
|
|
.btns{display:flex;flex-wrap:wrap;gap:.7rem;margin:0 0 .5rem;align-items:center}
|
|
.btn{display:inline-flex;align-items:center;gap:.5rem;font-family:var(--mono);
|
|
font-size:.72rem;letter-spacing:.14em;text-transform:uppercase;
|
|
padding:.8rem 1.4rem;border:1px solid var(--line-hi);border-radius:var(--r);
|
|
color:var(--ink);text-decoration:none;background:transparent;cursor:pointer;
|
|
line-height:1.2;font-weight:400}
|
|
.btn:hover{background:#141414;text-decoration:none;border-color:var(--soft)}
|
|
.btn.solid{background:var(--ink);color:#000;border-color:var(--ink)}
|
|
.btn.solid:hover{background:#d8d8d4}
|
|
.btn .chev{width:.7em;height:.7em;flex:0 0 auto;transition:transform .15s ease}
|
|
.btn[aria-expanded="true"] .chev{transform:rotate(180deg)}
|
|
|
|
/* Copy-prompt selector. Each row copies the prompt for its own target. */
|
|
.copy{position:relative;display:inline-flex}
|
|
.menu{position:absolute;top:calc(100% + .5rem);left:0;z-index:20;min-width:21rem;
|
|
background:var(--panel);border:1px solid var(--line-hi);border-radius:var(--r-soft);
|
|
padding:.4rem;display:flex;flex-direction:column;
|
|
box-shadow:0 12px 32px rgba(0,0,0,.75)}
|
|
.menu[hidden]{display:none}
|
|
.mi{display:flex;align-items:center;gap:.7rem;width:100%;text-align:left;
|
|
padding:.62rem .8rem;border:0;border-radius:calc(var(--r-soft) - .35rem);
|
|
background:transparent;color:var(--ink);font-family:var(--mono);font-size:.76rem;
|
|
letter-spacing:.04em;text-decoration:none;cursor:pointer;line-height:1.3;
|
|
white-space:nowrap}
|
|
.mi:hover,.mi:focus-visible{background:#181818;outline:none}
|
|
.mi .mark{width:1.05rem;height:1.05rem;color:var(--soft)}
|
|
.mi:hover .mark{color:var(--ink)}
|
|
.mi u{margin-left:auto;color:var(--faint);text-decoration:none;font-size:.68rem;
|
|
letter-spacing:.1em;text-transform:uppercase}
|
|
|
|
/* Works-with bar: one scrolling row, name and mark per agent. */
|
|
.strip{border-bottom:1px solid var(--line);background:var(--panel)}
|
|
.strip .in{max-width:60rem;margin:0 auto;padding:.9rem 1.75rem;display:flex;
|
|
align-items:center;gap:1.25rem;font-family:var(--mono);font-size:.72rem;
|
|
letter-spacing:.1em;text-transform:uppercase;color:var(--faint)}
|
|
.strip b{color:var(--soft);font-weight:400;flex:0 0 auto;
|
|
border-right:1px solid var(--line);padding-right:1.25rem;align-self:stretch;
|
|
display:flex;align-items:center}
|
|
.bar{display:flex;align-items:center;gap:.5rem;flex:1 1 auto;min-width:0;
|
|
overflow-x:auto;scrollbar-width:none;scroll-snap-type:x proximity;
|
|
-webkit-overflow-scrolling:touch;
|
|
-webkit-mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent);
|
|
mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent)}
|
|
.bar::-webkit-scrollbar{display:none}
|
|
.agent{display:inline-flex;align-items:center;gap:.45rem;flex:0 0 auto;
|
|
padding:.4rem .85rem;border:1px solid var(--line);border-radius:var(--r);
|
|
color:var(--soft);white-space:nowrap;scroll-snap-align:start;
|
|
text-decoration:none;background:var(--ground)}
|
|
.agent:hover{color:var(--ink);border-color:var(--line-hi);text-decoration:none}
|
|
.agent .mark{width:.95rem;height:.95rem}
|
|
|
|
h2{font-family:var(--mono);font-size:1.1rem;letter-spacing:-.01em;margin:2.75rem 0 .75rem;font-weight:500}
|
|
h3{font-size:1rem;margin:2rem 0 .5rem;font-weight:600}
|
|
h4,h5,h6{font-size:.94rem;margin:1.5rem 0 .5rem;font-weight:600}
|
|
p{margin:0 0 1rem;max-width:70ch}
|
|
ul,ol{margin:0 0 1rem;padding-left:1.35rem;max-width:70ch}
|
|
li{margin:.22rem 0}
|
|
a{color:var(--ink);text-decoration:underline;text-underline-offset:3px;
|
|
text-decoration-color:var(--line-hi)}
|
|
a:hover{text-decoration-color:var(--ink)}
|
|
a:focus-visible{outline:1px solid var(--ink);outline-offset:3px}
|
|
code{font-family:var(--mono);font-size:.85em;background:var(--panel);
|
|
border:1px solid var(--line);border-radius:5px;padding:.1em .36em}
|
|
pre{background:var(--panel);border:1px solid var(--line);padding:1rem 1.1rem;
|
|
border-radius:var(--r-soft);overflow-x:auto;margin:0 0 1.25rem}
|
|
pre code{background:none;border:0;padding:0;font-size:.82rem;line-height:1.7}
|
|
blockquote{margin:0 0 1rem;padding-left:1rem;border-left:1px solid var(--line-hi);color:var(--soft)}
|
|
hr{border:0;border-top:1px solid var(--line);margin:2.5rem 0}
|
|
table{border-collapse:collapse;width:100%;margin:0 0 1.25rem;font-size:.92rem;
|
|
display:block;overflow-x:auto}
|
|
th,td{border:1px solid var(--line);padding:.5rem .7rem;text-align:left}
|
|
|
|
.curl{display:block;background:var(--panel);border:1px solid var(--line);
|
|
border-radius:var(--r-soft);padding:.95rem 1.1rem;font-family:var(--mono);
|
|
font-size:.8rem;color:var(--soft);overflow-x:auto;margin:0 0 2.5rem;
|
|
white-space:pre;line-height:1.9}
|
|
.curl b{color:var(--ink);font-weight:400}
|
|
.curl i{font-style:normal;color:var(--faint)}
|
|
|
|
/* index */
|
|
#q{width:100%;padding:.85rem 1.2rem;font-family:var(--mono);font-size:.85rem;color:var(--ink);
|
|
background:var(--panel);border:1px solid var(--line);border-radius:var(--r);margin:0 0 .6rem}
|
|
#q::placeholder{color:var(--faint);text-transform:uppercase;letter-spacing:.1em;font-size:.72rem}
|
|
#q:focus{outline:none;border-color:var(--soft)}
|
|
#count{color:var(--faint);font-family:var(--mono);font-size:.72rem;letter-spacing:.1em;
|
|
text-transform:uppercase;margin:0 0 2rem}
|
|
.cat{margin:2.5rem 0 .5rem;font-family:var(--mono);font-size:.7rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);border-bottom:1px solid var(--line);
|
|
padding-bottom:.45rem}
|
|
.row{display:block;padding:.6rem .9rem;margin:0 -.9rem;border-bottom:1px solid var(--line);
|
|
border-radius:var(--r-soft);text-decoration:none}
|
|
.row:hover{background:#0a0a0a}
|
|
.nm{font-family:var(--mono);font-size:.84rem;color:var(--ink);display:block}
|
|
.ds{color:var(--soft);font-size:.9rem;display:block;margin-top:.12rem}
|
|
footer{margin-top:4rem;padding:2rem 0 4rem;border-top:1px solid var(--line);
|
|
color:var(--faint);font-size:.85rem}
|
|
footer a{color:var(--soft)}
|
|
.fmap{display:grid;gap:1.75rem;grid-template-columns:repeat(auto-fit,minmax(11rem,1fr));
|
|
margin:0 0 2rem;font-size:.85rem}
|
|
.fmap b{display:block;font-family:var(--mono);font-size:.68rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);font-weight:400;margin:0 0 .6rem;
|
|
border-bottom:1px solid var(--line);padding-bottom:.4rem}
|
|
.fmap a{display:inline-block;color:var(--ink);text-decoration:none;margin:.3rem .35rem 0 0}
|
|
.fmap a:hover{text-decoration:underline}
|
|
.fmap div{color:var(--faint);line-height:1.5}
|
|
.fend{border-top:1px solid var(--line);padding-top:1.25rem;font-size:.82rem}
|
|
|
|
@media (max-width:640px){
|
|
.hero .wrap{padding-top:3.25rem;padding-bottom:2.75rem}
|
|
.btns{gap:.5rem}
|
|
.btn{padding:.75rem 1.15rem;font-size:.68rem}
|
|
.copy,.copy .btn{width:100%;justify-content:center}
|
|
.menu{left:0;right:0;min-width:0}
|
|
.strip .in{flex-direction:column;align-items:flex-start;gap:.6rem;padding:.9rem 1.75rem}
|
|
.strip b{border-right:0;padding-right:0}
|
|
.bar{width:100%}
|
|
}
|
|
</style>
|
|
<link rel="alternate" type="text/markdown" href="/ml-huggingface-transformers.md">
|
|
|
|
<link rel="icon" href="/favicon.svg">
|
|
<link rel="canonical" href="https://hanzoskills.com/ml-huggingface-transformers">
|
|
</head>
|
|
<!-- Reading this with a machine? Every page here is also plain markdown.
|
|
This one: https://hanzoskills.com/ml-huggingface-transformers.md
|
|
Index: https://hanzoskills.com/llms.txt
|
|
Corpus: https://hanzoskills.com/all.md
|
|
Append .md to any path for the markdown twin. -->
|
|
<body>
|
|
<nav><div class="in">
|
|
<a class="brand" href="/"><svg class="mark" viewBox="0 0 67 67" fill="currentColor" aria-hidden="true"><path d="M22.21 67V44.6369H0V67H22.21Z"/><path d="M66.7038 22.3184H22.2534L0.0878906 44.6367H44.4634L66.7038 22.3184Z"/><path d="M22.21 0H0V22.3184H22.21V0Z"/><path d="M66.7198 0H44.5098V22.3184H66.7198V0Z"/><path d="M66.7198 67V44.6369H44.5098V67H66.7198Z"/></svg> HANZO<i>SKILLS</i></a>
|
|
<a class="up" href="/">All skills</a>
|
|
</div></nav>
|
|
|
|
<div class="wrap">
|
|
<h1>ml-huggingface-transformers</h1><p class="lede">Loading models, pipelines, inference, and tokenization with transformers library</p><h1>HuggingFace Transformers</h1>
|
|
<p><strong>Scope</strong>: Loading models, pipelines, inference, tokenizers, quantization, custom configurations <strong>Lines</strong>: ~365 <strong>Last Updated</strong>: 2025-10-25 <strong>Format Version</strong>: 1.0 (Atomic)</p>
|
|
<hr>
|
|
<h2>When to Use This Skill</h2>
|
|
<p>Activate this skill when:</p>
|
|
<ul><li>Loading pre-trained models for inference or fine-tuning</li><li>Using high-level pipelines for common NLP tasks</li><li>Implementing custom inference logic with models</li><li>Working with tokenizers for text preprocessing</li><li>Quantizing models for reduced memory usage</li><li>Configuring model architectures and parameters</li><li>Batching predictions for efficiency</li><li>Accelerating inference with GPU</li></ul>
|
|
<h2>Core Concepts</h2>
|
|
<h3>Auto Classes</h3>
|
|
<p><strong>Purpose</strong>: Automatically detect and load correct model/tokenizer classes</p>
|
|
<p><strong>Main Classes</strong>:</p>
|
|
<ul><li><code>AutoModel</code>: Base model without task-specific head</li><li><code>AutoModelForSequenceClassification</code>: Text classification</li><li><code>AutoModelForCausalLM</code>: Text generation (GPT-style)</li><li><code>AutoModelForSeq2SeqLM</code>: Sequence-to-sequence (T5, BART)</li><li><code>AutoTokenizer</code>: Tokenizer for any model</li></ul>
|
|
<pre><code class="lang-python">from transformers import AutoModel, AutoTokenizer
|
|
|
|
# Automatically loads correct classes
|
|
model = AutoModel.from_pretrained("bert-base-uncased")
|
|
tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")</code></pre>
|
|
<h3>Pipelines</h3>
|
|
<p><strong>High-Level API</strong>: One-line inference for common tasks</p>
|
|
<p><strong>Available Tasks</strong>:</p>
|
|
<ul><li><code>text-generation</code>: Generate text continuations</li><li><code>text-classification</code>: Classify text sentiment/topics</li><li><code>question-answering</code>: Answer questions from context</li><li><code>fill-mask</code>: Fill in masked tokens</li><li><code>summarization</code>: Summarize long text</li><li><code>translation</code>: Translate between languages</li><li><code>zero-shot-classification</code>: Classify without training</li><li><code>feature-extraction</code>: Get embeddings</li></ul>
|
|
<pre><code class="lang-python">from transformers import pipeline
|
|
|
|
# Create pipeline
|
|
classifier = pipeline("text-classification", model="distilbert-base-uncased-finetuned-sst-2-english")
|
|
|
|
# Single prediction
|
|
result = classifier("I love this product!")
|
|
# [{'label': 'POSITIVE', 'score': 0.9998}]
|
|
|
|
# Batch predictions
|
|
results = classifier(["Great!", "Terrible!", "It's okay"])</code></pre>
|
|
<h3>Tokenization</h3>
|
|
<p><strong>Process</strong>: Convert text to model inputs</p>
|
|
<p><strong>Key Operations</strong>:</p>
|
|
<ul><li>Encoding: Text → token IDs</li><li>Decoding: Token IDs → text</li><li>Special tokens: [CLS], [SEP], [PAD], [MASK]</li><li>Padding and truncation: Handle variable lengths</li></ul>
|
|
<pre><code class="lang-python">from transformers import AutoTokenizer
|
|
|
|
tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
|
|
|
|
# Encode text
|
|
inputs = tokenizer("Hello, world!", return_tensors="pt")
|
|
# {'input_ids': tensor([[101, 7592, 1010, 2088, 999, 102]]),
|
|
# 'attention_mask': tensor([[1, 1, 1, 1, 1, 1]])}
|
|
|
|
# Decode back to text
|
|
text = tokenizer.decode(inputs["input_ids"][0])
|
|
# "[CLS] hello, world! [SEP]"</code></pre>
|
|
<h3>Model Inference</h3>
|
|
<p><strong>Forward Pass</strong>: Get model predictions</p>
|
|
<pre><code class="lang-python">import torch
|
|
from transformers import AutoModelForSequenceClassification, AutoTokenizer
|
|
|
|
model = AutoModelForSequenceClassification.from_pretrained("distilbert-base-uncased-finetuned-sst-2-english")
|
|
tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased-finetuned-sst-2-english")
|
|
|
|
# Tokenize input
|
|
inputs = tokenizer("I love this!", return_tensors="pt")
|
|
|
|
# Get predictions
|
|
with torch.no_grad():
|
|
outputs = model(**inputs)
|
|
|
|
# Extract logits and probabilities
|
|
logits = outputs.logits
|
|
probs = torch.softmax(logits, dim=-1)
|
|
predicted_class = torch.argmax(probs, dim=-1).item()</code></pre>
|
|
<hr>
|
|
<h2>Patterns</h2>
|
|
<h3>Basic Model Loading</h3>
|
|
<pre><code class="lang-python">from transformers import AutoModel, AutoTokenizer
|
|
|
|
# Load model and tokenizer
|
|
model_name = "bert-base-uncased"
|
|
model = AutoModel.from_pretrained(model_name)
|
|
tokenizer = AutoTokenizer.from_pretrained(model_name)
|
|
|
|
# Load to GPU
|
|
model = model.to("cuda")
|
|
|
|
# Load with specific dtype
|
|
model = AutoModel.from_pretrained(model_name, torch_dtype=torch.float16)
|
|
|
|
# Load from local path
|
|
model = AutoModel.from_pretrained("/path/to/local/model")</code></pre>
|
|
<h3>Using Pipelines</h3>
|
|
<pre><code class="lang-python">from transformers import pipeline
|
|
|
|
# Text generation
|
|
generator = pipeline("text-generation", model="gpt2")
|
|
output = generator("Once upon a time", max_length=50, num_return_sequences=2)
|
|
|
|
# Question answering
|
|
qa = pipeline("question-answering", model="distilbert-base-cased-distilled-squad")
|
|
result = qa(
|
|
question="What is Python?",
|
|
context="Python is a high-level programming language known for its simplicity."
|
|
)
|
|
# {'answer': 'a high-level programming language', 'score': 0.98, 'start': 10, 'end': 43}
|
|
|
|
# Summarization
|
|
summarizer = pipeline("summarization", model="facebook/bart-large-cnn")
|
|
summary = summarizer(
|
|
"Long article text here...",
|
|
max_length=130,
|
|
min_length=30,
|
|
do_sample=False
|
|
)
|
|
|
|
# Zero-shot classification
|
|
classifier = pipeline("zero-shot-classification", model="facebook/bart-large-mnli")
|
|
result = classifier(
|
|
"This is a story about sports",
|
|
candidate_labels=["sports", "politics", "technology"]
|
|
)</code></pre>
|
|
<h3>Custom Inference</h3>
|
|
<pre><code class="lang-python">import torch
|
|
from transformers import AutoModelForCausalLM, AutoTokenizer
|
|
|
|
# Load model and tokenizer
|
|
model = AutoModelForCausalLM.from_pretrained("gpt2")
|
|
tokenizer = AutoTokenizer.from_pretrained("gpt2")
|
|
|
|
# Prepare input
|
|
prompt = "The future of AI is"
|
|
inputs = tokenizer(prompt, return_tensors="pt")
|
|
|
|
# Generate text
|
|
with torch.no_grad():
|
|
outputs = model.generate(
|
|
inputs["input_ids"],
|
|
max_length=50,
|
|
num_beams=5,
|
|
temperature=0.7,
|
|
top_k=50,
|
|
top_p=0.95,
|
|
do_sample=True,
|
|
pad_token_id=tokenizer.eos_token_id
|
|
)
|
|
|
|
# Decode output
|
|
generated_text = tokenizer.decode(outputs[0], skip_special_tokens=True)
|
|
print(generated_text)</code></pre>
|
|
<h3>Batch Processing</h3>
|
|
<pre><code class="lang-python">from transformers import AutoModelForSequenceClassification, AutoTokenizer
|
|
import torch
|
|
|
|
model = AutoModelForSequenceClassification.from_pretrained("distilbert-base-uncased-finetuned-sst-2-english")
|
|
tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased-finetuned-sst-2-english")
|
|
|
|
# Batch of texts
|
|
texts = [
|
|
"I love this product!",
|
|
"This is terrible.",
|
|
"It's okay, I guess."
|
|
]
|
|
|
|
# Tokenize with padding
|
|
inputs = tokenizer(texts, padding=True, truncation=True, return_tensors="pt")
|
|
|
|
# Batch inference
|
|
with torch.no_grad():
|
|
outputs = model(**inputs)
|
|
probs = torch.softmax(outputs.logits, dim=-1)
|
|
|
|
# Process results
|
|
for i, text in enumerate(texts):
|
|
positive_prob = probs[i][1].item()
|
|
print(f"{text}: {positive_prob:.2%} positive")</code></pre>
|
|
<h3>Model Quantization</h3>
|
|
<pre><code class="lang-python">from transformers import AutoModelForCausalLM, BitsAndBytesConfig
|
|
|
|
# 4-bit quantization config
|
|
quantization_config = BitsAndBytesConfig(
|
|
load_in_4bit=True,
|
|
bnb_4bit_compute_dtype=torch.float16,
|
|
bnb_4bit_use_double_quant=True,
|
|
bnb_4bit_quant_type="nf4"
|
|
)
|
|
|
|
# Load quantized model
|
|
model = AutoModelForCausalLM.from_pretrained(
|
|
"meta-llama/Llama-2-7b-hf",
|
|
quantization_config=quantization_config,
|
|
device_map="auto"
|
|
)
|
|
|
|
# 8-bit quantization (simpler)
|
|
model = AutoModelForCausalLM.from_pretrained(
|
|
"meta-llama/Llama-2-7b-hf",
|
|
load_in_8bit=True,
|
|
device_map="auto"
|
|
)</code></pre>
|
|
<h3>Working with Tokenizers</h3>
|
|
<pre><code class="lang-python">from transformers import AutoTokenizer
|
|
|
|
tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
|
|
|
|
# Basic encoding
|
|
tokens = tokenizer.encode("Hello, world!")
|
|
# [101, 7592, 1010, 2088, 999, 102]
|
|
|
|
# Encoding with special tokens control
|
|
tokens = tokenizer.encode("Hello, world!", add_special_tokens=False)
|
|
# [7592, 1010, 2088, 999]
|
|
|
|
# Batch encoding with padding
|
|
texts = ["Short text", "This is a longer text example"]
|
|
encoded = tokenizer(
|
|
texts,
|
|
padding=True,
|
|
truncation=True,
|
|
max_length=512,
|
|
return_tensors="pt"
|
|
)
|
|
|
|
# Get attention mask
|
|
attention_mask = encoded["attention_mask"]
|
|
|
|
# Decode with special tokens
|
|
text = tokenizer.decode(tokens, skip_special_tokens=False)
|
|
# "[CLS] hello, world! [SEP]"
|
|
|
|
# Decode without special tokens
|
|
text = tokenizer.decode(tokens, skip_special_tokens=True)
|
|
# "hello, world!"
|
|
|
|
# Token to string mapping
|
|
token_str = tokenizer.convert_ids_to_tokens([7592, 1010, 2088])
|
|
# ['hello', ',', 'world']</code></pre>
|
|
<h3>GPU Acceleration</h3>
|
|
<pre><code class="lang-python">import torch
|
|
from transformers import AutoModel, AutoTokenizer
|
|
|
|
# Check GPU availability
|
|
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
|
|
|
# Load model to GPU
|
|
model = AutoModel.from_pretrained("bert-base-uncased").to(device)
|
|
tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
|
|
|
|
# Process on GPU
|
|
text = "Example text"
|
|
inputs = tokenizer(text, return_tensors="pt").to(device)
|
|
|
|
with torch.no_grad():
|
|
outputs = model(**inputs)
|
|
|
|
# Move outputs back to CPU if needed
|
|
embeddings = outputs.last_hidden_state.cpu()
|
|
|
|
# Multi-GPU with device_map
|
|
from transformers import AutoModelForCausalLM
|
|
|
|
model = AutoModelForCausalLM.from_pretrained(
|
|
"meta-llama/Llama-2-7b-hf",
|
|
device_map="auto", # Automatically distribute across GPUs
|
|
torch_dtype=torch.float16
|
|
)</code></pre>
|
|
<h3>Custom Model Configuration</h3>
|
|
<pre><code class="lang-python">from transformers import AutoConfig, AutoModel
|
|
|
|
# Load default config
|
|
config = AutoConfig.from_pretrained("bert-base-uncased")
|
|
|
|
# Modify config
|
|
config.num_hidden_layers = 6 # Reduce layers
|
|
config.hidden_size = 512 # Reduce hidden size
|
|
config.num_attention_heads = 8
|
|
|
|
# Create model from custom config
|
|
model = AutoModel.from_config(config)
|
|
|
|
# Or modify during loading
|
|
model = AutoModel.from_pretrained(
|
|
"bert-base-uncased",
|
|
num_hidden_layers=6,
|
|
hidden_size=512
|
|
)
|
|
|
|
# Save custom config
|
|
config.save_pretrained("./my-custom-bert")</code></pre>
|
|
<h3>Streaming Generation</h3>
|
|
<pre><code class="lang-python">from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
|
|
from threading import Thread
|
|
|
|
model = AutoModelForCausalLM.from_pretrained("gpt2")
|
|
tokenizer = AutoTokenizer.from_pretrained("gpt2")
|
|
|
|
prompt = "The future of AI"
|
|
inputs = tokenizer(prompt, return_tensors="pt")
|
|
|
|
# Create streamer
|
|
streamer = TextIteratorStreamer(tokenizer, skip_special_tokens=True)
|
|
|
|
# Generate in background thread
|
|
generation_kwargs = dict(
|
|
inputs["input_ids"],
|
|
streamer=streamer,
|
|
max_length=100
|
|
)
|
|
thread = Thread(target=model.generate, kwargs=generation_kwargs)
|
|
thread.start()
|
|
|
|
# Stream output
|
|
print(prompt, end="")
|
|
for new_text in streamer:
|
|
print(new_text, end="", flush=True)
|
|
thread.join()</code></pre>
|
|
<hr>
|
|
<h2>Quick Reference</h2>
|
|
<h3>Common Model Classes</h3>
|
|
<pre><code>Class | Task
|
|
------------------------------------|----------------------------------
|
|
AutoModel | Base model (embeddings)
|
|
AutoModelForSequenceClassification | Text classification
|
|
AutoModelForCausalLM | Text generation (GPT)
|
|
AutoModelForMaskedLM | Masked language modeling (BERT)
|
|
AutoModelForSeq2SeqLM | Sequence-to-sequence (T5, BART)
|
|
AutoModelForQuestionAnswering | Question answering
|
|
AutoModelForTokenClassification | Named entity recognition</code></pre>
|
|
<h3>Pipeline Tasks</h3>
|
|
<pre><code>Pipeline | Example Model | Use Case
|
|
--------------------------|----------------------------|---------------------------
|
|
text-generation | gpt2, llama | Generate text
|
|
text-classification | distilbert-sst2 | Sentiment, topics
|
|
question-answering | distilbert-squad | Answer questions
|
|
fill-mask | bert-base-uncased | Fill [MASK] tokens
|
|
summarization | facebook/bart-large-cnn | Summarize text
|
|
translation | Helsinki-NLP/opus-mt | Translate languages
|
|
zero-shot-classification | facebook/bart-large-mnli | Classify without training
|
|
feature-extraction | sentence-transformers | Get embeddings</code></pre>
|
|
<h3>Key Parameters</h3>
|
|
<pre><code>Parameter | Default | Description
|
|
--------------------|---------|----------------------------------------
|
|
max_length | varies | Maximum sequence length
|
|
num_beams | 1 | Beam search width (1 = greedy)
|
|
temperature | 1.0 | Sampling temperature (lower = focused)
|
|
top_k | 50 | Top-k sampling
|
|
top_p | 1.0 | Nucleus sampling threshold
|
|
do_sample | False | Use sampling vs greedy decoding
|
|
pad_token_id | None | Token ID for padding
|
|
eos_token_id | None | Token ID for end-of-sequence</code></pre>
|
|
<h3>Key Guidelines</h3>
|
|
<pre><code>✅ DO: Use pipelines for simple tasks
|
|
✅ DO: Batch inputs for efficiency
|
|
✅ DO: Move models to GPU when available
|
|
✅ DO: Use torch.no_grad() during inference
|
|
✅ DO: Quantize large models to save memory
|
|
✅ DO: Handle padding and truncation properly
|
|
|
|
❌ DON'T: Load models in loops (cache outside)
|
|
❌ DON'T: Forget to set pad_token_id for generation
|
|
❌ DON'T: Process one item at a time (use batches)
|
|
❌ DON'T: Keep models in float32 unnecessarily</code></pre>
|
|
<hr>
|
|
<h2>Anti-Patterns</h2>
|
|
<h3>Critical Violations</h3>
|
|
<pre><code class="lang-python"># ❌ NEVER: Load model in loop
|
|
for text in texts:
|
|
model = AutoModel.from_pretrained("bert-base-uncased") # Extremely slow!
|
|
result = model(text)
|
|
|
|
# ✅ CORRECT: Load once, reuse
|
|
model = AutoModel.from_pretrained("bert-base-uncased")
|
|
for text in texts:
|
|
result = model(text)</code></pre>
|
|
<p>❌ <strong>Loading models repeatedly</strong>: Wastes time and memory ✅ <strong>Correct approach</strong>: Load once, reuse for all predictions</p>
|
|
<h3>Common Mistakes</h3>
|
|
<pre><code class="lang-python"># ❌ Don't: Ignore device placement
|
|
model = AutoModel.from_pretrained("bert-base-uncased")
|
|
inputs = tokenizer(text, return_tensors="pt").to("cuda")
|
|
outputs = model(**inputs) # Error: model on CPU, inputs on GPU
|
|
|
|
# ✅ Correct: Match devices
|
|
model = AutoModel.from_pretrained("bert-base-uncased").to("cuda")
|
|
inputs = tokenizer(text, return_tensors="pt").to("cuda")
|
|
outputs = model(**inputs)</code></pre>
|
|
<p>❌ <strong>Device mismatch</strong>: Runtime errors ✅ <strong>Better</strong>: Ensure model and inputs on same device</p>
|
|
<pre><code class="lang-python"># ❌ Don't: Forget no_grad during inference
|
|
outputs = model(**inputs) # Builds computation graph unnecessarily
|
|
|
|
# ✅ Correct: Use no_grad
|
|
with torch.no_grad():
|
|
outputs = model(**inputs)</code></pre>
|
|
<p>❌ <strong>Missing no_grad</strong>: Wastes memory tracking gradients ✅ <strong>Better</strong>: Always use <code>torch.no_grad()</code> for inference</p>
|
|
<pre><code class="lang-python"># ❌ Don't: Process items one by one
|
|
results = []
|
|
for text in texts:
|
|
inputs = tokenizer(text, return_tensors="pt")
|
|
output = model(**inputs)
|
|
results.append(output)
|
|
|
|
# ✅ Correct: Batch processing
|
|
inputs = tokenizer(texts, padding=True, truncation=True, return_tensors="pt")
|
|
outputs = model(**inputs)</code></pre>
|
|
<p>❌ <strong>No batching</strong>: Slow inference, underutilized GPU ✅ <strong>Better</strong>: Batch inputs for 10-100x speedup</p>
|
|
<pre><code class="lang-python"># ❌ Don't: Forget pad_token_id for generation
|
|
outputs = model.generate(inputs)
|
|
# Warning: Setting `pad_token_id` to `eos_token_id`
|
|
|
|
# ✅ Correct: Set explicitly
|
|
outputs = model.generate(
|
|
inputs,
|
|
pad_token_id=tokenizer.eos_token_id
|
|
)</code></pre>
|
|
<p>❌ <strong>Missing pad_token_id</strong>: Warnings and potential issues ✅ <strong>Better</strong>: Always set for generation tasks</p>
|
|
<hr>
|
|
<h2>Related Skills</h2>
|
|
<ul><li><code>ml/huggingface/huggingface-hub.md</code> - Downloading models from Hub</li><li><code>ml/huggingface/huggingface-datasets.md</code> - Loading datasets for training</li><li><code>ml/huggingface/huggingface-autotrain.md</code> - Fine-tuning models</li><li><code>ml/unsloth-finetuning.md</code> - Fast fine-tuning with Unsloth</li><li><code>ml/lora-peft-techniques.md</code> - Parameter-efficient fine-tuning</li><li><code>modal/modal-gpu-workloads.md</code> - GPU deployment for inference</li></ul>
|
|
<hr>
|
|
<p><strong>Last Updated</strong>: 2025-10-25 <strong>Format Version</strong>: 1.0 (Atomic)</p>
|
|
<footer>
|
|
<div class="fmap">
|
|
<div><b>Build</b>
|
|
<a href="https://hanzo.app">hanzo.app</a> app builder
|
|
<a href="https://hanzo.codes">hanzo.codes</a> code with an agent
|
|
<a href="https://hanzoskills.com">hanzoskills.com</a> what an agent reads</div>
|
|
<div><b>Run</b>
|
|
<a href="https://console.hanzo.ai">console.hanzo.ai</a> cloud console
|
|
<a href="https://api.hanzo.ai">api.hanzo.ai</a> the API</div>
|
|
<div><b>Account</b>
|
|
<a href="https://hanzo.id">hanzo.id</a> sign in
|
|
<a href="https://pay.hanzo.ai">pay.hanzo.ai</a> top up
|
|
<a href="https://billing.hanzo.ai">billing.hanzo.ai</a> invoices and usage</div>
|
|
<div><b>Company</b>
|
|
<a href="https://hanzo.ai">hanzo.ai</a> canonical
|
|
<a href="https://hanzo.ventures">hanzo.ventures</a> ventures</div>
|
|
</div>
|
|
<div class="fend"><a href="/ml-huggingface-transformers.md">This page as plain text</a> · <a href="/llms.txt">Index for agents</a></div>
|
|
</footer>
|
|
</div></body></html> |