mirror of
https://github.com/hanzoai/skills.git
synced 2026-11-19 00:47:36 +00:00
hanzoskills.com/llms.txt now opens with the install line, the API reference index and the company index, so an agent that lands on any one of the three can find the rest. robots.txt claimed 537 documents while llms.txt counts 920. The number is gone from robots.txt; llms.txt derives it. public/ is regenerated from scripts/site.py, which it had fallen behind: the copy-prompt menu and agent bar were in the generator and not in the committed pages. tests/site.mjs passes against the result. Co-authored-by: Hanzo Dev <[email protected]>
663 lines
30 KiB
HTML
663 lines
30 KiB
HTML
<!doctype html>
|
|
<html lang="en"><head>
|
|
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
<title>ml-huggingface-datasets</title>
|
|
<meta name="color-scheme" content="dark">
|
|
<style>
|
|
/* Committed to one visual world: a dark instrument panel. No light variant —
|
|
the dot field and the hairline rules only read on black, and a naive
|
|
inversion would wash both out. Every colour is painted explicitly so the page
|
|
never borrows a host background. */
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Pixel Square';src:url('/fonts/ZenPixel-Square.woff2') format('woff2');font-style:normal;font-display:swap}
|
|
|
|
:root{
|
|
--ink:#f4f4f2; --soft:#9c9c98; --faint:#6a6a66; --ground:#000; --panel:#0a0a0a;
|
|
--line:#232323; --line-hi:#3a3a3a; --accent:#f4f4f2;
|
|
--font-zen-sans:'Zen',-apple-system,BlinkMacSystemFont,"Segoe UI",Helvetica,Arial,sans-serif;
|
|
--font-zen-mono:'Zen Mono',ui-monospace,"SF Mono",SFMono-Regular,Menlo,Consolas,monospace;
|
|
--font-zen-pixel-square:'Zen Pixel Square',monospace;
|
|
--mono:var(--font-zen-mono);
|
|
--sans:var(--font-zen-sans);
|
|
/* Controls are pills and surfaces are soft, as on hanzo.ai. */
|
|
--r:999px; --r-soft:14px;
|
|
}
|
|
*{box-sizing:border-box}
|
|
html{-webkit-text-size-adjust:100%}
|
|
body{margin:0;background:var(--ground);color:var(--ink);font-family:var(--sans);
|
|
font-size:16px;line-height:1.65;-webkit-font-smoothing:antialiased}
|
|
.wrap{max-width:60rem;margin:0 auto;padding:0 1.75rem}
|
|
|
|
/* The dot field. A halftone made of one repeating radial-gradient, faded out
|
|
along the diagonal by a mask so it reads as a wave rather than wallpaper. */
|
|
/* No overflow clip here: the dot field is inset to the box already, and a clip
|
|
would cut off the copy-prompt menu. */
|
|
.hero{position:relative;border-bottom:1px solid var(--line)}
|
|
.hero::before{content:"";position:absolute;inset:0;
|
|
background-image:radial-gradient(var(--line-hi) 1px, transparent 1px);
|
|
background-size:7px 7px;
|
|
-webkit-mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
pointer-events:none}
|
|
.hero .wrap{position:relative;padding-top:5.5rem;padding-bottom:4rem}
|
|
|
|
nav{border-bottom:1px solid var(--line)}
|
|
nav .in{max-width:60rem;margin:0 auto;padding:1.1rem 1.75rem;display:flex;
|
|
justify-content:space-between;align-items:center;gap:1rem}
|
|
.brand{font-family:var(--mono);font-size:.85rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--ink);text-decoration:none;
|
|
display:inline-flex;align-items:center;gap:.6rem}
|
|
.brand i{font-style:normal;color:var(--faint)}
|
|
.mark{width:1.15em;height:1.15em;display:block;flex:0 0 auto}
|
|
nav a.up{font-family:var(--mono);font-size:.72rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--soft);text-decoration:none}
|
|
nav a.up:hover{color:var(--ink)}
|
|
|
|
h1{font-family:var(--mono);font-size:clamp(2rem,5.4vw,3.4rem);line-height:1.12;
|
|
letter-spacing:-.02em;margin:0 0 1.1rem;font-weight:500;text-wrap:balance}
|
|
h1 em{font-style:normal;color:var(--faint)}
|
|
.lede{color:var(--soft);font-size:1.05rem;max-width:52ch;margin:0 0 2.25rem}
|
|
|
|
.btns{display:flex;flex-wrap:wrap;gap:.7rem;margin:0 0 .5rem;align-items:center}
|
|
.btn{display:inline-flex;align-items:center;gap:.5rem;font-family:var(--mono);
|
|
font-size:.72rem;letter-spacing:.14em;text-transform:uppercase;
|
|
padding:.8rem 1.4rem;border:1px solid var(--line-hi);border-radius:var(--r);
|
|
color:var(--ink);text-decoration:none;background:transparent;cursor:pointer;
|
|
line-height:1.2;font-weight:400}
|
|
.btn:hover{background:#141414;text-decoration:none;border-color:var(--soft)}
|
|
.btn.solid{background:var(--ink);color:#000;border-color:var(--ink)}
|
|
.btn.solid:hover{background:#d8d8d4}
|
|
.btn .chev{width:.7em;height:.7em;flex:0 0 auto;transition:transform .15s ease}
|
|
.btn[aria-expanded="true"] .chev{transform:rotate(180deg)}
|
|
|
|
/* Copy-prompt selector. Each row copies the prompt for its own target. */
|
|
.copy{position:relative;display:inline-flex}
|
|
.menu{position:absolute;top:calc(100% + .5rem);left:0;z-index:20;min-width:21rem;
|
|
background:var(--panel);border:1px solid var(--line-hi);border-radius:var(--r-soft);
|
|
padding:.4rem;display:flex;flex-direction:column;
|
|
box-shadow:0 12px 32px rgba(0,0,0,.75)}
|
|
.menu[hidden]{display:none}
|
|
.mi{display:flex;align-items:center;gap:.7rem;width:100%;text-align:left;
|
|
padding:.62rem .8rem;border:0;border-radius:calc(var(--r-soft) - .35rem);
|
|
background:transparent;color:var(--ink);font-family:var(--mono);font-size:.76rem;
|
|
letter-spacing:.04em;text-decoration:none;cursor:pointer;line-height:1.3;
|
|
white-space:nowrap}
|
|
.mi:hover,.mi:focus-visible{background:#181818;outline:none}
|
|
.mi .mark{width:1.05rem;height:1.05rem;color:var(--soft)}
|
|
.mi:hover .mark{color:var(--ink)}
|
|
.mi u{margin-left:auto;color:var(--faint);text-decoration:none;font-size:.68rem;
|
|
letter-spacing:.1em;text-transform:uppercase}
|
|
|
|
/* Works-with bar: one scrolling row, name and mark per agent. */
|
|
.strip{border-bottom:1px solid var(--line);background:var(--panel)}
|
|
.strip .in{max-width:60rem;margin:0 auto;padding:.9rem 1.75rem;display:flex;
|
|
align-items:center;gap:1.25rem;font-family:var(--mono);font-size:.72rem;
|
|
letter-spacing:.1em;text-transform:uppercase;color:var(--faint)}
|
|
.strip b{color:var(--soft);font-weight:400;flex:0 0 auto;
|
|
border-right:1px solid var(--line);padding-right:1.25rem;align-self:stretch;
|
|
display:flex;align-items:center}
|
|
.bar{display:flex;align-items:center;gap:.5rem;flex:1 1 auto;min-width:0;
|
|
overflow-x:auto;scrollbar-width:none;scroll-snap-type:x proximity;
|
|
-webkit-overflow-scrolling:touch;
|
|
-webkit-mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent);
|
|
mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent)}
|
|
.bar::-webkit-scrollbar{display:none}
|
|
.agent{display:inline-flex;align-items:center;gap:.45rem;flex:0 0 auto;
|
|
padding:.4rem .85rem;border:1px solid var(--line);border-radius:var(--r);
|
|
color:var(--soft);white-space:nowrap;scroll-snap-align:start;
|
|
text-decoration:none;background:var(--ground)}
|
|
.agent:hover{color:var(--ink);border-color:var(--line-hi);text-decoration:none}
|
|
.agent .mark{width:.95rem;height:.95rem}
|
|
|
|
h2{font-family:var(--mono);font-size:1.1rem;letter-spacing:-.01em;margin:2.75rem 0 .75rem;font-weight:500}
|
|
h3{font-size:1rem;margin:2rem 0 .5rem;font-weight:600}
|
|
h4,h5,h6{font-size:.94rem;margin:1.5rem 0 .5rem;font-weight:600}
|
|
p{margin:0 0 1rem;max-width:70ch}
|
|
ul,ol{margin:0 0 1rem;padding-left:1.35rem;max-width:70ch}
|
|
li{margin:.22rem 0}
|
|
a{color:var(--ink);text-decoration:underline;text-underline-offset:3px;
|
|
text-decoration-color:var(--line-hi)}
|
|
a:hover{text-decoration-color:var(--ink)}
|
|
a:focus-visible{outline:1px solid var(--ink);outline-offset:3px}
|
|
code{font-family:var(--mono);font-size:.85em;background:var(--panel);
|
|
border:1px solid var(--line);border-radius:5px;padding:.1em .36em}
|
|
pre{background:var(--panel);border:1px solid var(--line);padding:1rem 1.1rem;
|
|
border-radius:var(--r-soft);overflow-x:auto;margin:0 0 1.25rem}
|
|
pre code{background:none;border:0;padding:0;font-size:.82rem;line-height:1.7}
|
|
blockquote{margin:0 0 1rem;padding-left:1rem;border-left:1px solid var(--line-hi);color:var(--soft)}
|
|
hr{border:0;border-top:1px solid var(--line);margin:2.5rem 0}
|
|
table{border-collapse:collapse;width:100%;margin:0 0 1.25rem;font-size:.92rem;
|
|
display:block;overflow-x:auto}
|
|
th,td{border:1px solid var(--line);padding:.5rem .7rem;text-align:left}
|
|
|
|
.curl{display:block;background:var(--panel);border:1px solid var(--line);
|
|
border-radius:var(--r-soft);padding:.95rem 1.1rem;font-family:var(--mono);
|
|
font-size:.8rem;color:var(--soft);overflow-x:auto;margin:0 0 2.5rem;
|
|
white-space:pre;line-height:1.9}
|
|
.curl b{color:var(--ink);font-weight:400}
|
|
.curl i{font-style:normal;color:var(--faint)}
|
|
|
|
/* index */
|
|
#q{width:100%;padding:.85rem 1.2rem;font-family:var(--mono);font-size:.85rem;color:var(--ink);
|
|
background:var(--panel);border:1px solid var(--line);border-radius:var(--r);margin:0 0 .6rem}
|
|
#q::placeholder{color:var(--faint);text-transform:uppercase;letter-spacing:.1em;font-size:.72rem}
|
|
#q:focus{outline:none;border-color:var(--soft)}
|
|
#count{color:var(--faint);font-family:var(--mono);font-size:.72rem;letter-spacing:.1em;
|
|
text-transform:uppercase;margin:0 0 2rem}
|
|
.cat{margin:2.5rem 0 .5rem;font-family:var(--mono);font-size:.7rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);border-bottom:1px solid var(--line);
|
|
padding-bottom:.45rem}
|
|
.row{display:block;padding:.6rem .9rem;margin:0 -.9rem;border-bottom:1px solid var(--line);
|
|
border-radius:var(--r-soft);text-decoration:none}
|
|
.row:hover{background:#0a0a0a}
|
|
.nm{font-family:var(--mono);font-size:.84rem;color:var(--ink);display:block}
|
|
.ds{color:var(--soft);font-size:.9rem;display:block;margin-top:.12rem}
|
|
footer{margin-top:4rem;padding:2rem 0 4rem;border-top:1px solid var(--line);
|
|
color:var(--faint);font-size:.85rem}
|
|
footer a{color:var(--soft)}
|
|
.fmap{display:grid;gap:1.75rem;grid-template-columns:repeat(auto-fit,minmax(11rem,1fr));
|
|
margin:0 0 2rem;font-size:.85rem}
|
|
.fmap b{display:block;font-family:var(--mono);font-size:.68rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);font-weight:400;margin:0 0 .6rem;
|
|
border-bottom:1px solid var(--line);padding-bottom:.4rem}
|
|
.fmap a{display:inline-block;color:var(--ink);text-decoration:none;margin:.3rem .35rem 0 0}
|
|
.fmap a:hover{text-decoration:underline}
|
|
.fmap div{color:var(--faint);line-height:1.5}
|
|
.fend{border-top:1px solid var(--line);padding-top:1.25rem;font-size:.82rem}
|
|
|
|
@media (max-width:640px){
|
|
.hero .wrap{padding-top:3.25rem;padding-bottom:2.75rem}
|
|
.btns{gap:.5rem}
|
|
.btn{padding:.75rem 1.15rem;font-size:.68rem}
|
|
.copy,.copy .btn{width:100%;justify-content:center}
|
|
.menu{left:0;right:0;min-width:0}
|
|
.strip .in{flex-direction:column;align-items:flex-start;gap:.6rem;padding:.9rem 1.75rem}
|
|
.strip b{border-right:0;padding-right:0}
|
|
.bar{width:100%}
|
|
}
|
|
</style>
|
|
<link rel="alternate" type="text/markdown" href="/ml-huggingface-datasets.md">
|
|
|
|
<link rel="icon" href="/favicon.svg">
|
|
<link rel="canonical" href="https://hanzoskills.com/ml-huggingface-datasets">
|
|
</head>
|
|
<!-- Reading this with a machine? Every page here is also plain markdown.
|
|
This one: https://hanzoskills.com/ml-huggingface-datasets.md
|
|
Index: https://hanzoskills.com/llms.txt
|
|
Corpus: https://hanzoskills.com/all.md
|
|
Append .md to any path for the markdown twin. -->
|
|
<body>
|
|
<nav><div class="in">
|
|
<a class="brand" href="/"><svg class="mark" viewBox="0 0 67 67" fill="currentColor" aria-hidden="true"><path d="M22.21 67V44.6369H0V67H22.21Z"/><path d="M66.7038 22.3184H22.2534L0.0878906 44.6367H44.4634L66.7038 22.3184Z"/><path d="M22.21 0H0V22.3184H22.21V0Z"/><path d="M66.7198 0H44.5098V22.3184H66.7198V0Z"/><path d="M66.7198 67V44.6369H44.5098V67H66.7198Z"/></svg> HANZO<i>SKILLS</i></a>
|
|
<a class="up" href="/">All skills</a>
|
|
</div></nav>
|
|
|
|
<div class="wrap">
|
|
<h1>ml-huggingface-datasets</h1><p class="lede">Loading, processing, and managing datasets with HuggingFace datasets library</p><h1>HuggingFace Datasets</h1>
|
|
<p><strong>Scope</strong>: Loading datasets, preprocessing, custom datasets, streaming, caching, data collators <strong>Lines</strong>: ~325 <strong>Last Updated</strong>: 2025-10-25 <strong>Format Version</strong>: 1.0 (Atomic)</p>
|
|
<hr>
|
|
<h2>When to Use This Skill</h2>
|
|
<p>Activate this skill when:</p>
|
|
<ul><li>Loading datasets from HuggingFace Hub</li><li>Processing large datasets that don't fit in memory</li><li>Creating custom datasets from CSV, JSON, or other formats</li><li>Applying transformations to dataset columns</li><li>Filtering and selecting dataset subsets</li><li>Preparing data for model training</li><li>Managing dataset caching and storage</li><li>Using data collators for dynamic batching</li></ul>
|
|
<h2>Core Concepts</h2>
|
|
<h3>Dataset Structure</h3>
|
|
<p><strong>DatasetDict</strong>: Container for multiple splits</p>
|
|
<pre><code class="lang-python">from datasets import load_dataset
|
|
|
|
dataset = load_dataset("imdb")
|
|
# DatasetDict({
|
|
# train: Dataset({features: ['text', 'label'], num_rows: 25000})
|
|
# test: Dataset({features: ['text', 'label'], num_rows: 25000})
|
|
# })</code></pre>
|
|
<p><strong>Dataset</strong>: Single split with rows and columns</p>
|
|
<pre><code class="lang-python">train_dataset = dataset["train"]
|
|
# Dataset({
|
|
# features: ['text', 'label'],
|
|
# num_rows: 25000
|
|
# })</code></pre>
|
|
<p><strong>Features</strong>: Column types and metadata</p>
|
|
<ul><li><code>Value</code>: Single value (int, float, string)</li><li><code>ClassLabel</code>: Categorical labels</li><li><code>Sequence</code>: List of values</li><li><code>Image</code>: Image data</li><li><code>Audio</code>: Audio data</li></ul>
|
|
<h3>Loading Datasets</h3>
|
|
<p><strong>From Hub</strong>:</p>
|
|
<pre><code class="lang-python">from datasets import load_dataset
|
|
|
|
# Load entire dataset
|
|
dataset = load_dataset("squad")
|
|
|
|
# Load specific split
|
|
train_data = load_dataset("squad", split="train")
|
|
|
|
# Load specific configuration
|
|
dataset = load_dataset("glue", "mrpc")
|
|
|
|
# Streaming mode (for large datasets)
|
|
dataset = load_dataset("c4", "en", streaming=True)</code></pre>
|
|
<p><strong>From Local Files</strong>:</p>
|
|
<pre><code class="lang-python"># CSV
|
|
dataset = load_dataset("csv", data_files="data.csv")
|
|
|
|
# JSON
|
|
dataset = load_dataset("json", data_files="data.json")
|
|
|
|
# Multiple files
|
|
dataset = load_dataset("json", data_files={
|
|
"train": "train.json",
|
|
"test": "test.json"
|
|
})
|
|
|
|
# Parquet
|
|
dataset = load_dataset("parquet", data_files="data.parquet")</code></pre>
|
|
<h3>Dataset Processing</h3>
|
|
<p><strong>Map</strong>: Apply function to each example</p>
|
|
<pre><code class="lang-python">def tokenize(example):
|
|
return tokenizer(example["text"], truncation=True, padding="max_length")
|
|
|
|
dataset = dataset.map(tokenize, batched=True)</code></pre>
|
|
<p><strong>Filter</strong>: Keep examples matching condition</p>
|
|
<pre><code class="lang-python"># Filter by condition
|
|
dataset = dataset.filter(lambda x: len(x["text"]) > 100)
|
|
|
|
# Filter with index
|
|
dataset = dataset.filter(lambda x, i: i % 2 == 0, with_indices=True)</code></pre>
|
|
<p><strong>Select</strong>: Choose specific examples</p>
|
|
<pre><code class="lang-python"># Select by index
|
|
small_dataset = dataset.select(range(1000))
|
|
|
|
# Select random sample
|
|
sample = dataset.shuffle(seed=42).select(range(100))</code></pre>
|
|
<hr>
|
|
<h2>Patterns</h2>
|
|
<h3>Loading Common Datasets</h3>
|
|
<pre><code class="lang-python">from datasets import load_dataset
|
|
|
|
# Text classification
|
|
imdb = load_dataset("imdb")
|
|
sst2 = load_dataset("glue", "sst2")
|
|
|
|
# Question answering
|
|
squad = load_dataset("squad")
|
|
squad_v2 = load_dataset("squad_v2")
|
|
|
|
# Summarization
|
|
cnn_dailymail = load_dataset("cnn_dailymail", "3.0.0")
|
|
|
|
# Translation
|
|
wmt = load_dataset("wmt14", "de-en")
|
|
|
|
# Named Entity Recognition
|
|
conll = load_dataset("conll2003")
|
|
|
|
# Image classification
|
|
cifar10 = load_dataset("cifar10")
|
|
mnist = load_dataset("mnist")</code></pre>
|
|
<h3>Processing with Map</h3>
|
|
<pre><code class="lang-python">from datasets import load_dataset
|
|
from transformers import AutoTokenizer
|
|
|
|
dataset = load_dataset("imdb")
|
|
tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
|
|
|
|
# Simple mapping
|
|
def tokenize_function(examples):
|
|
return tokenizer(examples["text"], truncation=True, padding="max_length")
|
|
|
|
tokenized_dataset = dataset.map(tokenize_function, batched=True)
|
|
|
|
# Map with multiple columns
|
|
def preprocess(examples):
|
|
# Tokenize text
|
|
tokenized = tokenizer(examples["text"], truncation=True)
|
|
# Add custom field
|
|
tokenized["length"] = [len(text.split()) for text in examples["text"]]
|
|
return tokenized
|
|
|
|
processed = dataset.map(preprocess, batched=True)
|
|
|
|
# Remove original columns
|
|
processed = dataset.map(
|
|
tokenize_function,
|
|
batched=True,
|
|
remove_columns=dataset["train"].column_names
|
|
)
|
|
|
|
# Parallel processing
|
|
processed = dataset.map(
|
|
tokenize_function,
|
|
batched=True,
|
|
num_proc=4 # Use 4 processes
|
|
)</code></pre>
|
|
<h3>Filtering and Selecting</h3>
|
|
<pre><code class="lang-python">from datasets import load_dataset
|
|
|
|
dataset = load_dataset("imdb", split="train")
|
|
|
|
# Filter by length
|
|
long_reviews = dataset.filter(lambda x: len(x["text"]) > 500)
|
|
|
|
# Filter by label
|
|
positive_only = dataset.filter(lambda x: x["label"] == 1)
|
|
|
|
# Complex filtering
|
|
def is_quality_review(example):
|
|
return len(example["text"]) > 100 and example["text"].count(".") > 3
|
|
|
|
quality_reviews = dataset.filter(is_quality_review)
|
|
|
|
# Select first 1000 examples
|
|
subset = dataset.select(range(1000))
|
|
|
|
# Select random sample
|
|
sample = dataset.shuffle(seed=42).select(range(100))
|
|
|
|
# Train/test split
|
|
split_dataset = dataset.train_test_split(test_size=0.2, seed=42)
|
|
train_data = split_dataset["train"]
|
|
test_data = split_dataset["test"]</code></pre>
|
|
<h3>Creating Custom Datasets</h3>
|
|
<pre><code class="lang-python">from datasets import Dataset, DatasetDict
|
|
|
|
# From dictionary
|
|
data = {
|
|
"text": ["Example 1", "Example 2", "Example 3"],
|
|
"label": [0, 1, 0]
|
|
}
|
|
dataset = Dataset.from_dict(data)
|
|
|
|
# From pandas DataFrame
|
|
import pandas as pd
|
|
|
|
df = pd.DataFrame({
|
|
"text": ["Example 1", "Example 2"],
|
|
"label": [0, 1]
|
|
})
|
|
dataset = Dataset.from_pandas(df)
|
|
|
|
# From generator (for large data)
|
|
def data_generator():
|
|
for i in range(1000):
|
|
yield {"text": f"Example {i}", "label": i % 2}
|
|
|
|
dataset = Dataset.from_generator(data_generator)
|
|
|
|
# Create DatasetDict with multiple splits
|
|
dataset_dict = DatasetDict({
|
|
"train": Dataset.from_dict(train_data),
|
|
"test": Dataset.from_dict(test_data)
|
|
})
|
|
|
|
# From CSV
|
|
dataset = Dataset.from_csv("data.csv")
|
|
|
|
# From JSON lines
|
|
dataset = Dataset.from_json("data.jsonl")</code></pre>
|
|
<h3>Streaming Large Datasets</h3>
|
|
<pre><code class="lang-python">from datasets import load_dataset
|
|
|
|
# Load in streaming mode
|
|
dataset = load_dataset("c4", "en", streaming=True)
|
|
|
|
# Iterate through examples
|
|
for example in dataset["train"]:
|
|
print(example["text"])
|
|
break # Process first example
|
|
|
|
# Take first N examples
|
|
from itertools import islice
|
|
|
|
first_1000 = list(islice(dataset["train"], 1000))
|
|
|
|
# Filter streaming dataset
|
|
filtered = dataset["train"].filter(lambda x: len(x["text"]) > 100)
|
|
|
|
# Map streaming dataset
|
|
tokenized = dataset["train"].map(lambda x: tokenizer(x["text"]))
|
|
|
|
# Shuffle streaming dataset
|
|
shuffled = dataset["train"].shuffle(seed=42, buffer_size=10000)</code></pre>
|
|
<h3>Dataset Features and Schema</h3>
|
|
<pre><code class="lang-python">from datasets import Dataset, Features, Value, ClassLabel, Sequence
|
|
|
|
# Define custom features
|
|
features = Features({
|
|
"text": Value("string"),
|
|
"label": ClassLabel(names=["negative", "positive"]),
|
|
"tokens": Sequence(Value("string")),
|
|
"score": Value("float32")
|
|
})
|
|
|
|
# Create dataset with features
|
|
data = {
|
|
"text": ["Good movie", "Bad movie"],
|
|
"label": [1, 0],
|
|
"tokens": [["good", "movie"], ["bad", "movie"]],
|
|
"score": [0.9, 0.1]
|
|
}
|
|
dataset = Dataset.from_dict(data, features=features)
|
|
|
|
# Access features
|
|
print(dataset.features)
|
|
# {'text': Value(dtype='string'),
|
|
# 'label': ClassLabel(names=['negative', 'positive']),
|
|
# 'tokens': Sequence(Value(dtype='string')),
|
|
# 'score': Value(dtype='float32')}
|
|
|
|
# Convert labels to integers
|
|
label_int = dataset["label"][0] # 1
|
|
|
|
# Convert integers to label names
|
|
label_name = dataset.features["label"].int2str(1) # "positive"</code></pre>
|
|
<h3>Caching and Storage</h3>
|
|
<pre><code class="lang-python">from datasets import load_dataset, disable_caching, enable_caching
|
|
|
|
# Default: caching enabled (~/.cache/huggingface/datasets/)
|
|
|
|
# Disable caching
|
|
disable_caching()
|
|
dataset = load_dataset("imdb")
|
|
|
|
# Re-enable caching
|
|
enable_caching()
|
|
|
|
# Custom cache directory
|
|
dataset = load_dataset("imdb", cache_dir="/custom/cache/path")
|
|
|
|
# Save dataset to disk
|
|
dataset.save_to_disk("./my_dataset")
|
|
|
|
# Load from disk
|
|
from datasets import load_from_disk
|
|
dataset = load_from_disk("./my_dataset")
|
|
|
|
# Clear cache for specific dataset
|
|
dataset.cleanup_cache_files()</code></pre>
|
|
<h3>Data Collators</h3>
|
|
<pre><code class="lang-python">from transformers import DataCollatorWithPadding, DataCollatorForLanguageModeling
|
|
from datasets import load_dataset
|
|
from transformers import AutoTokenizer
|
|
|
|
tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
|
|
dataset = load_dataset("imdb", split="train[:1000]")
|
|
|
|
# Tokenize dataset
|
|
def tokenize(examples):
|
|
return tokenizer(examples["text"], truncation=True)
|
|
|
|
tokenized = dataset.map(tokenize, batched=True, remove_columns=dataset.column_names)
|
|
|
|
# Dynamic padding collator
|
|
data_collator = DataCollatorWithPadding(tokenizer=tokenizer)
|
|
|
|
# Use with DataLoader
|
|
from torch.utils.data import DataLoader
|
|
|
|
dataloader = DataLoader(
|
|
tokenized,
|
|
batch_size=8,
|
|
collate_fn=data_collator
|
|
)
|
|
|
|
# Masked language modeling collator
|
|
mlm_collator = DataCollatorForLanguageModeling(
|
|
tokenizer=tokenizer,
|
|
mlm=True,
|
|
mlm_probability=0.15
|
|
)</code></pre>
|
|
<h3>Combining and Interleaving Datasets</h3>
|
|
<pre><code class="lang-python">from datasets import load_dataset, concatenate_datasets, interleave_datasets
|
|
|
|
# Load multiple datasets
|
|
dataset1 = load_dataset("imdb", split="train")
|
|
dataset2 = load_dataset("yelp_polarity", split="train")
|
|
|
|
# Concatenate (stack vertically)
|
|
combined = concatenate_datasets([dataset1, dataset2])
|
|
|
|
# Interleave (alternate examples)
|
|
interleaved = interleave_datasets([dataset1, dataset2])
|
|
|
|
# Interleave with custom probabilities
|
|
interleaved = interleave_datasets(
|
|
[dataset1, dataset2],
|
|
probabilities=[0.7, 0.3], # 70% from dataset1, 30% from dataset2
|
|
seed=42
|
|
)
|
|
|
|
# Interleave with stopping strategy
|
|
interleaved = interleave_datasets(
|
|
[dataset1, dataset2],
|
|
stopping_strategy="all_exhausted" # or "first_exhausted"
|
|
)</code></pre>
|
|
<hr>
|
|
<h2>Quick Reference</h2>
|
|
<h3>Loading Methods</h3>
|
|
<pre><code>Method | Source | Example
|
|
------------------------|---------------------------|----------------------------------
|
|
load_dataset(name) | HuggingFace Hub | load_dataset("imdb")
|
|
load_dataset(csv) | CSV file | load_dataset("csv", data_files="data.csv")
|
|
load_dataset(json) | JSON/JSONL file | load_dataset("json", data_files="data.jsonl")
|
|
Dataset.from_dict() | Python dictionary | Dataset.from_dict({"text": [...], "label": [...]})
|
|
Dataset.from_pandas() | Pandas DataFrame | Dataset.from_pandas(df)
|
|
Dataset.from_generator()| Python generator | Dataset.from_generator(gen_fn)
|
|
load_from_disk() | Saved dataset on disk | load_from_disk("./dataset")</code></pre>
|
|
<h3>Processing Methods</h3>
|
|
<pre><code>Method | Purpose | Example
|
|
------------------------|---------------------------|----------------------------------
|
|
map() | Apply function to all | dataset.map(tokenize, batched=True)
|
|
filter() | Keep matching examples | dataset.filter(lambda x: x["label"] == 1)
|
|
select() | Choose by index | dataset.select(range(100))
|
|
shuffle() | Randomize order | dataset.shuffle(seed=42)
|
|
train_test_split() | Split into train/test | dataset.train_test_split(test_size=0.2)
|
|
rename_column() | Rename column | dataset.rename_column("text", "sentence")
|
|
remove_columns() | Drop columns | dataset.remove_columns(["col1", "col2"])
|
|
flatten() | Unnest nested columns | dataset.flatten()</code></pre>
|
|
<h3>Key Parameters</h3>
|
|
<pre><code>Parameter | Default | Description
|
|
----------------|------------|----------------------------------------
|
|
batched | False | Process examples in batches (faster)
|
|
num_proc | None | Number of parallel processes
|
|
remove_columns | None | Columns to remove after mapping
|
|
batch_size | 1000 | Batch size for batched=True
|
|
cache_file_name | None | Custom cache file path
|
|
load_from_cache | True | Use cached results if available</code></pre>
|
|
<h3>Key Guidelines</h3>
|
|
<pre><code>✅ DO: Use batched=True for faster processing
|
|
✅ DO: Stream large datasets that don't fit in memory
|
|
✅ DO: Remove unnecessary columns to save memory
|
|
✅ DO: Use num_proc for CPU-intensive preprocessing
|
|
✅ DO: Define features for custom datasets
|
|
✅ DO: Use data collators for dynamic padding
|
|
|
|
❌ DON'T: Load entire dataset if you only need a subset
|
|
❌ DON'T: Forget to set seed for reproducible shuffling
|
|
❌ DON'T: Keep original columns if not needed
|
|
❌ DON'T: Process without batching (much slower)</code></pre>
|
|
<hr>
|
|
<h2>Anti-Patterns</h2>
|
|
<h3>Critical Violations</h3>
|
|
<pre><code class="lang-python"># ❌ NEVER: Process without batching for large datasets
|
|
dataset = dataset.map(tokenize) # Processes one example at a time
|
|
|
|
# ✅ CORRECT: Use batched processing
|
|
dataset = dataset.map(tokenize, batched=True) # 10-100x faster</code></pre>
|
|
<p>❌ <strong>No batching</strong>: Extremely slow for large datasets ✅ <strong>Correct approach</strong>: Always use <code>batched=True</code> for transformations</p>
|
|
<h3>Common Mistakes</h3>
|
|
<pre><code class="lang-python"># ❌ Don't: Keep unnecessary columns
|
|
tokenized = dataset.map(tokenize, batched=True)
|
|
# Still has original "text" column, wastes memory
|
|
|
|
# ✅ Correct: Remove columns not needed for training
|
|
tokenized = dataset.map(
|
|
tokenize,
|
|
batched=True,
|
|
remove_columns=dataset.column_names
|
|
)</code></pre>
|
|
<p>❌ <strong>Keeping unused columns</strong>: Wastes memory ✅ <strong>Better</strong>: Remove columns after processing</p>
|
|
<pre><code class="lang-python"># ❌ Don't: Load entire dataset when you need subset
|
|
dataset = load_dataset("c4", "en") # Hundreds of GB!
|
|
small_dataset = dataset["train"].select(range(1000))
|
|
|
|
# ✅ Correct: Use streaming or split parameter
|
|
dataset = load_dataset("c4", "en", split="train[:1000]", streaming=True)</code></pre>
|
|
<p>❌ <strong>Loading full dataset unnecessarily</strong>: Memory waste ✅ <strong>Better</strong>: Use <code>split</code> parameter or streaming</p>
|
|
<pre><code class="lang-python"># ❌ Don't: Forget to set seed for shuffling
|
|
shuffled = dataset.shuffle()
|
|
# Different order each run, not reproducible
|
|
|
|
# ✅ Correct: Set seed
|
|
shuffled = dataset.shuffle(seed=42)</code></pre>
|
|
<p>❌ <strong>No shuffle seed</strong>: Results not reproducible ✅ <strong>Better</strong>: Always set seed for reproducibility</p>
|
|
<pre><code class="lang-python"># ❌ Don't: Process serially when you can parallelize
|
|
processed = dataset.map(expensive_function, batched=True)
|
|
# Uses single process
|
|
|
|
# ✅ Correct: Use multiple processes
|
|
processed = dataset.map(
|
|
expensive_function,
|
|
batched=True,
|
|
num_proc=4 # 4x faster on 4 cores
|
|
)</code></pre>
|
|
<p>❌ <strong>No parallelization</strong>: Slow preprocessing ✅ <strong>Better</strong>: Use <code>num_proc</code> for CPU-heavy tasks</p>
|
|
<pre><code class="lang-python"># ❌ Don't: Create dataset without features for custom data
|
|
dataset = Dataset.from_dict({
|
|
"text": ["Example"],
|
|
"label": ["positive"] # Should be integer/ClassLabel
|
|
})
|
|
|
|
# ✅ Correct: Define features
|
|
features = Features({
|
|
"text": Value("string"),
|
|
"label": ClassLabel(names=["negative", "positive"])
|
|
})
|
|
dataset = Dataset.from_dict(
|
|
{"text": ["Example"], "label": [1]},
|
|
features=features
|
|
)</code></pre>
|
|
<p>❌ <strong>Missing features</strong>: Type inconsistencies ✅ <strong>Better</strong>: Define features explicitly for custom datasets</p>
|
|
<hr>
|
|
<h2>Related Skills</h2>
|
|
<ul><li><code>ml/huggingface/huggingface-hub.md</code> - Uploading datasets to Hub</li><li><code>ml/huggingface/huggingface-transformers.md</code> - Using datasets with models</li><li><code>ml/huggingface/huggingface-autotrain.md</code> - Training with datasets</li><li><code>ml/llm-dataset-preparation.md</code> - Preparing high-quality training data</li><li><code>database/postgres-optimization.md</code> - Storing processed datasets</li><li><code>data/etl-patterns.md</code> - ETL workflows for datasets</li></ul>
|
|
<hr>
|
|
<p><strong>Last Updated</strong>: 2025-10-25 <strong>Format Version</strong>: 1.0 (Atomic)</p>
|
|
<footer>
|
|
<div class="fmap">
|
|
<div><b>Build</b>
|
|
<a href="https://hanzo.app">hanzo.app</a> app builder
|
|
<a href="https://hanzo.codes">hanzo.codes</a> code with an agent
|
|
<a href="https://hanzoskills.com">hanzoskills.com</a> what an agent reads</div>
|
|
<div><b>Run</b>
|
|
<a href="https://console.hanzo.ai">console.hanzo.ai</a> cloud console
|
|
<a href="https://api.hanzo.ai">api.hanzo.ai</a> the API</div>
|
|
<div><b>Account</b>
|
|
<a href="https://hanzo.id">hanzo.id</a> sign in
|
|
<a href="https://pay.hanzo.ai">pay.hanzo.ai</a> top up
|
|
<a href="https://billing.hanzo.ai">billing.hanzo.ai</a> invoices and usage</div>
|
|
<div><b>Company</b>
|
|
<a href="https://hanzo.ai">hanzo.ai</a> canonical
|
|
<a href="https://hanzo.ventures">hanzo.ventures</a> ventures</div>
|
|
</div>
|
|
<div class="fend"><a href="/ml-huggingface-datasets.md">This page as plain text</a> · <a href="/llms.txt">Index for agents</a></div>
|
|
</footer>
|
|
</div></body></html> |