mirror of
https://github.com/hanzoai/skills.git
synced 2026-11-19 00:47:36 +00:00
hanzoskills.com/llms.txt now opens with the install line, the API reference index and the company index, so an agent that lands on any one of the three can find the rest. robots.txt claimed 537 documents while llms.txt counts 920. The number is gone from robots.txt; llms.txt derives it. public/ is regenerated from scripts/site.py, which it had fallen behind: the copy-prompt menu and agent bar were in the generator and not in the committed pages. tests/site.mjs passes against the result. Co-authored-by: Hanzo Dev <[email protected]>
605 lines
28 KiB
HTML
605 lines
28 KiB
HTML
<!doctype html>
|
|
<html lang="en"><head>
|
|
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
<title>ml-unsloth-finetuning</title>
|
|
<meta name="color-scheme" content="dark">
|
|
<style>
|
|
/* Committed to one visual world: a dark instrument panel. No light variant —
|
|
the dot field and the hairline rules only read on black, and a naive
|
|
inversion would wash both out. Every colour is painted explicitly so the page
|
|
never borrows a host background. */
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Pixel Square';src:url('/fonts/ZenPixel-Square.woff2') format('woff2');font-style:normal;font-display:swap}
|
|
|
|
:root{
|
|
--ink:#f4f4f2; --soft:#9c9c98; --faint:#6a6a66; --ground:#000; --panel:#0a0a0a;
|
|
--line:#232323; --line-hi:#3a3a3a; --accent:#f4f4f2;
|
|
--font-zen-sans:'Zen',-apple-system,BlinkMacSystemFont,"Segoe UI",Helvetica,Arial,sans-serif;
|
|
--font-zen-mono:'Zen Mono',ui-monospace,"SF Mono",SFMono-Regular,Menlo,Consolas,monospace;
|
|
--font-zen-pixel-square:'Zen Pixel Square',monospace;
|
|
--mono:var(--font-zen-mono);
|
|
--sans:var(--font-zen-sans);
|
|
/* Controls are pills and surfaces are soft, as on hanzo.ai. */
|
|
--r:999px; --r-soft:14px;
|
|
}
|
|
*{box-sizing:border-box}
|
|
html{-webkit-text-size-adjust:100%}
|
|
body{margin:0;background:var(--ground);color:var(--ink);font-family:var(--sans);
|
|
font-size:16px;line-height:1.65;-webkit-font-smoothing:antialiased}
|
|
.wrap{max-width:60rem;margin:0 auto;padding:0 1.75rem}
|
|
|
|
/* The dot field. A halftone made of one repeating radial-gradient, faded out
|
|
along the diagonal by a mask so it reads as a wave rather than wallpaper. */
|
|
/* No overflow clip here: the dot field is inset to the box already, and a clip
|
|
would cut off the copy-prompt menu. */
|
|
.hero{position:relative;border-bottom:1px solid var(--line)}
|
|
.hero::before{content:"";position:absolute;inset:0;
|
|
background-image:radial-gradient(var(--line-hi) 1px, transparent 1px);
|
|
background-size:7px 7px;
|
|
-webkit-mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
pointer-events:none}
|
|
.hero .wrap{position:relative;padding-top:5.5rem;padding-bottom:4rem}
|
|
|
|
nav{border-bottom:1px solid var(--line)}
|
|
nav .in{max-width:60rem;margin:0 auto;padding:1.1rem 1.75rem;display:flex;
|
|
justify-content:space-between;align-items:center;gap:1rem}
|
|
.brand{font-family:var(--mono);font-size:.85rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--ink);text-decoration:none;
|
|
display:inline-flex;align-items:center;gap:.6rem}
|
|
.brand i{font-style:normal;color:var(--faint)}
|
|
.mark{width:1.15em;height:1.15em;display:block;flex:0 0 auto}
|
|
nav a.up{font-family:var(--mono);font-size:.72rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--soft);text-decoration:none}
|
|
nav a.up:hover{color:var(--ink)}
|
|
|
|
h1{font-family:var(--mono);font-size:clamp(2rem,5.4vw,3.4rem);line-height:1.12;
|
|
letter-spacing:-.02em;margin:0 0 1.1rem;font-weight:500;text-wrap:balance}
|
|
h1 em{font-style:normal;color:var(--faint)}
|
|
.lede{color:var(--soft);font-size:1.05rem;max-width:52ch;margin:0 0 2.25rem}
|
|
|
|
.btns{display:flex;flex-wrap:wrap;gap:.7rem;margin:0 0 .5rem;align-items:center}
|
|
.btn{display:inline-flex;align-items:center;gap:.5rem;font-family:var(--mono);
|
|
font-size:.72rem;letter-spacing:.14em;text-transform:uppercase;
|
|
padding:.8rem 1.4rem;border:1px solid var(--line-hi);border-radius:var(--r);
|
|
color:var(--ink);text-decoration:none;background:transparent;cursor:pointer;
|
|
line-height:1.2;font-weight:400}
|
|
.btn:hover{background:#141414;text-decoration:none;border-color:var(--soft)}
|
|
.btn.solid{background:var(--ink);color:#000;border-color:var(--ink)}
|
|
.btn.solid:hover{background:#d8d8d4}
|
|
.btn .chev{width:.7em;height:.7em;flex:0 0 auto;transition:transform .15s ease}
|
|
.btn[aria-expanded="true"] .chev{transform:rotate(180deg)}
|
|
|
|
/* Copy-prompt selector. Each row copies the prompt for its own target. */
|
|
.copy{position:relative;display:inline-flex}
|
|
.menu{position:absolute;top:calc(100% + .5rem);left:0;z-index:20;min-width:21rem;
|
|
background:var(--panel);border:1px solid var(--line-hi);border-radius:var(--r-soft);
|
|
padding:.4rem;display:flex;flex-direction:column;
|
|
box-shadow:0 12px 32px rgba(0,0,0,.75)}
|
|
.menu[hidden]{display:none}
|
|
.mi{display:flex;align-items:center;gap:.7rem;width:100%;text-align:left;
|
|
padding:.62rem .8rem;border:0;border-radius:calc(var(--r-soft) - .35rem);
|
|
background:transparent;color:var(--ink);font-family:var(--mono);font-size:.76rem;
|
|
letter-spacing:.04em;text-decoration:none;cursor:pointer;line-height:1.3;
|
|
white-space:nowrap}
|
|
.mi:hover,.mi:focus-visible{background:#181818;outline:none}
|
|
.mi .mark{width:1.05rem;height:1.05rem;color:var(--soft)}
|
|
.mi:hover .mark{color:var(--ink)}
|
|
.mi u{margin-left:auto;color:var(--faint);text-decoration:none;font-size:.68rem;
|
|
letter-spacing:.1em;text-transform:uppercase}
|
|
|
|
/* Works-with bar: one scrolling row, name and mark per agent. */
|
|
.strip{border-bottom:1px solid var(--line);background:var(--panel)}
|
|
.strip .in{max-width:60rem;margin:0 auto;padding:.9rem 1.75rem;display:flex;
|
|
align-items:center;gap:1.25rem;font-family:var(--mono);font-size:.72rem;
|
|
letter-spacing:.1em;text-transform:uppercase;color:var(--faint)}
|
|
.strip b{color:var(--soft);font-weight:400;flex:0 0 auto;
|
|
border-right:1px solid var(--line);padding-right:1.25rem;align-self:stretch;
|
|
display:flex;align-items:center}
|
|
.bar{display:flex;align-items:center;gap:.5rem;flex:1 1 auto;min-width:0;
|
|
overflow-x:auto;scrollbar-width:none;scroll-snap-type:x proximity;
|
|
-webkit-overflow-scrolling:touch;
|
|
-webkit-mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent);
|
|
mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent)}
|
|
.bar::-webkit-scrollbar{display:none}
|
|
.agent{display:inline-flex;align-items:center;gap:.45rem;flex:0 0 auto;
|
|
padding:.4rem .85rem;border:1px solid var(--line);border-radius:var(--r);
|
|
color:var(--soft);white-space:nowrap;scroll-snap-align:start;
|
|
text-decoration:none;background:var(--ground)}
|
|
.agent:hover{color:var(--ink);border-color:var(--line-hi);text-decoration:none}
|
|
.agent .mark{width:.95rem;height:.95rem}
|
|
|
|
h2{font-family:var(--mono);font-size:1.1rem;letter-spacing:-.01em;margin:2.75rem 0 .75rem;font-weight:500}
|
|
h3{font-size:1rem;margin:2rem 0 .5rem;font-weight:600}
|
|
h4,h5,h6{font-size:.94rem;margin:1.5rem 0 .5rem;font-weight:600}
|
|
p{margin:0 0 1rem;max-width:70ch}
|
|
ul,ol{margin:0 0 1rem;padding-left:1.35rem;max-width:70ch}
|
|
li{margin:.22rem 0}
|
|
a{color:var(--ink);text-decoration:underline;text-underline-offset:3px;
|
|
text-decoration-color:var(--line-hi)}
|
|
a:hover{text-decoration-color:var(--ink)}
|
|
a:focus-visible{outline:1px solid var(--ink);outline-offset:3px}
|
|
code{font-family:var(--mono);font-size:.85em;background:var(--panel);
|
|
border:1px solid var(--line);border-radius:5px;padding:.1em .36em}
|
|
pre{background:var(--panel);border:1px solid var(--line);padding:1rem 1.1rem;
|
|
border-radius:var(--r-soft);overflow-x:auto;margin:0 0 1.25rem}
|
|
pre code{background:none;border:0;padding:0;font-size:.82rem;line-height:1.7}
|
|
blockquote{margin:0 0 1rem;padding-left:1rem;border-left:1px solid var(--line-hi);color:var(--soft)}
|
|
hr{border:0;border-top:1px solid var(--line);margin:2.5rem 0}
|
|
table{border-collapse:collapse;width:100%;margin:0 0 1.25rem;font-size:.92rem;
|
|
display:block;overflow-x:auto}
|
|
th,td{border:1px solid var(--line);padding:.5rem .7rem;text-align:left}
|
|
|
|
.curl{display:block;background:var(--panel);border:1px solid var(--line);
|
|
border-radius:var(--r-soft);padding:.95rem 1.1rem;font-family:var(--mono);
|
|
font-size:.8rem;color:var(--soft);overflow-x:auto;margin:0 0 2.5rem;
|
|
white-space:pre;line-height:1.9}
|
|
.curl b{color:var(--ink);font-weight:400}
|
|
.curl i{font-style:normal;color:var(--faint)}
|
|
|
|
/* index */
|
|
#q{width:100%;padding:.85rem 1.2rem;font-family:var(--mono);font-size:.85rem;color:var(--ink);
|
|
background:var(--panel);border:1px solid var(--line);border-radius:var(--r);margin:0 0 .6rem}
|
|
#q::placeholder{color:var(--faint);text-transform:uppercase;letter-spacing:.1em;font-size:.72rem}
|
|
#q:focus{outline:none;border-color:var(--soft)}
|
|
#count{color:var(--faint);font-family:var(--mono);font-size:.72rem;letter-spacing:.1em;
|
|
text-transform:uppercase;margin:0 0 2rem}
|
|
.cat{margin:2.5rem 0 .5rem;font-family:var(--mono);font-size:.7rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);border-bottom:1px solid var(--line);
|
|
padding-bottom:.45rem}
|
|
.row{display:block;padding:.6rem .9rem;margin:0 -.9rem;border-bottom:1px solid var(--line);
|
|
border-radius:var(--r-soft);text-decoration:none}
|
|
.row:hover{background:#0a0a0a}
|
|
.nm{font-family:var(--mono);font-size:.84rem;color:var(--ink);display:block}
|
|
.ds{color:var(--soft);font-size:.9rem;display:block;margin-top:.12rem}
|
|
footer{margin-top:4rem;padding:2rem 0 4rem;border-top:1px solid var(--line);
|
|
color:var(--faint);font-size:.85rem}
|
|
footer a{color:var(--soft)}
|
|
.fmap{display:grid;gap:1.75rem;grid-template-columns:repeat(auto-fit,minmax(11rem,1fr));
|
|
margin:0 0 2rem;font-size:.85rem}
|
|
.fmap b{display:block;font-family:var(--mono);font-size:.68rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);font-weight:400;margin:0 0 .6rem;
|
|
border-bottom:1px solid var(--line);padding-bottom:.4rem}
|
|
.fmap a{display:inline-block;color:var(--ink);text-decoration:none;margin:.3rem .35rem 0 0}
|
|
.fmap a:hover{text-decoration:underline}
|
|
.fmap div{color:var(--faint);line-height:1.5}
|
|
.fend{border-top:1px solid var(--line);padding-top:1.25rem;font-size:.82rem}
|
|
|
|
@media (max-width:640px){
|
|
.hero .wrap{padding-top:3.25rem;padding-bottom:2.75rem}
|
|
.btns{gap:.5rem}
|
|
.btn{padding:.75rem 1.15rem;font-size:.68rem}
|
|
.copy,.copy .btn{width:100%;justify-content:center}
|
|
.menu{left:0;right:0;min-width:0}
|
|
.strip .in{flex-direction:column;align-items:flex-start;gap:.6rem;padding:.9rem 1.75rem}
|
|
.strip b{border-right:0;padding-right:0}
|
|
.bar{width:100%}
|
|
}
|
|
</style>
|
|
<link rel="alternate" type="text/markdown" href="/ml-unsloth-finetuning.md">
|
|
|
|
<link rel="icon" href="/favicon.svg">
|
|
<link rel="canonical" href="https://hanzoskills.com/ml-unsloth-finetuning">
|
|
</head>
|
|
<!-- Reading this with a machine? Every page here is also plain markdown.
|
|
This one: https://hanzoskills.com/ml-unsloth-finetuning.md
|
|
Index: https://hanzoskills.com/llms.txt
|
|
Corpus: https://hanzoskills.com/all.md
|
|
Append .md to any path for the markdown twin. -->
|
|
<body>
|
|
<nav><div class="in">
|
|
<a class="brand" href="/"><svg class="mark" viewBox="0 0 67 67" fill="currentColor" aria-hidden="true"><path d="M22.21 67V44.6369H0V67H22.21Z"/><path d="M66.7038 22.3184H22.2534L0.0878906 44.6367H44.4634L66.7038 22.3184Z"/><path d="M22.21 0H0V22.3184H22.21V0Z"/><path d="M66.7198 0H44.5098V22.3184H66.7198V0Z"/><path d="M66.7198 67V44.6369H44.5098V67H66.7198Z"/></svg> HANZO<i>SKILLS</i></a>
|
|
<a class="up" href="/">All skills</a>
|
|
</div></nav>
|
|
|
|
<div class="wrap">
|
|
<h1>ml-unsloth-finetuning</h1><p class="lede">Fine-tuning LLMs (Llama, Mistral, Qwen) efficiently</p><h1>Unsloth Fine-Tuning</h1>
|
|
<p><strong>Scope</strong>: Fast LLM fine-tuning with Unsloth, memory optimization, multi-GPU, Flash Attention <strong>Lines</strong>: ~350 <strong>Last Updated</strong>: 2025-10-18</p>
|
|
<h2>When to Use This Skill</h2>
|
|
<p>Activate this skill when:</p>
|
|
<ul><li>Fine-tuning LLMs (Llama, Mistral, Qwen) efficiently</li><li>Working in memory-constrained GPU environments</li><li>Needing 2-5x faster training than standard methods</li><li>Optimizing training costs with reduced memory usage</li><li>Requiring rapid iteration on fine-tuning experiments</li><li>Training with consumer GPUs (24GB VRAM or less)</li></ul>
|
|
<h2>Core Concepts</h2>
|
|
<h3>Unsloth Architecture</h3>
|
|
<p><strong>Speed Optimizations</strong>:</p>
|
|
<ul><li>Flash Attention 2 integration for 2x faster attention</li><li>Custom CUDA kernels for backward pass optimization</li><li>Optimized RoPE (Rotary Position Embeddings) computation</li><li>Fused layernorm and cross-entropy operations</li><li>4-bit quantization with minimal accuracy loss</li></ul>
|
|
<p><strong>Memory Efficiency</strong>:</p>
|
|
<ul><li>Reduces VRAM usage by 30-60% vs standard training</li><li>Enables larger batch sizes on same hardware</li><li>Supports gradient checkpointing out of the box</li><li>Efficient mixed precision (bfloat16) handling</li></ul>
|
|
<p><strong>Compatibility</strong>:</p>
|
|
<ul><li>Works with PEFT/LoRA for parameter-efficient training</li><li>Supports Hugging Face Transformers ecosystem</li><li>Compatible with Modal.com GPU workloads</li><li>Drop-in replacement for standard fine-tuning</li></ul>
|
|
<h3>Supported Models</h3>
|
|
<p><strong>Llama Family</strong>:</p>
|
|
<ul><li>Llama 3.2 (1B, 3B), Llama 3.1 (8B, 70B, 405B)</li><li>Llama 3 (8B, 70B), Llama 2 (7B, 13B, 70B)</li></ul>
|
|
<p><strong>Mistral Family</strong>:</p>
|
|
<ul><li>Mistral 7B v0.3, Mixtral 8x7B, 8x22B</li><li>Mistral Small, Mistral NeMo</li></ul>
|
|
<p><strong>Other Models</strong>:</p>
|
|
<ul><li>Qwen 2.5 (0.5B - 72B), Qwen 2 (0.5B - 72B)</li><li>Gemma 2 (2B, 9B, 27B), Phi-3.5 (mini)</li><li>TinyLlama, OpenELM variants</li></ul>
|
|
<h3>Quantization Strategies</h3>
|
|
<p><strong>4-bit QLoRA</strong> (Recommended):</p>
|
|
<ul><li>Uses NF4 (Normal Float 4) quantization</li><li>Maintains 99%+ of full precision performance</li><li>Reduces memory by 4x vs FP16</li><li>Allows fine-tuning 70B models on 48GB GPU</li></ul>
|
|
<p><strong>16-bit Training</strong>:</p>
|
|
<ul><li>Full bfloat16 precision for maximum quality</li><li>Requires 2-4x more VRAM</li><li>Recommended for models <13B with sufficient VRAM</li></ul>
|
|
<hr>
|
|
<h2>Patterns</h2>
|
|
<h3>Basic Llama 3 Fine-Tuning</h3>
|
|
<pre><code class="lang-python">from unsloth import FastLanguageModel
|
|
import torch
|
|
|
|
# Load model with 4-bit quantization
|
|
model, tokenizer = FastLanguageModel.from_pretrained(
|
|
model_name="unsloth/llama-3-8b-bnb-4bit",
|
|
max_seq_length=2048,
|
|
dtype=None, # Auto-detect (bfloat16 for Ampere+)
|
|
load_in_4bit=True,
|
|
)
|
|
|
|
# Configure LoRA
|
|
model = FastLanguageModel.get_peft_model(
|
|
model,
|
|
r=16, # LoRA rank
|
|
target_modules=[
|
|
"q_proj", "k_proj", "v_proj", "o_proj",
|
|
"gate_proj", "up_proj", "down_proj"
|
|
],
|
|
lora_alpha=16,
|
|
lora_dropout=0, # Optimized for 0 dropout
|
|
bias="none",
|
|
use_gradient_checkpointing="unsloth", # Unsloth's optimized checkpointing
|
|
random_state=3407,
|
|
)</code></pre>
|
|
<h3>Dataset Preparation</h3>
|
|
<pre><code class="lang-python">from datasets import load_dataset
|
|
|
|
# Load dataset (Alpaca format)
|
|
dataset = load_dataset("yahma/alpaca-cleaned", split="train")
|
|
|
|
# Format for instruction tuning
|
|
alpaca_prompt = """Below is an instruction that describes a task. Write a response that appropriately completes the request.
|
|
|
|
### Instruction:
|
|
{}
|
|
|
|
### Response:
|
|
{}"""
|
|
|
|
def format_prompts(examples):
|
|
instructions = examples["instruction"]
|
|
outputs = examples["output"]
|
|
texts = []
|
|
|
|
for instruction, output in zip(instructions, outputs):
|
|
text = alpaca_prompt.format(instruction, output) + tokenizer.eos_token
|
|
texts.append(text)
|
|
|
|
return {"text": texts}
|
|
|
|
# Apply formatting
|
|
dataset = dataset.map(format_prompts, batched=True)</code></pre>
|
|
<h3>Training with SFTTrainer</h3>
|
|
<pre><code class="lang-python">from trl import SFTTrainer
|
|
from transformers import TrainingArguments
|
|
|
|
trainer = SFTTrainer(
|
|
model=model,
|
|
tokenizer=tokenizer,
|
|
train_dataset=dataset,
|
|
dataset_text_field="text",
|
|
max_seq_length=2048,
|
|
dataset_num_proc=2,
|
|
packing=False, # Can make training 5x faster for short sequences
|
|
args=TrainingArguments(
|
|
per_device_train_batch_size=2,
|
|
gradient_accumulation_steps=4, # Effective batch size = 8
|
|
warmup_steps=5,
|
|
max_steps=60, # Or num_train_epochs=1
|
|
learning_rate=2e-4,
|
|
fp16=not torch.cuda.is_bf16_supported(),
|
|
bf16=torch.cuda.is_bf16_supported(),
|
|
logging_steps=1,
|
|
optim="adamw_8bit", # Reduced memory optimizer
|
|
weight_decay=0.01,
|
|
lr_scheduler_type="linear",
|
|
seed=3407,
|
|
output_dir="outputs",
|
|
),
|
|
)
|
|
|
|
# Train
|
|
trainer_stats = trainer.train()</code></pre>
|
|
<h3>Multi-GPU Training</h3>
|
|
<pre><code class="lang-python"># Use with torchrun or Modal multi-GPU
|
|
import modal
|
|
|
|
app = modal.App("unsloth-multi-gpu")
|
|
|
|
image = (
|
|
modal.Image.debian_slim()
|
|
.pip_install("unsloth[colab-new] @ git+https://github.com/unslothai/unsloth.git")
|
|
.pip_install("xformers", "trl", "peft", "accelerate")
|
|
)
|
|
|
|
@app.function(
|
|
gpu="a100:2", # 2x A100
|
|
image=image,
|
|
timeout=7200
|
|
)
|
|
def multi_gpu_training():
|
|
from unsloth import FastLanguageModel
|
|
from accelerate import Accelerator
|
|
|
|
accelerator = Accelerator()
|
|
|
|
# Model automatically distributed across GPUs
|
|
model, tokenizer = FastLanguageModel.from_pretrained(
|
|
model_name="unsloth/llama-3-70b-bnb-4bit",
|
|
max_seq_length=4096,
|
|
dtype=None,
|
|
load_in_4bit=True,
|
|
device_map="auto" # Auto-distribute across GPUs
|
|
)
|
|
|
|
model = FastLanguageModel.get_peft_model(
|
|
model,
|
|
r=32, # Higher rank for 70B model
|
|
target_modules=["q_proj", "k_proj", "v_proj", "o_proj"],
|
|
lora_alpha=32,
|
|
)
|
|
|
|
# Train with accelerate
|
|
# (SFTTrainer handles multi-GPU automatically)</code></pre>
|
|
<h3>Saving and Loading</h3>
|
|
<pre><code class="lang-python"># Save LoRA adapter only (small, ~100MB)
|
|
model.save_pretrained("lora_model")
|
|
tokenizer.save_pretrained("lora_model")
|
|
|
|
# Save merged model (full weights)
|
|
model.save_pretrained_merged(
|
|
"merged_model",
|
|
tokenizer,
|
|
save_method="merged_16bit" # or "lora", "merged_4bit"
|
|
)
|
|
|
|
# Push to Hugging Face Hub
|
|
model.push_to_hub_merged(
|
|
"username/model-name",
|
|
tokenizer,
|
|
save_method="merged_16bit",
|
|
token="hf_..."
|
|
)
|
|
|
|
# Load for inference
|
|
model, tokenizer = FastLanguageModel.from_pretrained(
|
|
model_name="lora_model",
|
|
max_seq_length=2048,
|
|
dtype=None,
|
|
load_in_4bit=True,
|
|
)
|
|
FastLanguageModel.for_inference(model) # Enable fast inference mode</code></pre>
|
|
<h3>Inference After Training</h3>
|
|
<pre><code class="lang-python"># Enable fast inference (disables dropout, etc.)
|
|
FastLanguageModel.for_inference(model)
|
|
|
|
# Generate
|
|
inputs = tokenizer(
|
|
[alpaca_prompt.format(
|
|
"What is the capital of France?",
|
|
"" # Leave output empty for generation
|
|
)],
|
|
return_tensors="pt"
|
|
).to("cuda")
|
|
|
|
outputs = model.generate(
|
|
**inputs,
|
|
max_new_tokens=128,
|
|
use_cache=True,
|
|
temperature=1.5,
|
|
min_p=0.1
|
|
)
|
|
|
|
print(tokenizer.batch_decode(outputs))</code></pre>
|
|
<h3>Modal.com Deployment</h3>
|
|
<pre><code class="lang-python">import modal
|
|
|
|
app = modal.App("unsloth-finetune")
|
|
|
|
image = (
|
|
modal.Image.debian_slim()
|
|
.pip_install(
|
|
"unsloth[colab-new] @ git+https://github.com/unslothai/unsloth.git",
|
|
"xformers",
|
|
"trl",
|
|
"peft"
|
|
)
|
|
)
|
|
|
|
volume = modal.Volume.from_name("model-volume", create_if_missing=True)
|
|
|
|
@app.function(
|
|
gpu="l40s", # Recommended: great price/performance
|
|
image=image,
|
|
volumes={"/models": volume},
|
|
timeout=3600
|
|
)
|
|
def finetune_llama(
|
|
dataset_name: str,
|
|
output_path: str = "/models/finetuned"
|
|
):
|
|
from unsloth import FastLanguageModel
|
|
from trl import SFTTrainer
|
|
from transformers import TrainingArguments
|
|
from datasets import load_dataset
|
|
|
|
# Load model
|
|
model, tokenizer = FastLanguageModel.from_pretrained(
|
|
model_name="unsloth/llama-3-8b-bnb-4bit",
|
|
max_seq_length=2048,
|
|
dtype=None,
|
|
load_in_4bit=True,
|
|
)
|
|
|
|
# Apply LoRA
|
|
model = FastLanguageModel.get_peft_model(
|
|
model,
|
|
r=16,
|
|
target_modules=["q_proj", "k_proj", "v_proj", "o_proj"],
|
|
lora_alpha=16,
|
|
)
|
|
|
|
# Load dataset
|
|
dataset = load_dataset(dataset_name, split="train")
|
|
|
|
# Train
|
|
trainer = SFTTrainer(
|
|
model=model,
|
|
tokenizer=tokenizer,
|
|
train_dataset=dataset,
|
|
dataset_text_field="text",
|
|
max_seq_length=2048,
|
|
args=TrainingArguments(
|
|
per_device_train_batch_size=2,
|
|
gradient_accumulation_steps=4,
|
|
warmup_steps=5,
|
|
max_steps=100,
|
|
learning_rate=2e-4,
|
|
fp16=not torch.cuda.is_bf16_supported(),
|
|
bf16=torch.cuda.is_bf16_supported(),
|
|
logging_steps=1,
|
|
optim="adamw_8bit",
|
|
output_dir="/tmp/output",
|
|
),
|
|
)
|
|
|
|
trainer.train()
|
|
|
|
# Save to volume
|
|
model.save_pretrained(output_path)
|
|
tokenizer.save_pretrained(output_path)
|
|
volume.commit()
|
|
|
|
return {"status": "complete", "path": output_path}
|
|
|
|
@app.local_entrypoint()
|
|
def main():
|
|
result = finetune_llama.remote("yahma/alpaca-cleaned")
|
|
print(result)</code></pre>
|
|
<h3>Chat Template Fine-Tuning</h3>
|
|
<pre><code class="lang-python"># ChatML format
|
|
chat_template = """<|im_start|>system
|
|
{system}<|im_end|>
|
|
<|im_start|>user
|
|
{user}<|im_end|>
|
|
<|im_start|>assistant
|
|
{assistant}<|im_end|>"""
|
|
|
|
def format_chat(examples):
|
|
texts = []
|
|
for system, user, assistant in zip(
|
|
examples["system"],
|
|
examples["user"],
|
|
examples["assistant"]
|
|
):
|
|
text = chat_template.format(
|
|
system=system,
|
|
user=user,
|
|
assistant=assistant
|
|
) + tokenizer.eos_token
|
|
texts.append(text)
|
|
return {"text": texts}
|
|
|
|
dataset = dataset.map(format_chat, batched=True)</code></pre>
|
|
<hr>
|
|
<h2>Quick Reference</h2>
|
|
<h3>GPU Requirements by Model Size</h3>
|
|
<pre><code>Model Size | 4-bit VRAM | 16-bit VRAM | Recommended GPU
|
|
--------------|------------|-------------|------------------
|
|
1-3B | 6-8 GB | 12-16 GB | T4, L4
|
|
7-8B | 12-16 GB | 24-32 GB | L40S
|
|
13B | 16-20 GB | 32-40 GB | L40S, A100
|
|
70B | 40-48 GB | 140+ GB | A100 80GB, H100</code></pre>
|
|
<h3>Unsloth Installation</h3>
|
|
<pre><code class="lang-bash"># Standard installation
|
|
pip install "unsloth[colab-new] @ git+https://github.com/unslothai/unsloth.git"
|
|
pip install xformers trl peft accelerate
|
|
|
|
# For CUDA 11.8
|
|
pip install "unsloth[cu118-ampere-new] @ git+https://github.com/unslothai/unsloth.git"
|
|
|
|
# For CUDA 12.1+
|
|
pip install "unsloth[cu121-ampere-new] @ git+https://github.com/unslothai/unsloth.git"</code></pre>
|
|
<h3>Key Hyperparameters</h3>
|
|
<pre><code class="lang-python"># LoRA Configuration
|
|
r = 16 # Rank: 8-64 (higher = more params, better quality)
|
|
lora_alpha = 16 # Usually same as r
|
|
lora_dropout = 0 # Unsloth optimized for 0
|
|
target_modules = [ # All attention + MLP for best results
|
|
"q_proj", "k_proj", "v_proj", "o_proj",
|
|
"gate_proj", "up_proj", "down_proj"
|
|
]
|
|
|
|
# Training Configuration
|
|
batch_size = 2 # Per device (increase for larger GPUs)
|
|
gradient_accumulation = 4 # Effective batch = batch_size * accumulation
|
|
learning_rate = 2e-4 # Standard for LoRA
|
|
warmup_steps = 5 # 5-10% of total steps
|
|
max_seq_length = 2048 # Model's max (512-4096)</code></pre>
|
|
<h3>Common Commands</h3>
|
|
<pre><code class="lang-python"># Enable inference mode
|
|
FastLanguageModel.for_inference(model)
|
|
|
|
# Save adapter only
|
|
model.save_pretrained("path")
|
|
|
|
# Save merged 16-bit
|
|
model.save_pretrained_merged("path", tokenizer, save_method="merged_16bit")
|
|
|
|
# Save merged 4-bit (smallest)
|
|
model.save_pretrained_merged("path", tokenizer, save_method="merged_4bit")
|
|
|
|
# Push to Hub
|
|
model.push_to_hub_merged("user/model", tokenizer, save_method="merged_16bit")</code></pre>
|
|
<hr>
|
|
<h2>Anti-Patterns</h2>
|
|
<p>❌ <strong>Using standard Transformers for fine-tuning</strong>: Slower and uses more memory ✅ Use Unsloth for 2-5x speedup with same results</p>
|
|
<p>❌ <strong>Loading full precision models</strong>: Wastes VRAM on consumer GPUs ✅ Always use <code>load_in_4bit=True</code> for models >7B</p>
|
|
<p>❌ <strong>Wrong dtype for GPU architecture</strong>:</p>
|
|
<pre><code class="lang-python"># ❌ Bad: Forces FP16 on Ampere+ GPUs
|
|
args = TrainingArguments(fp16=True, bf16=False)
|
|
|
|
# ✅ Good: Auto-detect best dtype
|
|
args = TrainingArguments(
|
|
fp16=not torch.cuda.is_bf16_supported(),
|
|
bf16=torch.cuda.is_bf16_supported()
|
|
)</code></pre>
|
|
<p>❌ <strong>Tiny batch sizes without gradient accumulation</strong>: Unstable training ✅ Use <code>gradient_accumulation_steps</code> to increase effective batch size</p>
|
|
<p>❌ <strong>Too many LoRA target modules</strong>: Wastes memory for marginal gains ✅ Start with attention layers only: <code>["q_proj", "k_proj", "v_proj", "o_proj"]</code></p>
|
|
<p>❌ <strong>Ignoring sequence length</strong>: OOM errors or wasted compute ✅ Set <code>max_seq_length</code> to actual data length (analyze dataset first)</p>
|
|
<p>❌ <strong>Not using gradient checkpointing</strong>: Runs out of VRAM ✅ Always use <code>use_gradient_checkpointing="unsloth"</code> for large models</p>
|
|
<p>❌ <strong>Saving full model every epoch</strong>: Wastes disk space ✅ Save LoRA adapter only (~100MB) or push merged model once at end</p>
|
|
<p>❌ <strong>Using standard AdamW</strong>: Higher memory usage ✅ Use <code>optim="adamw_8bit"</code> for 2x memory reduction</p>
|
|
<hr>
|
|
<h2>Related Skills</h2>
|
|
<ul><li><code>lora-peft-techniques.md</code> - LoRA configuration, rank selection, merging</li><li><code>llm-dataset-preparation.md</code> - Dataset formatting, quality, validation</li><li><code>modal-gpu-workloads.md</code> - GPU selection, cost optimization, deployment</li><li><code>huggingface-autotrain.md</code> - Alternative no-code fine-tuning approach</li><li><code>llm-inference-optimization.md</code> - Post-training inference optimization</li><li><code>model-evaluation.md</code> - Evaluating fine-tuned model quality</li></ul>
|
|
<hr>
|
|
<p><strong>Last Updated</strong>: 2025-10-18 <strong>Format Version</strong>: 1.0 (Atomic)</p>
|
|
<footer>
|
|
<div class="fmap">
|
|
<div><b>Build</b>
|
|
<a href="https://hanzo.app">hanzo.app</a> app builder
|
|
<a href="https://hanzo.codes">hanzo.codes</a> code with an agent
|
|
<a href="https://hanzoskills.com">hanzoskills.com</a> what an agent reads</div>
|
|
<div><b>Run</b>
|
|
<a href="https://console.hanzo.ai">console.hanzo.ai</a> cloud console
|
|
<a href="https://api.hanzo.ai">api.hanzo.ai</a> the API</div>
|
|
<div><b>Account</b>
|
|
<a href="https://hanzo.id">hanzo.id</a> sign in
|
|
<a href="https://pay.hanzo.ai">pay.hanzo.ai</a> top up
|
|
<a href="https://billing.hanzo.ai">billing.hanzo.ai</a> invoices and usage</div>
|
|
<div><b>Company</b>
|
|
<a href="https://hanzo.ai">hanzo.ai</a> canonical
|
|
<a href="https://hanzo.ventures">hanzo.ventures</a> ventures</div>
|
|
</div>
|
|
<div class="fend"><a href="/ml-unsloth-finetuning.md">This page as plain text</a> · <a href="/llms.txt">Index for agents</a></div>
|
|
</footer>
|
|
</div></body></html> |