mirror of
https://github.com/hanzoai/skills.git
synced 2026-11-19 00:47:36 +00:00
hanzoskills.com/llms.txt now opens with the install line, the API reference index and the company index, so an agent that lands on any one of the three can find the rest. robots.txt claimed 537 documents while llms.txt counts 920. The number is gone from robots.txt; llms.txt derives it. public/ is regenerated from scripts/site.py, which it had fallen behind: the copy-prompt menu and agent bar were in the generator and not in the committed pages. tests/site.mjs passes against the result. Co-authored-by: Hanzo Dev <[email protected]>
507 lines
28 KiB
HTML
507 lines
28 KiB
HTML
<!doctype html>
|
|
<html lang="en"><head>
|
|
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
<title>ml-diffusion-model-basics</title>
|
|
<meta name="color-scheme" content="dark">
|
|
<style>
|
|
/* Committed to one visual world: a dark instrument panel. No light variant —
|
|
the dot field and the hairline rules only read on black, and a naive
|
|
inversion would wash both out. Every colour is painted explicitly so the page
|
|
never borrows a host background. */
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Pixel Square';src:url('/fonts/ZenPixel-Square.woff2') format('woff2');font-style:normal;font-display:swap}
|
|
|
|
:root{
|
|
--ink:#f4f4f2; --soft:#9c9c98; --faint:#6a6a66; --ground:#000; --panel:#0a0a0a;
|
|
--line:#232323; --line-hi:#3a3a3a; --accent:#f4f4f2;
|
|
--font-zen-sans:'Zen',-apple-system,BlinkMacSystemFont,"Segoe UI",Helvetica,Arial,sans-serif;
|
|
--font-zen-mono:'Zen Mono',ui-monospace,"SF Mono",SFMono-Regular,Menlo,Consolas,monospace;
|
|
--font-zen-pixel-square:'Zen Pixel Square',monospace;
|
|
--mono:var(--font-zen-mono);
|
|
--sans:var(--font-zen-sans);
|
|
/* Controls are pills and surfaces are soft, as on hanzo.ai. */
|
|
--r:999px; --r-soft:14px;
|
|
}
|
|
*{box-sizing:border-box}
|
|
html{-webkit-text-size-adjust:100%}
|
|
body{margin:0;background:var(--ground);color:var(--ink);font-family:var(--sans);
|
|
font-size:16px;line-height:1.65;-webkit-font-smoothing:antialiased}
|
|
.wrap{max-width:60rem;margin:0 auto;padding:0 1.75rem}
|
|
|
|
/* The dot field. A halftone made of one repeating radial-gradient, faded out
|
|
along the diagonal by a mask so it reads as a wave rather than wallpaper. */
|
|
/* No overflow clip here: the dot field is inset to the box already, and a clip
|
|
would cut off the copy-prompt menu. */
|
|
.hero{position:relative;border-bottom:1px solid var(--line)}
|
|
.hero::before{content:"";position:absolute;inset:0;
|
|
background-image:radial-gradient(var(--line-hi) 1px, transparent 1px);
|
|
background-size:7px 7px;
|
|
-webkit-mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
pointer-events:none}
|
|
.hero .wrap{position:relative;padding-top:5.5rem;padding-bottom:4rem}
|
|
|
|
nav{border-bottom:1px solid var(--line)}
|
|
nav .in{max-width:60rem;margin:0 auto;padding:1.1rem 1.75rem;display:flex;
|
|
justify-content:space-between;align-items:center;gap:1rem}
|
|
.brand{font-family:var(--mono);font-size:.85rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--ink);text-decoration:none;
|
|
display:inline-flex;align-items:center;gap:.6rem}
|
|
.brand i{font-style:normal;color:var(--faint)}
|
|
.mark{width:1.15em;height:1.15em;display:block;flex:0 0 auto}
|
|
nav a.up{font-family:var(--mono);font-size:.72rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--soft);text-decoration:none}
|
|
nav a.up:hover{color:var(--ink)}
|
|
|
|
h1{font-family:var(--mono);font-size:clamp(2rem,5.4vw,3.4rem);line-height:1.12;
|
|
letter-spacing:-.02em;margin:0 0 1.1rem;font-weight:500;text-wrap:balance}
|
|
h1 em{font-style:normal;color:var(--faint)}
|
|
.lede{color:var(--soft);font-size:1.05rem;max-width:52ch;margin:0 0 2.25rem}
|
|
|
|
.btns{display:flex;flex-wrap:wrap;gap:.7rem;margin:0 0 .5rem;align-items:center}
|
|
.btn{display:inline-flex;align-items:center;gap:.5rem;font-family:var(--mono);
|
|
font-size:.72rem;letter-spacing:.14em;text-transform:uppercase;
|
|
padding:.8rem 1.4rem;border:1px solid var(--line-hi);border-radius:var(--r);
|
|
color:var(--ink);text-decoration:none;background:transparent;cursor:pointer;
|
|
line-height:1.2;font-weight:400}
|
|
.btn:hover{background:#141414;text-decoration:none;border-color:var(--soft)}
|
|
.btn.solid{background:var(--ink);color:#000;border-color:var(--ink)}
|
|
.btn.solid:hover{background:#d8d8d4}
|
|
.btn .chev{width:.7em;height:.7em;flex:0 0 auto;transition:transform .15s ease}
|
|
.btn[aria-expanded="true"] .chev{transform:rotate(180deg)}
|
|
|
|
/* Copy-prompt selector. Each row copies the prompt for its own target. */
|
|
.copy{position:relative;display:inline-flex}
|
|
.menu{position:absolute;top:calc(100% + .5rem);left:0;z-index:20;min-width:21rem;
|
|
background:var(--panel);border:1px solid var(--line-hi);border-radius:var(--r-soft);
|
|
padding:.4rem;display:flex;flex-direction:column;
|
|
box-shadow:0 12px 32px rgba(0,0,0,.75)}
|
|
.menu[hidden]{display:none}
|
|
.mi{display:flex;align-items:center;gap:.7rem;width:100%;text-align:left;
|
|
padding:.62rem .8rem;border:0;border-radius:calc(var(--r-soft) - .35rem);
|
|
background:transparent;color:var(--ink);font-family:var(--mono);font-size:.76rem;
|
|
letter-spacing:.04em;text-decoration:none;cursor:pointer;line-height:1.3;
|
|
white-space:nowrap}
|
|
.mi:hover,.mi:focus-visible{background:#181818;outline:none}
|
|
.mi .mark{width:1.05rem;height:1.05rem;color:var(--soft)}
|
|
.mi:hover .mark{color:var(--ink)}
|
|
.mi u{margin-left:auto;color:var(--faint);text-decoration:none;font-size:.68rem;
|
|
letter-spacing:.1em;text-transform:uppercase}
|
|
|
|
/* Works-with bar: one scrolling row, name and mark per agent. */
|
|
.strip{border-bottom:1px solid var(--line);background:var(--panel)}
|
|
.strip .in{max-width:60rem;margin:0 auto;padding:.9rem 1.75rem;display:flex;
|
|
align-items:center;gap:1.25rem;font-family:var(--mono);font-size:.72rem;
|
|
letter-spacing:.1em;text-transform:uppercase;color:var(--faint)}
|
|
.strip b{color:var(--soft);font-weight:400;flex:0 0 auto;
|
|
border-right:1px solid var(--line);padding-right:1.25rem;align-self:stretch;
|
|
display:flex;align-items:center}
|
|
.bar{display:flex;align-items:center;gap:.5rem;flex:1 1 auto;min-width:0;
|
|
overflow-x:auto;scrollbar-width:none;scroll-snap-type:x proximity;
|
|
-webkit-overflow-scrolling:touch;
|
|
-webkit-mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent);
|
|
mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent)}
|
|
.bar::-webkit-scrollbar{display:none}
|
|
.agent{display:inline-flex;align-items:center;gap:.45rem;flex:0 0 auto;
|
|
padding:.4rem .85rem;border:1px solid var(--line);border-radius:var(--r);
|
|
color:var(--soft);white-space:nowrap;scroll-snap-align:start;
|
|
text-decoration:none;background:var(--ground)}
|
|
.agent:hover{color:var(--ink);border-color:var(--line-hi);text-decoration:none}
|
|
.agent .mark{width:.95rem;height:.95rem}
|
|
|
|
h2{font-family:var(--mono);font-size:1.1rem;letter-spacing:-.01em;margin:2.75rem 0 .75rem;font-weight:500}
|
|
h3{font-size:1rem;margin:2rem 0 .5rem;font-weight:600}
|
|
h4,h5,h6{font-size:.94rem;margin:1.5rem 0 .5rem;font-weight:600}
|
|
p{margin:0 0 1rem;max-width:70ch}
|
|
ul,ol{margin:0 0 1rem;padding-left:1.35rem;max-width:70ch}
|
|
li{margin:.22rem 0}
|
|
a{color:var(--ink);text-decoration:underline;text-underline-offset:3px;
|
|
text-decoration-color:var(--line-hi)}
|
|
a:hover{text-decoration-color:var(--ink)}
|
|
a:focus-visible{outline:1px solid var(--ink);outline-offset:3px}
|
|
code{font-family:var(--mono);font-size:.85em;background:var(--panel);
|
|
border:1px solid var(--line);border-radius:5px;padding:.1em .36em}
|
|
pre{background:var(--panel);border:1px solid var(--line);padding:1rem 1.1rem;
|
|
border-radius:var(--r-soft);overflow-x:auto;margin:0 0 1.25rem}
|
|
pre code{background:none;border:0;padding:0;font-size:.82rem;line-height:1.7}
|
|
blockquote{margin:0 0 1rem;padding-left:1rem;border-left:1px solid var(--line-hi);color:var(--soft)}
|
|
hr{border:0;border-top:1px solid var(--line);margin:2.5rem 0}
|
|
table{border-collapse:collapse;width:100%;margin:0 0 1.25rem;font-size:.92rem;
|
|
display:block;overflow-x:auto}
|
|
th,td{border:1px solid var(--line);padding:.5rem .7rem;text-align:left}
|
|
|
|
.curl{display:block;background:var(--panel);border:1px solid var(--line);
|
|
border-radius:var(--r-soft);padding:.95rem 1.1rem;font-family:var(--mono);
|
|
font-size:.8rem;color:var(--soft);overflow-x:auto;margin:0 0 2.5rem;
|
|
white-space:pre;line-height:1.9}
|
|
.curl b{color:var(--ink);font-weight:400}
|
|
.curl i{font-style:normal;color:var(--faint)}
|
|
|
|
/* index */
|
|
#q{width:100%;padding:.85rem 1.2rem;font-family:var(--mono);font-size:.85rem;color:var(--ink);
|
|
background:var(--panel);border:1px solid var(--line);border-radius:var(--r);margin:0 0 .6rem}
|
|
#q::placeholder{color:var(--faint);text-transform:uppercase;letter-spacing:.1em;font-size:.72rem}
|
|
#q:focus{outline:none;border-color:var(--soft)}
|
|
#count{color:var(--faint);font-family:var(--mono);font-size:.72rem;letter-spacing:.1em;
|
|
text-transform:uppercase;margin:0 0 2rem}
|
|
.cat{margin:2.5rem 0 .5rem;font-family:var(--mono);font-size:.7rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);border-bottom:1px solid var(--line);
|
|
padding-bottom:.45rem}
|
|
.row{display:block;padding:.6rem .9rem;margin:0 -.9rem;border-bottom:1px solid var(--line);
|
|
border-radius:var(--r-soft);text-decoration:none}
|
|
.row:hover{background:#0a0a0a}
|
|
.nm{font-family:var(--mono);font-size:.84rem;color:var(--ink);display:block}
|
|
.ds{color:var(--soft);font-size:.9rem;display:block;margin-top:.12rem}
|
|
footer{margin-top:4rem;padding:2rem 0 4rem;border-top:1px solid var(--line);
|
|
color:var(--faint);font-size:.85rem}
|
|
footer a{color:var(--soft)}
|
|
.fmap{display:grid;gap:1.75rem;grid-template-columns:repeat(auto-fit,minmax(11rem,1fr));
|
|
margin:0 0 2rem;font-size:.85rem}
|
|
.fmap b{display:block;font-family:var(--mono);font-size:.68rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);font-weight:400;margin:0 0 .6rem;
|
|
border-bottom:1px solid var(--line);padding-bottom:.4rem}
|
|
.fmap a{display:inline-block;color:var(--ink);text-decoration:none;margin:.3rem .35rem 0 0}
|
|
.fmap a:hover{text-decoration:underline}
|
|
.fmap div{color:var(--faint);line-height:1.5}
|
|
.fend{border-top:1px solid var(--line);padding-top:1.25rem;font-size:.82rem}
|
|
|
|
@media (max-width:640px){
|
|
.hero .wrap{padding-top:3.25rem;padding-bottom:2.75rem}
|
|
.btns{gap:.5rem}
|
|
.btn{padding:.75rem 1.15rem;font-size:.68rem}
|
|
.copy,.copy .btn{width:100%;justify-content:center}
|
|
.menu{left:0;right:0;min-width:0}
|
|
.strip .in{flex-direction:column;align-items:flex-start;gap:.6rem;padding:.9rem 1.75rem}
|
|
.strip b{border-right:0;padding-right:0}
|
|
.bar{width:100%}
|
|
}
|
|
</style>
|
|
<link rel="alternate" type="text/markdown" href="/ml-diffusion-model-basics.md">
|
|
|
|
<link rel="icon" href="/favicon.svg">
|
|
<link rel="canonical" href="https://hanzoskills.com/ml-diffusion-model-basics">
|
|
</head>
|
|
<!-- Reading this with a machine? Every page here is also plain markdown.
|
|
This one: https://hanzoskills.com/ml-diffusion-model-basics.md
|
|
Index: https://hanzoskills.com/llms.txt
|
|
Corpus: https://hanzoskills.com/all.md
|
|
Append .md to any path for the markdown twin. -->
|
|
<body>
|
|
<nav><div class="in">
|
|
<a class="brand" href="/"><svg class="mark" viewBox="0 0 67 67" fill="currentColor" aria-hidden="true"><path d="M22.21 67V44.6369H0V67H22.21Z"/><path d="M66.7038 22.3184H22.2534L0.0878906 44.6367H44.4634L66.7038 22.3184Z"/><path d="M22.21 0H0V22.3184H22.21V0Z"/><path d="M66.7198 0H44.5098V22.3184H66.7198V0Z"/><path d="M66.7198 67V44.6369H44.5098V67H66.7198Z"/></svg> HANZO<i>SKILLS</i></a>
|
|
<a class="up" href="/">All skills</a>
|
|
</div></nav>
|
|
|
|
<div class="wrap">
|
|
<h1>ml-diffusion-model-basics</h1><p class="lede">Understanding diffusion model theory and mechanics (forward/reverse process)</p><h1>Diffusion Model Basics</h1>
|
|
<p><strong>Scope</strong>: Diffusion model fundamentals, architectures, inference pipelines, and model selection <strong>Lines</strong>: ~340 <strong>Last Updated</strong>: 2025-10-18</p>
|
|
<h2>When to Use This Skill</h2>
|
|
<p>Activate this skill when:</p>
|
|
<ul><li>Understanding diffusion model theory and mechanics (forward/reverse process)</li><li>Implementing image generation with diffusion models</li><li>Selecting appropriate diffusion models for specific use cases</li><li>Configuring sampling parameters (schedulers, guidance, steps)</li><li>Debugging diffusion model inference issues</li><li>Comparing diffusion architectures (DDPM, DDIM, Stable Diffusion, SDXL)</li><li>Optimizing inference quality vs speed tradeoffs</li></ul>
|
|
<h2>Core Concepts</h2>
|
|
<h3>Forward and Reverse Diffusion Process</h3>
|
|
<p><strong>Forward Process (Noise Addition)</strong>:</p>
|
|
<ul><li>Gradually adds Gaussian noise to data over T timesteps</li><li>q(x_t | x_{t-1}) = N(x_t; √(1-β_t) x_{t-1}, β_t I)</li><li>β_t is variance schedule (increases from β_1 to β_T)</li><li>After T steps, x_T ≈ N(0, I) (pure noise)</li></ul>
|
|
<p><strong>Reverse Process (Denoising)</strong>:</p>
|
|
<ul><li>Learns to reverse noise addition: p_θ(x_{t-1} | x_t)</li><li>Neural network predicts noise ε_θ(x_t, t) at each timestep</li><li>Iteratively removes noise to generate samples</li><li>Conditioned on text embeddings for text-to-image</li></ul>
|
|
<p><strong>Key Insight</strong>: Training predicts noise, inference removes it step-by-step</p>
|
|
<h3>Noise Scheduling</h3>
|
|
<p><strong>Purpose</strong>: Controls noise addition/removal over time</p>
|
|
<p><strong>Common Schedulers</strong>:</p>
|
|
<ul><li><strong>DDPM</strong> (Denoising Diffusion Probabilistic Models): Original, slow but high quality</li><li><strong>DDIM</strong> (Denoising Diffusion Implicit Models): Deterministic, 10-50x faster</li><li><strong>PNDM</strong> (Pseudo Numerical Methods): Better numerical stability</li><li><strong>DPM-Solver</strong>: Adaptive step size, excellent speed/quality</li><li><strong>Euler/Euler Ancestral</strong>: Simple, good for artistic styles</li><li><strong>UniPC</strong>: Fast, high quality, good default choice</li></ul>
|
|
<p><strong>Scheduler Selection</strong>:</p>
|
|
<ul><li>Quality priority → DDPM, DPM-Solver++</li><li>Speed priority → DDIM, DPM-Solver, UniPC</li><li>Artistic/creative → Euler Ancestral</li><li>Deterministic → DDIM, DPM-Solver</li></ul>
|
|
<h3>UNet Architecture</h3>
|
|
<p><strong>Core Component</strong>: Modified UNet backbone for noise prediction</p>
|
|
<p><strong>Architecture</strong>:</p>
|
|
<ul><li><strong>Encoder</strong>: Downsampling with ResNet blocks + attention</li><li><strong>Bottleneck</strong>: Dense attention layers (most compute)</li><li><strong>Decoder</strong>: Upsampling with skip connections</li><li><strong>Time Embedding</strong>: Sinusoidal positional encoding for timestep t</li><li><strong>Cross-Attention</strong>: Inject text conditioning (CLIP embeddings)</li></ul>
|
|
<p><strong>Attention Mechanisms</strong>:</p>
|
|
<ul><li>Self-attention: Within image features</li><li>Cross-attention: Text-to-image conditioning</li><li>Spatial attention: Preserve spatial structure</li></ul>
|
|
<h3>Latent Diffusion Models (Stable Diffusion)</h3>
|
|
<p><strong>Key Innovation</strong>: Operate in compressed latent space, not pixel space</p>
|
|
<p><strong>Components</strong>:</p>
|
|
<ul><li><strong>VAE Encoder</strong>: Compress images to latent space (8x8x reduction)</li><li><strong>UNet</strong>: Diffusion in latent space (64x64 instead of 512x512)</li><li><strong>VAE Decoder</strong>: Reconstruct images from latents</li><li><strong>Text Encoder</strong>: CLIP (SD 1.x/2.x) or T5/CLIP (SDXL, SD3)</li></ul>
|
|
<p><strong>Benefits</strong>:</p>
|
|
<ul><li>8-10x faster than pixel-space diffusion</li><li>Lower VRAM requirements</li><li>Maintains image quality</li><li>Enables higher resolutions</li></ul>
|
|
<hr>
|
|
<h2>Patterns</h2>
|
|
<h3>Basic Text-to-Image Generation (Stable Diffusion 1.5)</h3>
|
|
<pre><code class="lang-python">from diffusers import StableDiffusionPipeline
|
|
import torch
|
|
|
|
# Load pipeline
|
|
pipe = StableDiffusionPipeline.from_pretrained(
|
|
"runwayml/stable-diffusion-v1-5",
|
|
torch_dtype=torch.float16,
|
|
safety_checker=None, # Optional: disable for speed
|
|
)
|
|
pipe = pipe.to("cuda")
|
|
|
|
# Generate image
|
|
prompt = "A serene mountain landscape at sunset, highly detailed, 4k"
|
|
negative_prompt = "blurry, low quality, distorted, ugly"
|
|
|
|
image = pipe(
|
|
prompt=prompt,
|
|
negative_prompt=negative_prompt,
|
|
num_inference_steps=50, # More steps = better quality
|
|
guidance_scale=7.5, # How strongly to follow prompt
|
|
height=512,
|
|
width=512,
|
|
generator=torch.Generator("cuda").manual_seed(42), # Reproducibility
|
|
).images[0]
|
|
|
|
image.save("output.png")</code></pre>
|
|
<p><strong>When to use</strong>:</p>
|
|
<ul><li>General purpose text-to-image generation</li><li>512x512 images (native resolution)</li><li>Quick prototyping and iteration</li></ul>
|
|
<h3>SDXL for Higher Quality</h3>
|
|
<pre><code class="lang-python">from diffusers import StableDiffusionXLPipeline
|
|
import torch
|
|
|
|
# SDXL has better text understanding and higher resolution
|
|
pipe = StableDiffusionXLPipeline.from_pretrained(
|
|
"stabilityai/stable-diffusion-xl-base-1.0",
|
|
torch_dtype=torch.float16,
|
|
variant="fp16",
|
|
use_safetensors=True,
|
|
)
|
|
pipe = pipe.to("cuda")
|
|
|
|
# Enable CPU offloading for lower VRAM (slower but uses ~6GB vs 12GB)
|
|
# pipe.enable_model_cpu_offload()
|
|
|
|
prompt = "Astronaut riding a horse on Mars, photorealistic, 8k uhd"
|
|
negative_prompt = "cartoon, painting, illustration"
|
|
|
|
image = pipe(
|
|
prompt=prompt,
|
|
negative_prompt=negative_prompt,
|
|
num_inference_steps=40, # SDXL converges faster
|
|
guidance_scale=7.0, # Lower than SD 1.5
|
|
height=1024, # Native resolution
|
|
width=1024,
|
|
).images[0]
|
|
|
|
image.save("sdxl_output.png")</code></pre>
|
|
<p><strong>Benefits</strong>:</p>
|
|
<ul><li>Better text comprehension (multiple text encoders)</li><li>Higher native resolution (1024x1024)</li><li>Improved image quality and details</li><li>Better anatomy and composition</li></ul>
|
|
<h3>Scheduler Comparison</h3>
|
|
<pre><code class="lang-python">from diffusers import (
|
|
StableDiffusionPipeline,
|
|
DDIMScheduler,
|
|
DPMSolverMultistepScheduler,
|
|
EulerAncestralDiscreteScheduler,
|
|
UniPCMultistepScheduler,
|
|
)
|
|
import torch
|
|
|
|
pipe = StableDiffusionPipeline.from_pretrained(
|
|
"runwayml/stable-diffusion-v1-5",
|
|
torch_dtype=torch.float16,
|
|
)
|
|
pipe = pipe.to("cuda")
|
|
|
|
prompt = "A magical forest with glowing mushrooms"
|
|
|
|
# Fast and deterministic (recommended default)
|
|
pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config)
|
|
image_unipc = pipe(prompt, num_inference_steps=20).images[0]
|
|
|
|
# High quality, slower
|
|
pipe.scheduler = DPMSolverMultistepScheduler.from_config(pipe.scheduler.config)
|
|
image_dpm = pipe(prompt, num_inference_steps=25).images[0]
|
|
|
|
# Artistic/creative (non-deterministic)
|
|
pipe.scheduler = EulerAncestralDiscreteScheduler.from_config(pipe.scheduler.config)
|
|
image_euler = pipe(prompt, num_inference_steps=30).images[0]
|
|
|
|
# Original DDIM (fast, deterministic)
|
|
pipe.scheduler = DDIMScheduler.from_config(pipe.scheduler.config)
|
|
image_ddim = pipe(prompt, num_inference_steps=50).images[0]</code></pre>
|
|
<p><strong>When to use each</strong>:</p>
|
|
<ul><li>UniPC: Default choice (fast + high quality)</li><li>DPM-Solver: Maximum quality, moderate speed</li><li>Euler Ancestral: Creative/artistic styles</li><li>DDIM: When you need exact determinism</li></ul>
|
|
<h3>Guidance Scale Tuning</h3>
|
|
<pre><code class="lang-python">import torch
|
|
from diffusers import StableDiffusionPipeline
|
|
|
|
pipe = StableDiffusionPipeline.from_pretrained(
|
|
"runwayml/stable-diffusion-v1-5",
|
|
torch_dtype=torch.float16,
|
|
).to("cuda")
|
|
|
|
prompt = "A red apple on a wooden table"
|
|
|
|
# Generate with different guidance scales
|
|
guidance_scales = [1.0, 3.0, 7.5, 12.0, 20.0]
|
|
images = []
|
|
|
|
for scale in guidance_scales:
|
|
image = pipe(
|
|
prompt=prompt,
|
|
guidance_scale=scale,
|
|
num_inference_steps=30,
|
|
generator=torch.Generator("cuda").manual_seed(42),
|
|
).images[0]
|
|
images.append(image)
|
|
image.save(f"guidance_{scale}.png")
|
|
|
|
# Guidance scale effects:
|
|
# 1.0-3.0: More creative, less prompt adherence, diverse
|
|
# 7.0-9.0: Balanced (recommended range)
|
|
# 10.0-15.0: Strong prompt adherence, less variation
|
|
# 15.0+: Over-saturated, artifacts, "burnt" look</code></pre>
|
|
<p><strong>Guidance Scale Guidelines</strong>:</p>
|
|
<ul><li>1.0-3.0: Exploratory, artistic freedom</li><li>7.0-8.0: General purpose (SD 1.5)</li><li>5.0-7.0: General purpose (SDXL)</li><li>10.0+: When prompt is very specific</li></ul>
|
|
<h3>Batch Generation</h3>
|
|
<pre><code class="lang-python">from diffusers import StableDiffusionPipeline
|
|
import torch
|
|
|
|
pipe = StableDiffusionPipeline.from_pretrained(
|
|
"runwayml/stable-diffusion-v1-5",
|
|
torch_dtype=torch.float16,
|
|
).to("cuda")
|
|
|
|
prompt = "A futuristic cityscape at night"
|
|
|
|
# Generate multiple images in one call (efficient)
|
|
images = pipe(
|
|
prompt=prompt,
|
|
num_images_per_prompt=4, # Batch size (watch VRAM)
|
|
num_inference_steps=30,
|
|
guidance_scale=7.5,
|
|
).images
|
|
|
|
# Save all images
|
|
for i, img in enumerate(images):
|
|
img.save(f"batch_{i}.png")</code></pre>
|
|
<p><strong>VRAM considerations</strong>:</p>
|
|
<ul><li>SD 1.5: ~2GB per image at 512x512</li><li>SDXL: ~4-5GB per image at 1024x1024</li><li>Reduce batch size if OOM</li></ul>
|
|
<h3>Image-to-Image Generation</h3>
|
|
<pre><code class="lang-python">from diffusers import StableDiffusionImg2ImgPipeline
|
|
from PIL import Image
|
|
import torch
|
|
|
|
# Load img2img pipeline
|
|
pipe = StableDiffusionImg2ImgPipeline.from_pretrained(
|
|
"runwayml/stable-diffusion-v1-5",
|
|
torch_dtype=torch.float16,
|
|
).to("cuda")
|
|
|
|
# Load input image
|
|
init_image = Image.open("input.png").convert("RGB")
|
|
init_image = init_image.resize((512, 512))
|
|
|
|
prompt = "A watercolor painting of the same scene"
|
|
|
|
# Generate variation
|
|
image = pipe(
|
|
prompt=prompt,
|
|
image=init_image,
|
|
strength=0.75, # 0.0 = no change, 1.0 = full generation
|
|
num_inference_steps=50,
|
|
guidance_scale=7.5,
|
|
).images[0]
|
|
|
|
image.save("img2img_output.png")</code></pre>
|
|
<p><strong>Strength parameter</strong>:</p>
|
|
<ul><li>0.0-0.3: Minor adjustments, style transfer</li><li>0.4-0.6: Moderate changes, keep composition</li><li>0.7-0.9: Major changes, loose reference</li><li>0.9-1.0: Almost like text-to-image</li></ul>
|
|
<h3>Inpainting</h3>
|
|
<pre><code class="lang-python">from diffusers import StableDiffusionInpaintPipeline
|
|
from PIL import Image
|
|
import torch
|
|
|
|
pipe = StableDiffusionInpaintPipeline.from_pretrained(
|
|
"runwayml/stable-diffusion-inpainting",
|
|
torch_dtype=torch.float16,
|
|
).to("cuda")
|
|
|
|
# Load image and mask (white = inpaint area)
|
|
image = Image.open("input.png").convert("RGB").resize((512, 512))
|
|
mask = Image.open("mask.png").convert("RGB").resize((512, 512))
|
|
|
|
prompt = "A red sports car"
|
|
|
|
result = pipe(
|
|
prompt=prompt,
|
|
image=image,
|
|
mask_image=mask,
|
|
num_inference_steps=50,
|
|
guidance_scale=7.5,
|
|
).images[0]
|
|
|
|
result.save("inpainted.png")</code></pre>
|
|
<p><strong>Use cases</strong>:</p>
|
|
<ul><li>Object removal/replacement</li><li>Background changes</li><li>Detail refinement</li><li>Creative editing</li></ul>
|
|
<hr>
|
|
<h2>Quick Reference</h2>
|
|
<h3>Model Comparison</h3>
|
|
<pre><code>Model | Resolution | VRAM | Speed | Quality | Best For
|
|
---------------|------------|--------|-------|---------|------------------
|
|
SD 1.5 | 512x512 | 4GB | Fast | Good | General, fast iteration
|
|
SD 2.1 | 768x768 | 6GB | Mid | Better | Higher res, improved quality
|
|
SDXL 1.0 | 1024x1024 | 8-12GB | Slow | Best | Production, high quality
|
|
SD 3.0 | 1024x1024 | 10GB+ | Mid | Best | Latest, best text understanding</code></pre>
|
|
<h3>Inference Parameters</h3>
|
|
<pre><code>Parameter | Range | Default | Effect
|
|
----------------------|------------|---------|--------------------------------
|
|
num_inference_steps | 20-100 | 50 | Quality vs speed (diminishing returns >50)
|
|
guidance_scale | 1.0-20.0 | 7.5 | Prompt adherence (7-9 optimal)
|
|
height/width | 512-1024 | 512 | Output resolution (multiple of 64)
|
|
num_images_per_prompt | 1-8 | 1 | Batch generation (VRAM limited)
|
|
strength (img2img) | 0.0-1.0 | 0.8 | How much to change input</code></pre>
|
|
<h3>Scheduler Selection</h3>
|
|
<pre><code>✅ DO: Use UniPC or DPM-Solver for best quality/speed balance
|
|
✅ DO: Use Euler Ancestral for artistic/creative outputs
|
|
✅ DO: Reduce steps with better schedulers (20-30 vs 50)
|
|
✅ DO: Match scheduler to use case (deterministic vs creative)
|
|
|
|
❌ DON'T: Use DDPM unless you need exact original algorithm
|
|
❌ DON'T: Use 100+ steps (waste of compute, minimal gain)
|
|
❌ DON'T: Ignore scheduler choice (huge impact on results)</code></pre>
|
|
<h3>Prompt Engineering</h3>
|
|
<pre><code>✅ DO: Be specific and descriptive
|
|
✅ DO: Use quality tags ("highly detailed", "4k", "professional")
|
|
✅ DO: Use negative prompts to avoid common issues
|
|
✅ DO: Mention style/medium if important ("oil painting", "photograph")
|
|
|
|
❌ DON'T: Be vague or ambiguous
|
|
❌ DON'T: Forget negative prompts (prevents common artifacts)
|
|
❌ DON'T: Over-complicate (model has limits on prompt length)</code></pre>
|
|
<hr>
|
|
<h2>Anti-Patterns</h2>
|
|
<p>❌ <strong>Using default scheduler without consideration</strong>: DDPM is slow, better options exist ✅ Use UniPC, DPM-Solver, or Euler Ancestral based on use case</p>
|
|
<p>❌ <strong>Ignoring guidance scale</strong>: Using default 7.5 for everything ✅ Tune guidance scale (7-9 for SD 1.5, 5-7 for SDXL)</p>
|
|
<p>❌ <strong>Too many inference steps</strong>: Using 100+ steps for marginal gains ✅ Use 20-30 steps with good scheduler, 40-50 max for quality</p>
|
|
<p>❌ <strong>Wrong resolution</strong>: Generating 1024x1024 with SD 1.5 ✅ Use native resolution (512 for SD 1.5, 1024 for SDXL)</p>
|
|
<p>❌ <strong>No negative prompts</strong>: Forgetting to specify what to avoid ✅ Always use negative prompts ("blurry, low quality, distorted")</p>
|
|
<p>❌ <strong>Not setting seed for debugging</strong>: Non-reproducible results ✅ Use fixed seed during development/debugging</p>
|
|
<p>❌ <strong>Loading full precision models</strong>: Using float32 on GPU ✅ Use float16/bfloat16 for 2x speed and 50% VRAM reduction</p>
|
|
<p>❌ <strong>Ignoring safety checker overhead</strong>: Keeping it on when not needed ✅ Disable safety checker for faster inference (if appropriate)</p>
|
|
<hr>
|
|
<h2>Related Skills</h2>
|
|
<ul><li><code>stable-diffusion-deployment.md</code> - Production deployment, API setup, optimization</li><li><code>diffusion-finetuning.md</code> - Fine-tuning with DreamBooth, LoRA, textual inversion</li><li><code>modal-gpu-workloads.md</code> - GPU selection and configuration for diffusion models</li><li><code>modal-web-endpoints.md</code> - Creating API endpoints for diffusion inference</li><li><code>lora-peft-techniques.md</code> - LoRA for parameter-efficient fine-tuning</li></ul>
|
|
<hr>
|
|
<p><strong>Last Updated</strong>: 2025-10-18 <strong>Format Version</strong>: 1.0 (Atomic)</p>
|
|
<footer>
|
|
<div class="fmap">
|
|
<div><b>Build</b>
|
|
<a href="https://hanzo.app">hanzo.app</a> app builder
|
|
<a href="https://hanzo.codes">hanzo.codes</a> code with an agent
|
|
<a href="https://hanzoskills.com">hanzoskills.com</a> what an agent reads</div>
|
|
<div><b>Run</b>
|
|
<a href="https://console.hanzo.ai">console.hanzo.ai</a> cloud console
|
|
<a href="https://api.hanzo.ai">api.hanzo.ai</a> the API</div>
|
|
<div><b>Account</b>
|
|
<a href="https://hanzo.id">hanzo.id</a> sign in
|
|
<a href="https://pay.hanzo.ai">pay.hanzo.ai</a> top up
|
|
<a href="https://billing.hanzo.ai">billing.hanzo.ai</a> invoices and usage</div>
|
|
<div><b>Company</b>
|
|
<a href="https://hanzo.ai">hanzo.ai</a> canonical
|
|
<a href="https://hanzo.ventures">hanzo.ventures</a> ventures</div>
|
|
</div>
|
|
<div class="fend"><a href="/ml-diffusion-model-basics.md">This page as plain text</a> · <a href="/llms.txt">Index for agents</a></div>
|
|
</footer>
|
|
</div></body></html> |