mirror of
https://github.com/hanzoai/skills.git
synced 2026-11-19 00:47:36 +00:00
915 lines
42 KiB
HTML
915 lines
42 KiB
HTML
<!doctype html>
|
|
<html lang="en"><head>
|
|
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
<title>hip-0035</title>
|
|
<meta name="color-scheme" content="dark">
|
|
<style>
|
|
/* Committed to one visual world: a dark instrument panel. No light variant —
|
|
the dot field and the hairline rules only read on black, and a naive
|
|
inversion would wash both out. Every colour is painted explicitly so the page
|
|
never borrows a host background. */
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Pixel Square';src:url('/fonts/ZenPixel-Square.woff2') format('woff2');font-style:normal;font-display:swap}
|
|
|
|
:root{
|
|
--ink:#f4f4f2; --soft:#9c9c98; --faint:#6a6a66; --ground:#000; --panel:#0a0a0a;
|
|
--line:#232323; --line-hi:#3a3a3a; --accent:#f4f4f2;
|
|
--font-zen-sans:'Zen',-apple-system,BlinkMacSystemFont,"Segoe UI",Helvetica,Arial,sans-serif;
|
|
--font-zen-mono:'Zen Mono',ui-monospace,"SF Mono",SFMono-Regular,Menlo,Consolas,monospace;
|
|
--font-zen-pixel-square:'Zen Pixel Square',monospace;
|
|
--mono:var(--font-zen-mono);
|
|
--sans:var(--font-zen-sans);
|
|
/* Controls are pills and surfaces are soft, as on hanzo.ai. */
|
|
--r:999px; --r-soft:14px;
|
|
}
|
|
*{box-sizing:border-box}
|
|
html{-webkit-text-size-adjust:100%}
|
|
body{margin:0;background:var(--ground);color:var(--ink);font-family:var(--sans);
|
|
font-size:16px;line-height:1.65;-webkit-font-smoothing:antialiased}
|
|
.wrap{max-width:60rem;margin:0 auto;padding:0 1.75rem}
|
|
|
|
/* The dot field. A halftone made of one repeating radial-gradient, faded out
|
|
along the diagonal by a mask so it reads as a wave rather than wallpaper. */
|
|
/* No overflow clip here: the dot field is inset to the box already, and a clip
|
|
would cut off the copy-prompt menu. */
|
|
.hero{position:relative;border-bottom:1px solid var(--line)}
|
|
.hero::before{content:"";position:absolute;inset:0;
|
|
background-image:radial-gradient(var(--line-hi) 1px, transparent 1px);
|
|
background-size:7px 7px;
|
|
-webkit-mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
pointer-events:none}
|
|
.hero .wrap{position:relative;padding-top:5.5rem;padding-bottom:4rem}
|
|
|
|
nav{border-bottom:1px solid var(--line)}
|
|
nav .in{max-width:60rem;margin:0 auto;padding:1.1rem 1.75rem;display:flex;
|
|
justify-content:space-between;align-items:center;gap:1rem}
|
|
.brand{font-family:var(--mono);font-size:.85rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--ink);text-decoration:none;
|
|
display:inline-flex;align-items:center;gap:.6rem}
|
|
.brand i{font-style:normal;color:var(--faint)}
|
|
.mark{width:1.15em;height:1.15em;display:block;flex:0 0 auto}
|
|
nav a.up{font-family:var(--mono);font-size:.72rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--soft);text-decoration:none}
|
|
nav a.up:hover{color:var(--ink)}
|
|
|
|
h1{font-family:var(--mono);font-size:clamp(2rem,5.4vw,3.4rem);line-height:1.12;
|
|
letter-spacing:-.02em;margin:0 0 1.1rem;font-weight:500;text-wrap:balance}
|
|
h1 em{font-style:normal;color:var(--faint)}
|
|
.lede{color:var(--soft);font-size:1.05rem;max-width:52ch;margin:0 0 2.25rem}
|
|
|
|
.btns{display:flex;flex-wrap:wrap;gap:.7rem;margin:0 0 .5rem;align-items:center}
|
|
.btn{display:inline-flex;align-items:center;gap:.5rem;font-family:var(--mono);
|
|
font-size:.72rem;letter-spacing:.14em;text-transform:uppercase;
|
|
padding:.8rem 1.4rem;border:1px solid var(--line-hi);border-radius:var(--r);
|
|
color:var(--ink);text-decoration:none;background:transparent;cursor:pointer;
|
|
line-height:1.2;font-weight:400}
|
|
.btn:hover{background:#141414;text-decoration:none;border-color:var(--soft)}
|
|
.btn.solid{background:var(--ink);color:#000;border-color:var(--ink)}
|
|
.btn.solid:hover{background:#d8d8d4}
|
|
.btn .chev{width:.7em;height:.7em;flex:0 0 auto;transition:transform .15s ease}
|
|
.btn[aria-expanded="true"] .chev{transform:rotate(180deg)}
|
|
|
|
/* Copy-prompt selector. Each row copies the prompt for its own target. */
|
|
.copy{position:relative;display:inline-flex}
|
|
.menu{position:absolute;top:calc(100% + .5rem);left:0;z-index:20;min-width:21rem;
|
|
background:var(--panel);border:1px solid var(--line-hi);border-radius:var(--r-soft);
|
|
padding:.4rem;display:flex;flex-direction:column;
|
|
box-shadow:0 12px 32px rgba(0,0,0,.75)}
|
|
.menu[hidden]{display:none}
|
|
.mi{display:flex;align-items:center;gap:.7rem;width:100%;text-align:left;
|
|
padding:.62rem .8rem;border:0;border-radius:calc(var(--r-soft) - .35rem);
|
|
background:transparent;color:var(--ink);font-family:var(--mono);font-size:.76rem;
|
|
letter-spacing:.04em;text-decoration:none;cursor:pointer;line-height:1.3;
|
|
white-space:nowrap}
|
|
.mi:hover,.mi:focus-visible{background:#181818;outline:none}
|
|
.mi .mark{width:1.05rem;height:1.05rem;color:var(--soft)}
|
|
.mi:hover .mark{color:var(--ink)}
|
|
.mi u{margin-left:auto;color:var(--faint);text-decoration:none;font-size:.68rem;
|
|
letter-spacing:.1em;text-transform:uppercase}
|
|
|
|
/* Works-with bar: one scrolling row, name and mark per agent. */
|
|
.strip{border-bottom:1px solid var(--line);background:var(--panel)}
|
|
.strip .in{max-width:60rem;margin:0 auto;padding:.9rem 1.75rem;display:flex;
|
|
align-items:center;gap:1.25rem;font-family:var(--mono);font-size:.72rem;
|
|
letter-spacing:.1em;text-transform:uppercase;color:var(--faint)}
|
|
.strip b{color:var(--soft);font-weight:400;flex:0 0 auto;
|
|
border-right:1px solid var(--line);padding-right:1.25rem;align-self:stretch;
|
|
display:flex;align-items:center}
|
|
.bar{display:flex;align-items:center;gap:.5rem;flex:1 1 auto;min-width:0;
|
|
overflow-x:auto;scrollbar-width:none;scroll-snap-type:x proximity;
|
|
-webkit-overflow-scrolling:touch;
|
|
-webkit-mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent);
|
|
mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent)}
|
|
.bar::-webkit-scrollbar{display:none}
|
|
.agent{display:inline-flex;align-items:center;gap:.45rem;flex:0 0 auto;
|
|
padding:.4rem .85rem;border:1px solid var(--line);border-radius:var(--r);
|
|
color:var(--soft);white-space:nowrap;scroll-snap-align:start;
|
|
text-decoration:none;background:var(--ground)}
|
|
.agent:hover{color:var(--ink);border-color:var(--line-hi);text-decoration:none}
|
|
.agent .mark{width:.95rem;height:.95rem}
|
|
|
|
h2{font-family:var(--mono);font-size:1.1rem;letter-spacing:-.01em;margin:2.75rem 0 .75rem;font-weight:500}
|
|
h3{font-size:1rem;margin:2rem 0 .5rem;font-weight:600}
|
|
h4,h5,h6{font-size:.94rem;margin:1.5rem 0 .5rem;font-weight:600}
|
|
p{margin:0 0 1rem;max-width:70ch}
|
|
ul,ol{margin:0 0 1rem;padding-left:1.35rem;max-width:70ch}
|
|
li{margin:.22rem 0}
|
|
a{color:var(--ink);text-decoration:underline;text-underline-offset:3px;
|
|
text-decoration-color:var(--line-hi)}
|
|
a:hover{text-decoration-color:var(--ink)}
|
|
a:focus-visible{outline:1px solid var(--ink);outline-offset:3px}
|
|
code{font-family:var(--mono);font-size:.85em;background:var(--panel);
|
|
border:1px solid var(--line);border-radius:5px;padding:.1em .36em}
|
|
pre{background:var(--panel);border:1px solid var(--line);padding:1rem 1.1rem;
|
|
border-radius:var(--r-soft);overflow-x:auto;margin:0 0 1.25rem}
|
|
pre code{background:none;border:0;padding:0;font-size:.82rem;line-height:1.7}
|
|
blockquote{margin:0 0 1rem;padding-left:1rem;border-left:1px solid var(--line-hi);color:var(--soft)}
|
|
hr{border:0;border-top:1px solid var(--line);margin:2.5rem 0}
|
|
table{border-collapse:collapse;width:100%;margin:0 0 1.25rem;font-size:.92rem;
|
|
display:block;overflow-x:auto}
|
|
th,td{border:1px solid var(--line);padding:.5rem .7rem;text-align:left}
|
|
|
|
.curl{display:block;background:var(--panel);border:1px solid var(--line);
|
|
border-radius:var(--r-soft);padding:.95rem 1.1rem;font-family:var(--mono);
|
|
font-size:.8rem;color:var(--soft);overflow-x:auto;margin:0 0 2.5rem;
|
|
white-space:pre;line-height:1.9}
|
|
.curl b{color:var(--ink);font-weight:400}
|
|
.curl i{font-style:normal;color:var(--faint)}
|
|
|
|
/* index */
|
|
#q{width:100%;padding:.85rem 1.2rem;font-family:var(--mono);font-size:.85rem;color:var(--ink);
|
|
background:var(--panel);border:1px solid var(--line);border-radius:var(--r);margin:0 0 .6rem}
|
|
#q::placeholder{color:var(--faint);text-transform:uppercase;letter-spacing:.1em;font-size:.72rem}
|
|
#q:focus{outline:none;border-color:var(--soft)}
|
|
#count{color:var(--faint);font-family:var(--mono);font-size:.72rem;letter-spacing:.1em;
|
|
text-transform:uppercase;margin:0 0 2rem}
|
|
.cat{margin:2.5rem 0 .5rem;font-family:var(--mono);font-size:.7rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);border-bottom:1px solid var(--line);
|
|
padding-bottom:.45rem}
|
|
.row{display:block;padding:.6rem .9rem;margin:0 -.9rem;border-bottom:1px solid var(--line);
|
|
border-radius:var(--r-soft);text-decoration:none}
|
|
.row:hover{background:#0a0a0a}
|
|
.nm{font-family:var(--mono);font-size:.84rem;color:var(--ink);display:block}
|
|
.ds{color:var(--soft);font-size:.9rem;display:block;margin-top:.12rem}
|
|
footer{margin-top:4rem;padding:2rem 0 4rem;border-top:1px solid var(--line);
|
|
color:var(--faint);font-size:.85rem}
|
|
footer a{color:var(--soft)}
|
|
.fmap{display:grid;gap:1.75rem;grid-template-columns:repeat(auto-fit,minmax(11rem,1fr));
|
|
margin:0 0 2rem;font-size:.85rem}
|
|
.fmap b{display:block;font-family:var(--mono);font-size:.68rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);font-weight:400;margin:0 0 .6rem;
|
|
border-bottom:1px solid var(--line);padding-bottom:.4rem}
|
|
.fmap a{display:inline-block;color:var(--ink);text-decoration:none;margin:.3rem .35rem 0 0}
|
|
.fmap a:hover{text-decoration:underline}
|
|
.fmap div{color:var(--faint);line-height:1.5}
|
|
.fend{border-top:1px solid var(--line);padding-top:1.25rem;font-size:.82rem}
|
|
|
|
@media (max-width:640px){
|
|
.hero .wrap{padding-top:3.25rem;padding-bottom:2.75rem}
|
|
.btns{gap:.5rem}
|
|
.btn{padding:.75rem 1.15rem;font-size:.68rem}
|
|
.copy,.copy .btn{width:100%;justify-content:center}
|
|
.menu{left:0;right:0;min-width:0}
|
|
.strip .in{flex-direction:column;align-items:flex-start;gap:.6rem;padding:.9rem 1.75rem}
|
|
.strip b{border-right:0;padding-right:0}
|
|
.bar{width:100%}
|
|
}
|
|
</style>
|
|
<link rel="alternate" type="text/markdown" href="/hip-0035.md">
|
|
|
|
<link rel="icon" href="/favicon.svg">
|
|
<link rel="canonical" href="https://hanzoskills.com/hip-0035">
|
|
</head>
|
|
<!-- Reading this with a machine? Every page here is also plain markdown.
|
|
This one: https://hanzoskills.com/hip-0035.md
|
|
Index: https://hanzoskills.com/llms.txt
|
|
Corpus: https://hanzoskills.com/all.md
|
|
Append .md to any path for the markdown twin. -->
|
|
<body>
|
|
<nav><div class="in">
|
|
<a class="brand" href="/"><svg class="mark" viewBox="0 0 67 67" fill="currentColor" aria-hidden="true"><path d="M22.21 67V44.6369H0V67H22.21Z"/><path d="M66.7038 22.3184H22.2534L0.0878906 44.6367H44.4634L66.7038 22.3184Z"/><path d="M22.21 0H0V22.3184H22.21V0Z"/><path d="M66.7198 0H44.5098V22.3184H66.7198V0Z"/><path d="M66.7198 67V44.6369H44.5098V67H66.7198Z"/></svg> HANZO<i>SKILLS</i></a>
|
|
<a class="up" href="/">All skills</a>
|
|
</div></nav>
|
|
|
|
<div class="wrap">
|
|
<h1>hip-0035</h1><p class="lede">HIP-35: Image & Video Generation Standard. Status Draft. Hanzo's own standard — read this before implementing against it.</p><h1>HIP-0035: Image & Video Generation Standard</h1>
|
|
<h2>Abstract</h2>
|
|
<p>This proposal defines the standard for visual AI generation workflows across the Hanzo ecosystem, encompassing image generation, video synthesis, and 3D asset creation. It specifies two complementary interfaces: <strong>Hanzo Studio</strong>, a node-based visual programming environment for composing diffusion pipelines, and <strong>Hanzo Painter</strong>, a simplified prompt-to-image interface for casual generation. Both share a common backend compute layer with queue-based GPU scheduling.</p>
|
|
<p><strong>Repositories</strong>:</p>
|
|
<ul><li>Studio: <a href="https://github.com/hanzoai/studio">github.com/hanzoai/studio</a></li><li>Painter: <a href="https://github.com/hanzoai/painter">github.com/hanzoai/painter</a></li></ul>
|
|
<p><strong>Ports</strong>:</p>
|
|
<ul><li>Studio: 8188 (ComfyUI default)</li><li>Painter: 3035</li></ul>
|
|
<p><strong>Docker</strong>:</p>
|
|
<ul><li><code>hanzoai/studio:latest</code></li><li><code>hanzoai/painter:latest</code></li></ul>
|
|
<h2>Design Philosophy</h2>
|
|
<p>This section explains the architectural reasoning behind every major decision. These are not arbitrary choices -- each one follows from a specific constraint or tradeoff.</p>
|
|
<h3>Why Self-Hosted Over Cloud APIs</h3>
|
|
<p>Consider the economics of image generation at scale:</p>
|
|
<p>| Approach | Cost per image | Cold start | Control | |----------|---------------|------------|---------| | Replicate (SDXL) | $0.0023 | 2-10s | None | | RunPod Serverless | $0.0019 | 3-15s | Limited | | Midjourney | $0.01-0.04 | 0s (queued) | None | | Self-hosted (A100) | $0.0002-0.0005 | 0s | Full |</p>
|
|
<p>At 10,000 generations/day, self-hosted saves $5,000-15,000/month. At 100,000/day, the savings are transformative. Self-hosting also eliminates cold starts (models stay loaded in VRAM), enables custom model merges and LoRAs, and keeps generated content on our infrastructure.</p>
|
|
<p>The tradeoff is operational complexity: we must manage GPU hardware, model storage, and queue scheduling. This is acceptable because Hanzo already operates GPU infrastructure for LLM inference (HIP-0004).</p>
|
|
<h2>Specification</h2>
|
|
<h3>Node Graph Architecture</h3>
|
|
<p>The execution model is a directed acyclic graph (DAG) where each node represents a discrete operation:</p>
|
|
<pre><code>┌──────────────┐ ┌───────────────┐ ┌──────────────┐
|
|
│ Load SDXL │────▶│ CLIP Encode │────▶│ KSampler │
|
|
│ Checkpoint │ │ (Positive) │ │ │
|
|
└──────────────┘ └───────────────┘ │ model ──────│
|
|
│ positive ───│
|
|
┌──────────────┐ ┌───────────────┐ │ negative ───│
|
|
│ Empty Latent │────▶│ │────▶│ latent ─────│
|
|
│ Image │ │ CLIP Encode │ └──────┬───────┘
|
|
└──────────────┘ │ (Negative) │ │
|
|
└───────────────┘ ▼
|
|
┌──────────────┐
|
|
│ VAE Decode │
|
|
└──────┬───────┘
|
|
│
|
|
▼
|
|
┌──────────────┐
|
|
│ Save Image │
|
|
└──────────────┘</code></pre>
|
|
<h4>Node Interface</h4>
|
|
<pre><code class="lang-typescript">interface NodeDefinition {
|
|
// Unique node type identifier
|
|
type: string;
|
|
|
|
// Human-readable display name
|
|
display_name: string;
|
|
|
|
// Category for UI grouping
|
|
category: string;
|
|
|
|
// Input slots
|
|
inputs: Record<string, {
|
|
type: DataType;
|
|
required: boolean;
|
|
default?: any;
|
|
tooltip?: string;
|
|
// For numeric inputs
|
|
min?: number;
|
|
max?: number;
|
|
step?: number;
|
|
// For enum inputs
|
|
options?: string[];
|
|
}>;
|
|
|
|
// Output slots
|
|
outputs: Record<string, {
|
|
type: DataType;
|
|
}>;
|
|
|
|
// Execution function identifier
|
|
function: string;
|
|
}
|
|
|
|
type DataType =
|
|
| "MODEL" // Diffusion model
|
|
| "CLIP" // Text encoder
|
|
| "VAE" // Variational autoencoder
|
|
| "CONDITIONING" // Encoded text/image conditioning
|
|
| "LATENT" // Latent space tensor
|
|
| "IMAGE" // Decoded pixel image (RGB float32)
|
|
| "MASK" // Single-channel mask
|
|
| "CONTROL_NET" // ControlNet model
|
|
| "LORA" // LoRA weights
|
|
| "INT" // Integer parameter
|
|
| "FLOAT" // Float parameter
|
|
| "STRING" // Text parameter
|
|
| "BOOLEAN"; // Boolean parameter</code></pre>
|
|
<h4>Workflow Format</h4>
|
|
<p>Workflows are serialized as JSON for storage, versioning, and sharing:</p>
|
|
<pre><code class="lang-json">{
|
|
"version": 1,
|
|
"nodes": {
|
|
"1": {
|
|
"type": "CheckpointLoaderSimple",
|
|
"inputs": {
|
|
"ckpt_name": "sdxl_base_1.0.safetensors"
|
|
}
|
|
},
|
|
"2": {
|
|
"type": "CLIPTextEncode",
|
|
"inputs": {
|
|
"text": "a photorealistic mountain landscape at sunset",
|
|
"clip": ["1", "CLIP"]
|
|
}
|
|
},
|
|
"3": {
|
|
"type": "CLIPTextEncode",
|
|
"inputs": {
|
|
"text": "blurry, low quality, watermark",
|
|
"clip": ["1", "CLIP"]
|
|
}
|
|
},
|
|
"4": {
|
|
"type": "EmptyLatentImage",
|
|
"inputs": {
|
|
"width": 1024,
|
|
"height": 1024,
|
|
"batch_size": 1
|
|
}
|
|
},
|
|
"5": {
|
|
"type": "KSampler",
|
|
"inputs": {
|
|
"model": ["1", "MODEL"],
|
|
"positive": ["2", "CONDITIONING"],
|
|
"negative": ["3", "CONDITIONING"],
|
|
"latent_image": ["4", "LATENT"],
|
|
"seed": 42,
|
|
"steps": 30,
|
|
"cfg": 7.5,
|
|
"sampler_name": "euler_ancestral",
|
|
"scheduler": "karras",
|
|
"denoise": 1.0
|
|
}
|
|
},
|
|
"6": {
|
|
"type": "VAEDecode",
|
|
"inputs": {
|
|
"samples": ["5", "LATENT"],
|
|
"vae": ["1", "VAE"]
|
|
}
|
|
},
|
|
"7": {
|
|
"type": "SaveImage",
|
|
"inputs": {
|
|
"images": ["6", "IMAGE"],
|
|
"filename_prefix": "hanzo_gen"
|
|
}
|
|
}
|
|
}
|
|
}</code></pre>
|
|
<h3>Model Format Support</h3>
|
|
<pre><code class="lang-yaml">Formats:
|
|
safetensors:
|
|
description: Primary format. Memory-mapped, fast loading, no pickle exploits.
|
|
extensions: [".safetensors"]
|
|
recommended: true
|
|
|
|
GGUF:
|
|
description: Quantized format for reduced VRAM usage.
|
|
extensions: [".gguf"]
|
|
use_case: Running large models on consumer GPUs (8-16GB VRAM).
|
|
|
|
ONNX:
|
|
description: Cross-platform inference format.
|
|
extensions: [".onnx"]
|
|
use_case: Candle backend, DirectML (Windows), CoreML (macOS).</code></pre>
|
|
<h3>Supported Architectures</h3>
|
|
<pre><code class="lang-yaml">Image Generation:
|
|
Stable Diffusion 1.5:
|
|
resolution: 512x512
|
|
vram: 4GB minimum
|
|
status: Legacy, widely supported
|
|
lora_compatible: true
|
|
|
|
Stable Diffusion XL:
|
|
resolution: 1024x1024
|
|
vram: 8GB minimum
|
|
status: Production standard
|
|
lora_compatible: true
|
|
refiner: Optional two-stage pipeline
|
|
|
|
Stable Diffusion 3 / 3.5:
|
|
resolution: 1024x1024
|
|
vram: 12GB minimum
|
|
status: Current generation
|
|
text_encoders: [CLIP-L, CLIP-G, T5-XXL]
|
|
architecture: MMDiT (Multi-Modal Diffusion Transformer)
|
|
|
|
Flux:
|
|
resolution: Up to 2048x2048
|
|
vram: 12-24GB
|
|
status: Current generation
|
|
variants: [dev, schnell, pro]
|
|
architecture: Rectified flow transformer
|
|
lora_compatible: true
|
|
|
|
Video Generation:
|
|
Stable Video Diffusion (SVD):
|
|
input: Single image
|
|
output: 14-25 frames
|
|
resolution: 576x1024
|
|
vram: 16GB minimum
|
|
|
|
AnimateDiff:
|
|
input: Text prompt or image + motion module
|
|
output: 16-32 frames
|
|
resolution: 512x512
|
|
vram: 12GB minimum
|
|
|
|
CogVideo:
|
|
input: Text prompt
|
|
output: Up to 6 seconds
|
|
resolution: 480x720
|
|
vram: 24GB minimum
|
|
|
|
External Proxies:
|
|
DALL-E 3:
|
|
type: API proxy via HIP-0004
|
|
resolution: 1024x1024, 1024x1792, 1792x1024
|
|
note: Routed through LLM Gateway, billed per generation</code></pre>
|
|
<h3>LoRA, ControlNet, and IP-Adapter</h3>
|
|
<p>These are the three primary conditioning mechanisms for guiding generation beyond text prompts:</p>
|
|
<pre><code class="lang-yaml">LoRA (Low-Rank Adaptation):
|
|
purpose: Fine-tuned style or subject weights merged at inference time
|
|
format: safetensors
|
|
application: Merged into MODEL before sampling
|
|
stacking: Multiple LoRAs can be applied with independent strength weights
|
|
example_node: LoraLoader
|
|
inputs:
|
|
model: MODEL
|
|
clip: CLIP
|
|
lora_name: "pixel_art_v2.safetensors"
|
|
strength_model: 0.8
|
|
strength_clip: 0.8
|
|
|
|
ControlNet:
|
|
purpose: Spatial conditioning from reference images (edges, depth, pose)
|
|
preprocessors:
|
|
- Canny (edge detection)
|
|
- Depth (MiDaS, Zoe)
|
|
- OpenPose (human skeleton)
|
|
- Scribble (hand-drawn guides)
|
|
- Tile (detail preservation for upscaling)
|
|
- Inpainting (masked region fill)
|
|
application: Injected as additional conditioning into the sampler
|
|
example_node: ControlNetApply
|
|
inputs:
|
|
conditioning: CONDITIONING
|
|
control_net: CONTROL_NET
|
|
image: IMAGE (preprocessed reference)
|
|
strength: 0.75
|
|
|
|
IP-Adapter:
|
|
purpose: Image-prompt conditioning (style transfer from reference images)
|
|
application: Encodes a reference image into CLIP embedding space
|
|
variants: [ip-adapter, ip-adapter-plus, ip-adapter-face]
|
|
example_node: IPAdapterApply
|
|
inputs:
|
|
model: MODEL
|
|
image: IMAGE (reference)
|
|
weight: 0.7
|
|
noise: 0.3</code></pre>
|
|
<h3>API Specification</h3>
|
|
<h4>Queue a Workflow</h4>
|
|
<pre><code class="lang-yaml">POST /api/prompt
|
|
Headers:
|
|
Authorization: Bearer <token>
|
|
Content-Type: application/json
|
|
Body:
|
|
client_id: string # WebSocket client ID for progress updates
|
|
prompt: WorkflowJSON # The node graph (see Workflow Format above)
|
|
extra_data:
|
|
extra_pnginfo: # Metadata embedded in output PNG
|
|
workflow: WorkflowJSON
|
|
Response:
|
|
prompt_id: string # UUID for tracking
|
|
number: integer # Queue position
|
|
node_errors: {} # Validation errors (empty if valid)</code></pre>
|
|
<h4>Check Queue Status</h4>
|
|
<pre><code class="lang-yaml">GET /api/queue
|
|
Response:
|
|
queue_running: # Currently executing
|
|
- prompt_id: string
|
|
workflow: WorkflowJSON
|
|
started: timestamp
|
|
queue_pending: # Waiting for GPU
|
|
- prompt_id: string
|
|
number: integer
|
|
queued: timestamp</code></pre>
|
|
<h4>Get Generation History</h4>
|
|
<pre><code class="lang-yaml">GET /api/history
|
|
Query:
|
|
prompt_id?: string # Filter by specific generation
|
|
max_items?: integer # Pagination (default: 200)
|
|
Response:
|
|
<prompt_id>:
|
|
prompt: WorkflowJSON
|
|
outputs:
|
|
<node_id>:
|
|
images:
|
|
- filename: string
|
|
subfolder: string
|
|
type: "output"
|
|
status:
|
|
status_str: "success" | "error"
|
|
completed: boolean
|
|
messages: [[timestamp, message], ...]</code></pre>
|
|
<h4>Retrieve Generated Image</h4>
|
|
<pre><code class="lang-yaml">GET /api/view
|
|
Query:
|
|
filename: string
|
|
subfolder?: string
|
|
type: "output" | "input" | "temp"
|
|
Response:
|
|
Content-Type: image/png | image/jpeg | image/webp
|
|
Body: Raw image bytes</code></pre>
|
|
<h4>WebSocket Progress</h4>
|
|
<pre><code class="lang-yaml">WS /ws?clientId=<client_id>
|
|
|
|
# Server -> Client messages:
|
|
|
|
# Execution started
|
|
{"type": "execution_start", "data": {"prompt_id": "..."}}
|
|
|
|
# Node execution progress
|
|
{"type": "progress", "data": {"value": 15, "max": 30, "prompt_id": "..."}}
|
|
|
|
# Node execution complete
|
|
{"type": "executed", "data": {"node": "5", "output": {"images": [...]}}}
|
|
|
|
# Full execution complete
|
|
{"type": "execution_complete", "data": {"prompt_id": "..."}}
|
|
|
|
# Error during execution
|
|
{"type": "execution_error", "data": {"prompt_id": "...", "node_id": "5", "exception_message": "..."}}</code></pre>
|
|
<h3>Output Formats</h3>
|
|
<pre><code class="lang-yaml">Images:
|
|
PNG:
|
|
default: true
|
|
metadata: Workflow JSON embedded in PNG tEXt chunk
|
|
use_case: Lossless, preserves workflow for re-import
|
|
JPEG:
|
|
quality: 85-95
|
|
use_case: Web delivery, smaller file size
|
|
WebP:
|
|
quality: 80-95
|
|
use_case: Web delivery, best compression ratio
|
|
|
|
Video:
|
|
MP4:
|
|
codec: H.264 or H.265
|
|
fps: 8-30 (configurable)
|
|
use_case: Standard video delivery
|
|
GIF:
|
|
use_case: Short animations, social media
|
|
max_frames: 64
|
|
WebM:
|
|
codec: VP9
|
|
use_case: Web-optimized video</code></pre>
|
|
<h3>Painter API (Simplified Interface)</h3>
|
|
<p>Painter exposes a high-level API that maps to Studio workflows internally:</p>
|
|
<pre><code class="lang-yaml">POST /api/generate
|
|
Headers:
|
|
Authorization: Bearer <token>
|
|
Body:
|
|
prompt: string # Positive prompt
|
|
negative_prompt?: string # Negative prompt (default: quality negatives)
|
|
model?: string # Model name (default: "flux-schnell")
|
|
width?: integer # Output width (default: 1024)
|
|
height?: integer # Output height (default: 1024)
|
|
steps?: integer # Sampling steps (default: model-dependent)
|
|
cfg_scale?: float # Classifier-free guidance (default: 7.0)
|
|
seed?: integer # Seed for reproducibility (-1 for random)
|
|
style?: string # Style preset name
|
|
num_images?: integer # Batch count (1-4, default: 1)
|
|
reference_image?: string # URL or base64 for IP-Adapter
|
|
control_image?: string # URL or base64 for ControlNet
|
|
control_type?: string # "canny" | "depth" | "pose" | "scribble"
|
|
Response:
|
|
generation_id: string
|
|
images:
|
|
- url: string
|
|
width: integer
|
|
height: integer
|
|
seed: integer
|
|
workflow_id: string # Can be opened in Studio
|
|
usage:
|
|
model: string
|
|
steps: integer
|
|
compute_ms: integer
|
|
cost_credits: float</code></pre>
|
|
<h2>Implementation</h2>
|
|
<h3>System Architecture</h3>
|
|
<pre><code>┌──────────────────────────────────────────────────────────────────┐
|
|
│ Clients │
|
|
│ ┌──────────┐ ┌──────────┐ ┌──────────┐ ┌────────────────┐ │
|
|
│ │ Studio │ │ Painter │ │ API │ │ MCP Tool │ │
|
|
│ │ (Nodes) │ │ (React) │ │ Clients │ │ (HIP-0010) │ │
|
|
│ └─────┬─────┘ └────┬─────┘ └────┬─────┘ └───────┬────────┘ │
|
|
└────────┼──────────────┼─────────────┼────────────────┼───────────┘
|
|
│ │ │ │
|
|
▼ ▼ ▼ ▼
|
|
┌──────────────────────────────────────────────────────────────────┐
|
|
│ Studio API Server (:8188) │
|
|
│ ┌──────────────┐ ┌──────────────┐ ┌────────────────────────┐ │
|
|
│ │ Workflow │ │ Prompt │ │ Model Manager │ │
|
|
│ │ Validator │ │ Queue │ │ (load/unload/cache) │ │
|
|
│ └──────────────┘ └──────┬───────┘ └────────────────────────┘ │
|
|
└────────────────────────────┼─────────────────────────────────────┘
|
|
│
|
|
▼
|
|
┌──────────────────────────────────────────────────────────────────┐
|
|
│ GPU Worker Pool │
|
|
│ ┌──────────┐ ┌──────────┐ ┌──────────┐ ┌──────────┐ │
|
|
│ │ Worker 0 │ │ Worker 1 │ │ Worker 2 │ │ Worker N │ │
|
|
│ │ A100 80G │ │ A100 80G │ │ RTX 4090 │ │ (Candle) │ │
|
|
│ │ PyTorch │ │ PyTorch │ │ PyTorch │ │ CPU/Metal│ │
|
|
│ └──────────┘ └──────────┘ └──────────┘ └──────────┘ │
|
|
└──────────────────────────────────────────────────────────────────┘
|
|
│ │ │ │
|
|
▼ ▼ ▼ ▼
|
|
┌──────────────────────────────────────────────────────────────────┐
|
|
│ Storage Layer │
|
|
│ ┌───────────────┐ ┌───────────────┐ ┌───────────────────┐ │
|
|
│ │ Model Storage │ │ Output Storage│ │ Workflow Storage │ │
|
|
│ │ (S3/MinIO) │ │ (S3/MinIO) │ │ (SQL ) │ │
|
|
│ │ safetensors │ │ PNG/MP4 │ │ JSON documents │ │
|
|
│ └───────────────┘ └───────────────┘ └───────────────────┘ │
|
|
└──────────────────────────────────────────────────────────────────┘</code></pre>
|
|
<h3>GPU Scheduling</h3>
|
|
<p>The prompt queue implements a priority-based FIFO scheduler:</p>
|
|
<pre><code class="lang-python">class GPUScheduler:
|
|
"""
|
|
Assigns queued prompts to available GPU workers.
|
|
Workers are selected based on VRAM capacity and loaded models.
|
|
"""
|
|
def schedule(self, prompt: QueuedPrompt) -> Worker:
|
|
# 1. Determine required models from workflow
|
|
required_models = self.extract_models(prompt.workflow)
|
|
total_vram = self.estimate_vram(required_models)
|
|
|
|
# 2. Prefer worker that already has models loaded (avoid reload)
|
|
for worker in self.workers:
|
|
if worker.has_models_loaded(required_models):
|
|
return worker
|
|
|
|
# 3. Find worker with sufficient free VRAM
|
|
for worker in self.available_workers():
|
|
if worker.free_vram >= total_vram:
|
|
return worker
|
|
|
|
# 4. If no worker fits, queue for next available
|
|
return self.enqueue_waiting(prompt)
|
|
|
|
def estimate_vram(self, models: list[str]) -> int:
|
|
"""
|
|
SDXL base: ~6.5GB
|
|
SDXL refiner: ~6.5GB
|
|
Flux dev: ~12GB (fp16) or ~6GB (fp8)
|
|
ControlNet: ~1.5GB each
|
|
LoRA: ~200MB each (merged, no extra VRAM at inference)
|
|
T5-XXL: ~10GB (fp16) or ~5GB (fp8)
|
|
"""
|
|
return sum(self.model_vram_map[m] for m in models)</code></pre>
|
|
<h3>Model Storage</h3>
|
|
<p>Models are stored in Object Storage (S3/MinIO) and cached locally on GPU workers:</p>
|
|
<pre><code class="lang-yaml">Storage Layout:
|
|
s3://hanzo-models/
|
|
checkpoints/
|
|
sdxl_base_1.0.safetensors # 6.94GB
|
|
flux1-dev.safetensors # 23.8GB (fp16)
|
|
flux1-dev-fp8.safetensors # 11.9GB (fp8)
|
|
sd3.5_large.safetensors # 16.5GB
|
|
loras/
|
|
pixel_art_v2.safetensors # 150MB
|
|
film_grain.safetensors # 200MB
|
|
controlnet/
|
|
control_v11p_sd15_canny.safetensors
|
|
controlnet-sdxl-depth.safetensors
|
|
vae/
|
|
sdxl_vae.safetensors
|
|
clip/
|
|
t5xxl_fp16.safetensors
|
|
upscale/
|
|
4x-UltraSharp.pth
|
|
|
|
Worker Local Cache:
|
|
/models/ # Fast NVMe SSD
|
|
checkpoints/ # Most-used models pinned
|
|
loras/ # LRU eviction
|
|
controlnet/
|
|
Cache Policy:
|
|
max_size: 200GB per worker
|
|
eviction: LRU with pinning support
|
|
prefetch: Preload models referenced in queued prompts</code></pre>
|
|
<h3>Hanzo Custom Nodes</h3>
|
|
<p>Studio ships with Hanzo-specific custom nodes that integrate with the broader ecosystem:</p>
|
|
<pre><code class="lang-yaml">Custom Nodes:
|
|
HanzoModelLoader:
|
|
description: Load models from Hanzo Object Storage with automatic caching
|
|
inputs:
|
|
model_id: STRING # Hanzo model registry ID
|
|
precision: ENUM # fp32, fp16, fp8, int8
|
|
outputs:
|
|
model: MODEL
|
|
clip: CLIP
|
|
vae: VAE
|
|
|
|
HanzoSaveToStorage:
|
|
description: Save output directly to Hanzo Object Storage
|
|
inputs:
|
|
images: IMAGE
|
|
bucket: STRING
|
|
path: STRING
|
|
format: ENUM # png, jpeg, webp
|
|
|
|
HanzoContentFilter:
|
|
description: NSFW detection gate -- blocks unsafe content
|
|
inputs:
|
|
images: IMAGE
|
|
threshold: FLOAT # 0.0-1.0 (default: 0.85)
|
|
outputs:
|
|
safe_images: IMAGE
|
|
flagged: BOOLEAN
|
|
|
|
HanzoBillingMeter:
|
|
description: Records compute usage for billing via HIP-0004
|
|
inputs:
|
|
images: IMAGE # Pass-through
|
|
user_id: STRING
|
|
model_name: STRING
|
|
steps: INT
|
|
outputs:
|
|
images: IMAGE # Unchanged pass-through
|
|
|
|
HanzoWatermark:
|
|
description: Optional invisible watermark for provenance tracking
|
|
inputs:
|
|
images: IMAGE
|
|
metadata: STRING # JSON metadata to embed
|
|
outputs:
|
|
images: IMAGE</code></pre>
|
|
<h3>Deployment</h3>
|
|
<h4>Docker Compose (Development)</h4>
|
|
<pre><code class="lang-yaml">services:
|
|
studio:
|
|
image: hanzoai/studio:latest
|
|
ports:
|
|
- "8188:8188"
|
|
volumes:
|
|
- ./models:/models
|
|
- ./output:/output
|
|
- ./workflows:/workflows
|
|
environment:
|
|
- HANZO_API_KEY=${HANZO_API_KEY}
|
|
- S3_ENDPOINT=${S3_ENDPOINT}
|
|
- S3_ACCESS_KEY=${S3_ACCESS_KEY}
|
|
- S3_SECRET_KEY=${S3_SECRET_KEY}
|
|
- S3_MODEL_BUCKET=hanzo-models
|
|
deploy:
|
|
resources:
|
|
reservations:
|
|
devices:
|
|
- driver: nvidia
|
|
count: all
|
|
capabilities: [gpu]
|
|
|
|
painter:
|
|
image: hanzoai/painter:latest
|
|
ports:
|
|
- "3035:3035"
|
|
environment:
|
|
- STUDIO_URL=http://studio:8188
|
|
- HANZO_API_KEY=${HANZO_API_KEY}
|
|
- DATABASE_URL=postgresql://user:pass@db:5432/painter
|
|
depends_on:
|
|
- studio
|
|
|
|
db:
|
|
image: postgres:16-alpine
|
|
environment:
|
|
- POSTGRES_DB=painter
|
|
- POSTGRES_USER=user
|
|
- POSTGRES_PASSWORD=pass
|
|
volumes:
|
|
- painter_data:/var/lib/postgresql/data
|
|
|
|
volumes:
|
|
painter_data:</code></pre>
|
|
<h4>Kubernetes (Production)</h4>
|
|
<pre><code class="lang-yaml">apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: studio-worker
|
|
namespace: hanzo
|
|
spec:
|
|
replicas: 3
|
|
selector:
|
|
matchLabels:
|
|
app: studio-worker
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: studio-worker
|
|
spec:
|
|
containers:
|
|
- name: studio
|
|
image: hanzoai/studio:latest
|
|
ports:
|
|
- containerPort: 8188
|
|
resources:
|
|
limits:
|
|
nvidia.com/gpu: 1
|
|
memory: "32Gi"
|
|
requests:
|
|
nvidia.com/gpu: 1
|
|
memory: "24Gi"
|
|
volumeMounts:
|
|
- name: model-cache
|
|
mountPath: /models
|
|
env:
|
|
- name: HANZO_API_KEY
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: hanzo-secrets
|
|
key: api-key
|
|
nodeSelector:
|
|
gpu-type: a100
|
|
volumes:
|
|
- name: model-cache
|
|
hostPath:
|
|
path: /mnt/nvme/models
|
|
type: DirectoryOrCreate</code></pre>
|
|
<h2>Security Considerations</h2>
|
|
<h3>Content Moderation</h3>
|
|
<p>All generated images pass through the <code>HanzoContentFilter</code> node before delivery:</p>
|
|
<ol><li><strong>NSFW Detection</strong>: Multi-label classifier (nudity, violence, hate symbols) with configurable threshold</li><li><strong>Prompt Filtering</strong>: Blocklist + semantic similarity check against known harmful prompts</li><li><strong>Watermarking</strong>: Optional invisible watermark encoding generation metadata (user, timestamp, model, prompt hash) for provenance tracking</li><li><strong>Audit Log</strong>: Every generation is logged with prompt, parameters, user ID, and moderation result</li></ol>
|
|
<h3>Rate Limiting</h3>
|
|
<pre><code class="lang-yaml">Tiers:
|
|
free:
|
|
generations_per_day: 50
|
|
max_resolution: 1024x1024
|
|
max_steps: 30
|
|
max_batch: 1
|
|
models: [flux-schnell, sdxl]
|
|
|
|
pro:
|
|
generations_per_day: 1000
|
|
max_resolution: 2048x2048
|
|
max_steps: 50
|
|
max_batch: 4
|
|
models: all
|
|
priority_queue: true
|
|
|
|
enterprise:
|
|
generations_per_day: unlimited
|
|
max_resolution: 4096x4096
|
|
max_steps: 100
|
|
max_batch: 8
|
|
models: all
|
|
priority_queue: true
|
|
dedicated_gpu: optional</code></pre>
|
|
<h3>Model Provenance</h3>
|
|
<p>Every model in the registry includes:</p>
|
|
<ul><li><strong>Source</strong>: Original model card URL (HuggingFace, Civitai)</li><li><strong>License</strong>: SPDX identifier (CreativeML-OpenRAIL-M, Apache-2.0, etc.)</li><li><strong>SHA-256</strong>: Hash of the safetensors file for integrity verification</li><li><strong>Scan status</strong>: Result of malware/pickle scan (safetensors files are inherently safe; .ckpt files are scanned)</li></ul>
|
|
<h3>Input Validation</h3>
|
|
<ul><li>Maximum prompt length: 10,000 characters</li><li>Maximum workflow size: 1MB JSON</li><li>Maximum node count per workflow: 500</li><li>Image upload limits: 20MB per image, PNG/JPEG/WebP only</li><li>All user-provided file paths are sandboxed to prevent directory traversal</li></ul>
|
|
<h2>Performance Targets</h2>
|
|
<pre><code class="lang-yaml">Image Generation:
|
|
SDXL 1024x1024 30 steps (A100): < 3 seconds
|
|
Flux schnell 1024x1024 4 steps (A100): < 2 seconds
|
|
Flux dev 1024x1024 30 steps (A100): < 8 seconds
|
|
|
|
Video Generation:
|
|
SVD 14 frames 576x1024 (A100): < 30 seconds
|
|
AnimateDiff 16 frames 512x512 (A100): < 15 seconds
|
|
|
|
Queue:
|
|
Time to first byte (prompt submission to WebSocket ack): < 100ms
|
|
Queue throughput: > 100 generations/minute per A100
|
|
|
|
Storage:
|
|
Model load time (NVMe cache hit): < 5 seconds
|
|
Model load time (S3 fetch, 10GB model): < 60 seconds
|
|
Output image delivery (S3): < 200ms</code></pre>
|
|
<h2>Integration Points</h2>
|
|
<pre><code>HIP-0004 (LLM Gateway) ──▶ DALL-E proxy, prompt enhancement via LLM
|
|
HIP-0010 (MCP) ──▶ "generate_image" tool for AI agents
|
|
HIP-0013 (Workflows) ──▶ Image generation as workflow step
|
|
HIP-0017 (Analytics) ──▶ Generation events, usage metrics
|
|
HIP-0018 (Payments) ──▶ Credit billing per generation
|
|
HIP-0019 (Candle) ──▶ Rust inference backend for CPU/Metal
|
|
HIP-0032 (Storage) ──▶ Model weights and output image storage</code></pre>
|
|
<h2>References</h2>
|
|
<ol><li><a href="https://docs.comfy.org/">ComfyUI Documentation</a></li><li><a href="https://arxiv.org/abs/2112.10752">Stable Diffusion Paper (Rombach et al., 2022)</a></li><li><a href="https://blackforestlabs.ai/">Flux Architecture (Black Forest Labs)</a></li><li><a href="https://arxiv.org/abs/2302.05543">ControlNet (Zhang et al., 2023)</a></li><li><a href="https://arxiv.org/abs/2308.06721">IP-Adapter (Ye et al., 2023)</a></li><li><a href="https://arxiv.org/abs/2106.09685">LoRA (Hu et al., 2021)</a></li><li><a href="./hip-0004-llm-gateway-unified-ai-provider-interface.md">HIP-4: LLM Gateway</a></li><li><a href="./hip-0019-tensor-operations-standard.md">HIP-19: Tensor Operations Standard</a></li><li><a href="https://github.com/hanzoai/studio">Hanzo Studio Repository</a></li><li><a href="https://github.com/hanzoai/painter">Hanzo Painter Repository</a></li></ol>
|
|
<h2>Copyright</h2>
|
|
<p>Copyright and related rights waived via <a href="https://creativecommons.org/publicdomain/zero/1.0/">CC0</a>.</p>
|
|
<footer>
|
|
<div class="fmap">
|
|
<div><b>Build</b>
|
|
<a href="https://hanzo.app">hanzo.app</a> app builder
|
|
<a href="https://hanzo.codes">hanzo.codes</a> code with an agent
|
|
<a href="https://hanzoskills.com">hanzoskills.com</a> what an agent reads</div>
|
|
<div><b>Run</b>
|
|
<a href="https://console.hanzo.ai">console.hanzo.ai</a> cloud console
|
|
<a href="https://api.hanzo.ai">api.hanzo.ai</a> the API</div>
|
|
<div><b>Account</b>
|
|
<a href="https://hanzo.id">hanzo.id</a> sign in
|
|
<a href="https://pay.hanzo.ai">pay.hanzo.ai</a> top up
|
|
<a href="https://billing.hanzo.ai">billing.hanzo.ai</a> invoices and usage</div>
|
|
<div><b>Company</b>
|
|
<a href="https://hanzo.ai">hanzo.ai</a> canonical
|
|
<a href="https://hanzo.ventures">hanzo.ventures</a> ventures</div>
|
|
</div>
|
|
<div class="fend"><a href="/hip-0035.md">This page as plain text</a> · <a href="/llms.txt">Index for agents</a></div>
|
|
</footer>
|
|
</div>
|
|
</body></html> |