mirror of
https://github.com/hanzoai/skills.git
synced 2026-11-19 00:47:36 +00:00
hanzoskills.com/llms.txt now opens with the install line, the API reference index and the company index, so an agent that lands on any one of the three can find the rest. robots.txt claimed 537 documents while llms.txt counts 920. The number is gone from robots.txt; llms.txt derives it. public/ is regenerated from scripts/site.py, which it had fallen behind: the copy-prompt menu and agent bar were in the generator and not in the committed pages. tests/site.mjs passes against the result. Co-authored-by: Hanzo Dev <[email protected]>
873 lines
63 KiB
HTML
873 lines
63 KiB
HTML
<!doctype html>
|
||
<html lang="en"><head>
|
||
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
||
<title>hip-0060</title>
|
||
<meta name="color-scheme" content="dark">
|
||
<style>
|
||
/* Committed to one visual world: a dark instrument panel. No light variant —
|
||
the dot field and the hairline rules only read on black, and a naive
|
||
inversion would wash both out. Every colour is painted explicitly so the page
|
||
never borrows a host background. */
|
||
@font-face{font-family:'Zen';src:url('/fonts/Zen-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
||
@font-face{font-family:'Zen';src:url('/fonts/Zen-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
||
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
||
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
||
@font-face{font-family:'Zen Pixel Square';src:url('/fonts/ZenPixel-Square.woff2') format('woff2');font-style:normal;font-display:swap}
|
||
|
||
:root{
|
||
--ink:#f4f4f2; --soft:#9c9c98; --faint:#6a6a66; --ground:#000; --panel:#0a0a0a;
|
||
--line:#232323; --line-hi:#3a3a3a; --accent:#f4f4f2;
|
||
--font-zen-sans:'Zen',-apple-system,BlinkMacSystemFont,"Segoe UI",Helvetica,Arial,sans-serif;
|
||
--font-zen-mono:'Zen Mono',ui-monospace,"SF Mono",SFMono-Regular,Menlo,Consolas,monospace;
|
||
--font-zen-pixel-square:'Zen Pixel Square',monospace;
|
||
--mono:var(--font-zen-mono);
|
||
--sans:var(--font-zen-sans);
|
||
/* Controls are pills and surfaces are soft, as on hanzo.ai. */
|
||
--r:999px; --r-soft:14px;
|
||
}
|
||
*{box-sizing:border-box}
|
||
html{-webkit-text-size-adjust:100%}
|
||
body{margin:0;background:var(--ground);color:var(--ink);font-family:var(--sans);
|
||
font-size:16px;line-height:1.65;-webkit-font-smoothing:antialiased}
|
||
.wrap{max-width:60rem;margin:0 auto;padding:0 1.75rem}
|
||
|
||
/* The dot field. A halftone made of one repeating radial-gradient, faded out
|
||
along the diagonal by a mask so it reads as a wave rather than wallpaper. */
|
||
/* No overflow clip here: the dot field is inset to the box already, and a clip
|
||
would cut off the copy-prompt menu. */
|
||
.hero{position:relative;border-bottom:1px solid var(--line)}
|
||
.hero::before{content:"";position:absolute;inset:0;
|
||
background-image:radial-gradient(var(--line-hi) 1px, transparent 1px);
|
||
background-size:7px 7px;
|
||
-webkit-mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
||
mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
||
pointer-events:none}
|
||
.hero .wrap{position:relative;padding-top:5.5rem;padding-bottom:4rem}
|
||
|
||
nav{border-bottom:1px solid var(--line)}
|
||
nav .in{max-width:60rem;margin:0 auto;padding:1.1rem 1.75rem;display:flex;
|
||
justify-content:space-between;align-items:center;gap:1rem}
|
||
.brand{font-family:var(--mono);font-size:.85rem;letter-spacing:.14em;
|
||
text-transform:uppercase;color:var(--ink);text-decoration:none;
|
||
display:inline-flex;align-items:center;gap:.6rem}
|
||
.brand i{font-style:normal;color:var(--faint)}
|
||
.mark{width:1.15em;height:1.15em;display:block;flex:0 0 auto}
|
||
nav a.up{font-family:var(--mono);font-size:.72rem;letter-spacing:.14em;
|
||
text-transform:uppercase;color:var(--soft);text-decoration:none}
|
||
nav a.up:hover{color:var(--ink)}
|
||
|
||
h1{font-family:var(--mono);font-size:clamp(2rem,5.4vw,3.4rem);line-height:1.12;
|
||
letter-spacing:-.02em;margin:0 0 1.1rem;font-weight:500;text-wrap:balance}
|
||
h1 em{font-style:normal;color:var(--faint)}
|
||
.lede{color:var(--soft);font-size:1.05rem;max-width:52ch;margin:0 0 2.25rem}
|
||
|
||
.btns{display:flex;flex-wrap:wrap;gap:.7rem;margin:0 0 .5rem;align-items:center}
|
||
.btn{display:inline-flex;align-items:center;gap:.5rem;font-family:var(--mono);
|
||
font-size:.72rem;letter-spacing:.14em;text-transform:uppercase;
|
||
padding:.8rem 1.4rem;border:1px solid var(--line-hi);border-radius:var(--r);
|
||
color:var(--ink);text-decoration:none;background:transparent;cursor:pointer;
|
||
line-height:1.2;font-weight:400}
|
||
.btn:hover{background:#141414;text-decoration:none;border-color:var(--soft)}
|
||
.btn.solid{background:var(--ink);color:#000;border-color:var(--ink)}
|
||
.btn.solid:hover{background:#d8d8d4}
|
||
.btn .chev{width:.7em;height:.7em;flex:0 0 auto;transition:transform .15s ease}
|
||
.btn[aria-expanded="true"] .chev{transform:rotate(180deg)}
|
||
|
||
/* Copy-prompt selector. Each row copies the prompt for its own target. */
|
||
.copy{position:relative;display:inline-flex}
|
||
.menu{position:absolute;top:calc(100% + .5rem);left:0;z-index:20;min-width:21rem;
|
||
background:var(--panel);border:1px solid var(--line-hi);border-radius:var(--r-soft);
|
||
padding:.4rem;display:flex;flex-direction:column;
|
||
box-shadow:0 12px 32px rgba(0,0,0,.75)}
|
||
.menu[hidden]{display:none}
|
||
.mi{display:flex;align-items:center;gap:.7rem;width:100%;text-align:left;
|
||
padding:.62rem .8rem;border:0;border-radius:calc(var(--r-soft) - .35rem);
|
||
background:transparent;color:var(--ink);font-family:var(--mono);font-size:.76rem;
|
||
letter-spacing:.04em;text-decoration:none;cursor:pointer;line-height:1.3;
|
||
white-space:nowrap}
|
||
.mi:hover,.mi:focus-visible{background:#181818;outline:none}
|
||
.mi .mark{width:1.05rem;height:1.05rem;color:var(--soft)}
|
||
.mi:hover .mark{color:var(--ink)}
|
||
.mi u{margin-left:auto;color:var(--faint);text-decoration:none;font-size:.68rem;
|
||
letter-spacing:.1em;text-transform:uppercase}
|
||
|
||
/* Works-with bar: one scrolling row, name and mark per agent. */
|
||
.strip{border-bottom:1px solid var(--line);background:var(--panel)}
|
||
.strip .in{max-width:60rem;margin:0 auto;padding:.9rem 1.75rem;display:flex;
|
||
align-items:center;gap:1.25rem;font-family:var(--mono);font-size:.72rem;
|
||
letter-spacing:.1em;text-transform:uppercase;color:var(--faint)}
|
||
.strip b{color:var(--soft);font-weight:400;flex:0 0 auto;
|
||
border-right:1px solid var(--line);padding-right:1.25rem;align-self:stretch;
|
||
display:flex;align-items:center}
|
||
.bar{display:flex;align-items:center;gap:.5rem;flex:1 1 auto;min-width:0;
|
||
overflow-x:auto;scrollbar-width:none;scroll-snap-type:x proximity;
|
||
-webkit-overflow-scrolling:touch;
|
||
-webkit-mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent);
|
||
mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent)}
|
||
.bar::-webkit-scrollbar{display:none}
|
||
.agent{display:inline-flex;align-items:center;gap:.45rem;flex:0 0 auto;
|
||
padding:.4rem .85rem;border:1px solid var(--line);border-radius:var(--r);
|
||
color:var(--soft);white-space:nowrap;scroll-snap-align:start;
|
||
text-decoration:none;background:var(--ground)}
|
||
.agent:hover{color:var(--ink);border-color:var(--line-hi);text-decoration:none}
|
||
.agent .mark{width:.95rem;height:.95rem}
|
||
|
||
h2{font-family:var(--mono);font-size:1.1rem;letter-spacing:-.01em;margin:2.75rem 0 .75rem;font-weight:500}
|
||
h3{font-size:1rem;margin:2rem 0 .5rem;font-weight:600}
|
||
h4,h5,h6{font-size:.94rem;margin:1.5rem 0 .5rem;font-weight:600}
|
||
p{margin:0 0 1rem;max-width:70ch}
|
||
ul,ol{margin:0 0 1rem;padding-left:1.35rem;max-width:70ch}
|
||
li{margin:.22rem 0}
|
||
a{color:var(--ink);text-decoration:underline;text-underline-offset:3px;
|
||
text-decoration-color:var(--line-hi)}
|
||
a:hover{text-decoration-color:var(--ink)}
|
||
a:focus-visible{outline:1px solid var(--ink);outline-offset:3px}
|
||
code{font-family:var(--mono);font-size:.85em;background:var(--panel);
|
||
border:1px solid var(--line);border-radius:5px;padding:.1em .36em}
|
||
pre{background:var(--panel);border:1px solid var(--line);padding:1rem 1.1rem;
|
||
border-radius:var(--r-soft);overflow-x:auto;margin:0 0 1.25rem}
|
||
pre code{background:none;border:0;padding:0;font-size:.82rem;line-height:1.7}
|
||
blockquote{margin:0 0 1rem;padding-left:1rem;border-left:1px solid var(--line-hi);color:var(--soft)}
|
||
hr{border:0;border-top:1px solid var(--line);margin:2.5rem 0}
|
||
table{border-collapse:collapse;width:100%;margin:0 0 1.25rem;font-size:.92rem;
|
||
display:block;overflow-x:auto}
|
||
th,td{border:1px solid var(--line);padding:.5rem .7rem;text-align:left}
|
||
|
||
.curl{display:block;background:var(--panel);border:1px solid var(--line);
|
||
border-radius:var(--r-soft);padding:.95rem 1.1rem;font-family:var(--mono);
|
||
font-size:.8rem;color:var(--soft);overflow-x:auto;margin:0 0 2.5rem;
|
||
white-space:pre;line-height:1.9}
|
||
.curl b{color:var(--ink);font-weight:400}
|
||
.curl i{font-style:normal;color:var(--faint)}
|
||
|
||
/* index */
|
||
#q{width:100%;padding:.85rem 1.2rem;font-family:var(--mono);font-size:.85rem;color:var(--ink);
|
||
background:var(--panel);border:1px solid var(--line);border-radius:var(--r);margin:0 0 .6rem}
|
||
#q::placeholder{color:var(--faint);text-transform:uppercase;letter-spacing:.1em;font-size:.72rem}
|
||
#q:focus{outline:none;border-color:var(--soft)}
|
||
#count{color:var(--faint);font-family:var(--mono);font-size:.72rem;letter-spacing:.1em;
|
||
text-transform:uppercase;margin:0 0 2rem}
|
||
.cat{margin:2.5rem 0 .5rem;font-family:var(--mono);font-size:.7rem;letter-spacing:.16em;
|
||
text-transform:uppercase;color:var(--faint);border-bottom:1px solid var(--line);
|
||
padding-bottom:.45rem}
|
||
.row{display:block;padding:.6rem .9rem;margin:0 -.9rem;border-bottom:1px solid var(--line);
|
||
border-radius:var(--r-soft);text-decoration:none}
|
||
.row:hover{background:#0a0a0a}
|
||
.nm{font-family:var(--mono);font-size:.84rem;color:var(--ink);display:block}
|
||
.ds{color:var(--soft);font-size:.9rem;display:block;margin-top:.12rem}
|
||
footer{margin-top:4rem;padding:2rem 0 4rem;border-top:1px solid var(--line);
|
||
color:var(--faint);font-size:.85rem}
|
||
footer a{color:var(--soft)}
|
||
.fmap{display:grid;gap:1.75rem;grid-template-columns:repeat(auto-fit,minmax(11rem,1fr));
|
||
margin:0 0 2rem;font-size:.85rem}
|
||
.fmap b{display:block;font-family:var(--mono);font-size:.68rem;letter-spacing:.16em;
|
||
text-transform:uppercase;color:var(--faint);font-weight:400;margin:0 0 .6rem;
|
||
border-bottom:1px solid var(--line);padding-bottom:.4rem}
|
||
.fmap a{display:inline-block;color:var(--ink);text-decoration:none;margin:.3rem .35rem 0 0}
|
||
.fmap a:hover{text-decoration:underline}
|
||
.fmap div{color:var(--faint);line-height:1.5}
|
||
.fend{border-top:1px solid var(--line);padding-top:1.25rem;font-size:.82rem}
|
||
|
||
@media (max-width:640px){
|
||
.hero .wrap{padding-top:3.25rem;padding-bottom:2.75rem}
|
||
.btns{gap:.5rem}
|
||
.btn{padding:.75rem 1.15rem;font-size:.68rem}
|
||
.copy,.copy .btn{width:100%;justify-content:center}
|
||
.menu{left:0;right:0;min-width:0}
|
||
.strip .in{flex-direction:column;align-items:flex-start;gap:.6rem;padding:.9rem 1.75rem}
|
||
.strip b{border-right:0;padding-right:0}
|
||
.bar{width:100%}
|
||
}
|
||
</style>
|
||
<link rel="alternate" type="text/markdown" href="/hip-0060.md">
|
||
|
||
<link rel="icon" href="/favicon.svg">
|
||
<link rel="canonical" href="https://hanzoskills.com/hip-0060">
|
||
</head>
|
||
<!-- Reading this with a machine? Every page here is also plain markdown.
|
||
This one: https://hanzoskills.com/hip-0060.md
|
||
Index: https://hanzoskills.com/llms.txt
|
||
Corpus: https://hanzoskills.com/all.md
|
||
Append .md to any path for the markdown twin. -->
|
||
<body>
|
||
<nav><div class="in">
|
||
<a class="brand" href="/"><svg class="mark" viewBox="0 0 67 67" fill="currentColor" aria-hidden="true"><path d="M22.21 67V44.6369H0V67H22.21Z"/><path d="M66.7038 22.3184H22.2534L0.0878906 44.6367H44.4634L66.7038 22.3184Z"/><path d="M22.21 0H0V22.3184H22.21V0Z"/><path d="M66.7198 0H44.5098V22.3184H66.7198V0Z"/><path d="M66.7198 67V44.6369H44.5098V67H66.7198Z"/></svg> HANZO<i>SKILLS</i></a>
|
||
<a class="up" href="/">All skills</a>
|
||
</div></nav>
|
||
|
||
<div class="wrap">
|
||
<h1>hip-0060</h1><p class="lede">HIP-60: Serverless Functions (FaaS) Standard. Status Final. Hanzo architectural specification.</p><h1>HIP-0060: Serverless Functions (FaaS) Standard</h1>
|
||
<h2>Abstract</h2>
|
||
<p>This proposal defines the Serverless Functions standard for the Hanzo ecosystem. <strong>Hanzo Functions</strong> is a Function-as-a-Service platform for event-driven AI workloads. What ships today is a per-org function registry plus sandboxed invocation; the Knative execution plane, GPU pools and snapshotting specified further down are the design for where that execution grows, and each such section says so where it diverges from the code.</p>
|
||
<p>Functions are the smallest deployable unit of compute in the Hanzo platform. Where the Inference Engine (HIP-0043) runs persistent model-serving processes and the Edge layer (HIP-0050) runs V8 isolates for lightweight request processing, Functions occupy the middle ground: containerized, stateless units of work that spin up on demand, execute, and disappear. They are the correct abstraction for bursty, event-driven AI workloads that do not justify a long-running service.</p>
|
||
<p>The registry accepts six runtimes today — <code>node</code>, <code>python</code>, <code>go</code>, <code>deno</code>, <code>bash</code> and <code>container</code> (BYO image) — the closed set in <code>apps/functions/functions.go:52-55</code>, mapped to the sandbox executor's language table. The pre-warmed AI base images this paragraph used to promise (PyTorch/ONNX/Candle pre-installed) are design, not shipped artifacts.</p>
|
||
<p>AI-specific triggers connect functions to the broader Hanzo infrastructure: model inference events from the LLM Gateway (HIP-0004), webhook delivery, scheduled retraining cycles, data pipeline stages from Hanzo Stream (HIP-1061), and async task invocation from Hanzo MQ (HIP-0055). Functions can also be deployed to the Edge (HIP-0050) for latency-sensitive invocation without GPU requirements.</p>
|
||
<p><strong>Serving</strong>: <code>apps/functions</code> in <code>hanzoai/cloud</code>, at <code>/v1/functions</code> (<code>manifest/apps.go:176</code>) — the registry and invoke endpoint ship inside the cloud binary (<code>plugin/functions</code>), not as a standalone service with its own ports, image or <code>hanzo-fn</code> binary; the CLI is the generated <code>hanzo functions</code> command group (HIP-1030)</p>
|
||
<h2>Motivation</h2>
|
||
<h3>The Bursty Inference Problem</h3>
|
||
<p>AI workloads are fundamentally bursty. A customer fine-tunes a model once a week. A webhook fires when a document is uploaded and needs embedding. A nightly job recomputes recommendation scores. A Slack bot classifies incoming messages and routes them to the right team.</p>
|
||
<p>None of these workloads justify a long-running service. A persistent Kubernetes Deployment with 2 replicas, running 24/7 to handle a webhook that fires 50 times a day, wastes 99.9% of its allocated compute. Multiply this across dozens of AI-adjacent microservices and the waste is substantial -- hundreds of dollars per month in idle GPU and CPU time.</p>
|
||
<p>The serverless model eliminates this waste. A function scales to zero when idle and scales up within seconds when triggered. You pay (in cluster resources) only for the milliseconds of actual execution. For the webhook that fires 50 times a day, the cost drops from "always-on Deployment" to "50 cold starts + 50 executions."</p>
|
||
<h3>The GPU Cold Start Problem</h3>
|
||
<p>Generic FaaS platforms (AWS Lambda, Google Cloud Functions, Knative with default configuration) are designed for CPU workloads. Cold start time is typically 100-500ms for a pre-built container, which is acceptable for HTTP request handling.</p>
|
||
<p>GPU functions are a different story. A function that runs a small inference model must:</p>
|
||
<ol><li>Schedule onto a node with an available GPU (0-30 seconds if the cluster has spare capacity, minutes if a new node must be provisioned)</li><li>Pull the container image (2-10 seconds for a base Python + PyTorch image, which is 2-8GB)</li><li>Initialize the CUDA runtime (1-3 seconds)</li><li>Load model weights into GPU memory (1-10 seconds depending on model size)</li><li>Execute the actual inference (10-500ms)</li></ol>
|
||
<p>Steps 1-4 can take 30+ seconds. For a function that processes a webhook in 200ms, spending 30 seconds on cold start is absurd. The cold start is 150x longer than the useful work.</p>
|
||
<p>Hanzo Functions solves this with three mechanisms: <strong>pre-warmed GPU pools</strong> (step 1 is eliminated), <strong>container snapshots</strong> (steps 2-3 are reduced to <1 second), and <strong>model caching</strong> (step 4 is eliminated for recently-used models). The result is GPU function cold starts under 2 seconds for models already in the cache, and under 5 seconds for first-time model loads.</p>
|
||
<h3>The Event-Driven AI Pipeline Problem</h3>
|
||
<p>Modern AI applications are not monolithic inference endpoints. They are pipelines of discrete steps:</p>
|
||
<pre><code>Document uploaded
|
||
--> Extract text (CPU, 500ms)
|
||
--> Chunk text (CPU, 50ms)
|
||
--> Generate embeddings (GPU, 200ms per chunk)
|
||
--> Store in vector DB (CPU, 100ms)
|
||
--> Trigger reranking index update (CPU, 2s)
|
||
--> Send notification (CPU, 50ms)</code></pre>
|
||
<p>Each step has different resource requirements (CPU vs. GPU), different scaling characteristics (embedding generation is the bottleneck), and different failure modes (vector DB write might fail independently of embedding generation). Running this pipeline in a single service means the entire service needs GPU access even though only one step uses the GPU.</p>
|
||
<p>Functions let you decompose the pipeline into independent units. The embedding step runs on a GPU function. Every other step runs on a CPU function. The GPU function scales independently based on its queue depth. If the embedding step fails, it retries independently without re-running text extraction.</p>
|
||
<p>The glue between steps is Hanzo Stream (HIP-1061) for durable event-driven pipelines and Hanzo MQ (HIP-0055) for async task invocation. Functions subscribe to stream topics or MQ subjects and are triggered automatically when events arrive.</p>
|
||
<h3>Why Not Just Use Kubernetes Jobs</h3>
|
||
<p>Kubernetes Jobs are the simplest way to run one-off compute tasks. They schedule a pod, run a container to completion, and clean up. Why build a FaaS layer on top?</p>
|
||
<p>Three reasons: <strong>cold start optimization</strong>, <strong>event binding</strong>, and <strong>developer experience</strong>.</p>
|
||
<p><strong>Cold start</strong>: A Kubernetes Job starts from scratch every time. It pulls the image, initializes the runtime, and loads dependencies. There is no concept of a warm pool, container reuse, or model caching. Knative Serving (which underlies Hanzo Functions) maintains a pool of warm containers that are immediately available for new requests. The difference is 200ms vs. 30 seconds for a GPU workload.</p>
|
||
<p><strong>Event binding</strong>: Kubernetes Jobs have no built-in trigger mechanism. You need an external system (CronJob for schedules, a webhook receiver for HTTP events, a custom controller for Kafka consumption) to create Jobs in response to events. Hanzo Functions provides a declarative trigger configuration: "run this function when a message arrives on <code>mq.batch.embeddings</code>" or "run this function on a cron schedule." The trigger-to-function binding is managed by the platform, not by the developer.</p>
|
||
<p><strong>Developer experience</strong>: A Kubernetes Job requires writing a Dockerfile, a Job YAML manifest, understanding pod scheduling, resource limits, service accounts, and image pull secrets. A Hanzo Function requires writing a function in Python/Go/Rust/TypeScript, pointing the CLI at it, and declaring a trigger. The platform handles containerization, scheduling, scaling, and monitoring.</p>
|
||
<h2>Design Philosophy</h2>
|
||
<h3>Why Knative Over OpenFaaS and AWS Lambda</h3>
|
||
<p>The three major approaches to serverless on Kubernetes are Knative, OpenFaaS, and Lambda-compatible runtimes (Firecracker/Lambda containers). Each makes different tradeoffs.</p>
|
||
<p><strong>AWS Lambda</strong> is the gold standard for serverless developer experience: write a function, deploy it, never think about infrastructure. But Lambda is a proprietary AWS service. Running Lambda-compatible runtimes on Kubernetes (via Firecracker or Lambda Web Adapter) gives you the API surface but not the operational benefits. You still manage the cluster, the networking, and the scaling. And Lambda's runtime contract (256MB /tmp, 15-minute timeout, no GPU support) is designed for web APIs, not AI workloads. There is no path to GPU-attached Lambda functions on self-hosted infrastructure.</p>
|
||
<p><strong>OpenFaaS</strong> is a lightweight, Kubernetes-native FaaS framework. It is simpler than Knative: fewer CRDs, no Istio dependency, and a straightforward watchdog pattern (HTTP → container → response). However, OpenFaaS has two limitations for AI workloads. First, its autoscaler is basic: it scales based on requests-per-second, not queue depth or GPU utilization. For AI functions where a single request takes 10 seconds of GPU time, request-rate scaling is the wrong signal. Second, OpenFaaS does not support scale-to-zero natively (the "zero-scale" feature requires a separate component and has a 5-10 second wake-up penalty with no warm pool concept).</p>
|
||
<p><strong>Knative Serving</strong> is the most sophisticated option. It provides:</p>
|
||
<ol><li><strong>Scale-to-zero with warm pools</strong>: Knative maintains a configurable number of warm instances (the "initial scale" and "min scale" settings). When traffic drops to zero, instances drain gracefully. When traffic returns, warm instances handle requests immediately while new instances spin up.</li></ol>
|
||
<ol><li><strong>Concurrency-based autoscaling</strong>: Knative scales based on concurrent requests per instance, not requests per second. This is the correct signal for AI functions: if each instance can handle 1 concurrent GPU inference, and 10 requests arrive, Knative scales to 10 instances. OpenFaaS's rate-based scaling would see "10 requests in 1 second" and might scale differently.</li></ol>
|
||
<ol><li><strong>Revision management</strong>: Knative supports traffic splitting between function revisions. Deploy a new version, route 10% of traffic to it, monitor GPU utilization and latency, then promote to 100%. This is essential for AI functions where a new model version might have different memory or latency characteristics.</li></ol>
|
||
<ol><li><strong>Kubernetes-native</strong>: Knative uses standard Kubernetes primitives (Deployments, Services, HPAs). No proprietary abstractions. When something breaks, you debug with <code>kubectl</code>, not a vendor-specific CLI.</li></ol>
|
||
<p><strong>Trade-off acknowledged</strong>: Knative is more complex than OpenFaaS. It has more CRDs, a heavier control plane, and a steeper learning curve. We accept this because its autoscaling model and revision management are correct for GPU workloads, and its Kubernetes-native design means we are not locked into a framework-specific operational model.</p>
|
||
<p>| Factor | AWS Lambda | OpenFaaS | Knative | |--------|-----------|----------|---------| | GPU support | No | Manual (no first-class) | Via custom runtime (this HIP) | | Scale-to-zero | Native | Plugin (slow wake) | Native (warm pool) | | Autoscaling signal | Concurrency | Request rate | Concurrency | | Revision/traffic split | Aliases + weights | No | Native | | K8s integration | External | Lightweight | Deep (CRDs, Services) | | Operational complexity | None (managed) | Low | Medium | | Vendor lock-in | AWS only | None | None | | Cold start (CPU) | 100-500ms | 1-5s | 500ms-2s | | Cold start (GPU) | N/A | 30s+ | 2-5s (with this HIP) |</p>
|
||
<h3>Why AI Needs Serverless</h3>
|
||
<p>The dominant pattern for AI model serving is persistent deployments: load a model into GPU memory and keep it there, serving requests indefinitely. This is correct for high-traffic models (zen-72b serving thousands of requests per minute). But the AI ecosystem has a long tail of workloads that are poorly served by persistent deployments:</p>
|
||
<ol><li><strong>Bursty inference</strong>: A customer's fine-tuned model receives 100 requests during business hours and zero requests overnight. A persistent deployment wastes 16 hours of GPU time per day. A function scales to zero overnight and wakes in seconds when the first morning request arrives.</li></ol>
|
||
<ol><li><strong>Event-driven pipelines</strong>: Document processing, image annotation, video transcription, and data enrichment are triggered by uploads, not by user requests. They run for seconds or minutes, not hours. Functions are the natural execution model.</li></ol>
|
||
<ol><li><strong>Scheduled retraining</strong>: Nightly model retraining, weekly evaluation runs, and monthly dataset refreshes are periodic GPU workloads. Functions with cron triggers replace the CronJob + custom-container pattern with a single declarative configuration.</li></ol>
|
||
<ol><li><strong>Webhook handlers</strong>: OAuth callbacks, payment confirmations, CI/CD notifications, and third-party API webhooks need lightweight, stateless request handling. Functions handle these without a dedicated microservice per webhook source.</li></ol>
|
||
<ol><li><strong>Prototyping and experimentation</strong>: Researchers need to deploy a model for a demo, test a new preprocessing step, or run a one-off evaluation. Functions let them deploy in seconds without writing Dockerfiles or Kubernetes manifests.</li></ol>
|
||
<p>The principle: <strong>persistent deployments for steady-state serving; functions for everything else.</strong></p>
|
||
<h3>Why Not Edge Functions for Everything</h3>
|
||
<p>Hanzo Edge (HIP-0050) provides V8 isolate-based functions at globally distributed PoPs. Why not use Edge for all serverless workloads?</p>
|
||
<p>Because Edge functions run on CPU-only infrastructure. They execute JavaScript/TypeScript in V8 isolates with sub-millisecond cold starts and <128MB memory limits. This is perfect for authentication, routing, caching, and small-model inference (embeddings, classification).</p>
|
||
<p>But Edge functions cannot:</p>
|
||
<ul><li>Attach to GPUs for model inference</li><li>Run Python, Go, or Rust code natively</li><li>Use more than 128MB of memory</li><li>Execute for longer than 100ms of CPU time (configurable, but fundamentally limited by the isolate model)</li></ul>
|
||
<p>Hanzo Functions fills the gap between Edge (millisecond-scale, CPU-only, globally distributed) and the Inference Engine (persistent, GPU-attached, origin-only). Functions run in containers with full OS access, arbitrary memory limits, GPU attachment, and execution times measured in seconds or minutes.</p>
|
||
<pre><code>Workload Type │ Execution Model │ Cold Start │ GPU │ Duration
|
||
───────────────────────┼──────────────────────┼─────────────┼─────┼──────────
|
||
Auth/routing/caching │ Edge (HIP-0050) │ <1ms │ No │ <100ms
|
||
Webhooks/ETL/pipelines │ Functions (HIP-0060) │ 500ms-5s │ Opt │ <15min
|
||
Model serving │ Engine (HIP-0043) │ 2-30s │ Yes │ Persistent
|
||
Training/fine-tuning │ ML Pipeline (HIP-57) │ Minutes │ Yes │ Hours</code></pre>
|
||
<h2>Specification</h2>
|
||
<h3>The shipped surface</h3>
|
||
<p>Every address is under <code>/v1/functions</code> — nine paths, published in <code>plugin/functions/openapi.json</code> and documented route-by-route in the package doc (<code>apps/functions/functions.go:11-27</code>):</p>
|
||
<pre><code>GET /v1/functions list
|
||
POST /v1/functions create / redeploy
|
||
GET /v1/functions/metrics invocation chart + cost (derived from real rows)
|
||
GET /v1/functions/triggers all triggers (HTTP)
|
||
GET /v1/functions/deployments current deployments
|
||
GET /v1/functions/secrets mounted secret NAMES (values live in KMS)
|
||
GET /v1/functions/{name} detail
|
||
DELETE /v1/functions/{name} delete, with its invocations
|
||
GET /v1/functions/{name}/invocations recent invocations
|
||
GET /v1/functions/{name}/logs last invocation output
|
||
POST /v1/functions/{name}/invoke run it</code></pre>
|
||
<p><strong>Store.</strong> The per-org function registry: runtime, source (≤256 KiB), resource limits, and the NAMES of mounted secrets — never a secret value (<code>apps/functions/store.go</code>, <code>apps/functions/functions.go:4-9</code>). Invocation rows live beside it, and every metric shown is derived from them; nothing is interpolated.</p>
|
||
<p><strong>Execution.</strong> Invoke delegates to a sandbox through <code>apps/exec</code> — the binary NEVER runs org code in-process, and an unconfigured sandbox fails closed with 503 rather than fabricating a result (<code>apps/functions/functions.go:26-29</code>, <code>apps/functions/invoke.go:20-38</code>).</p>
|
||
<p><strong>Tenancy.</strong> The org is the validated principal (HIP-0026); the registry's org column is enforced on every query, and the billing subject is <code>principal.Ledger</code>, never a caller-supplied field.</p>
|
||
<p><strong>Meter.</strong> Functions is <strong>metered</strong> — the plugin declares <code>Price: cloud.Metered</code> (<code>plugin/functions/main.go:21</code>), so a write to this surface requires commercial standing before it runs (<code>spend.go:302</code>). Two debits land on the caller's org ledger through the ONE shared <code>cloud.ResourceMeter</code>, and either is independently free at a zero rate:</p>
|
||
<ul><li>the flat per-invocation fee, <code>CLOUD_FUNCTION_FEE_CENTS</code></li></ul>
|
||
<p> (<code>apps/functions/functions.go:61-69</code>), gated BEFORE the sandbox runs — an org that cannot cover it gets 402 and no work happens (<code>apps/functions/invoke.go:169-171</code>);</p>
|
||
<ul><li>the compute fee in GB-seconds — (memory GB) × (wall-clock s), the unit the</li></ul>
|
||
<p> industry bills serverless on — at <code>CLOUD_FUNCTION_GBSEC_CENTS</code>, default $1.00/GB-s, integer-exact with half-up rounding (<code>apps/functions/gbseconds.go</code>).</p>
|
||
<p><strong>Events.</strong> None: the capability publishes nothing on the bus, so a customer's webhooks receive nothing from it.</p>
|
||
<p><strong>Observability.</strong> Nothing beyond the request span every route gets; <code>GET /v1/functions/metrics</code> is a read over the org's own invocation rows, not a telemetry exporter.</p>
|
||
<p><strong>Stage.</strong> <strong>ga</strong> (HIP-0139 §8): functions is agentic-OS core — the unit of compute agents deploy to.</p>
|
||
<p><strong>Upstreams.</strong> None: no OSS project is forked, embedded or mirrored in <code>apps/functions</code>; Knative, CRIU and the runtime images named in this document are referenced designs, not vendored code.</p>
|
||
<h3>Architecture</h3>
|
||
<pre><code> ┌────────────────────────────────────────────┐
|
||
│ Hanzo Functions │
|
||
│ Control Plane :8060 │
|
||
│ │
|
||
│ ┌──────────┐ ┌──────────┐ ┌──────────┐ │
|
||
│ │ Function │ │ Trigger │ │ Revision │ │
|
||
│ │ Registry │ │ Manager │ │ Router │ │
|
||
│ └──────────┘ └──────────┘ └──────────┘ │
|
||
└────────────────────┬───────────────────────┘
|
||
│
|
||
┌────────────────────────┼────────────────────────┐
|
||
│ │ │
|
||
┌──────┴──────┐ ┌───────┴──────┐ ┌──────┴──────┐
|
||
│ Knative │ │ Knative │ │ Knative │
|
||
│ Serving │ │ Serving │ │ Serving │
|
||
│ (CPU Pool) │ │ (GPU Pool) │ │ (Edge Sync) │
|
||
│ │ │ │ │ │
|
||
│ Python │ │ Python+CUDA │ │ Sync to │
|
||
│ Go │ │ Rust+Candle │ │ HIP-0050 │
|
||
│ Rust │ │ │ │ │
|
||
│ TypeScript │ │ │ │ │
|
||
└──────┬──────┘ └──────┬───────┘ └─────────────┘
|
||
│ │
|
||
┌─────────┴──────────┐ ┌───────┴───────────┐
|
||
│ │ │ │
|
||
┌──┴───┐ ┌──────┐ ┌──┴──┐ │ ┌──────┐ ┌─────┐ │
|
||
│Stream│ │ MQ │ │HTTP │ │ │Model │ │GPU │ │
|
||
│HIP-1061│ │HIP-55│ │ │ │ │Cache │ │Pool │ │
|
||
└──────┘ └──────┘ └─────┘ │ └──────┘ └─────┘ │
|
||
└───────────────────┘</code></pre>
|
||
<p>The architecture has three layers:</p>
|
||
<ol><li><strong>Control Plane</strong> (port 8060): Manages function definitions, trigger bindings, revision history, and deployment orchestration. Talks to the Knative API server to create/update Knative Services.</li></ol>
|
||
<ol><li><strong>Execution Plane</strong>: Knative Serving manages the lifecycle of function instances. CPU functions run on the standard node pool. GPU functions run on GPU-labeled nodes with pre-warmed CUDA containers. Edge-compatible functions are synced to the Edge control plane (HIP-0050) for deployment as V8 isolates.</li></ol>
|
||
<ol><li><strong>Event Plane</strong>: Trigger sources (Stream, MQ, HTTP, Cron) are connected to functions via the Trigger Manager. When an event arrives, the Trigger Manager invokes the function through the Knative invocation proxy (port 8061).</li></ol>
|
||
<h3>Function Definition</h3>
|
||
<p>A function is defined by a manifest file (<code>function.yaml</code>) and source code:</p>
|
||
<pre><code class="lang-yaml"># function.yaml
|
||
name: embed-document
|
||
runtime: python
|
||
version: 1.0.0
|
||
entry: handler.embed
|
||
description: Generate embeddings for uploaded documents
|
||
|
||
resources:
|
||
cpu: "500m"
|
||
memory: "1Gi"
|
||
gpu: "nvidia.com/gpu: 1" # Optional: request a GPU
|
||
timeout: 300s # Maximum execution time
|
||
concurrency: 4 # Max concurrent requests per instance
|
||
|
||
scaling:
|
||
min_instances: 0 # Scale to zero when idle
|
||
max_instances: 20 # Hard ceiling
|
||
target_concurrency: 1 # Scale up when concurrency exceeds this
|
||
scale_down_delay: 300s # Wait 5 min before scaling down
|
||
|
||
triggers:
|
||
- type: mq
|
||
source: mq.batch.embeddings
|
||
consumer_group: fn-embed-document
|
||
|
||
- type: http
|
||
path: /v1/functions/embed-document
|
||
methods: [POST]
|
||
|
||
- type: cron
|
||
schedule: "0 2 * * *" # Daily at 02:00 UTC
|
||
payload: '{"mode": "reindex"}'
|
||
|
||
environment:
|
||
MODEL_NAME: bge-large-en-v1.5
|
||
BATCH_SIZE: "32"
|
||
|
||
secrets:
|
||
- name: hanzo-api-key
|
||
env: HANZO_API_KEY</code></pre>
|
||
<h4>Manifest Fields</h4>
|
||
<p>| Field | Type | Required | Description | |-------|------|----------|-------------| | <code>name</code> | string | yes | Function name. Lowercase, alphanumeric + hyphens. Globally unique within the org. | | <code>runtime</code> | enum | yes | <code>python</code>, <code>go</code>, <code>rust</code>, <code>typescript</code> | | <code>version</code> | semver | yes | Function version. New versions create new Knative revisions. | | <code>entry</code> | string | yes | Entrypoint. Format depends on runtime (see Runtime Specification). | | <code>description</code> | string | no | Human-readable description. | | <code>resources.cpu</code> | string | no | CPU request (Kubernetes format). Default: <code>250m</code>. | | <code>resources.memory</code> | string | no | Memory request. Default: <code>256Mi</code>. | | <code>resources.gpu</code> | string | no | GPU resource request. Omit for CPU-only functions. | | <code>resources.timeout</code> | duration | no | Max execution time. Default: <code>60s</code>. Max: <code>900s</code> (15 minutes). | | <code>resources.concurrency</code> | int | no | Max concurrent requests per instance. Default: <code>1</code> for GPU, <code>10</code> for CPU. | | <code>scaling.min_instances</code> | int | no | Minimum instances. <code>0</code> enables scale-to-zero. Default: <code>0</code>. | | <code>scaling.max_instances</code> | int | no | Maximum instances. Default: <code>10</code>. | | <code>scaling.target_concurrency</code> | int | no | Target concurrency for autoscaling. Default: <code>1</code>. | | <code>scaling.scale_down_delay</code> | duration | no | Grace period before scaling down. Default: <code>300s</code>. | | <code>triggers</code> | list | yes | One or more trigger definitions (see Trigger Specification). | | <code>environment</code> | map | no | Environment variables injected into the function container. | | <code>secrets</code> | list | no | KMS secrets (HIP-0027) injected as environment variables. |</p>
|
||
<h3>Runtime Specification</h3>
|
||
<p>Each runtime provides a base container image with pre-installed dependencies and a standard invocation contract.</p>
|
||
<h4>Python Runtime</h4>
|
||
<p>Base image: <code>ghcr.io/hanzoai/fn-python:3.12</code></p>
|
||
<p>Pre-installed: <code>torch</code>, <code>transformers</code>, <code>onnxruntime</code>, <code>numpy</code>, <code>httpx</code>, <code>pydantic</code></p>
|
||
<pre><code class="lang-python"># handler.py
|
||
from hanzo.functions import Context, Response
|
||
|
||
def embed(ctx: Context) -> Response:
|
||
"""Generate embeddings for a document."""
|
||
doc_id = ctx.data["document_id"]
|
||
text = fetch_document(doc_id)
|
||
|
||
model = ctx.model("bge-large-en-v1.5") # Loaded from model cache
|
||
embeddings = model.encode(text, batch_size=32)
|
||
|
||
store_embeddings(doc_id, embeddings)
|
||
|
||
return Response(
|
||
status=200,
|
||
body={"document_id": doc_id, "dimensions": len(embeddings[0])}
|
||
)</code></pre>
|
||
<p>The <code>Context</code> object provides:</p>
|
||
<p>| Attribute | Type | Description | |-----------|------|-------------| | <code>ctx.data</code> | dict | Parsed trigger payload (JSON body, MQ message data, Stream event data) | | <code>ctx.headers</code> | dict | HTTP headers (for HTTP triggers) or message headers | | <code>ctx.trigger</code> | TriggerInfo | Trigger metadata (type, source, timestamp) | | <code>ctx.model(name)</code> | Model | Load a model from the GPU model cache | | <code>ctx.kv</code> | KVClient | KV client (HIP-1164) | | <code>ctx.storage</code> | StorageClient | Object storage client (HIP-0405) | | <code>ctx.publish(subject, data)</code> | None | Publish to MQ (HIP-0055) or Stream (HIP-1061) | | <code>ctx.log</code> | Logger | Structured logger with request correlation ID |</p>
|
||
<h4>Go Runtime</h4>
|
||
<p>Base image: <code>ghcr.io/hanzoai/fn-go:1.22</code></p>
|
||
<p>Pre-installed: ONNX Runtime C bindings, Hanzo SDK</p>
|
||
<pre><code class="lang-go">// handler.go
|
||
package main
|
||
|
||
import (
|
||
"github.com/hanzoai/functions/sdk"
|
||
)
|
||
|
||
func Classify(ctx *sdk.Context) *sdk.Response {
|
||
text := ctx.Data["text"].(string)
|
||
|
||
model, err := ctx.Model("distilbert-intent")
|
||
if err != nil {
|
||
return sdk.Error(500, err)
|
||
}
|
||
|
||
result, err := model.Predict(text)
|
||
if err != nil {
|
||
return sdk.Error(500, err)
|
||
}
|
||
|
||
return sdk.OK(map[string]interface{}{
|
||
"label": result.Label,
|
||
"confidence": result.Score,
|
||
})
|
||
}</code></pre>
|
||
<h4>Rust Runtime</h4>
|
||
<p>Base image: <code>ghcr.io/hanzoai/fn-rust:1.77</code></p>
|
||
<p>Pre-installed: Candle (HIP-0019), <code>tokio</code>, <code>serde</code>, <code>reqwest</code></p>
|
||
<pre><code class="lang-rust">// src/handler.rs
|
||
use hanzo_functions::{Context, Response, Result};
|
||
|
||
pub async fn transcribe(ctx: Context) -> Result<Response> {
|
||
let audio_url: String = ctx.data().get("audio_url")?;
|
||
let audio = ctx.storage().get(&audio_url).await?;
|
||
|
||
let model = ctx.model("whisper-small").await?;
|
||
let transcript = model.transcribe(&audio).await?;
|
||
|
||
ctx.publish("mq.pipeline.transcription.complete", &serde_json::json!({
|
||
"audio_url": audio_url,
|
||
"transcript": transcript,
|
||
})).await?;
|
||
|
||
Ok(Response::ok(serde_json::json!({
|
||
"transcript": transcript,
|
||
})))
|
||
}</code></pre>
|
||
<h4>TypeScript Runtime</h4>
|
||
<p>Base image: <code>ghcr.io/hanzoai/fn-typescript:22</code></p>
|
||
<p>Pre-installed: <code>@xenova/transformers</code>, <code>onnxruntime-node</code>, Hanzo SDK</p>
|
||
<pre><code class="lang-typescript">// handler.ts
|
||
import { Context, Response } from '@hanzo/functions'
|
||
|
||
export async function webhook(ctx: Context): Promise<Response> {
|
||
const event = ctx.data as WebhookPayload
|
||
|
||
// Classify the incoming webhook
|
||
const intent = await ctx.model('distilbert-intent').predict(event.text)
|
||
|
||
// Route based on classification
|
||
if (intent.label === 'support') {
|
||
await ctx.publish('mq.notify.support', {
|
||
channel: 'slack',
|
||
message: event.text,
|
||
metadata: { confidence: intent.score },
|
||
})
|
||
}
|
||
|
||
return Response.ok({ routed: true, intent: intent.label })
|
||
}</code></pre>
|
||
<h3>Trigger Specification</h3>
|
||
<p>Triggers connect external events to function invocations. A function can have multiple triggers of different types.</p>
|
||
<h4>HTTP Trigger</h4>
|
||
<p>Exposes the function as an HTTP endpoint on the invocation proxy (port 8061).</p>
|
||
<pre><code class="lang-yaml">triggers:
|
||
- type: http
|
||
path: /v1/functions/my-function
|
||
methods: [GET, POST]
|
||
auth: required # "required" (default), "optional", "none"
|
||
rate_limit: 100/min # Per-API-key rate limit</code></pre>
|
||
<p>HTTP triggers create a Knative Route that maps the path to the function's Knative Service. Authentication is handled by the invocation proxy using IAM (HIP-0026) JWT validation.</p>
|
||
<h4>MQ Trigger</h4>
|
||
<p>Invokes the function when a message arrives on a NATS JetStream subject.</p>
|
||
<pre><code class="lang-yaml">triggers:
|
||
- type: mq
|
||
source: mq.batch.embeddings
|
||
consumer_group: fn-embed-document
|
||
batch_size: 1 # Messages per invocation (default: 1)
|
||
max_batch_wait: 5s # Max wait to fill batch</code></pre>
|
||
<p>The Trigger Manager runs a NATS consumer in the specified consumer group. When a message arrives, it invokes the function via HTTP and acknowledges the message only after the function returns successfully. If the function fails, the message is nacked and redelivered per the MQ queue's retry policy.</p>
|
||
<h4>Stream Trigger (HIP-1061)</h4>
|
||
<p>Invokes the function when an event is published to a Kafka topic.</p>
|
||
<pre><code class="lang-yaml">triggers:
|
||
- type: stream
|
||
topic: llm_usage
|
||
consumer_group: fn-usage-aggregator
|
||
batch_size: 100 # Events per invocation
|
||
max_batch_wait: 10s
|
||
start_offset: latest # "latest" or "earliest"</code></pre>
|
||
<p>The Trigger Manager runs a Kafka consumer. Events are batched and delivered to the function as an array in <code>ctx.data</code>. The consumer commits offsets only after successful function execution.</p>
|
||
<h4>Cron Trigger</h4>
|
||
<p>Invokes the function on a schedule.</p>
|
||
<pre><code class="lang-yaml">triggers:
|
||
- type: cron
|
||
schedule: "0 */6 * * *" # Every 6 hours (standard cron syntax)
|
||
timezone: UTC
|
||
payload: '{"type": "full_reconcile"}'</code></pre>
|
||
<p>The Trigger Manager uses an internal scheduler (backed by SQL, not Kubernetes CronJobs) to fire cron triggers. This avoids the Kubernetes CronJob limitation of 1-minute granularity and provides better observability through the management API.</p>
|
||
<h4>Inference Event Trigger</h4>
|
||
<p>Invokes the function in response to LLM Gateway (HIP-0004) inference events. This is a convenience trigger built on top of the Stream trigger that filters <code>llm_usage</code> events.</p>
|
||
<pre><code class="lang-yaml">triggers:
|
||
- type: inference
|
||
models: [zen-7b, zen-14b] # Filter by model
|
||
event_types: [completed, failed] # Filter by outcome
|
||
min_tokens: 1000 # Filter by token count</code></pre>
|
||
<h3>GPU Function Execution</h3>
|
||
<p>GPU functions are the distinguishing feature of Hanzo Functions. This section specifies the mechanisms that make GPU cold starts tolerable.</p>
|
||
<h4>Pre-Warmed GPU Pool</h4>
|
||
<p>The cluster maintains a pool of GPU-equipped pods that are pre-initialized with the CUDA runtime and a base function container. These pods are idle but warm -- the CUDA context is loaded, the GPU driver is initialized, and the base container is running.</p>
|
||
<p>When a GPU function is invoked:</p>
|
||
<ol><li>The scheduler selects a warm pod from the pool (0ms scheduling delay).</li><li>The function code and dependencies are injected into the warm pod via a volume mount (200-500ms).</li><li>The model is loaded from the model cache (0ms if cached, 1-10s if not).</li><li>The function executes.</li><li>After execution, the pod returns to the pool for reuse.</li></ol>
|
||
<p>Pool sizing:</p>
|
||
<pre><code class="lang-yaml">gpu_pool:
|
||
size: 4 # Pre-warmed GPU pods
|
||
gpu_type: nvidia-a10g # GPU type for pool pods
|
||
idle_timeout: 600s # Return to pool after 10 min idle
|
||
max_model_cache: 8Gi # Per-pod model cache size
|
||
preload_models: # Models loaded at pool startup
|
||
- bge-large-en-v1.5
|
||
- whisper-small
|
||
- distilbert-intent</code></pre>
|
||
<h4>Container Snapshots</h4>
|
||
<p>Traditional container cold start involves pulling the image, unpacking layers, and initializing the runtime. For a Python + PyTorch image (5GB+), this takes 5-15 seconds even from a local registry.</p>
|
||
<p>Hanzo Functions uses <strong>container snapshots</strong> (CRIU-based checkpoint/restore) to reduce cold start to <1 second:</p>
|
||
<ol><li><strong>Snapshot creation</strong>: When a function is deployed, the platform runs the container, initializes the runtime (imports, CUDA setup, model loading), and creates a CRIU checkpoint of the process state.</li><li><strong>Snapshot restore</strong>: On cold start, instead of starting the container from scratch, the platform restores the checkpoint. All imports are loaded, CUDA is initialized, and models are in memory. The function is ready to execute in <1 second.</li></ol>
|
||
<p>Snapshots are stored in Object Storage (HIP-0405) and cached on local SSD at each node. They are invalidated when the function code or runtime version changes.</p>
|
||
<pre><code>Traditional cold start: Pull image (5s) → Start container (1s) → Import torch (3s) → Load CUDA (2s) → Load model (3s) = 14s
|
||
Snapshot cold start: Restore checkpoint (800ms) → Ready</code></pre>
|
||
<p>Snapshot support is available for Python and Rust runtimes. Go and TypeScript runtimes already have fast cold starts (<1s) without snapshots due to their lightweight initialization.</p>
|
||
<h4>Model Cache</h4>
|
||
<p>GPU functions frequently load the same models. The model cache is a node-local LRU cache backed by NVMe SSD that stores model weights in a ready-to-load format.</p>
|
||
<pre><code>Model requested by function
|
||
--> Check node-local cache (NVMe SSD)
|
||
--> Hit: mmap into GPU memory (50-200ms)
|
||
--> Miss: Download from Object Storage (HIP-0405) (1-10s)
|
||
--> Cache locally
|
||
--> mmap into GPU memory</code></pre>
|
||
<p>The cache operates at the node level, not the pod level. Multiple function pods on the same node share the cache. Cache eviction uses LRU with a configurable size limit (default: 50GB per node).</p>
|
||
<p>Cache metrics:</p>
|
||
<pre><code>hanzo_fn_model_cache_hits_total{node, model}
|
||
hanzo_fn_model_cache_misses_total{node, model}
|
||
hanzo_fn_model_cache_size_bytes{node}
|
||
hanzo_fn_model_cache_evictions_total{node}
|
||
hanzo_fn_model_load_duration_seconds{model}</code></pre>
|
||
<h3>Control Plane API</h3>
|
||
<p>The management surface this section used to table — ports 8060/8061, revisions, traffic splits, async invocation, a GPU-pool read — is design the shipped registry does not serve. The served control surface is exactly the eleven operations of "The shipped surface" above, and the served document (HIP-1030) is its enumeration; an operation absent there is not callable and MUST NOT be assumed by a client. Revision management and traffic splitting arrive, if they arrive, with the Knative execution plane.</p>
|
||
<h3>Invocation Protocol</h3>
|
||
<p>Functions are invoked via HTTP POST to the function's Knative Service endpoint. The invocation proxy (port 8061) handles routing, authentication, and trigger-specific payload transformation.</p>
|
||
<h4>Request Format</h4>
|
||
<pre><code class="lang-json">POST /v1/functions/embed-document/invoke
|
||
Content-Type: application/json
|
||
Authorization: Bearer <jwt>
|
||
X-Function-Trigger: mq
|
||
X-Function-Trigger-Source: mq.batch.embeddings
|
||
X-Request-ID: req_01HQ3X7K8M2N4P5R6S7T8U9V0W
|
||
|
||
{
|
||
"document_id": "doc_abc123",
|
||
"options": { "model": "bge-large-en-v1.5" }
|
||
}</code></pre>
|
||
<h4>Response Format</h4>
|
||
<pre><code class="lang-json">HTTP/1.1 200 OK
|
||
Content-Type: application/json
|
||
X-Function-Name: embed-document
|
||
X-Function-Revision: embed-document-00003
|
||
X-Function-Duration-Ms: 1250
|
||
X-Function-Instance: fn-embed-document-00003-deployment-abc12-xyz
|
||
|
||
{
|
||
"document_id": "doc_abc123",
|
||
"dimensions": 1024,
|
||
"chunks_processed": 42
|
||
}</code></pre>
|
||
<h3>Traffic Splitting</h3>
|
||
<p>Functions support gradual rollouts via traffic splitting between revisions:</p>
|
||
<pre><code class="lang-bash"># Deploy new version (creates revision 4)
|
||
hanzo-fn deploy --name embed-document --source ./src
|
||
|
||
# Route 10% of traffic to the new revision
|
||
hanzo-fn traffic embed-document --revision 4 --percent 10
|
||
|
||
# Monitor metrics, then promote
|
||
hanzo-fn traffic embed-document --revision 4 --percent 100</code></pre>
|
||
<p>Traffic splitting is implemented via Knative's traffic configuration:</p>
|
||
<pre><code class="lang-yaml">traffic:
|
||
- revisionName: embed-document-00003
|
||
percent: 90
|
||
- revisionName: embed-document-00004
|
||
percent: 10</code></pre>
|
||
<h3>Metrics</h3>
|
||
<p>There is no <code>hanzo_fn_*</code> Prometheus family: the shipped metrics surface is <code>GET /v1/functions/metrics</code>, a windowed series of REAL invocation counts, statuses and cost derived from the org's own invocation rows (<code>apps/functions/metrics.go</code>) — nothing interpolated, nothing invented. A node-level exporter for pools, caches and snapshot restore times belongs to the execution plane that would own those mechanisms.</p>
|
||
<h2>Implementation</h2>
|
||
<h3>CLI</h3>
|
||
<p>There is no <code>hanzo-fn</code> binary: the developer interface is the generated <code>hanzo functions</code> command group, projected from the served document like every other capability's CLI (HIP-1030). The verbs below are the intended ergonomics for that group; the deployment and compose manifests that follow describe the standalone service this HIP no longer ships and are retained only as the execution-plane design.</p>
|
||
<pre><code class="lang-bash"># Initialize a new function project
|
||
hanzo-fn init --runtime python --name my-function
|
||
|
||
# Local development (runs function locally with hot reload)
|
||
hanzo-fn dev --port 8080
|
||
|
||
# Deploy to cluster
|
||
hanzo-fn deploy --name my-function --source ./src
|
||
|
||
# Invoke a deployed function
|
||
hanzo-fn invoke my-function --data '{"key": "value"}'
|
||
|
||
# View logs
|
||
hanzo-fn logs my-function --follow
|
||
|
||
# View metrics
|
||
hanzo-fn metrics my-function
|
||
|
||
# List all functions
|
||
hanzo-fn list
|
||
|
||
# Delete a function
|
||
hanzo-fn delete my-function</code></pre>
|
||
<h3>Kubernetes Deployment</h3>
|
||
<h4>Control Plane</h4>
|
||
<pre><code class="lang-yaml">apiVersion: apps/v1
|
||
kind: Deployment
|
||
metadata:
|
||
name: hanzo-functions
|
||
namespace: hanzo
|
||
spec:
|
||
replicas: 2
|
||
selector:
|
||
matchLabels:
|
||
app: hanzo-functions
|
||
template:
|
||
spec:
|
||
containers:
|
||
- name: control-plane
|
||
image: ghcr.io/hanzoai/functions:latest
|
||
args: ["serve", "--config", "/etc/functions/config.yaml"]
|
||
ports:
|
||
- containerPort: 8060
|
||
name: api
|
||
- containerPort: 8061
|
||
name: invoke
|
||
- containerPort: 9060
|
||
name: metrics
|
||
resources:
|
||
requests: { cpu: "500m", memory: "512Mi" }
|
||
limits: { cpu: "2000m", memory: "2Gi" }
|
||
env:
|
||
- name: DATABASE_URL
|
||
valueFrom:
|
||
secretKeyRef:
|
||
name: hanzo-functions-db
|
||
key: url
|
||
- name: NATS_URL
|
||
value: nats://localhost:4222
|
||
- name: KAFKA_BROKERS
|
||
value: localhost:9092
|
||
volumeMounts:
|
||
- name: config
|
||
mountPath: /etc/functions
|
||
volumes:
|
||
- name: config
|
||
configMap:
|
||
name: functions-config</code></pre>
|
||
<h4>GPU Pool DaemonSet</h4>
|
||
<pre><code class="lang-yaml">apiVersion: apps/v1
|
||
kind: DaemonSet
|
||
metadata:
|
||
name: hanzo-fn-gpu-pool
|
||
namespace: hanzo
|
||
spec:
|
||
selector:
|
||
matchLabels:
|
||
app: hanzo-fn-gpu-pool
|
||
template:
|
||
spec:
|
||
nodeSelector:
|
||
nvidia.com/gpu.present: "true"
|
||
containers:
|
||
- name: gpu-warm
|
||
image: ghcr.io/hanzoai/fn-python:3.12-cuda
|
||
command: ["hanzo-fn-agent", "--mode=pool", "--model-cache=/cache"]
|
||
resources:
|
||
requests:
|
||
cpu: "1000m"
|
||
memory: "4Gi"
|
||
nvidia.com/gpu: "1"
|
||
limits:
|
||
cpu: "4000m"
|
||
memory: "16Gi"
|
||
nvidia.com/gpu: "1"
|
||
volumeMounts:
|
||
- name: model-cache
|
||
mountPath: /cache
|
||
- name: snapshots
|
||
mountPath: /snapshots
|
||
volumes:
|
||
- name: model-cache
|
||
hostPath:
|
||
path: /var/hanzo/model-cache
|
||
type: DirectoryOrCreate
|
||
- name: snapshots
|
||
hostPath:
|
||
path: /var/hanzo/snapshots
|
||
type: DirectoryOrCreate</code></pre>
|
||
<h3>Docker Development</h3>
|
||
<pre><code class="lang-yaml"># compose.yml
|
||
services:
|
||
functions:
|
||
image: ghcr.io/hanzoai/functions:latest
|
||
ports:
|
||
- "8060:8060"
|
||
- "8061:8061"
|
||
- "9060:9060"
|
||
environment:
|
||
# No per-app database. State follows HIP-0144: per-tenant by default,
|
||
# the column store for events, the ONE shared sql only if a server is
|
||
# genuinely required.
|
||
STORE: tenant
|
||
NATS_URL: nats://nats:4222
|
||
KAFKA_BROKERS: kafka:9092
|
||
OBJECT_STORAGE_URL: http://minio:9000
|
||
GPU_POOL_ENABLED: "false" # No GPU in dev
|
||
volumes:
|
||
- ./config.yaml:/etc/functions/config.yaml
|
||
|
||
# Local function runner (no Knative needed for dev)
|
||
function-runner:
|
||
image: ghcr.io/hanzoai/fn-python:3.12
|
||
ports:
|
||
- "8080:8080"
|
||
volumes:
|
||
- ./my-function:/app
|
||
command: ["hanzo-fn-agent", "--mode=dev", "--source=/app"]</code></pre>
|
||
<h3>Configuration</h3>
|
||
<pre><code class="lang-yaml"># config.yaml
|
||
server:
|
||
api_port: 8060
|
||
invoke_port: 8061
|
||
metrics_port: 9060
|
||
|
||
database:
|
||
url: ${DATABASE_URL}
|
||
max_connections: 20
|
||
|
||
knative:
|
||
namespace: hanzo-functions
|
||
domain: fn.hanzo.ai
|
||
|
||
triggers:
|
||
nats:
|
||
url: ${NATS_URL}
|
||
credentials:
|
||
user: functions-trigger
|
||
password: ${NATS_PASSWORD}
|
||
kafka:
|
||
brokers: ${KAFKA_BROKERS}
|
||
group_prefix: fn-
|
||
cron:
|
||
store: database # Cron state in SQL
|
||
|
||
gpu_pool:
|
||
enabled: true
|
||
size: 4
|
||
gpu_type: nvidia.com/gpu
|
||
idle_timeout: 600s
|
||
max_model_cache: 50Gi
|
||
preload_models:
|
||
- bge-large-en-v1.5
|
||
- whisper-small
|
||
|
||
snapshots:
|
||
enabled: true
|
||
storage: s3://hanzo-functions/snapshots/
|
||
max_age: 7d
|
||
|
||
runtimes:
|
||
python:
|
||
image: ghcr.io/hanzoai/fn-python:3.12
|
||
gpu_image: ghcr.io/hanzoai/fn-python:3.12-cuda
|
||
default_timeout: 60s
|
||
max_timeout: 900s
|
||
go:
|
||
image: ghcr.io/hanzoai/fn-go:1.22
|
||
default_timeout: 30s
|
||
max_timeout: 300s
|
||
rust:
|
||
image: ghcr.io/hanzoai/fn-rust:1.77
|
||
gpu_image: ghcr.io/hanzoai/fn-rust:1.77-cuda
|
||
default_timeout: 60s
|
||
max_timeout: 900s
|
||
typescript:
|
||
image: ghcr.io/hanzoai/fn-typescript:22
|
||
default_timeout: 30s
|
||
max_timeout: 300s
|
||
|
||
observability:
|
||
log_level: info
|
||
trace_sampling: 0.1
|
||
metrics_namespace: hanzo_fn</code></pre>
|
||
<h2>Security Considerations</h2>
|
||
<p>What an attacker gets from the wrong implementation: an invoke path that runs org code in-process is arbitrary code execution inside the binary that holds every tenant's stores — which is why execution is delegated to the sandbox and fails closed when the sandbox is absent; a registry that stored secret VALUES would turn a function listing into credential disclosure — it stores names, and values stay in KMS; and an invoke gated after the work instead of before it is free compute for an unfunded org — the gate runs first and 402s (<code>apps/functions/invoke.go:169-171</code>).</p>
|
||
<h3>Authentication and Authorization</h3>
|
||
<ul><li><strong>Function deployment</strong>: Requires IAM authentication with <code>functions:deploy</code> permission scoped to the organization.</li><li><strong>HTTP trigger invocation</strong>: Requires IAM JWT or API key. Configurable per trigger (<code>auth: required</code>, <code>optional</code>, or <code>none</code>).</li><li><strong>MQ/Stream triggers</strong>: Authenticated via NATS/Kafka credentials managed by the control plane. Individual functions do not handle message bus authentication.</li><li><strong>Control plane API</strong>: All endpoints require IAM authentication with appropriate <code>functions:*</code> permissions.</li></ul>
|
||
<h3>Network Policies</h3>
|
||
<p>Functions can only communicate with:</p>
|
||
<ul><li>Hanzo internal services (IAM, KV, Object Storage, MQ, Stream) via their cluster-internal endpoints</li><li>External URLs explicitly allowlisted in the function configuration</li></ul>
|
||
<pre><code class="lang-yaml">apiVersion: networking.k8s.io/v1
|
||
kind: NetworkPolicy
|
||
metadata:
|
||
name: hanzo-functions-egress
|
||
namespace: hanzo-functions
|
||
spec:
|
||
podSelector:
|
||
matchLabels:
|
||
app.kubernetes.io/managed-by: hanzo-functions
|
||
egress:
|
||
- to:
|
||
- namespaceSelector:
|
||
matchLabels:
|
||
name: hanzo
|
||
ports:
|
||
- protocol: TCP
|
||
port: 4222 # NATS
|
||
- protocol: TCP
|
||
port: 9092 # Kafka
|
||
- protocol: TCP
|
||
port: 6379 # KV
|
||
- protocol: TCP
|
||
port: 9000 # MinIO
|
||
- protocol: TCP
|
||
port: 5432 # SQL
|
||
policyTypes:
|
||
- Egress</code></pre>
|
||
<h3>Execution Time Limits</h3>
|
||
<p>Functions have a hard timeout (default 60s, max 900s). The invocation proxy terminates functions that exceed their timeout and returns a 504 Gateway Timeout to the caller. For MQ/Stream triggers, the message is nacked and redelivered.</p>
|
||
<p>This prevents runaway GPU consumption from buggy or malicious functions. A function that enters an infinite loop will be killed after its timeout, and the GPU is returned to the pool.</p>
|
||
<h3>Supply Chain Security</h3>
|
||
<p>Function container images are built from Hanzo-maintained base images. These base images are:</p>
|
||
<ul><li>Built from minimal base images (distroless for Go/Rust, slim for Python/TypeScript)</li><li>Scanned for CVEs on every build via Trivy</li><li>Signed with cosign and verified at deployment time</li><li>Pinned to specific digests in the function manifest (not mutable tags)</li></ul>
|
||
<p>User function code is injected into these base images at deployment time. The control plane validates that the source archive does not contain executable binaries, symlinks outside the function directory, or files larger than 100MB.</p>
|
||
<h2>Relationship to Other HIPs</h2>
|
||
<p>| HIP | Relationship | |-----|-------------| | <strong>HIP-4</strong> (LLM Gateway) | Inference event trigger source. Functions process LLM usage events. | | <strong>HIP-19</strong> (Tensor Operations) | Candle library used in Rust GPU runtime for tensor operations. | | <strong>HIP-26</strong> (IAM) | Authentication for function deployment and HTTP trigger invocation. | | <strong>HIP-27</strong> (KMS) | Secret injection into function environments. | | <strong>HIP-1164</strong> (KV Store) | Functions access KV via <code>ctx.kv</code> for caching and state. | | <strong>HIP-1061</strong> (Event Streaming) | Stream trigger consumes Kafka topics. Functions publish to Stream. | | <strong>HIP-132</strong> (Telemetry) | Prometheus metrics and structured logging. | | <strong>HIP-405</strong> (Object Storage) | Model cache storage. Container snapshot storage. Function access via <code>ctx.storage</code>. | | <strong>HIP-106</strong> (Cloud) | Functions are a deployment target within the Cloud platform. | | <strong>HIP-43</strong> (Inference Engine) | Persistent serving complement. Engine for steady-state; Functions for bursty. | | <strong>HIP-50</strong> (Edge Computing) | TypeScript functions sync to Edge for latency-sensitive CPU workloads. | | <strong>HIP-57</strong> (ML Pipeline) | Pipeline stages can be implemented as functions. Retraining triggers. | | <strong>HIP-105</strong> (In-Process Extension Runtime) | Complementary, different workload class. HIP-60 runs full containerized functions in Knative pods (cold start in seconds, GPU-attachable). HIP-105 runs in-process wasm/JS/Go extensions inside a host service (cold start in microseconds, no pod). Rule of thumb: if the work justifies a fresh pod, HIP-60; if it's a hot-path validator or per-record hook, HIP-105. |</p>
|
||
<h2>References</h2>
|
||
<ol><li><a href="https://knative.dev/docs/serving/">Knative Serving Documentation</a></li><li><a href="https://knative.dev/docs/serving/autoscaling/">Knative Autoscaling</a></li><li><a href="https://criu.org/Main_Page">CRIU: Checkpoint/Restore in Userspace</a></li><li><a href="https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/">NVIDIA Container Toolkit</a></li><li><a href="./hip-0004-llm-gateway-unified-ai-provider-interface.md">HIP-4: LLM Gateway</a></li><li><a href="./hip-1061-mq-queues-and-streams.md">HIP-1061: MQ — Queues and Streams</a></li><li><a href="./hip-0043-llm-inference-engine-standard.md">HIP-43: LLM Inference Engine Standard</a></li><li><a href="./hip-0050-edge-computing-standard.md">HIP-50: Edge Computing Standard</a></li><li>HIP-57: ML Pipeline & Training Standard</li><li><a href="https://docs.aws.amazon.com/lambda/latest/dg/lambda-runtime-environment.html">AWS Lambda Execution Environment</a></li><li><a href="https://github.com/hanzoai/functions">Hanzo Functions Repository</a></li></ol>
|
||
<h2>Copyright</h2>
|
||
<p>Copyright and related rights waived via <a href="https://creativecommons.org/publicdomain/zero/1.0/">CC0</a>.</p>
|
||
<footer>
|
||
<div class="fmap">
|
||
<div><b>Build</b>
|
||
<a href="https://hanzo.app">hanzo.app</a> app builder
|
||
<a href="https://hanzo.codes">hanzo.codes</a> code with an agent
|
||
<a href="https://hanzoskills.com">hanzoskills.com</a> what an agent reads</div>
|
||
<div><b>Run</b>
|
||
<a href="https://console.hanzo.ai">console.hanzo.ai</a> cloud console
|
||
<a href="https://api.hanzo.ai">api.hanzo.ai</a> the API</div>
|
||
<div><b>Account</b>
|
||
<a href="https://hanzo.id">hanzo.id</a> sign in
|
||
<a href="https://pay.hanzo.ai">pay.hanzo.ai</a> top up
|
||
<a href="https://billing.hanzo.ai">billing.hanzo.ai</a> invoices and usage</div>
|
||
<div><b>Company</b>
|
||
<a href="https://hanzo.ai">hanzo.ai</a> canonical
|
||
<a href="https://hanzo.ventures">hanzo.ventures</a> ventures</div>
|
||
</div>
|
||
<div class="fend"><a href="/hip-0060.md">This page as plain text</a> · <a href="/llms.txt">Index for agents</a></div>
|
||
</footer>
|
||
</div></body></html> |