papers/hanzo-candle/hanzo-candle.tex

% Hanzo Candle: A Rust ML Framework for High-Performance Inference
\documentclass[11pt,twocolumn]{article}
\usepackage{../shared/hanzocover}
\usepackage{amsmath, amssymb, amsthm}
\usepackage{mathtools}
\usepackage{bm}
\usepackage{graphicx}
\usepackage{booktabs}
\usepackage{hyperref}
\usepackage{enumitem}
\usepackage{algorithm}
\usepackage{algpseudocode}
\usepackage{xcolor}
\usepackage{float}
\usepackage{subcaption}
\usepackage{multirow}
\usepackage{listings}
\input{../shared/lstlang}
\usepackage[numbers]{natbib}
\hypersetup{colorlinks=true,linkcolor=black,citecolor=blue,urlcolor=blue}

\newtheorem{theorem}{Theorem}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{definition}{Definition}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{corollary}[theorem]{Corollary}

\lstset{
  basicstyle=\ttfamily\small,
  breaklines=true,
  frame=single,
  numbers=left,
  numberstyle=\tiny,
  xleftmargin=2em,
}

\title{Hanzo Candle: A Rust ML Framework for High-Performance Inference}
\author{
    Antje Worring, Zach Kelling \quad
    David Wei \quad
    Marcus Chen \\
    \textit{Hanzo AI Research} \\
    \texttt{[email protected]}
}
\date{August 2024}

\begin{document}

\hanzocoverpage

\begin{abstract}
We present \textbf{Hanzo Candle}, a Rust-based machine learning framework designed for high-performance inference of large language models and multimodal models. While Python-based frameworks (PyTorch, TensorFlow) dominate model training, their runtime overhead, garbage collection pauses, and GIL contention make them suboptimal for latency-sensitive inference serving. Candle provides a pure-Rust tensor library with three key innovations: (i) a \emph{zero-copy tensor engine} that eliminates memory allocation overhead through arena-based allocation and view-based reshaping, achieving 2.3x lower memory allocation rates than PyTorch during inference; (ii) a \emph{fused kernel compiler} that automatically fuses sequential operations into single GPU kernels, reducing kernel launch overhead by 47\% on transformer architectures; and (iii) a \emph{type-safe model definition API} that leverages Rust's type system to catch dimension mismatches, dtype errors, and device placement bugs at compile time rather than runtime. We evaluate Candle on inference benchmarks spanning language models (Llama 3, Mistral, Phi), vision models (CLIP, SAM), and multimodal models (LLaVA), demonstrating 1.4--2.8x throughput improvements over PyTorch and 1.2--1.6x over ONNX Runtime, with 3.1x lower P99 latency variance due to the absence of garbage collection. Candle supports CUDA, Metal, and CPU backends, enabling deployment from data center GPUs to Apple Silicon laptops. The framework has been adopted for production inference serving at Hanzo AI, processing 47 million inference requests over 14 months.
\end{abstract}

\input{sections/01-introduction}
\input{sections/02-tensor-engine}
\input{sections/03-fused-kernel-compiler}
\input{sections/04-type-safe-model-api}
\input{sections/05-backend-architecture}
\input{sections/06-model-zoo}
\input{sections/07-evaluation}
\input{sections/08-production-deployment}
\input{sections/09-related-work}
\input{sections/10-discussion}
\input{sections/99-conclusion}

\end{document}