papers/hanzo-candle/sections/99-conclusion.tex

\section{Conclusion}

\label{sec:conclusion}

We have presented Hanzo Candle, a Rust ML framework designed for high-performance inference. By leveraging Rust's ownership system for zero-GC memory management, arena-based allocation for reduced fragmentation, automatic kernel fusion for reduced launch overhead, and a type-safe API for compile-time error detection, Candle achieves 1.4--2.8x throughput improvements over PyTorch with 3.1x lower latency variance. Production deployment at Hanzo AI serving 47M inference requests over 14 months demonstrates the practical viability of Rust as an inference runtime. Candle is open-source and available at \texttt{github.com/hanzoai/candle}.

\bibliographystyle{plain}
\begin{thebibliography}{30}

\bibitem{burn2023}
Burn.
\newblock Burn: Dynamic deep learning framework in {Rust}.
\newblock \emph{GitHub Repository}, 2023.

\bibitem{dao2022flashattention}
T.~Dao, D.~Y. Fu, S.~Ermon, A.~Rudra, and C.~R\'e.
\newblock {FlashAttention}: Fast and memory-efficient exact attention with {IO}-awareness.
\newblock In \emph{NeurIPS}, 2022.

\bibitem{dao2023flashattention2}
T.~Dao.
\newblock {FlashAttention-2}: Faster attention with better parallelism and work partitioning.
\newblock \emph{arXiv preprint arXiv:2307.08691}, 2023.

\bibitem{ggml2023}
G.~Gerganov.
\newblock {ggml}: Tensor library for machine learning.
\newblock \emph{GitHub Repository}, 2023.

\bibitem{jung2018rustbelt}
R.~Jung, J.-H. Jourdan, R.~Krebbers, and D.~Dreyer.
\newblock {RustBelt}: Securing the foundations of the {Rust} programming language.
\newblock \emph{POPL}, 2018.

\bibitem{klabnik2019rust}
S.~Klabnik and C.~Nichols.
\newblock \emph{The Rust Programming Language}.
\newblock No Starch Press, 2nd edition, 2019.

\bibitem{kwon2023vllm}
W.~Kwon, Z.~Li, S.~Zhuang, et~al.
\newblock Efficient memory management for large language model serving with {PagedAttention}.
\newblock In \emph{SOSP}, 2023.

\bibitem{llamacpp2023}
G.~Gerganov.
\newblock {llama.cpp}: LLM inference in {C/C++}.
\newblock \emph{GitHub Repository}, 2023.

\bibitem{milakov2018online}
M.~Milakov and N.~Gimelshein.
\newblock Online normalizer calculation for softmax.
\newblock \emph{arXiv preprint arXiv:1805.02867}, 2018.

\bibitem{onnxruntime2019}
Microsoft.
\newblock {ONNX Runtime}: Cross-platform, high performance {ML} inferencing.
\newblock \emph{GitHub Repository}, 2019.

\bibitem{paszke2019pytorch}
A.~Paszke, S.~Gross, F.~Massa, et~al.
\newblock {PyTorch}: An imperative style, high-performance deep learning library.
\newblock In \emph{NeurIPS}, 2019.

\bibitem{tensorrt2017}
NVIDIA.
\newblock {TensorRT}: High-performance deep learning inference.
\newblock \emph{NVIDIA Developer}, 2017.

\bibitem{tchrs2023}
tch-rs.
\newblock Rust bindings for the {C++} {API} of {PyTorch}.
\newblock \emph{GitHub Repository}, 2023.

\bibitem{thunderkittens2024}
B.~Spector, S.~Arora, and C.~R\'e.
\newblock {ThunderKittens}: Simple, fast, and adorable {AI} kernels.
\newblock \emph{arXiv preprint arXiv:2410.20399}, 2024.

\bibitem{tillet2019triton}
P.~Tillet, H.~T. Kung, and D.~Cox.
\newblock Triton: An intermediate language and compiler for tiled neural network computations.
\newblock In \emph{MAPL}, 2019.

\bibitem{tract2023}
Tract.
\newblock Tract: Tiny, no-nonsense, self-contained, {ONNX} and {TF Lite} inference.
\newblock \emph{GitHub Repository}, 2023.

\bibitem{whispercpp2023}
G.~Gerganov.
\newblock {whisper.cpp}: Port of {Whisper} in {C/C++}.
\newblock \emph{GitHub Repository}, 2023.

\bibitem{zheng2024sglang}
L.~Zheng, L.~Yin, Z.~Xie, et~al.
\newblock {SGLang}: Efficient execution of structured language model programs.
\newblock \emph{arXiv preprint arXiv:2312.07104}, 2024.

\bibitem{levis2024candle}
L.~Levis, N.~Patry, and P.~Delattre.
\newblock Candle: Minimalist {ML} framework for {Rust}.
\newblock \emph{GitHub Repository}, 2024.

\bibitem{pope2023efficiently}
R.~Pope, S.~Douglas, A.~Chowdhery, et~al.
\newblock Efficiently scaling transformer inference.
\newblock In \emph{MLSys}, 2023.

\bibitem{aminabadi2022deepspeed}
R.~Y. Aminabadi, S.~Rajbhandari, M.~Zhang, et~al.
\newblock {DeepSpeed} inference: Enabling efficient inference of transformer models at unprecedented scale.
\newblock In \emph{SC22}, 2022.

\bibitem{sheng2023flexgen}
Y.~Sheng, L.~Zheng, B.~Yuan, et~al.
\newblock {FlexGen}: High-throughput generative inference of large language models with a single {GPU}.
\newblock In \emph{ICML}, 2023.

\bibitem{agrawal2024taming}
A.~Agrawal, N.~Kedia, A.~Panwar, et~al.
\newblock Taming throughput-latency tradeoff in {LLM} inference with {Sarathi-Serve}.
\newblock In \emph{OSDI}, 2024.

\bibitem{ansel2024pytorch2}
J.~Ansel, E.~Yang, H.~He, et~al.
\newblock {PyTorch 2}: Faster machine learning through dynamic {Python} bytecode transformation.
\newblock In \emph{ASPLOS}, 2024.

\bibitem{vaswani2017attention}
A.~Vaswani, N.~Shazeer, N.~Parmar, et~al.
\newblock Attention is all you need.
\newblock In \emph{NeurIPS}, 2017.

\end{thebibliography}