papers/hanzo-engine/main.tex

\documentclass[11pt]{article}
\usepackage{../shared/hanzocover}
\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{amsmath,amssymb}
\usepackage{graphicx}
\usepackage{booktabs}
\usepackage{xcolor}
\usepackage{tikz}
\usetikzlibrary{shapes,arrows.meta,positioning,fit,backgrounds,calc}
\usepackage[hidelinks]{hyperref}

\emergencystretch=2em

\definecolor{hzblue}{rgb}{0.12,0.30,0.55}
\definecolor{hzslate}{rgb}{0.22,0.25,0.30}
\definecolor{hzgreen}{rgb}{0.10,0.45,0.25}
\definecolor{hzred}{rgb}{0.60,0.16,0.16}

% =====================================================================
% Numbers-as-data contract. Every measured cell defaults to \runpending.
% The harness (scripts/bench_stats.py) writes results-data.tex, which
% \renewcommand's the macros below with values that are a pure function of
% the raw per-repetition samples committed under engine/bench-runs. Until
% that file exists we typeset "run pending", never a fabricated number.
% Macro scheme (letters only): \r<Backend><Role><Field>
%   Backend in {Cuda,Metal,Rocm,Vulkan};
%   Role    in {Decode,Prefill,PrefillRagged,PrefillLong,PrefillMax};
%   Field   H=hanzo t/s, Hci=95% CI half-width, L=llama t/s,
%           R=ratio hanzo/llama, V=verdict {WIN,LOSS,PARITY}.
% =====================================================================
\newcommand{\runpending}{\textit{\small run pending}}
\makeatletter
\newcommand{\pend}[1]{\@ifundefined{#1}{\expandafter\newcommand\csname #1\endcsname{\runpending}}{}}
\newcommand{\pendcell}[1]{\pend{r#1H}\pend{r#1Hci}\pend{r#1L}\pend{r#1R}\pend{r#1V}}
\makeatother
\pendcell{CudaDecode}\pendcell{CudaPrefill}\pendcell{CudaPrefillRagged}\pendcell{CudaPrefillLong}\pendcell{CudaPrefillMax}
\pendcell{MetalDecode}\pendcell{MetalPrefill}\pendcell{MetalPrefillRagged}\pendcell{MetalPrefillLong}\pendcell{MetalPrefillMax}
\pendcell{RocmDecode}\pendcell{RocmPrefill}\pendcell{RocmPrefillRagged}\pendcell{RocmPrefillLong}\pendcell{RocmPrefillMax}
\pendcell{VulkanDecode}\pendcell{VulkanPrefill}\pendcell{VulkanPrefillRagged}\pendcell{VulkanPrefillLong}\pendcell{VulkanPrefillMax}
% Bandwidth-wall roofline cells (Vulkan / ROCm effective GB/s and rooflines)
\pend{rVulkanEffBW}\pend{rVulkanRoofBW}\pend{rVulkanReach}
\pend{rRocmEffBW}\pend{rRocmRoofBW}\pend{rRocmReach}
\pend{rCudaEffBW}\pend{rCudaRoofBW}\pend{rCudaReach}
\pend{rMetalEffBW}\pend{rMetalRoofBW}\pend{rMetalReach}
% Architecture row: prompt-lookup speculation (the accepted-draft multiple), reported BESIDE
% the kernel rows, never blended -- it is an algorithmic speedup, orthogonal to kernel efficiency.
\pend{rSpecSustainMult}\pend{rSpecSustainVsLl}\pend{rSpecCodeMult}\pend{rSpecCodeVsLl}
\pend{rSpecNovelMult}\pend{rSpecNovelVsLl}\pend{rSpecGamma}
\IfFileExists{results-data.tex}{\input{results-data.tex}}{}
\IfFileExists{results-spec.tex}{\input{results-spec.tex}}{}

\title{hanzo-engine: A Portable Single-Binary LLM Inference Engine, Measured Against llama.cpp Across Four GPU Backends}
\author{Zach Kelling\thanks{Corresponding author: [email protected]} \\
    \textit{Hanzo Industries Inc (Techstars '17)} \\
    Los Angeles, California \\
    \texttt{https://hanzo.ai}}
\date{Hanzo Research}

\begin{document}

\hanzocoverpage

\twocolumn[
  \begin{@twocolumnfalse}
  \maketitle
  \begin{abstract}
  \noindent
  A production LLM inference engine faces a portability--performance dilemma: the
  fastest engines are backend-specialized (TensorRT-LLM for NVIDIA, MLX for Apple),
  while the portable engine of record, \texttt{llama.cpp}, sets the cross-platform
  bar. We ask a narrow, falsifiable question: can one Rust binary, compiled from a
  single kernel source through an auto-fusion DSL, \emph{meet or beat} \texttt{llama.cpp}
  on its own strongest per-target build across CUDA, Metal, ROCm, and Vulkan? We answer
  it with a benchmark protocol designed to survive hostile review---strongest-baseline
  ($\texttt{llama.cpp}$ built with its best flags per target, SHA recorded), cache-cold,
  contention-gated, both prefill and decode phases, ragged prompt shapes, $N{=}7$
  repetitions with Student-$t$ 95\% confidence intervals, byte-exact quality gates,
  and a fully pinned manifest (our commit, crate versions, model SHA-256, driver, OS).
  We report the measured board \emph{including where we lose and why}, and the answer is
  nuanced. Prefill is at parity: a clean WIN on CUDA at long context
  (\rCudaPrefillMaxR$\times$), parity-to-slightly-behind on Metal and ROCm, and a
  size-sensitive weakness on Vulkan that we root-cause (\S\ref{sec:results}). Decode, on the
  1.7B headline and \emph{greedy-vs-greedy} (a sampling-parity control added mid-study to correct
a stochastic-sampler artifact, \S\ref{sec:results}), trails \texttt{llama.cpp}---modestly except
on Vulkan---on all four backends---\rCudaDecodeR$\times$ (CUDA),
  \rMetalDecodeR$\times$ (Metal), \rRocmDecodeR$\times$ (ROCm), \rVulkanDecodeR$\times$
  (Vulkan)---and we decompose that gap rather than dress it. On the integrated APU it is
  bandwidth \emph{utilization}: we sustain \rVulkanReach\ (Vulkan) and \rRocmReach\ (ROCm) of
  the effective bandwidth \texttt{llama.cpp} itself extracts, against a \emph{measured}
  213~GB/s device peak---a coalescing-and-submission gap, not an ALU deficit
  (\S\ref{sec:bw}). On discrete memory (CUDA, Metal) it is a fixed per-layer overhead that
  amortizes only slightly with model size once sampling parity is enforced. We do not claim a performance
  crown; we claim a portable single binary whose exact standing against the portable baseline
  is now measured, reproducible, and reported without varnish. Where a measurement is
  not yet in hand we typeset it as \emph{run pending} rather than invent a number.
  \end{abstract}
  \vspace{1.5em}
  \end{@twocolumnfalse}
]

\section{Introduction}\label{sec:intro}
Two facts about LLM inference engines are in tension. First, the fastest engines are
specialized to one accelerator: TensorRT-LLM~\cite{tensorrtllm} compiles for NVIDIA,
and vendor kernels win by exploiting one architecture's memory hierarchy and matrix
units. Second, the engine most practitioners actually deploy on heterogeneous
hardware is \texttt{llama.cpp}~\cite{llamacpp}, whose portability across CPU, CUDA,
Metal, ROCm, and Vulkan---from one C/C\texttt{++} source---made it the de facto
cross-platform baseline. Portability and peak performance are usually traded off: a
portable engine carries an abstraction tax.

\textbf{hanzo-engine} (github.com/hanzoai/engine), a Rust engine descended from
mistral.rs~\cite{mistralrs}, targets the same portability envelope as
\texttt{llama.cpp} but generates its GPU kernels from a single source through an
auto-fusion DSL (hanzo-fusion) that lowers to each backend (CUDA, Metal, ROCm/HIP,
Vulkan/SPIR-V). The engineering bet is that a fusion compiler can recover the
abstraction tax---emitting fused, backend-idiomatic kernels that match hand-written
ones---so that one binary need not concede performance for portability.

This paper does not argue that bet rhetorically; it measures it. Our contributions:
\begin{itemize}\itemsep2pt
  \item \textbf{A benchmark protocol} (\S\ref{sec:method}) for cross-engine LLM
  throughput claims that is defensible under adversarial review: strongest per-target
  baseline with recorded SHA and build flags; cache-cold loads; a contention gate that
  refuses to measure on a busy accelerator; ragged (non-16-aligned) prompt shapes;
  $N{=}7$ repetitions with Student-$t$ confidence intervals and a coefficient-of-variation
  flag; byte-exact / recall quality gates for lossy options; and a pinned reproducibility
  manifest.
  \item \textbf{A reproducible harness} (\S\ref{sec:repro}): a count-based sampler that is
  immune by construction to self-reported-rate errors, a driver that records the manifest
  and runs both engines identically, and an analysis script that computes every published
  statistic as a pure function of the committed raw samples.
  \item \textbf{The measured board} (\S\ref{sec:results}) across four GPU backends and the
  Qwen3 family, with confidence intervals, both phases, and every loss reported plainly.
  \item \textbf{A bandwidth-wall analysis} (\S\ref{sec:bw}): the integrated-APU Vulkan and
  ROCm decode gaps are shown to be memory-bandwidth-bound, not kernel-quality-bound, via a
  roofline argument with measured effective bandwidth---a result we consider a contribution,
  not an embarrassment.
\end{itemize}

We scope our claim precisely. We do \emph{not} claim datacenter batch-serving supremacy;
that ground belongs to vLLM~\cite{paged2023} and SGLang~\cite{sglang2024} on server GPUs.
Nor---the measurements are blunt about this---do we claim to beat \texttt{llama.cpp} across
the board: on the small-model headline we are at parity on prefill (a WIN only on CUDA at
long context) and behind on decode, by a margin we localize and bound. What we claim is a
\emph{portable single binary whose standing against the portable baseline is measured rather
than asserted}, competitive on prefill, with a decode gap that is bandwidth-utilization on
the APU and amortizing fixed overhead on discrete memory---plus two production-grade lossless
features (prompt-lookup speculative decoding and an opt-in KV-cache quantization) verified
byte-exact. The contribution is the protocol, the reproducible harness, and the honest board,
as much as the engine.

\section{Related Work}\label{sec:related}
\textbf{Portable inference.} \texttt{llama.cpp}~\cite{llamacpp} established the
single-source, multi-backend design point and the \texttt{llama-bench} measurement
convention (warm-up, repeated prefill/decode timing) we adopt for both engines.
It is our baseline throughout.

\textbf{Paged KV and serving.} vLLM's PagedAttention~\cite{paged2023} eliminated KV
fragmentation and, with Orca-style iteration-level scheduling~\cite{orca2022}, defines
modern continuous batching. SGLang~\cite{sglang2024} adds RadixAttention for prefix
reuse. hanzo-engine ports the vLLM-v1 paged design and Orca scheduling to the same
portable substrate; we validate the batching behavior rather than claim novelty in it.

\textbf{Attention kernels.} FlashAttention~\cite{flashattention2022} and its
successor~\cite{flashattention2_2023} make attention IO-aware; our backends implement
flash-style fused attention (single-query strided-KV GQA on decode) generated by the
fusion DSL.

\textbf{KV-cache quantization.} KIVI~\cite{kivi2024} showed tuning-free asymmetric
low-bit KV quantization. Our opt-in int8 per-token KV cache follows the KIVI/KVQuant
symmetric-per-token lineage and is gated by a byte-exact CPU oracle.

\textbf{Speculative decoding.} Speculative decoding~\cite{leviathan2023} accelerates
memory-bound decode by verifying cheap drafts. Prompt-lookup decoding~\cite{promptlookup2023}
is a training-free, quantization-agnostic special case that drafts from the sequence's own
$n$-gram history; we ship it in the production decode path with a byte-identical greedy
guarantee.

\section{System Architecture}\label{sec:arch}
\input{figures/architecture}
hanzo-engine is one Rust workspace producing one binary per target. Four elements bear
on the measurements.

\textbf{Fusion DSL.} Kernels are written once in hanzo-fusion, an auto-fusion compiler on
the hanzo-cl substrate, and lowered to CUDA, Metal, HIP, and SPIR-V. Block-structured MoE,
a fused router gate, and fused GQA flash-SDPA are all DSL-generated and bit-exact across
backends. Portability is therefore a property of \emph{one} kernel source, not of $N$
hand-maintained backends.

\textbf{Paged KV cache.} A vLLM-v1-faithful paged attention allocates the KV cache in
fixed blocks with per-sequence block tables and context lengths, enabling non-fragmenting
memory and mixed-length batches.

\textbf{Continuous batching.} The paged decode path uses Orca iteration-level
scheduling~\cite{orca2022}: every running sequence decodes each step, with preemption
reserved for genuine KV-block exhaustion. Length divergence no longer sheds sequences,
so a length-mixed batch runs at its true occupancy rather than collapsing to an effective
batch of one. Measured greedy on the APU (ROCm, Qwen3-1.7B), aggregate decode throughput scales
from $102$ to $226$~tok/s as concurrency goes $1{\to}8$---a $2.2\times$ batching gain, not the
$8\times$ a compute-bound datacenter GPU would show, because the same GTT bandwidth ceiling
(\S\ref{sec:bw}) caps how far batching can amortize weight streaming on this hardware. The
scheduler enables the batch; the memory system bounds its payoff.

\textbf{Lossless acceleration.} Two decode-time options preserve greedy output exactly.
Prompt-lookup speculative decoding drafts from the tail $n$-gram of the sequence's own
history; because the draft carries no logits, the batched verifier accepts a token only
when it equals the target's own argmax, so greedy output is byte-identical with the
proposer on or off. An opt-in int8 per-token KV cache (\S\ref{sec:arch}) quantizes each
cached K/V vector to signed int8 with one f32 scale; the default cache is byte-unchanged.

\section{Methodology}\label{sec:method}
A throughput ratio between two engines is only as credible as the discipline behind it.
Our protocol is built to remove every systematic bias we could identify, and to expose
the residual noise rather than hide it.

\textbf{Strongest baseline.} \texttt{llama.cpp} is built per target with its best backend
flags (CUDA/HIP with flash-attention and CUDA graphs; Vulkan with coopmat), and we record
its git SHA and the CMake cache. Comparing against a weak baseline is the most common way
a ``win'' evaporates; we compare against \texttt{llama.cpp}'s strongest per-target build.

\textbf{Identical instrument, both engines.} Both engines are timed by wall-clock over a
fixed token count---\texttt{llama-bench}'s own convention. Using GPU timestamps for one
engine and wall-clock for the other would inject a cross-engine bias larger than the effect
we measure; identical instrumentation is the fair choice. Our sampler
(\S\ref{sec:repro}) records raw per-repetition (wall, tokens) pairs and \emph{ignores}
self-reported rates, so it cannot be fooled by an engine's internal counters---the failure
mode behind a prior spurious result.

\textbf{Contention gate.} A shared workstation is never perfectly idle. Before every timed
run the driver refuses to proceed if the GPU is $>15\%$ busy or if any GPU/compile
competitor (profiler, compiler, another engine) is running; desktop CPU load is recorded as
advisory. Because decode is GPU-compute-bound, its timing is robust to CPU load, which the
coefficient-of-variation flag confirms empirically.

\textbf{Sampling parity (a required control).} Both engines must decode under the same
sampler, stated next to every rate. \texttt{llama-bench} decodes greedily; we therefore force
greedy argmax on our side too (top-$k{=}1$, device path), so the measured rate isolates
model evaluation, not sampler cost. This control is not optional: an unset temperature
defaults to $1.0$ and, absent top-$k$, routes decode through a full-vocabulary CPU multinomial
whose per-token tax (backend-variable, $1.5\text{--}1.8$~ms on CPU-sampled backends, $\approx 0$
where the sampler is on-device) silently inflates the apparent decode gap---an artifact we hit
and corrected (\S\ref{sec:results}). The prefix cache and EOS stop are disabled so every run
does the full work.

\textbf{Ragged shapes.} Aligned-only prompt lengths hide alignment-dependent kernel bugs
(the $m \bmod 16$ class). We measure prefill at an aligned size (512), a ragged size (500),
and a size sweep (2048, 4096) whose linear regime lets us separate pure prefill compute
from fixed per-request overhead.

\textbf{Statistics.} Each cell is $N{=}7$ repetitions after warm-up. We report the mean and
a Student-$t$ 95\% confidence interval, and flag any cell whose coefficient of variation
exceeds 5\%. Ratios carry a delta-method interval; a verdict is WIN only if the ratio
interval lies entirely above 1, LOSS if entirely below, else PARITY.

\textbf{Quality gates.} Lossless options are held to identity, not just speed: prompt-lookup
and the default KV cache are byte-identical to baseline greedy output; the opt-in int8 KV
cache is argmax-identical at 128 tokens and preserves exact ordered recall at 256.

\textbf{Pinned manifest.} Every run records the engine commit, crate versions
(hanzo-ml, backend kernels), the \texttt{llama.cpp} SHA, the model path and SHA-256, the
GPU and driver, the OS, the exact flags, and the environment. The board is reproducible
from the manifest alone.

\section{Results}\label{sec:results}
Every number below is emitted by the harness from committed raw samples; a cell that had
not been run would read \emph{run pending} rather than a guess. The headline model is
Qwen3-1.7B-Q4\_K\_M, the one GGUF common to all boxes (byte-identical for both engines,
throughput being weight-value-independent at fixed shape and quant); a Qwen3-4B control on
the two discrete-memory backends probes the model-size effect. The cut line is engine 1.7.87 @
\texttt{1ac219c43}, hanzo-ml 0.11.88, hanzo-rocm-kernels 0.11.36, hanzo-metal-kernels
0.11.44.

\begin{table}[t]\centering\small
\caption{Measured board, Qwen3-1.7B-Q4\_K\_M, engine 1.7.87 (eager decode), quiet box,
decode greedy-vs-greedy. t/s as mean$\pm$95\% CI over $N{=}7$. Ratio is hanzo-engine /
\texttt{llama.cpp} (strongest per-target build); verdict at the 95\% level. Prefill sizes:
512 aligned, 500 ragged, 2048/4096 compute-bound sweep.}
\label{tab:board}
\IfFileExists{results-board.tex}{\input{results-board.tex}}{%
\begin{tabular}{@{}llrrrrl@{}}\toprule
backend & phase & $n$ & hanzo & llama & ratio & verdict \\\midrule
\runpending & & & & & & \\\bottomrule
\end{tabular}}
\end{table}

\textbf{A sampling-parity correction (erratum).} An earlier cut of this board measured our
decode with a \emph{stochastic} sampler while \texttt{llama-bench} measured greedy---an
apples-to-oranges control we had not stated, and a real artifact we correct here in the open.
Our harness left \texttt{temperature} unset; the engine defaults an unset temperature to
\(1.0\) and, with no top-\(k\), samples the full \(151{,}936\)-token vocabulary by a
\emph{CPU} multinomial each step---a per-token tax that idles the GPU and is absent from
greedy \texttt{llama-bench}. Corrected to greedy-vs-greedy (top-\(k{=}1\), device argmax),
decode rises materially: Metal \(0.72\!\to\!0.92\), ROCm \(0.72\!\to\!0.85\), Vulkan
\(0.52\!\to\!0.60\). CUDA is unchanged at \(0.84\): its sampler runs on-device (a cuda-gated
top-1 path), so the multinomial never fires---measured tax \(\approx\!0\) vs
\(1.5\text{--}1.8\)~ms/token on the CPU-sampled backends, which also explains why the tax was
backend-variable. Table~\ref{tab:board} is the corrected greedy-vs-greedy measurement;
the stochastic figures are retained in the run logs (\texttt{ARTIFACT-stochastic-*.json}).
Sampling config for \emph{both} engines is now a required, stated control (\S\ref{sec:method}).
Prefill is unaffected---it scores one generated token, so the tax is negligible. We report the
artifact rather than silently restate the numbers; the discipline, not the erratum, is the point.

\textbf{Decode.} Decode is the latency-critical, memory-bound phase. Greedy-vs-greedy,
hanzo-engine reaches \rMetalDecodeR$\times$ \texttt{llama.cpp} on Metal (near parity),
\rRocmDecodeR$\times$ on ROCm, \rCudaDecodeR$\times$ on CUDA, and \rVulkanDecodeR$\times$ on
Vulkan---a loss on all four, but a modest one except on Vulkan. On the integrated APU
(ROCm, Vulkan) the residual is bandwidth utilization (\S\ref{sec:bw}); on discrete memory
(CUDA, Metal) it is a small fixed per-layer overhead, not a kernel-quality gap.

\textbf{Conditions and versioning.} The board is measured on a \emph{quiet} box (accelerator
idle, contention gate passed). Sub-parity decode retains a residual load sensitivity through
the host-side dispatch path: the Vulkan greedy row is $93.9{\pm}1.7$~tok/s idle, but the same
binary and config on an agent-loaded box reads $76.9{\pm}8.7$ (CV $8\%$)---the same
measurement, not a contradiction, and the reason we gate on idle and flag CV. The board is
also stamped to one engine release: 1.7.87, \emph{eager} decode with no GPU command graph. It
is a moving baseline. Three kernel efforts in review will shift specific rows at the next
release---a Vulkan dense-decode command graph (measured $+22.3\%$ same-load, projecting the
Vulkan decode row from $0.60$ to a \emph{measured} $0.71$ on the shipped 1.7.90/ml-0.11.90 binary---credited to the dense-Qwen3 decode command graph, not the 1.7.90 sampler edits, since greedy already used host argmax pre-1.7.90 (the sort-skip touches only stochastic decode). A same-binary \texttt{VK\_GRAPHS} A/B on the shipped release isolates it directly: graph-off $0.58\times$ (exactly the eager baseline), graph-on $0.72\times$---a $+24.8\%$ decode gain from the command graph, matching its independently-isolated $+22.3\%$), a CUDA matvec-geometry port (toward
parity), and the ragged-BMM store, measured to lift Vulkan pp500 from $0.25$ to $0.51\times$ (a $2.17\times$ whole-prefill jump, CV $26\%\!\to\!1.2\%$, aligned pp512 bit-exact)---so
we version the board by engine release rather than present it as fixed.

\textbf{Prefill.} Prefill is compute-bound (a batched GEMM over the prompt). The whole-request
sampler charges prefill with a fixed per-request overhead that \texttt{llama-bench}'s pure
prefill excludes; we therefore read the compute-bound ratio from the large-prompt sweep
(\rRocmPrefillMaxR$\times$ on ROCm at 4096, \rCudaPrefillMaxR$\times$ on CUDA), where the
overhead is amortized, and report the small-prompt cells for completeness. On Vulkan the
ratio instead \emph{falls} with prompt length (\rVulkanPrefillR$\times$ at 512 to
\rVulkanPrefillMaxR$\times$ at 4096): a genuine long-context prefill weakness we root-caused
rather than hid. It is not submission chunking---disabling the ring-safety submission bound
(\texttt{VK\_WORK\_CAP=0}) leaves the cliff and instead hangs the 4096 prefill on the driver
ring, and bounded caps of $2\text{--}4\times$ the default move the rate by under $2\%$---and
it is not the weight GEMM, which the roofline probe clocks at $17\text{--}24$~TFLOP/s
(near-peak). The superlinear falloff is the $O(n^2)$ prefill attention geometry, isolating
the next kernel to fuse. A ragged (non-16-aligned) prompt compounds it: Vulkan
\texttt{pp500} is $\sim3\times$ slower than \texttt{pp512} and high-variance, an
alignment-sensitive path ROCm does not exhibit.

\textbf{Component measurements.} Three lossless/near-lossless mechanisms are verified at the
kernel level from the engine's run logs and re-checked here.
\emph{(i) ROCm min-bias.} Folding the k-quant activation scale once per block (rather than per
output element) leaves the Q4\_K prefill GEMM bit-exact---byte-identical 401-character greedy
decode on Qwen3-4B---while removing arithmetic: measured by rocprofv3 over 648/648 equal
dispatches, $\text{SQ\_INSTS\_VALU}$ falls $6.6\%$ on the asymmetric \texttt{qmmq\_q4k\_f16}
target and moves $+0.01\%$ on the symmetric \texttt{qmmq\_q6k\_f16} control that elides the
term. This is an isolated-effect measurement: treatment and control differ only in the branch
under test. \emph{(ii) Prompt-lookup.} On a grounded code-repeat prompt the proposer achieves
$85\%$ draft acceptance (mean $6.83$ of $8$), emitting $\sim$47 tokens in $\sim$6 target
forwards where the baseline needs 48, with greedy output byte-identical on or off; the
wall-clock win is memory-bandwidth-regime dependent (break-even on compute-bound CPU, a
multiplicative win on memory-bound decode). \emph{(iii) int8 KV.} The opt-in int8 per-token KV
cache is argmax-identical to f16 at 128 greedy tokens and preserves exact ordered 12-item
recall at 256, gated by a 6-test CPU oracle; the default cache is byte-unchanged.

\textbf{Architecture row: prompt-lookup speculation.} Table~\ref{tab:spec} is reported
\emph{beside}, and never blended into, the kernel board: it measures an \emph{algorithmic}
speedup---the accepted-draft multiple over non-speculative decode---orthogonal to the
per-token kernel efficiency of Table~\ref{tab:board}. Prompt-lookup drafts $\gamma$ tokens from
the sequence's own $n$-gram history ($n_{\max}{=}3$), and the batched verifier accepts only
what greedy decode would itself emit, so output is byte-identical with speculation on or off---a
lossless guarantee we assert by hashing the completion (SHA-256 equal, lookup on vs off), not a
quality/speed trade. The realized multiple is workload-dependent by construction, so following
the shared cross-backend protocol we measure three verbatim classes at sampling parity (greedy):
\emph{sustained-reuse} (a repeating log whose tail $n$-gram recurs, where drafts land),
\emph{code-copy} (verbatim then novel, mixed reuse), and \emph{novel} (free generation, little
recurrence, where the method correctly stands down toward $1\times$). Gamma is swept per backend
($4,8,16$) on the reuse class: a fixed per-verify-round overhead must be amortized over accepted
tokens, so the break-even $\gamma$ is backend-specific. On Metal the reuse class reaches
$\rSpecSustainMult\times$ the non-speculative rate at $\gamma{=}\rSpecGamma$
($\rSpecSustainVsLl\times$ \texttt{llama.cpp}, a WIN), while novel holds
$\rSpecNovelVsLl\times$; the same protocol runs on the other backends for a comparable row.
These spec figures are stamped to the spec cut and re-measured at battery time (the manifest
version disambiguates): a verify-round matvec fix landing as ml~0.11.91 lifts the novel floor
to $\sim\!0.71\times$, lossless, without touching the kernel board.

\begin{table}[t]\centering\small
\caption{Architecture row: prompt-lookup speculative decoding, greedy, lossless (completion
SHA-256 identical with lookup on/off). Multiple is speculative $\div$ non-speculative decode
t/s (best over the $\gamma$ sweep); vs-llama is speculative t/s $\div$ \texttt{llama.cpp}
greedy. Metal shown; other backends run the same verbatim protocol. Reported beside, not
blended into, Table~\ref{tab:board}.}
\label{tab:spec}
\begin{tabular}{@{}lrr@{}}
\toprule
workload & best multiple ($\gamma$) & vs.\ \texttt{llama.cpp} \\
\midrule
sustained-reuse & \rSpecSustainMult$\times$ & \rSpecSustainVsLl$\times$ \\
code-copy       & \rSpecCodeMult$\times$    & \rSpecCodeVsLl$\times$    \\
novel           & \rSpecNovelMult$\times$   & \rSpecNovelVsLl$\times$   \\
\bottomrule
\end{tabular}
\end{table}

\textbf{Model size.} The 1.7B headline is a deliberately unforgiving choice: a small model
amortizes our fixed per-layer glue over the least compute. Table~\ref{tab:size} measures the
same backends on Qwen3-4B-Q4\_K\_M, greedy-vs-greedy. Decode improves with size but only
slightly---Metal $0.92\!\to\!0.94$, CUDA $0.84\!\to\!0.88$---once sampling parity is enforced.
This is itself a correction: in the stochastic board the apparent size effect looked far
larger (Metal $0.72\!\to\!0.81$) because the CPU-sampler tax is a bigger \emph{fraction} of a
faster small-model decode, so removing it shrinks the small model's deficit and most of the
gap with it. The real model-size effect is modest; the large one was mostly the artifact.

\begin{table}[t]\centering\small
\caption{Model-size effect, Qwen3-4B-Q4\_K\_M (per-box, native GGUF). Same protocol as
Table~\ref{tab:board}.}
\label{tab:size}
\IfFileExists{results-board-4b.tex}{\input{results-board-4b.tex}}{%
\begin{tabular}{@{}llrrrrl@{}}\toprule
backend & phase & $n$ & hanzo & llama & ratio & verdict \\\midrule
\runpending & & & & & & \\\bottomrule
\end{tabular}}
\end{table}

\section{The Bandwidth Wall}\label{sec:bw}
The below-parity decode ratios on the integrated APU (Radeon 8060S, gfx1151, shared LPDDR5X
via GTT) are memory-bandwidth-bound, not arithmetic-bound. Decode of a dense model streams
the full weight set once per token, so the token rate is bounded by
$\text{BW}_{\text{eff}}/\text{bytes-per-token}$~\cite{roofline2009}; because both engines
stream the \emph{same} quantized weights, the decode ratio \emph{is} the effective-bandwidth
ratio, independent of any byte estimate. Taking the model's \(1.28\)~GB as an upper bound on
bytes streamed per token, hanzo-engine sustains \rVulkanEffBW~GB/s on Vulkan against the
\rVulkanRoofBW~GB/s that \texttt{llama.cpp} sustains on the same device, and \rRocmEffBW\
vs.\ \rRocmRoofBW~GB/s on ROCm. The engine's device-to-device roofline probe measures the
APU's sustainable peak at \(213\)~GB/s (the LPDDR5X-8000 / 256-bit theoretical peak is
\(256\)~GB/s), and against that measured peak \texttt{llama.cpp}'s Vulkan decode already
sustains \(199\)~GB/s---\(93\%\) of peak, effectively \emph{at} the wall.

We, by contrast, do not claim to be at the wall. We reach \rVulkanReach\ (Vulkan) and
\rRocmReach\ (ROCm) of the bandwidth \texttt{llama.cpp} extracts from this APU---about
\(56\%\) of the measured peak on Vulkan---so about \(40\%\) of the bandwidth
\texttt{llama.cpp} reaches is left on the table. That residual is a bandwidth \emph{utilization} gap---uncoalesced
global-memory traffic and per-step command record/submit overhead---not an ALU gap: at the
kernel level our dp4a decode matvec already matches \texttt{llama.cpp}'s. It is therefore
improvable engineering (memory coalescing to raise cache-cold GTT bandwidth; a CPU-side
command graph to cut record/submit latency), and it is bounded above by the \(213\)~GB/s
hard wall of unified LPDDR5X that \texttt{llama.cpp} already reaches. On discrete-memory targets
(CUDA, Metal), where bandwidth is abundant relative to this 1.7B model, the same fused
kernels reach parity (\S\ref{sec:results}). The lesson generalizes: on APU-class edge
hardware, decode SOTA is won by raising sustained bandwidth utilization (coalescing,
command-graph submission) and by algorithmic reductions in bytes-per-token (KV quantization,
speculative verification), not by further ALU tuning.

\section{Limitations and Honest-Reporting Notes}\label{sec:limits}
We report the following plainly. \emph{(1) Prefill definition.} Our count-based sampler's
prefill includes per-request overhead absent from \texttt{llama-bench}'s pure prefill; we
mitigate with a size sweep but the small-prompt prefill cells understate our compute rate and
are labeled as such. \emph{(2) Shared workstation.} The APU box is a live workstation; timing
is taken under a contention gate and desktop load is recorded, with any residual noise
surfaced by the CV flag rather than smoothed away. \emph{(3) Scope.} We measure single-stream
throughput and a batching sanity check, not datacenter batch serving at scale; vLLM and SGLang
own that regime and we do not contest it here. \emph{(4) Model breadth.} The cross-backend
board fixes one model for comparability; MoE and larger dense results are reported per box
where the weights are resident. \emph{(5) Claims we overturned.} Where our independent
measurement disagrees with a prior internal number, the measured value in Table~\ref{tab:board}
stands and the discrepancy is noted in the run log, not reconciled by adjustment.

\section{Reproducibility}\label{sec:repro}
The harness lives in the engine repository under \texttt{scripts/}. \texttt{hanzo-bench
-{}-json} emits raw per-repetition \texttt{[wall\_s, tokens]} samples plus backend and version;
it records token \emph{counts} from the engine's usage and times with wall-clock, ignoring any
self-reported rate. \texttt{scripts/dossier.sh} runs the contention gate, writes the pinned
manifest (\S\ref{sec:method}), and drives \texttt{hanzo-bench} and \texttt{llama-bench} over the
shape matrix on the same GGUF. \texttt{scripts/bench\_stats.py} ingests the run directory and
computes the board, the confidence intervals, and this paper's \texttt{results-data.tex}---every
statistic a pure function of the committed samples. Each run directory under
\texttt{engine/bench-runs/} contains the manifest, the raw JSON for both engines, the contention
log, and the derived board. The appendix lists the exact invocation per backend.

\section{Conclusion}
One binary, one kernel source, four GPU backends. Measured against \texttt{llama.cpp}'s
strongest per-target build under a protocol designed for hostile review, hanzo-engine holds
parity on prefill---a WIN on CUDA at long context---and trails on 1.7B decode across all
four backends---modestly, near parity on Metal, wider only on Vulkan---by a margin we localize
rather than dress: bandwidth \emph{utilization} on the
integrated APU (memory-bound, not ALU-bound, improvable by coalescing and command-graph
submission and bounded by a measured 213~GB/s ceiling), and an amortizing fixed per-layer
overhead on discrete memory that a 4B control narrows only slightly. That is a more modest result
than the internal claims it was written to audit, several of which it overturns against a
properly-built baseline---which is the point. The honest board, losses and all, is the
credibility; the reproducible harness makes every number in it checkable from the committed
samples; and the gaps it exposes are a work list, not a verdict.

\appendix
\section{Reproducibility Appendix}\label{sec:appendix}
The cut line is engine 1.7.87 @ \texttt{1ac219c43}, hanzo-ml 0.11.88,
hanzo-rocm-kernels 0.11.36, and \texttt{llama.cpp}~@~\texttt{0821c5f}; the headline
model is \texttt{Qwen3-1.7B-Q4\_K\_M} (SHA-256 recorded per run). The harness lives on
branch \texttt{bench/dossier} of \texttt{github.com/hanzoai/engine}. One backend is
measured per invocation, on a verified-quiet box:
{\footnotesize
\begin{verbatim}
scripts/dossier.sh --backend {rocm|vulkan|metal|cuda} \
  --hanzo-bench target/release/hanzo-bench \
  --llama-bench <llama.cpp>/build/bin/llama-bench \
  --model <same.gguf> --model-tag qwen3-1p7b \
  -p 512,500,2048,4096 -n 128 -r 7 \
  --llama-dir <llama.cpp>
python3 scripts/bench_stats.py <run-dir>
scripts/assemble_paper.sh papers/hanzo-engine <run-dir>...
\end{verbatim}}
\noindent
\texttt{llama.cpp} is built per target with its strongest backend: CUDA and HIP with
flash-attention and CUDA/HIP graphs (\texttt{-fa 1}), Vulkan with \texttt{KHR\_coopmat},
Metal with its embedded library. Each run directory under \texttt{engine/bench-runs/}
carries \texttt{manifest.json} (engine commit, crate and kernel versions,
\texttt{llama.cpp} SHA, model SHA-256, GPU, OS, flags, environment),
\texttt{quiet-gate.txt}, the raw per-repetition JSON for both engines, and the derived
board; \texttt{results-data.tex} and \texttt{results-board.tex} in this directory are
generated from those samples. The first repetition of each hanzo cell is discarded as
per-shape warmup (\texttt{llama-bench} warms each shape internally); the raw samples are
committed so any reader can recompute with it included.

\bibliographystyle{plain}
\bibliography{references}

\end{document}