mirror of
https://github.com/hanzoai/papers.git
synced 2026-10-22 17:20:17 +00:00
103 lines
2.9 KiB
TeX
103 lines
2.9 KiB
TeX
\section{Evaluation}
|
|
|
|
\label{sec:evaluation}
|
|
|
|
\subsection{Benchmark Applications}
|
|
|
|
We evaluate Hanzo Platform using 15 representative AI application architectures:
|
|
|
|
\begin{table}[H]
|
|
\centering
|
|
\small
|
|
\begin{tabular}{llc}
|
|
\toprule
|
|
\textbf{App Type} & \textbf{Components} & \textbf{GPU?} \\
|
|
\midrule
|
|
Chat API & LLM Gateway, Redis & No \\
|
|
RAG App & LLM, Vector DB, Web & No \\
|
|
Code Assistant & LLM, Git, Sandbox & No \\
|
|
Image Gen API & Diffusion model, S3 & Yes \\
|
|
Voice Agent & STT, LLM, TTS & Yes \\
|
|
Search Engine & Embedding, Index, LLM & Opt. \\
|
|
Recommendation & ML model, DB, API & Opt. \\
|
|
Content Moderation & Classifier, Queue & Yes \\
|
|
Document Processor & OCR, LLM, Storage & Opt. \\
|
|
Fine-tuning Service & Training, Eval, API & Yes \\
|
|
\bottomrule
|
|
\end{tabular}
|
|
\caption{Subset of 10/15 benchmark AI applications.}
|
|
\label{tab:apps}
|
|
\end{table}
|
|
|
|
\subsection{Deployment Time Comparison}
|
|
|
|
Time from \texttt{git push} to serving first request:
|
|
|
|
\begin{table}[H]
|
|
\centering
|
|
\small
|
|
\begin{tabular}{lccccc}
|
|
\toprule
|
|
\textbf{App Type} & \textbf{Hanzo} & \textbf{Vercel} & \textbf{Rly.} & \textbf{Fly} & \textbf{Render} \\
|
|
\midrule
|
|
Chat API & 1.1m & 0.8m & 2.4m & 1.9m & 3.2m \\
|
|
RAG App & 2.3m & 4.1m & 5.7m & 4.8m & 7.1m \\
|
|
Image Gen & 6.8m & N/A & 28m & 15m & 35m \\
|
|
Voice Agent & 5.2m & N/A & 22m & 12m & 29m \\
|
|
Fine-tuning & 12.1m & N/A & N/A & 41m & N/A \\
|
|
\midrule
|
|
\textbf{Average} & \textbf{5.5m} & --- & 14.5m & 14.9m & 18.6m \\
|
|
\bottomrule
|
|
\end{tabular}
|
|
\caption{Deployment times. N/A = platform does not support GPU workloads.}
|
|
\label{tab:deploy_time}
|
|
\end{table}
|
|
|
|
\subsection{Cost Comparison}
|
|
|
|
Monthly cost for running a medium-traffic RAG application (1000 req/hour):
|
|
|
|
\begin{table}[H]
|
|
\centering
|
|
\small
|
|
\begin{tabular}{lcccc}
|
|
\toprule
|
|
\textbf{Component} & \textbf{Hanzo} & \textbf{AWS} & \textbf{Fly} & \textbf{Rly.} \\
|
|
\midrule
|
|
Compute & \$47 & \$89 & \$62 & \$58 \\
|
|
Database & \$15 & \$42 & \$20 & \$25 \\
|
|
Vector DB & \$12 & \$67 & \$35 & N/A \\
|
|
LLM API & \$180 & \$180 & \$180 & \$180 \\
|
|
Storage & \$5 & \$12 & \$8 & \$7 \\
|
|
\midrule
|
|
\textbf{Total} & \textbf{\$259} & \$390 & \$305 & \$270+ \\
|
|
\bottomrule
|
|
\end{tabular}
|
|
\caption{Monthly cost comparison for a medium-traffic RAG application.}
|
|
\label{tab:cost}
|
|
\end{table}
|
|
|
|
\subsection{Developer Experience Survey}
|
|
|
|
We surveyed 120 developers who deployed AI applications on multiple platforms:
|
|
|
|
\begin{table}[H]
|
|
\centering
|
|
\small
|
|
\begin{tabular}{lcccc}
|
|
\toprule
|
|
\textbf{Dimension} & \textbf{Hanzo} & \textbf{Vercel} & \textbf{Fly} & \textbf{AWS} \\
|
|
\midrule
|
|
Setup ease & 4.6/5 & 4.7/5 & 3.8/5 & 2.4/5 \\
|
|
AI support & 4.8/5 & 2.9/5 & 3.2/5 & 3.8/5 \\
|
|
Cost clarity & 4.5/5 & 3.4/5 & 3.9/5 & 2.1/5 \\
|
|
Debugging & 4.3/5 & 3.8/5 & 3.5/5 & 3.2/5 \\
|
|
Scaling & 4.4/5 & 4.2/5 & 4.0/5 & 4.5/5 \\
|
|
Overall & \textbf{4.5/5} & 3.8/5 & 3.7/5 & 3.2/5 \\
|
|
\bottomrule
|
|
\end{tabular}
|
|
\caption{Developer experience survey results (120 respondents).}
|
|
\label{tab:dx_survey}
|
|
\end{table}
|
|
|