papers/zen/zen-coder_whitepaper.tex

\documentclass[11pt,a4paper]{article}
\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{amsmath,amsfonts,amssymb}
\usepackage{graphicx}
\usepackage{hyperref}
\usepackage{listings}
\usepackage[dvipsnames]{xcolor}
\usepackage{booktabs}
\usepackage{float}
\usepackage{geometry}
\geometry{margin=1in}

% Color definitions
\definecolor{zenblue}{RGB}{41,121,255}
\definecolor{zengreen}{RGB}{52,199,89}
\definecolor{zenorange}{RGB}{255,149,0}
\definecolor{codegray}{RGB}{245,245,245}

% Hyperref setup
\hypersetup{
    colorlinks=true,
    linkcolor=zenblue,
    urlcolor=zenblue,
    citecolor=zenblue
}

% Code listing setup
\lstset{
    backgroundcolor=\color{codegray},
    basicstyle=\ttfamily\small,
    breaklines=true,
    captionpos=b,
    frame=single,
    numbers=left,
    numberstyle=\tiny\color{gray}
}

\title{
    \vspace{-2cm}
    \Large \textbf{Zen AI Model Family} \\
    \vspace{0.5cm}
    \Huge \textbf{Zen Coder} \\
    \vspace{0.3cm}
    \large Agentic Code Generation Models (4B - 1T) \\
    \vspace{0.5cm}
    \normalsize Technical Report v2.0
}

\author{
    Antje Worring, Hanzo AI Research\thanks{[email protected]} \\
    \texttt{[email protected]} \\
    \\
    Zoo Labs Foundation \\
    \texttt{[email protected]}
}

\date{December 2025}

\begin{document}

\maketitle

\begin{abstract}
We present \textbf{Zen Coder}, a family of agentic code generation models ranging from 4B to 1T parameters,
trained on the \textbf{Zen Agentic Dataset}---8.47 billion tokens of real-world Claude Code sessions,
git history, and professional software development spanning 15 years across 1,452 repositories.
Unlike synthetic datasets, our training data captures actual debugging workflows, multi-file refactoring decisions,
tool use patterns, and error recovery from production AI development. The model family includes dense models
(4B, 24B, 123B) and MoE architectures (358B, 1T), covering edge deployment to frontier capabilities.
\end{abstract}

\tableofcontents
\newpage

\section{Introduction}

The emergence of agentic AI systems---where models interact with tools, execute multi-step plans, and
maintain context across complex workflows---has created demand for models specifically trained on
real agentic programming patterns. \textbf{Zen Coder} addresses this by training on actual Claude Code
debug sessions rather than synthetic instruction-following data.

\subsection{Model Family}

\begin{table}[H]
\centering
\begin{tabular}{llllll}
\toprule
\textbf{Model} & \textbf{Size} & \textbf{Base} & \textbf{VRAM} & \textbf{Context} & \textbf{Status} \\
\midrule
Zen Coder 4B & 4B & Zen-4B-Instruct & 8 GB & 32K & Trained \\
Zen Coder 24B & 24B & Zen Coder 24B-Instruct & 24 GB & 256K & Trained \\
Zen Coder 123B & 123B & Zen Coder 123B & 128 GB & 256K & Training \\
Zen Coder Max & 358B (MoE) & Zen Coder Max & 180 GB & 200K & Planned \\
Zen Coder Ultra & 1T (MoE) & Zen MoDE Ultra & 256 GB & 128K & Planned \\
\bottomrule
\end{tabular}
\caption{Zen Coder Model Family}
\end{table}

\subsection{Key Innovations}
\begin{itemize}
    \item \textbf{Real Agentic Data}: Trained on actual Claude Code sessions, not synthetic instruction data
    \item \textbf{Production Code}: 15 years of battle-tested software across AI, Web3, cryptography
    \item \textbf{Multi-Architecture}: Dense (4B-123B) and MoE (358B-1T) models for different deployment needs
    \item \textbf{Open Training}: Full training framework available via \href{https://github.com/zenlm/zen-trainer}{zen-trainer}
\end{itemize}

\section{Training Data: Zen Agentic Dataset}

The \textbf{Zen Agentic Dataset} comprises 8.47 billion tokens derived from real software development
and Claude Code interactions, distinguishing it from synthetic datasets.

\begin{table}[H]
\centering
\begin{tabular}{lrrr}
\toprule
\textbf{Data Source} & \textbf{Tokens (B)} & \textbf{\%} & \textbf{Description} \\
\midrule
Git History & 4.03 & 48\% & 15 years of commits, diffs, source files \\
Claude Debug Sessions & 2.42 & 29\% & Real debugging workflows with tool use \\
Claude Conversations & 1.14 & 13\% & Architecture discussions, code reviews \\
Claude Interactions & 0.86 & 10\% & Multi-turn coding assistance \\
\midrule
\textbf{Total} & \textbf{8.47} & 100\% & 3.35M training samples \\
\bottomrule
\end{tabular}
\caption{Zen Agentic Dataset Composition}
\end{table}

\subsection{Dataset Statistics}

\begin{table}[H]
\centering
\begin{tabular}{ll}
\toprule
\textbf{Metric} & \textbf{Value} \\
\midrule
Total Tokens & 8.47 billion \\
Training Samples & 3.35 million \\
Validation Samples & 100,000 \\
Total Size & 27 GB \\
Repositories & 1,452 \\
Time Span & 15 years (2010-2025) \\
\bottomrule
\end{tabular}
\caption{Dataset Statistics}
\end{table}

\subsection{Domain Coverage}

\subsubsection{Agentic AI \& LLM Infrastructure}
\begin{itemize}
    \item Model Context Protocol (MCP) - 260+ tool implementations
    \item Multi-agent orchestration - Claude, GPT-4, Gemini, Zen integrations
    \item Agent frameworks - Planning, memory, tool use, reflection
    \item LLM Gateway - Unified proxy for 100+ providers
\end{itemize}

\subsubsection{Web3 \& Blockchain}
\begin{itemize}
    \item Smart contracts - Solidity, Vyper (ERC20, ERC721, ERC1155, DeFi)
    \item Consensus engines - Quasar, BFT, DAG-based protocols
    \item Cross-chain bridges - Multi-VM architecture
    \item DeFi protocols - AMMs, lending, staking, governance
\end{itemize}

\subsubsection{Cryptography \& Security}
\begin{itemize}
    \item Post-quantum cryptography - Kyber, Dilithium, SPHINCS+
    \item Threshold cryptography - MPC, secret sharing, DKG
    \item Zero-knowledge proofs - zkSNARKs, zkSTARKs experimentation
    \item Key management - HD wallets, hardware integration
\end{itemize}

\subsubsection{Modern Development}
\begin{itemize}
    \item Full-stack TypeScript - Next.js 14+, React 18+, Node.js
    \item Systems programming - Rust, Go, Python, C/C++
    \item DevOps - Docker, Kubernetes, CI/CD pipelines
    \item Real-time systems - Event sourcing, CQRS, message queues
\end{itemize}

\subsection{Language Distribution}

\begin{table}[H]
\centering
\begin{tabular}{lll}
\toprule
\textbf{Tier 1 (Core)} & \textbf{Tier 2 (Infrastructure)} & \textbf{Tier 3 (Specialized)} \\
\midrule
Python & SQL & Solidity \\
TypeScript & Bash/Shell & C/C++ \\
JavaScript & YAML/TOML & Protobuf \\
Rust & Dockerfile & GraphQL \\
Go & Makefile & Move \\
\bottomrule
\end{tabular}
\caption{Language Coverage by Tier}
\end{table}

\section{Model Architectures}

\subsection{Dense Models (4B, 24B, 123B)}

The smaller models use dense transformer architectures optimized for different deployment scenarios:

\begin{table}[H]
\centering
\begin{tabular}{llll}
\toprule
\textbf{Model} & \textbf{Base Architecture} & \textbf{Layers} & \textbf{Hidden Dim} \\
\midrule
Zen Coder 4B & Zen-4B-Instruct & 40 & 2560 \\
Zen Coder 24B & Zen Coder 24B-Instruct & 56 & 5120 \\
Zen Coder 123B & Zen Coder 123B & 80 & 8192 \\
\bottomrule
\end{tabular}
\caption{Dense Model Specifications}
\end{table}

\subsection{MoE Models (358B, 1T)}

The larger models employ Mixture of Experts architectures for efficient scaling:

\begin{table}[H]
\centering
\begin{tabular}{llll}
\toprule
\textbf{Model} & \textbf{Total Params} & \textbf{Active Params} & \textbf{Experts} \\
\midrule
Zen Coder Max & 358B & ~60B & 128 experts, top-8 \\
Zen Coder Ultra & 1T & ~150B & 256 experts, top-8 \\
\bottomrule
\end{tabular}
\caption{MoE Model Specifications}
\end{table}

\section{Training Methodology}

\subsection{Training Framework}

We developed \href{https://github.com/zenlm/zen-trainer}{zen-trainer}, an open-source framework
supporting multiple backends:

\begin{itemize}
    \item \textbf{MLX}: Apple Silicon optimization (M1/M2/M3 Pro/Max/Ultra)
    \item \textbf{Unsloth}: 2x faster NVIDIA training with memory optimization
    \item \textbf{DeepSpeed}: Multi-GPU and multi-node training
\end{itemize}

\subsection{Fine-tuning Configuration}

\begin{table}[H]
\centering
\begin{tabular}{lllll}
\toprule
\textbf{Model} & \textbf{LoRA r} & \textbf{LoRA $\alpha$} & \textbf{Batch} & \textbf{LR} \\
\midrule
4B & 64 & 128 & 4 & 2e-4 \\
24B & 32 & 64 & 2 & 1e-4 \\
123B & 16 & 32 & 1 & 5e-5 \\
Max & 16 & 32 & 1 & 5e-6 \\
Ultra & 8 & 16 & 1 & 1e-6 \\
\bottomrule
\end{tabular}
\caption{QLoRA Fine-tuning Hyperparameters}
\end{table}

\subsection{Training Costs}

For 3.35M samples (8.47B tokens) on 8xH200 @ \$35/hr:

\begin{table}[H]
\centering
\begin{tabular}{llll}
\toprule
\textbf{Model} & \textbf{Cloud Hours} & \textbf{Cloud Cost} & \textbf{Local (Mac Studio)} \\
\midrule
Zen Coder 4B & 9h & \$326 & 2 days (FREE) \\
Zen Coder 24B & 23h & \$814 & 5 days (FREE) \\
Zen Coder 123B & 62h & \$2,171 & 13 days (FREE) \\
Zen Coder Max & 116h & \$4,071 & 19 days (FREE) \\
Zen Coder Ultra & 310h & \$10,856 & N/A (too large) \\
\bottomrule
\end{tabular}
\caption{Training Cost Estimates}
\end{table}

\section{Usage}

\subsection{Installation}

\begin{lstlisting}[language=bash, caption=Install zen-trainer]
pip install zen-trainer
\end{lstlisting}

\subsection{Training Example}

\begin{lstlisting}[language=Python, caption=Fine-tuning with zen-trainer]
from zen_trainer import ZenTrainer

trainer = ZenTrainer(
    model_key="zen-coder-4b",
    dataset_path="hanzoai/zen-agentic-dataset-private",
    output_dir="./output/zen-coder-4b",
)
trainer.train()
\end{lstlisting}

\subsection{Inference Example}

\begin{lstlisting}[language=Python, caption=Using Zen Coder for inference]
from transformers import AutoModelForCausalLM, AutoTokenizer

model = AutoModelForCausalLM.from_pretrained(
    "zenlm/zen-coder-4b",
    torch_dtype="auto",
    device_map="auto"
)
tokenizer = AutoTokenizer.from_pretrained("zenlm/zen-coder-4b")

messages = [
    {"role": "user", "content": "Write a Python function to validate email addresses"}
]

text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
inputs = tokenizer(text, return_tensors="pt").to(model.device)
outputs = model.generate(**inputs, max_new_tokens=512)
print(tokenizer.decode(outputs[0], skip_special_tokens=True))
\end{lstlisting}

\section{What Makes This Unique}

\subsection{Real Agentic Programming}

Unlike synthetic datasets, the Zen Agentic Dataset contains \textbf{actual Claude Code sessions} showing:
\begin{itemize}
    \item Real debugging workflows with trial and error
    \item Complex multi-file refactoring decisions
    \item Architecture discussions and trade-offs
    \item Tool use patterns (file ops, search, git, tests)
    \item Error recovery and iterative refinement
\end{itemize}

\subsection{Production Code Quality}
\begin{itemize}
    \item Code that shipped to production systems
    \item Security-audited smart contracts
    \item Performance-optimized infrastructure
    \item Battle-tested patterns from real deployments
\end{itemize}

\section{Evaluation}

Models are evaluated on agentic coding benchmarks including:
\begin{itemize}
    \item \textbf{SWE-bench Verified}: Real GitHub issue resolution
    \item \textbf{TAU-Bench}: Tool-agent-user interaction
    \item \textbf{BFCL V3}: Berkeley Function Call Leaderboard
    \item \textbf{Terminal-Bench}: Terminal environment tasks
    \item \textbf{LiveCodeBench}: Real-time coding challenges
\end{itemize}

\section{Licensing \& Access}

\subsection{Models}
\begin{itemize}
    \item Zen Coder 4B, 24B: Apache 2.0
    \item Zen Coder 123B: Zen Commercial License v1.0
    \item Zen Coder Max: Zen Commercial License v1.0
    \item Zen Coder Ultra: Zen Commercial License v1.0
\end{itemize}

\subsection{Dataset Access}
The Zen Agentic Dataset is available for research and commercial licensing.
Contact \texttt{[email protected]} for access.

\section{Supported Organizations}

\begin{table}[H]
\centering
\begin{tabular}{lll}
\toprule
\textbf{Organization} & \textbf{Focus} & \textbf{Role} \\
\midrule
Hanzo AI & AI infrastructure & Primary maintainer \\
Zen LM & Open model research & Model training \\
Zoo Labs & Decentralized AI & Research grants \\
Lux Network & AI compute settlement & Infrastructure \\
\bottomrule
\end{tabular}
\caption{Supporting Organizations}
\end{table}

\section{Conclusion}

Zen Coder represents a new approach to code generation model training: using real agentic programming
data rather than synthetic instructions. By training on actual Claude Code sessions from 15 years of
production software development, these models learn genuine debugging patterns, tool use workflows,
and the iterative nature of real programming.

The complete model family---from 4B for edge deployment to 1T for frontier capabilities---provides
options for every use case. The open training framework (\href{https://github.com/zenlm/zen-trainer}{zen-trainer})
enables the community to train custom models on their own data.

\section*{Links}

\begin{itemize}
    \item Models: \href{https://huggingface.co/zenlm}{huggingface.co/zenlm}
    \item Dataset: \href{https://huggingface.co/datasets/hanzoai/zen-agentic-dataset}{huggingface.co/datasets/hanzoai/zen-agentic-dataset}
    \item Training: \href{https://github.com/zenlm/zen-trainer}{github.com/zenlm/zen-trainer}
    \item Website: \href{https://zenlm.org}{zenlm.org}
    \item Contact: \href{mailto:[email protected]}{[email protected]}
\end{itemize}

\begin{thebibliography}{99}

\bibitem{roziere2024codellama}
B.~Rozi\`ere, J.~Gehring, F.~Gloeckle, S.~Sootla, I.~Gat, X.~E.~Tan,
Y.~Adi, J.~Liu, R.~Sauvestre, T.~Remez, et~al.
\newblock Code {Llama}: Open foundation models for code.
\newblock \emph{arXiv preprint arXiv:2308.12950}, 2024.

\bibitem{li2023starcoder}
R.~Li, L.~B.~Allal, Y.~Zi, N.~Muennighoff, D.~Kocetkov, C.~Mou,
M.~Marone, C.~Akiki, J.~Li, J.~Chim, et~al.
\newblock {StarCoder}: May the source be with you!
\newblock \emph{arXiv preprint arXiv:2305.06161}, 2023.

\bibitem{guo2024deepseek}
D.~Guo, Q.~Zhu, D.~Yang, Z.~Xie, K.~Dong, W.~Zhang, G.~Chen, X.~Bi,
Y.~Wu, Y.~K.~Li, et~al.
\newblock {DeepSeek-Coder}: When the large language model meets programming.
\newblock \emph{arXiv preprint arXiv:2401.14196}, 2024.

\bibitem{chen2021humaneval}
M.~Chen, J.~Tworek, H.~Jun, Q.~Yuan, H.~Pinto de~Oliveira, J.~Kaplan,
H.~Edwards, Y.~Burda, N.~Joseph, G.~Brockman, et~al.
\newblock Evaluating large language models trained on code.
\newblock \emph{arXiv preprint arXiv:2107.03374}, 2021.

\bibitem{jimenez2024swebench}
C.~E.~Jimenez, J.~Yang, A.~Wettig, S.~Yao, K.~Pei, O.~Press, and
K.~Narasimhan.
\newblock {SWE-bench}: Can language models resolve real-world {GitHub} issues?
\newblock In \emph{ICLR}, 2024.

\bibitem{hu2022lora}
E.~J.~Hu, Y.~Shen, P.~Wallis, Z.~Allen-Zhu, Y.~Li, S.~Wang, L.~Wang, and
W.~Chen.
\newblock {LoRA}: Low-rank adaptation of large language models.
\newblock In \emph{ICLR}, 2022.

\bibitem{shazeer2017moe}
N.~Shazeer, A.~Mirhoseini, K.~Maziarz, A.~Davis, Q.~Le, G.~Hinton, and
J.~Dean.
\newblock Outrageously large neural networks: The sparsely-gated
mixture-of-experts layer.
\newblock In \emph{ICLR}, 2017.

\bibitem{vaswani2017attention}
A.~Vaswani, N.~Shazeer, N.~Parmar, J.~Uszkoreit, L.~Jones, A.~N.~Gomez,
\L.~Kaiser, and I.~Polosukhin.
\newblock Attention is all you need.
\newblock In \emph{NeurIPS}, 2017.

\end{thebibliography}

\end{document}