papers/zoo-hamiltonian-llm/zoo-hamiltonian-llm.tex

\documentclass[11pt,letterpaper]{article}
\usepackage{shared/zoocover}

% Packages
\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{amsmath,amssymb,amsthm}
\usepackage{algorithm}
\usepackage{algorithmic}
\usepackage{graphicx}
\usepackage{hyperref}
\usepackage{xcolor}
\usepackage{booktabs}
\usepackage{tikz}
\usetikzlibrary{shapes,arrows,positioning,calc}
\usepackage{listings}
\input{shared/lstlang}
\usepackage{caption}
\usepackage{subcaption}
% geometry loaded by zoocover

% Theorem environments
\newtheorem{theorem}{Theorem}[section]
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{definition}{Definition}[section]
\newtheorem{example}{Example}[section]
\newtheorem{remark}{Remark}[section]

% Custom commands
\newcommand{\Zoo}{\textsc{Zoo}}
\newcommand{\NFT}{\textsc{nft}}
\newcommand{\ERC}{\textsc{erc}}
\newcommand{\LLM}{\textsc{llm}}
\newcommand{\DAO}{\textsc{dao}}
\newcommand{\HLLM}{\textsc{hllm}}
\newcommand{\R}{\mathbb{R}}
\newcommand{\E}{\mathbb{E}}
\newcommand{\KL}{\mathrm{KL}}

% Hyperref setup
\hypersetup{
    colorlinks=true,
    linkcolor=blue,
    citecolor=blue,
    urlcolor=blue
}

% Title and authors
\title{\textbf{Hamiltonian Large Language Models:\\Energy-Based Inference for Zoo AI}}

\author{
Antje Worring, Zoo Labs Foundation Research\thanks{[email protected]}\\
\textit{Hanzo Industries \quad Lux Industries \quad Zoo Labs Foundation}\\
\texttt{[email protected]}
}

\date{December 2024}

\begin{document}

\zoocoverpage

\begin{abstract}
We introduce \emph{Hamiltonian Large Language Models} (HLLMs), a reformulation of autoregressive language modeling through the lens of Hamiltonian mechanics. Where standard transformers define next-token prediction as a chain of conditional softmax distributions, HLLMs introduce a conserved energy function $H(q, p)$ over token embeddings $q$ and latent momenta $p$ that governs inference dynamics. We show that the energy conservation property of Hamiltonian flow---$|H(q_t, p_t) - H(q_0, p_0)| \leq C\epsilon^2$ under leapfrog integration---provides concentration bounds on multi-step inference that are strictly tighter than those available from attention weight norms alone. We further develop a Hidden Markov gating mechanism that leverages Hamiltonian dual variables for stable MoE expert routing, and a group sequence optimization procedure for training. Experiments on long-context reasoning benchmarks show that HLLMs maintain coherence at 128K tokens where standard transformers degrade, while achieving comparable perplexity on standard language modeling tasks. HLLMs serve as the reasoning backbone of Zoo's Eco-1 architecture.

\textbf{Keywords}: Hamiltonian mechanics, energy-based models, language models, long-context reasoning, mixture of experts
\end{abstract}

\section{Introduction}

Autoregressive language models generate text by sampling from $p(x_t \mid x_{<t})$ at each step. This formulation is memoryless in a specific sense: the model has no explicit dynamical constraint preventing the latent state from drifting arbitrarily far from its initial configuration over many steps. Attention mechanisms provide long-range connections, but the attention weight matrix $A \in \R^{T \times T}$ has spectral norm that grows with sequence length, and no conservation law prevents the latent representation from accumulating errors.

We propose a complementary view. Rather than treating inference as sequential sampling, we model it as a dynamical system with a conserved quantity. The Hamiltonian $H(q, p)$ defines the total energy of the system, where $q$ represents token embeddings (position) and $p$ represents latent momenta that encode the direction and magnitude of the model's reasoning trajectory. Hamilton's equations constrain the evolution:

\begin{equation}
\frac{dq}{dt} = \frac{\partial H}{\partial p}, \quad \frac{dp}{dt} = -\frac{\partial H}{\partial q}
\label{eq:hamilton}
\end{equation}

The fundamental property is that $H$ is constant along trajectories of this system. Discretization via the leapfrog integrator introduces bounded error $O(\epsilon^2)$ per step, giving us a formal guarantee on how far the model's latent state can drift during long inference chains.

\subsection{Contributions}

\begin{enumerate}
    \item A formulation of language model inference as Hamiltonian dynamics with provable energy conservation bounds (Section~\ref{sec:formulation}).
    \item A Hidden Markov gating mechanism for MoE routing derived from Hamiltonian dual variables (Section~\ref{sec:moe}).
    \item A group sequence optimization training procedure that stabilizes expert selection via policy-level rewards (Section~\ref{sec:training}).
    \item Experimental validation showing tighter coherence at 128K context with comparable perplexity to standard transformers (Section~\ref{sec:experiments}).
\end{enumerate}

\section{Background}

\subsection{Hamiltonian Mechanics}

A Hamiltonian system is defined by a smooth function $H: \R^{2d} \to \R$ over phase space $(q, p) \in \R^d \times \R^d$. The time evolution is governed by Equation~\ref{eq:hamilton}. The flow $\phi_t: (q_0, p_0) \mapsto (q_t, p_t)$ is \emph{symplectic}: it preserves the symplectic form $\omega = \sum_i dq_i \wedge dp_i$, and consequently preserves phase-space volume (Liouville's theorem) and the Hamiltonian value.

\begin{theorem}[Energy Conservation~\cite{leimkuhler2004simulating}]
\label{thm:conservation}
For the leapfrog integrator with step size $\epsilon$ applied to a Hamiltonian $H \in C^3$:
\[
|H(q_n, p_n) - H(q_0, p_0)| \leq C \epsilon^2
\]
for all $n$ up to exponentially long times $T = O(e^{c/\epsilon})$, where $C$ depends on the third derivatives of $H$.
\end{theorem}

\subsection{Energy-Based Models}

An energy-based model (EBM)~\cite{lecun2006tutorial} defines a probability distribution via $p(x) = \frac{1}{Z} \exp(-E(x))$ where $E: \R^d \to \R$ is the energy function and $Z = \int \exp(-E(x)) dx$ is the partition function. Inference amounts to finding low-energy configurations. Boltzmann machines~\cite{hinton2006reducing}, restricted Boltzmann machines, and score-based diffusion models~\cite{song2021score} are instances of this framework.

\subsection{Transformer Attention}

Standard multi-head attention computes:
\[
\text{Attn}(Q, K, V) = \text{softmax}\!\left(\frac{QK^\top}{\sqrt{d_k}}\right) V
\]
The attention matrix $A = \text{softmax}(QK^\top / \sqrt{d_k})$ is row-stochastic but has no conservation law constraining its evolution across layers.

\section{HLLM Formulation}
\label{sec:formulation}

\subsection{Phase Space for Token Sequences}

Given a sequence of $T$ tokens with embeddings $x_1, \ldots, x_T \in \R^d$, we define the phase space state as:

\begin{definition}[HLLM Phase Space]
The position vector $q = (x_1, \ldots, x_T) \in \R^{Td}$ concatenates all token embeddings. The momentum vector $p \in \R^{Td}$ is initialized from a learned prior $p \sim \mathcal{N}(\mu_\theta(q), \Sigma_\theta(q))$ conditioned on the input.
\end{definition}

The Hamiltonian decomposes as:

\begin{equation}
H(q, p) = U(q) + K(p)
\label{eq:hllm_hamiltonian}
\end{equation}

where the potential energy $U(q)$ is the negative log-likelihood of the token sequence under a learned model:

\begin{equation}
U(q) = -\sum_{t=1}^{T} \log p_\theta(x_t \mid x_{<t})
\label{eq:potential}
\end{equation}

and the kinetic energy $K(p)$ is:

\begin{equation}
K(p) = \frac{1}{2} p^\top M^{-1} p
\label{eq:kinetic}
\end{equation}

where $M$ is a learned mass matrix (positive definite, parameterized as $M = L L^\top + \sigma^2 I$).

\subsection{Leapfrog Inference}

Inference proceeds by integrating Hamilton's equations via the leapfrog (Stormer-Verlet) method:

\begin{algorithm}[t]
\caption{HLLM Leapfrog Inference}
\label{alg:leapfrog}
\begin{algorithmic}[1]
\STATE \textbf{Input}: Initial embeddings $q_0$, step size $\epsilon$, steps $L$
\STATE \textbf{Output}: Refined embeddings $q_L$
\STATE Sample $p_0 \sim \mathcal{N}(\mu_\theta(q_0), \Sigma_\theta(q_0))$
\STATE $p_{1/2} \leftarrow p_0 - \frac{\epsilon}{2} \nabla_q U(q_0)$
\FOR{$\ell = 1$ to $L$}
    \STATE $q_\ell \leftarrow q_{\ell-1} + \epsilon \, M^{-1} p_{\ell-1/2}$
    \IF{$\ell < L$}
        \STATE $p_{\ell+1/2} \leftarrow p_{\ell-1/2} - \epsilon \, \nabla_q U(q_\ell)$
    \ELSE
        \STATE $p_L \leftarrow p_{L-1/2} - \frac{\epsilon}{2} \nabla_q U(q_L)$
    \ENDIF
\ENDFOR
\RETURN $q_L$
\end{algorithmic}
\end{algorithm}

\begin{theorem}[HLLM Inference Concentration]
\label{thm:concentration}
Let $(q_0, p_0)$ be the initial phase-space state and $(q_L, p_L)$ the state after $L$ leapfrog steps with step size $\epsilon$. If $U \in C^3$ with $\|\nabla^3 U\| \leq B$, then:
\[
\|q_L - q_L^*\|_2 \leq \frac{BL\epsilon^2}{2}\|M^{-1}\|_{\text{op}}
\]
where $q_L^*$ is the exact solution. Consequently, the token-level prediction error is bounded by a quantity independent of sequence length $T$ (depending only on $L$ and $\epsilon$).
\end{theorem}

\begin{proof}
The leapfrog integrator is a second-order symplectic method. The local truncation error per step is $O(\epsilon^3)$ for both $q$ and $p$. Over $L$ steps, the global error in $q$ is $O(L\epsilon^2)$ by standard ODE error analysis. The factor $\|M^{-1}\|_{\text{op}}$ enters from the momentum-to-position coupling $\dot{q} = M^{-1}p$. Crucially, symplecticity prevents error growth beyond this linear bound---unlike generic Runge-Kutta methods where errors can grow exponentially.
\end{proof}

\subsection{Comparison with Attention}

In a standard $L$-layer transformer, the residual stream after layer $\ell$ is:
\[
h_\ell = h_{\ell-1} + \text{Attn}_\ell(h_{\ell-1}) + \text{FFN}_\ell(h_{\ell-1})
\]
No conservation law constrains $\|h_L - h_0\|$. The norm can grow as $O(L \cdot \max_\ell \|\text{Attn}_\ell + \text{FFN}_\ell\|)$, which is not bounded by any architectural invariant.

In contrast, HLLM's energy conservation ensures:
\[
|U(q_L) - U(q_0)| \leq |K(p_0) - K(p_L)| + C\epsilon^2 \leq K(p_0) + C\epsilon^2
\]

Since $K(p_0) = \frac{1}{2}p_0^\top M^{-1} p_0$ is finite and controlled by the prior, the potential energy shift---and hence the change in log-likelihood landscape---is bounded.

\section{Hamiltonian MoE Routing}
\label{sec:moe}

\subsection{Hidden Markov Gating}

Standard MoE routing uses a learned gating function $g(x) = \text{softmax}(W_g x)$ independently per token. This leads to expert oscillation: the selected expert can change abruptly between adjacent tokens. We introduce a Hidden Markov Model (HMM) over expert assignments, where the Hamiltonian's dual variables govern transitions.

\begin{definition}[Hamiltonian HMM Gating]
Let $z_t \in \{1, \ldots, K\}$ denote the expert assignment at position $t$. The transition probability is:
\[
P(z_t = k \mid z_{t-1} = j, p_t) = \frac{\exp(-\beta \|p_t - \mu_k\|^2) \cdot T_{jk}}{\sum_{k'} \exp(-\beta \|p_t - \mu_{k'}\|^2) \cdot T_{jk'}}
\]
where $T \in \R^{K \times K}_{\geq 0}$ is a learned transition matrix (row-stochastic), $\mu_k$ is the momentum centroid for expert $k$, and $\beta > 0$ is an inverse temperature.
\end{definition}

The transition matrix $T$ encourages \emph{regime persistence}: once the model enters a reasoning mode (e.g., mathematical deduction), it tends to stay there rather than oscillating.

\begin{proposition}[Regime Persistence]
Let $\lambda_2(T)$ denote the second-largest eigenvalue of $T$. If $\lambda_2(T) > 1 - \delta$ for small $\delta > 0$, then the expected regime duration is $\Omega(1/\delta)$ steps.
\end{proposition}

\subsection{Dual Variable Routing}

The momentum $p_t$ acts as a natural routing signal. In Hamiltonian mechanics, momentum encodes the ``direction'' of the system's trajectory. We use this to route tokens to experts whose specialization matches the current reasoning direction:

\begin{lstlisting}[language=Python,caption=Hamiltonian MoE routing]
def hamiltonian_moe_route(q, p, experts, T_matrix,
                          beta=1.0, top_k=2):
    # Momentum-based affinity to each expert
    affinities = []
    for k, expert in enumerate(experts):
        dist = (p - expert.momentum_centroid).norm(dim=-1)
        affinities.append(-beta * dist**2)
    affinities = torch.stack(affinities, dim=-1)

    # HMM transition prior from previous expert
    if hasattr(q, 'prev_expert'):
        transition_prior = T_matrix[q.prev_expert]
        affinities = affinities + transition_prior.log()

    # Top-k selection
    weights = F.softmax(affinities, dim=-1)
    top_vals, top_idx = weights.topk(top_k, dim=-1)
    top_vals = top_vals / top_vals.sum(dim=-1, keepdim=True)

    return top_idx, top_vals
\end{lstlisting}

\section{Training}
\label{sec:training}

\subsection{Loss Function}

The HLLM training loss combines three terms:

\begin{equation}
\mathcal{L} = \mathcal{L}_{\text{NLL}} + \lambda_H \mathcal{L}_{\text{energy}} + \lambda_g \mathcal{L}_{\text{gate}}
\label{eq:loss}
\end{equation}

\begin{itemize}
    \item $\mathcal{L}_{\text{NLL}} = -\sum_t \log p_\theta(x_t \mid x_{<t})$: standard next-token prediction.
    \item $\mathcal{L}_{\text{energy}} = \frac{1}{L}\sum_{\ell=1}^{L} (H(q_\ell, p_\ell) - H(q_0, p_0))^2$: penalizes energy conservation violations.
    \item $\mathcal{L}_{\text{gate}} = \sum_k (\bar{g}_k - 1/K)^2$: load balance regularizer.
\end{itemize}

\subsection{Group Sequence Optimization}

Training MoE models is unstable because expert selection is discrete. We introduce \emph{group sequence optimization} (GSO): rather than optimizing per-token expert assignments, we optimize over contiguous groups of tokens (``reasoning segments'') and assign rewards at the group level.

\begin{algorithm}[t]
\caption{Group Sequence Optimization}
\label{alg:gso}
\begin{algorithmic}[1]
\STATE \textbf{Input}: Sequence $x_1, \ldots, x_T$, group size $G$
\STATE Partition into groups $S_1, \ldots, S_{\lceil T/G \rceil}$
\FOR{each group $S_i$}
    \STATE Compute expert assignments via HMM gating
    \STATE Forward pass through selected experts
    \STATE Compute group reward $r_i = -\mathcal{L}_{\text{NLL}}(S_i) - \lambda_H \mathcal{L}_{\text{energy}}(S_i)$
\ENDFOR
\STATE Update gating parameters via REINFORCE with group rewards
\STATE Update expert parameters via standard backpropagation
\end{algorithmic}
\end{algorithm}

\begin{theorem}[GSO Variance Reduction]
The variance of the REINFORCE gradient estimator under GSO with group size $G$ is $O(1/G)$ times the variance of per-token REINFORCE, assuming bounded rewards.
\end{theorem}

\begin{proof}
Each group aggregates $G$ individual token rewards. By the law of total variance and the bounded reward assumption $|r_t| \leq R$:
\[
\text{Var}\!\left[\sum_{t \in S_i} r_t \nabla_\phi \log \pi_\phi(z_t)\right] \leq G \cdot R^2 \cdot \text{Var}[\nabla_\phi \log \pi_\phi(z_t)]
\]
The group-level estimator uses $\bar{r}_i = \frac{1}{G}\sum_{t \in S_i} r_t$, reducing the reward variance by $1/G$ while the gradient direction is shared across the group.
\end{proof}

\section{Architecture Details}

\subsection{Model Configurations}

\begin{table}[t]
\centering
\begin{tabular}{lccc}
\toprule
\textbf{Parameter} & \textbf{HLLM-7B} & \textbf{HLLM-32B} & \textbf{HLLM-72B} \\
\midrule
Hidden size & 4096 & 5120 & 8192 \\
Layers & 32 & 64 & 80 \\
Attention heads & 32 & 40 & 64 \\
KV heads (GQA) & 8 & 8 & 8 \\
Experts & 8 & 8 & 16 \\
Active experts & 2 & 2 & 4 \\
Leapfrog steps & 4 & 6 & 8 \\
Step size $\epsilon$ & 0.01 & 0.01 & 0.005 \\
Context length & 32K & 128K & 128K \\
\bottomrule
\end{tabular}
\caption{HLLM model configurations.}
\label{tab:configs}
\end{table}

\subsection{Mass Matrix Parameterization}

The mass matrix $M$ controls how momentum translates to position updates. We parameterize it as a block-diagonal matrix with per-layer blocks:

\begin{equation}
M = \text{diag}(M_1, M_2, \ldots, M_L), \quad M_\ell = L_\ell L_\ell^\top + \sigma^2 I
\end{equation}

where $L_\ell \in \R^{d \times r}$ with $r \ll d$ is a low-rank factor, and $\sigma^2$ is a scalar ensuring positive definiteness. This allows different layers to have different ``inertia,'' controlling how quickly the representation changes in each subspace.

\section{Experiments}
\label{sec:experiments}

\subsection{Long-Context Reasoning}

We evaluate on the RULER benchmark~\cite{hsieh2024ruler} at context lengths from 4K to 128K tokens.

\begin{table}[t]
\centering
\begin{tabular}{lcccc}
\toprule
\textbf{Model} & \textbf{4K} & \textbf{16K} & \textbf{64K} & \textbf{128K} \\
\midrule
Transformer-32B & 94.2 & 91.7 & 83.1 & 71.4 \\
Mamba-32B & 93.8 & 91.2 & 86.4 & 79.3 \\
HLLM-32B & 93.9 & 92.1 & 89.7 & 86.2 \\
\bottomrule
\end{tabular}
\caption{RULER accuracy (\%) at various context lengths.}
\label{tab:ruler}
\end{table}

The key observation: at 128K tokens, HLLM-32B retains 86.2\% accuracy versus the transformer's 71.4\%. The energy conservation bound (Theorem~\ref{thm:concentration}) manifests as slower degradation with length.

\subsection{Energy Conservation Verification}

We measure the empirical energy drift $|H(q_L, p_L) - H(q_0, p_0)|$ across inference:

\begin{table}[t]
\centering
\begin{tabular}{lccc}
\toprule
\textbf{Context Length} & \textbf{Mean Drift} & \textbf{Max Drift} & \textbf{Theoretical Bound} \\
\midrule
4K & 0.0021 & 0.0089 & 0.0100 \\
16K & 0.0024 & 0.0093 & 0.0100 \\
64K & 0.0028 & 0.0097 & 0.0100 \\
128K & 0.0031 & 0.0099 & 0.0100 \\
\bottomrule
\end{tabular}
\caption{Energy drift with $\epsilon = 0.01$. Drift grows sublinearly with context length, consistent with $O(\epsilon^2)$ bound.}
\label{tab:drift}
\end{table}

\subsection{Language Modeling Perplexity}

On standard benchmarks, HLLM achieves comparable perplexity:

\begin{table}[t]
\centering
\begin{tabular}{lcc}
\toprule
\textbf{Model} & \textbf{WikiText-103} & \textbf{C4} \\
\midrule
Transformer-32B & 5.12 & 7.84 \\
HLLM-32B & 5.28 & 7.91 \\
\bottomrule
\end{tabular}
\caption{Perplexity on language modeling benchmarks.}
\label{tab:perplexity}
\end{table}

The 3\% perplexity increase is the cost of the Hamiltonian constraint; the benefit is the long-context stability shown in Table~\ref{tab:ruler}.

\subsection{MoE Routing Stability}

We compare expert assignment entropy (lower = more stable routing) across methods:

\begin{table}[t]
\centering
\begin{tabular}{lcc}
\toprule
\textbf{Routing Method} & \textbf{Assignment Entropy} & \textbf{Regime Duration} \\
\midrule
Softmax top-2 & 2.41 & 3.2 tokens \\
Hash routing & 2.08 & 8.1 tokens \\
HMM gating (ours) & 1.73 & 14.6 tokens \\
\bottomrule
\end{tabular}
\caption{Expert routing stability metrics.}
\label{tab:routing}
\end{table}

\section{Integration with Zoo Architecture}

HLLM serves as the reasoning backbone of Eco-1 (ZIP-0003). The potential energy $U(q)$ is computed by the language modeling head, while the Z-JEPA cross-modal encoder contributes terms to $U$ from video, audio, and 3D inputs. The momentum $p$ is shared across modalities, ensuring that the conservation law applies to the full multimodal inference chain.

BitDelta personalization (ZIP-0007) modifies the potential energy:
\begin{equation}
U_{\text{user}}(q) = U(q; W_b + \gamma \cdot \text{sign}(\delta_u))
\end{equation}
Since the personalized weights differ from the base by a bounded perturbation, the energy conservation bound (Theorem~\ref{thm:conservation}) still holds with a modified constant $C' = C + O(\gamma)$.

\section{Related Work}

\textbf{Hamiltonian neural networks.} Greydanus et al.~\cite{greydanus2019hamiltonian} learn Hamiltonian dynamics from data but apply the framework to physical systems, not language. Toth et al.~\cite{toth2020hamiltonian} extend this to generative models. Our work applies Hamiltonian structure to the inference process of an LLM.

\textbf{Energy-based language models.} Residual energy-based models~\cite{bakhtin2021residual} use EBMs for text generation but do not enforce conservation laws. Our approach differs in that the Hamiltonian provides an explicit conserved quantity during inference.

\textbf{MoE routing.} Mixtral~\cite{jiang2024mixtral} uses top-2 softmax gating. Switch Transformer~\cite{fedus2022switch} uses top-1. Expert Choice~\cite{zhou2022mixture} lets experts choose tokens. None use HMM-based gating with conservation-derived routing signals.

\textbf{Long-context transformers.} Approaches include sparse attention~\cite{child2019generating}, linear attention~\cite{katharopoulos2020transformers}, and state-space models~\cite{gu2024mamba}. These modify the attention mechanism; HLLMs address the problem at the dynamical-system level.

\section{Conclusion}

HLLMs introduce a physically motivated inductive bias---energy conservation---into language model inference. The formal guarantee of Theorem~\ref{thm:concentration} translates to measurable improvements in long-context coherence (86.2\% vs.\ 71.4\% at 128K tokens) at a modest cost in short-context perplexity (3\% increase). The Hidden Markov gating mechanism derived from Hamiltonian dual variables produces stable expert routing with 4.6$\times$ longer regime durations than standard softmax gating. Group sequence optimization reduces REINFORCE gradient variance by $O(1/G)$, enabling stable MoE training. As the reasoning backbone of Zoo's Eco-1 architecture, HLLMs provide the stable inference foundation on which multimodal perception, personalization, and active inference planning are built.

\section*{Acknowledgments}

This work was supported by Zoo Labs Foundation. We thank the Zen modeling team for base model infrastructure and the Hanzo compute team for GPU allocation.

\bibliographystyle{plain}
\begin{thebibliography}{35}

\bibitem{leimkuhler2004simulating}
B.~Leimkuhler and S.~Reich, \emph{Simulating Hamiltonian Dynamics}. Cambridge University Press, 2004.

\bibitem{neal2011mcmc}
R.~M.~Neal, ``MCMC Using Hamiltonian Dynamics,'' in \emph{Handbook of Markov Chain Monte Carlo}, Chapman \& Hall, 2011.

\bibitem{lecun2006tutorial}
Y.~LeCun, S.~Chopra, R.~Hadsell, M.~Ranzato, and F.~J.~Huang, ``A Tutorial on Energy-Based Learning,'' in \emph{Predicting Structured Data}, MIT Press, 2006.

\bibitem{hinton2006reducing}
G.~E.~Hinton and R.~R.~Salakhutdinov, ``Reducing the Dimensionality of Data with Neural Networks,'' \emph{Science}, vol.~313, pp.~504--507, 2006.

\bibitem{song2021score}
Y.~Song, J.~Sohl-Dickstein, D.~P.~Kingma, A.~Kumar, S.~Ermon, and B.~Poole, ``Score-Based Generative Modeling through Stochastic Differential Equations,'' in \emph{Proc.\ ICLR}, 2021.

\bibitem{vaswani2017attention}
A.~Vaswani \emph{et al.}, ``Attention Is All You Need,'' in \emph{Proc.\ NeurIPS}, 2017.

\bibitem{greydanus2019hamiltonian}
S.~Greydanus, M.~Dzamba, and J.~Sprague, ``Hamiltonian Neural Networks,'' in \emph{Proc.\ NeurIPS}, 2019.

\bibitem{toth2020hamiltonian}
P.~Toth, D.~J.~Rezende, A.~Jaegle, S.~Racaniere, A.~Botev, and I.~Higgins, ``Hamiltonian Generative Networks,'' in \emph{Proc.\ ICLR}, 2020.

\bibitem{bakhtin2021residual}
A.~Bakhtin, Y.~Deng, S.~Gross, M.~Ott, M.~A.~Ranzato, and A.~Szlam, ``Residual Energy-Based Models for Text,'' \emph{J.~Machine Learning Research}, vol.~22, no.~40, pp.~1--41, 2021.

\bibitem{jiang2024mixtral}
A.~Q.~Jiang \emph{et al.}, ``Mixtral of Experts,'' \emph{arXiv:2401.04088}, 2024.

\bibitem{fedus2022switch}
W.~Fedus, B.~Zoph, and N.~Shazeer, ``Switch Transformers: Scaling to Trillion Parameter Models with Simple and Efficient Sparsity,'' \emph{J.~Machine Learning Research}, vol.~23, no.~120, pp.~1--39, 2022.

\bibitem{zhou2022mixture}
Y.~Zhou, T.~Lei, H.~Liu, N.~Du, Y.~Huang, V.~Zhao, A.~Dai, Z.~Chen, Q.~Le, and J.~Laudon, ``Mixture-of-Experts with Expert Choice Routing,'' in \emph{Proc.\ NeurIPS}, 2022.

\bibitem{child2019generating}
R.~Child, S.~Gray, A.~Radford, and I.~Sutskever, ``Generating Long Sequences with Sparse Transformers,'' \emph{arXiv:1904.10509}, 2019.

\bibitem{katharopoulos2020transformers}
A.~Katharopoulos, A.~Vyas, N.~Pappas, and F.~Fleuret, ``Transformers Are RNNs: Fast Autoregressive Transformers with Linear Attention,'' in \emph{Proc.\ ICML}, 2020.

\bibitem{gu2024mamba}
A.~Gu and T.~Dao, ``Mamba: Linear-Time Sequence Modeling with Selective State Spaces,'' \emph{arXiv:2312.00752}, 2024.

\bibitem{hsieh2024ruler}
C.-P.~Hsieh \emph{et al.}, ``RULER: What's the Real Context Size of Your Long-Context Language Models?'' \emph{arXiv:2404.06654}, 2024.

\bibitem{touvron2023llama}
H.~Touvron \emph{et al.}, ``LLaMA: Open and Efficient Foundation Language Models,'' \emph{arXiv:2302.13971}, 2023.

\bibitem{liu2024bitdelta}
J.~Liu, N.~Dedhia, S.~Wu, H.~Hu, and C.~De~Sa, ``BitDelta: Your Fine-Tune May Only Be Worth One Bit,'' in \emph{Proc.\ NeurIPS}, 2024.

\bibitem{friston2010free}
K.~Friston, ``The Free-Energy Principle: A Unified Brain Theory?'' \emph{Nature Reviews Neuroscience}, vol.~11, no.~2, pp.~127--138, 2010.

\bibitem{schmidhuber2015deep}
J.~Schmidhuber, ``Deep Learning in Neural Networks: An Overview,'' \emph{Neural Networks}, vol.~61, pp.~85--117, 2015.

\bibitem{lecun2022path}
Y.~LeCun, ``A Path Towards Autonomous Machine Intelligence,'' Open Review, version 0.9.2, 2022.

\bibitem{shazeer2017outrageously}
N.~Shazeer, A.~Mirhoseini, K.~Maziarz, A.~Davis, Q.~Le, G.~Hinton, and J.~Dean, ``Outrageously Large Neural Networks: The Sparsely-Gated Mixture-of-Experts Layer,'' in \emph{Proc.\ ICLR}, 2017.

\bibitem{baum1966statistical}
L.~E.~Baum and T.~Petrie, ``Statistical Inference for Probabilistic Functions of Finite State Markov Chains,'' \emph{The Annals of Mathematical Statistics}, vol.~37, no.~6, pp.~1554--1563, 1966.

\bibitem{betancourt2017conceptual}
M.~Betancourt, ``A Conceptual Introduction to Hamiltonian Monte Carlo,'' \emph{arXiv:1701.02434}, 2017.

\bibitem{devlin2019bert}
J.~Devlin, M.-W.~Chang, K.~Lee, and K.~Toutanova, ``BERT: Pre-Training of Deep Bidirectional Transformers for Language Understanding,'' in \emph{Proc.\ NAACL-HLT}, 2019.

\bibitem{dai2019transformer}
Z.~Dai, Z.~Yang, Y.~Yang, J.~Carbonell, Q.~Le, and R.~Salakhutdinov, ``Transformer-XL: Attentive Language Models Beyond a Fixed-Length Context,'' in \emph{Proc.\ ACL}, 2019.

\bibitem{hairer2006geometric}
E.~Hairer, C.~Lubich, and G.~Wanner, \emph{Geometric Numerical Integration: Structure-Preserving Algorithms for Ordinary Differential Equations}, 2nd ed. Springer, 2006.

\bibitem{arnol2013mathematical}
V.~I.~Arnold, \emph{Mathematical Methods of Classical Mechanics}, 2nd ed. Springer, 2013.

\end{thebibliography}

\end{document}