papers/hanzo-jin/sections/04-training.tex

\section{Training}


\subsection{Pre-training Objectives}

\paragraph{Within-Modality JEPA.} Standard masked prediction within each modality.

\paragraph{Cross-Modal JEPA.} Predict masked regions in one modality given context from another:

\begin{equation}
    \mathcal{L}_{\text{cross-modal}} = \mathbb{E}_{x_v, x_a} \left[ \|g(f(x_v)) - f(x_a^{\text{mask}})\|^2 \right]
\end{equation}

\paragraph{Contrastive Alignment.} InfoNCE loss aligns corresponding cross-modal pairs:

\begin{equation}
    \mathcal{L}_{\text{align}} = -\log \frac{\exp(z_v \cdot z_a / \tau)}{\sum_j \exp(z_v \cdot z_a^j / \tau)}
\end{equation}

\paragraph{Total Loss.}
\begin{equation}
    \mathcal{L} = \alpha \mathcal{L}_{\text{JEPA}} + \beta \mathcal{L}_{\text{cross-modal}} + \gamma \mathcal{L}_{\text{align}}
\end{equation}

\subsection{Curriculum Learning}

Training proceeds in stages:
\begin{enumerate}
    \item Single-modality JEPA (epochs 1-50)
    \item Cross-modal prediction (epochs 51-100)
    \item Full multimodal with z-JEPA (epochs 101-150)
\end{enumerate}

\subsection{Scaling}

\begin{table}[h]
\centering
\caption{Jin model configurations.}
\label{tab:scale}
\begin{tabular}{lrrr}
\toprule
\textbf{Model} & \textbf{Params} & \textbf{Layers} & \textbf{Experts} \\
\midrule
Jin-Nano & 1B & 6 & 8 \\
Jin-Base & 7B & 12 & 16 \\
Jin-Large & 70B & 24 & 64 \\
\bottomrule
\end{tabular}
\end{table}