papers/zen/zen-video-i2v_whitepaper.tex

\documentclass[11pt,a4paper]{article}
\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{amsmath,amsfonts,amssymb}
\usepackage{graphicx}
\usepackage{hyperref}
\usepackage{listings}
\usepackage{color}
\usepackage{booktabs}
\usepackage{float}
\usepackage{geometry}
\geometry{margin=1in}
\definecolor{zenblue}{RGB}{41,121,255}
\hypersetup{colorlinks=true,linkcolor=zenblue,urlcolor=zenblue,citecolor=zenblue}

\title{\textbf{Zen-Video-I2V: Photorealistic Image-to-Video Generation\\
with Controllable Motion Dynamics}\\[0.5em]
\large Technical Whitepaper v2025.04}
\author{Hanzo AI Research, Zen LM Research Team\\
\texttt{[email protected]}\\
\href{https://papers.zenlm.org}{papers.zenlm.org}}
\date{April 2025}

\begin{document}
\maketitle

\begin{abstract}
Zen-Video-I2V is a 14 billion parameter image-to-video generation model that animates static input images into fluid video sequences up to 10 seconds at 1080p resolution, with controllable motion direction, intensity, and style. The model achieves a Fr\'{e}chet Video Distance (FVD) of 289 on UCF-101, a Warping Error of 0.023 (measuring temporal warp consistency), and 91.3\% temporal consistency on the Video Quality Assessment benchmark. Zen-Video-I2V supports both single-image animation (producing plausible dynamic motion from scene content) and image-sequence interpolation (generating smooth transitions between provided keyframes). This paper describes the model architecture, motion conditioning mechanisms, training methodology, and comprehensive video quality evaluations.
\end{abstract}

\tableofcontents
\newpage

\section{Introduction}

The transition from image generation to video generation requires solving two qualitatively different problems. Image generation learns the distribution of plausible spatial configurations. Video generation must additionally learn the distribution of plausible temporal dynamics: how objects in a scene move, deform, and interact over time while maintaining physical and visual coherence across frames.

Image-to-video generation constrains this problem by conditioning on a real input image, fixing the spatial configuration and shifting the task to learning what temporal evolutions are consistent with that fixed starting state. This constraint is practically valuable: it allows users to start from real photographs, generated images, or concept art and animate them into video without requiring full video synthesis from text prompts.

Zen-Video-I2V advances image-to-video generation through three contributions:

\begin{enumerate}
  \item \textbf{Dual-path motion conditioning}: Separate conditioning pathways for camera motion (dolly, pan, tilt, zoom, orbit) and scene motion (object-level dynamics), enabling independent control of each.
  \item \textbf{Physical plausibility constraints}: A physics-informed regularizer trained on simulation data penalizes animations that violate basic physical constraints (rigid body motion, fluid dynamics, cloth behavior).
  \item \textbf{Temporal attention at scale}: A full-resolution temporal attention mechanism (not limited to downsampled feature maps) operating across all frames simultaneously, producing high temporal coherence even in long-form (10 second) generations.
\end{enumerate}

\subsection{Model Overview}

\begin{table}[H]
\centering
\caption{Zen-Video-I2V Model Specification}
\begin{tabular}{ll}
\toprule
\textbf{Parameter} & \textbf{Value} \\
\midrule
Architecture & Diffusion Transformer (DiT) with temporal attention \\
Total Parameters & 14B \\
Output Resolution & 720p (1280$\times$720) or 1080p (1920$\times$1080) \\
Output Duration & 2--10 seconds (configurable) \\
Frame Rate & 24 fps (configurable: 8, 16, 24 fps) \\
FVD on UCF-101 & 289 \\
Warping Error & 0.023 \\
Temporal Consistency & 91.3\% \\
Version & v2025.04 \\
Release Date & April 2025 \\
\bottomrule
\end{tabular}
\end{table}

\section{Architecture}

\subsection{Diffusion Transformer with Temporal Attention}

Zen-Video-I2V is built on a Diffusion Transformer (DiT) \cite{dit} backbone extended with full-resolution temporal attention. The model operates in the latent space of a pretrained video VAE \cite{vae} that encodes video frames at 8$\times$ spatial compression with 4-frame temporal compression, reducing a 1080p 240-frame video to a 135$\times$240$\times$60-frame latent volume.

The transformer architecture alternates between spatial attention (across spatial positions within a frame) and temporal attention (across frames at fixed spatial positions):

\begin{align}
  h^{\text{spatial}} &= \text{SelfAttn}_{\text{spatial}}(h) \\
  h^{\text{temporal}} &= \text{SelfAttn}_{\text{temporal}}(h^{\text{spatial}})
\end{align}

Temporal attention is applied at full latent resolution, not downsampled, which enables the model to maintain fine-grained temporal coherence in high-frequency details (hair, fabric texture, water surface). The computational cost of full-resolution temporal attention is managed through FlashAttention-3 \cite{flashattn3} and temporal sparse attention patterns for long (10s) generations.

\subsection{Dual-Path Motion Conditioning}

Motion control is provided through two independent conditioning pathways injected via cross-attention:

\textbf{Camera Motion Pathway}: Encodes camera trajectory as a sequence of 6-DoF camera poses $\{(R_t, T_t)\}_{t=1}^F$ where $R_t \in SO(3)$ is rotation and $T_t \in \mathbb{R}^3$ is translation. The camera trajectory is encoded by a lightweight 6-layer MLP producing a 512-dimensional embedding per frame.

\textbf{Scene Motion Pathway}: Encodes object-level dynamics as an optical flow field $\{u_t\}_{t=1}^F$ where $u_t \in \mathbb{R}^{H \times W \times 2}$ is the dense flow from frame $t-1$ to frame $t$. The flow field is encoded by a small 4-layer U-Net producing a 256-dimensional embedding at the latent spatial resolution.

The two pathways are combined through learned gating:

\begin{equation}
  c_{\text{motion}} = \sigma(g) \cdot c_{\text{camera}} + (1 - \sigma(g)) \cdot c_{\text{scene}}
\end{equation}

where $g$ is a learned gating scalar conditioned on the generation context. In practice, the gate learns to balance camera and scene motion conditioning based on the input image content: outdoor scenes with clear horizon lines favor camera conditioning, while close-up portraits or product shots favor scene motion conditioning.

\subsection{Image Conditioning}

The input image is encoded by the video VAE and injected into the generation process as:

\begin{enumerate}
  \item \textbf{First-frame conditioning}: The encoded input image is fixed as the first latent frame, with the model trained to denoise subsequent frames conditioned on this fixed start state.
  \item \textbf{Global image embedding}: A CLIP image encoder extracts a semantic embedding of the input image that conditions all transformer layers via cross-attention, maintaining semantic consistency throughout the generated video.
  \item \textbf{Pixel-level conditioning}: For high-fidelity reconstruction of the input image's specific textures and details, ControlNet-style feature injection \cite{controlnet} copies early encoder features directly into the decoder path.
\end{enumerate}

\subsection{Physics-Informed Regularization}

Physical plausibility is enforced through an auxiliary physics regularizer trained on simulation data from three domains:

\begin{enumerate}
  \item \textbf{Rigid body}: Motion from a physics engine simulator covering 50K object types.
  \item \textbf{Fluid}: Water surface simulation data from a fluid dynamics solver.
  \item \textbf{Cloth and soft body}: Garment simulation covering 20K clothing styles.
\end{enumerate}

The regularizer $R_\psi$ takes a generated optical flow sequence and predicts a physics plausibility score. This score is used as an auxiliary loss during training:

\begin{equation}
  \mathcal{L}_{\text{phys}} = -\log R_\psi(u_{1:F})
\end{equation}

The physics regularizer is frozen during video model training (trained separately on simulation data only), serving as a learned prior that penalizes motions that violate physical constraints without requiring differentiable physics simulation.

\section{Training Methodology}

\subsection{Training Data}

\begin{table}[H]
\centering
\caption{Training Data Composition}
\begin{tabular}{lrr}
\toprule
\textbf{Source} & \textbf{Video Hours} & \textbf{Fraction} \\
\midrule
Licensed stock footage & 2,400,000 & 48.0\% \\
Documentary and nature video & 800,000 & 16.0\% \\
Film and television (licensed) & 600,000 & 12.0\% \\
User-generated video (consented) & 500,000 & 10.0\% \\
Physics simulation renders & 400,000 & 8.0\% \\
3D rendering and animation & 300,000 & 6.0\% \\
\midrule
\textbf{Total} & \textbf{5,000,000} & \textbf{100\%} \\
\bottomrule
\end{tabular}
\end{table}

All training data was filtered for quality (resolution $\geq$ 720p, motion blur $<$ threshold, compression artifact score $<$ threshold) and processed to remove frames with faces not covered by consent agreements.

\subsection{Training Curriculum}

Training proceeds through three stages:

\textbf{Stage 1 -- Short clip pretraining} (2 seconds, 48 frames): The model learns basic temporal coherence and motion dynamics on short clips, where the temporal attention computation is manageable at full resolution.

\textbf{Stage 2 -- Mid-length extension} (5 seconds, 120 frames): The context length is extended through curriculum continuation training, where the model is initialized from Stage 1 weights and fine-tuned on longer clips with sparse temporal attention for positions beyond 48 frames.

\textbf{Stage 3 -- Long-form fine-tuning} (10 seconds, 240 frames): Final stage fine-tuning on high-quality long clips (filtered for production quality), with the sparse temporal attention pattern expanded to cover the full temporal extent.

\subsection{Noise Schedule}

Zen-Video-I2V uses a continuous-time diffusion process with a learned noise schedule that is adaptive per-frame: the noise level at frame $t$ is conditioned on the frame index, applying higher signal-to-noise ratios for early frames (closer to the conditioning image) and allowing more variance for later frames (more uncertain motion trajectories):

\begin{equation}
  \sigma_t^{(f)} = \sigma_{\min} + \frac{f}{F}(\sigma_{\max} - \sigma_{\min}), \quad f \in \{1, \ldots, F\}
\end{equation}

This schedule encourages the model to generate temporally consistent short-range dynamics while maintaining temporal diversity over longer horizons.

\section{Evaluation}

\subsection{Video Quality Benchmarks}

\begin{table}[H]
\centering
\caption{Video Quality Benchmark Results}
\begin{tabular}{lcccc}
\toprule
\textbf{Model} & \textbf{FVD (UCF-101)} $\downarrow$ & \textbf{Warping Error} $\downarrow$ & \textbf{Temporal Consistency} $\uparrow$ & \textbf{PSNR (1st frame)} $\uparrow$ \\
\midrule
Zen-Video-I2V (14B) & \textbf{289} & \textbf{0.023} & \textbf{91.3\%} & 42.1 dB \\
Comparable I2V model A & 341 & 0.031 & 87.6\% & 40.8 dB \\
Comparable I2V model B & 387 & 0.038 & 84.2\% & 39.4 dB \\
Comparable I2V model C & 312 & 0.027 & 89.1\% & 41.3 dB \\
\bottomrule
\end{tabular}
\end{table}

\subsection{Human Evaluation}

\begin{table}[H]
\centering
\caption{Human Evaluation MOS (1--5 scale, 500 clips rated by 3 raters each)}
\begin{tabular}{lcccc}
\toprule
\textbf{Model} & \textbf{Realism} & \textbf{Temporal Coherence} & \textbf{Motion Quality} & \textbf{Overall} \\
\midrule
Zen-Video-I2V (14B) & \textbf{4.1} & \textbf{4.0} & \textbf{3.9} & \textbf{4.0} \\
Comparable model A & 3.7 & 3.5 & 3.4 & 3.5 \\
Comparable model B & 3.4 & 3.3 & 3.1 & 3.3 \\
\bottomrule
\end{tabular}
\end{table}

\subsection{Motion Control Evaluation}

We evaluate the precision of the dual-path motion control on a benchmark of 200 clips with specified camera trajectories and object motion targets:

\begin{table}[H]
\centering
\caption{Motion Control Accuracy}
\begin{tabular}{lcc}
\toprule
\textbf{Motion Type} & \textbf{Direction Accuracy} & \textbf{Intensity Correlation} \\
\midrule
Camera pan / tilt & 94.2\% & 0.91 \\
Camera dolly (zoom equivalent) & 91.7\% & 0.88 \\
Camera orbit & 88.3\% & 0.84 \\
Object translation & 87.4\% & 0.82 \\
Object deformation (cloth, fluid) & 79.1\% & 0.76 \\
\textbf{Average} & \textbf{88.1\%} & \textbf{0.84} \\
\bottomrule
\end{tabular}
\end{table}

\subsection{Resolution and Duration Scaling}

\begin{table}[H]
\centering
\caption{Quality vs. Resolution and Duration}
\begin{tabular}{lcccc}
\toprule
\textbf{Resolution} & \textbf{Duration} & \textbf{FVD} & \textbf{Temporal Consistency} & \textbf{Gen Time} \\
\midrule
720p & 2s & 241 & 93.7\% & 18s \\
720p & 5s & 267 & 92.1\% & 44s \\
720p & 10s & 289 & 91.3\% & 87s \\
1080p & 2s & 253 & 93.1\% & 31s \\
1080p & 5s & 278 & 91.4\% & 74s \\
1080p & 10s & 302 & 90.7\% & 148s \\
\bottomrule
\end{tabular}
\end{table}

Generation times measured on 4$\times$H100 80GB SXM5 (FP8, 50 DDIM steps).

\subsection{Image-Sequence Interpolation}

When provided with multiple keyframe images, Zen-Video-I2V generates smooth interpolated video transitions:

\begin{table}[H]
\centering
\caption{Keyframe Interpolation Quality}
\begin{tabular}{lccc}
\toprule
\textbf{Metric} & \textbf{Zen-Video-I2V} & \textbf{Linear Interpolation} & \textbf{Morph baseline} \\
\midrule
Perceptual quality MOS & 4.2/5 & 2.1/5 & 3.1/5 \\
Boundary frame fidelity (PSNR) & 41.3 dB & 44.2 dB & 38.7 dB \\
Intermediate plausibility MOS & 4.0/5 & 1.8/5 & 2.9/5 \\
\bottomrule
\end{tabular}
\end{table}

\subsection{Per-Domain Performance}

\begin{table}[H]
\centering
\caption{FVD by Content Domain}
\begin{tabular}{lcc}
\toprule
\textbf{Domain} & \textbf{Zen-Video-I2V FVD} & \textbf{Best Comparable FVD} \\
\midrule
Nature / landscape & 243 & 298 \\
Human portrait & 318 & 394 \\
Animal motion & 271 & 337 \\
Urban / architecture & 261 & 319 \\
Product / object & 234 & 287 \\
Abstract / artistic & 312 & 381 \\
\bottomrule
\end{tabular}
\end{table}

Human portrait is the most challenging domain due to the perceptual sensitivity to uncanny motion artifacts in faces; Zen-Video-I2V's physics-informed regularizer is less effective for biological motion than for physical objects.

\section{Deployment}

\subsection{Inference Requirements}

\begin{table}[H]
\centering
\caption{Inference Hardware Requirements}
\begin{tabular}{llll}
\toprule
\textbf{Config} & \textbf{Hardware} & \textbf{Resolution} & \textbf{Gen Time (5s)} \\
\midrule
High quality & 4 $\times$ H100 80GB & 1080p & 74s \\
Standard & 4 $\times$ H100 80GB & 720p & 44s \\
Fast (reduced steps) & 2 $\times$ H100 80GB & 720p & 28s \\
\bottomrule
\end{tabular}
\end{table}

\subsection{API Interface}

Zen-Video-I2V is accessible through the Zen LM API with the following parameters:

\begin{itemize}
  \item \texttt{image}: Input image (JPEG/PNG, min 512px on shorter edge).
  \item \texttt{duration}: Output duration in seconds (2.0--10.0, default 5.0).
  \item \texttt{resolution}: Output resolution (``720p'' or ``1080p'').
  \item \texttt{camera\_motion}: Camera trajectory type (``static'', ``dolly\_in'', ``dolly\_out'', ``pan\_left'', ``pan\_right'', ``orbit'', or custom 6-DoF trajectory).
  \item \texttt{motion\_intensity}: Overall motion scale (0.0--1.0, default 0.5).
  \item \texttt{fps}: Output frame rate (8, 16, or 24).
  \item \texttt{seed}: Reproducibility seed for deterministic generation.
\end{itemize}

\section{Safety Considerations}

\subsection{Identity and Consent}

Zen-Video-I2V includes a face detection classifier that identifies human faces in input images. Generation involving detected faces requires:

\begin{itemize}
  \item Explicit consent acknowledgment from the API caller that they have rights to animate the depicted individual.
  \item Invisible watermarking applied to all generated video (C2PA-compatible provenance metadata).
  \item Rate limiting and audit logging for accounts generating face-containing animations.
\end{itemize}

\subsection{Deepfake Mitigation}

Zen-Video-I2V applies Content Authenticity Initiative (CAI) metadata to all generated videos, marking them as AI-generated in a tamper-evident way. The model is also trained to degrade gracefully (reduce quality) on inputs that appear to be composite images designed for deceptive impersonation (face-swapped images, morphed composites).

\section{Related Work}

Image-to-video generation has advanced through GAN-based \cite{vidgan}, flow-based \cite{flowgen}, and diffusion-based \cite{stablevideo,svd} approaches. Temporal attention mechanisms in video diffusion have been studied in \cite{tune-a-video,videocrafter}. Motion controllability has been explored through trajectory conditioning \cite{dragnuwa} and optical flow guidance \cite{motionctrl}. Zen-Video-I2V advances this line of work through dual-path motion conditioning, physics-informed regularization, and full-resolution temporal attention at 14B parameter scale.

\section{Conclusion}

Zen-Video-I2V establishes new state-of-the-art results on image-to-video generation benchmarks (FVD 289, Warping Error 0.023, Temporal Consistency 91.3\%) while introducing dual-path motion control and physics-informed regularization that improve the plausibility and controllability of generated animations. At 14B parameters, the model generates photorealistic 10-second 1080p video from single images, suitable for creative production, product visualization, and interactive media applications. The physics-informed regularizer and full-resolution temporal attention architecture provide strong foundations for future extensions to longer-form and higher-resolution video generation.

\begin{thebibliography}{9}
\bibitem{dit} Peebles, W. \& Xie, S. (2023). Scalable Diffusion Models with Transformers. ICCV 2023.
\bibitem{vae} Kingma, D.P. \& Welling, M. (2014). Auto-Encoding Variational Bayes. ICLR 2014.
\bibitem{flashattn3} Shah, J. et al. (2024). FlashAttention-3: Fast and Accurate Attention with Asynchrony and Low-precision. arXiv:2407.08608.
\bibitem{controlnet} Zhang, L. et al. (2023). Adding Conditional Control to Text-to-Image Diffusion Models. ICCV 2023.
\bibitem{vidgan} Vondrick, C. et al. (2016). Generating Videos with Scene Dynamics. NeurIPS 2016.
\bibitem{flowgen} Holynski, A. et al. (2021). Animating Pictures with Eulerian Motion Fields. CVPR 2021.
\bibitem{stablevideo} Chai, L. et al. (2023). StableVideo: Text-driven Consistency-aware Diffusion Video Editing. ICCV 2023.
\bibitem{svd} Blattmann, A. et al. (2023). Stable Video Diffusion: Scaling Latent Video Diffusion Models to Large Datasets. arXiv:2311.15127.
\bibitem{tune-a-video} Wu, J.Z. et al. (2023). Tune-A-Video: One-Shot Tuning of Image Diffusion Models for Text-to-Video Generation. ICCV 2023.
\bibitem{videocrafter} He, Y. et al. (2023). VideoCrafter1: Open Diffusion Models for High-Quality Video Generation. arXiv:2310.19512.
\bibitem{dragnuwa} Wang, S. et al. (2024). DragNUWA: Fine-grained Control in Video Generation by Integrating Text, Image, and Trajectory. ECCV 2024.
\bibitem{motionctrl} Wang, Z. et al. (2024). MotionCtrl: A Unified and Flexible Motion Controller for Video Generation. SIGGRAPH 2024.
\end{thebibliography}

\end{document}