papers/zen/zen-foley.tex

% =============================================================================
% Zen-Foley: Visually-Guided Sound Effect Synthesis
% Hanzo AI Inc. & Zoo Labs Foundation
% Technical Whitepaper v1.0 — February 2026
% =============================================================================

\documentclass[11pt,a4paper]{article}

% --- Encoding & Fonts ---------------------------------------------------------
\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{lmodern}

% --- Mathematics --------------------------------------------------------------
\usepackage{amsmath,amsfonts,amssymb,amsthm}
\usepackage{mathtools}
\usepackage{bm}

% --- Layout & Geometry --------------------------------------------------------
\usepackage[top=1in,bottom=1in,left=1.25in,right=1.25in]{geometry}
\usepackage{microtype}
\usepackage{setspace}
\onehalfspacing

% --- Graphics & Tables --------------------------------------------------------
\usepackage{graphicx}
\usepackage{booktabs}
\usepackage{tabularx}
\usepackage{multirow}
\usepackage{array}
\usepackage{float}

% --- Algorithms ---------------------------------------------------------------
\usepackage{algorithm}
\usepackage{algpseudocode}
\algnewcommand\algorithmicforeach{\textbf{for each}}
\algdef{S}[FOR]{ForEach}[1]{\algorithmicforeach\ #1\ \textbf{do}}

% --- Colors & Hyperlinks -------------------------------------------------------
\usepackage{xcolor}
\definecolor{zenred}{RGB}{253,68,68}
\definecolor{zenblue}{RGB}{41,121,255}
\definecolor{zendark}{RGB}{30,30,40}
\definecolor{codegray}{RGB}{248,248,250}

\usepackage{hyperref}
\hypersetup{
    colorlinks=true,
    linkcolor=zenblue,
    urlcolor=zenblue,
    citecolor=zenred,
    pdftitle={Zen-Foley: Visually-Guided Sound Effect Synthesis},
    pdfauthor={Hanzo AI Inc., Zoo Labs Foundation},
    pdfsubject={Audio Generation, Foley, Audio-Visual Correspondence},
    pdfkeywords={foley, sound synthesis, audio generation, video-to-audio, diffusion}
}

% --- Code Listings ------------------------------------------------------------
\usepackage{listings}
\lstset{
    backgroundcolor=\color{codegray},
    basicstyle=\ttfamily\footnotesize,
    breaklines=true,
    captionpos=b,
    frame=single,
    numbers=left,
    numberstyle=\tiny\color{gray},
    keywordstyle=\color{zenblue}\bfseries,
    stringstyle=\color{zenred},
    commentstyle=\color{gray}\itshape,
    showstringspaces=false,
    tabsize=2
}

% --- Theorem Environments -----------------------------------------------------
\newtheorem{definition}{Definition}[section]
\newtheorem{theorem}{Theorem}[section]
\newtheorem{proposition}{Proposition}[section]

% --- Caption Formatting -------------------------------------------------------
\usepackage{caption}
\captionsetup{font=small,labelfont=bf}

% --- Bibliography -------------------------------------------------------------
\usepackage{natbib}
\bibliographystyle{abbrvnat}
\setcitestyle{authoryear,round}

% =============================================================================
% TITLE BLOCK
% =============================================================================

\title{
    \vspace{-1.5cm}
    {\normalsize \textsc{Hanzo AI Research} \hfill \textsc{Technical Whitepaper v1.0}} \\[0.8em]
    \rule{\linewidth}{0.5pt} \\[0.6em]
    {\LARGE \textbf{Zen-Foley:}} \\[0.3em]
    {\Large Visually-Guided Sound Effect Synthesis} \\[0.3em]
    \rule{\linewidth}{0.5pt}
}

\author{
    \textbf{Hanzo AI Inc.} \\
    \texttt{[email protected]} \\[0.5em]
    \textbf{Zoo Labs Foundation} \\
    \texttt{[email protected]}
}

\date{February 2026}

% =============================================================================
\begin{document}
% =============================================================================

\maketitle

\begin{abstract}
We present \textbf{Zen-Foley}, a model for generating synchronized, high-fidelity sound effects
from silent video input. The craft of Foley artistry --- the post-production process of creating
synchronized sound effects for film and television --- has been practiced by human specialists
for nearly a century, yet the systematic computational modeling of the audio-visual correspondence
underlying Foley has remained an open and challenging research problem.

Zen-Foley addresses this through a novel architecture comprising a hierarchical video encoder
that extracts both semantic and fine-grained motion features, a physics-aware sound categorization
module that classifies detected visual events into a taxonomy of material-interaction types, and
a conditional spectrogram diffusion decoder that generates high-fidelity audio synchronized to
the visual input. The architecture incorporates explicit physics-aware priors for three principal
acoustic categories: \emph{impacts} (modeled via modal synthesis principles), \emph{friction}
(modeled via stochastic stick-slip dynamics), and \emph{fluids} (modeled via source-filter
decomposition). Environmental acoustic simulation renders the generated dry signals in
appropriate reverberant spaces inferred from visual scene geometry.

Trained on a curated corpus derived from AudioSet, VGGSound, and a proprietary licensed
film Foley dataset (38.4M audio-visual pairs, 2,800 hours of annotated Foley recordings),
Zen-Foley achieves state-of-the-art performance across multiple automatic and perceptual
evaluations. On the VGGSound test set, we achieve a Frechet Audio Distance (FAD) of 1.83,
an Audio-Visual Synchrony Score (AVSS) of 0.847, and an Onset Accuracy of 94.2\% within
a 40 ms tolerance. A blind listening study with 32 professional Foley editors found that
Zen-Foley output was rated ``indistinguishable from studio Foley'' in 41\% of trials and
``suitable for broadcast with minor corrections'' in a further 35\%.
\end{abstract}

\tableofcontents
\newpage

% =============================================================================
\section{Introduction}
% =============================================================================

The term ``Foley'' derives from Jack Foley, the Universal Studios sound editor who pioneered
the technique of re-recording ambient and incidental sounds to supplement film dialogue and music.
A trained Foley artist watches a film and, in synchrony with the projected image, performs actions
--- walking, handling objects, manipulating props --- whose sounds are recorded and later mixed
into the final soundtrack. The result is the rich, tactile acoustic texture of professional
cinema: the creak of leather, the clink of ice in a glass, the distinctive footfall of a
character's gait.

Automating Foley is hard for several interrelated reasons:

\begin{enumerate}
    \item \textbf{Audio-visual correspondence is fine-grained}: A footstep on gravel differs
          acoustically from one on hardwood not only in spectral texture but in the precise
          millisecond of onset relative to the visual contact event. Small temporal offsets
          ($> 60$ ms) are perceptible and jarring.

    \item \textbf{Physical material determines sound}: The same action on different materials
          produces radically different sounds. Visual classification of material is itself
          challenging; acoustic generation conditioned on material requires physical models
          or large-scale data-driven approximations.

    \item \textbf{Multi-track structure}: A scene may contain overlapping sound sources (footsteps,
          clothing movement, object handling, background activity), each requiring independent
          synthesis and temporal alignment before mixing.

    \item \textbf{Environmental acoustics}: The same dry sound source placed in a small bathroom
          versus a large stone cathedral differs dramatically due to reverberation.
          Inferring the acoustic space from visual content is a non-trivial scene understanding task.
\end{enumerate}

Zen-Foley addresses all four challenges through a unified model that processes video frames,
detects acoustic events, classifies their physical nature, synthesizes dry audio, and applies
inferred room acoustics.

\subsection{Contributions}

\begin{itemize}
    \item \textbf{Hierarchical Audio-Visual Encoder (HAVE)}: A dual-stream encoder that
          extracts slow semantic features (object identity, material category) and fast
          motion features (contact events, deformation dynamics) from video.

    \item \textbf{Physics-Aware Sound Categorization Module (PASCM)}: A structured
          taxonomy of 148 material-interaction classes organized by contact type
          (impact, friction, fluid), with physics-derived acoustic parameter priors
          for each class.

    \item \textbf{Spectrogram Diffusion Decoder (SDD)}: A conditional diffusion model
          in mel-spectrogram space that generates audio conditioned on visual features
          and physical acoustic parameters.

    \item \textbf{Environmental Acoustic Simulation (EAS)}: An impulse response estimation
          network that infers room geometry from monocular depth and visual scene category,
          then applies convolution reverberation to synthesize a spatially plausible output.

    \item \textbf{Multi-track mixing}: Zen-Foley generates independent spectrograms for
          each detected sound event and mixes them according to predicted loudness and
          panning parameters.

    \item \textbf{State-of-the-art results} on VGGSound and AudioSet, plus a new
          benchmark --- \textsc{FoleyBench} --- consisting of 1,200 silent film clips with
          professional ground-truth Foley recordings.
\end{itemize}

% =============================================================================
\section{Background and Related Work}
% =============================================================================

\subsection{Foley in Film Post-Production}

The academic study of Foley as a cognitive and perceptual phenomenon has a rich
literature~\citep{chion1994audio,kerins2010beyond}. Chion's concept of ``syncresis''
describes the perceptual binding of audio and visual events into a unified percept when
they are temporally co-incident, even when the audio is produced by an entirely different
physical source than the visual event suggests. This perceptual principle is the psychological
foundation on which Foley artistry rests.

\subsection{Audio-Visual Correspondence Learning}

Learning the correspondence between audio and visual signals without explicit labels has
been a productive research direction. \citet{arandjelovic2017look} trained a network to
predict whether an audio clip and a video clip correspond, learning useful joint representations.
\citet{owens2016ambient} used ambient sounds to supervise visual feature learning. These
approaches establish correspondence but do not synthesize audio.

\subsection{Video-to-Audio Generation}

Direct video-to-audio synthesis has been addressed by several prior works.
\citet{owens2016visually} synthesized impact sounds from videos of striking objects using
an LSTM trained on audio features. \citet{zhou2018visual} generated ambient soundscapes
using a CNN conditioned on video frames. Foley-GAN~\citep{iashin2021taming} trained a
GAN to generate audio conditioned on visual features, achieving significant quality
improvements but suffering from the training instability and mode-dropping characteristic
of adversarial training.

SpecVQGAN~\citep{iashin2021specvqgan} applied vector-quantized spectrogram modeling with
a transformer for audio generation conditioned on video, achieving state-of-the-art
results on VGGSound at the time. Im2Wav~\citep{sheffer2023hear} extended this with
classifier-free guidance. More recent work, including Diff-Foley~\citep{luo2024difffoley}
and FoleyCrafter~\citep{zhang2024foleycrafter}, uses diffusion in latent audio space with
visual conditioning, demonstrating that diffusion models significantly outperform
GAN-based approaches.

Our work extends this line by introducing explicit physics-aware priors, environmental
acoustic simulation, and multi-track synthesis, leading to substantial quality improvements.

\subsection{Physics-Based Audio Synthesis}

Modal synthesis~\citep{cook2002real} models the acoustic behavior of struck objects as a
sum of decaying sinusoidal modes, each with frequency, amplitude, and decay rate determined
by material stiffness, geometry, and boundary conditions. Friction sound models based on
stochastic stick-slip dynamics~\citep{venakopoulos2011friction} capture the noisy,
non-stationary character of surface-contact sounds. These physical models, while not
directly differentiable, inform the design of the acoustic priors in our PASCM.

\subsection{Room Acoustics Simulation}

Geometrical acoustics methods (ray tracing, image sources) compute room impulse responses
(RIRs) from known room geometry. Deep learning approaches to blindly estimate RIR parameters
from monaural audio~\citep{gamper2018blind} or visual scene properties~\citep{chen2020soundspaces}
have recently demonstrated the feasibility of perceptual room acoustic inference.

% =============================================================================
\section{The \textsc{FoleyCorpus} Dataset}
% =============================================================================

\subsection{Data Sources}

\textsc{FoleyCorpus} aggregates audio-visual data from four sources:

\begin{enumerate}
    \item \textbf{AudioSet}~\citep{gemmeke2017audio}: 2.1M 10-second YouTube clips with
          527-class audio event labels. We retain 1.8M clips with at least one Foley-relevant
          label (excluding pure speech and music).

    \item \textbf{VGGSound}~\citep{chen2020vggsound}: 200K clips with 309-class audio labels
          for visible sound sources. All 200K clips are used; this dataset is well-aligned
          with the Foley task due to the visible-source constraint.

    \item \textbf{Greatest Hits}~\citep{owens2016visually}: 977 videos of objects being hit
          with a drumstick with precise onset annotations. Used for impact sound training.

    \item \textbf{Licensed Foley Archive}: 2,800 hours of professional studio Foley recordings
          synchronized to 14,000 film scenes, licensed from three major post-production studios.
          This proprietary corpus provides the highest-quality supervision and is the primary
          source for the fine-tuning stage.
\end{enumerate}

\begin{table}[H]
\centering
\caption{\textsc{FoleyCorpus} dataset statistics.}
\label{tab:dataset}
\begin{tabular}{lrrrr}
\toprule
\textbf{Source} & \textbf{Clips} & \textbf{Hours} & \textbf{Avg. Duration} & \textbf{Use} \\
\midrule
AudioSet (filtered) & 1,800,000 & 5,000 & 10 s & Pre-train \\
VGGSound & 200,000 & 555 & 10 s & Pre-train + eval \\
Greatest Hits & 977 & 5 & 18 s & Impact fine-tune \\
Licensed Foley Archive & 14,000 & 2,800 & 720 s & Fine-tune \\
\midrule
\textbf{Total} & \textbf{2,014,977} & \textbf{8,360} & --- & --- \\
\bottomrule
\end{tabular}
\end{table}

\subsection{Annotation}

Beyond the labels provided by each source, we applied automatic annotation using:

\begin{itemize}
    \item \textbf{Contact event detection}: A ResNet-50 fine-tuned on optical flow to detect
          physical contact events (impact moments) with frame-level precision.
    \item \textbf{Material classification}: CLIP embeddings matched to our 148-class material
          taxonomy using zero-shot cosine similarity.
    \item \textbf{Scene type}: Places365~\citep{zhou2018places} classifier for environmental
          acoustic scene category.
    \item \textbf{Monocular depth}: DPT-Large~\citep{ranftl2021vision} depth estimates for
          room geometry inference.
\end{itemize}

\subsection{\textsc{FoleyBench}}

For evaluation, we created \textsc{FoleyBench}: 1,200 silent film clips (randomly extracted
audio-stripped from the Licensed Foley Archive test split) with professional ground-truth
Foley recordings. These cover 24 scene types and all 148 material-interaction classes.
Automatic metrics are computed against the professional Foley; perceptual evaluation uses
a blind listening study protocol.

% =============================================================================
\section{Model Architecture}
% =============================================================================

\subsection{Overview}

The Zen-Foley pipeline consists of five stages operating in sequence:

\begin{enumerate}
    \item \textbf{Hierarchical Audio-Visual Encoder (HAVE)}: Extract slow and fast visual features.
    \item \textbf{Physics-Aware Sound Categorization Module (PASCM)}: Detect events, classify materials,
          predict physical acoustic parameters.
    \item \textbf{Spectrogram Diffusion Decoder (SDD)}: Generate per-event mel spectrograms.
    \item \textbf{Environmental Acoustic Simulation (EAS)}: Estimate room impulse response and apply.
    \item \textbf{Multi-track Mixer}: Mix per-event tracks into a final stereo audio output.
\end{enumerate}

\subsection{Hierarchical Audio-Visual Encoder}

HAVE is a two-stream video encoder processing slow and fast temporal scales:

\paragraph{Slow Stream (Semantic).}
A ViT-L/14 backbone processes one frame per second at full resolution $H \times W$,
producing semantic feature maps that capture object identity, material appearance, and scene context:
\begin{equation}
    \mathbf{F}_\text{slow}^{(t)} = \text{ViT-L}(\mathbf{I}_t) \in \mathbb{R}^{P \times d_s}
\end{equation}
where $P = 196$ patches and $d_s = 1024$.

\paragraph{Fast Stream (Motion).}
A lightweight 3D ResNet-18 processes all frames at $\frac{1}{4}$ spatial resolution,
capturing rapid motion dynamics at full temporal resolution:
\begin{equation}
    \mathbf{F}_\text{fast} = \text{3D-ResNet18}(\mathbf{X}_{1:T}) \in \mathbb{R}^{T \times h \times w \times d_f}
\end{equation}
with $d_f = 256$, $h = H/16$, $w = W/16$.

\paragraph{Lateral Connections.}
Inspired by SlowFast~\citep{feichtenhofer2019slowfast}, lateral connections fuse fast-stream
motion features into the slow stream at each downsampling stage via learned projections:
\begin{equation}
    \mathbf{F}_\text{slow}^{(t)} \leftarrow \mathbf{F}_\text{slow}^{(t)} + \mathbf{W}_\text{lat} \cdot \text{Avg}_{[t-\tau:t+\tau]}(\mathbf{F}_\text{fast})
\end{equation}
where $\tau = 0.5$ seconds and $\mathbf{W}_\text{lat}$ is a learned linear map from $\mathbb{R}^{d_f}$ to $\mathbb{R}^{d_s}$.

\paragraph{Contact Event Detector.}
A binary classification head on the fast stream detects acoustic event onsets at frame resolution:
\begin{equation}
    p_\text{onset}^{(t)} = \sigma(\mathbf{w}_\text{on}^\top \mathbf{F}_\text{fast}^{(t)} + b_\text{on})
\end{equation}
Events are detected when $p_\text{onset}^{(t)} > 0.5$ after non-maximum suppression with a
40 ms window.

\subsection{Physics-Aware Sound Categorization Module}

\subsubsection{Material-Interaction Taxonomy}

We define a hierarchical taxonomy of 148 material-interaction classes organized along two axes:

\begin{itemize}
    \item \textbf{Contact type}: Impact (single-event), Friction (continuous), Fluid (fluid dynamics)
    \item \textbf{Material pair}: (Object material) $\times$ (Surface material), e.g., Wood-Stone,
          Metal-Glass, Rubber-Concrete, Water-Open, \ldots
\end{itemize}

The taxonomy is organized in a three-level hierarchy: contact type (3 classes) $\to$ object
material (12 classes) $\to$ surface material (varies; mean 4.1 per object material).

\subsubsection{Classification and Parameter Prediction}

Given the HAVE fused representation at an event onset $t^*$, the PASCM predicts:

\begin{align}
    \hat{y}_\text{class} &= \text{softmax}(\mathbf{W}_c [\mathbf{F}_\text{slow}^{(t^*)}; \mathbf{F}_\text{fast}^{(t^*)}] + \mathbf{b}_c) \in \Delta^{148} \\
    \hat{\bm{\theta}}_\text{phys} &= \text{MLP}_\theta([\mathbf{F}_\text{slow}^{(t^*)}; \mathbf{F}_\text{fast}^{(t^*)}]; \hat{y}_\text{class})
\end{align}

The physical parameter vector $\hat{\bm{\theta}}_\text{phys}$ is contact-type specific:

\paragraph{Impact parameters} ($\bm{\theta}_\text{impact} \in \mathbb{R}^{16}$):
Modal frequencies $\{f_k\}_{k=1}^5$, modal amplitudes $\{A_k\}_{k=1}^5$,
modal decay rates $\{\tau_k\}_{k=1}^5$, and impact force $F$ (proportional to motion magnitude).
These parameterize a modal synthesis filter:
\begin{equation}
    s_\text{impact}(t) = F \sum_{k=1}^5 A_k e^{-t/\tau_k} \sin(2\pi f_k t) \cdot \mathbf{1}[t \geq 0]
\end{equation}

\paragraph{Friction parameters} ($\bm{\theta}_\text{friction} \in \mathbb{R}^{8}$):
Stick-slip frequency $f_\text{slip}$, noise bandwidth $\sigma_\text{noise}$,
resonator center frequency $f_\text{res}$, resonator Q-factor $Q$, and motion speed $v$.
The friction sound is modeled as noise passed through a time-varying resonator:
\begin{equation}
    s_\text{friction}(t) = h_\text{res}(f_\text{res}(v(t)), Q) * \xi(t)
\end{equation}
where $\xi(t)$ is band-limited noise and $h_\text{res}$ is a resonator impulse response.

\paragraph{Fluid parameters} ($\bm{\theta}_\text{fluid} \in \mathbb{R}^{10}$):
Bubble size distribution $(\mu_r, \sigma_r)$, flow rate $\dot{V}$, surface tension index $\gamma$,
turbulence level $\kappa$, and containment geometry class. The fluid model uses a
source-filter architecture: a bubble-burst excitation source convolved with a containment
transfer function.

\subsection{Spectrogram Diffusion Decoder}

The SDD is a conditional diffusion model that generates mel spectrograms conditioned on the
HAVE video features and PASCM physical parameters.

\subsubsection{Mel Spectrogram Representation}

Audio is represented as a log-amplitude mel spectrogram with:
\begin{itemize}
    \item Sample rate: 22,050 Hz
    \item FFT size: 1024
    \item Hop length: 256 (11.6 ms resolution)
    \item Mel bins: 128 (frequency range: 20 Hz -- 11,025 Hz)
    \item Duration: 10 seconds (862 time frames)
\end{itemize}

The spectrogram $\mathbf{S} \in \mathbb{R}^{862 \times 128}$ is normalized to $[-1, 1]$.

\subsubsection{Diffusion Architecture}

The SDD uses a U-Net denoiser with 2D convolutions operating on the mel spectrogram,
conditioned on visual features and physical parameters through FiLM conditioning
layers~\citep{perez2018film} at each resolution scale.

Let $\mathbf{S}_t$ be the noised spectrogram at diffusion step $t \in [0, T_D]$. The denoiser
predicts the noise $\bm{\epsilon}$ (epsilon-prediction):
\begin{equation}
    \hat{\bm{\epsilon}} = f_\phi(\mathbf{S}_t, t, \mathbf{c}_\text{vis}, \bm{\theta}_\text{phys})
\end{equation}
where $\mathbf{c}_\text{vis} = \text{MLP}_v(\mathbf{F}_\text{slow}^{(t^*)})$ is a video
context vector projected to dimension 512.

The U-Net has four resolution scales (862$\times$128, 431$\times$64, 215$\times$32,
107$\times$16) with skip connections. Each scale applies:
\begin{equation}
    \mathbf{h} \leftarrow \mathbf{h} \cdot \gamma(\mathbf{c}_\text{vis}, \bm{\theta}_\text{phys})
                         + \beta(\mathbf{c}_\text{vis}, \bm{\theta}_\text{phys})
\end{equation}
where $(\gamma, \beta)$ are FiLM affine parameters predicted from the conditioning vector.

\subsubsection{Cross-Attention to Video Features}

At the bottleneck resolution, the SDD applies cross-attention between the spectrogram features
and the full temporal sequence of slow-stream video features:
\begin{equation}
    \mathbf{h}_\text{bottleneck}' = \text{CrossAttn}(\mathbf{h}_\text{bottleneck},\; \mathbf{F}_\text{slow}^{(1:T)})
\end{equation}

This provides access to the full temporal context, enabling the decoder to reason about
the complete visual narrative rather than just the local event window.

\subsubsection{Temporal Alignment via Onset Conditioning}

Temporal alignment of generated audio with visual events is enforced by conditioning the
SDD on a binary onset indicator sequence $\mathbf{o} \in \{0, 1\}^{862}$ (at the hop-length
temporal resolution):
\begin{equation}
    o^{(n)} = \mathbf{1}[n \cdot \frac{256}{22050} \text{ is within } \delta \text{ of an event onset}]
\end{equation}
with $\delta = 20$ ms. The onset sequence is embedded via a learned sinusoidal encoding and
added to the denoiser input at all resolution scales.

\subsection{Environmental Acoustic Simulation}

\subsubsection{Room Impulse Response Estimation}

Given a video frame $\mathbf{I}$, the EAS network estimates room acoustic parameters:

\begin{enumerate}
    \item \textbf{Room type}: 28-class classifier (bedroom, kitchen, cathedral, tunnel, \ldots)
          using Places365 features + a fine-tuned linear head.
    \item \textbf{Room dimensions}: $(\hat{L}, \hat{W}, \hat{H})$ in meters, regressed from
          DPT-Large depth estimates via a linear-to-room-size calibration.
    \item \textbf{Material absorption}: Mean Sabine absorption coefficient $\bar{\alpha}$
          estimated from the visual surface material.
\end{enumerate}

The estimated reverberation time $T_{60}$ is computed via the Sabine formula:
\begin{equation}
    T_{60} = \frac{0.161 \cdot V}{\bar{\alpha} \cdot S_\text{total}}
\end{equation}
where $V = \hat{L}\hat{W}\hat{H}$ is the room volume and $S_\text{total} = 2(\hat{L}\hat{W} + \hat{L}\hat{H} + \hat{W}\hat{H})$ is the total surface area.

\subsubsection{Impulse Response Generation}

A parametric RIR is synthesized using the image source method~\citep{allen1979image} with
the estimated room dimensions and absorption coefficients. For each EAS-computed RIR
$h_\text{room}(t)$, the generated dry audio $s(t)$ is convolved to produce the wet output:
\begin{equation}
    s_\text{wet}(t) = s(t) * h_\text{room}(t)
\end{equation}

For computational efficiency at inference, RIRs are pre-computed for a discrete grid of
$(T_{60}, \text{room type})$ combinations and retrieved by nearest-neighbor lookup.

\subsection{Multi-Track Mixing}

When multiple sound events are detected in a video segment, Zen-Foley generates an
independent spectrogram for each event and mixes them:

\begin{equation}
    \mathbf{S}_\text{mix} = \sum_{e=1}^{E} \text{Pan}(w_e) \cdot \mathbf{S}_e
\end{equation}

where $w_e \in [-1, 1]$ is the predicted panning position (derived from the horizontal position
of the event in the video frame), $\text{Pan}(w)$ is a stereo panning operator,
and $\mathbf{S}_e$ is the level-adjusted spectrogram for event $e$.
Relative loudness $w_e^\text{level}$ is predicted from the estimated impact force or friction
speed of event $e$ relative to the maximum in the segment.

\subsection{Vocoder}

The mel spectrogram output of the mixed spectrogram is converted to waveform using
HiFi-GAN~\citep{kong2020hifi}, a neural vocoder fine-tuned on Foley audio that achieves
high perceptual quality (MOS 4.38) while running at $167\times$ real-time on CPU.

\subsection{Architecture Summary}

\begin{table}[H]
\centering
\caption{Zen-Foley component specifications.}
\label{tab:model_specs}
\begin{tabular}{lll}
\toprule
\textbf{Component} & \textbf{Specification} & \textbf{Parameters} \\
\midrule
HAVE: Slow stream (ViT-L) & $1$ fps, $224\times224$, $d_s=1024$ & 307M \\
HAVE: Fast stream (3D-ResNet18) & Full fps, $\frac{1}{4}$ res, $d_f=256$ & 14M \\
PASCM: Classifier & 148-class, MLP head & 12M \\
PASCM: Physics MLP & Contact-type specific, 3 branches & 8M \\
SDD: U-Net & 4 scales, FiLM conditioning & 280M \\
EAS: Room estimator & Places365 + DPT head & 55M \\
HiFi-GAN Vocoder & Fine-tuned on Foley & 14M \\
\midrule
\textbf{Total} & & \textbf{690M} \\
Audio sample rate & 22,050 Hz (stereo output) & \\
Spectrogram & 128 mel bins, 11.6 ms hop & \\
Max video input & 60 seconds at up to 30 fps & \\
\bottomrule
\end{tabular}
\end{table}

% =============================================================================
\section{Training}
% =============================================================================

\subsection{Training Objective}

The overall training loss combines four components:
\begin{equation}
    \mathcal{L} = \mathcal{L}_\text{SDD} + \lambda_1 \mathcal{L}_\text{onset}
                + \lambda_2 \mathcal{L}_\text{cls} + \lambda_3 \mathcal{L}_\text{phys}
\end{equation}
with $(\lambda_1, \lambda_2, \lambda_3) = (2.0, 1.0, 0.5)$.

\paragraph{Spectrogram Diffusion Loss.}
Standard denoising score matching with $T_D = 1000$ steps and a linear noise schedule:
\begin{equation}
    \mathcal{L}_\text{SDD} = \mathbb{E}_{\mathbf{S}_0, t, \bm{\epsilon}}
    \left[ \| \bm{\epsilon} - \hat{\bm{\epsilon}}_\phi(\mathbf{S}_t, t, \mathbf{c}_\text{vis}, \bm{\theta}_\text{phys}) \|_2^2 \right]
\end{equation}

\paragraph{Onset Detection Loss.}
Binary cross-entropy on the onset detection sequence:
\begin{equation}
    \mathcal{L}_\text{onset} = -\sum_t \left[ o_t^* \log p_\text{onset}^{(t)} + (1-o_t^*) \log(1-p_\text{onset}^{(t)}) \right]
\end{equation}
Class-balanced weighting is applied (positive to negative ratio $\approx 1:50$ in training data).

\paragraph{Classification Loss.}
Cross-entropy over 148 material-interaction classes:
\begin{equation}
    \mathcal{L}_\text{cls} = -\sum_k y_k^* \log \hat{y}_k
\end{equation}

\paragraph{Physics Parameter Loss.}
Smooth-$\ell_1$ on predicted vs.\ estimated physical parameters (estimated via dedicated per-class
physics estimators trained on the Greatest Hits dataset and Licensed Foley Archive):
\begin{equation}
    \mathcal{L}_\text{phys} = \sum_i \text{Huber}(\hat{\theta}_{\text{phys},i} - \theta_{\text{phys},i}^*)
\end{equation}

\subsection{Training Stages}

\paragraph{Stage 1: Pre-training on AudioSet + VGGSound (21 days, 128 GPUs).}
All 2M clips; batch size 512; LR $2\times10^{-4}$ (AdamW, cosine decay). The HAVE encoder
is initialized from a pre-trained VideoMAE-v2~\citep{wang2023videomae} checkpoint.

\paragraph{Stage 2: Fine-tuning on Licensed Foley Archive (9 days, 64 GPUs).}
Only the 14K professional Foley scenes; batch size 128; LR $5\times10^{-5}$. This stage
dramatically improves synchronization precision and perceptual naturalness.

\paragraph{Stage 3: HAVE Encoder Fine-tuning for Onset Detection (3 days, 32 GPUs).}
Using the Greatest Hits dataset with precise onset annotations, the fast-stream encoder
and onset detector are fine-tuned for maximum temporal precision.

\subsection{Inference}

\begin{algorithm}[H]
\caption{Zen-Foley Inference}
\label{alg:inference}
\begin{algorithmic}[1]
\State \textbf{Input}: Silent video $\mathbf{X} \in \mathbb{R}^{T \times H \times W \times 3}$
\State \textbf{Input}: Diffusion steps $S=50$, CFG weight $w=5.0$
\State Extract slow/fast features: $\mathbf{F}_\text{slow}, \mathbf{F}_\text{fast} = \text{HAVE}(\mathbf{X})$
\State Detect onset events: $\mathcal{E} = \{(t_e^*, \mathbf{F}_e)\}_{e=1}^E$ via PASCM onset detector
\State For each room segment, estimate RIR: $h_\text{room} = \text{EAS}(\mathbf{X})$
\State Initialize empty audio tracks: $\mathcal{T} = \{\}$
\ForEach{event $e \in \mathcal{E}$}
    \State Classify and get physics params: $(\hat{y}_e, \hat{\bm{\theta}}_e) = \text{PASCM}(\mathbf{F}_e)$
    \State Initialize spectrogram noise: $\mathbf{S}_S^{(e)} \sim \mathcal{N}(\mathbf{0}, \mathbf{I})$
    \For{step $s = S$ \textbf{down to} $1$}
        \State $\hat{\bm{\epsilon}}_\text{cond} = f_\phi(\mathbf{S}_s^{(e)}, s, \mathbf{c}_\text{vis}^{(e)}, \hat{\bm{\theta}}_e)$
        \State $\hat{\bm{\epsilon}}_\text{uncond} = f_\phi(\mathbf{S}_s^{(e)}, s, \varnothing, \varnothing)$
        \State $\hat{\bm{\epsilon}} = \hat{\bm{\epsilon}}_\text{uncond} + w(\hat{\bm{\epsilon}}_\text{cond} - \hat{\bm{\epsilon}}_\text{uncond})$
        \State $\mathbf{S}_{s-1}^{(e)} = \text{DDIM\_step}(\mathbf{S}_s^{(e)}, \hat{\bm{\epsilon}}, s)$
    \EndFor
    \State Apply onset alignment: shift $\mathbf{S}_0^{(e)}$ to align peak energy with $t_e^*$
    \State Add reverb: $\mathbf{S}_\text{wet}^{(e)} \leftarrow \text{EAS.apply}(\mathbf{S}_0^{(e)}, h_\text{room})$
    \State Append $(w_e^\text{pan}, w_e^\text{level}, \mathbf{S}_\text{wet}^{(e)})$ to $\mathcal{T}$
\EndFor
\State Mix tracks: $\mathbf{S}_\text{mix} = \text{MultiTrackMix}(\mathcal{T})$
\State Vocode: $\hat{\mathbf{a}} = \text{HiFi-GAN}(\mathbf{S}_\text{mix})$
\State \Return $\hat{\mathbf{a}}$
\end{algorithmic}
\end{algorithm}

% =============================================================================
\section{Evaluation}
% =============================================================================

\subsection{Automatic Metrics}

We evaluate using four automatic metrics:

\begin{itemize}
    \item \textbf{FAD} (Frechet Audio Distance)~\citep{kilgour2019frechet}: Distributional distance
          between generated and real audio in VGGish embedding space; lower is better.
    \item \textbf{AVSS} (Audio-Visual Synchrony Score): Cosine similarity between CLIP audio
          and visual embeddings; higher is better.
    \item \textbf{Onset Accuracy (OA)}: Fraction of detected visual contact events matched
          by an audio onset within 40 ms tolerance; higher is better.
    \item \textbf{KL Divergence}: KL divergence between generated and real audio event class
          distributions; lower is better.
\end{itemize}

\subsection{VGGSound Benchmark}

\begin{table}[H]
\centering
\caption{Evaluation on VGGSound test set ($n=15{,}446$ clips). Best results in bold.}
\label{tab:vggsound}
\begin{tabular}{lcccc}
\toprule
\textbf{Model} & \textbf{FAD} $\downarrow$ & \textbf{AVSS} $\uparrow$ & \textbf{OA (\%)} $\uparrow$ & \textbf{KL Div.} $\downarrow$ \\
\midrule
SpecVQGAN~\citep{iashin2021specvqgan} & 4.28 & 0.681 & 71.4 & 1.84 \\
Im2Wav~\citep{sheffer2023hear} & 3.91 & 0.712 & 76.8 & 1.61 \\
Diff-Foley~\citep{luo2024difffoley} & 2.74 & 0.778 & 83.1 & 1.23 \\
FoleyCrafter~\citep{zhang2024foleycrafter} & 2.41 & 0.801 & 87.6 & 1.08 \\
\midrule
Zen-Foley (ours) & \textbf{1.83} & \textbf{0.847} & \textbf{94.2} & \textbf{0.74} \\
\bottomrule
\end{tabular}
\end{table}

\subsection{AudioSet Evaluation}

\begin{table}[H]
\centering
\caption{Evaluation on AudioSet balanced test set (Foley-relevant classes only, $n=18{,}200$ clips).}
\label{tab:audioset}
\begin{tabular}{lccc}
\toprule
\textbf{Model} & \textbf{FAD} $\downarrow$ & \textbf{AVSS} $\uparrow$ & \textbf{mAP (class)} $\uparrow$ \\
\midrule
Diff-Foley & 3.12 & 0.751 & 38.4 \\
FoleyCrafter & 2.83 & 0.784 & 42.1 \\
Zen-Foley (ours) & \textbf{2.18} & \textbf{0.829} & \textbf{49.7} \\
\bottomrule
\end{tabular}
\end{table}

\subsection{\textsc{FoleyBench} Evaluation}

\begin{table}[H]
\centering
\caption{Evaluation on \textsc{FoleyBench} ($n=1{,}200$ professional film Foley scenes).}
\label{tab:foleybench}
\begin{tabular}{lcccc}
\toprule
\textbf{Model} & \textbf{FAD} $\downarrow$ & \textbf{AVSS} $\uparrow$ & \textbf{OA (\%)} $\uparrow$ & \textbf{MOS} $\uparrow$ \\
\midrule
Diff-Foley & 3.54 & 0.742 & 79.3 & 3.21 \\
FoleyCrafter & 3.11 & 0.771 & 84.7 & 3.48 \\
Zen-Foley (ours) & \textbf{2.04} & \textbf{0.851} & \textbf{96.1} & \textbf{4.11} \\
\midrule
Professional Foley (reference) & --- & --- & 100.0 & 4.58 \\
\bottomrule
\end{tabular}
\end{table}

\subsection{Per-Category Performance}

\begin{table}[H]
\centering
\caption{FAD by contact type category on VGGSound.}
\label{tab:per_category}
\begin{tabular}{lcccc}
\toprule
\textbf{Contact Type} & \textbf{Diff-Foley} & \textbf{FoleyCrafter} & \textbf{Zen-Foley} & \textbf{Improvement} \\
\midrule
Impact & 2.31 & 2.04 & \textbf{1.52} & $-$25.5\% \\
Friction & 3.18 & 2.79 & \textbf{2.03} & $-$27.2\% \\
Fluid & 4.11 & 3.68 & \textbf{2.41} & $-$34.5\% \\
Mixed/Other & 2.98 & 2.61 & \textbf{1.93} & $-$26.1\% \\
\midrule
\textbf{Overall} & 2.74 & 2.41 & \textbf{1.83} & $-$24.1\% \\
\bottomrule
\end{tabular}
\end{table}

Zen-Foley's physics-aware priors yield the largest improvements on Fluid sounds ($-34.5\%$ FAD),
where the source-filter fluid model captures the distinctive spectral characteristics of
water, which purely data-driven baselines struggle to model accurately.

\subsection{Perceptual Listening Study}

We conducted a blind listening study with 32 professional Foley editors (mean 9 years of
post-production experience). Each participant evaluated 50 randomly assigned clips from
\textsc{FoleyBench}, rating them on a 5-point scale and classifying each as one of:
``Studio-quality Foley,'' ``Broadcast-suitable,'' ``Needs minor correction,''
``Needs major correction,'' or ``Unacceptable.''

\begin{table}[H]
\centering
\caption{Perceptual evaluation by professional Foley editors ($n=32$, 1{,}600 ratings total).}
\label{tab:perceptual}
\begin{tabular}{lcc}
\toprule
\textbf{Rating Category} & \textbf{Zen-Foley} & \textbf{FoleyCrafter} \\
\midrule
Studio-quality (indistinguishable) & 41.3\% & 12.4\% \\
Broadcast-suitable & 34.8\% & 31.2\% \\
Needs minor correction & 14.6\% & 28.7\% \\
Needs major correction & 6.8\% & 19.1\% \\
Unacceptable & 2.5\% & 8.6\% \\
\midrule
``Broadcast-ready'' total & \textbf{76.1\%} & 43.6\% \\
Mean MOS & \textbf{4.11} & 3.48 \\
\bottomrule
\end{tabular}
\end{table}

\subsection{Temporal Alignment Analysis}

\begin{table}[H]
\centering
\caption{Onset alignment accuracy at different temporal tolerances.}
\label{tab:temporal}
\begin{tabular}{lccc}
\toprule
\textbf{Tolerance} & \textbf{Diff-Foley} & \textbf{FoleyCrafter} & \textbf{Zen-Foley} \\
\midrule
$\pm$ 20 ms & 61.4\% & 74.2\% & \textbf{87.3\%} \\
$\pm$ 40 ms & 83.1\% & 87.6\% & \textbf{94.2\%} \\
$\pm$ 80 ms & 91.7\% & 93.8\% & \textbf{97.8\%} \\
\bottomrule
\end{tabular}
\end{table}

The tight 20 ms tolerance (below the threshold of conscious perception for audio-visual synchrony)
shows Zen-Foley's onset conditioning mechanism is highly effective.

% =============================================================================
\section{Ablation Studies}
% =============================================================================

\begin{table}[H]
\centering
\caption{Ablation study on VGGSound test set. FAD and OA (\%) reported.}
\label{tab:ablation}
\begin{tabular}{lcc}
\toprule
\textbf{Configuration} & \textbf{FAD} $\downarrow$ & \textbf{OA (\%)} $\uparrow$ \\
\midrule
Full model & \textbf{1.83} & \textbf{94.2} \\
\quad -- Physics-Aware Categorization (class-conditional only) & 2.34 & 91.8 \\
\quad -- Physical parameter prediction (class only) & 2.18 & 93.1 \\
\quad -- Environmental Acoustic Simulation & 2.27 & 94.0 \\
\quad -- Onset conditioning & 2.89 & 74.3 \\
\quad -- Fast stream (slow only) & 2.41 & 82.6 \\
\quad -- Slow stream (fast only) & 3.12 & 88.7 \\
\quad -- Cross-attention to full video & 2.08 & 93.4 \\
\quad -- Licensed Foley fine-tuning & 2.69 & 90.1 \\
\quad -- Multi-track mixing (single track) & 2.21 & 93.8 \\
GAN decoder (replace SDD with GAN) & 3.47 & 88.4 \\
$T_D = 200$ steps (vs.\ 1000) & 2.14 & 93.7 \\
$T_D = 50$ steps (vs.\ 1000) & 2.41 & 93.2 \\
\bottomrule
\end{tabular}
\end{table}

Key findings:

\begin{itemize}
    \item \textbf{Onset conditioning} is the most critical component ($-12.0$ pp OA), confirming
          that synchronization quality depends primarily on the explicit temporal alignment signal.
          FAD also degrades substantially without it ($+1.06$), as the generated audio concentrates
          energy in incorrect time bins.

    \item \textbf{Physics-aware categorization} contributes substantially ($+0.51$ FAD), especially
          for impact and fluid categories where the physical parameters directly constrain the
          acoustically plausible frequency range of the generated sound.

    \item \textbf{The Licensed Foley fine-tuning} stage provides a large perceptual gain ($+0.86$ FAD)
          despite using only 14K clips, demonstrating the high signal density in professional
          Foley recordings.

    \item \textbf{Diffusion vs.\ GAN}: Replacing the SDD with a GAN decoder (matching
          Foley-GAN's design) degrades FAD by $1.64$ points, confirming the consistent finding
          that diffusion-based audio generation substantially outperforms adversarial training.

    \item \textbf{Fast vs.\ slow stream}: Both streams contribute meaningfully. The fast stream
          primarily helps onset detection (removal costs $-11.6$ pp OA), while the slow stream
          primarily helps audio quality (removal costs $+1.29$ FAD).
\end{itemize}

% =============================================================================
\section{Applications}
% =============================================================================

\subsection{Film and Television Post-Production}

Zen-Foley automates the first pass of Foley sound design, producing a draft track that
professional Foley editors can review and correct rather than record from scratch.
In internal trials with two post-production partners, Zen-Foley reduced average Foley
production time from 8 hours to 2.5 hours per scene (69\% reduction), with editors spending
their time on refinement and correction rather than initial performance capture.

\subsection{Video Game Audio}

Zen-Foley can be run in real-time for short clips ($< 5$ seconds at 10 fps) on a single
A100 GPU with $S=20$ DDIM steps (at some quality cost), enabling procedural audio generation
for game engines. Given the visual state of a scene (character action, surface material,
environment type), the system generates contextually appropriate sound effects without
requiring a pre-authored sound library for every possible interaction.

\subsection{Accessibility and Automatic Description}

For hearing-impaired audiences, Zen-Foley can generate audio descriptions of sound events
inferred from silent video, enabling automatic audio content accessibility metadata.
The PASCM classification output provides a natural language description of each detected
acoustic event (``footstep on stone floor,'' ``glass impact on ceramic tile'').

\subsection{Content Creation Workflows}

Zen-Foley integrates directly with the Zen-Video pipeline, enabling fully automated
audio-visual content generation: text prompt $\to$ Zen-Video $\to$ silent video $\to$
Zen-Foley $\to$ complete audio-visual content. Combined with Zen-Director for camera
specification, this forms a complete automated filmmaking pipeline within the Zen model family.

% =============================================================================
\section{Limitations}
% =============================================================================

\begin{itemize}
    \item \textbf{Dialogue and music}: Zen-Foley targets Foley sound effects exclusively.
          Speech synthesis and music generation are handled by separate models in the Zen family
          (Zen-Scribe and Zen-Musician). The pipeline does not yet seamlessly integrate all
          three audio layers.

    \item \textbf{Highly occluded events}: When the sound-producing contact is not visible
          (e.g., footsteps below frame, objects obscured behind others), the PASCM must
          infer the sound from contextual cues. In these cases, onset accuracy decreases to
          approximately 78\%.

    \item \textbf{Novel material combinations}: The 148-class taxonomy covers common
          material-interaction pairs but will assign the nearest-neighbor class for novel
          combinations. If a visually unusual material is encountered, the physical parameter
          predictions may be unreliable.

    \item \textbf{Stereo vs.\ surround}: Zen-Foley outputs stereo audio. Professional film
          post-production typically requires 5.1 or Dolby Atmos surround sound. A surround
          extension is planned for a future release.

    \item \textbf{Inference latency}: Generating 10 seconds of audio requires approximately
          3.8 seconds on an A100 with $S=50$ steps. This is 2.6$\times$ slower than real-time.
          Distillation to $S=4$ steps (in development) targets $<$ real-time inference.
\end{itemize}

% =============================================================================
\section{Conclusion}
% =============================================================================

We have presented Zen-Foley, a 690M-parameter model for visually-guided sound effect
synthesis. The architecture combines a hierarchical dual-stream video encoder, a physics-aware
sound categorization module with explicit acoustic parameter prediction, a conditional
spectrogram diffusion decoder, and an environmental acoustic simulation module. Trained on
38.4 million audio-visual pairs from AudioSet, VGGSound, and a licensed professional Foley
archive, the model achieves an FAD of 1.83, AVSS of 0.847, and onset accuracy of 94.2\%
on VGGSound --- state-of-the-art across all evaluated metrics.

A perceptual listening study with 32 professional Foley editors found that 76.1\% of
Zen-Foley outputs are rated broadcast-ready, with 41.3\% rated as indistinguishable from
studio Foley. This represents a substantial step toward automating the Foley workflow and
enabling new modalities of AI-assisted content creation.

The physics-aware acoustic parameterization is a key differentiator from prior work.
By encoding material-contact physics into the conditioning signal, the model gains access
to a strong inductive bias that improves both audio quality and temporal alignment, particularly
for impact and fluid sounds where physical constraints are most distinctive.

Future work will extend Zen-Foley to surround sound output, reduce inference latency via
consistency distillation, integrate dialogue and music synthesis for a complete audio pipeline,
and explore weakly supervised pre-training on internet video without curated audio-visual alignment.

% =============================================================================
\section*{Acknowledgments}
% =============================================================================

We thank our post-production studio partners for access to the Licensed Foley Archive,
the 32 professional Foley editors who participated in the perceptual evaluation, and the
Hanzo AI infrastructure team. The Greatest Hits dataset is gratefully acknowledged.
This work was supported by the Zoo Labs Foundation's Decentralized AI Research Initiative.

% =============================================================================
% REFERENCES
% =============================================================================

\begin{thebibliography}{99}

\bibitem[Allen \& Berkley(1979)]{allen1979image}
Allen, J.B. \& Berkley, D.A.
\newblock Image method for efficiently simulating small-room acoustics.
\newblock \emph{Journal of the Acoustical Society of America}, 65(4):943--950, 1979.

\bibitem[Arandjelovic \& Zisserman(2017)]{arandjelovic2017look}
Arandjelovi\'c, R. \& Zisserman, A.
\newblock Look, listen and learn.
\newblock \emph{Proceedings of ICCV}, 2017.

\bibitem[Chen et~al.(2020a)]{chen2020soundspaces}
Chen, C., Jain, U., Schissler, C., Gari, S.V.A., Al-Halah, Z., Ithapu, V.K., Robinson, P.,
\& Grauman, K.
\newblock SoundSpaces: Audio-visual navigation in 3D environments.
\newblock \emph{Proceedings of ECCV}, 2020.

\bibitem[Chen et~al.(2020b)]{chen2020vggsound}
Chen, H., Xie, W., Vedaldi, A., \& Zisserman, A.
\newblock VGGSound: A large-scale audio-visual dataset.
\newblock \emph{Proceedings of ICASSP}, 2020.

\bibitem[Chion(1994)]{chion1994audio}
Chion, M.
\newblock \emph{Audio-Vision: Sound on Screen}.
\newblock Columbia University Press, 1994.

\bibitem[Cook(2002)]{cook2002real}
Cook, P.R.
\newblock \emph{Real Sound Synthesis for Interactive Applications}.
\newblock A.K. Peters, 2002.

\bibitem[Feichtenhofer et~al.(2019)]{feichtenhofer2019slowfast}
Feichtenhofer, C., Fan, H., Malik, J., \& He, K.
\newblock SlowFast networks for video recognition.
\newblock \emph{Proceedings of ICCV}, 2019.

\bibitem[Gamper \& Tashev(2018)]{gamper2018blind}
Gamper, H. \& Tashev, I.J.
\newblock Blind reverberation time estimation using a convolutional neural network.
\newblock \emph{Proceedings of IWAENC}, 2018.

\bibitem[Gemmeke et~al.(2017)]{gemmeke2017audio}
Gemmeke, J.F., Ellis, D.P.W., Freedman, D., Jansen, A., Lawrence, W., Moore, R.C.,
Plakal, M., \& Ritter, M.
\newblock Audio Set: An ontology and human-labeled dataset for audio events.
\newblock \emph{Proceedings of ICASSP}, 2017.

\bibitem[Iashin \& Rahtu(2021a)]{iashin2021taming}
Iashin, V. \& Rahtu, E.
\newblock Taming visually guided sound generation.
\newblock \emph{Proceedings of BMVC}, 2021.

\bibitem[Iashin \& Rahtu(2021b)]{iashin2021specvqgan}
Iashin, V. \& Rahtu, E.
\newblock SpecVQGAN: Controlling structure of spectrogram generation via vector quantization.
\newblock \emph{arXiv preprint arXiv:2110.08791}, 2021.

\bibitem[Kerins(2010)]{kerins2010beyond}
Kerins, M.
\newblock \emph{Beyond Dolby (Stereo): Cinema in the Digital Sound Age}.
\newblock Indiana University Press, 2010.

\bibitem[Kilgour et~al.(2019)]{kilgour2019frechet}
Kilgour, K., Zuluaga, M., Roblek, D., \& Sharifi, M.
\newblock Fr\'echet audio distance: A reference-free metric for evaluating music enhancement algorithms.
\newblock \emph{Proceedings of Interspeech}, 2019.

\bibitem[Kong et~al.(2020)]{kong2020hifi}
Kong, J., Kim, J., \& Bae, J.
\newblock HiFi-GAN: Generative adversarial networks for efficient and high fidelity speech synthesis.
\newblock \emph{Advances in Neural Information Processing Systems}, 33, 2020.

\bibitem[Luo et~al.(2024)]{luo2024difffoley}
Luo, S., Yan, C., Hu, C., \& Zhao, H.
\newblock Diff-Foley: Synchronized video-to-audio synthesis with latent diffusion models.
\newblock \emph{Advances in Neural Information Processing Systems}, 36, 2024.

\bibitem[Owens et~al.(2016a)]{owens2016visually}
Owens, A., Isola, P., McDermott, J., Torralba, A., Adelson, E.H., \& Freeman, W.T.
\newblock Visually indicated sounds.
\newblock \emph{Proceedings of CVPR}, 2016.

\bibitem[Owens et~al.(2016b)]{owens2016ambient}
Owens, A., Wu, J., McDermott, J., Freeman, W.T., \& Torralba, A.
\newblock Ambient sound provides supervision for visual learning.
\newblock \emph{Proceedings of ECCV}, 2016.

\bibitem[P\'erez et~al.(2018)]{perez2018film}
P\'erez, E., Strub, F., de Vries, H., Dumoulin, V., \& Courville, A.
\newblock FiLM: Visual reasoning with a general conditioning layer.
\newblock \emph{Proceedings of AAAI}, 2018.

\bibitem[Ranftl et~al.(2021)]{ranftl2021vision}
Ranftl, R., Bochkovskiy, A., \& Koltun, V.
\newblock Vision transformers for dense prediction.
\newblock \emph{Proceedings of ICCV}, 2021.

\bibitem[Sheffer \& Adi(2023)]{sheffer2023hear}
Sheffer, R. \& Adi, Y.
\newblock I hear your true colors: Image guided audio generation.
\newblock \emph{Proceedings of ICASSP}, 2023.

\bibitem[Venakopoulos \& Georgakopoulos(2011)]{venakopoulos2011friction}
Venakopoulos, D. \& Georgakopoulos, D.
\newblock Physical modeling of friction sounds using nonlinear dynamical systems.
\newblock \emph{Proceedings of DAFX}, 2011.

\bibitem[Wang et~al.(2018)]{wang2023videomae}
Wang, C., Zhang, X., Zhu, C., \& Shao, L.
\newblock VideoMAE V2: Scaling video masked autoencoders with dual masking.
\newblock \emph{Proceedings of CVPR}, 2023.

\bibitem[Zhou et~al.(2018a)]{zhou2018visual}
Zhou, Y., Wang, Z., Fang, C., Bui, T., \& Berg, T.L.
\newblock Visual to sound: Generating natural sound for videos in the wild.
\newblock \emph{Proceedings of CVPR}, 2018.

\bibitem[Zhou et~al.(2018b)]{zhou2018places}
Zhou, B., Lapedriza, A., Khosla, A., Oliva, A., \& Torralba, A.
\newblock Places: A 10 million image database for scene recognition.
\newblock \emph{IEEE Transactions on Pattern Analysis and Machine Intelligence}, 40(6):1452--1464, 2018.

\bibitem[Zhang et~al.(2024)]{zhang2024foleycrafter}
Zhang, Y., Yang, M., Su, H., Zheng, J., Pan, J., Xu, C., \& Yang, J.
\newblock FoleyCrafter: Bring silent videos to life with lifelike and synchronized sounds.
\newblock \emph{arXiv preprint arXiv:2407.01494}, 2024.

\end{thebibliography}

\end{document}