mirror of
https://github.com/zenlm/vjepa2.git
synced 2026-10-30 19:33:29 +00:00
* fix: correct CSV logger column headers to match logged data (#43) - Change 'loss' column to 'train_acc' - Change 'acc' column to 'val_acc' - Headers now accurately reflect train_acc and val_acc values being logged * fixing various typos in diverse texts (#40) * Comments addressing PR #15 and Issue #65 (#75) * Comments addressing PR #15 and Issue #65 * Link to PR #15 * fixing miscellaneous typos in different files (#77) * fixing typos in 3 files (#78) * Bump black from 24.4.2 to 26.3.1 (#127) Bumps [black](https://github.com/psf/black) from 24.4.2 to 26.3.1. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md) - [Commits](https://github.com/psf/black/compare/24.4.2...26.3.1) --- updated-dependencies: - dependency-name: black dependency-version: 26.3.1 dependency-type: direct:production ... Signed-off-by: dependabot[bot] <[email protected]> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * V-JEPA 2.1 (#130) * V-JEPA 2.1 Initial commit * Update arxiv * Update configs * Fix eval paths * Fix teaser (#133) * Clean Teaser * Clean Teaser * Fix figure (#143) --------- Signed-off-by: dependabot[bot] <[email protected]> Co-authored-by: KK <[email protected]> Co-authored-by: Didier Durand <[email protected]> Co-authored-by: Russell Howes <[email protected]> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Adrien Bardes <[email protected]>
150 lines
5.6 KiB
Python
150 lines
5.6 KiB
Python
"""
|
|
Copyright (c) Meta Platforms, Inc. and affiliates.
|
|
|
|
This source code is licensed under the MIT license found in the
|
|
LICENSE file in the root directory of this source tree.
|
|
------------------------------------------------------------------------------
|
|
|
|
modelcustom API requirements:
|
|
|
|
API requirements for Encoder module:
|
|
1) Needs to be a pytorch module with 'forward()' function protocol:
|
|
:param x: (Tensor) Video clip (shape=[batch_size x num_channels x num_frames x height x width])
|
|
:returns: (Tensor) Representations of video clip (shape=[batch_size x num_encoder_tokens x feature_dim])
|
|
2) Needs to have a public attribute called 'embed_dim' (int) describing its
|
|
output feature dimension.
|
|
|
|
API requirements for Predictor module:
|
|
1) Needs to be a pytorch module with 'forward()' function protocol:
|
|
:param x: (Tensor) Video clip tokens (shape=[batch_size x num_encoder_tokens x feature_dim])
|
|
:param anticipation_time: (Tensor) Seconds into the future to predict for each sample in batch
|
|
(shape=[batch_size])
|
|
:returns: (Tensor) Representations of future frames (shape=[batch_size x num_output_tokens x feature_dim])
|
|
2) Needs to have a public attribute called 'embed_dim' (int) describing its
|
|
output feature dimension.
|
|
"""
|
|
|
|
import logging
|
|
|
|
import torch
|
|
import torch.nn as nn
|
|
|
|
import src.models.vision_transformer as vit
|
|
from src.masks.utils import apply_masks
|
|
from src.models.utils.pos_embs import get_1d_sincos_pos_embed
|
|
|
|
logging.basicConfig()
|
|
logger = logging.getLogger()
|
|
logger.setLevel(logging.INFO)
|
|
|
|
|
|
def init_module(
|
|
resolution: int,
|
|
frames_per_clip: int,
|
|
checkpoint: str,
|
|
# --
|
|
model_kwargs: dict,
|
|
wrapper_kwargs: dict,
|
|
):
|
|
logger.info(f"Loading pretrained model from {checkpoint}")
|
|
checkpoint = torch.load(checkpoint, map_location="cpu")
|
|
|
|
enc_kwargs = model_kwargs["encoder"]
|
|
enc_ckp_key = enc_kwargs.get("checkpoint_key")
|
|
enc_model_name = enc_kwargs.get("model_name")
|
|
|
|
model = vit.__dict__[enc_model_name](img_size=resolution, num_frames=frames_per_clip, **enc_kwargs)
|
|
|
|
pretrained_dict = checkpoint[enc_ckp_key]
|
|
# --
|
|
pretrained_dict = {k.replace("module.", ""): v for k, v in pretrained_dict.items()}
|
|
pretrained_dict = {k.replace("backbone.", ""): v for k, v in pretrained_dict.items()}
|
|
for k, v in model.state_dict().items():
|
|
if k not in pretrained_dict:
|
|
logger.info(f'key "{k}" could not be found in loaded state dict')
|
|
elif pretrained_dict[k].shape != v.shape:
|
|
logger.info(f'key "{k}" is of different shape in model and loaded state dict')
|
|
pretrained_dict[k] = v
|
|
msg = model.load_state_dict(pretrained_dict, strict=False)
|
|
logger.info(f"loaded pretrained model with msg: {msg}")
|
|
print(model)
|
|
|
|
model = ClipAggregation(
|
|
model,
|
|
tubelet_size=model.tubelet_size,
|
|
**wrapper_kwargs,
|
|
)
|
|
del checkpoint
|
|
return model
|
|
|
|
|
|
class ClipAggregation(nn.Module):
|
|
"""
|
|
Process each clip independently and concatenate all tokens
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
model,
|
|
tubelet_size=2,
|
|
max_frames=128,
|
|
use_pos_embed=False,
|
|
):
|
|
super().__init__()
|
|
self.model = model
|
|
self.tubelet_size = tubelet_size
|
|
self.embed_dim = embed_dim = model.embed_dim
|
|
self.num_heads = model.num_heads
|
|
|
|
# 1D-temporal pos-embedding
|
|
self.pos_embed = None
|
|
if use_pos_embed:
|
|
max_T = max_frames // tubelet_size
|
|
self.pos_embed = nn.Parameter(torch.zeros(1, max_T, embed_dim), requires_grad=False)
|
|
sincos = get_1d_sincos_pos_embed(embed_dim, max_T)
|
|
self.pos_embed.copy_(torch.from_numpy(sincos).float().unsqueeze(0))
|
|
|
|
def forward(self, x, clip_indices=None):
|
|
|
|
num_clips = len(x)
|
|
num_views_per_clip = len(x[0])
|
|
B, C, F, H, W = x[0][0].size()
|
|
|
|
# Concatenate all spatial and temporal views along batch dimension
|
|
x = [torch.cat(xi, dim=0) for xi in x]
|
|
x = torch.cat(x, dim=0)
|
|
|
|
outputs = self.model(x)
|
|
|
|
def multiviews_postprocess(outputs):
|
|
_, N, D = outputs.size()
|
|
T = F // self.tubelet_size # num temporal indices
|
|
S = N // T # num spatial tokens
|
|
|
|
# Unroll outputs into a 2D array [spatial_views x temporal_views]
|
|
eff_B = B * num_views_per_clip
|
|
all_outputs = [[] for _ in range(num_views_per_clip)]
|
|
for i in range(num_clips):
|
|
o = outputs[i * eff_B : (i + 1) * eff_B]
|
|
for j in range(num_views_per_clip):
|
|
all_outputs[j].append(o[j * B : (j + 1) * B])
|
|
|
|
for i, outputs in enumerate(all_outputs):
|
|
# Concatenate along temporal dimension
|
|
outputs = [o.reshape(B, T, S, D) for o in outputs]
|
|
outputs = torch.cat(outputs, dim=1).flatten(1, 2)
|
|
# Compute positional embedding
|
|
if (self.pos_embed is not None) and (clip_indices is not None):
|
|
_indices = [c[:, :: self.tubelet_size] for c in clip_indices]
|
|
pos_embed = self.pos_embed.repeat(B, 1, 1) # [B, max_T, D]
|
|
pos_embed = apply_masks(pos_embed, _indices, concat=False) # list(Tensor([B, T, D]))
|
|
pos_embed = torch.cat(pos_embed, dim=1) # concatenate along temporal dimension
|
|
pos_embed = pos_embed.unsqueeze(2).repeat(1, 1, S, 1) # [B, T*num_clips, S, D]
|
|
pos_embed = pos_embed.flatten(1, 2)
|
|
outputs += pos_embed
|
|
all_outputs[i] = outputs
|
|
|
|
return all_outputs
|
|
|
|
return multiviews_postprocess(outputs)
|