mirror of
https://github.com/zenlm/vjepa2.git
synced 2026-10-30 19:33:29 +00:00
Co-authored-by: Nicolas Ballas <[email protected]> Co-authored-by: Adrien Bardes <[email protected]> Co-authored-by: David Fan <[email protected]> Co-authored-by: Quentin Garrido <[email protected]> Co-authored-by: Russell Howes <[email protected]> Co-authored-by: Mojtaba Komeili <[email protected]> Co-authored-by: Matthew Muckley <[email protected]> Co-authored-by: Koustuv Sinha <[email protected]> Co-authored-by: Artem Zholus <[email protected]>
84 lines
3.0 KiB
Python
84 lines
3.0 KiB
Python
"""
|
|
Copyright (c) Meta Platforms, Inc. and affiliates.
|
|
|
|
This source code is licensed under the MIT license found in the
|
|
LICENSE file in the root directory of this source tree.
|
|
------------------------------------------------------------------------------
|
|
|
|
modelcustom API requirements:
|
|
|
|
API requirements for Encoder module:
|
|
1) Needs to be a pytorch module with 'forward()' function protocol:
|
|
:param x: (Tensor) Video clip (shape=[batch_size x num_channels x num_frames x height x width])
|
|
:returns: (Tensor) Representations of video clip (shape=[batch_size x num_encoder_tokens x feature_dim])
|
|
2) Needs to have a public attribute called 'embed_dim' (int) describing its
|
|
output feature dimension.
|
|
|
|
API requirements for Predictor module:
|
|
1) Needs to be a pytorch module with 'forward()' function protocol:
|
|
:param x: (Tensor) Video clip tokens (shape=[batch_size x num_encoder_tokens x feature_dim])
|
|
:param anticipation_time: (Tensor) Seconds into the future to predict for each sample in batch
|
|
(shape=[batch_size])
|
|
:returns: (Tensor) Representations of future frames (shape=[batch_size x num_output_tokens x feature_dim])
|
|
2) Needs to have a public attribute called 'embed_dim' (int) describing its
|
|
output feature dimension.
|
|
"""
|
|
|
|
import logging
|
|
|
|
import torch
|
|
|
|
import src.models.vision_transformer as vit
|
|
|
|
logging.basicConfig()
|
|
logger = logging.getLogger()
|
|
logger.setLevel(logging.INFO)
|
|
|
|
|
|
def init_module(
|
|
resolution: int,
|
|
checkpoint: str,
|
|
# --
|
|
model_kwargs: dict,
|
|
wrapper_kwargs: dict,
|
|
**kwargs,
|
|
):
|
|
logger.info(f"Loading pretrained model from {checkpoint=}")
|
|
checkpoint = torch.load(checkpoint, map_location="cpu")
|
|
|
|
img_as_video_nframes = wrapper_kwargs.get("img_as_video_nframes")
|
|
# --
|
|
enc_kwargs = model_kwargs["encoder"]
|
|
enc_ckp_key = enc_kwargs.get("checkpoint_key")
|
|
enc_model_name = enc_kwargs.get("model_name")
|
|
|
|
model = vit.__dict__[enc_model_name](
|
|
input_size=resolution,
|
|
num_frames=img_as_video_nframes,
|
|
**enc_kwargs,
|
|
)
|
|
|
|
def forward_prehook(module, input):
|
|
input = input[0] # [B, C, H, W]
|
|
input = input.unsqueeze(2).repeat(1, 1, img_as_video_nframes, 1, 1)
|
|
return input
|
|
|
|
model.register_forward_pre_hook(forward_prehook)
|
|
|
|
pretrained_dict = checkpoint[enc_ckp_key]
|
|
# --
|
|
pretrained_dict = {k.replace("module.", ""): v for k, v in pretrained_dict.items()}
|
|
pretrained_dict = {k.replace("backbone.", ""): v for k, v in pretrained_dict.items()}
|
|
for k, v in model.state_dict().items():
|
|
if k not in pretrained_dict:
|
|
logger.info(f'key "{k}" could not be found in loaded state dict')
|
|
elif pretrained_dict[k].shape != v.shape:
|
|
logger.info(f'key "{k}" is of different shape in model and loaded state dict')
|
|
pretrained_dict[k] = v
|
|
msg = model.load_state_dict(pretrained_dict, strict=False)
|
|
logger.info(f"loaded pretrained model with msg: {msg}")
|
|
print(model)
|
|
|
|
del checkpoint
|
|
return model
|