mirror of
https://github.com/zenlm/vjepa2.git
synced 2026-10-30 19:33:29 +00:00
Co-authored-by: Nicolas Ballas <[email protected]> Co-authored-by: Adrien Bardes <[email protected]> Co-authored-by: David Fan <[email protected]> Co-authored-by: Quentin Garrido <[email protected]> Co-authored-by: Russell Howes <[email protected]> Co-authored-by: Mojtaba Komeili <[email protected]> Co-authored-by: Matthew Muckley <[email protected]> Co-authored-by: Koustuv Sinha <[email protected]> Co-authored-by: Artem Zholus <[email protected]>
76 lines
2.3 KiB
Python
76 lines
2.3 KiB
Python
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
|
#
|
|
# This source code is licensed under the license found in the
|
|
# LICENSE file in the root directory of this source tree.
|
|
|
|
import numpy as np
|
|
import torch.nn.functional as F
|
|
|
|
from .mpc_utils import cem, compute_new_pose
|
|
|
|
|
|
class WorldModel(object):
|
|
|
|
def __init__(
|
|
self,
|
|
encoder,
|
|
predictor,
|
|
tokens_per_frame,
|
|
transform,
|
|
mpc_args={
|
|
"rollout": 2,
|
|
"samples": 400,
|
|
"topk": 10,
|
|
"cem_steps": 10,
|
|
"momentum_mean": 0.15,
|
|
"momentum_std": 0.15,
|
|
"maxnorm": 0.05,
|
|
"verbose": True,
|
|
},
|
|
normalize_reps=True,
|
|
device="cuda:0",
|
|
):
|
|
super().__init__()
|
|
self.encoder = encoder
|
|
self.predictor = predictor
|
|
self.normalize_reps = normalize_reps
|
|
self.transform = transform
|
|
self.tokens_per_frame = tokens_per_frame
|
|
self.device = device
|
|
self.mpc_args = mpc_args
|
|
|
|
def encode(self, image):
|
|
clip = np.expand_dims(image, axis=0)
|
|
clip = self.transform(clip)[None, :]
|
|
B, C, T, H, W = clip.size()
|
|
clip = clip.permute(0, 2, 1, 3, 4).flatten(0, 1).unsqueeze(2).repeat(1, 1, 2, 1, 1)
|
|
clip = clip.to(self.device, non_blocking=True)
|
|
h = self.encoder(clip)
|
|
h = h.view(B, T, -1, h.size(-1)).flatten(1, 2)
|
|
if self.normalize_reps:
|
|
h = F.layer_norm(h, (h.size(-1),))
|
|
return h
|
|
|
|
def infer_next_action(self, rep, pose, goal_rep, close_gripper=None):
|
|
|
|
def step_predictor(reps, actions, poses):
|
|
B, T, N_T, D = reps.size()
|
|
reps = reps.flatten(1, 2)
|
|
next_rep = self.predictor(reps, actions, poses)[:, -self.tokens_per_frame :]
|
|
if self.normalize_reps:
|
|
next_rep = F.layer_norm(next_rep, (next_rep.size(-1),))
|
|
next_rep = next_rep.view(B, 1, N_T, D)
|
|
next_pose = compute_new_pose(poses[:, -1:], actions[:, -1:])
|
|
return next_rep, next_pose
|
|
|
|
mpc_action = cem(
|
|
context_frame=rep,
|
|
context_pose=pose,
|
|
goal_frame=goal_rep,
|
|
world_model=step_predictor,
|
|
close_gripper=close_gripper,
|
|
**self.mpc_args,
|
|
)[0]
|
|
|
|
return mpc_action
|