tangger/openpi
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

debugging with open loop inference and replay

Browse Source
This commit is contained in:
pi4
2025-01-03 04:41:43 -08:00
parent aa669f6cdc
commit d536f68daf
2 changed files with 28 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
examples/ur_sim/env.py
View File

@@ -23,8 +23,8 @@ sys.argv = [sys.argv[0]] + other_args # clear out sys.argv for hydra
# launch omniverse app # launch omniverse app
args_cli.enable_cameras = True args_cli.enable_cameras = True
# args_cli.headless = True args_cli.headless = True
args_cli.headless = False # args_cli.headless = False
app_launcher = AppLauncher(args_cli) app_launcher = AppLauncher(args_cli)
simulation_app = app_launcher.app simulation_app = app_launcher.app
@@ -37,6 +37,8 @@ import torch
import gymnasium import gymnasium
import numpy as np import numpy as np
np.set_printoptions(precision=3)
from pathlib import Path from pathlib import Path
from openpi_client.runtime import environment as _environment from openpi_client.runtime import environment as _environment
from typing_extensions import override from typing_extensions import override
@@ -70,8 +72,9 @@ class URSimEnvironment(_environment.Environment):
) )
sim_assets = { sim_assets = {
"pi_scene_v2_static": DATA_PATH/"pi_scene_v2", # "pi_scene_v2_static": DATA_PATH/"pi_scene_v2",
"bottle": DATA_PATH/"pi_objects/bottle", "pi_scene_v5_static": DATA_PATH/"pi_scene_v5",
# "bottle": DATA_PATH/"pi_objects/bottle",
"plate": DATA_PATH/"pi_objects/plate", "plate": DATA_PATH/"pi_objects/plate",
"robot": DATA_PATH/"pi_robot/", "robot": DATA_PATH/"pi_robot/",
} }
@@ -115,6 +118,7 @@ class URSimEnvironment(_environment.Environment):
# scale gripper from [0,1] to [-1,1] # scale gripper from [0,1] to [-1,1]
action = action.copy() action = action.copy()
action[-1] = action[-1] * 2 - 1 action[-1] = action[-1] * 2 - 1
print(action)
##### #####
# action = np.zeros(7) # action = np.zeros(7)
@@ -132,8 +136,15 @@ class URSimEnvironment(_environment.Environment):
img1 = self._last_obs["observation/base_0_camera/rgb/image"] img1 = self._last_obs["observation/base_0_camera/rgb/image"]
img2 = self._last_obs["observation/wrist_0_camera/rgb/image"] img2 = self._last_obs["observation/wrist_0_camera/rgb/image"]
# img3 = self._last_obs["base"][0]
# img4 = self._last_obs["wrist"][0]
img3 = self._last_obs["_observation/base_0_camera/rgb/image"]
img4 = self._last_obs["_observation/wrist_0_camera/rgb/image"]
# big_img = np.concatenate([img1, img2, img3, img4], axis=1)
big_img = np.concatenate([img1, img2], axis=1) big_img = np.concatenate([img1, img2], axis=1)
splats = np.concatenate([img3, img4], axis=1)
cv2.imshow("big_img", cv2.cvtColor(big_img, cv2.COLOR_RGB2BGR)) cv2.imshow("big_img", cv2.cvtColor(big_img, cv2.COLOR_RGB2BGR))
cv2.imshow("splat", cv2.cvtColor(splats, cv2.COLOR_RGB2BGR))
cv2.waitKey(1) cv2.waitKey(1)
self.step += 1 self.step += 1
@@ -148,12 +159,13 @@ class URSimEnvironment(_environment.Environment):
data["observation/base_0_camera/rgb/image"] = gym_obs["splat"]["base_cam"] data["observation/base_0_camera/rgb/image"] = gym_obs["splat"]["base_cam"]
data["observation/wrist_0_camera/rgb/image"] = gym_obs["splat"]["wrist_cam"] data["observation/wrist_0_camera/rgb/image"] = gym_obs["splat"]["wrist_cam"]
# data["observation/base_0_camera/rgb/image"] = (self.file["observation/base_0_camera/rgb/image_224_224"][self.step]) data["base"] = self._gym.get_image(camera_name="base_cam")
# data["observation/wrist_0_camera/rgb/image"] = (self.file["observation/wrist_0_camera/rgb/image_224_224"][self.step]) data["wrist"] = self._gym.get_image(camera_name="wrist_cam")
# data["observation/base_0_camera/rgb/image"] = (self.file["observation/base_0_camera/rgb/image_256_320"][self.step])
# data["observation/wrist_0_camera/rgb/image"] = (self.file["observation/wrist_0_camera/rgb/image_256_320"][self.step]) data["_observation/base_0_camera/rgb/image"] = (self.file["observation/base_0_camera/rgb/image_224_224"][self.step])
# data["observation/ur5e/joints/position"] = self.file["observation/ur5e/joints/position"][self.step] data["_observation/wrist_0_camera/rgb/image"] = (self.file["observation/wrist_0_camera/rgb/image_224_224"][self.step])
# data["observation/robotiq_gripper/gripper/position"] = self.file["observation/robotiq_gripper/gripper/position"][self.step] data["_observation/ur5e/joints/position"] = self.file["observation/ur5e/joints/position"][self.step]
data["_observation/robotiq_gripper/gripper/position"] = self.file["observation/robotiq_gripper/gripper/position"][self.step]
# #
# print(data["observation/ur5e/joints/position"]) # print(data["observation/ur5e/joints/position"])

7
examples/ur_sim/saver.py
View File

@@ -24,7 +24,12 @@ class VideoSaver(_subscriber.Subscriber):
def on_step(self, observation: dict, action: dict) -> None: def on_step(self, observation: dict, action: dict) -> None:
img1 = observation["observation/base_0_camera/rgb/image"] img1 = observation["observation/base_0_camera/rgb/image"]
img2 = observation["observation/wrist_0_camera/rgb/image"] img2 = observation["observation/wrist_0_camera/rgb/image"]
big_img = np.concatenate([img1, img2], axis=1) # img1 = observation["observation/base_0_camera/rgb/image"]
# img2 = observation["observation/wrist_0_camera/rgb/image"]
# img3 = observation["base"][0]
# img4 = observation["wrist"][0]
big_img = np.concatenate([img1, img2,], axis=1)
self._images.append(big_img) self._images.append(big_img)
# im = observation["image"][0] # [C, H, W] # im = observation["image"][0] # [C, H, W]
# im = np.transpose(im, (1, 2, 0)) # [H, W, C] # im = np.transpose(im, (1, 2, 0)) # [H, W, C]