From af6bca72706139c81caf3026d1295b771ee22fa3 Mon Sep 17 00:00:00 2001 From: hangX Date: Sun, 15 Mar 2026 20:23:21 +0800 Subject: [PATCH] =?UTF-8?q?=E5=A4=9Aview?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../teleoperation/teleop_se3_agent.py | 57 +++++++++++++++++++ .../teleoperation/teleop_xr_agent.py | 51 +++++++++++++++++ .../mindrobot_left_arm_ik_env_cfg.py | 44 +++++++++++++- 3 files changed, 150 insertions(+), 2 deletions(-) diff --git a/scripts/environments/teleoperation/teleop_se3_agent.py b/scripts/environments/teleoperation/teleop_se3_agent.py index 26776ec..6a86441 100644 --- a/scripts/environments/teleoperation/teleop_se3_agent.py +++ b/scripts/environments/teleoperation/teleop_se3_agent.py @@ -92,6 +92,45 @@ if args_cli.enable_pinocchio: import isaaclab_tasks.manager_based.locomanipulation.pick_place # noqa: F401 import isaaclab_tasks.manager_based.manipulation.pick_place # noqa: F401 + +# ===================================================================== +# Robot Camera Viewport Utilities (pure GUI — no --enable_cameras needed) +# ===================================================================== + +# Resolved prim paths for env_0. +# These match the camera prims embedded in mindrobot_cd2.usd. +# Run with the diagnostic block once to confirm exact paths. +_ROBOT_CAM_PRIMS: dict[str, str] = { + "Left Hand": "/World/envs/env_0/Robot/rm_65_fb_left/Link_6/robot___left_8_02/cam_left_hand", + "Right Hand": "/World/envs/env_0/Robot/rm_65_b_right/Link6/robot___right_8_2/cam_right_hand", + "Head": "/World/envs/env_0/Robot/robot_head/cam_head", + "Chest": "/World/envs/env_0/Robot/robot_trunk/cam_chest", +} + +_robot_viewports: dict[str, object] = {} + + +def create_robot_viewports() -> None: + """Create 4 viewport windows bound to robot camera prims (GUI only).""" + try: + import omni.kit.viewport.utility as vp_util + except ImportError: + logger.warning("[Viewport] omni.kit.viewport.utility not available.") + return + for name, cam_path in _ROBOT_CAM_PRIMS.items(): + vp_win = vp_util.create_viewport_window( + f"Robot {name} View", width=640, height=360 + ) + vp_win.viewport_api.camera_path = cam_path + _robot_viewports[name] = vp_win + print(f"[INFO] Viewport 'Robot {name} View' → {cam_path}") + + +def rebind_robot_viewports() -> None: + """Re-bind viewports to camera paths after env reset.""" + for name, vp_win in _robot_viewports.items(): + vp_win.viewport_api.camera_path = _ROBOT_CAM_PRIMS[name] + # import logger logger = logging.getLogger(__name__) @@ -274,6 +313,23 @@ def main() -> None: env.reset() teleop_interface.reset() + # -- Diagnostic: print all Camera prims in the stage -- + try: + import omni.usd + from pxr import UsdGeom + stage = omni.usd.get_context().get_stage() + print("\n[DIAG] All Camera prims in stage:") + for prim in stage.Traverse(): + if UsdGeom.Camera(prim): + print(f" {prim.GetPath()}") + print("[DIAG] Done.\n") + except Exception as _diag_e: + print(f"[DIAG] Could not traverse stage: {_diag_e}") + # -- End diagnostic -- + + # Create viewport windows bound to robot cameras (no --enable_cameras needed) + create_robot_viewports() + print("Teleoperation started. Press 'R' to reset the environment.") # simulate environment @@ -296,6 +352,7 @@ def main() -> None: if should_reset_recording_instance: env.reset() teleop_interface.reset() + rebind_robot_viewports() should_reset_recording_instance = False print("Environment reset complete") except Exception as e: diff --git a/scripts/environments/teleoperation/teleop_xr_agent.py b/scripts/environments/teleoperation/teleop_xr_agent.py index deaede2..52ad8aa 100644 --- a/scripts/environments/teleoperation/teleop_xr_agent.py +++ b/scripts/environments/teleoperation/teleop_xr_agent.py @@ -82,6 +82,53 @@ from xr_utils import XrClient, transform_xr_pose, quat_diff_as_rotvec_xyzw, is_v from xr_utils.geometry import R_HEADSET_TO_WORLD +# ===================================================================== +# Robot Camera Viewport Utilities +# ===================================================================== + +# Resolved prim paths for env_0 — must match CameraCfg prim_path with +# {ENV_REGEX_NS} → /World/envs/env_0 +_ROBOT_CAM_PRIMS: dict[str, str] = { + "Left Hand": "/World/envs/env_0/Robot/rm_65_fb_left/Link_6/robot___left_8_02/cam_left_hand", + "Right Hand": "/World/envs/env_0/Robot/rm_65_b_right/Link6/robot___right_8_2/cam_right_hand", + "Head": "/World/envs/env_0/Robot/robot_head/cam_head", + "Chest": "/World/envs/env_0/Robot/robot_trunk/cam_chest", +} + +# Stores created viewport window objects +_robot_viewports: dict[str, object] = {} + + +def create_robot_viewports() -> None: + """Create 4 viewport windows and bind each to a robot camera prim. + + Must be called after env.reset() so all prims exist on the USD stage. + """ + try: + import omni.kit.viewport.utility as vp_util + except ImportError: + logger.warning("[Viewport] omni.kit.viewport.utility not available — skipping viewport creation.") + return + + for name, cam_path in _ROBOT_CAM_PRIMS.items(): + vp_win = vp_util.create_viewport_window( + f"Robot {name} View", width=640, height=360 + ) + vp_win.viewport_api.camera_path = cam_path + _robot_viewports[name] = vp_win + print(f"[INFO] Viewport 'Robot {name} View' bound to: {cam_path}") + + +def rebind_robot_viewports() -> None: + """Re-bind all robot viewports to their camera paths. + + Call this after every env reset so viewports stay locked to the cameras. + """ + for name, vp_win in _robot_viewports.items(): + cam_path = _ROBOT_CAM_PRIMS[name] + vp_win.viewport_api.camera_path = cam_path + + # ===================================================================== # Teleoperation Interface for XR # ===================================================================== @@ -430,6 +477,9 @@ def main() -> None: clear_ik_target_state() teleop_interface.reset() + # Create 4 viewport windows bound to the robot cameras + create_robot_viewports() + print("\n" + "=" * 50) print(" 🚀 Teleoperation Started!") print(" 🎮 Use the TRIGGER to open/close gripper.") @@ -450,6 +500,7 @@ def main() -> None: obs, _ = env.reset() clear_ik_target_state() teleop_interface.reset() + rebind_robot_viewports() should_reset = False sim_frame = 0 continue diff --git a/source/mindbot/mindbot/tasks/manager_based/il/open_drybox/mindrobot_left_arm_ik_env_cfg.py b/source/mindbot/mindbot/tasks/manager_based/il/open_drybox/mindrobot_left_arm_ik_env_cfg.py index a3faa03..53d4432 100644 --- a/source/mindbot/mindbot/tasks/manager_based/il/open_drybox/mindrobot_left_arm_ik_env_cfg.py +++ b/source/mindbot/mindbot/tasks/manager_based/il/open_drybox/mindrobot_left_arm_ik_env_cfg.py @@ -32,7 +32,8 @@ from isaaclab.sim.spawners.from_files.from_files_cfg import GroundPlaneCfg from isaaclab.utils import configclass from isaaclab.markers.config import FRAME_MARKER_CFG from isaaclab.devices.openxr import XrCfg - +from isaaclab.sensors import CameraCfg +from isaaclab.envs import ViewerCfg # 导入基础配置和MDP函数 from isaaclab_tasks.manager_based.manipulation.stack import mdp @@ -101,7 +102,41 @@ class MindRobotTeleopSceneCfg(InteractiveSceneCfg): robot: ArticulationCfg = MINDBOT_HIGH_PD_CFG.replace( prim_path="{ENV_REGEX_NS}/Robot" ) - + # Cameras — CameraCfg sensors are only needed when reading pixel data (e.g. IL data collection). + # For viewport display only, use omni.kit.viewport.utility to bind the camera prim directly. + # Uncomment below AND add --enable_cameras flag only when sensor data is required. + # left_hand_camera = CameraCfg( + # prim_path="{ENV_REGEX_NS}/Robot/rm_65_fb_left/Link_6/robot___left_8_02/cam_left_hand", + # update_period=1 / 30, + # height=480, + # width=640, + # data_types=["rgb"], + # spawn=None, + # ) + # right_hand_camera = CameraCfg( + # prim_path="{ENV_REGEX_NS}/Robot/rm_65_b_right/Link6/robot___right_8_2/cam_right_hand", + # update_period=1 / 30, + # height=480, + # width=640, + # data_types=["rgb"], + # spawn=None, + # ) + # head_camera = CameraCfg( + # prim_path="{ENV_REGEX_NS}/Robot/robot_head/cam_head", + # update_period=1 / 30, + # height=480, + # width=640, + # data_types=["rgb"], + # spawn=None, + # ) + # chest_camera = CameraCfg( + # prim_path="{ENV_REGEX_NS}/Robot/robot_trunk/cam_chest", + # update_period=1 / 30, + # height=480, + # width=640, + # data_types=["rgb"], + # spawn=None, + # ) # Lighting light = AssetBaseCfg( prim_path="/World/light", @@ -294,6 +329,11 @@ class MindRobotLeftArmIKEnvCfg(ManagerBasedRLEnvCfg): """Post initialization.""" super().__post_init__() + self.viewer = ViewerCfg( + eye=(2.0, -1.5, 1.8), # 相机位置 + lookat=(0.0, 0.0, 0.8), # 瞄准点（机器人腰部高度附近） + origin_type="world", + ) # General settings self.decimation = 2 self.episode_length_s = 50.0