多view

scripts/environments/teleoperation/teleop_se3_agent.py

@@ -92,6 +92,45 @@ if args_cli.enable_pinocchio:
     import isaaclab_tasks.manager_based.locomanipulation.pick_place  # noqa: F401
     import isaaclab_tasks.manager_based.manipulation.pick_place  # noqa: F401
 
+
+# =====================================================================
+# Robot Camera Viewport Utilities  (pure GUI — no --enable_cameras needed)
+# =====================================================================
+
+# Resolved prim paths for env_0.
+# These match the camera prims embedded in mindrobot_cd2.usd.
+# Run with the diagnostic block once to confirm exact paths.
+_ROBOT_CAM_PRIMS: dict[str, str] = {
+    "Left Hand":  "/World/envs/env_0/Robot/rm_65_fb_left/Link_6/robot___left_8_02/cam_left_hand",
+    "Right Hand": "/World/envs/env_0/Robot/rm_65_b_right/Link6/robot___right_8_2/cam_right_hand",
+    "Head":       "/World/envs/env_0/Robot/robot_head/cam_head",
+    "Chest":      "/World/envs/env_0/Robot/robot_trunk/cam_chest",
+}
+
+_robot_viewports: dict[str, object] = {}
+
+
+def create_robot_viewports() -> None:
+    """Create 4 viewport windows bound to robot camera prims (GUI only)."""
+    try:
+        import omni.kit.viewport.utility as vp_util
+    except ImportError:
+        logger.warning("[Viewport] omni.kit.viewport.utility not available.")
+        return
+    for name, cam_path in _ROBOT_CAM_PRIMS.items():
+        vp_win = vp_util.create_viewport_window(
+            f"Robot {name} View", width=640, height=360
+        )
+        vp_win.viewport_api.camera_path = cam_path
+        _robot_viewports[name] = vp_win
+        print(f"[INFO] Viewport 'Robot {name} View' → {cam_path}")
+
+
+def rebind_robot_viewports() -> None:
+    """Re-bind viewports to camera paths after env reset."""
+    for name, vp_win in _robot_viewports.items():
+        vp_win.viewport_api.camera_path = _ROBOT_CAM_PRIMS[name]
+
 # import logger
 logger = logging.getLogger(__name__)
 
@@ -274,6 +313,23 @@ def main() -> None:
     env.reset()
     teleop_interface.reset()
 
+    # -- Diagnostic: print all Camera prims in the stage --
+    try:
+        import omni.usd
+        from pxr import UsdGeom
+        stage = omni.usd.get_context().get_stage()
+        print("\n[DIAG] All Camera prims in stage:")
+        for prim in stage.Traverse():
+            if UsdGeom.Camera(prim):
+                print(f"  {prim.GetPath()}")
+        print("[DIAG] Done.\n")
+    except Exception as _diag_e:
+        print(f"[DIAG] Could not traverse stage: {_diag_e}")
+    # -- End diagnostic --
+
+    # Create viewport windows bound to robot cameras (no --enable_cameras needed)
+    create_robot_viewports()
+
     print("Teleoperation started. Press 'R' to reset the environment.")
 
     # simulate environment
@@ -296,6 +352,7 @@ def main() -> None:
                 if should_reset_recording_instance:
                     env.reset()
                     teleop_interface.reset()
+                    rebind_robot_viewports()
                     should_reset_recording_instance = False
                     print("Environment reset complete")
         except Exception as e:

scripts/environments/teleoperation/teleop_xr_agent.py

@@ -82,6 +82,53 @@ from xr_utils import XrClient, transform_xr_pose, quat_diff_as_rotvec_xyzw, is_v
 from xr_utils.geometry import R_HEADSET_TO_WORLD
 
 
+# =====================================================================
+# Robot Camera Viewport Utilities
+# =====================================================================
+
+# Resolved prim paths for env_0 — must match CameraCfg prim_path with
+# {ENV_REGEX_NS} → /World/envs/env_0
+_ROBOT_CAM_PRIMS: dict[str, str] = {
+    "Left Hand":  "/World/envs/env_0/Robot/rm_65_fb_left/Link_6/robot___left_8_02/cam_left_hand",
+    "Right Hand": "/World/envs/env_0/Robot/rm_65_b_right/Link6/robot___right_8_2/cam_right_hand",
+    "Head":       "/World/envs/env_0/Robot/robot_head/cam_head",
+    "Chest":      "/World/envs/env_0/Robot/robot_trunk/cam_chest",
+}
+
+# Stores created viewport window objects
+_robot_viewports: dict[str, object] = {}
+
+
+def create_robot_viewports() -> None:
+    """Create 4 viewport windows and bind each to a robot camera prim.
+
+    Must be called after env.reset() so all prims exist on the USD stage.
+    """
+    try:
+        import omni.kit.viewport.utility as vp_util
+    except ImportError:
+        logger.warning("[Viewport] omni.kit.viewport.utility not available — skipping viewport creation.")
+        return
+
+    for name, cam_path in _ROBOT_CAM_PRIMS.items():
+        vp_win = vp_util.create_viewport_window(
+            f"Robot {name} View", width=640, height=360
+        )
+        vp_win.viewport_api.camera_path = cam_path
+        _robot_viewports[name] = vp_win
+        print(f"[INFO] Viewport 'Robot {name} View' bound to: {cam_path}")
+
+
+def rebind_robot_viewports() -> None:
+    """Re-bind all robot viewports to their camera paths.
+
+    Call this after every env reset so viewports stay locked to the cameras.
+    """
+    for name, vp_win in _robot_viewports.items():
+        cam_path = _ROBOT_CAM_PRIMS[name]
+        vp_win.viewport_api.camera_path = cam_path
+
+
 # =====================================================================
 # Teleoperation Interface for XR
 # =====================================================================
@@ -430,6 +477,9 @@ def main() -> None:
     clear_ik_target_state()
     teleop_interface.reset()
 
+    # Create 4 viewport windows bound to the robot cameras
+    create_robot_viewports()
+
     print("\n" + "=" * 50)
     print(" 🚀 Teleoperation Started!")
     print(" 🎮 Use the TRIGGER to open/close gripper.")
@@ -450,6 +500,7 @@ def main() -> None:
                     obs, _ = env.reset()
                     clear_ik_target_state()
                     teleop_interface.reset()
+                    rebind_robot_viewports()
                     should_reset = False
                     sim_frame = 0
                     continue

source/mindbot/mindbot/tasks/manager_based/il/open_drybox/mindrobot_left_arm_ik_env_cfg.py

@@ -32,7 +32,8 @@ from isaaclab.sim.spawners.from_files.from_files_cfg import GroundPlaneCfg
 from isaaclab.utils import configclass
 from isaaclab.markers.config import FRAME_MARKER_CFG
 from isaaclab.devices.openxr import XrCfg
-
+from isaaclab.sensors import CameraCfg
+from isaaclab.envs import ViewerCfg
 # 导入基础配置和MDP函数
 from isaaclab_tasks.manager_based.manipulation.stack import mdp
 
@@ -101,7 +102,41 @@ class MindRobotTeleopSceneCfg(InteractiveSceneCfg):
     robot: ArticulationCfg = MINDBOT_HIGH_PD_CFG.replace(
         prim_path="{ENV_REGEX_NS}/Robot"
     )
-
+    # Cameras — CameraCfg sensors are only needed when reading pixel data (e.g. IL data collection).
+    # For viewport display only, use omni.kit.viewport.utility to bind the camera prim directly.
+    # Uncomment below AND add --enable_cameras flag only when sensor data is required.
+    # left_hand_camera = CameraCfg(
+    #     prim_path="{ENV_REGEX_NS}/Robot/rm_65_fb_left/Link_6/robot___left_8_02/cam_left_hand",
+    #     update_period=1 / 30,
+    #     height=480,
+    #     width=640,
+    #     data_types=["rgb"],
+    #     spawn=None,
+    # )
+    # right_hand_camera = CameraCfg(
+    #     prim_path="{ENV_REGEX_NS}/Robot/rm_65_b_right/Link6/robot___right_8_2/cam_right_hand",
+    #     update_period=1 / 30,
+    #     height=480,
+    #     width=640,
+    #     data_types=["rgb"],
+    #     spawn=None,
+    # )
+    # head_camera = CameraCfg(
+    #     prim_path="{ENV_REGEX_NS}/Robot/robot_head/cam_head",
+    #     update_period=1 / 30,
+    #     height=480,
+    #     width=640,
+    #     data_types=["rgb"],
+    #     spawn=None,
+    # )
+    # chest_camera = CameraCfg(
+    #     prim_path="{ENV_REGEX_NS}/Robot/robot_trunk/cam_chest",
+    #     update_period=1 / 30,
+    #     height=480,
+    #     width=640,
+    #     data_types=["rgb"],
+    #     spawn=None,
+    # )
     # Lighting
     light = AssetBaseCfg(
         prim_path="/World/light",
@@ -294,6 +329,11 @@ class MindRobotLeftArmIKEnvCfg(ManagerBasedRLEnvCfg):
         """Post initialization."""
         super().__post_init__()
 
+        self.viewer = ViewerCfg(
+            eye=(2.0, -1.5, 1.8),      # 相机位置
+            lookat=(0.0, 0.0, 0.8),    # 瞄准点（机器人腰部高度附近）
+            origin_type="world",
+        )
         # General settings
         self.decimation = 2
         self.episode_length_s = 50.0