fix some bugs

lerobot/common/robot_devices/motors/realman_dual.py

@@ -46,6 +46,7 @@ class RealmanDualMotorsBus:
     """
     def __init__(self, config: RealmanDualMotorsBusConfig):
         self.config = config
+        # import pdb; pdb.set_trace()
         self._initialize_arms()
         self._initialize_parameters()
         self._initialize_positions()

lerobot/common/robot_devices/robots/configs.py

@@ -756,7 +756,7 @@ class RealmanDualRobotConfig(RobotConfig):
     min_gripper: int = 10
     servo_config_file: str = "/home/maic/LYT/lerobot/lerobot/common/robot_devices/teleop/servo_dual.yaml"
     left_end_control_guid: str = '0300b14bff1100003708000010010000'
-    right_end_control_guid: str = '0300509d5e040000120b000009050000'
+    right_end_control_guid: str = '030003f05e0400008e02000010010000'
 
     follower_arm: dict[str, MotorsBusConfig] = field(
         default_factory=lambda: {

lerobot/common/robot_devices/robots/realman_dual.py

@@ -14,6 +14,7 @@ from lerobot.common.robot_devices.motors.utils import get_motor_names, make_moto
 from lerobot.common.robot_devices.cameras.utils import make_cameras_from_configs
 from lerobot.common.robot_devices.utils import RobotDeviceAlreadyConnectedError, RobotDeviceNotConnectedError
 from lerobot.common.robot_devices.robots.configs import RealmanDualRobotConfig
+from lerobot.common.robot_devices.robots.utils import ask_llm
 
 
 
@@ -28,7 +29,6 @@ class RealmanDualRobot:
         
         # build cameras
         self.cameras = make_cameras_from_configs(self.config.cameras)
-        
         # build realman motors
         self.piper_motors = make_motors_buses_from_configs(self.config.follower_arm)
         self.arm = self.piper_motors['main']
@@ -239,6 +239,9 @@ class RealmanDualRobot:
 
             if record_data:
                 data = self._prepare_record_data()
+                if data[0]:
+                #     # ask_llm("将超声仪左下角试管架上的试管移动到超声仪中", data[0])
+                    pass
                 return data
             
         except Exception as e:

lerobot/common/robot_devices/robots/utils.py

@@ -12,8 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import Protocol
+conversation_history = []
+conversation_client = None
 
+
+from typing import Protocol, Dict
+
+# Robot configuration imports
 from lerobot.common.robot_devices.robots.configs import (
     AlohaRobotConfig,
     KochBimanualRobotConfig,
@@ -29,28 +34,38 @@ from lerobot.common.robot_devices.robots.configs import (
     RealmanDualRobotConfig
 )
 
+# Added library imports for LLM interaction
+from openai import OpenAI
+import base64
+import os
+import cv2
+import torch
+import numpy as np
+from pynput import keyboard
+import time
+import json 
+from datetime import datetime 
 
 def get_arm_id(name, arm_type):
-    """Returns the string identifier of a robot arm. For instance, for a bimanual manipulator
+    """Returns the string identifier of a robot arm."""
-    like Aloha, it could be left_follower, right_follower, left_leader, or right_leader.
-    """
     return f"{name}_{arm_type}"
 
 
 class Robot(Protocol):
-    # TODO(rcadene, aliberts): Add unit test checking the protocol is implemented in the corresponding classes
     robot_type: str
     features: dict
+    cameras: Dict 
 
     def connect(self): ...
     def run_calibration(self): ...
     def teleop_step(self, record_data=False): ...
-    def capture_observation(self): ...
+    def capture_observation(self) -> Dict: ...
     def send_action(self, action): ...
     def disconnect(self): ...
 
 
 def make_robot_config(robot_type: str, **kwargs) -> RobotConfig:
+    # ... (此函数内容保持不变) ...
     if robot_type == "aloha":
         return AlohaRobotConfig(**kwargs)
     elif robot_type == "koch":
@@ -69,37 +84,60 @@ def make_robot_config(robot_type: str, **kwargs) -> RobotConfig:
         return LeKiwiRobotConfig(**kwargs)
     elif robot_type == 'realman':
         return RealmanRobotConfig(**kwargs)
-
+    elif robot_type == 'realman_dual':
+        return RealmanDualRobotConfig(**kwargs)
     else:
         raise ValueError(f"Robot type '{robot_type}' is not available.")
 
 
 def make_robot_from_config(config: RobotConfig):
+    # ... (此函数内容保持不变) ...
     if isinstance(config, ManipulatorRobotConfig):
         from lerobot.common.robot_devices.robots.manipulator import ManipulatorRobot
-
         return ManipulatorRobot(config)
     elif isinstance(config, LeKiwiRobotConfig):
         from lerobot.common.robot_devices.robots.mobile_manipulator import MobileManipulator
-
         return MobileManipulator(config)
-    
     elif isinstance(config, RealmanRobotConfig):
         from lerobot.common.robot_devices.robots.realman import RealmanRobot
-
         return RealmanRobot(config)
-    
     elif isinstance(config, RealmanDualRobotConfig):
         from lerobot.common.robot_devices.robots.realman_dual import RealmanDualRobot
-
         return RealmanDualRobot(config)
-
     else:
         from lerobot.common.robot_devices.robots.stretch import StretchRobot
-
         return StretchRobot(config)
 
 
 def make_robot(robot_type: str, **kwargs) -> Robot:
     config = make_robot_config(robot_type, **kwargs)
     return make_robot_from_config(config)
+
+# -------------------- LLM 交互功能区 --------------------
+
+def encode_image_to_base64(image_tensor: torch.Tensor) -> str:
+    """将PyTorch张量格式的图像编码为Base64字符串"""
+    image_np = image_tensor.cpu().numpy().astype(np.uint8)
+    image_bgr = cv2.cvtColor(image_np, cv2.COLOR_RGB2BGR)
+    success, buffer = cv2.imencode(".jpg", image_bgr)
+    if not success:
+        raise ValueError("图像编码失败")
+    return base64.b64encode(buffer).decode("utf-8")
+
+# 在文件顶部添加全局变量来管理会话
+conversation_history = []
+conversation_client = None
+
+def ask_llm(query: str, state: dict):
+
+    prompt = """ """
+    
+    api_key = os.getenv("OPENAI_API_KEY")
+    base_url = os.getenv("OPENAI_BASE_URL")
+    client = OpenAI(api_key=api_key, base_url=base_url)
+    # keys = [key for key in state]
+    # import pdb
+    # pdb.set_trace()
+
+
+    pass

lerobot/common/robot_devices/teleop/gamepad.py

@@ -4,8 +4,8 @@ import time
 import logging
 from typing import Dict
 from dataclasses import dataclass
-from .find_gamepad import find_controller_index
+from lerobot.common.robot_devices.teleop.find_gamepad import find_controller_index
-from .servo_server import ServoArmServer
+from lerobot.common.robot_devices.teleop.servo_server import ServoArmServer
 
 
 class RealmanAlohaMaster:

lerobot/common/robot_devices/teleop/realman_aloha_dual.py

@@ -2,7 +2,7 @@ import time
 import logging
 from typing import Dict
 from dataclasses import dataclass
-from .gamepad import RealmanAlohaMaster, DummyEndposeMaster
+from lerobot.common.robot_devices.teleop.gamepad import RealmanAlohaMaster, DummyEndposeMaster
 
 
 @dataclass

realman.md

@@ -33,16 +33,14 @@ pip install pygame
 见lerobot_piper_tutorial/1. 🤗 LeRobot：新增机械臂的一般流程.pdf
 
 # Teleoperate
-```bash
+```python
-cd piper_scripts/
-bash can_activate.sh can0 1000000
-
-cd ..
 python lerobot/scripts/control_robot.py \
-    --robot.type=piper \
+--robot.type=realman_dual \
-    --robot.inference_time=false \
+--robot.inference_time=false \    
-    --control.type=teleoperate
+--control.type=teleoperate \
+--control.display_data=true
 ```
+display_data=true turn on run.io else turn off.
 
 # Record
 Set dataset root path