From a87dce9e3ff0636be9ec2444d0b2e6f991e1665d Mon Sep 17 00:00:00 2001 From: yutang Date: Fri, 1 Aug 2025 10:02:18 +0800 Subject: [PATCH] fix some bugs --- .../robot_devices/motors/realman_dual.py | 1 + .../common/robot_devices/robots/configs.py | 2 +- .../robot_devices/robots/realman_dual.py | 5 +- lerobot/common/robot_devices/robots/utils.py | 68 +++++++++++++++---- .../common/robot_devices/teleop/gamepad.py | 4 +- .../teleop/realman_aloha_dual.py | 2 +- realman.md | 14 ++-- 7 files changed, 68 insertions(+), 28 deletions(-) diff --git a/lerobot/common/robot_devices/motors/realman_dual.py b/lerobot/common/robot_devices/motors/realman_dual.py index ef8fe344..e2d1ca0e 100644 --- a/lerobot/common/robot_devices/motors/realman_dual.py +++ b/lerobot/common/robot_devices/motors/realman_dual.py @@ -46,6 +46,7 @@ class RealmanDualMotorsBus: """ def __init__(self, config: RealmanDualMotorsBusConfig): self.config = config + # import pdb; pdb.set_trace() self._initialize_arms() self._initialize_parameters() self._initialize_positions() diff --git a/lerobot/common/robot_devices/robots/configs.py b/lerobot/common/robot_devices/robots/configs.py index 9cb5b7f3..b7e573fb 100644 --- a/lerobot/common/robot_devices/robots/configs.py +++ b/lerobot/common/robot_devices/robots/configs.py @@ -756,7 +756,7 @@ class RealmanDualRobotConfig(RobotConfig): min_gripper: int = 10 servo_config_file: str = "/home/maic/LYT/lerobot/lerobot/common/robot_devices/teleop/servo_dual.yaml" left_end_control_guid: str = '0300b14bff1100003708000010010000' - right_end_control_guid: str = '0300509d5e040000120b000009050000' + right_end_control_guid: str = '030003f05e0400008e02000010010000' follower_arm: dict[str, MotorsBusConfig] = field( default_factory=lambda: { diff --git a/lerobot/common/robot_devices/robots/realman_dual.py b/lerobot/common/robot_devices/robots/realman_dual.py index 32ddf27c..109dfc25 100644 --- a/lerobot/common/robot_devices/robots/realman_dual.py +++ b/lerobot/common/robot_devices/robots/realman_dual.py @@ -14,6 +14,7 @@ from lerobot.common.robot_devices.motors.utils import get_motor_names, make_moto from lerobot.common.robot_devices.cameras.utils import make_cameras_from_configs from lerobot.common.robot_devices.utils import RobotDeviceAlreadyConnectedError, RobotDeviceNotConnectedError from lerobot.common.robot_devices.robots.configs import RealmanDualRobotConfig +from lerobot.common.robot_devices.robots.utils import ask_llm @@ -28,7 +29,6 @@ class RealmanDualRobot: # build cameras self.cameras = make_cameras_from_configs(self.config.cameras) - # build realman motors self.piper_motors = make_motors_buses_from_configs(self.config.follower_arm) self.arm = self.piper_motors['main'] @@ -239,6 +239,9 @@ class RealmanDualRobot: if record_data: data = self._prepare_record_data() + if data[0]: + # # ask_llm("将超声仪左下角试管架上的试管移动到超声仪中", data[0]) + pass return data except Exception as e: diff --git a/lerobot/common/robot_devices/robots/utils.py b/lerobot/common/robot_devices/robots/utils.py index ac2b9eed..1f740fb4 100644 --- a/lerobot/common/robot_devices/robots/utils.py +++ b/lerobot/common/robot_devices/robots/utils.py @@ -12,8 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Protocol +conversation_history = [] +conversation_client = None + +from typing import Protocol, Dict + +# Robot configuration imports from lerobot.common.robot_devices.robots.configs import ( AlohaRobotConfig, KochBimanualRobotConfig, @@ -29,28 +34,38 @@ from lerobot.common.robot_devices.robots.configs import ( RealmanDualRobotConfig ) +# Added library imports for LLM interaction +from openai import OpenAI +import base64 +import os +import cv2 +import torch +import numpy as np +from pynput import keyboard +import time +import json +from datetime import datetime def get_arm_id(name, arm_type): - """Returns the string identifier of a robot arm. For instance, for a bimanual manipulator - like Aloha, it could be left_follower, right_follower, left_leader, or right_leader. - """ + """Returns the string identifier of a robot arm.""" return f"{name}_{arm_type}" class Robot(Protocol): - # TODO(rcadene, aliberts): Add unit test checking the protocol is implemented in the corresponding classes robot_type: str features: dict + cameras: Dict def connect(self): ... def run_calibration(self): ... def teleop_step(self, record_data=False): ... - def capture_observation(self): ... + def capture_observation(self) -> Dict: ... def send_action(self, action): ... def disconnect(self): ... def make_robot_config(robot_type: str, **kwargs) -> RobotConfig: + # ... (此函数内容保持不变) ... if robot_type == "aloha": return AlohaRobotConfig(**kwargs) elif robot_type == "koch": @@ -69,37 +84,60 @@ def make_robot_config(robot_type: str, **kwargs) -> RobotConfig: return LeKiwiRobotConfig(**kwargs) elif robot_type == 'realman': return RealmanRobotConfig(**kwargs) - + elif robot_type == 'realman_dual': + return RealmanDualRobotConfig(**kwargs) else: raise ValueError(f"Robot type '{robot_type}' is not available.") def make_robot_from_config(config: RobotConfig): + # ... (此函数内容保持不变) ... if isinstance(config, ManipulatorRobotConfig): from lerobot.common.robot_devices.robots.manipulator import ManipulatorRobot - return ManipulatorRobot(config) elif isinstance(config, LeKiwiRobotConfig): from lerobot.common.robot_devices.robots.mobile_manipulator import MobileManipulator - return MobileManipulator(config) - elif isinstance(config, RealmanRobotConfig): from lerobot.common.robot_devices.robots.realman import RealmanRobot - return RealmanRobot(config) - elif isinstance(config, RealmanDualRobotConfig): from lerobot.common.robot_devices.robots.realman_dual import RealmanDualRobot - return RealmanDualRobot(config) - else: from lerobot.common.robot_devices.robots.stretch import StretchRobot - return StretchRobot(config) def make_robot(robot_type: str, **kwargs) -> Robot: config = make_robot_config(robot_type, **kwargs) return make_robot_from_config(config) + +# -------------------- LLM 交互功能区 -------------------- + +def encode_image_to_base64(image_tensor: torch.Tensor) -> str: + """将PyTorch张量格式的图像编码为Base64字符串""" + image_np = image_tensor.cpu().numpy().astype(np.uint8) + image_bgr = cv2.cvtColor(image_np, cv2.COLOR_RGB2BGR) + success, buffer = cv2.imencode(".jpg", image_bgr) + if not success: + raise ValueError("图像编码失败") + return base64.b64encode(buffer).decode("utf-8") + +# 在文件顶部添加全局变量来管理会话 +conversation_history = [] +conversation_client = None + +def ask_llm(query: str, state: dict): + + prompt = """ """ + + api_key = os.getenv("OPENAI_API_KEY") + base_url = os.getenv("OPENAI_BASE_URL") + client = OpenAI(api_key=api_key, base_url=base_url) + # keys = [key for key in state] + # import pdb + # pdb.set_trace() + + + pass \ No newline at end of file diff --git a/lerobot/common/robot_devices/teleop/gamepad.py b/lerobot/common/robot_devices/teleop/gamepad.py index 759bfc97..b40e3098 100644 --- a/lerobot/common/robot_devices/teleop/gamepad.py +++ b/lerobot/common/robot_devices/teleop/gamepad.py @@ -4,8 +4,8 @@ import time import logging from typing import Dict from dataclasses import dataclass -from .find_gamepad import find_controller_index -from .servo_server import ServoArmServer +from lerobot.common.robot_devices.teleop.find_gamepad import find_controller_index +from lerobot.common.robot_devices.teleop.servo_server import ServoArmServer class RealmanAlohaMaster: diff --git a/lerobot/common/robot_devices/teleop/realman_aloha_dual.py b/lerobot/common/robot_devices/teleop/realman_aloha_dual.py index 1bb43728..559c6f46 100644 --- a/lerobot/common/robot_devices/teleop/realman_aloha_dual.py +++ b/lerobot/common/robot_devices/teleop/realman_aloha_dual.py @@ -2,7 +2,7 @@ import time import logging from typing import Dict from dataclasses import dataclass -from .gamepad import RealmanAlohaMaster, DummyEndposeMaster +from lerobot.common.robot_devices.teleop.gamepad import RealmanAlohaMaster, DummyEndposeMaster @dataclass diff --git a/realman.md b/realman.md index bad50b93..28bda1fc 100644 --- a/realman.md +++ b/realman.md @@ -33,16 +33,14 @@ pip install pygame 见lerobot_piper_tutorial/1. 🤗 LeRobot：新增机械臂的一般流程.pdf # Teleoperate -```bash -cd piper_scripts/ -bash can_activate.sh can0 1000000 - -cd .. +```python python lerobot/scripts/control_robot.py \ - --robot.type=piper \ - --robot.inference_time=false \ - --control.type=teleoperate +--robot.type=realman_dual \ +--robot.inference_time=false \ +--control.type=teleoperate \ +--control.display_data=true ``` +display_data=true turn on run.io else turn off. # Record Set dataset root path