Hardware API redesign (#777)

Co-authored-by: Pepijn <138571049+pkooij@users.noreply.github.com>
Co-authored-by: Steven Palma <imstevenpmwork@ieee.org>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Steven Palma <steven.palma@huggingface.co>
Co-authored-by: Adil Zouitine <adilzouitinegm@gmail.com>
Co-authored-by: Pepijn <pepijn@huggingface.co>

examples/lekiwi/evaluate.py

@@ -0,0 +1,38 @@
+import torch
+
+from lerobot.common.policies.act.modeling_act import ACTPolicy
+from lerobot.common.robots.lekiwi.config_lekiwi import LeKiwiClientConfig
+from lerobot.common.robots.lekiwi.lekiwi_client import LeKiwiClient
+from lerobot.common.utils.control_utils import predict_action
+from lerobot.common.utils.utils import get_safe_torch_device
+
+NB_CYCLES_CLIENT_CONNECTION = 1000
+
+robot_config = LeKiwiClientConfig(remote_ip="172.18.134.136", id="lekiwi")
+robot = LeKiwiClient(robot_config)
+
+robot.connect()
+
+policy = ACTPolicy.from_pretrained("pepijn223/act_lekiwi_circle")
+policy.reset()
+
+print("Running inference")
+i = 0
+while i < NB_CYCLES_CLIENT_CONNECTION:
+    obs = robot.get_observation()
+
+    for key, value in obs.items():
+        if isinstance(value, torch.Tensor):
+            obs[key] = value.numpy()
+
+    action_values = predict_action(
+        obs, policy, get_safe_torch_device(policy.config.device), policy.config.use_amp
+    )
+    action = {
+        key: action_values[i].item() if isinstance(action_values[i], torch.Tensor) else action_values[i]
+        for i, key in enumerate(robot.action_features)
+    }
+    robot.send_action(action)
+    i += 1
+
+robot.disconnect()

examples/lekiwi/record.py

@@ -0,0 +1,67 @@
+import time
+
+from lerobot.common.datasets.lerobot_dataset import LeRobotDataset
+from lerobot.common.datasets.utils import hw_to_dataset_features
+from lerobot.common.robots.lekiwi.config_lekiwi import LeKiwiClientConfig
+from lerobot.common.robots.lekiwi.lekiwi_client import LeKiwiClient
+from lerobot.common.teleoperators.keyboard import KeyboardTeleop, KeyboardTeleopConfig
+from lerobot.common.teleoperators.so100_leader import SO100Leader, SO100LeaderConfig
+
+NB_CYCLES_CLIENT_CONNECTION = 250
+
+leader_arm_config = SO100LeaderConfig(port="/dev/tty.usbmodem58760431551")
+leader_arm = SO100Leader(leader_arm_config)
+
+keyboard_config = KeyboardTeleopConfig()
+keyboard = KeyboardTeleop(keyboard_config)
+
+robot_config = LeKiwiClientConfig(remote_ip="172.18.134.136", id="lekiwi")
+robot = LeKiwiClient(robot_config)
+
+action_features = hw_to_dataset_features(robot.action_features, "action")
+obs_features = hw_to_dataset_features(robot.observation_features, "observation")
+dataset_features = {**action_features, **obs_features}
+
+dataset = LeRobotDataset.create(
+    repo_id="user/lekiwi" + str(int(time.time())),
+    fps=10,
+    features=dataset_features,
+    robot_type=robot.name,
+)
+
+leader_arm.connect()
+keyboard.connect()
+robot.connect()
+
+if not robot.is_connected or not leader_arm.is_connected or not keyboard.is_connected:
+    exit()
+
+print("Starting LeKiwi teleoperation")
+i = 0
+while i < NB_CYCLES_CLIENT_CONNECTION:
+    arm_action = leader_arm.get_action()
+    arm_action = {f"arm_{k}": v for k, v in arm_action.items()}
+
+    keyboard_keys = keyboard.get_action()
+
+    base_action = robot._from_keyboard_to_base_action(keyboard_keys)
+
+    action = {**arm_action, **base_action} if len(base_action) > 0 else arm_action
+
+    action_sent = robot.send_action(action)
+    observation = robot.get_observation()
+
+    frame = {**action_sent, **observation}
+    task = "Dummy Example Task Dataset"
+
+    dataset.add_frame(frame, task)
+    i += 1
+
+print("Disconnecting Teleop Devices and LeKiwi Client")
+robot.disconnect()
+leader_arm.disconnect()
+keyboard.disconnect()
+
+print("Uploading dataset to the hub")
+dataset.save_episode()
+dataset.push_to_hub()

examples/lekiwi/replay.py

@@ -0,0 +1,25 @@
+import time
+
+from lerobot.common.datasets.lerobot_dataset import LeRobotDataset
+from lerobot.common.robots.lekiwi.config_lekiwi import LeKiwiClientConfig
+from lerobot.common.robots.lekiwi.lekiwi_client import LeKiwiClient
+from lerobot.common.utils.robot_utils import busy_wait
+
+robot_config = LeKiwiClientConfig(remote_ip="172.18.134.136", id="lekiwi")
+robot = LeKiwiClient(robot_config)
+
+dataset = LeRobotDataset("pepijn223/lekiwi1749025613", episodes=[0])
+
+robot.connect()
+
+print("Replaying episode…")
+for _, action_array in enumerate(dataset.hf_dataset["action"]):
+    t0 = time.perf_counter()
+
+    action = {name: float(action_array[i]) for i, name in enumerate(dataset.features["action"]["names"])}
+    robot.send_action(action)
+
+    busy_wait(max(1.0 / dataset.fps - (time.perf_counter() - t0), 0.0))
+
+print("Disconnecting LeKiwi Client")
+robot.disconnect()

examples/lekiwi/teleoperate.py

@@ -0,0 +1,32 @@
+from lerobot.common.robots.lekiwi import LeKiwiClient, LeKiwiClientConfig
+from lerobot.common.teleoperators.keyboard.teleop_keyboard import KeyboardTeleop, KeyboardTeleopConfig
+from lerobot.common.teleoperators.so100_leader import SO100Leader, SO100LeaderConfig
+
+robot_config = LeKiwiClientConfig(remote_ip="172.18.134.136", id="my_lekiwi")
+
+teleop__arm_config = SO100LeaderConfig(
+    port="/dev/tty.usbmodem58760431551",
+    id="my_awesome_leader_arm",
+)
+
+teleop_keyboard_config = KeyboardTeleopConfig(
+    id="my_laptop_keyboard",
+)
+
+robot = LeKiwiClient(robot_config)
+teleop_arm = SO100Leader(teleop__arm_config)
+telep_keyboard = KeyboardTeleop(teleop_keyboard_config)
+robot.connect()
+teleop_arm.connect()
+telep_keyboard.connect()
+
+while True:
+    observation = robot.get_observation()
+
+    arm_action = teleop_arm.get_action()
+    arm_action = {f"arm_{k}": v for k, v in arm_action.items()}
+
+    keyboard_keys = telep_keyboard.get_action()
+    base_action = robot._from_keyboard_to_base_action(keyboard_keys)
+
+    robot.send_action(arm_action | base_action)