Merge remote-tracking branch 'origin/main' into user/rcadene/2024_09_10_train_aloha

lerobot/common/robot_devices/cameras/intelrealsense.py

@@ -332,9 +332,9 @@ class IntelRealSenseCamera:
                 f"Can't set {self.height=} for IntelRealSenseCamera({self.camera_index}). Actual value is {actual_height}."
             )
 
-        self.fps = actual_fps
+        self.fps = round(actual_fps)
-        self.width = actual_width
+        self.width = round(actual_width)
-        self.height = actual_height
+        self.height = round(actual_height)
 
         self.is_connected = True
 

lerobot/common/robot_devices/cameras/opencv.py

@@ -263,6 +263,15 @@ class OpenCVCamera:
             # when other threads are used to save the images.
             setNumThreads(1)
 
+        camera_idx = f"/dev/video{self.camera_index}" if platform.system() == "Linux" else self.camera_index
+        # First create a temporary camera trying to access `camera_index`,
+        # and verify it is a valid camera by calling `isOpened`.
+        tmp_camera = VideoCapture(camera_idx)
+        is_camera_open = tmp_camera.isOpened()
+        # Release camera to make it accessible for `find_camera_indices`
+        tmp_camera.release()
+        del tmp_camera
+
         camera_idx = f"/dev/video{self.camera_index}" if platform.system() == "Linux" else self.camera_index
         with self.lock:
             # First create a temporary camera trying to access `camera_index`,
@@ -289,40 +298,39 @@ class OpenCVCamera:
         # Secondly, create the camera that will be used downstream.
         # Note: For some unknown reason, calling `isOpened` blocks the camera which then
         # needs to be re-created.
-        with self.lock:
+        self.camera = VideoCapture(camera_idx)
-            self.camera = VideoCapture(camera_idx)
 
-            if self.fps is not None:
+        if self.fps is not None:
-                self.camera.set(CAP_PROP_FPS, self.fps)
+            self.camera.set(CAP_PROP_FPS, self.fps)
-            if self.width is not None:
+        if self.width is not None:
-                self.camera.set(CAP_PROP_FRAME_WIDTH, self.width)
+            self.camera.set(CAP_PROP_FRAME_WIDTH, self.width)
-            if self.height is not None:
+        if self.height is not None:
-                self.camera.set(CAP_PROP_FRAME_HEIGHT, self.height)
+            self.camera.set(CAP_PROP_FRAME_HEIGHT, self.height)
 
-            actual_fps = self.camera.get(CAP_PROP_FPS)
+        actual_fps = self.camera.get(CAP_PROP_FPS)
-            actual_width = self.camera.get(CAP_PROP_FRAME_WIDTH)
+        actual_width = self.camera.get(CAP_PROP_FRAME_WIDTH)
-            actual_height = self.camera.get(CAP_PROP_FRAME_HEIGHT)
+        actual_height = self.camera.get(CAP_PROP_FRAME_HEIGHT)
 
-            # Using `math.isclose` since actual fps can be a float (e.g. 29.9 instead of 30)
+        # Using `math.isclose` since actual fps can be a float (e.g. 29.9 instead of 30)
-            if self.fps is not None and not math.isclose(self.fps, actual_fps, rel_tol=1e-3):
+        if self.fps is not None and not math.isclose(self.fps, actual_fps, rel_tol=1e-3):
-                # Using `OSError` since it's a broad that encompasses issues related to device communication
+            # Using `OSError` since it's a broad that encompasses issues related to device communication
-                raise OSError(
+            raise OSError(
-                    f"Can't set {self.fps=} for OpenCVCamera({self.camera_index}). Actual value is {actual_fps}."
+                f"Can't set {self.fps=} for OpenCVCamera({self.camera_index}). Actual value is {actual_fps}."
-                )
+            )
-            if self.width is not None and self.width != actual_width:
+        if self.width is not None and self.width != actual_width:
-                raise OSError(
+            raise OSError(
-                    f"Can't set {self.width=} for OpenCVCamera({self.camera_index}). Actual value is {actual_width}."
+                f"Can't set {self.width=} for OpenCVCamera({self.camera_index}). Actual value is {actual_width}."
-                )
+            )
-            if self.height is not None and self.height != actual_height:
+        if self.height is not None and self.height != actual_height:
-                raise OSError(
+            raise OSError(
-                    f"Can't set {self.height=} for OpenCVCamera({self.camera_index}). Actual value is {actual_height}."
+                f"Can't set {self.height=} for OpenCVCamera({self.camera_index}). Actual value is {actual_height}."
-                )
+            )
 
-            self.fps = int(actual_fps)
+        self.fps = round(actual_fps)
-            self.width = int(actual_width)
+        self.width = round(actual_width)
-            self.height = int(actual_height)
+        self.height = round(actual_height)
 
-            self.is_connected = True
+        self.is_connected = True
 
     def read(self, temporary_color_mode: str | None = None) -> np.ndarray:
         """Read a frame from the camera returned in the format (height, width, channels)
@@ -338,8 +346,7 @@ class OpenCVCamera:
 
         start_time = time.perf_counter()
 
-        with self.lock:
+        ret, color_image = self.camera.read()
-            ret, color_image = self.camera.read()
 
         if not ret:
             raise OSError(f"Can't capture color image from camera {self.camera_index}.")
@@ -374,8 +381,7 @@ class OpenCVCamera:
         # log the utc time at which the image was received
         self.logs["timestamp_utc"] = capture_timestamp_utc()
 
-        with self.lock:
+        self.color_image = color_image
-            self.color_image = color_image
 
         return color_image
 
@@ -401,7 +407,6 @@ class OpenCVCamera:
 
         num_tries = 0
         while True:
-            # Do not use `with self.lock` here, as it reduces fps
             if self.color_image is not None:
                 return self.color_image
 
@@ -417,16 +422,14 @@ class OpenCVCamera:
             )
 
         if self.thread is not None:
-            with self.lock:
+            self.stop_event.set()
-                self.stop_event.set()
             self.thread.join()  # wait for the thread to finish
             self.thread = None
             self.stop_event = None
 
-        with self.lock:
+        self.camera.release()
-            self.camera.release()
+        self.camera = None
-            self.camera = None
+        self.is_connected = False
-            self.is_connected = False
 
     def __del__(self):
         with self.lock:

lerobot/scripts/control_robot.py

@@ -370,6 +370,7 @@ def record(
     tags=None,
     num_image_writers_per_camera=4,
     force_override=False,
+    display_cameras=True,
 ):
     # TODO(rcadene): Add option to record logs
     # TODO(rcadene): Clean this function via decomposition in higher level functions
@@ -403,7 +404,7 @@ def record(
         episode_index = 0
 
     if is_headless():
-        logging.info(
+        logging.warning(
             "Headless environment detected. On-screen cameras display and keyboard inputs will not be available."
         )
 
@@ -471,7 +472,7 @@ def record(
         else:
             observation = robot.capture_observation()
 
-        if not is_headless():
+        if display_cameras and not is_headless():
             image_keys = [key for key in observation if "image" in key]
             for key in image_keys:
                 cv2.imshow(key, cv2.cvtColor(observation[key].numpy(), cv2.COLOR_RGB2BGR))
@@ -515,7 +516,7 @@ def record(
                 for key in image_keys:
                     frame_queue.put((observation[key], key, frame_index, episode_index, videos_dir))
 
-                if not is_headless():
+                if display_cameras and not is_headless():
                     image_keys = [key for key in observation if "image" in key]
                     for key in image_keys:
                         cv2.imshow(key, cv2.cvtColor(observation[key].numpy(), cv2.COLOR_RGB2BGR))
@@ -663,7 +664,7 @@ def record(
         stop_workers(frame_workers, frame_queue)
 
     robot.disconnect()
-    if not is_headless():
+    if display_cameras and not is_headless():
         cv2.destroyAllWindows()
 
     num_episodes = episode_index

poetry.lock

@@ -1,4 +1,4 @@
-# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand.
+# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand.
 
 [[package]]
 name = "absl-py"
@@ -2406,7 +2406,6 @@ description = "Nvidia JIT LTO Library"
 optional = false
 python-versions = ">=3"
 files = [
-    {file = "nvidia_nvjitlink_cu12-12.5.82-py3-none-manylinux2014_aarch64.whl", hash = "sha256:98103729cc5226e13ca319a10bbf9433bbbd44ef64fe72f45f067cacc14b8d27"},
     {file = "nvidia_nvjitlink_cu12-12.5.82-py3-none-manylinux2014_x86_64.whl", hash = "sha256:f9b37bc5c8cf7509665cb6ada5aaa0ce65618f2332b7d3e78e9790511f111212"},
     {file = "nvidia_nvjitlink_cu12-12.5.82-py3-none-win_amd64.whl", hash = "sha256:e782564d705ff0bf61ac3e1bf730166da66dd2fe9012f111ede5fc49b64ae697"},
 ]
@@ -4578,7 +4577,7 @@ dora = ["gym-dora"]
 dynamixel = ["dynamixel-sdk", "pynput"]
 intelrealsense = ["pyrealsense2"]
 pusht = ["gym-pusht"]
-test = ["pytest", "pytest-cov"]
+test = ["pyserial", "pytest", "pytest-cov"]
 umi = ["imagecodecs"]
 video-benchmark = ["pandas", "scikit-image"]
 xarm = ["gym-xarm"]
@@ -4586,4 +4585,4 @@ xarm = ["gym-xarm"]
 [metadata]
 lock-version = "2.0"
 python-versions = ">=3.10,<3.13"
-content-hash = "4e86e7b1eba959daff1edbc7d41df690e82e065f75de2cf0cde705d596565004"
+content-hash = "5e4f6b9727d67a37d1c6d94af7661bb688a0866afd30878c5e523b8e768deac6"

pyproject.toml

@@ -67,7 +67,7 @@ pynput = {version = ">=1.7.7", optional = true}
 # TODO(rcadene, salibert): 71.0.1 has a bug
 setuptools = {version = "!=71.0.1", optional = true}
 pyrealsense2 = {version = ">=2.55.1.6486", markers = "sys_platform != 'darwin'", optional = true}
-pyserial = ">=3.5"
+pyserial = {version = ">=3.5", optional = true}
 
 
 [tool.poetry.extras]
@@ -76,7 +76,7 @@ pusht = ["gym-pusht"]
 xarm = ["gym-xarm"]
 aloha = ["gym-aloha"]
 dev = ["pre-commit", "debugpy"]
-test = ["pytest", "pytest-cov"]
+test = ["pytest", "pytest-cov", "pyserial"]
 umi = ["imagecodecs"]
 video_benchmark = ["scikit-image", "pandas"]
 dynamixel = ["dynamixel-sdk", "pynput"]

tests/test_control_robot.py

@@ -36,11 +36,20 @@ from tests.utils import DEFAULT_CONFIG_PATH, DEVICE, TEST_ROBOT_TYPES, require_r
 
 @pytest.mark.parametrize("robot_type, mock", TEST_ROBOT_TYPES)
 @require_robot
-def test_teleoperate(request, robot_type, mock):
+def test_teleoperate(tmpdir, request, robot_type, mock):
     if mock:
         request.getfixturevalue("patch_builtins_input")
 
-    robot = make_robot(robot_type, mock=mock)
+        # Create an empty calibration directory to trigger manual calibration
+        # and avoid writing calibration files in user .cache/calibration folder
+        tmpdir = Path(tmpdir)
+        calibration_dir = tmpdir / robot_type
+        overrides = [f"calibration_dir={calibration_dir}"]
+    else:
+        # Use the default .cache/calibration folder when mock=False
+        overrides = None
+
+    robot = make_robot(robot_type, overrides=overrides, mock=mock)
     teleoperate(robot, teleop_time_s=1)
     teleoperate(robot, fps=30, teleop_time_s=1)
     teleoperate(robot, fps=60, teleop_time_s=1)
@@ -53,6 +62,7 @@ def test_calibrate(tmpdir, request, robot_type, mock):
     if mock:
         request.getfixturevalue("patch_builtins_input")
 
+    # Create an empty calibration directory to trigger manual calibration
     tmpdir = Path(tmpdir)
     calibration_dir = tmpdir / robot_type
     overrides_calibration_dir = [f"calibration_dir={calibration_dir}"]
@@ -65,13 +75,21 @@ def test_calibrate(tmpdir, request, robot_type, mock):
 @pytest.mark.parametrize("robot_type, mock", TEST_ROBOT_TYPES)
 @require_robot
 def test_record_without_cameras(tmpdir, request, robot_type, mock):
+    # Avoid using cameras
+    overrides = ["~cameras"]
+
     if mock:
         request.getfixturevalue("patch_builtins_input")
 
-    root = Path(tmpdir)
+        # Create an empty calibration directory to trigger manual calibration
+        # and avoid writing calibration files in user .cache/calibration folder
+        calibration_dir = Path(tmpdir) / robot_type
+        overrides.append(f"calibration_dir={calibration_dir}")
+
+    root = Path(tmpdir) / "data"
     repo_id = "lerobot/debug"
 
-    robot = make_robot(robot_type, overrides=["~cameras"], mock=mock)
+    robot = make_robot(robot_type, overrides=overrides, mock=mock)
     record(
         robot,
         fps=30,
@@ -92,10 +110,24 @@ def test_record_and_replay_and_policy(tmpdir, request, robot_type, mock):
     if mock:
         request.getfixturevalue("patch_builtins_input")
 
-    root = Path(tmpdir)
+        # Create an empty calibration directory to trigger manual calibration
+        # and avoid writing calibration files in user .cache/calibration folder
+        calibration_dir = Path(tmpdir) / robot_type
+        overrides = [f"calibration_dir={calibration_dir}"]
+    else:
+        # Use the default .cache/calibration folder when mock=False
+        overrides = None
+
+    if robot_type == "aloha":
+        pytest.skip("TODO(rcadene): enable test once aloha_real and act_aloha_real are merged")
+
+    env_name = "koch_real"
+    policy_name = "act_koch_real"
+
+    root = Path(tmpdir) / "data"
     repo_id = "lerobot/debug"
 
-    robot = make_robot(robot_type, mock=mock)
+    robot = make_robot(robot_type, overrides=overrides, mock=mock)
     dataset = record(
         robot,
         fps=30,
@@ -107,6 +139,8 @@ def test_record_and_replay_and_policy(tmpdir, request, robot_type, mock):
         push_to_hub=False,
         # TODO(rcadene, aliberts): test video=True
         video=False,
+        # TODO(rcadene): display cameras through cv2 sometimes crashes on mac
+        display_cameras=False,
     )
 
     replay(robot, episode=0, fps=30, root=root, repo_id=repo_id)
@@ -147,6 +181,7 @@ def test_record_and_replay_and_policy(tmpdir, request, robot_type, mock):
         run_compute_stats=False,
         push_to_hub=False,
         video=False,
+        display_cameras=False,
     )
 
     del robot

tests/test_robots.py

@@ -40,6 +40,8 @@ def test_robot(tmpdir, request, robot_type, mock):
     # TODO(rcadene): test logs
     # TODO(rcadene): add compatibility with other robots
 
+    robot_kwargs = {"robot_type": robot_type}
+
     if robot_type == "aloha" and mock:
         # To simplify unit test, we do not rerun manual calibration for Aloha mock=True.
         # Instead, we use the files from '.cache/calibration/aloha_default'
@@ -52,15 +54,16 @@ def test_robot(tmpdir, request, robot_type, mock):
         tmpdir = Path(tmpdir)
         calibration_dir = tmpdir / robot_type
         overrides_calibration_dir = [f"calibration_dir={calibration_dir}"]
+        robot_kwargs["calibration_dir"] = calibration_dir
 
     # Test connecting without devices raises an error
-    robot = ManipulatorRobot()
+    robot = ManipulatorRobot(**robot_kwargs)
     with pytest.raises(ValueError):
         robot.connect()
     del robot
 
     # Test using robot before connecting raises an error
-    robot = ManipulatorRobot()
+    robot = ManipulatorRobot(**robot_kwargs)
     with pytest.raises(RobotDeviceNotConnectedError):
         robot.teleop_step()
     with pytest.raises(RobotDeviceNotConnectedError):
@@ -84,8 +87,9 @@ def test_robot(tmpdir, request, robot_type, mock):
     with pytest.raises(RobotDeviceAlreadyConnectedError):
         robot.connect()
 
-    # Test disconnecting with `__del__`
+    # TODO(rcadene, aliberts): Test disconnecting with `__del__` instead of `disconnect`
-    del robot
+    # del robot
+    robot.disconnect()
 
     # Test teleop can run
     robot = make_robot(robot_type, overrides=overrides_calibration_dir, mock=mock)
@@ -136,4 +140,3 @@ def test_robot(tmpdir, request, robot_type, mock):
         assert not robot.leader_arms[name].is_connected
     for name in robot.cameras:
         assert not robot.cameras[name].is_connected
-    del robot

tests/utils.py

@@ -275,6 +275,8 @@ def make_robot(robot_type: str, overrides: list[str] | None = None, mock=False)
     if mock:
         overrides = [] if overrides is None else copy(overrides)
 
+        # Explicitely add mock argument to the cameras and set it to true
+        # TODO(rcadene, aliberts): redesign when we drop hydra
         if robot_type == "koch":
             overrides.append("+leader_arms.main.mock=true")
             overrides.append("+follower_arms.main.mock=true")