diff --git a/lerobot/common/robot_devices/cameras/intelrealsense.py b/lerobot/common/robot_devices/cameras/intelrealsense.py index c8bb5f07..c949109b 100644 --- a/lerobot/common/robot_devices/cameras/intelrealsense.py +++ b/lerobot/common/robot_devices/cameras/intelrealsense.py @@ -332,9 +332,9 @@ class IntelRealSenseCamera: f"Can't set {self.height=} for IntelRealSenseCamera({self.camera_index}). Actual value is {actual_height}." ) - self.fps = actual_fps - self.width = actual_width - self.height = actual_height + self.fps = round(actual_fps) + self.width = round(actual_width) + self.height = round(actual_height) self.is_connected = True diff --git a/lerobot/common/robot_devices/cameras/opencv.py b/lerobot/common/robot_devices/cameras/opencv.py index 35e548ca..c7b0737e 100644 --- a/lerobot/common/robot_devices/cameras/opencv.py +++ b/lerobot/common/robot_devices/cameras/opencv.py @@ -263,6 +263,15 @@ class OpenCVCamera: # when other threads are used to save the images. setNumThreads(1) + camera_idx = f"/dev/video{self.camera_index}" if platform.system() == "Linux" else self.camera_index + # First create a temporary camera trying to access `camera_index`, + # and verify it is a valid camera by calling `isOpened`. + tmp_camera = VideoCapture(camera_idx) + is_camera_open = tmp_camera.isOpened() + # Release camera to make it accessible for `find_camera_indices` + tmp_camera.release() + del tmp_camera + camera_idx = f"/dev/video{self.camera_index}" if platform.system() == "Linux" else self.camera_index with self.lock: # First create a temporary camera trying to access `camera_index`, @@ -289,40 +298,39 @@ class OpenCVCamera: # Secondly, create the camera that will be used downstream. # Note: For some unknown reason, calling `isOpened` blocks the camera which then # needs to be re-created. - with self.lock: - self.camera = VideoCapture(camera_idx) + self.camera = VideoCapture(camera_idx) - if self.fps is not None: - self.camera.set(CAP_PROP_FPS, self.fps) - if self.width is not None: - self.camera.set(CAP_PROP_FRAME_WIDTH, self.width) - if self.height is not None: - self.camera.set(CAP_PROP_FRAME_HEIGHT, self.height) + if self.fps is not None: + self.camera.set(CAP_PROP_FPS, self.fps) + if self.width is not None: + self.camera.set(CAP_PROP_FRAME_WIDTH, self.width) + if self.height is not None: + self.camera.set(CAP_PROP_FRAME_HEIGHT, self.height) - actual_fps = self.camera.get(CAP_PROP_FPS) - actual_width = self.camera.get(CAP_PROP_FRAME_WIDTH) - actual_height = self.camera.get(CAP_PROP_FRAME_HEIGHT) + actual_fps = self.camera.get(CAP_PROP_FPS) + actual_width = self.camera.get(CAP_PROP_FRAME_WIDTH) + actual_height = self.camera.get(CAP_PROP_FRAME_HEIGHT) - # Using `math.isclose` since actual fps can be a float (e.g. 29.9 instead of 30) - if self.fps is not None and not math.isclose(self.fps, actual_fps, rel_tol=1e-3): - # Using `OSError` since it's a broad that encompasses issues related to device communication - raise OSError( - f"Can't set {self.fps=} for OpenCVCamera({self.camera_index}). Actual value is {actual_fps}." - ) - if self.width is not None and self.width != actual_width: - raise OSError( - f"Can't set {self.width=} for OpenCVCamera({self.camera_index}). Actual value is {actual_width}." - ) - if self.height is not None and self.height != actual_height: - raise OSError( - f"Can't set {self.height=} for OpenCVCamera({self.camera_index}). Actual value is {actual_height}." - ) + # Using `math.isclose` since actual fps can be a float (e.g. 29.9 instead of 30) + if self.fps is not None and not math.isclose(self.fps, actual_fps, rel_tol=1e-3): + # Using `OSError` since it's a broad that encompasses issues related to device communication + raise OSError( + f"Can't set {self.fps=} for OpenCVCamera({self.camera_index}). Actual value is {actual_fps}." + ) + if self.width is not None and self.width != actual_width: + raise OSError( + f"Can't set {self.width=} for OpenCVCamera({self.camera_index}). Actual value is {actual_width}." + ) + if self.height is not None and self.height != actual_height: + raise OSError( + f"Can't set {self.height=} for OpenCVCamera({self.camera_index}). Actual value is {actual_height}." + ) - self.fps = int(actual_fps) - self.width = int(actual_width) - self.height = int(actual_height) + self.fps = round(actual_fps) + self.width = round(actual_width) + self.height = round(actual_height) - self.is_connected = True + self.is_connected = True def read(self, temporary_color_mode: str | None = None) -> np.ndarray: """Read a frame from the camera returned in the format (height, width, channels) @@ -338,8 +346,7 @@ class OpenCVCamera: start_time = time.perf_counter() - with self.lock: - ret, color_image = self.camera.read() + ret, color_image = self.camera.read() if not ret: raise OSError(f"Can't capture color image from camera {self.camera_index}.") @@ -374,8 +381,7 @@ class OpenCVCamera: # log the utc time at which the image was received self.logs["timestamp_utc"] = capture_timestamp_utc() - with self.lock: - self.color_image = color_image + self.color_image = color_image return color_image @@ -401,7 +407,6 @@ class OpenCVCamera: num_tries = 0 while True: - # Do not use `with self.lock` here, as it reduces fps if self.color_image is not None: return self.color_image @@ -417,16 +422,14 @@ class OpenCVCamera: ) if self.thread is not None: - with self.lock: - self.stop_event.set() + self.stop_event.set() self.thread.join() # wait for the thread to finish self.thread = None self.stop_event = None - with self.lock: - self.camera.release() - self.camera = None - self.is_connected = False + self.camera.release() + self.camera = None + self.is_connected = False def __del__(self): with self.lock: diff --git a/lerobot/scripts/control_robot.py b/lerobot/scripts/control_robot.py index ca486105..2a83d6db 100644 --- a/lerobot/scripts/control_robot.py +++ b/lerobot/scripts/control_robot.py @@ -370,6 +370,7 @@ def record( tags=None, num_image_writers_per_camera=4, force_override=False, + display_cameras=True, ): # TODO(rcadene): Add option to record logs # TODO(rcadene): Clean this function via decomposition in higher level functions @@ -403,7 +404,7 @@ def record( episode_index = 0 if is_headless(): - logging.info( + logging.warning( "Headless environment detected. On-screen cameras display and keyboard inputs will not be available." ) @@ -471,7 +472,7 @@ def record( else: observation = robot.capture_observation() - if not is_headless(): + if display_cameras and not is_headless(): image_keys = [key for key in observation if "image" in key] for key in image_keys: cv2.imshow(key, cv2.cvtColor(observation[key].numpy(), cv2.COLOR_RGB2BGR)) @@ -515,7 +516,7 @@ def record( for key in image_keys: frame_queue.put((observation[key], key, frame_index, episode_index, videos_dir)) - if not is_headless(): + if display_cameras and not is_headless(): image_keys = [key for key in observation if "image" in key] for key in image_keys: cv2.imshow(key, cv2.cvtColor(observation[key].numpy(), cv2.COLOR_RGB2BGR)) @@ -663,7 +664,7 @@ def record( stop_workers(frame_workers, frame_queue) robot.disconnect() - if not is_headless(): + if display_cameras and not is_headless(): cv2.destroyAllWindows() num_episodes = episode_index diff --git a/poetry.lock b/poetry.lock index ad641ac9..a25cdcf7 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. [[package]] name = "absl-py" @@ -2406,7 +2406,6 @@ description = "Nvidia JIT LTO Library" optional = false python-versions = ">=3" files = [ - {file = "nvidia_nvjitlink_cu12-12.5.82-py3-none-manylinux2014_aarch64.whl", hash = "sha256:98103729cc5226e13ca319a10bbf9433bbbd44ef64fe72f45f067cacc14b8d27"}, {file = "nvidia_nvjitlink_cu12-12.5.82-py3-none-manylinux2014_x86_64.whl", hash = "sha256:f9b37bc5c8cf7509665cb6ada5aaa0ce65618f2332b7d3e78e9790511f111212"}, {file = "nvidia_nvjitlink_cu12-12.5.82-py3-none-win_amd64.whl", hash = "sha256:e782564d705ff0bf61ac3e1bf730166da66dd2fe9012f111ede5fc49b64ae697"}, ] @@ -4578,7 +4577,7 @@ dora = ["gym-dora"] dynamixel = ["dynamixel-sdk", "pynput"] intelrealsense = ["pyrealsense2"] pusht = ["gym-pusht"] -test = ["pytest", "pytest-cov"] +test = ["pyserial", "pytest", "pytest-cov"] umi = ["imagecodecs"] video-benchmark = ["pandas", "scikit-image"] xarm = ["gym-xarm"] @@ -4586,4 +4585,4 @@ xarm = ["gym-xarm"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.13" -content-hash = "4e86e7b1eba959daff1edbc7d41df690e82e065f75de2cf0cde705d596565004" +content-hash = "5e4f6b9727d67a37d1c6d94af7661bb688a0866afd30878c5e523b8e768deac6" diff --git a/pyproject.toml b/pyproject.toml index 9ffaa648..11924441 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,7 +67,7 @@ pynput = {version = ">=1.7.7", optional = true} # TODO(rcadene, salibert): 71.0.1 has a bug setuptools = {version = "!=71.0.1", optional = true} pyrealsense2 = {version = ">=2.55.1.6486", markers = "sys_platform != 'darwin'", optional = true} -pyserial = ">=3.5" +pyserial = {version = ">=3.5", optional = true} [tool.poetry.extras] @@ -76,7 +76,7 @@ pusht = ["gym-pusht"] xarm = ["gym-xarm"] aloha = ["gym-aloha"] dev = ["pre-commit", "debugpy"] -test = ["pytest", "pytest-cov"] +test = ["pytest", "pytest-cov", "pyserial"] umi = ["imagecodecs"] video_benchmark = ["scikit-image", "pandas"] dynamixel = ["dynamixel-sdk", "pynput"] diff --git a/tests/test_control_robot.py b/tests/test_control_robot.py index 452c709f..7b913083 100644 --- a/tests/test_control_robot.py +++ b/tests/test_control_robot.py @@ -36,11 +36,20 @@ from tests.utils import DEFAULT_CONFIG_PATH, DEVICE, TEST_ROBOT_TYPES, require_r @pytest.mark.parametrize("robot_type, mock", TEST_ROBOT_TYPES) @require_robot -def test_teleoperate(request, robot_type, mock): +def test_teleoperate(tmpdir, request, robot_type, mock): if mock: request.getfixturevalue("patch_builtins_input") - robot = make_robot(robot_type, mock=mock) + # Create an empty calibration directory to trigger manual calibration + # and avoid writing calibration files in user .cache/calibration folder + tmpdir = Path(tmpdir) + calibration_dir = tmpdir / robot_type + overrides = [f"calibration_dir={calibration_dir}"] + else: + # Use the default .cache/calibration folder when mock=False + overrides = None + + robot = make_robot(robot_type, overrides=overrides, mock=mock) teleoperate(robot, teleop_time_s=1) teleoperate(robot, fps=30, teleop_time_s=1) teleoperate(robot, fps=60, teleop_time_s=1) @@ -53,6 +62,7 @@ def test_calibrate(tmpdir, request, robot_type, mock): if mock: request.getfixturevalue("patch_builtins_input") + # Create an empty calibration directory to trigger manual calibration tmpdir = Path(tmpdir) calibration_dir = tmpdir / robot_type overrides_calibration_dir = [f"calibration_dir={calibration_dir}"] @@ -65,13 +75,21 @@ def test_calibrate(tmpdir, request, robot_type, mock): @pytest.mark.parametrize("robot_type, mock", TEST_ROBOT_TYPES) @require_robot def test_record_without_cameras(tmpdir, request, robot_type, mock): + # Avoid using cameras + overrides = ["~cameras"] + if mock: request.getfixturevalue("patch_builtins_input") - root = Path(tmpdir) + # Create an empty calibration directory to trigger manual calibration + # and avoid writing calibration files in user .cache/calibration folder + calibration_dir = Path(tmpdir) / robot_type + overrides.append(f"calibration_dir={calibration_dir}") + + root = Path(tmpdir) / "data" repo_id = "lerobot/debug" - robot = make_robot(robot_type, overrides=["~cameras"], mock=mock) + robot = make_robot(robot_type, overrides=overrides, mock=mock) record( robot, fps=30, @@ -92,10 +110,24 @@ def test_record_and_replay_and_policy(tmpdir, request, robot_type, mock): if mock: request.getfixturevalue("patch_builtins_input") - root = Path(tmpdir) + # Create an empty calibration directory to trigger manual calibration + # and avoid writing calibration files in user .cache/calibration folder + calibration_dir = Path(tmpdir) / robot_type + overrides = [f"calibration_dir={calibration_dir}"] + else: + # Use the default .cache/calibration folder when mock=False + overrides = None + + if robot_type == "aloha": + pytest.skip("TODO(rcadene): enable test once aloha_real and act_aloha_real are merged") + + env_name = "koch_real" + policy_name = "act_koch_real" + + root = Path(tmpdir) / "data" repo_id = "lerobot/debug" - robot = make_robot(robot_type, mock=mock) + robot = make_robot(robot_type, overrides=overrides, mock=mock) dataset = record( robot, fps=30, @@ -107,6 +139,8 @@ def test_record_and_replay_and_policy(tmpdir, request, robot_type, mock): push_to_hub=False, # TODO(rcadene, aliberts): test video=True video=False, + # TODO(rcadene): display cameras through cv2 sometimes crashes on mac + display_cameras=False, ) replay(robot, episode=0, fps=30, root=root, repo_id=repo_id) @@ -147,6 +181,7 @@ def test_record_and_replay_and_policy(tmpdir, request, robot_type, mock): run_compute_stats=False, push_to_hub=False, video=False, + display_cameras=False, ) del robot diff --git a/tests/test_robots.py b/tests/test_robots.py index 93c1a1fe..72f0c944 100644 --- a/tests/test_robots.py +++ b/tests/test_robots.py @@ -40,6 +40,8 @@ def test_robot(tmpdir, request, robot_type, mock): # TODO(rcadene): test logs # TODO(rcadene): add compatibility with other robots + robot_kwargs = {"robot_type": robot_type} + if robot_type == "aloha" and mock: # To simplify unit test, we do not rerun manual calibration for Aloha mock=True. # Instead, we use the files from '.cache/calibration/aloha_default' @@ -52,15 +54,16 @@ def test_robot(tmpdir, request, robot_type, mock): tmpdir = Path(tmpdir) calibration_dir = tmpdir / robot_type overrides_calibration_dir = [f"calibration_dir={calibration_dir}"] + robot_kwargs["calibration_dir"] = calibration_dir # Test connecting without devices raises an error - robot = ManipulatorRobot() + robot = ManipulatorRobot(**robot_kwargs) with pytest.raises(ValueError): robot.connect() del robot # Test using robot before connecting raises an error - robot = ManipulatorRobot() + robot = ManipulatorRobot(**robot_kwargs) with pytest.raises(RobotDeviceNotConnectedError): robot.teleop_step() with pytest.raises(RobotDeviceNotConnectedError): @@ -84,8 +87,9 @@ def test_robot(tmpdir, request, robot_type, mock): with pytest.raises(RobotDeviceAlreadyConnectedError): robot.connect() - # Test disconnecting with `__del__` - del robot + # TODO(rcadene, aliberts): Test disconnecting with `__del__` instead of `disconnect` + # del robot + robot.disconnect() # Test teleop can run robot = make_robot(robot_type, overrides=overrides_calibration_dir, mock=mock) @@ -136,4 +140,3 @@ def test_robot(tmpdir, request, robot_type, mock): assert not robot.leader_arms[name].is_connected for name in robot.cameras: assert not robot.cameras[name].is_connected - del robot diff --git a/tests/utils.py b/tests/utils.py index 69a1743c..0c4b94d8 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -275,6 +275,8 @@ def make_robot(robot_type: str, overrides: list[str] | None = None, mock=False) if mock: overrides = [] if overrides is None else copy(overrides) + # Explicitely add mock argument to the cameras and set it to true + # TODO(rcadene, aliberts): redesign when we drop hydra if robot_type == "koch": overrides.append("+leader_arms.main.mock=true") overrides.append("+follower_arms.main.mock=true")