From 47d5008407af46a06cf9bfa69b0c7c7471e3ace8 Mon Sep 17 00:00:00 2001 From: Steven Palma Date: Wed, 21 May 2025 15:41:52 +0200 Subject: [PATCH] chore(cameras): address unresolved conversations --- lerobot/common/cameras/camera.py | 2 ++ lerobot/common/cameras/configs.py | 5 ++-- .../common/cameras/opencv/camera_opencv.py | 16 ++++++++--- .../cameras/realsense/camera_realsense.py | 27 +++++++------------ lerobot/common/cameras/utils.py | 2 +- tests/cameras/test_opencv.py | 4 ++- tests/cameras/test_realsense.py | 2 +- 7 files changed, 32 insertions(+), 26 deletions(-) diff --git a/lerobot/common/cameras/camera.py b/lerobot/common/cameras/camera.py index 1937205b1..0ef853402 100644 --- a/lerobot/common/cameras/camera.py +++ b/lerobot/common/cameras/camera.py @@ -36,6 +36,7 @@ class Camera(abc.ABC): fps (int | None): Configured frames per second width (int | None): Frame width in pixels height (int | None): Frame height in pixels + warmup_time (int | None): Time reading frames before returning from connect (in seconds) Example: class MyCamera(Camera): @@ -55,6 +56,7 @@ class Camera(abc.ABC): self.fps: int | None = config.fps self.width: int | None = config.width self.height: int | None = config.height + self.warmup_time: int | None = config.warmup_time @property @abc.abstractmethod diff --git a/lerobot/common/cameras/configs.py b/lerobot/common/cameras/configs.py index 1f8dda865..6189db4f9 100644 --- a/lerobot/common/cameras/configs.py +++ b/lerobot/common/cameras/configs.py @@ -21,12 +21,12 @@ from enum import Enum import draccus -class ColorMode(Enum): +class ColorMode(str, Enum): RGB = "rgb" BGR = "bgr" -class Cv2Rotation(Enum): +class Cv2Rotation(int, Enum): NO_ROTATION = 0 ROTATE_90 = 90 ROTATE_180 = 180 @@ -38,6 +38,7 @@ class CameraConfig(draccus.ChoiceRegistry, abc.ABC): fps: int | None = None width: int | None = None height: int | None = None + warmup_time: int | None = None @property def type(self) -> str: diff --git a/lerobot/common/cameras/opencv/camera_opencv.py b/lerobot/common/cameras/opencv/camera_opencv.py index d4ba26cd8..fd252bdc3 100644 --- a/lerobot/common/cameras/opencv/camera_opencv.py +++ b/lerobot/common/cameras/opencv/camera_opencv.py @@ -113,6 +113,7 @@ class OpenCVCamera(Camera): self.fps = config.fps self.color_mode = config.color_mode + self.warmup_time = config.warmup_time self.videocapture: cv2.VideoCapture | None = None @@ -121,7 +122,7 @@ class OpenCVCamera(Camera): self.frame_queue: queue.Queue = queue.Queue(maxsize=1) self.rotation: int | None = get_cv2_rotation(config.rotation) - self.backend: int = get_cv2_backend() # NOTE(Steven): If we specify backend the opencv open fails + self.backend: int = get_cv2_backend() if self.height and self.width: if self.rotation in [cv2.ROTATE_90_CLOCKWISE, cv2.ROTATE_90_COUNTERCLOCKWISE]: @@ -157,7 +158,7 @@ class OpenCVCamera(Camera): # blocking in multi-threaded applications, especially during data collection. cv2.setNumThreads(1) - self.videocapture = cv2.VideoCapture(self.index_or_path) + self.videocapture = cv2.VideoCapture(self.index_or_path, self.backend) if not self.videocapture.isOpened(): self.videocapture.release() @@ -170,8 +171,15 @@ class OpenCVCamera(Camera): self._configure_capture_settings() if warmup: - logger.debug(f"Reading a warm-up frame for {self.index_or_path}...") - self.read() # NOTE(Steven): For now we just read one frame, we could also loop for X frames/secs + if self.warmup_time is None: + raise ValueError( + f"Warmup time is not set for {self}. Please set a warmup time in the configuration." + ) + logger.debug(f"Reading a warm-up frames for {self} for {self.warmup_time} seconds...") + start_time = time.time() + while time.time() - start_time < self.warmup_time: + self.read() + time.sleep(0.1) logger.debug(f"Camera {self.index_or_path} connected and configured successfully.") diff --git a/lerobot/common/cameras/realsense/camera_realsense.py b/lerobot/common/cameras/realsense/camera_realsense.py index 2733315da..d06eb3ddb 100644 --- a/lerobot/common/cameras/realsense/camera_realsense.py +++ b/lerobot/common/cameras/realsense/camera_realsense.py @@ -18,7 +18,6 @@ Provides the RealSenseCamera class for capturing frames from Intel RealSense cam import contextlib import logging -import math import queue import time from threading import Event, Thread @@ -129,6 +128,7 @@ class RealSenseCamera(Camera): self.fps = config.fps self.color_mode = config.color_mode self.use_depth = config.use_depth + self.warmup_time = config.warmup_time self.rs_pipeline: rs.pipeline | None = None self.rs_profile: rs.pipeline_profile | None = None @@ -186,8 +186,15 @@ class RealSenseCamera(Camera): self._validate_capture_settings() if warmup: - logger.debug(f"Reading a warm-up frame for {self}...") - self.read() # NOTE(Steven): For now we just read one frame, we could also loop for X frames/secs + if self.warmup_time is None: + raise ValueError( + f"Warmup time is not set for {self}. Please set a warmup time in the configuration." + ) + logger.debug(f"Reading a warm-up frames for {self} for {self.warmup_time} seconds...") + start_time = time.time() + while time.time() - start_time < self.warmup_time: + self.read() + time.sleep(0.1) logger.info(f"{self} connected.") @@ -314,8 +321,6 @@ class RealSenseCamera(Camera): if self.fps is None: self.fps = stream.fps() - else: - self._validate_fps(stream) if self.width is None or self.height is None: actual_width = int(round(stream.width())) @@ -333,20 +338,8 @@ class RealSenseCamera(Camera): if self.use_depth: stream = self.rs_profile.get_stream(rs.stream.depth).as_video_stream_profile() - self._validate_fps(stream) self._validate_width_and_height(stream) - def _validate_fps(self, stream: rs.video_stream_profile) -> None: - """Validates and sets the internal FPS based on actual stream FPS.""" - actual_fps = stream.fps() - - # Use math.isclose for robust float comparison - if not math.isclose(self.fps, actual_fps, rel_tol=1e-3): - raise RuntimeError( - f"Failed to set requested FPS {self.fps} for {self}. Actual value reported: {actual_fps}." - ) - logger.debug(f"FPS set to {actual_fps} for {self}.") - def _validate_width_and_height(self, stream) -> None: """Validates and sets the internal capture width and height based on actual stream width.""" actual_width = int(round(stream.width())) diff --git a/lerobot/common/cameras/utils.py b/lerobot/common/cameras/utils.py index 809fe0851..d95703da5 100644 --- a/lerobot/common/cameras/utils.py +++ b/lerobot/common/cameras/utils.py @@ -62,7 +62,7 @@ def get_cv2_backend() -> int: import cv2 available_cv2_backends = { - "Linux": cv2.CAP_DSHOW, + "Linux": cv2.CAP_V4L2, "Windows": cv2.CAP_AVFOUNDATION, "Darwin": cv2.CAP_ANY, } diff --git a/tests/cameras/test_opencv.py b/tests/cameras/test_opencv.py index 56ba3bb29..dda2a4fba 100644 --- a/tests/cameras/test_opencv.py +++ b/tests/cameras/test_opencv.py @@ -145,7 +145,9 @@ def test_async_read_timeout(): try: with pytest.raises(TimeoutError): - camera.async_read(timeout_ms=0) + camera.async_read( + timeout_ms=0 + ) # NOTE(Steven): This is flaky as sdometimes we actually get a frame finally: if camera.is_connected: camera.disconnect() diff --git a/tests/cameras/test_realsense.py b/tests/cameras/test_realsense.py index 91882991f..e0662fb2e 100644 --- a/tests/cameras/test_realsense.py +++ b/tests/cameras/test_realsense.py @@ -110,7 +110,7 @@ def test_read_depth(): camera = RealSenseCamera(config) camera.connect(warmup=False) - img = camera.read_depth(timeout_ms=500) # NOTE(Steven): Reading depth takes longer + img = camera.read_depth(timeout_ms=1000) # NOTE(Steven): Reading depth takes longer assert isinstance(img, np.ndarray)