chore(cameras): address unresolved conversations

lerobot/common/cameras/camera.py

@@ -36,6 +36,7 @@ class Camera(abc.ABC):
         fps (int | None): Configured frames per second
         width (int | None): Frame width in pixels
         height (int | None): Frame height in pixels
+        warmup_time (int | None): Time reading frames before returning from connect (in seconds)
 
     Example:
         class MyCamera(Camera):
@@ -55,6 +56,7 @@ class Camera(abc.ABC):
         self.fps: int | None = config.fps
         self.width: int | None = config.width
         self.height: int | None = config.height
+        self.warmup_time: int | None = config.warmup_time
 
     @property
     @abc.abstractmethod

lerobot/common/cameras/configs.py

@@ -21,12 +21,12 @@ from enum import Enum
 import draccus
 
 
-class ColorMode(Enum):
+class ColorMode(str, Enum):
     RGB = "rgb"
     BGR = "bgr"
 
 
-class Cv2Rotation(Enum):
+class Cv2Rotation(int, Enum):
     NO_ROTATION = 0
     ROTATE_90 = 90
     ROTATE_180 = 180
@@ -38,6 +38,7 @@ class CameraConfig(draccus.ChoiceRegistry, abc.ABC):
     fps: int | None = None
     width: int | None = None
     height: int | None = None
+    warmup_time: int | None = None
 
     @property
     def type(self) -> str:

lerobot/common/cameras/opencv/camera_opencv.py

@@ -113,6 +113,7 @@ class OpenCVCamera(Camera):
 
         self.fps = config.fps
         self.color_mode = config.color_mode
+        self.warmup_time = config.warmup_time
 
         self.videocapture: cv2.VideoCapture | None = None
 
@@ -121,7 +122,7 @@ class OpenCVCamera(Camera):
         self.frame_queue: queue.Queue = queue.Queue(maxsize=1)
 
         self.rotation: int | None = get_cv2_rotation(config.rotation)
-        self.backend: int = get_cv2_backend()  # NOTE(Steven): If we specify backend the opencv open fails
+        self.backend: int = get_cv2_backend()
 
         if self.height and self.width:
             if self.rotation in [cv2.ROTATE_90_CLOCKWISE, cv2.ROTATE_90_COUNTERCLOCKWISE]:
@@ -157,7 +158,7 @@ class OpenCVCamera(Camera):
         # blocking in multi-threaded applications, especially during data collection.
         cv2.setNumThreads(1)
 
-        self.videocapture = cv2.VideoCapture(self.index_or_path)
+        self.videocapture = cv2.VideoCapture(self.index_or_path, self.backend)
 
         if not self.videocapture.isOpened():
             self.videocapture.release()
@@ -170,8 +171,15 @@ class OpenCVCamera(Camera):
         self._configure_capture_settings()
 
         if warmup:
-            logger.debug(f"Reading a warm-up frame for {self.index_or_path}...")
-            self.read()  # NOTE(Steven): For now we just read one frame, we could also loop for X frames/secs
+            if self.warmup_time is None:
+                raise ValueError(
+                    f"Warmup time is not set for {self}. Please set a warmup time in the configuration."
+                )
+            logger.debug(f"Reading a warm-up frames for {self} for {self.warmup_time} seconds...")
+            start_time = time.time()
+            while time.time() - start_time < self.warmup_time:
+                self.read()
+                time.sleep(0.1)
 
         logger.debug(f"Camera {self.index_or_path} connected and configured successfully.")
 

lerobot/common/cameras/realsense/camera_realsense.py

@@ -18,7 +18,6 @@ Provides the RealSenseCamera class for capturing frames from Intel RealSense cam
 
 import contextlib
 import logging
-import math
 import queue
 import time
 from threading import Event, Thread
@@ -129,6 +128,7 @@ class RealSenseCamera(Camera):
         self.fps = config.fps
         self.color_mode = config.color_mode
         self.use_depth = config.use_depth
+        self.warmup_time = config.warmup_time
 
         self.rs_pipeline: rs.pipeline | None = None
         self.rs_profile: rs.pipeline_profile | None = None
@@ -186,8 +186,15 @@ class RealSenseCamera(Camera):
         self._validate_capture_settings()
 
         if warmup:
-            logger.debug(f"Reading a warm-up frame for {self}...")
-            self.read()  # NOTE(Steven): For now we just read one frame, we could also loop for X frames/secs
+            if self.warmup_time is None:
+                raise ValueError(
+                    f"Warmup time is not set for {self}. Please set a warmup time in the configuration."
+                )
+            logger.debug(f"Reading a warm-up frames for {self} for {self.warmup_time} seconds...")
+            start_time = time.time()
+            while time.time() - start_time < self.warmup_time:
+                self.read()
+                time.sleep(0.1)
 
         logger.info(f"{self} connected.")
 
@@ -314,8 +321,6 @@ class RealSenseCamera(Camera):
 
         if self.fps is None:
             self.fps = stream.fps()
-        else:
-            self._validate_fps(stream)
 
         if self.width is None or self.height is None:
             actual_width = int(round(stream.width()))
@@ -333,20 +338,8 @@ class RealSenseCamera(Camera):
 
         if self.use_depth:
             stream = self.rs_profile.get_stream(rs.stream.depth).as_video_stream_profile()
-            self._validate_fps(stream)
             self._validate_width_and_height(stream)
 
-    def _validate_fps(self, stream: rs.video_stream_profile) -> None:
-        """Validates and sets the internal FPS based on actual stream FPS."""
-        actual_fps = stream.fps()
-
-        # Use math.isclose for robust float comparison
-        if not math.isclose(self.fps, actual_fps, rel_tol=1e-3):
-            raise RuntimeError(
-                f"Failed to set requested FPS {self.fps} for {self}. Actual value reported: {actual_fps}."
-            )
-        logger.debug(f"FPS set to {actual_fps} for {self}.")
-
     def _validate_width_and_height(self, stream) -> None:
         """Validates and sets the internal capture width and height based on actual stream width."""
         actual_width = int(round(stream.width()))

lerobot/common/cameras/utils.py

@@ -62,7 +62,7 @@ def get_cv2_backend() -> int:
     import cv2
 
     available_cv2_backends = {
-        "Linux": cv2.CAP_DSHOW,
+        "Linux": cv2.CAP_V4L2,
         "Windows": cv2.CAP_AVFOUNDATION,
         "Darwin": cv2.CAP_ANY,
     }

tests/cameras/test_opencv.py

@@ -145,7 +145,9 @@ def test_async_read_timeout():
 
     try:
         with pytest.raises(TimeoutError):
-            camera.async_read(timeout_ms=0)
+            camera.async_read(
+                timeout_ms=0
+            )  # NOTE(Steven): This is flaky as sdometimes we actually get a frame
     finally:
         if camera.is_connected:
             camera.disconnect()

tests/cameras/test_realsense.py

@@ -110,7 +110,7 @@ def test_read_depth():
     camera = RealSenseCamera(config)
     camera.connect(warmup=False)
 
-    img = camera.read_depth(timeout_ms=500)  # NOTE(Steven): Reading depth takes longer
+    img = camera.read_depth(timeout_ms=1000)  # NOTE(Steven): Reading depth takes longer
     assert isinstance(img, np.ndarray)