From aa1d906802fbe2a55111dc15e31b748c86773cae Mon Sep 17 00:00:00 2001 From: hls <56255627+forgetwhatuwant@users.noreply.github.com> Date: Mon, 20 Oct 2025 20:19:21 +0800 Subject: [PATCH] Enhance OpenCVCamera with FOURCC for MJPEG support and validation (#1558) * Enhance OpenCVCamera with FOURCC support and validation - Added FOURCC configuration option to OpenCVCamera and OpenCVCameraConfig for specifying video format. - Implemented _validate_fourcc method to validate and set the camera's FOURCC code. - Updated _configure_capture_settings to apply FOURCC settings before FPS and resolution. - Enhanced camera detection to include default FOURCC code in camera info. - Updated documentation to reflect new FOURCC parameter and its implications on performance. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Add tests for FOURCC configuration in OpenCVCamera - Implemented tests to validate FOURCC configuration and its application in OpenCVCamera. - Added checks for valid FOURCC codes and ensured that invalid codes raise appropriate errors. - Included a test for camera connection functionality using specified FOURCC settings. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix circular import in __init__.py - change to relative import * Update src/lerobot/cameras/opencv/configuration_opencv.py Co-authored-by: Steven Palma Signed-off-by: hls <56255627+forgetwhatuwant@users.noreply.github.com> * Update src/lerobot/cameras/opencv/configuration_opencv.py Co-authored-by: Steven Palma Signed-off-by: hls <56255627+forgetwhatuwant@users.noreply.github.com> * fix(camera_opencv): ensure MSMF hardware transform compatibility on Windows before importing OpenCV * This change reverts the import from a relative import (.) back to the absolute import (lerobot.) as it was previously * opencv/config: satisfy Ruff SIM102 by merging nested if for fourcc validation * style(opencv/config): apply ruff-format changes --------- Signed-off-by: hls <56255627+forgetwhatuwant@users.noreply.github.com> Signed-off-by: Steven Palma Co-authored-by: forgetwhatuwant Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Steven Palma --- src/lerobot/cameras/opencv/camera_opencv.py | 31 +++++++++++++- .../cameras/opencv/configuration_opencv.py | 14 ++++++- tests/cameras/test_opencv.py | 40 +++++++++++++++++++ 3 files changed, 82 insertions(+), 3 deletions(-) diff --git a/src/lerobot/cameras/opencv/camera_opencv.py b/src/lerobot/cameras/opencv/camera_opencv.py index 9e278dfd..708340ab 100644 --- a/src/lerobot/cameras/opencv/camera_opencv.py +++ b/src/lerobot/cameras/opencv/camera_opencv.py @@ -181,12 +181,14 @@ class OpenCVCamera(Camera): def _configure_capture_settings(self) -> None: """ - Applies the specified FPS, width, and height settings to the connected camera. + Applies the specified FOURCC, FPS, width, and height settings to the connected camera. This method attempts to set the camera properties via OpenCV. It checks if the camera successfully applied the settings and raises an error if not. + FOURCC is set first (if specified) as it can affect the available FPS and resolution options. Args: + fourcc: The desired FOURCC code (e.g., "MJPG", "YUYV"). If None, auto-detect. fps: The desired frames per second. If None, the setting is skipped. width: The desired capture width. If None, the setting is skipped. height: The desired capture height. If None, the setting is skipped. @@ -200,6 +202,9 @@ class OpenCVCamera(Camera): if not self.is_connected: raise DeviceNotConnectedError(f"Cannot configure settings for {self} as it is not connected.") + # Set FOURCC first (if specified) as it can affect available FPS/resolution options + if self.config.fourcc is not None: + self._validate_fourcc() if self.videocapture is None: raise DeviceNotConnectedError(f"{self} videocapture is not initialized") @@ -235,6 +240,23 @@ class OpenCVCamera(Camera): if not success or not math.isclose(self.fps, actual_fps, rel_tol=1e-3): raise RuntimeError(f"{self} failed to set fps={self.fps} ({actual_fps=}).") + def _validate_fourcc(self) -> None: + """Validates and sets the camera's FOURCC code.""" + + fourcc_code = cv2.VideoWriter_fourcc(*self.config.fourcc) + success = self.videocapture.set(cv2.CAP_PROP_FOURCC, fourcc_code) + actual_fourcc_code = self.videocapture.get(cv2.CAP_PROP_FOURCC) + + # Convert actual FOURCC code back to string for comparison + actual_fourcc_code_int = int(actual_fourcc_code) + actual_fourcc = "".join([chr((actual_fourcc_code_int >> 8 * i) & 0xFF) for i in range(4)]) + + if not success or actual_fourcc != self.config.fourcc: + logger.warning( + f"{self} failed to set fourcc={self.config.fourcc} (actual={actual_fourcc}, success={success}). " + f"Continuing with default format." + ) + def _validate_width_and_height(self) -> None: """Validates and sets the camera's frame capture width and height.""" @@ -287,6 +309,12 @@ class OpenCVCamera(Camera): default_height = int(camera.get(cv2.CAP_PROP_FRAME_HEIGHT)) default_fps = camera.get(cv2.CAP_PROP_FPS) default_format = camera.get(cv2.CAP_PROP_FORMAT) + + # Get FOURCC code and convert to string + default_fourcc_code = camera.get(cv2.CAP_PROP_FOURCC) + default_fourcc_code_int = int(default_fourcc_code) + default_fourcc = "".join([chr((default_fourcc_code_int >> 8 * i) & 0xFF) for i in range(4)]) + camera_info = { "name": f"OpenCV Camera @ {target}", "type": "OpenCV", @@ -294,6 +322,7 @@ class OpenCVCamera(Camera): "backend_api": camera.getBackendName(), "default_stream_profile": { "format": default_format, + "fourcc": default_fourcc, "width": default_width, "height": default_height, "fps": default_fps, diff --git a/src/lerobot/cameras/opencv/configuration_opencv.py b/src/lerobot/cameras/opencv/configuration_opencv.py index b66fb31b..37a42861 100644 --- a/src/lerobot/cameras/opencv/configuration_opencv.py +++ b/src/lerobot/cameras/opencv/configuration_opencv.py @@ -35,8 +35,9 @@ class OpenCVCameraConfig(CameraConfig): OpenCVCameraConfig(0, 30, 1280, 720) # 1280x720 @ 30FPS OpenCVCameraConfig(/dev/video4, 60, 640, 480) # 640x480 @ 60FPS - # Advanced configurations - OpenCVCameraConfig(128422271347, 30, 640, 480, rotation=Cv2Rotation.ROTATE_90) # With 90° rotation + # Advanced configurations with FOURCC format + OpenCVCameraConfig(128422271347, 30, 640, 480, rotation=Cv2Rotation.ROTATE_90, fourcc="MJPG") # With 90° rotation and MJPG format + OpenCVCameraConfig(0, 30, 1280, 720, fourcc="YUYV") # With YUYV format ``` Attributes: @@ -48,15 +49,19 @@ class OpenCVCameraConfig(CameraConfig): color_mode: Color mode for image output (RGB or BGR). Defaults to RGB. rotation: Image rotation setting (0°, 90°, 180°, or 270°). Defaults to no rotation. warmup_s: Time reading frames before returning from connect (in seconds) + fourcc: FOURCC code for video format (e.g., "MJPG", "YUYV", "I420"). Defaults to None (auto-detect). Note: - Only 3-channel color output (RGB/BGR) is currently supported. + - FOURCC codes must be 4-character strings (e.g., "MJPG", "YUYV"). Some common FOUCC codes: https://learn.microsoft.com/en-us/windows/win32/medfound/video-fourccs#fourcc-constants + - Setting FOURCC can help achieve higher frame rates on some cameras. """ index_or_path: int | Path color_mode: ColorMode = ColorMode.RGB rotation: Cv2Rotation = Cv2Rotation.NO_ROTATION warmup_s: int = 1 + fourcc: str | None = None def __post_init__(self) -> None: if self.color_mode not in (ColorMode.RGB, ColorMode.BGR): @@ -73,3 +78,8 @@ class OpenCVCameraConfig(CameraConfig): raise ValueError( f"`rotation` is expected to be in {(Cv2Rotation.NO_ROTATION, Cv2Rotation.ROTATE_90, Cv2Rotation.ROTATE_180, Cv2Rotation.ROTATE_270)}, but {self.rotation} is provided." ) + + if self.fourcc is not None and (not isinstance(self.fourcc, str) or len(self.fourcc) != 4): + raise ValueError( + f"`fourcc` must be a 4-character string (e.g., 'MJPG', 'YUYV'), but '{self.fourcc}' is provided." + ) diff --git a/tests/cameras/test_opencv.py b/tests/cameras/test_opencv.py index a3d98a67..3cf3793b 100644 --- a/tests/cameras/test_opencv.py +++ b/tests/cameras/test_opencv.py @@ -155,6 +155,46 @@ def test_async_read_before_connect(): _ = camera.async_read() +def test_fourcc_configuration(): + """Test FourCC configuration validation and application.""" + + # Test MJPG specifically (main use case) + config = OpenCVCameraConfig(index_or_path=DEFAULT_PNG_FILE_PATH, fourcc="MJPG") + camera = OpenCVCamera(config) + assert camera.config.fourcc == "MJPG" + + # Test a few other common formats + valid_fourcc_codes = ["YUYV", "YUY2", "RGB3"] + + for fourcc in valid_fourcc_codes: + config = OpenCVCameraConfig(index_or_path=DEFAULT_PNG_FILE_PATH, fourcc=fourcc) + camera = OpenCVCamera(config) + assert camera.config.fourcc == fourcc + + # Test invalid FOURCC codes + invalid_fourcc_codes = ["ABC", "ABCDE", ""] + + for fourcc in invalid_fourcc_codes: + with pytest.raises(ValueError): + OpenCVCameraConfig(index_or_path=DEFAULT_PNG_FILE_PATH, fourcc=fourcc) + + +def test_fourcc_with_camera(): + """Test FourCC functionality with actual camera connection.""" + config = OpenCVCameraConfig(index_or_path=DEFAULT_PNG_FILE_PATH, fourcc="MJPG") + camera = OpenCVCamera(config) + + # Connect should work with MJPG specified + camera.connect(warmup=False) + assert camera.is_connected + + # Read should work normally + img = camera.read() + assert isinstance(img, np.ndarray) + + camera.disconnect() + + @pytest.mark.parametrize("index_or_path", TEST_IMAGE_PATHS, ids=TEST_IMAGE_SIZES) @pytest.mark.parametrize( "rotation",