From be5336f9d3b60b2f9364ed2afe27c5a1b0fe8d5f Mon Sep 17 00:00:00 2001 From: jess-moss Date: Fri, 20 Sep 2024 14:53:15 -0500 Subject: [PATCH 1/2] Made script to save camera images instead of these funcitons existing in the camera driver classes. --- examples/7_get_started_with_real_robot.md | 3 +- .../robot_devices/cameras/intelrealsense.py | 141 +--------------- .../common/robot_devices/cameras/opencv.py | 125 +------------- lerobot/scripts/save_images_from_cameras.py | 158 ++++++++++++++++++ 4 files changed, 168 insertions(+), 259 deletions(-) create mode 100644 lerobot/scripts/save_images_from_cameras.py diff --git a/examples/7_get_started_with_real_robot.md b/examples/7_get_started_with_real_robot.md index 50a2c645..97529e95 100644 --- a/examples/7_get_started_with_real_robot.md +++ b/examples/7_get_started_with_real_robot.md @@ -544,7 +544,8 @@ To instantiate an [`OpenCVCamera`](../lerobot/common/robot_devices/cameras/openc To find the camera indices, run the following utility script, which will save a few frames from each detected camera: ```bash -python lerobot/common/robot_devices/cameras/opencv.py \ +python lerobot/scripts/save_images_from_cameras.py \ + --driver opencv \ --images-dir outputs/images_from_opencv_cameras ``` diff --git a/lerobot/common/robot_devices/cameras/intelrealsense.py b/lerobot/common/robot_devices/cameras/intelrealsense.py index 4806bf78..99866e23 100644 --- a/lerobot/common/robot_devices/cameras/intelrealsense.py +++ b/lerobot/common/robot_devices/cameras/intelrealsense.py @@ -2,28 +2,21 @@ This file contains utilities for recording frames from Intel Realsense cameras. """ -import argparse -import concurrent.futures -import logging -import shutil import threading import time import traceback from dataclasses import dataclass, replace -from pathlib import Path from threading import Thread import cv2 import numpy as np import pyrealsense2 as rs -from PIL import Image from lerobot.common.robot_devices.utils import ( RobotDeviceAlreadyConnectedError, RobotDeviceNotConnectedError, ) from lerobot.common.utils.utils import capture_timestamp_utc -from lerobot.scripts.control_robot import busy_wait SERIAL_NUMBER_INDEX = 1 @@ -46,89 +39,6 @@ def find_camera_indices(raise_when_empty=True) -> list[int]: return camera_ids -def save_image(img_array, camera_idx, frame_index, images_dir): - try: - img = Image.fromarray(img_array) - path = images_dir / f"camera_{camera_idx}_frame_{frame_index:06d}.png" - path.parent.mkdir(parents=True, exist_ok=True) - img.save(str(path), quality=100) - logging.info(f"Saved image: {path}") - except Exception as e: - logging.error(f"Failed to save image for camera {camera_idx} frame {frame_index}: {e}") - - -def save_images_from_cameras( - images_dir: Path, - camera_ids: list[int] | None = None, - fps=None, - width=None, - height=None, - record_time_s=2, -): - """ - Initializes all the cameras and saves images to the directory. Useful to visually identify the camera - associated to a given camera index. - """ - if camera_ids is None: - camera_ids = find_camera_indices() - - print("Connecting cameras") - cameras = [] - for cam_idx in camera_ids: - camera = IntelRealSenseCamera(cam_idx, fps=fps, width=width, height=height) - camera.connect() - print( - f"IntelRealSenseCamera({camera.camera_index}, fps={camera.fps}, width={camera.width}, height={camera.height}, color_mode={camera.color_mode})" - ) - cameras.append(camera) - - images_dir = Path(images_dir) - if images_dir.exists(): - shutil.rmtree( - images_dir, - ) - images_dir.mkdir(parents=True, exist_ok=True) - - print(f"Saving images to {images_dir}") - frame_index = 0 - start_time = time.perf_counter() - try: - with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor: - while True: - now = time.perf_counter() - - for camera in cameras: - # If we use async_read when fps is None, the loop will go full speed, and we will end up - # saving the same images from the cameras multiple times until the RAM/disk is full. - image = camera.read() if fps is None else camera.async_read() - if image is None: - print("No Frame") - bgr_converted_image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR) - - executor.submit( - save_image, - bgr_converted_image, - camera.camera_index, - frame_index, - images_dir, - ) - - if fps is not None: - dt_s = time.perf_counter() - now - busy_wait(1 / fps - dt_s) - - if time.perf_counter() - start_time > record_time_s: - break - - print(f"Frame: {frame_index:04d}\tLatency (ms): {(time.perf_counter() - now) * 1000:.2f}") - - frame_index += 1 - finally: - print(f"Images have been saved to {images_dir}") - for camera in cameras: - camera.disconnect() - - @dataclass class IntelRealSenseCameraConfig: """ @@ -173,7 +83,9 @@ class IntelRealSenseCamera: To find the camera indices of your cameras, you can run our utility script that will save a few frames for each camera: ```bash - python lerobot/common/robot_devices/cameras/intelrealsense.py --images-dir outputs/images_from_intelrealsense_cameras + python lerobot/scripts/save_images_from_cameras.py \ + --driver intelrealsense \ + --images-dir outputs/images_from_intelrealsense_cameras ``` When an IntelRealSenseCamera is instantiated, if no specific config is provided, the default fps, width, height and color_mode @@ -274,7 +186,7 @@ class IntelRealSenseCamera: if self.camera_index not in available_cam_ids: raise ValueError( f"`camera_index` is expected to be one of these available cameras {available_cam_ids}, but {self.camera_index} is provided instead. " - "To find the camera index you should use, run `python lerobot/common/robot_devices/cameras/intelrealsense.py`." + "To find the camera index you should use, run `python lerobot/scripts/save_images_from_cameras.py --driver intelrealsense`." ) raise OSError(f"Can't access IntelRealSenseCamera({self.camera_index}).") @@ -401,48 +313,3 @@ class IntelRealSenseCamera: def __del__(self): if getattr(self, "is_connected", False): self.disconnect() - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Save a few frames using `IntelRealSenseCamera` for all cameras connected to the computer, or a selected subset." - ) - parser.add_argument( - "--camera-ids", - type=int, - nargs="*", - default=None, - help="List of camera indices used to instantiate the `IntelRealSenseCamera`. If not provided, find and use all available camera indices.", - ) - parser.add_argument( - "--fps", - type=int, - default=30, - help="Set the number of frames recorded per seconds for all cameras. If not provided, use the default fps of each camera.", - ) - parser.add_argument( - "--width", - type=str, - default=640, - help="Set the width for all cameras. If not provided, use the default width of each camera.", - ) - parser.add_argument( - "--height", - type=str, - default=480, - help="Set the height for all cameras. If not provided, use the default height of each camera.", - ) - parser.add_argument( - "--images-dir", - type=Path, - default="outputs/images_from_intelrealsense_cameras", - help="Set directory to save a few frames for each camera.", - ) - parser.add_argument( - "--record-time-s", - type=float, - default=2.0, - help="Set the number of seconds used to record the frames. By default, 2 seconds.", - ) - args = parser.parse_args() - save_images_from_cameras(**vars(args)) diff --git a/lerobot/common/robot_devices/cameras/opencv.py b/lerobot/common/robot_devices/cameras/opencv.py index b066a451..9da6d8bb 100644 --- a/lerobot/common/robot_devices/cameras/opencv.py +++ b/lerobot/common/robot_devices/cameras/opencv.py @@ -2,11 +2,8 @@ This file contains utilities for recording frames from cameras. For more info look at `OpenCVCamera` docstring. """ -import argparse -import concurrent.futures import math import platform -import shutil import threading import time from dataclasses import dataclass, replace @@ -15,12 +12,10 @@ from threading import Thread import cv2 import numpy as np -from PIL import Image from lerobot.common.robot_devices.utils import ( RobotDeviceAlreadyConnectedError, RobotDeviceNotConnectedError, - busy_wait, ) from lerobot.common.utils.utils import capture_timestamp_utc @@ -70,75 +65,6 @@ def find_camera_indices(raise_when_empty=False, max_index_search_range=MAX_OPENC return camera_ids -def save_image(img_array, camera_index, frame_index, images_dir): - img = Image.fromarray(img_array) - path = images_dir / f"camera_{camera_index:02d}_frame_{frame_index:06d}.png" - path.parent.mkdir(parents=True, exist_ok=True) - img.save(str(path), quality=100) - - -def save_images_from_cameras( - images_dir: Path, camera_ids: list[int] | None = None, fps=None, width=None, height=None, record_time_s=2 -): - """ - Initializes all the cameras and saves images to the directory. Useful to visually identify the camera - associated to a given camera index. - """ - if camera_ids is None: - camera_ids = find_camera_indices() - - print("Connecting cameras") - cameras = [] - for cam_idx in camera_ids: - camera = OpenCVCamera(cam_idx, fps=fps, width=width, height=height) - camera.connect() - print( - f"OpenCVCamera({camera.camera_index}, fps={camera.fps}, width={camera.width}, " - f"height={camera.height}, color_mode={camera.color_mode})" - ) - cameras.append(camera) - - images_dir = Path(images_dir) - if images_dir.exists(): - shutil.rmtree( - images_dir, - ) - images_dir.mkdir(parents=True, exist_ok=True) - - print(f"Saving images to {images_dir}") - frame_index = 0 - start_time = time.perf_counter() - with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor: - while True: - now = time.perf_counter() - - for camera in cameras: - # If we use async_read when fps is None, the loop will go full speed, and we will endup - # saving the same images from the cameras multiple times until the RAM/disk is full. - image = camera.read() if fps is None else camera.async_read() - - executor.submit( - save_image, - image, - camera.camera_index, - frame_index, - images_dir, - ) - - if fps is not None: - dt_s = time.perf_counter() - now - busy_wait(1 / fps - dt_s) - - if time.perf_counter() - start_time > record_time_s: - break - - print(f"Frame: {frame_index:04d}\tLatency (ms): {(time.perf_counter() - now) * 1000:.2f}") - - frame_index += 1 - - print(f"Images have been saved to {images_dir}") - - @dataclass class OpenCVCameraConfig: """ @@ -175,7 +101,9 @@ class OpenCVCamera: To find the camera indices of your cameras, you can run our utility script that will be save a few frames for each camera: ```bash - python lerobot/common/robot_devices/cameras/opencv.py --images-dir outputs/images_from_opencv_cameras + python lerobot/scripts/save_images_from_cameras.py \ + --driver opencv \ + --images-dir outputs/images_from_opencv_cameras ``` When an OpenCVCamera is instantiated, if no specific config is provided, the default fps, width, height and color_mode @@ -248,7 +176,7 @@ class OpenCVCamera: if self.camera_index not in available_cam_ids: raise ValueError( f"`camera_index` is expected to be one of these available cameras {available_cam_ids}, but {self.camera_index} is provided instead. " - "To find the camera index you should use, run `python lerobot/common/robot_devices/cameras/opencv.py`." + "To find the camera index you should use, run `python lerobot/scripts/save_images_from_cameras.py --driver opencv`." ) raise OSError(f"Can't access OpenCVCamera({self.camera_index}).") @@ -384,48 +312,3 @@ class OpenCVCamera: def __del__(self): if getattr(self, "is_connected", False): self.disconnect() - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Save a few frames using `OpenCVCamera` for all cameras connected to the computer, or a selected subset." - ) - parser.add_argument( - "--camera-ids", - type=int, - nargs="*", - default=None, - help="List of camera indices used to instantiate the `OpenCVCamera`. If not provided, find and use all available camera indices.", - ) - parser.add_argument( - "--fps", - type=int, - default=None, - help="Set the number of frames recorded per seconds for all cameras. If not provided, use the default fps of each camera.", - ) - parser.add_argument( - "--width", - type=str, - default=None, - help="Set the width for all cameras. If not provided, use the default width of each camera.", - ) - parser.add_argument( - "--height", - type=str, - default=None, - help="Set the height for all cameras. If not provided, use the default height of each camera.", - ) - parser.add_argument( - "--images-dir", - type=Path, - default="outputs/images_from_opencv_cameras", - help="Set directory to save a few frames for each camera.", - ) - parser.add_argument( - "--record-time-s", - type=float, - default=2.0, - help="Set the number of seconds used to record the frames. By default, 2 seconds.", - ) - args = parser.parse_args() - save_images_from_cameras(**vars(args)) diff --git a/lerobot/scripts/save_images_from_cameras.py b/lerobot/scripts/save_images_from_cameras.py new file mode 100644 index 00000000..0bd700dd --- /dev/null +++ b/lerobot/scripts/save_images_from_cameras.py @@ -0,0 +1,158 @@ +import argparse +import concurrent.futures +import importlib +import shutil +import time +from pathlib import Path + +import cv2 +from PIL import Image + +from lerobot.scripts.control_robot import busy_wait + + +def save_image(img_array, camera_index, frame_index, images_dir): + try: + img = Image.fromarray(img_array) + path = images_dir / f"camera_{camera_index:02d}_frame_{frame_index:06d}.png" + path.parent.mkdir(parents=True, exist_ok=True) + img.save(str(path), quality=100) + print(f"Image saved to: {path}") + except Exception as e: + print(f"Failed to save image: {e}") + + +def save_images_from_cameras( + driver: str, + images_dir: Path, + camera_ids: list[int] | None = None, + fps=None, + width=None, + height=None, + record_time_s=2, +): + """ + Initializes all the cameras and saves images to the directory. Useful to visually identify the camera + associated to a given camera index. + """ + # Dynamically import the appropriate camera class based on the brand + if driver == "intelrealsense": + camera_module = importlib.import_module("lerobot.common.robot_devices.cameras.intelrealsense") + camera_class = camera_module.IntelRealSenseCamera + find_camera_indices = camera_module.find_camera_indices + elif driver == "opencv": + camera_module = importlib.import_module("lerobot.common.robot_devices.cameras.opencv") + camera_class = camera_module.OpenCVCamera + find_camera_indices = camera_module.find_camera_indices + else: + raise ValueError( + f"Unsupported camera driver: {driver}. Note: the drivers we currently support are opencv and intelrealsense." + ) + + if camera_ids is None: + camera_ids = find_camera_indices() + + print("Connecting cameras") + cameras = [] + for cam_idx in camera_ids: + camera = camera_class(cam_idx, fps=fps, width=width, height=height) + camera.connect() + print( + f"{camera.__class__.__name__}({camera.camera_index}, fps={camera.fps}, width={camera.width}, height={camera.height}, color_mode={camera.color_mode})" + ) + cameras.append(camera) + + images_dir = Path(images_dir) + if images_dir.exists(): + shutil.rmtree(images_dir) + images_dir.mkdir(parents=True, exist_ok=True) + + print(f"Saving images to {images_dir}") + frame_index = 0 + start_time = time.perf_counter() + + # Use ThreadPoolExecutor for saving images asynchronously + with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor: + try: + while True: + now = time.perf_counter() + + for camera in cameras: + # Capture image + image = camera.read() if fps is None else camera.async_read() + if image is None: + print("No Frame") + else: + bgr_converted_image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR) + + # Submit the save_image function to be executed in the background + executor.submit( + save_image, + bgr_converted_image, + camera.camera_index, + frame_index, + images_dir, + ) + + if fps is not None: + dt_s = time.perf_counter() - now + busy_wait(1 / fps - dt_s) + + if time.perf_counter() - start_time > record_time_s: + break + + print(f"Frame: {frame_index:04d}\tLatency (ms): {(time.perf_counter() - now) * 1000:.2f}") + + frame_index += 1 + finally: + print(f"Images have been saved to {images_dir}") + for camera in cameras: + camera.disconnect() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Save a few frames for all cameras connected to the computer, or a selected subset." + ) + parser.add_argument( + "--driver", type=str, required=True, help="Camera driver (e.g., intelrealsense, opencv)" + ) + parser.add_argument( + "--camera-ids", + type=int, + nargs="*", + default=None, + help="List of camera indices used to instantiate the `IntelRealSenseCamera`. If not provided, find and use all available camera indices.", + ) + parser.add_argument( + "--fps", + type=int, + default=30, + help="Set the number of frames recorded per second for all cameras. If not provided, use the default fps of each camera.", + ) + parser.add_argument( + "--width", + type=str, + default=640, + help="Set the width for all cameras. If not provided, use the default width of each camera.", + ) + parser.add_argument( + "--height", + type=str, + default=480, + help="Set the height for all cameras. If not provided, use the default height of each camera.", + ) + parser.add_argument( + "--images-dir", + type=Path, + default="outputs/images_from_cameras", + help="Set directory to save a few frames for each camera.", + ) + parser.add_argument( + "--record-time-s", + type=float, + default=2.0, + help="Set the number of seconds used to record the frames. By default, 2 seconds.", + ) + args = parser.parse_args() + save_images_from_cameras(**vars(args)) From 34dcd05a96121c1e3e8cff7737b0a3607b9a1255 Mon Sep 17 00:00:00 2001 From: jess-moss Date: Fri, 20 Sep 2024 14:53:15 -0500 Subject: [PATCH 2/2] Made script to save camera images instead of these funcitons existing in the camera driver classes. Fixed test_cameras.py script so it passes. --- tests/test_cameras.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/test_cameras.py b/tests/test_cameras.py index 0d5d9442..f5cb0e09 100644 --- a/tests/test_cameras.py +++ b/tests/test_cameras.py @@ -7,12 +7,15 @@ pytest -sx tests/test_cameras.py::test_camera ``` """ +from pathlib import Path + import numpy as np import pytest from lerobot import available_robots -from lerobot.common.robot_devices.cameras.opencv import OpenCVCamera, save_images_from_cameras +from lerobot.common.robot_devices.cameras.opencv import OpenCVCamera from lerobot.common.robot_devices.utils import RobotDeviceAlreadyConnectedError, RobotDeviceNotConnectedError +from lerobot.scripts.save_images_from_cameras import save_images_from_cameras from tests.utils import require_robot CAMERA_INDEX = 2 @@ -131,7 +134,8 @@ def test_camera(request, robot_type): del camera -@pytest.mark.parametrize("robot_type", available_robots) -@require_robot -def test_save_images_from_cameras(tmpdir, request, robot_type): - save_images_from_cameras(tmpdir, record_time_s=1) +def test_save_images_from_cameras(): + # Set the path relative to the project root (lerobot) where 'outputs' already exists + images_dir = Path("outputs/images_from_opencv_cameras") + images_dir.mkdir(parents=True, exist_ok=True) + save_images_from_cameras(driver="opencv", images_dir=images_dir)