add realman shadow src
Some checks failed
Secret Leaks / trufflehog (push) Has been cancelled

This commit is contained in:
2025-06-07 11:29:43 +08:00
parent e079566597
commit cf8df17d3a
98 changed files with 14215 additions and 0 deletions

View File

@@ -0,0 +1,4 @@
__pycache__/
*.pyc
*.pyo
*.pt

View File

@@ -0,0 +1,33 @@
[tool.poetry]
name = "shadow_camera"
version = "0.1.0"
description = "camera class, currently includes realsense"
readme = "README.md"
authors = ["Shadow <qiuchengzhan@gmail.com>"]
license = "MIT"
#include = ["realman_vision/pytransform/_pytransform.so",]
classifiers = [
"Operating System :: POSIX :: Linux amd64",
"Programming Language :: Python :: 3.10",
]
[tool.poetry.dependencies]
python = ">=3.9"
numpy = ">=2.0.1"
opencv-python = ">=4.10.0.84"
pyrealsense2 = ">=2.55.1.6486"
[tool.poetry.dev-dependencies] # 列出开发时所需的依赖项,比如测试、文档生成等工具。
pytest = ">=8.3"
black = ">=24.10.0"
[tool.poetry.plugins."scripts"] # 定义命令行脚本,使得用户可以通过命令行运行指定的函数。
[tool.poetry.group.dev.dependencies]
[build-system]
requires = ["poetry-core>=1.8.4"]
build-backend = "poetry.core.masonry.api"

View File

@@ -0,0 +1 @@
__version__ = '0.1.0'

View File

@@ -0,0 +1,38 @@
from abc import ABCMeta, abstractmethod
class BaseCamera(metaclass=ABCMeta):
"""摄像头基类"""
def __init__(self):
pass
@abstractmethod
def start_camera(self):
"""启动相机"""
pass
@abstractmethod
def stop_camera(self):
"""停止相机"""
pass
@abstractmethod
def set_resolution(self, resolution_width, resolution_height):
"""设置相机彩色图像分辨率"""
pass
@abstractmethod
def set_frame_rate(self, fps):
"""设置相机彩色图像帧率"""
pass
@abstractmethod
def read_frame(self):
"""读取一帧彩色图像和深度图像"""
pass
@abstractmethod
def get_camera_intrinsics(self):
"""获取彩色图像和深度图像的内参"""
pass

View File

@@ -0,0 +1,38 @@
from shadow_camera import base_camera
import cv2
class OpenCVCamera(base_camera.BaseCamera):
"""基于OpenCV的摄像头类"""
def __init__(self, device_id=0):
"""初始化视频捕获
参数:
device_id: 摄像头设备ID
"""
self.cap = cv2.VideoCapture(device_id)
def get_frame(self):
"""获取当前帧
返回:
frame: 当前帧的图像数据,取不到时返回None
"""
ret, frame = self.cap.read()
return frame if ret else None
def get_frame_info(self):
"""获取当前帧信息
返回:
dict: 帧信息字典
"""
width = int(self.cap.get(cv2.CAP_PROP_FRAME_WIDTH))
height = int(self.cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
channels = int(self.cap.get(cv2.CAP_PROP_FRAME_CHANNELS))
return {
'width': width,
'height': height,
'channels': channels
}

View File

@@ -0,0 +1,280 @@
import time
import logging
import numpy as np
import pyrealsense2 as rs
import base_camera
# 设置日志配置
logging.basicConfig(
level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
)
class RealSenseCamera(base_camera.BaseCamera):
"""Intel RealSense相机类"""
def __init__(self, serial_num=None, is_depth_frame=False):
"""
初始化相机对象
:param serial_num: 相机序列号默认为None
"""
super().__init__()
self._color_resolution = [640, 480]
self._depth_resolution = [640, 480]
self._color_frames_rate = 30
self._depth_frames_rate = 15
self.timestamp = 0
self.color_timestamp = 0
self.depth_timestamp = 0
self._colorizer = rs.colorizer()
self._config = rs.config()
self.is_depth_frame = is_depth_frame
self.camera_on = False
self.serial_num = serial_num
def get_serial_num(self):
serial_num = {}
context = rs.context()
devices = context.query_devices() # 获取所有设备
if len(context.devices) > 0:
for i, device in enumerate(devices):
serial_num[i] = device.get_info(rs.camera_info.serial_number)
logging.info(f"Detected serial numbers: {serial_num}")
return serial_num
def _set_config(self):
if self.serial_num is not None:
logging.info(f"Setting device with serial number: {self.serial_num}")
self._config.enable_device(self.serial_num)
self._config.enable_stream(
rs.stream.color,
self._color_resolution[0],
self._color_resolution[1],
rs.format.rgb8,
self._color_frames_rate,
)
if self.is_depth_frame:
self._config.enable_stream(
rs.stream.depth,
self._depth_resolution[0],
self._depth_resolution[1],
rs.format.z16,
self._depth_frames_rate,
)
def start_camera(self):
"""
启动相机并获取内参信息,如果后续调用帧对齐,则内参均为彩色内参
"""
self._pipeline = rs.pipeline()
if self.is_depth_frame:
self.point_cloud = rs.pointcloud()
self._align = rs.align(rs.stream.color)
self._set_config()
self.profile = self._pipeline.start(self._config)
if self.is_depth_frame:
self._depth_intrinsics = (
self.profile.get_stream(rs.stream.depth)
.as_video_stream_profile()
.get_intrinsics()
)
self._color_intrinsics = (
self.profile.get_stream(rs.stream.color)
.as_video_stream_profile()
.get_intrinsics()
)
self.camera_on = True
logging.info("Camera started successfully")
logging.info(
f"Camera started with color resolution: {self._color_resolution}, depth resolution: {self._depth_resolution}"
)
logging.info(
f"Color FPS: {self._color_frames_rate}, Depth FPS: {self._depth_frames_rate}"
)
def stop_camera(self):
"""
停止相机
"""
self._pipeline.stop()
self.camera_on = False
logging.info("Camera stopped")
def set_resolution(self, color_resolution, depth_resolution):
self._color_resolution = color_resolution
self._depth_resolution = depth_resolution
logging.info(
"Optional color resolution:"
"[320, 180] [320, 240] [424, 240] [640, 360] [640, 480]"
"[848, 480] [960, 540] [1280, 720] [1920, 1080]"
)
logging.info(
"Optional depth resolution:"
"[256, 144] [424, 240] [480, 270] [640, 360] [640, 400]"
"[640, 480] [848, 100] [848, 480] [1280, 720] [1280, 800]"
)
logging.info(f"Set color resolution to: {color_resolution}")
logging.info(f"Set depth resolution to: {depth_resolution}")
def set_frame_rate(self, color_fps, depth_fps):
self._color_frames_rate = color_fps
self._depth_frames_rate = depth_fps
logging.info("Optional color fps: 6 15 30 60 ")
logging.info("Optional depth fps: 6 15 30 60 90 100 300")
logging.info(f"Set color FPS to: {color_fps}")
logging.info(f"Set depth FPS to: {depth_fps}")
# TODO: 调节白平衡进行补偿
# def set_exposure(self, exposure):
def read_frame(self, is_color=True, is_depth=True, is_colorized_depth=False, is_point_cloud=False):
"""
读取一帧彩色图像和深度图像
:return: 彩色图像和深度图像的NumPy数组
"""
while not self.camera_on:
time.sleep(0.5)
color_image = None
depth_image = None
colorized_depth = None
point_cloud = None
try:
frames = self._pipeline.wait_for_frames()
if is_color:
color_frame = frames.get_color_frame()
color_image = np.asanyarray(color_frame.get_data())
else:
color_image = None
if is_depth:
depth_frame = frames.get_depth_frame()
depth_image = np.asanyarray(depth_frame.get_data())
else:
depth_image = None
colorized_depth = (
np.asanyarray(self._colorizer.colorize(depth_frame).get_data())
if is_colorized_depth
else None
)
point_cloud = (
np.asanyarray(self.point_cloud.calculate(depth_frame).get_vertices())
if is_point_cloud
else None
)
# 获取时间戳单位为ms对齐后color时间戳 > depth = aligned选择color
self.color_timestamp = color_frame.get_timestamp()
if self.is_depth_frame:
self.depth_timestamp = depth_frame.get_timestamp()
except Exception as e:
logging.warning(e)
if "Frame didn't arrive within 5000" in str(e):
logging.warning("Frame didn't arrive within 5000ms, resetting device")
self.stop_camera()
self.start_camera()
return color_image, depth_image, colorized_depth, point_cloud
def read_align_frame(self, is_color=True, is_depth=True, is_colorized_depth=False, is_point_cloud=False):
"""
读取一帧对齐的彩色图像和深度图像
:return: 彩色图像和深度图像的NumPy数组
"""
while not self.camera_on:
time.sleep(0.5)
try:
frames = self._pipeline.wait_for_frames()
aligned_frames = self._align.process(frames)
aligned_color_frame = aligned_frames.get_color_frame()
self._aligned_depth_frame = aligned_frames.get_depth_frame()
color_image = np.asanyarray(aligned_color_frame.get_data())
depth_image = np.asanyarray(self._aligned_depth_frame.get_data())
colorized_depth = (
np.asanyarray(
self._colorizer.colorize(self._aligned_depth_frame).get_data()
)
if is_colorized_depth
else None
)
if is_point_cloud:
points = self.point_cloud.calculate(self._aligned_depth_frame)
# 将元组数据转换为 NumPy 数组
point_cloud = np.array(
[[point[0], point[1], point[2]] for point in points.get_vertices()]
)
else:
point_cloud = None
# 获取时间戳单位为ms对齐后color时间戳 > depth = aligned选择color
self.timestamp = aligned_color_frame.get_timestamp()
return color_image, depth_image, colorized_depth, point_cloud
except Exception as e:
if "Frame didn't arrive within 5000" in str(e):
logging.warning("Frame didn't arrive within 5000ms, resetting device")
self.stop_camera()
self.start_camera()
# device = self.profile.get_device()
# device.hardware_reset()
def get_camera_intrinsics(self):
"""
获取彩色图像和深度图像的内参信息
:return: 彩色图像和深度图像的内参信息
"""
# 宽高:.width, .height; 焦距:.fx, .fy; 像素坐标:.ppx, .ppy; 畸变系数:.coeffs
logging.info("Getting camera intrinsics")
logging.info(
"Width and height: .width, .height; Focal length: .fx, .fy; Pixel coordinates: .ppx, .ppy; Distortion coefficient: .coeffs"
)
return self._color_intrinsics, self._depth_intrinsics
def get_3d_camera_coordinate(self, depth_pixel, align=False):
"""
获取深度相机坐标系下的三维坐标
:param depth_pixel:深度像素坐标
:param align: 是否对齐
:return: 深度值和相机坐标
"""
if not hasattr(self, "_aligned_depth_frame"):
raise AttributeError(
"Aligned depth frame not set. Call read_align_frame() first."
)
distance = self._aligned_depth_frame.get_distance(
depth_pixel[0], depth_pixel[1]
)
intrinsics = self._color_intrinsics if align else self._depth_intrinsics
camera_coordinate = rs.rs2_deproject_pixel_to_point(
intrinsics, depth_pixel, distance
)
return distance, camera_coordinate
if __name__ == "__main__":
camera = RealSenseCamera(is_depth_frame=False)
camera.get_serial_num()
camera.start_camera()
# camera.set_frame_rate(60, 60)
color_image, depth_image, colorized_depth, point_cloud = camera.read_frame()
camera.stop_camera()
logging.info(f"Color image shape: {color_image.shape}")
# logging.info(f"Depth image shape: {depth_image.shape}")
# logging.info(f"Colorized depth image shape: {colorized_depth.shape}")
# logging.info(f"Point cloud shape: {point_cloud.shape}")
logging.info(f"Color timestamp: {camera.timestamp}")
# logging.info(f"Depth timestamp: {camera.depth_timestamp}")
logging.info(f"Color timestamp: {camera.color_timestamp}")
# logging.info(f"Depth timestamp: {camera.depth_timestamp}")
logging.info("Test passed")

View File

@@ -0,0 +1,101 @@
import pyrealsense2 as rs
import numpy as np
import h5py
import time
import threading
import keyboard # 用于监听键盘输入
# 全局变量
is_recording = False # 标志位,控制录制状态
color_images = [] # 存储彩色图像
depth_images = [] # 存储深度图像
timestamps = [] # 存储时间戳
# 配置D435相机
def configure_camera():
pipeline = rs.pipeline()
config = rs.config()
config.enable_stream(rs.stream.color, 640, 480, rs.format.bgr8, 30) # 彩色图像流
config.enable_stream(rs.stream.depth, 640, 480, rs.format.z16, 30) # 深度图像流
pipeline.start(config)
return pipeline
# 监听键盘输入,控制录制状态
def listen_for_keyboard():
global is_recording
while True:
if keyboard.is_pressed('s'): # 按下 's' 开始录制
is_recording = True
print("Recording started.")
time.sleep(0.5) # 防止重复触发
elif keyboard.is_pressed('q'): # 按下 'q' 停止录制
is_recording = False
print("Recording stopped.")
time.sleep(0.5) # 防止重复触发
elif keyboard.is_pressed('e'): # 按下 'e' 退出程序
print("Exiting program.")
exit()
time.sleep(0.1)
# 采集图像数据
def capture_frames(pipeline):
global is_recording, color_images, depth_images, timestamps
try:
while True:
if is_recording:
frames = pipeline.wait_for_frames()
color_frame = frames.get_color_frame()
depth_frame = frames.get_depth_frame()
if not color_frame or not depth_frame:
continue
# 获取当前时间戳
timestamp = time.time()
# 将图像转换为numpy数组
color_image = np.asanyarray(color_frame.get_data())
depth_image = np.asanyarray(depth_frame.get_data())
# 存储数据
color_images.append(color_image)
depth_images.append(depth_image)
timestamps.append(timestamp)
print(f"Captured frame at {timestamp}")
else:
time.sleep(0.1) # 如果未录制,等待一段时间
finally:
pipeline.stop()
# 保存为HDF5文件
def save_to_hdf5(color_images, depth_images, timestamps, filename="output.h5"):
with h5py.File(filename, "w") as f:
f.create_dataset("color_images", data=np.array(color_images), compression="gzip")
f.create_dataset("depth_images", data=np.array(depth_images), compression="gzip")
f.create_dataset("timestamps", data=np.array(timestamps), compression="gzip")
print(f"Data saved to {filename}")
# 主函数
def main():
global is_recording, color_images, depth_images, timestamps
# 启动键盘监听线程
keyboard_thread = threading.Thread(target=listen_for_keyboard)
keyboard_thread.daemon = True
keyboard_thread.start()
# 配置相机
pipeline = configure_camera()
# 开始采集图像
capture_frames(pipeline)
# 录制结束后保存数据
if color_images and depth_images and timestamps:
save_to_hdf5(color_images, depth_images, timestamps, "mobile_aloha_data.h5")
if __name__ == "__main__":
main()

View File

@@ -0,0 +1,152 @@
import os
import cv2
import time
import numpy as np
from os import path
import pyrealsense2 as rs
from shadow_camera import realsense
import logging
def test_camera():
camera = realsense.RealSenseCamera('241122071186')
camera.start_camera()
while True:
# result = camera.read_align_frame()
# if result is None:
# print('is None')
# continue
# start_time = time.time()
color_image, depth_image, colorized_depth, vtx = camera.read_frame()
color_image = cv2.cvtColor(color_image, cv2.COLOR_RGB2BGR)
print(f"color_image: {color_image.shape}")
# print(f"Time: {end_time - start_time}")
cv2.imshow("bgr_image", color_image)
if cv2.waitKey(1) & 0xFF == ord("q"):
break
camera.stop_camera()
def test_get_serial_num():
camera = realsense.RealSenseCamera()
device = camera.get_serial_num()
class CameraCapture:
def __init__(self, camera_serial_num=None, save_dir="./save"):
self._camera_serial_num = camera_serial_num
self._color_save_dir = path.join(save_dir, "color")
self._depth_save_dir = path.join(save_dir, "depth")
os.makedirs(save_dir, exist_ok=True)
os.makedirs(self._color_save_dir, exist_ok=True)
os.makedirs(self._depth_save_dir, exist_ok=True)
def get_serial_num(self):
self._camera_serial_num = {}
camera_names = ["left", "right", "head", "table"]
context = rs.context()
devices = context.query_devices() # 获取所有设备
if len(context.devices) > 0:
for i, device in enumerate(devices):
self._camera_serial_num[camera_names[i]] = device.get_info(
rs.camera_info.serial_number
)
print(self._camera_serial_num)
return self._camera_serial_num
def start_camera(self):
if self._camera_serial_num is None:
self.get_serial_num()
self._camera_left = realsense.RealSenseCamera(self._camera_serial_num["left"])
self._camera_right = realsense.RealSenseCamera(self._camera_serial_num["right"])
self._camera_head = realsense.RealSenseCamera(self._camera_serial_num["head"])
self._camera_left.start_camera()
self._camera_right.start_camera()
self._camera_head.start_camera()
def stop_camera(self):
self._camera_left.stop_camera()
self._camera_right.stop_camera()
self._camera_head.stop_camera()
def _save_datas(self, timestamp, color_image, depth_image, camera_name):
color_filename = path.join(
self._color_save_dir, f"{timestamp}" + camera_name + ".jpg"
)
depth_filename = path.join(
self._depth_save_dir, f"{timestamp}" + camera_name + ".png"
)
cv2.imwrite(color_filename, color_image)
cv2.imwrite(depth_filename, depth_image)
def capture_images(self):
while True:
(
color_image_left,
depth_image_left,
_,
_,
) = self._camera_left.read_align_frame()
(
color_image_right,
depth_image_right,
_,
_,
) = self._camera_right.read_align_frame()
(
color_image_head,
depth_image_head,
_,
point_cloud3,
) = self._camera_head.read_align_frame()
bgr_color_image_left = cv2.cvtColor(color_image_left, cv2.COLOR_RGB2BGR)
bgr_color_image_right = cv2.cvtColor(color_image_right, cv2.COLOR_RGB2BGR)
bgr_color_image_head = cv2.cvtColor(color_image_head, cv2.COLOR_RGB2BGR)
timestamp = time.time() * 1000
cv2.imshow("Camera left", bgr_color_image_left)
cv2.imshow("Camera right", bgr_color_image_right)
cv2.imshow("Camera head", bgr_color_image_head)
# self._save_datas(
# timestamp, bgr_color_image_left, depth_image_left, "left"
# )
# self._save_datas(
# timestamp, bgr_color_image_right, depth_image_right, "right"
# )
# self._save_datas(
# timestamp, bgr_color_image_head, depth_image_head, "head"
# )
if cv2.waitKey(1) & 0xFF == ord("q"):
break
cv2.destroyAllWindows()
if __name__ == "__main__":
#test_camera()
test_get_serial_num()
"""
输入相机序列号制定左右相机:
dict{'left': '241222075132', 'right': '242322076532', 'head': '242322076532'}
保存路径:
str./save
输入为空,自动分配相机序列号(不指定左、右、头部),保存路径为./save
"""
# capture = CameraCapture()
# capture.get_serial_num()
# test_get_serial_num()
# capture.start_camera()
# capture.capture_images()
# capture.stop_camera()

View File

@@ -0,0 +1,71 @@
import pytest
import pyrealsense2 as rs
from shadow_camera.realsense import RealSenseCamera
class TestRealSenseCamera:
@pytest.fixture(autouse=True)
def setup_camera(self):
self.camera = RealSenseCamera()
def test_get_serial_num(self):
serial_nums = self.camera.get_serial_num()
assert isinstance(serial_nums, dict)
assert len(serial_nums) > 0
def test_start_stop_camera(self):
self.camera.start_camera()
assert self.camera.camera_on is True
self.camera.stop_camera()
assert self.camera.camera_on is False
def test_set_resolution(self):
color_resolution = [1280, 720]
depth_resolution = [1280, 720]
self.camera.set_resolution(color_resolution, depth_resolution)
assert self.camera._color_resolution == color_resolution
assert self.camera._depth_resolution == depth_resolution
def test_set_frame_rate(self):
color_fps = 60
depth_fps = 60
self.camera.set_frame_rate(color_fps, depth_fps)
assert self.camera._color_frames_rate == color_fps
assert self.camera._depth_frames_rate == depth_fps
def test_read_frame(self):
self.camera.start_camera()
color_image, depth_image, colorized_depth, point_cloud = (
self.camera.read_frame()
)
assert color_image is not None
assert depth_image is not None
self.camera.stop_camera()
def test_read_align_frame(self):
self.camera.start_camera()
color_image, depth_image, colorized_depth, point_cloud = (
self.camera.read_align_frame()
)
assert color_image is not None
assert depth_image is not None
self.camera.stop_camera()
def test_get_camera_intrinsics(self):
self.camera.start_camera()
color_intrinsics, depth_intrinsics = self.camera.get_camera_intrinsics()
assert color_intrinsics is not None
assert depth_intrinsics is not None
self.camera.stop_camera()
def test_get_3d_camera_coordinate(self):
self.camera.start_camera()
# 先调用 read_align_frame 方法以确保 _aligned_depth_frame 被设置
self.camera.read_align_frame()
depth_pixel = [320, 240]
distance, camera_coordinate = self.camera.get_3d_camera_coordinate(
depth_pixel, align=True
)
assert distance > 0
assert len(camera_coordinate) == 3
self.camera.stop_camera()