Add CloudXR VR streaming support for PICO 4 Ultra (Early Access)
Replaces manual H264/TCP stereo streaming with NVIDIA CloudXR for
higher-quality stereoscopic rendering and lower latency.
Changes:
- teleop_xr_agent.py: add --cloudxr flag (enables Isaac Sim XR mode,
disables manual StreamingManager)
- deps/cloudxr/: NVIDIA CloudXR.js SDK (Early Access) with Isaac Lab
teleop React web client
- deps/cloudxr/Dockerfile.wss.proxy: HAProxy WSS proxy for PICO 4 Ultra
HTTPS mode (routes wss://48322 → ws://49100)
- deps/cloudxr/isaac/webpack.dev.js: disable file watching to avoid
EMFILE errors with large node_modules
- deps/cloudxr/INSTALL.md: full setup guide
Usage:
# Start CloudXR Runtime + Isaac Lab
cd ~/IsaacLab && ./docker/container.py start \
--files docker-compose.cloudxr-runtime.patch.yaml \
--env-file .env.cloudxr-runtime
# Run teleop with CloudXR
~/IsaacLab/isaaclab.sh -p teleop_xr_agent.py \
--task Isaac-MindRobot-2i-DualArm-IK-Abs-v0 --cloudxr
# Serve web client
cd deps/cloudxr/isaac && npm run dev-server:https
This commit is contained in:
125
deps/cloudxr/helpers/BrowserCapabilities.ts
vendored
Normal file
125
deps/cloudxr/helpers/BrowserCapabilities.ts
vendored
Normal file
@@ -0,0 +1,125 @@
|
||||
interface CapabilityCheck {
|
||||
name: string;
|
||||
required: boolean;
|
||||
check: () => boolean | Promise<boolean>;
|
||||
message: string;
|
||||
}
|
||||
|
||||
const capabilities: CapabilityCheck[] = [
|
||||
{
|
||||
name: 'WebGL2',
|
||||
required: true,
|
||||
check: () => {
|
||||
const canvas = document.createElement('canvas');
|
||||
const gl = canvas.getContext('webgl2');
|
||||
return gl !== null;
|
||||
},
|
||||
message: 'WebGL2 is required for rendering',
|
||||
},
|
||||
{
|
||||
name: 'WebXR',
|
||||
required: true,
|
||||
check: () => {
|
||||
return 'xr' in navigator;
|
||||
},
|
||||
message: 'WebXR is required for VR/AR functionality',
|
||||
},
|
||||
{
|
||||
name: 'RTCPeerConnection',
|
||||
required: true,
|
||||
check: () => {
|
||||
return 'RTCPeerConnection' in window;
|
||||
},
|
||||
message: 'RTCPeerConnection is required for WebRTC streaming',
|
||||
},
|
||||
{
|
||||
name: 'requestVideoFrameCallback',
|
||||
required: true,
|
||||
check: () => {
|
||||
const video = document.createElement('video');
|
||||
return typeof video.requestVideoFrameCallback === 'function';
|
||||
},
|
||||
message: 'HTMLVideoElement.requestVideoFrameCallback is required for video frame processing',
|
||||
},
|
||||
{
|
||||
name: 'Canvas.captureStream',
|
||||
required: true,
|
||||
check: () => {
|
||||
const canvas = document.createElement('canvas');
|
||||
return typeof canvas.captureStream === 'function';
|
||||
},
|
||||
message: 'Canvas.captureStream is required for video streaming',
|
||||
},
|
||||
{
|
||||
name: 'AV1 Codec Support',
|
||||
required: false,
|
||||
check: async () => {
|
||||
try {
|
||||
// Check if MediaCapabilities API is available
|
||||
if (!navigator.mediaCapabilities) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Check MediaCapabilities for AV1 decoding support
|
||||
const config = {
|
||||
type: 'webrtc' as MediaDecodingType,
|
||||
video: {
|
||||
contentType: 'video/av1',
|
||||
width: 1920,
|
||||
height: 1080,
|
||||
framerate: 60,
|
||||
bitrate: 15000000, // 15 Mbps
|
||||
},
|
||||
};
|
||||
|
||||
const result = await navigator.mediaCapabilities.decodingInfo(config);
|
||||
return result.supported;
|
||||
} catch (error) {
|
||||
console.warn('Error checking AV1 support:', error);
|
||||
return false;
|
||||
}
|
||||
},
|
||||
message: 'AV1 codec support is recommended for optimal streaming quality',
|
||||
},
|
||||
];
|
||||
|
||||
export async function checkCapabilities(): Promise<{
|
||||
success: boolean;
|
||||
failures: string[];
|
||||
warnings: string[];
|
||||
}> {
|
||||
const failures: string[] = [];
|
||||
const warnings: string[] = [];
|
||||
const requiredFailures: string[] = [];
|
||||
|
||||
for (const capability of capabilities) {
|
||||
try {
|
||||
const result = await Promise.resolve(capability.check());
|
||||
if (!result) {
|
||||
if (capability.required) {
|
||||
requiredFailures.push(capability.message);
|
||||
console.error(`Required capability missing: ${capability.message}`);
|
||||
} else {
|
||||
warnings.push(capability.message);
|
||||
console.warn(`Optional capability missing: ${capability.message}`);
|
||||
}
|
||||
failures.push(capability.message);
|
||||
}
|
||||
} catch (error) {
|
||||
if (capability.required) {
|
||||
requiredFailures.push(capability.message);
|
||||
console.error(`Error checking required capability ${capability.name}:`, error);
|
||||
} else {
|
||||
warnings.push(capability.message);
|
||||
console.warn(`Error checking optional capability ${capability.name}:`, error);
|
||||
}
|
||||
failures.push(capability.message);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
success: requiredFailures.length === 0,
|
||||
failures,
|
||||
warnings,
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user