Files
mindbot/deps/cloudxr/helpers/BrowserCapabilities.ts
yt lee 623e05f250 Add CloudXR VR streaming support for PICO 4 Ultra (Early Access)
Replaces manual H264/TCP stereo streaming with NVIDIA CloudXR for
higher-quality stereoscopic rendering and lower latency.

Changes:
- teleop_xr_agent.py: add --cloudxr flag (enables Isaac Sim XR mode,
  disables manual StreamingManager)
- deps/cloudxr/: NVIDIA CloudXR.js SDK (Early Access) with Isaac Lab
  teleop React web client
- deps/cloudxr/Dockerfile.wss.proxy: HAProxy WSS proxy for PICO 4 Ultra
  HTTPS mode (routes wss://48322 → ws://49100)
- deps/cloudxr/isaac/webpack.dev.js: disable file watching to avoid
  EMFILE errors with large node_modules
- deps/cloudxr/INSTALL.md: full setup guide

Usage:
  # Start CloudXR Runtime + Isaac Lab
  cd ~/IsaacLab && ./docker/container.py start \
      --files docker-compose.cloudxr-runtime.patch.yaml \
      --env-file .env.cloudxr-runtime

  # Run teleop with CloudXR
  ~/IsaacLab/isaaclab.sh -p teleop_xr_agent.py \
      --task Isaac-MindRobot-2i-DualArm-IK-Abs-v0 --cloudxr

  # Serve web client
  cd deps/cloudxr/isaac && npm run dev-server:https
2026-03-26 14:29:03 +08:00

126 lines
3.4 KiB
TypeScript

interface CapabilityCheck {
name: string;
required: boolean;
check: () => boolean | Promise<boolean>;
message: string;
}
const capabilities: CapabilityCheck[] = [
{
name: 'WebGL2',
required: true,
check: () => {
const canvas = document.createElement('canvas');
const gl = canvas.getContext('webgl2');
return gl !== null;
},
message: 'WebGL2 is required for rendering',
},
{
name: 'WebXR',
required: true,
check: () => {
return 'xr' in navigator;
},
message: 'WebXR is required for VR/AR functionality',
},
{
name: 'RTCPeerConnection',
required: true,
check: () => {
return 'RTCPeerConnection' in window;
},
message: 'RTCPeerConnection is required for WebRTC streaming',
},
{
name: 'requestVideoFrameCallback',
required: true,
check: () => {
const video = document.createElement('video');
return typeof video.requestVideoFrameCallback === 'function';
},
message: 'HTMLVideoElement.requestVideoFrameCallback is required for video frame processing',
},
{
name: 'Canvas.captureStream',
required: true,
check: () => {
const canvas = document.createElement('canvas');
return typeof canvas.captureStream === 'function';
},
message: 'Canvas.captureStream is required for video streaming',
},
{
name: 'AV1 Codec Support',
required: false,
check: async () => {
try {
// Check if MediaCapabilities API is available
if (!navigator.mediaCapabilities) {
return false;
}
// Check MediaCapabilities for AV1 decoding support
const config = {
type: 'webrtc' as MediaDecodingType,
video: {
contentType: 'video/av1',
width: 1920,
height: 1080,
framerate: 60,
bitrate: 15000000, // 15 Mbps
},
};
const result = await navigator.mediaCapabilities.decodingInfo(config);
return result.supported;
} catch (error) {
console.warn('Error checking AV1 support:', error);
return false;
}
},
message: 'AV1 codec support is recommended for optimal streaming quality',
},
];
export async function checkCapabilities(): Promise<{
success: boolean;
failures: string[];
warnings: string[];
}> {
const failures: string[] = [];
const warnings: string[] = [];
const requiredFailures: string[] = [];
for (const capability of capabilities) {
try {
const result = await Promise.resolve(capability.check());
if (!result) {
if (capability.required) {
requiredFailures.push(capability.message);
console.error(`Required capability missing: ${capability.message}`);
} else {
warnings.push(capability.message);
console.warn(`Optional capability missing: ${capability.message}`);
}
failures.push(capability.message);
}
} catch (error) {
if (capability.required) {
requiredFailures.push(capability.message);
console.error(`Error checking required capability ${capability.name}:`, error);
} else {
warnings.push(capability.message);
console.warn(`Error checking optional capability ${capability.name}:`, error);
}
failures.push(capability.message);
}
}
return {
success: requiredFailures.length === 0,
failures,
warnings,
};
}