Files
mindbot/deps/cloudxr/helpers/utils.ts
yt lee 623e05f250 Add CloudXR VR streaming support for PICO 4 Ultra (Early Access)
Replaces manual H264/TCP stereo streaming with NVIDIA CloudXR for
higher-quality stereoscopic rendering and lower latency.

Changes:
- teleop_xr_agent.py: add --cloudxr flag (enables Isaac Sim XR mode,
  disables manual StreamingManager)
- deps/cloudxr/: NVIDIA CloudXR.js SDK (Early Access) with Isaac Lab
  teleop React web client
- deps/cloudxr/Dockerfile.wss.proxy: HAProxy WSS proxy for PICO 4 Ultra
  HTTPS mode (routes wss://48322 → ws://49100)
- deps/cloudxr/isaac/webpack.dev.js: disable file watching to avoid
  EMFILE errors with large node_modules
- deps/cloudxr/INSTALL.md: full setup guide

Usage:
  # Start CloudXR Runtime + Isaac Lab
  cd ~/IsaacLab && ./docker/container.py start \
      --files docker-compose.cloudxr-runtime.patch.yaml \
      --env-file .env.cloudxr-runtime

  # Run teleop with CloudXR
  ~/IsaacLab/isaaclab.sh -p teleop_xr_agent.py \
      --task Isaac-MindRobot-2i-DualArm-IK-Abs-v0 --cloudxr

  # Serve web client
  cd deps/cloudxr/isaac && npm run dev-server:https
2026-03-26 14:29:03 +08:00

294 lines
10 KiB
TypeScript

/**
* Parses URL parameters and returns them as an object
* @param location - Optional location object (defaults to window.location)
* @returns Object with URL parameters as key-value pairs
*/
export function getUrlParams(location: Location = window.location): Record<string, string> {
const params: Record<string, string> = {};
const queryString = location.search.substring(1);
if (queryString) {
const pairs = queryString.split('&');
for (const pair of pairs) {
const [key, value = ''] = pair.split('=');
params[decodeURIComponent(key)] = decodeURIComponent(value);
}
}
return params;
}
/**
* Enables localStorage functionality for form elements
* @param element - The HTML input or select element to enable localStorage for
* @param key - The localStorage key to use for saving/loading the value
*/
export function enableLocalStorage(element: HTMLInputElement | HTMLSelectElement, key: string) {
// Check if localStorage is already enabled for this element and key
const localStorageKey = `__localStorageEnabled_${key}`;
if ((element as any)[localStorageKey]) {
console.warn(`localStorage already enabled for ${key}, skipping`);
return;
}
// Load saved value from localStorage
try {
// Check if the key exists in localStorage, not just if it has values
if (localStorage.hasOwnProperty(key)) {
const savedValue = localStorage.getItem(key);
element.value = savedValue || '';
console.info(`Loaded saved ${key} from localStorage:`, savedValue);
}
} catch (error) {
console.warn(`${key}: Failed to load saved value from localStorage:`, error);
}
// Set up event listener to save value when changed
const changeHandler = () => {
try {
// Always save the value, even if it's empty
localStorage.setItem(key, element.value);
console.info(`Saved ${key} to localStorage:`, JSON.stringify(element.value));
} catch (error) {
console.warn(`${key}: Failed to save to localStorage:`, error);
}
};
element.addEventListener('change', changeHandler);
// Mark this element as having localStorage enabled for this key
(element as any)[localStorageKey] = true;
}
/**
* Strips protocol prefixes (http:// or https://) from a URL string
* @param url - The URL string to clean
* @returns The URL without protocol prefix
*/
function stripProtocol(url: string): string {
return url.replace(/^https?:\/\//, '');
}
/**
* Connection configuration object containing server connection details
*/
export interface ConnectionConfiguration {
/** Final server IP address (may be modified for HTTPS proxy routing) */
serverIP: string;
/** Final port number (fixed to 443 for HTTPS, user-provided for HTTP) */
port: number;
/** Whether the connection will use secure protocol (HTTPS/WSS) */
useSecureConnection: boolean;
}
/**
* CloudXR configuration interface containing all streaming settings
*/
export interface CloudXRConfig {
/** IP address of the CloudXR streaming server */
serverIP: string;
/** Port number for the CloudXR server connection */
port: number;
/** Whether to use secure connection (HTTPS/WSS) or insecure (HTTP/WS) */
useSecureConnection: boolean;
/** Width of each eye in pixels (must be multiple of 16) */
perEyeWidth: number;
/** Height of each eye in pixels (must be multiple of 16) */
perEyeHeight: number;
/** Target frame rate for the XR device in frames per second (FPS) */
deviceFrameRate: number;
/** Maximum streaming bitrate in Megabits per second (Mbps) */
maxStreamingBitrateMbps: number;
/** XR immersive mode: 'ar' for augmented reality, 'vr' for virtual reality */
immersiveMode: 'ar' | 'vr';
/** Application identifier string for the CloudXR session */
app: string;
/** Type of server being connected to */
serverType: string;
/** Optional proxy URL for HTTPS routing (e.g., 'https://proxy.example.com/'); if empty, uses direct WSS connection */
proxyUrl: string;
/** Preferred XR reference space for tracking and positioning */
referenceSpaceType: 'auto' | 'local-floor' | 'local' | 'viewer' | 'unbounded';
/** XR reference space offset along X axis in meters (positive is right) */
xrOffsetX?: number;
/** XR reference space offset along Y axis in meters (positive is up) */
xrOffsetY?: number;
/** XR reference space offset along Z axis in meters (positive is backward) */
xrOffsetZ?: number;
}
/**
* Determines connection configuration based on protocol and user inputs
* Supports both direct WSS connections and proxy routing for HTTPS
*
* @param serverIP - The user-provided server IP address
* @param port - The user-provided port number
* @param proxyUrl - Optional proxy URL for HTTPS routing (if provided, uses proxy routing; otherwise direct connection)
* @param location - Optional location object (defaults to window.location)
* @returns Object containing server IP, port, and security settings
* @throws {Error} When proxy URL format is invalid (must start with https://)
*/
export function getConnectionConfig(
serverIP: string,
port: number,
proxyUrl: string,
location: Location = window.location
): ConnectionConfiguration {
let finalServerIP = '';
let finalPort = port;
let finalUseSecureConnection = false;
// Determine if we should use secure connection based on page protocol
if (location.protocol === 'https:') {
console.info('Running on HTTPS protocol - using secure WebSocket (WSS)');
finalUseSecureConnection = true;
// Check if proxy URL is provided for routing
const trimmedProxyUrl = proxyUrl?.trim();
if (trimmedProxyUrl) {
// Proxy routing mode
console.info('Proxy URL provided - using proxy routing mode');
if (!trimmedProxyUrl.startsWith('https://')) {
throw new Error('Proxy URL must start with https://. Received: ' + trimmedProxyUrl);
}
// Use port 443 for proxy routing (standard HTTPS port)
finalPort = 443;
// Route through proxy: if specific server IP provided, append it to proxy URL
if (serverIP && serverIP !== 'localhost' && serverIP !== '127.0.0.1') {
// Route to proxy with IP
const cleanServerIP = stripProtocol(serverIP);
// Clean proxy URL - strip protocol and trailing slash
const cleanProxyUrl = stripProtocol(trimmedProxyUrl).replace(/\/$/, '');
finalServerIP = `${cleanProxyUrl}/${cleanServerIP}`;
console.info(`Using HTTPS proxy with IP: ${finalServerIP}`);
} else {
// Route to proxy without IP
finalServerIP = stripProtocol(trimmedProxyUrl).replace(/\/$/, '');
console.info(`Using HTTPS proxy without specific IP: ${finalServerIP}`);
}
} else {
// Direct WSS connection mode
console.info('No proxy URL - using direct WSS connection');
// Handle server IP input
if (serverIP && serverIP !== 'localhost' && serverIP !== '127.0.0.1') {
finalServerIP = stripProtocol(serverIP);
console.info('Using user-provided server IP:', finalServerIP);
} else {
finalServerIP = new URL(location.href).hostname;
console.info('Using default server IP from window location:', finalServerIP);
}
// Use user-provided port for direct WSS
if (port && !isNaN(port)) {
finalPort = port;
console.info('Using user-provided port:', finalPort);
}
}
} else {
// HTTP protocol - direct WS connection
console.info('Running on HTTP protocol - using insecure WebSocket (WS)');
finalUseSecureConnection = false;
// Handle server IP input
if (serverIP && serverIP !== 'localhost' && serverIP !== '127.0.0.1') {
finalServerIP = stripProtocol(serverIP);
console.info('Using user-provided server IP:', finalServerIP);
} else {
finalServerIP = new URL(location.href).hostname;
console.info('Using default server IP from window location:', finalServerIP);
}
// Handle port input
if (port && !isNaN(port)) {
finalPort = port;
console.info('Using user-provided port:', finalPort);
}
}
return {
serverIP: finalServerIP,
port: finalPort,
useSecureConnection: finalUseSecureConnection,
} as ConnectionConfiguration;
}
/**
* Sets up certificate acceptance link for self-signed certificates in HTTPS mode
* Shows a link to accept certificates when using direct WSS connection (no proxy)
*
* @param serverIpInput - Input element for server IP address
* @param portInput - Input element for port number
* @param proxyUrlInput - Input element for proxy URL
* @param certAcceptanceLink - Container element for the certificate link
* @param certLink - Anchor element for the certificate URL
* @param location - Optional location object (defaults to window.location)
* @returns Cleanup function to remove event listeners
*/
export function setupCertificateAcceptanceLink(
serverIpInput: HTMLInputElement,
portInput: HTMLInputElement,
proxyUrlInput: HTMLInputElement,
certAcceptanceLink: HTMLElement,
certLink: HTMLAnchorElement,
location: Location = window.location
): () => void {
/**
* Updates the certificate acceptance link based on current configuration
* Shows link only when in HTTPS mode without proxy (direct WSS)
*/
const updateCertLink = () => {
const isHttps = location.protocol === 'https:';
const hasProxy = proxyUrlInput.value.trim().length > 0;
const portValue = parseInt(portInput.value, 10);
const defaultPort = hasProxy ? 443 : 48322;
const port = portValue || defaultPort;
// Show link only in HTTPS mode without proxy
if (isHttps && !hasProxy) {
let serverIp = serverIpInput.value.trim();
if (!serverIp) {
serverIp = new URL(location.href).hostname;
}
const url = `https://${serverIp}:${port}/`;
certAcceptanceLink.style.display = 'block';
certLink.href = url;
certLink.textContent = `Click ${url} to accept cert`;
} else {
certAcceptanceLink.style.display = 'none';
}
};
// Add event listeners to update link when inputs change
serverIpInput.addEventListener('input', updateCertLink);
portInput.addEventListener('input', updateCertLink);
proxyUrlInput.addEventListener('input', updateCertLink);
// Initial update after localStorage values are restored
setTimeout(updateCertLink, 0);
// Return cleanup function to remove event listeners
return () => {
serverIpInput.removeEventListener('input', updateCertLink);
portInput.removeEventListener('input', updateCertLink);
proxyUrlInput.removeEventListener('input', updateCertLink);
};
}