Add CloudXR VR streaming support for PICO 4 Ultra (Early Access)
Replaces manual H264/TCP stereo streaming with NVIDIA CloudXR for
higher-quality stereoscopic rendering and lower latency.
Changes:
- teleop_xr_agent.py: add --cloudxr flag (enables Isaac Sim XR mode,
disables manual StreamingManager)
- deps/cloudxr/: NVIDIA CloudXR.js SDK (Early Access) with Isaac Lab
teleop React web client
- deps/cloudxr/Dockerfile.wss.proxy: HAProxy WSS proxy for PICO 4 Ultra
HTTPS mode (routes wss://48322 → ws://49100)
- deps/cloudxr/isaac/webpack.dev.js: disable file watching to avoid
EMFILE errors with large node_modules
- deps/cloudxr/INSTALL.md: full setup guide
Usage:
# Start CloudXR Runtime + Isaac Lab
cd ~/IsaacLab && ./docker/container.py start \
--files docker-compose.cloudxr-runtime.patch.yaml \
--env-file .env.cloudxr-runtime
# Run teleop with CloudXR
~/IsaacLab/isaaclab.sh -p teleop_xr_agent.py \
--task Isaac-MindRobot-2i-DualArm-IK-Abs-v0 --cloudxr
# Serve web client
cd deps/cloudxr/isaac && npm run dev-server:https
This commit is contained in:
515
deps/cloudxr/simple/src/main.ts
vendored
Normal file
515
deps/cloudxr/simple/src/main.ts
vendored
Normal file
@@ -0,0 +1,515 @@
|
||||
/**
|
||||
* CloudXR.js Simple Example - WebXR Streaming Application
|
||||
*
|
||||
* CloudXR streams XR content from a powerful server to lightweight clients (think Netflix for VR/AR).
|
||||
* Server does the heavy rendering, client displays video and sends back tracking data.
|
||||
*
|
||||
* Key Flow:
|
||||
* 1. constructor() - Initialize UI and check browser support
|
||||
* 2. connectToCloudXR() - Connect to server (called on CONNECT button click)
|
||||
* 3. initializeWebGL() - Set up graphics rendering
|
||||
* 4. createXRSession() - Enter VR/AR mode
|
||||
* 5. createCloudXRSession() - Configure CloudXR streaming
|
||||
* 6. onXRFrame() - Render loop: send tracking, receive & display video frames
|
||||
*/
|
||||
|
||||
import { checkCapabilities } from '@helpers/BrowserCapabilities';
|
||||
import { loadIWERIfNeeded } from '@helpers/LoadIWER';
|
||||
import { overridePressureObserver } from '@helpers/overridePressureObserver';
|
||||
import {
|
||||
enableLocalStorage,
|
||||
getConnectionConfig,
|
||||
setupCertificateAcceptanceLink,
|
||||
} from '@helpers/utils';
|
||||
import { getOrCreateCanvas, logOrThrow } from '@helpers/WebGlUtils';
|
||||
import * as CloudXR from '@nvidia/cloudxr';
|
||||
|
||||
// Override PressureObserver early to catch errors from buggy browser implementations
|
||||
overridePressureObserver();
|
||||
|
||||
/**
|
||||
* CloudXR Client - Main Application Class
|
||||
*
|
||||
* Architecture: WebXR (hardware access) + WebGL (rendering) + CloudXR (streaming)
|
||||
*/
|
||||
class CloudXRClient {
|
||||
// UI Elements - Form inputs and display elements
|
||||
private startButton: HTMLButtonElement;
|
||||
private exitButton: HTMLButtonElement;
|
||||
private serverIpInput: HTMLInputElement;
|
||||
private portInput: HTMLInputElement;
|
||||
private proxyUrlInput: HTMLInputElement;
|
||||
private immersiveSelect: HTMLSelectElement;
|
||||
private deviceFrameRateSelect: HTMLSelectElement;
|
||||
private maxStreamingBitrateMbpsSelect: HTMLSelectElement;
|
||||
private proxyDefaultText: HTMLElement;
|
||||
private statusMessageBox: HTMLElement;
|
||||
private statusMessageText: HTMLElement;
|
||||
private perEyeWidthInput: HTMLInputElement;
|
||||
private perEyeHeightInput: HTMLInputElement;
|
||||
private referenceSpaceSelect: HTMLSelectElement;
|
||||
private xrOffsetXInput: HTMLInputElement;
|
||||
private xrOffsetYInput: HTMLInputElement;
|
||||
private xrOffsetZInput: HTMLInputElement;
|
||||
private certAcceptanceLink: HTMLElement;
|
||||
private certLink: HTMLAnchorElement;
|
||||
private enablePoseSmoothingSelect: HTMLSelectElement;
|
||||
private posePredictionFactorInput: HTMLInputElement;
|
||||
private posePredictionFactorValue: HTMLElement;
|
||||
|
||||
// Core Session Components
|
||||
private xrSession: XRSession | null = null; // WebXR session (hardware access)
|
||||
private cloudxrSession: CloudXR.Session | null = null; // CloudXR session (streaming)
|
||||
private gl: WebGL2RenderingContext | null = null; // WebGL context (rendering)
|
||||
private baseLayer: XRWebGLLayer | null = null; // Bridge between WebXR and WebGL
|
||||
private deviceFrameRate: number = 0; // Target frame rate for XR session
|
||||
private hasSetTargetFrameRate: boolean = false; // Track if we've set target frame rate
|
||||
|
||||
/**
|
||||
* Initialize UI, enable localStorage, and check WebXR support
|
||||
*/
|
||||
constructor() {
|
||||
// Get references to all UI elements
|
||||
this.startButton = document.getElementById('startButton') as HTMLButtonElement;
|
||||
this.exitButton = document.getElementById('exitButton') as HTMLButtonElement;
|
||||
this.serverIpInput = document.getElementById('serverIpInput') as HTMLInputElement;
|
||||
this.portInput = document.getElementById('portInput') as HTMLInputElement;
|
||||
this.proxyUrlInput = document.getElementById('proxyUrl') as HTMLInputElement;
|
||||
this.immersiveSelect = document.getElementById('immersive') as HTMLSelectElement;
|
||||
this.deviceFrameRateSelect = document.getElementById('deviceFrameRate') as HTMLSelectElement;
|
||||
this.maxStreamingBitrateMbpsSelect = document.getElementById(
|
||||
'maxStreamingBitrateMbps'
|
||||
) as HTMLSelectElement;
|
||||
this.proxyDefaultText = document.getElementById('proxyDefaultText') as HTMLElement;
|
||||
this.statusMessageBox = document.getElementById('statusMessageBox') as HTMLElement;
|
||||
this.statusMessageText = document.getElementById('statusMessageText') as HTMLElement;
|
||||
this.perEyeWidthInput = document.getElementById('perEyeWidth') as HTMLInputElement;
|
||||
this.perEyeHeightInput = document.getElementById('perEyeHeight') as HTMLInputElement;
|
||||
this.referenceSpaceSelect = document.getElementById('referenceSpace') as HTMLSelectElement;
|
||||
this.xrOffsetXInput = document.getElementById('xrOffsetX') as HTMLInputElement;
|
||||
this.xrOffsetYInput = document.getElementById('xrOffsetY') as HTMLInputElement;
|
||||
this.xrOffsetZInput = document.getElementById('xrOffsetZ') as HTMLInputElement;
|
||||
this.certAcceptanceLink = document.getElementById('certAcceptanceLink') as HTMLElement;
|
||||
this.certLink = document.getElementById('certLink') as HTMLAnchorElement;
|
||||
this.enablePoseSmoothingSelect = document.getElementById(
|
||||
'enablePoseSmoothing'
|
||||
) as HTMLSelectElement;
|
||||
this.posePredictionFactorInput = document.getElementById(
|
||||
'posePredictionFactor'
|
||||
) as HTMLInputElement;
|
||||
this.posePredictionFactorValue = document.getElementById(
|
||||
'posePredictionFactorValue'
|
||||
) as HTMLElement;
|
||||
|
||||
// Enable localStorage to persist user settings
|
||||
enableLocalStorage(this.serverIpInput, 'serverIp');
|
||||
enableLocalStorage(this.portInput, 'port');
|
||||
enableLocalStorage(this.proxyUrlInput, 'proxyUrl');
|
||||
enableLocalStorage(this.immersiveSelect, 'immersiveMode');
|
||||
enableLocalStorage(this.deviceFrameRateSelect, 'deviceFrameRate');
|
||||
enableLocalStorage(this.maxStreamingBitrateMbpsSelect, 'maxStreamingBitrateMbps');
|
||||
enableLocalStorage(this.perEyeWidthInput, 'perEyeWidth');
|
||||
enableLocalStorage(this.perEyeHeightInput, 'perEyeHeight');
|
||||
enableLocalStorage(this.referenceSpaceSelect, 'referenceSpace');
|
||||
enableLocalStorage(this.xrOffsetXInput, 'xrOffsetX');
|
||||
enableLocalStorage(this.xrOffsetYInput, 'xrOffsetY');
|
||||
enableLocalStorage(this.xrOffsetZInput, 'xrOffsetZ');
|
||||
enableLocalStorage(this.enablePoseSmoothingSelect, 'enablePoseSmoothing');
|
||||
enableLocalStorage(this.posePredictionFactorInput, 'posePredictionFactor');
|
||||
|
||||
// Update slider value display when it changes
|
||||
this.posePredictionFactorInput.addEventListener('input', () => {
|
||||
this.posePredictionFactorValue.textContent = this.posePredictionFactorInput.value;
|
||||
});
|
||||
// Set initial display value
|
||||
this.posePredictionFactorValue.textContent = this.posePredictionFactorInput.value;
|
||||
|
||||
// Configure proxy information and port placeholder based on protocol
|
||||
if (window.location.protocol === 'https:') {
|
||||
this.proxyDefaultText.textContent =
|
||||
'Optional: Leave empty for direct WSS connection, or provide URL for proxy routing (e.g., https://proxy.example.com/)';
|
||||
this.portInput.placeholder = 'Port (default: 48322, or 443 if proxy URL set)';
|
||||
} else {
|
||||
this.proxyDefaultText.textContent = 'Not needed for HTTP - uses direct WS connection';
|
||||
this.portInput.placeholder = 'Port (default: 49100)';
|
||||
}
|
||||
|
||||
this.startButton.addEventListener('click', () => this.connectToCloudXR());
|
||||
this.exitButton.addEventListener('click', () => this.xrSession?.end());
|
||||
|
||||
// Set up certificate acceptance link
|
||||
setupCertificateAcceptanceLink(
|
||||
this.serverIpInput,
|
||||
this.portInput,
|
||||
this.proxyUrlInput,
|
||||
this.certAcceptanceLink,
|
||||
this.certLink
|
||||
);
|
||||
|
||||
this.checkWebXRSupport();
|
||||
}
|
||||
|
||||
/**
|
||||
* Check browser support: WebXR, WebGL2, WebRTC, and video frame callbacks
|
||||
* Also loads Immersive Web Emulator if needed (for desktop development)
|
||||
*/
|
||||
private async checkWebXRSupport(): Promise<void> {
|
||||
const { supportsImmersive, iwerLoaded } = await loadIWERIfNeeded();
|
||||
if (!supportsImmersive) {
|
||||
this.showStatus('Immersive mode not supported', 'error');
|
||||
this.startButton.disabled = true;
|
||||
return;
|
||||
}
|
||||
|
||||
this.startButton.disabled = true;
|
||||
this.startButton.innerHTML = 'CONNECT (checking capabilities)';
|
||||
|
||||
const result = await checkCapabilities();
|
||||
if (!result.success) {
|
||||
this.showStatus(
|
||||
'Browser does not meet required capabilities:\n' + result.failures.join('\n'),
|
||||
'error'
|
||||
);
|
||||
this.startButton.innerHTML = 'CONNECT (capabilities check failed)';
|
||||
return;
|
||||
}
|
||||
|
||||
if (result.warnings.length > 0) {
|
||||
this.showStatus('Performance notice:\n' + result.warnings.join('\n'), 'info');
|
||||
} else if (iwerLoaded) {
|
||||
// Include IWER status in the final success message
|
||||
this.showStatus(
|
||||
'CloudXR.js SDK is supported. Ready to connect!\nUsing IWER (Immersive Web Emulator Runtime) - Emulating Meta Quest 3.',
|
||||
'info'
|
||||
);
|
||||
} else {
|
||||
this.showStatus('CloudXR.js SDK is supported. Ready to connect!', 'success');
|
||||
}
|
||||
|
||||
this.startButton.disabled = false;
|
||||
this.startButton.innerHTML = 'CONNECT';
|
||||
}
|
||||
|
||||
private showStatus(message: string, type: 'success' | 'error' | 'info'): void {
|
||||
this.statusMessageText.textContent = message;
|
||||
this.statusMessageBox.className = `status-message-box show ${type}`;
|
||||
console[type === 'error' ? 'error' : 'info'](message);
|
||||
}
|
||||
|
||||
/**
|
||||
* Main connection flow - orchestrates WebGL, WebXR, and CloudXR setup
|
||||
* Steps: Read config → Initialize WebGL → Create XR session → Connect to CloudXR server
|
||||
*/
|
||||
private async connectToCloudXR(): Promise<void> {
|
||||
// Read configuration from UI form
|
||||
const serverIp = this.serverIpInput.value.trim() || 'localhost';
|
||||
|
||||
// Determine default port based on connection type and proxy usage
|
||||
const useSecureConnection = window.location.protocol === 'https:';
|
||||
const portValue = parseInt(this.portInput.value, 10);
|
||||
const proxyUrl = this.proxyUrlInput.value;
|
||||
const hasProxy = proxyUrl.trim().length > 0;
|
||||
|
||||
let defaultPort = 49100; // HTTP default (direct CloudXR Runtime connection)
|
||||
if (useSecureConnection) {
|
||||
defaultPort = hasProxy ? 443 : 48322; // HTTPS with proxy → 443, HTTPS without → 48322
|
||||
}
|
||||
|
||||
const port = portValue || defaultPort;
|
||||
const perEyeWidth = parseInt(this.perEyeWidthInput.value, 10) || 2048;
|
||||
const perEyeHeight = parseInt(this.perEyeHeightInput.value, 10) || 1792;
|
||||
const deviceFrameRate = parseInt(this.deviceFrameRateSelect.value, 10);
|
||||
const maxStreamingBitrateKbps = parseInt(this.maxStreamingBitrateMbpsSelect.value, 10) * 1000;
|
||||
const immersiveMode = this.immersiveSelect.value as 'ar' | 'vr';
|
||||
const referenceSpaceType = this.referenceSpaceSelect.value as XRReferenceSpaceType;
|
||||
const xrOffsetX = (parseFloat(this.xrOffsetXInput.value) || 0) / 100; // cm to meters
|
||||
const xrOffsetY = (parseFloat(this.xrOffsetYInput.value) || 0) / 100;
|
||||
const xrOffsetZ = (parseFloat(this.xrOffsetZInput.value) || 0) / 100;
|
||||
|
||||
try {
|
||||
this.startButton.disabled = true;
|
||||
this.startButton.innerHTML = 'CONNECT (connecting)';
|
||||
this.showStatus(`Connecting to Server ${serverIp}:${port}...`, 'info');
|
||||
|
||||
// Initialize WebGL, WebXR session, and reference space
|
||||
await this.initializeWebGL();
|
||||
await this.createXRSession(immersiveMode, deviceFrameRate);
|
||||
|
||||
let referenceSpace = await this.getReferenceSpace(referenceSpaceType);
|
||||
if (xrOffsetX !== 0 || xrOffsetY !== 0 || xrOffsetZ !== 0) {
|
||||
const offsetTransform = new XRRigidTransform(
|
||||
{ x: xrOffsetX, y: xrOffsetY, z: xrOffsetZ },
|
||||
{ x: 0, y: 0, z: 0, w: 1 }
|
||||
);
|
||||
referenceSpace = referenceSpace.getOffsetReferenceSpace(offsetTransform);
|
||||
}
|
||||
|
||||
// Create CloudXR session and connect to server
|
||||
await this.createCloudXRSession(
|
||||
serverIp,
|
||||
port,
|
||||
proxyUrl,
|
||||
perEyeWidth,
|
||||
perEyeHeight,
|
||||
maxStreamingBitrateKbps,
|
||||
referenceSpace
|
||||
);
|
||||
|
||||
this.cloudxrSession!.connect();
|
||||
this.startButton.innerHTML = 'CONNECT (waiting for streaming)';
|
||||
this.showStatus(`Connected to Server ${serverIp}:${port}...`, 'info');
|
||||
} catch (error) {
|
||||
this.showStatus(`Connection failed: ${error}`, 'error');
|
||||
this.startButton.disabled = false;
|
||||
this.startButton.innerHTML = 'CONNECT';
|
||||
|
||||
if (this.xrSession) {
|
||||
try {
|
||||
await this.xrSession.end();
|
||||
} catch (endError) {
|
||||
console.error('Error ending XR session during cleanup:', endError);
|
||||
this.clearSessionReferences();
|
||||
}
|
||||
} else {
|
||||
this.clearSessionReferences();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize WebGL2 context for rendering (high-performance, XR-compatible)
|
||||
*/
|
||||
private async initializeWebGL(): Promise<void> {
|
||||
const webglCanvas = getOrCreateCanvas('webglCanvas');
|
||||
const gl = webglCanvas.getContext('webgl2', {
|
||||
alpha: true,
|
||||
depth: true,
|
||||
stencil: false,
|
||||
desynchronized: false,
|
||||
antialias: false, // No antialiasing (video already rendered)
|
||||
failIfMajorPerformanceCaveat: true,
|
||||
powerPreference: 'high-performance', // Use discrete GPU if available
|
||||
premultipliedAlpha: false,
|
||||
preserveDrawingBuffer: false,
|
||||
}) as WebGL2RenderingContext;
|
||||
|
||||
if (!gl) throw new Error('Failed to create WebGL2 context');
|
||||
|
||||
await gl.makeXRCompatible(); // Required before using with XRWebGLLayer
|
||||
this.gl = gl;
|
||||
logOrThrow('Creating WebGL context', this.gl);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create WebXR session, XRWebGLLayer, and start render loop
|
||||
*/
|
||||
private async createXRSession(
|
||||
immersiveMode: 'ar' | 'vr',
|
||||
deviceFrameRate: number
|
||||
): Promise<void> {
|
||||
const mode = immersiveMode === 'vr' ? 'immersive-vr' : 'immersive-ar';
|
||||
const options = {
|
||||
requiredFeatures: ['local-floor'],
|
||||
optionalFeatures: ['hand-tracking', 'high-fixed-foveation-level'],
|
||||
};
|
||||
|
||||
// Try requested mode, fallback to alternative if unsupported
|
||||
try {
|
||||
this.xrSession = await navigator.xr!.requestSession(mode, options);
|
||||
} catch (error) {
|
||||
console.warn(`${mode} session failed, trying alternative:`, error);
|
||||
const altMode = immersiveMode === 'vr' ? 'immersive-ar' : 'immersive-vr';
|
||||
this.xrSession = await navigator.xr!.requestSession(altMode, options);
|
||||
}
|
||||
|
||||
// Create XRWebGLLayer - provides framebuffer for CloudXR to render into
|
||||
this.baseLayer = new XRWebGLLayer(this.xrSession, this.gl!, {
|
||||
alpha: true,
|
||||
antialias: false,
|
||||
depth: true,
|
||||
framebufferScaleFactor: 1.2,
|
||||
ignoreDepthValues: false,
|
||||
stencil: false,
|
||||
});
|
||||
|
||||
// Store frame rate for later use in render loop
|
||||
this.deviceFrameRate = deviceFrameRate;
|
||||
this.hasSetTargetFrameRate = false;
|
||||
|
||||
this.xrSession.updateRenderState({ baseLayer: this.baseLayer });
|
||||
this.xrSession.addEventListener('end', () => this.handleXRSessionEnd());
|
||||
this.xrSession.requestAnimationFrame(this.onXRFrame.bind(this));
|
||||
}
|
||||
|
||||
/**
|
||||
* Get XR reference space with fallbacks
|
||||
* Reference space types: 'local-floor' (room-scale), 'local' (seated), 'viewer' (head-locked)
|
||||
*/
|
||||
private async getReferenceSpace(
|
||||
referenceSpaceType: XRReferenceSpaceType
|
||||
): Promise<XRReferenceSpace> {
|
||||
try {
|
||||
return await this.xrSession!.requestReferenceSpace(referenceSpaceType);
|
||||
} catch (error) {
|
||||
console.warn(`'${referenceSpaceType}' not supported, trying fallbacks...`);
|
||||
try {
|
||||
return await this.xrSession!.requestReferenceSpace('local-floor');
|
||||
} catch {
|
||||
try {
|
||||
return await this.xrSession!.requestReferenceSpace('local');
|
||||
} catch {
|
||||
return await this.xrSession!.requestReferenceSpace('viewer');
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Configure CloudXR session and set up event handlers
|
||||
* Establishes WebRTC connection, receives video stream, sends tracking data
|
||||
*/
|
||||
private async createCloudXRSession(
|
||||
serverIp: string,
|
||||
port: number,
|
||||
proxyUrl: string,
|
||||
perEyeWidth: number,
|
||||
perEyeHeight: number,
|
||||
maxStreamingBitrateKbps: number,
|
||||
referenceSpace: XRReferenceSpace
|
||||
): Promise<void> {
|
||||
const connectionConfig = getConnectionConfig(serverIp, port, proxyUrl);
|
||||
|
||||
const sessionOptions: CloudXR.SessionOptions = {
|
||||
serverAddress: connectionConfig.serverIP,
|
||||
serverPort: connectionConfig.port,
|
||||
useSecureConnection: connectionConfig.useSecureConnection,
|
||||
gl: this.gl!,
|
||||
perEyeWidth, // Stream resolution: width = perEyeWidth * 2 (side-by-side)
|
||||
perEyeHeight, // Stream resolution: height = perEyeHeight * 9/4 (includes metadata)
|
||||
referenceSpace,
|
||||
deviceFrameRate: parseInt(this.deviceFrameRateSelect.value, 10),
|
||||
maxStreamingBitrateKbps,
|
||||
enablePoseSmoothing: this.enablePoseSmoothingSelect.value === 'true',
|
||||
posePredictionFactor: parseFloat(this.posePredictionFactorInput.value),
|
||||
telemetry: {
|
||||
enabled: true,
|
||||
appInfo: { version: '6.0.0-beta', product: 'CloudXR.js WebGL Example' },
|
||||
},
|
||||
};
|
||||
|
||||
const delegates: CloudXR.SessionDelegates = {
|
||||
onStreamStarted: () => {
|
||||
console.log('CloudXR stream started');
|
||||
this.startButton.innerHTML = 'CONNECT (streaming)';
|
||||
this.exitButton.style.display = 'block';
|
||||
this.showStatus('Streaming started!', 'success');
|
||||
},
|
||||
onStreamStopped: (error?: Error) => {
|
||||
if (error) {
|
||||
console.error('Stream stopped with error:', error);
|
||||
this.showStatus(`Stream stopped: ${error}`, 'error');
|
||||
} else {
|
||||
console.log('Stream stopped normally');
|
||||
this.showStatus('Stream stopped', 'info');
|
||||
}
|
||||
|
||||
if (this.xrSession) {
|
||||
this.xrSession
|
||||
.end()
|
||||
.catch(endError => console.error('Error ending XR session:', endError))
|
||||
.finally(() => (this.exitButton.style.display = 'none'));
|
||||
} else {
|
||||
this.exitButton.style.display = 'none';
|
||||
}
|
||||
|
||||
this.startButton.disabled = false;
|
||||
this.startButton.innerHTML = 'CONNECT';
|
||||
},
|
||||
onWebGLStateChangeBegin: () => console.debug('WebGL state change begin'),
|
||||
onWebGLStateChangeEnd: () => console.debug('WebGL state change end'),
|
||||
onServerMessageReceived: (messageData: Uint8Array) => {
|
||||
const messageString = new TextDecoder().decode(messageData);
|
||||
console.debug('Server message:', messageString);
|
||||
},
|
||||
};
|
||||
|
||||
try {
|
||||
this.cloudxrSession = CloudXR.createSession(sessionOptions, delegates);
|
||||
} catch (error) {
|
||||
console.error('Failed to create CloudXR session:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Main render loop - runs every frame (72-120 FPS)
|
||||
* Sends tracking data to server, receives video frame, renders to display
|
||||
*/
|
||||
private async onXRFrame(timestamp: DOMHighResTimeStamp, frame: XRFrame): Promise<void> {
|
||||
this.xrSession!.requestAnimationFrame(this.onXRFrame.bind(this));
|
||||
|
||||
// Set target frame rate on first frame only
|
||||
if (!this.hasSetTargetFrameRate && 'updateTargetFrameRate' in this.xrSession!) {
|
||||
this.hasSetTargetFrameRate = true;
|
||||
try {
|
||||
await this.xrSession!.updateTargetFrameRate(this.deviceFrameRate);
|
||||
console.debug(
|
||||
`Target frame rate set to ${this.deviceFrameRate}, current: ${this.xrSession!.frameRate}`
|
||||
);
|
||||
} catch (error) {
|
||||
console.error('Failed to set target frame rate:', error);
|
||||
}
|
||||
}
|
||||
|
||||
if (!this.cloudxrSession) {
|
||||
console.debug('Skipping frame, CloudXR session not created yet');
|
||||
return;
|
||||
}
|
||||
|
||||
if (this.cloudxrSession.state !== CloudXR.SessionState.Connected) {
|
||||
console.debug('Skipping frame, session not ready');
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
// Send tracking (head/hand positions) → Receive video → Render
|
||||
this.cloudxrSession.sendTrackingStateToServer(timestamp, frame);
|
||||
this.gl!.bindFramebuffer(this.gl!.FRAMEBUFFER, this.baseLayer!.framebuffer);
|
||||
this.cloudxrSession.render(timestamp, frame, this.baseLayer!);
|
||||
} catch (error) {
|
||||
console.error('Error in render frame:', error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Cleanup when XR session ends (user exits, removes headset, or error occurs)
|
||||
*/
|
||||
private handleXRSessionEnd(): void {
|
||||
try {
|
||||
if (this.cloudxrSession) {
|
||||
this.cloudxrSession.disconnect();
|
||||
this.cloudxrSession = null;
|
||||
}
|
||||
|
||||
this.clearSessionReferences();
|
||||
|
||||
this.startButton.disabled = false;
|
||||
this.startButton.innerHTML = 'CONNECT';
|
||||
this.exitButton.style.display = 'none';
|
||||
} catch (error) {
|
||||
this.showStatus(`Disconnect error: ${error}`, 'error');
|
||||
}
|
||||
}
|
||||
|
||||
private clearSessionReferences(): void {
|
||||
this.baseLayer = null;
|
||||
this.xrSession = null;
|
||||
this.gl = null;
|
||||
this.hasSetTargetFrameRate = false;
|
||||
}
|
||||
}
|
||||
|
||||
// Application entry point - wait for DOM to load, then initialize client
|
||||
document.addEventListener('DOMContentLoaded', () => {
|
||||
new CloudXRClient();
|
||||
});
|
||||
Reference in New Issue
Block a user