Files
mindbot/deps/cloudxr/helpers/WebGLStateBinding.ts
yt lee 623e05f250 Add CloudXR VR streaming support for PICO 4 Ultra (Early Access)
Replaces manual H264/TCP stereo streaming with NVIDIA CloudXR for
higher-quality stereoscopic rendering and lower latency.

Changes:
- teleop_xr_agent.py: add --cloudxr flag (enables Isaac Sim XR mode,
  disables manual StreamingManager)
- deps/cloudxr/: NVIDIA CloudXR.js SDK (Early Access) with Isaac Lab
  teleop React web client
- deps/cloudxr/Dockerfile.wss.proxy: HAProxy WSS proxy for PICO 4 Ultra
  HTTPS mode (routes wss://48322 → ws://49100)
- deps/cloudxr/isaac/webpack.dev.js: disable file watching to avoid
  EMFILE errors with large node_modules
- deps/cloudxr/INSTALL.md: full setup guide

Usage:
  # Start CloudXR Runtime + Isaac Lab
  cd ~/IsaacLab && ./docker/container.py start \
      --files docker-compose.cloudxr-runtime.patch.yaml \
      --env-file .env.cloudxr-runtime

  # Run teleop with CloudXR
  ~/IsaacLab/isaaclab.sh -p teleop_xr_agent.py \
      --task Isaac-MindRobot-2i-DualArm-IK-Abs-v0 --cloudxr

  # Serve web client
  cd deps/cloudxr/isaac && npm run dev-server:https
2026-03-26 14:29:03 +08:00

287 lines
8.7 KiB
TypeScript

import { WebGLStateTracker, WebGLState } from './WebGLState';
import { apply } from './WebGLStateApply';
/**
* BoundWebGLState - A WebGL2 context with automatic state tracking
*
* This is the original WebGL2 context with state-changing functions rebound
* to call the tracker before forwarding to the original implementation.
*/
export class BoundWebGLState {
private gl: WebGL2RenderingContext;
private tracker: WebGLStateTracker;
private originalFunctions: Map<string, Function>;
private savedState?: WebGLState;
private trackingEnabled = true;
constructor(
gl: WebGL2RenderingContext,
tracker: WebGLStateTracker,
originalFunctions: Map<string, Function>
) {
this.gl = gl;
this.tracker = tracker;
this.originalFunctions = originalFunctions;
}
/**
* Get the internal state tracker
*/
private getTracker(): WebGLStateTracker {
return this.tracker;
}
/**
* Save the current tracked WebGL state
* This clones the state so it can be restored later
*/
save(): void {
this.savedState = this.tracker.getState();
}
/**
* Restore the previously saved WebGL state
* This applies the saved state back to the WebGL context
* @throws {Error} If no state has been saved
*/
restore(): void {
if (!this.savedState) {
throw new Error('No state has been saved. Call save() before restore().');
}
// Save the current tracking state and enable tracking during restore
// This ensures the tracker stays synchronized with actual GL state
const wasTrackingEnabled = this.trackingEnabled;
this.trackingEnabled = true;
const currentState = this.tracker.getState();
apply(this.gl, this.savedState, currentState);
// Restore the original tracking state
this.trackingEnabled = wasTrackingEnabled;
}
enableTracking(enable: boolean): void {
this.trackingEnabled = enable;
}
_enabled(): boolean {
return this.trackingEnabled;
}
/**
* Revert all tracked functions back to their original implementations
* This removes state tracking from the WebGL context
*/
revert(): void {
const glAny = this.gl as any;
for (const [name, originalFunction] of this.originalFunctions.entries()) {
glAny[name] = originalFunction;
}
// Clear the map
this.originalFunctions.clear();
// Remove the stored BoundWebGLState from the GL context
delete glAny.__cloudxrBoundState;
}
}
/**
* Bind a WebGL2 context with automatic state tracking
*
* Rebinds state-changing methods on the WebGL context to automatically track
* state changes before forwarding to the original implementation.
*
* @param gl - The WebGL2RenderingContext to wrap
* @returns A BoundWebGLState instance that provides access to the tracker and revert functionality
*
* @example
* ```typescript
* const canvas = document.getElementById('canvas') as HTMLCanvasElement;
* const gl = canvas.getContext('webgl2')!;
* const binding = bindGL(gl);
*
* // Use gl like a normal WebGL context - it's now tracked
* gl.bindBuffer(gl.ARRAY_BUFFER, buffer);
* gl.bindVertexArray(vao);
*
* // Save the current state
* binding.save();
*
* // Make some temporary changes
* gl.bindBuffer(gl.ARRAY_BUFFER, tempBuffer);
* gl.enable(gl.BLEND);
*
* // Restore the saved state
* binding.restore();
*
* // Access tracked state
* const state = binding.getTracker().getState();
* console.log(state.buffers?.arrayBuffer); // The bound buffer
*
* // When done, revert to stop tracking
* binding.revert();
* ```
*/
export function bindGL(gl: WebGL2RenderingContext): BoundWebGLState {
const glAny = gl as any;
// Check if this GL context is already wrapped - prevent double-wrapping
if (glAny.__cloudxrBoundState) {
console.warn(
'WebGL context is already wrapped with state tracking. Returning existing BoundWebGLState.'
);
return glAny.__cloudxrBoundState;
}
// Create the tracker
const tracker = new WebGLStateTracker();
// Store original functions for later reversion
const originalFunctions = new Map<string, Function>();
const wrappedFunctions = new Map<string, Function>();
const state = new BoundWebGLState(gl, tracker, originalFunctions);
// Store the BoundWebGLState on the GL context to prevent double-wrapping
glAny.__cloudxrBoundState = state;
// Helper function to bind a method
const bind = (name: string, trackerMethod: Function) => {
// CRITICAL: Store the original BEFORE we replace it, otherwise we'll store the wrapper
const originalMethod = glAny[name];
if (!originalMethod) {
throw new Error('Original method not found for ' + name);
}
if (originalMethod === wrappedFunctions.get(name)) {
throw new Error('Wrapped function already bound for ' + name);
}
const original = originalMethod.bind(gl);
originalFunctions.set(name, original);
const wrappedFunction = (...args: any[]) => {
if (state._enabled()) {
trackerMethod.apply(tracker, args);
}
return original(...args);
};
wrappedFunctions.set(name, wrappedFunction);
glAny[name] = wrappedFunction;
};
// Buffer bindings
bind('bindBuffer', tracker.bindBuffer);
bind('bindBufferBase', tracker.bindBufferBase);
bind('bindBufferRange', tracker.bindBufferRange);
// Buffer lifecycle tracking (for validation without GPU calls)
const originalCreateBuffer = glAny.createBuffer.bind(gl);
originalFunctions.set('createBuffer', originalCreateBuffer);
glAny.createBuffer = (): WebGLBuffer | null => {
const buffer = originalCreateBuffer();
if (buffer) {
tracker.createBuffer(buffer);
}
return buffer;
};
bind('deleteBuffer', tracker.deleteBuffer);
// VAO and vertex attributes
bind('bindVertexArray', tracker.bindVertexArray);
bind('deleteVertexArray', tracker.deleteVertexArray);
bind('enableVertexAttribArray', tracker.enableVertexAttribArray);
bind('disableVertexAttribArray', tracker.disableVertexAttribArray);
bind('vertexAttribPointer', tracker.vertexAttribPointer);
bind('vertexAttribIPointer', tracker.vertexAttribIPointer);
bind('vertexAttribDivisor', tracker.vertexAttribDivisor);
// Texture bindings
bind('activeTexture', tracker.activeTexture);
bind('bindTexture', tracker.bindTexture);
// Program binding
bind('useProgram', tracker.useProgram);
// Framebuffer bindings
bind('bindFramebuffer', tracker.bindFramebuffer);
bind('framebufferTexture2D', tracker.framebufferTexture2D);
bind('framebufferRenderbuffer', tracker.framebufferRenderbuffer);
bind('framebufferTextureLayer', tracker.framebufferTextureLayer);
bind('drawBuffers', tracker.drawBuffers);
bind('readBuffer', tracker.readBuffer);
// Renderbuffer binding
bind('bindRenderbuffer', tracker.bindRenderbuffer);
// Transform feedback
bind('bindTransformFeedback', tracker.bindTransformFeedback);
bind('beginTransformFeedback', tracker.beginTransformFeedback);
bind('endTransformFeedback', tracker.endTransformFeedback);
bind('pauseTransformFeedback', tracker.pauseTransformFeedback);
bind('resumeTransformFeedback', tracker.resumeTransformFeedback);
// Capabilities (enable/disable)
bind('enable', tracker.enable);
bind('disable', tracker.disable);
// Viewport and scissor
bind('viewport', tracker.viewport);
bind('scissor', tracker.scissor);
// Clear values
bind('clearColor', tracker.clearColor);
bind('clearDepth', tracker.clearDepth);
bind('clearStencil', tracker.clearStencil);
// Blend state
bind('blendColor', tracker.blendColor);
bind('blendEquation', tracker.blendEquation);
bind('blendEquationSeparate', tracker.blendEquationSeparate);
bind('blendFunc', tracker.blendFunc);
bind('blendFuncSeparate', tracker.blendFuncSeparate);
// Depth state
bind('depthFunc', tracker.depthFunc);
bind('depthMask', tracker.depthMask);
bind('depthRange', tracker.depthRange);
// Stencil state
bind('stencilFunc', tracker.stencilFunc);
bind('stencilFuncSeparate', tracker.stencilFuncSeparate);
bind('stencilMask', tracker.stencilMask);
bind('stencilMaskSeparate', tracker.stencilMaskSeparate);
bind('stencilOp', tracker.stencilOp);
bind('stencilOpSeparate', tracker.stencilOpSeparate);
// Color mask
bind('colorMask', tracker.colorMask);
// Culling and face orientation
bind('cullFace', tracker.cullFace);
bind('frontFace', tracker.frontFace);
// Line width
bind('lineWidth', tracker.lineWidth);
// Polygon offset
bind('polygonOffset', tracker.polygonOffset);
// Sample coverage
bind('sampleCoverage', tracker.sampleCoverage);
// Pixel store parameters
bind('pixelStorei', tracker.pixelStorei);
// Sampler binding
bind('bindSampler', tracker.bindSampler);
// Query operations
bind('beginQuery', tracker.beginQuery);
bind('endQuery', tracker.endQuery);
return state;
}