Add CloudXR VR streaming support for PICO 4 Ultra (Early Access)
Replaces manual H264/TCP stereo streaming with NVIDIA CloudXR for
higher-quality stereoscopic rendering and lower latency.
Changes:
- teleop_xr_agent.py: add --cloudxr flag (enables Isaac Sim XR mode,
disables manual StreamingManager)
- deps/cloudxr/: NVIDIA CloudXR.js SDK (Early Access) with Isaac Lab
teleop React web client
- deps/cloudxr/Dockerfile.wss.proxy: HAProxy WSS proxy for PICO 4 Ultra
HTTPS mode (routes wss://48322 → ws://49100)
- deps/cloudxr/isaac/webpack.dev.js: disable file watching to avoid
EMFILE errors with large node_modules
- deps/cloudxr/INSTALL.md: full setup guide
Usage:
# Start CloudXR Runtime + Isaac Lab
cd ~/IsaacLab && ./docker/container.py start \
--files docker-compose.cloudxr-runtime.patch.yaml \
--env-file .env.cloudxr-runtime
# Run teleop with CloudXR
~/IsaacLab/isaaclab.sh -p teleop_xr_agent.py \
--task Isaac-MindRobot-2i-DualArm-IK-Abs-v0 --cloudxr
# Serve web client
cd deps/cloudxr/isaac && npm run dev-server:https
This commit is contained in:
555
deps/cloudxr/react/src/App.tsx
vendored
Normal file
555
deps/cloudxr/react/src/App.tsx
vendored
Normal file
@@ -0,0 +1,555 @@
|
||||
/**
|
||||
* App.tsx - Main CloudXR React Application
|
||||
*
|
||||
* This is the root component of the CloudXR React example application. It sets up:
|
||||
* - WebXR session management and XR store configuration
|
||||
* - CloudXR server configuration (IP, port, stream settings)
|
||||
* - UI state management (connection status, session state)
|
||||
* - Integration between CloudXR rendering component and UI components
|
||||
* - Entry point for AR/VR experiences with CloudXR streaming
|
||||
*
|
||||
* The app integrates with the HTML interface which provides a "CONNECT" button
|
||||
* to enter AR mode and displays the CloudXR UI with controls for teleop actions
|
||||
* and disconnect when in XR mode.
|
||||
*/
|
||||
|
||||
import { checkCapabilities } from '@helpers/BrowserCapabilities';
|
||||
import { loadIWERIfNeeded } from '@helpers/LoadIWER';
|
||||
import { overridePressureObserver } from '@helpers/overridePressureObserver';
|
||||
import * as CloudXR from '@nvidia/cloudxr';
|
||||
import { Environment } from '@react-three/drei';
|
||||
import { Canvas } from '@react-three/fiber';
|
||||
import { setPreferredColorScheme } from '@react-three/uikit';
|
||||
import { XR, createXRStore, noEvents, PointerEvents, XROrigin, useXR } from '@react-three/xr';
|
||||
import { useState, useMemo, useEffect, useRef } from 'react';
|
||||
|
||||
import { CloudXR2DUI } from './CloudXR2DUI';
|
||||
import CloudXRComponent from './CloudXRComponent';
|
||||
import CloudXR3DUI from './CloudXRUI';
|
||||
|
||||
// Override PressureObserver early to catch errors from buggy browser implementations
|
||||
overridePressureObserver();
|
||||
|
||||
const store = createXRStore({
|
||||
foveation: 0,
|
||||
emulate: false, // Disable IWER emulation from react in favor of custom iwer loading function
|
||||
// Configure WebXR input profiles to use local assets
|
||||
// Use relative path from current page location
|
||||
baseAssetPath: `${new URL('.', window.location).href}npm/@webxr-input-profiles/assets@${process.env.WEBXR_ASSETS_VERSION}/dist/profiles/`,
|
||||
hand: {
|
||||
model: false, // Disable hand models but keep pointer functionality
|
||||
},
|
||||
});
|
||||
|
||||
setPreferredColorScheme('dark');
|
||||
|
||||
const START_TELEOP_COMMAND = {
|
||||
type: 'teleop_command',
|
||||
message: {
|
||||
command: 'start teleop',
|
||||
},
|
||||
} as const;
|
||||
|
||||
// Environment component like controller-test
|
||||
function NonAREnvironment() {
|
||||
// Use local HDR file instead of preset so client doesn't need to download it from CDN
|
||||
return (
|
||||
<Environment
|
||||
blur={0.2}
|
||||
background={false}
|
||||
environmentIntensity={2}
|
||||
files="assets/hdri/potsdamer_platz_1k.hdr"
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
function App() {
|
||||
const COUNTDOWN_MAX_SECONDS = 9;
|
||||
const COUNTDOWN_STORAGE_KEY = 'cxr.react.countdownSeconds';
|
||||
// 2D UI management
|
||||
const [cloudXR2DUI, setCloudXR2DUI] = useState<CloudXR2DUI | null>(null);
|
||||
// IWER loading state
|
||||
const [iwerLoaded, setIwerLoaded] = useState(false);
|
||||
// Capability state management
|
||||
const [capabilitiesValid, setCapabilitiesValid] = useState(false);
|
||||
const capabilitiesCheckedRef = useRef(false);
|
||||
// Connection state management
|
||||
const [isConnected, setIsConnected] = useState(false);
|
||||
// Session status management
|
||||
const [sessionStatus, setSessionStatus] = useState('Disconnected');
|
||||
// Error message management
|
||||
const [errorMessage, setErrorMessage] = useState('');
|
||||
// CloudXR session reference
|
||||
const [cloudXRSession, setCloudXRSession] = useState<CloudXR.Session | null>(null);
|
||||
// XR mode state for UI visibility
|
||||
const [isXRMode, setIsXRMode] = useState(false);
|
||||
// Server address being used for connection
|
||||
const [serverAddress, setServerAddress] = useState<string>('');
|
||||
// Teleop countdown and state
|
||||
const [isCountingDown, setIsCountingDown] = useState(false);
|
||||
const [countdownRemaining, setCountdownRemaining] = useState(0);
|
||||
const [isTeleopRunning, setIsTeleopRunning] = useState(false);
|
||||
const countdownTimerRef = useRef<number | null>(null);
|
||||
const [countdownDuration, setCountdownDuration] = useState<number>(() => {
|
||||
try {
|
||||
const saved = localStorage.getItem(COUNTDOWN_STORAGE_KEY);
|
||||
if (saved != null) {
|
||||
const value = parseInt(saved, 10);
|
||||
if (!isNaN(value)) {
|
||||
return Math.min(COUNTDOWN_MAX_SECONDS, Math.max(0, value));
|
||||
}
|
||||
}
|
||||
} catch (_) {}
|
||||
return 3;
|
||||
});
|
||||
|
||||
// Persist countdown duration on change
|
||||
useEffect(() => {
|
||||
try {
|
||||
localStorage.setItem(COUNTDOWN_STORAGE_KEY, String(countdownDuration));
|
||||
} catch (_) {}
|
||||
}, [countdownDuration]);
|
||||
|
||||
// Load IWER first (must happen before anything else)
|
||||
// Note: React Three Fiber's emulation is disabled (emulate: false) to avoid conflicts
|
||||
useEffect(() => {
|
||||
const loadIWER = async () => {
|
||||
const { supportsImmersive, iwerLoaded: wasIwerLoaded } = await loadIWERIfNeeded();
|
||||
if (!supportsImmersive) {
|
||||
setErrorMessage('Immersive mode not supported');
|
||||
setIwerLoaded(false);
|
||||
setCapabilitiesValid(false);
|
||||
capabilitiesCheckedRef.current = false; // Reset check flag on failure
|
||||
return;
|
||||
}
|
||||
// IWER loaded successfully, now we can proceed with capability checks
|
||||
setIwerLoaded(true);
|
||||
// Store whether IWER was loaded for status message display later
|
||||
if (wasIwerLoaded) {
|
||||
sessionStorage.setItem('iwerWasLoaded', 'true');
|
||||
}
|
||||
};
|
||||
|
||||
loadIWER();
|
||||
}, []);
|
||||
|
||||
// Update button state when IWER fails and UI becomes ready
|
||||
useEffect(() => {
|
||||
if (cloudXR2DUI && !iwerLoaded && !capabilitiesValid) {
|
||||
cloudXR2DUI.setStartButtonState(true, 'CONNECT (immersive mode not supported)');
|
||||
}
|
||||
}, [cloudXR2DUI, iwerLoaded, capabilitiesValid]);
|
||||
|
||||
// Check capabilities once CloudXR2DUI is ready and IWER is loaded
|
||||
useEffect(() => {
|
||||
const checkCapabilitiesOnce = async () => {
|
||||
if (!cloudXR2DUI || !iwerLoaded) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Guard: only check capabilities once
|
||||
if (capabilitiesCheckedRef.current) {
|
||||
return;
|
||||
}
|
||||
capabilitiesCheckedRef.current = true;
|
||||
|
||||
// Disable button and show checking status
|
||||
cloudXR2DUI.setStartButtonState(true, 'CONNECT (checking capabilities)');
|
||||
|
||||
let result: { success: boolean; failures: string[]; warnings: string[] } = {
|
||||
success: false,
|
||||
failures: [],
|
||||
warnings: [],
|
||||
};
|
||||
try {
|
||||
result = await checkCapabilities();
|
||||
} catch (error) {
|
||||
cloudXR2DUI.showStatus(`Capability check error: ${error}`, 'error');
|
||||
setCapabilitiesValid(false);
|
||||
cloudXR2DUI.setStartButtonState(true, 'CONNECT (capability check failed)');
|
||||
capabilitiesCheckedRef.current = false; // Reset on error for potential retry
|
||||
return;
|
||||
}
|
||||
if (!result.success) {
|
||||
cloudXR2DUI.showStatus(
|
||||
'Browser does not meet required capabilities:\n' + result.failures.join('\n'),
|
||||
'error'
|
||||
);
|
||||
setCapabilitiesValid(false);
|
||||
cloudXR2DUI.setStartButtonState(true, 'CONNECT (capability check failed)');
|
||||
capabilitiesCheckedRef.current = false; // Reset on failure for potential retry
|
||||
return;
|
||||
}
|
||||
|
||||
// Show final status message with IWER info if applicable
|
||||
const iwerWasLoaded = sessionStorage.getItem('iwerWasLoaded') === 'true';
|
||||
if (result.warnings.length > 0) {
|
||||
cloudXR2DUI.showStatus('Performance notice:\n' + result.warnings.join('\n'), 'info');
|
||||
} else if (iwerWasLoaded) {
|
||||
// Include IWER status in the final success message
|
||||
cloudXR2DUI.showStatus(
|
||||
'CloudXR.js SDK is supported. Ready to connect!\nUsing IWER (Immersive Web Emulator Runtime) - Emulating Meta Quest 3.',
|
||||
'info'
|
||||
);
|
||||
} else {
|
||||
cloudXR2DUI.showStatus('CloudXR.js SDK is supported. Ready to connect!', 'success');
|
||||
}
|
||||
|
||||
setCapabilitiesValid(true);
|
||||
cloudXR2DUI.setStartButtonState(false, 'CONNECT');
|
||||
};
|
||||
|
||||
checkCapabilitiesOnce();
|
||||
}, [cloudXR2DUI, iwerLoaded]);
|
||||
|
||||
// Track config changes to trigger re-renders when form values change
|
||||
const [configVersion, setConfigVersion] = useState(0);
|
||||
|
||||
// Initialize CloudXR2DUI
|
||||
useEffect(() => {
|
||||
// Create and initialize the 2D UI manager
|
||||
const ui = new CloudXR2DUI(() => {
|
||||
// Callback when configuration changes
|
||||
setConfigVersion(v => v + 1);
|
||||
});
|
||||
ui.initialize();
|
||||
ui.setupConnectButtonHandler(
|
||||
async () => {
|
||||
// Start XR session
|
||||
if (ui.getConfiguration().immersiveMode === 'ar') {
|
||||
await store.enterAR();
|
||||
} else if (ui.getConfiguration().immersiveMode === 'vr') {
|
||||
await store.enterVR();
|
||||
} else {
|
||||
setErrorMessage('Unrecognized immersive mode');
|
||||
}
|
||||
},
|
||||
(error: Error) => {
|
||||
setErrorMessage(`Failed to start XR session: ${error}`);
|
||||
}
|
||||
);
|
||||
|
||||
setCloudXR2DUI(ui);
|
||||
|
||||
// Cleanup function
|
||||
return () => {
|
||||
if (ui) {
|
||||
ui.cleanup();
|
||||
}
|
||||
};
|
||||
}, []);
|
||||
|
||||
// Update HTML error message display when error state changes
|
||||
useEffect(() => {
|
||||
if (cloudXR2DUI) {
|
||||
if (errorMessage) {
|
||||
cloudXR2DUI.showError(errorMessage);
|
||||
} else {
|
||||
cloudXR2DUI.hideError();
|
||||
}
|
||||
}
|
||||
}, [errorMessage, cloudXR2DUI]);
|
||||
|
||||
// Listen for XR session state changes to update button and UI visibility
|
||||
useEffect(() => {
|
||||
const handleXRStateChange = () => {
|
||||
const xrState = store.getState();
|
||||
|
||||
if (xrState.mode === 'immersive-ar' || xrState.mode === 'immersive-vr') {
|
||||
// XR session is active
|
||||
setIsXRMode(true);
|
||||
if (cloudXR2DUI) {
|
||||
cloudXR2DUI.setStartButtonState(true, 'CONNECT (XR session active)');
|
||||
}
|
||||
} else {
|
||||
// XR session ended
|
||||
setIsXRMode(false);
|
||||
if (cloudXR2DUI) {
|
||||
cloudXR2DUI.setStartButtonState(false, 'CONNECT');
|
||||
}
|
||||
|
||||
if (xrState.error) {
|
||||
setErrorMessage(`XR session error: ${xrState.error}`);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Subscribe to XR state changes
|
||||
const unsubscribe = store.subscribe(handleXRStateChange);
|
||||
|
||||
// Cleanup
|
||||
return () => {
|
||||
unsubscribe();
|
||||
setIsXRMode(false);
|
||||
};
|
||||
}, [cloudXR2DUI]);
|
||||
|
||||
// CloudXR status change handler
|
||||
const handleStatusChange = (connected: boolean, status: string) => {
|
||||
setIsConnected(connected);
|
||||
setSessionStatus(status);
|
||||
};
|
||||
|
||||
// UI Event Handlers
|
||||
const handleStartTeleop = () => {
|
||||
console.log('Start Teleop pressed');
|
||||
|
||||
if (!cloudXRSession) {
|
||||
console.error('CloudXR session not available');
|
||||
return;
|
||||
}
|
||||
|
||||
if (isCountingDown || isTeleopRunning) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Begin countdown before starting teleop (immediately if 0)
|
||||
if (countdownDuration <= 0) {
|
||||
setIsCountingDown(false);
|
||||
setCountdownRemaining(0);
|
||||
|
||||
try {
|
||||
cloudXRSession.sendServerMessage(START_TELEOP_COMMAND);
|
||||
console.log('Start teleop command sent');
|
||||
setIsTeleopRunning(true);
|
||||
} catch (error) {
|
||||
console.error('Failed to send teleop command:', error);
|
||||
setIsTeleopRunning(false);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
setIsCountingDown(true);
|
||||
setCountdownRemaining(countdownDuration);
|
||||
|
||||
countdownTimerRef.current = window.setInterval(() => {
|
||||
setCountdownRemaining(prev => {
|
||||
if (prev <= 1) {
|
||||
// Countdown finished
|
||||
if (countdownTimerRef.current !== null) {
|
||||
clearInterval(countdownTimerRef.current);
|
||||
countdownTimerRef.current = null;
|
||||
}
|
||||
setIsCountingDown(false);
|
||||
|
||||
// Send start teleop command
|
||||
|
||||
try {
|
||||
cloudXRSession.sendServerMessage(START_TELEOP_COMMAND);
|
||||
console.log('Start teleop command sent');
|
||||
setIsTeleopRunning(true);
|
||||
} catch (error) {
|
||||
console.error('Failed to send teleop command:', error);
|
||||
setIsTeleopRunning(false);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
return prev - 1;
|
||||
});
|
||||
}, 1000);
|
||||
};
|
||||
|
||||
const handleStopTeleop = () => {
|
||||
console.log('Stop Teleop pressed');
|
||||
|
||||
// If countdown is active, cancel it and reset state
|
||||
if (isCountingDown) {
|
||||
if (countdownTimerRef.current !== null) {
|
||||
clearInterval(countdownTimerRef.current);
|
||||
countdownTimerRef.current = null;
|
||||
}
|
||||
setIsCountingDown(false);
|
||||
setCountdownRemaining(0);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!cloudXRSession) {
|
||||
console.error('CloudXR session not available');
|
||||
return;
|
||||
}
|
||||
|
||||
// Send stop teleop command
|
||||
const teleopCommand = {
|
||||
type: 'teleop_command',
|
||||
message: {
|
||||
command: 'stop teleop',
|
||||
},
|
||||
};
|
||||
|
||||
try {
|
||||
cloudXRSession.sendServerMessage(teleopCommand);
|
||||
console.log('Stop teleop command sent');
|
||||
setIsTeleopRunning(false);
|
||||
} catch (error) {
|
||||
console.error('Failed to send teleop command:', error);
|
||||
}
|
||||
};
|
||||
|
||||
const handleResetTeleop = () => {
|
||||
console.log('Reset Teleop pressed');
|
||||
|
||||
// Cancel any active countdown
|
||||
if (countdownTimerRef.current !== null) {
|
||||
clearInterval(countdownTimerRef.current);
|
||||
countdownTimerRef.current = null;
|
||||
}
|
||||
setIsCountingDown(false);
|
||||
setCountdownRemaining(0);
|
||||
|
||||
if (!cloudXRSession) {
|
||||
console.error('CloudXR session not available');
|
||||
return;
|
||||
}
|
||||
|
||||
// Send stop teleop command first
|
||||
const stopCommand = {
|
||||
type: 'teleop_command',
|
||||
message: {
|
||||
command: 'stop teleop',
|
||||
},
|
||||
};
|
||||
|
||||
// Send reset teleop command
|
||||
const resetCommand = {
|
||||
type: 'teleop_command',
|
||||
message: {
|
||||
command: 'reset teleop',
|
||||
},
|
||||
};
|
||||
|
||||
try {
|
||||
cloudXRSession.sendServerMessage(stopCommand);
|
||||
console.log('Stop teleop command sent');
|
||||
cloudXRSession.sendServerMessage(resetCommand);
|
||||
console.log('Reset teleop command sent');
|
||||
setIsTeleopRunning(false);
|
||||
} catch (error) {
|
||||
console.error('Failed to send teleop commands:', error);
|
||||
}
|
||||
};
|
||||
|
||||
const handleDisconnect = () => {
|
||||
console.log('Disconnect pressed');
|
||||
|
||||
// Cleanup countdown state on disconnect
|
||||
if (countdownTimerRef.current !== null) {
|
||||
clearInterval(countdownTimerRef.current);
|
||||
countdownTimerRef.current = null;
|
||||
}
|
||||
setIsCountingDown(false);
|
||||
setCountdownRemaining(0);
|
||||
setIsTeleopRunning(false);
|
||||
|
||||
const xrState = store.getState();
|
||||
const session = xrState.session;
|
||||
if (session) {
|
||||
session.end().catch((err: unknown) => {
|
||||
setErrorMessage(
|
||||
`Failed to end XR session: ${err instanceof Error ? err.message : String(err)}`
|
||||
);
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
// Countdown configuration handlers (0-5 seconds)
|
||||
const handleIncreaseCountdown = () => {
|
||||
if (isCountingDown) return;
|
||||
setCountdownDuration(prev => Math.min(COUNTDOWN_MAX_SECONDS, prev + 1));
|
||||
};
|
||||
|
||||
const handleDecreaseCountdown = () => {
|
||||
if (isCountingDown) return;
|
||||
setCountdownDuration(prev => Math.max(0, prev - 1));
|
||||
};
|
||||
|
||||
// Memo config based on configVersion (manual dependency tracker incremented on config changes)
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
const config = useMemo(
|
||||
() => (cloudXR2DUI ? cloudXR2DUI.getConfiguration() : null),
|
||||
[cloudXR2DUI, configVersion]
|
||||
);
|
||||
|
||||
// Sync XR mode state to body class for CSS styling
|
||||
useEffect(() => {
|
||||
if (isXRMode) {
|
||||
document.body.classList.add('xr-mode');
|
||||
} else {
|
||||
document.body.classList.remove('xr-mode');
|
||||
}
|
||||
|
||||
return () => {
|
||||
document.body.classList.remove('xr-mode');
|
||||
};
|
||||
}, [isXRMode]);
|
||||
|
||||
return (
|
||||
<>
|
||||
<Canvas
|
||||
events={noEvents}
|
||||
style={{
|
||||
background: '#000',
|
||||
width: '100vw',
|
||||
height: '100vh',
|
||||
position: 'fixed',
|
||||
top: 0,
|
||||
left: 0,
|
||||
zIndex: -1,
|
||||
}}
|
||||
gl={{
|
||||
preserveDrawingBuffer: true, // Keep buffer for custom rendering
|
||||
antialias: true,
|
||||
}}
|
||||
camera={{ position: [0, 0, 0.65] }}
|
||||
onWheel={e => {
|
||||
e.preventDefault();
|
||||
}}
|
||||
>
|
||||
<PointerEvents batchEvents={false} />
|
||||
<XR store={store}>
|
||||
<NonAREnvironment />
|
||||
<XROrigin />
|
||||
{cloudXR2DUI && config && (
|
||||
<>
|
||||
<CloudXRComponent
|
||||
config={config}
|
||||
onStatusChange={handleStatusChange}
|
||||
onError={error => {
|
||||
if (cloudXR2DUI) {
|
||||
cloudXR2DUI.showError(error);
|
||||
}
|
||||
}}
|
||||
onSessionReady={setCloudXRSession}
|
||||
onServerAddress={setServerAddress}
|
||||
/>
|
||||
<CloudXR3DUI
|
||||
onStartTeleop={handleStartTeleop}
|
||||
onStopTeleop={handleStopTeleop}
|
||||
onDisconnect={handleDisconnect}
|
||||
onResetTeleop={handleResetTeleop}
|
||||
serverAddress={serverAddress || config.serverIP}
|
||||
sessionStatus={sessionStatus}
|
||||
playLabel={
|
||||
isTeleopRunning
|
||||
? 'Running'
|
||||
: isCountingDown
|
||||
? `Starting in ${countdownRemaining} sec...`
|
||||
: 'Play'
|
||||
}
|
||||
playDisabled={isCountingDown || isTeleopRunning}
|
||||
countdownSeconds={countdownDuration}
|
||||
onCountdownIncrease={handleIncreaseCountdown}
|
||||
onCountdownDecrease={handleDecreaseCountdown}
|
||||
countdownDisabled={isCountingDown}
|
||||
position={[0, 1.6, -1.8]}
|
||||
rotation={[0, 0, 0]}
|
||||
/>
|
||||
</>
|
||||
)}
|
||||
</XR>
|
||||
</Canvas>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
export default App;
|
||||
Reference in New Issue
Block a user