/** * App.tsx - Main CloudXR React Application * * This is the root component of the CloudXR React example application. It sets up: * - WebXR session management and XR store configuration * - CloudXR server configuration (IP, port, stream settings) * - UI state management (connection status, session state) * - Integration between CloudXR rendering component and UI components * - Entry point for AR/VR experiences with CloudXR streaming * * The app integrates with the HTML interface which provides a "CONNECT" button * to enter AR mode and displays the CloudXR UI with controls for teleop actions * and disconnect when in XR mode. */ import { checkCapabilities } from '@helpers/BrowserCapabilities'; import { loadIWERIfNeeded } from '@helpers/LoadIWER'; import { overridePressureObserver } from '@helpers/overridePressureObserver'; import * as CloudXR from '@nvidia/cloudxr'; import { Environment } from '@react-three/drei'; import { Canvas } from '@react-three/fiber'; import { setPreferredColorScheme } from '@react-three/uikit'; import { XR, createXRStore, noEvents, PointerEvents, XROrigin, useXR } from '@react-three/xr'; import { useState, useMemo, useEffect, useRef } from 'react'; import { CloudXR2DUI } from './CloudXR2DUI'; import CloudXRComponent from './CloudXRComponent'; import CloudXR3DUI from './CloudXRUI'; // Override PressureObserver early to catch errors from buggy browser implementations overridePressureObserver(); const store = createXRStore({ foveation: 0, emulate: false, // Disable IWER emulation from react in favor of custom iwer loading function // Configure WebXR input profiles to use local assets // Use relative path from current page location baseAssetPath: `${new URL('.', window.location).href}npm/@webxr-input-profiles/assets@${process.env.WEBXR_ASSETS_VERSION}/dist/profiles/`, hand: { model: false, // Disable hand models but keep pointer functionality }, }); setPreferredColorScheme('dark'); const START_TELEOP_COMMAND = { type: 'teleop_command', message: { command: 'start teleop', }, } as const; // Environment component like controller-test function NonAREnvironment() { // Use local HDR file instead of preset so client doesn't need to download it from CDN return ( ); } function App() { const COUNTDOWN_MAX_SECONDS = 9; const COUNTDOWN_STORAGE_KEY = 'cxr.react.countdownSeconds'; // 2D UI management const [cloudXR2DUI, setCloudXR2DUI] = useState(null); // IWER loading state const [iwerLoaded, setIwerLoaded] = useState(false); // Capability state management const [capabilitiesValid, setCapabilitiesValid] = useState(false); const capabilitiesCheckedRef = useRef(false); // Connection state management const [isConnected, setIsConnected] = useState(false); // Session status management const [sessionStatus, setSessionStatus] = useState('Disconnected'); // Error message management const [errorMessage, setErrorMessage] = useState(''); // CloudXR session reference const [cloudXRSession, setCloudXRSession] = useState(null); // XR mode state for UI visibility const [isXRMode, setIsXRMode] = useState(false); // Server address being used for connection const [serverAddress, setServerAddress] = useState(''); // Teleop countdown and state const [isCountingDown, setIsCountingDown] = useState(false); const [countdownRemaining, setCountdownRemaining] = useState(0); const [isTeleopRunning, setIsTeleopRunning] = useState(false); const countdownTimerRef = useRef(null); const [countdownDuration, setCountdownDuration] = useState(() => { try { const saved = localStorage.getItem(COUNTDOWN_STORAGE_KEY); if (saved != null) { const value = parseInt(saved, 10); if (!isNaN(value)) { return Math.min(COUNTDOWN_MAX_SECONDS, Math.max(0, value)); } } } catch (_) {} return 3; }); // Persist countdown duration on change useEffect(() => { try { localStorage.setItem(COUNTDOWN_STORAGE_KEY, String(countdownDuration)); } catch (_) {} }, [countdownDuration]); // Load IWER first (must happen before anything else) // Note: React Three Fiber's emulation is disabled (emulate: false) to avoid conflicts useEffect(() => { const loadIWER = async () => { const { supportsImmersive, iwerLoaded: wasIwerLoaded } = await loadIWERIfNeeded(); if (!supportsImmersive) { setErrorMessage('Immersive mode not supported'); setIwerLoaded(false); setCapabilitiesValid(false); capabilitiesCheckedRef.current = false; // Reset check flag on failure return; } // IWER loaded successfully, now we can proceed with capability checks setIwerLoaded(true); // Store whether IWER was loaded for status message display later if (wasIwerLoaded) { sessionStorage.setItem('iwerWasLoaded', 'true'); } }; loadIWER(); }, []); // Update button state when IWER fails and UI becomes ready useEffect(() => { if (cloudXR2DUI && !iwerLoaded && !capabilitiesValid) { cloudXR2DUI.setStartButtonState(true, 'CONNECT (immersive mode not supported)'); } }, [cloudXR2DUI, iwerLoaded, capabilitiesValid]); // Check capabilities once CloudXR2DUI is ready and IWER is loaded useEffect(() => { const checkCapabilitiesOnce = async () => { if (!cloudXR2DUI || !iwerLoaded) { return; } // Guard: only check capabilities once if (capabilitiesCheckedRef.current) { return; } capabilitiesCheckedRef.current = true; // Disable button and show checking status cloudXR2DUI.setStartButtonState(true, 'CONNECT (checking capabilities)'); let result: { success: boolean; failures: string[]; warnings: string[] } = { success: false, failures: [], warnings: [], }; try { result = await checkCapabilities(); } catch (error) { cloudXR2DUI.showStatus(`Capability check error: ${error}`, 'error'); setCapabilitiesValid(false); cloudXR2DUI.setStartButtonState(true, 'CONNECT (capability check failed)'); capabilitiesCheckedRef.current = false; // Reset on error for potential retry return; } if (!result.success) { cloudXR2DUI.showStatus( 'Browser does not meet required capabilities:\n' + result.failures.join('\n'), 'error' ); setCapabilitiesValid(false); cloudXR2DUI.setStartButtonState(true, 'CONNECT (capability check failed)'); capabilitiesCheckedRef.current = false; // Reset on failure for potential retry return; } // Show final status message with IWER info if applicable const iwerWasLoaded = sessionStorage.getItem('iwerWasLoaded') === 'true'; if (result.warnings.length > 0) { cloudXR2DUI.showStatus('Performance notice:\n' + result.warnings.join('\n'), 'info'); } else if (iwerWasLoaded) { // Include IWER status in the final success message cloudXR2DUI.showStatus( 'CloudXR.js SDK is supported. Ready to connect!\nUsing IWER (Immersive Web Emulator Runtime) - Emulating Meta Quest 3.', 'info' ); } else { cloudXR2DUI.showStatus('CloudXR.js SDK is supported. Ready to connect!', 'success'); } setCapabilitiesValid(true); cloudXR2DUI.setStartButtonState(false, 'CONNECT'); }; checkCapabilitiesOnce(); }, [cloudXR2DUI, iwerLoaded]); // Track config changes to trigger re-renders when form values change const [configVersion, setConfigVersion] = useState(0); // Initialize CloudXR2DUI useEffect(() => { // Create and initialize the 2D UI manager const ui = new CloudXR2DUI(() => { // Callback when configuration changes setConfigVersion(v => v + 1); }); ui.initialize(); ui.setupConnectButtonHandler( async () => { // Start XR session if (ui.getConfiguration().immersiveMode === 'ar') { await store.enterAR(); } else if (ui.getConfiguration().immersiveMode === 'vr') { await store.enterVR(); } else { setErrorMessage('Unrecognized immersive mode'); } }, (error: Error) => { setErrorMessage(`Failed to start XR session: ${error}`); } ); setCloudXR2DUI(ui); // Cleanup function return () => { if (ui) { ui.cleanup(); } }; }, []); // Update HTML error message display when error state changes useEffect(() => { if (cloudXR2DUI) { if (errorMessage) { cloudXR2DUI.showError(errorMessage); } else { cloudXR2DUI.hideError(); } } }, [errorMessage, cloudXR2DUI]); // Listen for XR session state changes to update button and UI visibility useEffect(() => { const handleXRStateChange = () => { const xrState = store.getState(); if (xrState.mode === 'immersive-ar' || xrState.mode === 'immersive-vr') { // XR session is active setIsXRMode(true); if (cloudXR2DUI) { cloudXR2DUI.setStartButtonState(true, 'CONNECT (XR session active)'); } } else { // XR session ended setIsXRMode(false); if (cloudXR2DUI) { cloudXR2DUI.setStartButtonState(false, 'CONNECT'); } if (xrState.error) { setErrorMessage(`XR session error: ${xrState.error}`); } } }; // Subscribe to XR state changes const unsubscribe = store.subscribe(handleXRStateChange); // Cleanup return () => { unsubscribe(); setIsXRMode(false); }; }, [cloudXR2DUI]); // CloudXR status change handler const handleStatusChange = (connected: boolean, status: string) => { setIsConnected(connected); setSessionStatus(status); }; // UI Event Handlers const handleStartTeleop = () => { console.log('Start Teleop pressed'); if (!cloudXRSession) { console.error('CloudXR session not available'); return; } if (isCountingDown || isTeleopRunning) { return; } // Begin countdown before starting teleop (immediately if 0) if (countdownDuration <= 0) { setIsCountingDown(false); setCountdownRemaining(0); try { cloudXRSession.sendServerMessage(START_TELEOP_COMMAND); console.log('Start teleop command sent'); setIsTeleopRunning(true); } catch (error) { console.error('Failed to send teleop command:', error); setIsTeleopRunning(false); } return; } setIsCountingDown(true); setCountdownRemaining(countdownDuration); countdownTimerRef.current = window.setInterval(() => { setCountdownRemaining(prev => { if (prev <= 1) { // Countdown finished if (countdownTimerRef.current !== null) { clearInterval(countdownTimerRef.current); countdownTimerRef.current = null; } setIsCountingDown(false); // Send start teleop command try { cloudXRSession.sendServerMessage(START_TELEOP_COMMAND); console.log('Start teleop command sent'); setIsTeleopRunning(true); } catch (error) { console.error('Failed to send teleop command:', error); setIsTeleopRunning(false); } return 0; } return prev - 1; }); }, 1000); }; const handleStopTeleop = () => { console.log('Stop Teleop pressed'); // If countdown is active, cancel it and reset state if (isCountingDown) { if (countdownTimerRef.current !== null) { clearInterval(countdownTimerRef.current); countdownTimerRef.current = null; } setIsCountingDown(false); setCountdownRemaining(0); return; } if (!cloudXRSession) { console.error('CloudXR session not available'); return; } // Send stop teleop command const teleopCommand = { type: 'teleop_command', message: { command: 'stop teleop', }, }; try { cloudXRSession.sendServerMessage(teleopCommand); console.log('Stop teleop command sent'); setIsTeleopRunning(false); } catch (error) { console.error('Failed to send teleop command:', error); } }; const handleResetTeleop = () => { console.log('Reset Teleop pressed'); // Cancel any active countdown if (countdownTimerRef.current !== null) { clearInterval(countdownTimerRef.current); countdownTimerRef.current = null; } setIsCountingDown(false); setCountdownRemaining(0); if (!cloudXRSession) { console.error('CloudXR session not available'); return; } // Send stop teleop command first const stopCommand = { type: 'teleop_command', message: { command: 'stop teleop', }, }; // Send reset teleop command const resetCommand = { type: 'teleop_command', message: { command: 'reset teleop', }, }; try { cloudXRSession.sendServerMessage(stopCommand); console.log('Stop teleop command sent'); cloudXRSession.sendServerMessage(resetCommand); console.log('Reset teleop command sent'); setIsTeleopRunning(false); } catch (error) { console.error('Failed to send teleop commands:', error); } }; const handleDisconnect = () => { console.log('Disconnect pressed'); // Cleanup countdown state on disconnect if (countdownTimerRef.current !== null) { clearInterval(countdownTimerRef.current); countdownTimerRef.current = null; } setIsCountingDown(false); setCountdownRemaining(0); setIsTeleopRunning(false); const xrState = store.getState(); const session = xrState.session; if (session) { session.end().catch((err: unknown) => { setErrorMessage( `Failed to end XR session: ${err instanceof Error ? err.message : String(err)}` ); }); } }; // Countdown configuration handlers (0-5 seconds) const handleIncreaseCountdown = () => { if (isCountingDown) return; setCountdownDuration(prev => Math.min(COUNTDOWN_MAX_SECONDS, prev + 1)); }; const handleDecreaseCountdown = () => { if (isCountingDown) return; setCountdownDuration(prev => Math.max(0, prev - 1)); }; // Memo config based on configVersion (manual dependency tracker incremented on config changes) // eslint-disable-next-line react-hooks/exhaustive-deps const config = useMemo( () => (cloudXR2DUI ? cloudXR2DUI.getConfiguration() : null), [cloudXR2DUI, configVersion] ); // Sync XR mode state to body class for CSS styling useEffect(() => { if (isXRMode) { document.body.classList.add('xr-mode'); } else { document.body.classList.remove('xr-mode'); } return () => { document.body.classList.remove('xr-mode'); }; }, [isXRMode]); return ( <> { e.preventDefault(); }} > {cloudXR2DUI && config && ( <> { if (cloudXR2DUI) { cloudXR2DUI.showError(error); } }} onSessionReady={setCloudXRSession} onServerAddress={setServerAddress} /> )} ); } export default App;