mindbot/deps/cloudxr/react/src/App.tsx

/**
 * App.tsx - Main CloudXR React Application
 *
 * This is the root component of the CloudXR React example application. It sets up:
 * - WebXR session management and XR store configuration
 * - CloudXR server configuration (IP, port, stream settings)
 * - UI state management (connection status, session state)
 * - Integration between CloudXR rendering component and UI components
 * - Entry point for AR/VR experiences with CloudXR streaming
 *
 * The app integrates with the HTML interface which provides a "CONNECT" button
 * to enter AR mode and displays the CloudXR UI with controls for teleop actions
 * and disconnect when in XR mode.
 */

import { checkCapabilities } from '@helpers/BrowserCapabilities';
import { loadIWERIfNeeded } from '@helpers/LoadIWER';
import { overridePressureObserver } from '@helpers/overridePressureObserver';
import * as CloudXR from '@nvidia/cloudxr';
import { Environment } from '@react-three/drei';
import { Canvas } from '@react-three/fiber';
import { setPreferredColorScheme } from '@react-three/uikit';
import { XR, createXRStore, noEvents, PointerEvents, XROrigin, useXR } from '@react-three/xr';
import { useState, useMemo, useEffect, useRef } from 'react';

import { CloudXR2DUI } from './CloudXR2DUI';
import CloudXRComponent from './CloudXRComponent';
import CloudXR3DUI from './CloudXRUI';

// Override PressureObserver early to catch errors from buggy browser implementations
overridePressureObserver();

const store = createXRStore({
  foveation: 0,
  emulate: false, // Disable IWER emulation from react in favor of custom iwer loading function
  // Configure WebXR input profiles to use local assets
  // Use relative path from current page location
  baseAssetPath: `${new URL('.', window.location).href}npm/@webxr-input-profiles/assets@${process.env.WEBXR_ASSETS_VERSION}/dist/profiles/`,
  hand: {
    model: false, // Disable hand models but keep pointer functionality
  },
});

setPreferredColorScheme('dark');

const START_TELEOP_COMMAND = {
  type: 'teleop_command',
  message: {
    command: 'start teleop',
  },
} as const;

// Environment component like controller-test
function NonAREnvironment() {
  // Use local HDR file instead of preset so client doesn't need to download it from CDN
  return (
    <Environment
      blur={0.2}
      background={false}
      environmentIntensity={2}
      files="assets/hdri/potsdamer_platz_1k.hdr"
    />
  );
}

function App() {
  const COUNTDOWN_MAX_SECONDS = 9;
  const COUNTDOWN_STORAGE_KEY = 'cxr.react.countdownSeconds';
  // 2D UI management
  const [cloudXR2DUI, setCloudXR2DUI] = useState<CloudXR2DUI | null>(null);
  // IWER loading state
  const [iwerLoaded, setIwerLoaded] = useState(false);
  // Capability state management
  const [capabilitiesValid, setCapabilitiesValid] = useState(false);
  const capabilitiesCheckedRef = useRef(false);
  // Connection state management
  const [isConnected, setIsConnected] = useState(false);
  // Session status management
  const [sessionStatus, setSessionStatus] = useState('Disconnected');
  // Error message management
  const [errorMessage, setErrorMessage] = useState('');
  // CloudXR session reference
  const [cloudXRSession, setCloudXRSession] = useState<CloudXR.Session | null>(null);
  // XR mode state for UI visibility
  const [isXRMode, setIsXRMode] = useState(false);
  // Server address being used for connection
  const [serverAddress, setServerAddress] = useState<string>('');
  // Teleop countdown and state
  const [isCountingDown, setIsCountingDown] = useState(false);
  const [countdownRemaining, setCountdownRemaining] = useState(0);
  const [isTeleopRunning, setIsTeleopRunning] = useState(false);
  const countdownTimerRef = useRef<number | null>(null);
  const [countdownDuration, setCountdownDuration] = useState<number>(() => {
    try {
      const saved = localStorage.getItem(COUNTDOWN_STORAGE_KEY);
      if (saved != null) {
        const value = parseInt(saved, 10);
        if (!isNaN(value)) {
          return Math.min(COUNTDOWN_MAX_SECONDS, Math.max(0, value));
        }
      }
    } catch (_) {}
    return 3;
  });

  // Persist countdown duration on change
  useEffect(() => {
    try {
      localStorage.setItem(COUNTDOWN_STORAGE_KEY, String(countdownDuration));
    } catch (_) {}
  }, [countdownDuration]);

  // Load IWER first (must happen before anything else)
  // Note: React Three Fiber's emulation is disabled (emulate: false) to avoid conflicts
  useEffect(() => {
    const loadIWER = async () => {
      const { supportsImmersive, iwerLoaded: wasIwerLoaded } = await loadIWERIfNeeded();
      if (!supportsImmersive) {
        setErrorMessage('Immersive mode not supported');
        setIwerLoaded(false);
        setCapabilitiesValid(false);
        capabilitiesCheckedRef.current = false; // Reset check flag on failure
        return;
      }
      // IWER loaded successfully, now we can proceed with capability checks
      setIwerLoaded(true);
      // Store whether IWER was loaded for status message display later
      if (wasIwerLoaded) {
        sessionStorage.setItem('iwerWasLoaded', 'true');
      }
    };

    loadIWER();
  }, []);

  // Update button state when IWER fails and UI becomes ready
  useEffect(() => {
    if (cloudXR2DUI && !iwerLoaded && !capabilitiesValid) {
      cloudXR2DUI.setStartButtonState(true, 'CONNECT (immersive mode not supported)');
    }
  }, [cloudXR2DUI, iwerLoaded, capabilitiesValid]);

  // Check capabilities once CloudXR2DUI is ready and IWER is loaded
  useEffect(() => {
    const checkCapabilitiesOnce = async () => {
      if (!cloudXR2DUI || !iwerLoaded) {
        return;
      }

      // Guard: only check capabilities once
      if (capabilitiesCheckedRef.current) {
        return;
      }
      capabilitiesCheckedRef.current = true;

      // Disable button and show checking status
      cloudXR2DUI.setStartButtonState(true, 'CONNECT (checking capabilities)');

      let result: { success: boolean; failures: string[]; warnings: string[] } = {
        success: false,
        failures: [],
        warnings: [],
      };
      try {
        result = await checkCapabilities();
      } catch (error) {
        cloudXR2DUI.showStatus(`Capability check error: ${error}`, 'error');
        setCapabilitiesValid(false);
        cloudXR2DUI.setStartButtonState(true, 'CONNECT (capability check failed)');
        capabilitiesCheckedRef.current = false; // Reset on error for potential retry
        return;
      }
      if (!result.success) {
        cloudXR2DUI.showStatus(
          'Browser does not meet required capabilities:\n' + result.failures.join('\n'),
          'error'
        );
        setCapabilitiesValid(false);
        cloudXR2DUI.setStartButtonState(true, 'CONNECT (capability check failed)');
        capabilitiesCheckedRef.current = false; // Reset on failure for potential retry
        return;
      }

      // Show final status message with IWER info if applicable
      const iwerWasLoaded = sessionStorage.getItem('iwerWasLoaded') === 'true';
      if (result.warnings.length > 0) {
        cloudXR2DUI.showStatus('Performance notice:\n' + result.warnings.join('\n'), 'info');
      } else if (iwerWasLoaded) {
        // Include IWER status in the final success message
        cloudXR2DUI.showStatus(
          'CloudXR.js SDK is supported. Ready to connect!\nUsing IWER (Immersive Web Emulator Runtime) - Emulating Meta Quest 3.',
          'info'
        );
      } else {
        cloudXR2DUI.showStatus('CloudXR.js SDK is supported. Ready to connect!', 'success');
      }

      setCapabilitiesValid(true);
      cloudXR2DUI.setStartButtonState(false, 'CONNECT');
    };

    checkCapabilitiesOnce();
  }, [cloudXR2DUI, iwerLoaded]);

  // Track config changes to trigger re-renders when form values change
  const [configVersion, setConfigVersion] = useState(0);

  // Initialize CloudXR2DUI
  useEffect(() => {
    // Create and initialize the 2D UI manager
    const ui = new CloudXR2DUI(() => {
      // Callback when configuration changes
      setConfigVersion(v => v + 1);
    });
    ui.initialize();
    ui.setupConnectButtonHandler(
      async () => {
        // Start XR session
        if (ui.getConfiguration().immersiveMode === 'ar') {
          await store.enterAR();
        } else if (ui.getConfiguration().immersiveMode === 'vr') {
          await store.enterVR();
        } else {
          setErrorMessage('Unrecognized immersive mode');
        }
      },
      (error: Error) => {
        setErrorMessage(`Failed to start XR session: ${error}`);
      }
    );

    setCloudXR2DUI(ui);

    // Cleanup function
    return () => {
      if (ui) {
        ui.cleanup();
      }
    };
  }, []);

  // Update HTML error message display when error state changes
  useEffect(() => {
    if (cloudXR2DUI) {
      if (errorMessage) {
        cloudXR2DUI.showError(errorMessage);
      } else {
        cloudXR2DUI.hideError();
      }
    }
  }, [errorMessage, cloudXR2DUI]);

  // Listen for XR session state changes to update button and UI visibility
  useEffect(() => {
    const handleXRStateChange = () => {
      const xrState = store.getState();

      if (xrState.mode === 'immersive-ar' || xrState.mode === 'immersive-vr') {
        // XR session is active
        setIsXRMode(true);
        if (cloudXR2DUI) {
          cloudXR2DUI.setStartButtonState(true, 'CONNECT (XR session active)');
        }
      } else {
        // XR session ended
        setIsXRMode(false);
        if (cloudXR2DUI) {
          cloudXR2DUI.setStartButtonState(false, 'CONNECT');
        }

        if (xrState.error) {
          setErrorMessage(`XR session error: ${xrState.error}`);
        }
      }
    };

    // Subscribe to XR state changes
    const unsubscribe = store.subscribe(handleXRStateChange);

    // Cleanup
    return () => {
      unsubscribe();
      setIsXRMode(false);
    };
  }, [cloudXR2DUI]);

  // CloudXR status change handler
  const handleStatusChange = (connected: boolean, status: string) => {
    setIsConnected(connected);
    setSessionStatus(status);
  };

  // UI Event Handlers
  const handleStartTeleop = () => {
    console.log('Start Teleop pressed');

    if (!cloudXRSession) {
      console.error('CloudXR session not available');
      return;
    }

    if (isCountingDown || isTeleopRunning) {
      return;
    }

    // Begin countdown before starting teleop (immediately if 0)
    if (countdownDuration <= 0) {
      setIsCountingDown(false);
      setCountdownRemaining(0);

      try {
        cloudXRSession.sendServerMessage(START_TELEOP_COMMAND);
        console.log('Start teleop command sent');
        setIsTeleopRunning(true);
      } catch (error) {
        console.error('Failed to send teleop command:', error);
        setIsTeleopRunning(false);
      }
      return;
    }

    setIsCountingDown(true);
    setCountdownRemaining(countdownDuration);

    countdownTimerRef.current = window.setInterval(() => {
      setCountdownRemaining(prev => {
        if (prev <= 1) {
          // Countdown finished
          if (countdownTimerRef.current !== null) {
            clearInterval(countdownTimerRef.current);
            countdownTimerRef.current = null;
          }
          setIsCountingDown(false);

          // Send start teleop command

          try {
            cloudXRSession.sendServerMessage(START_TELEOP_COMMAND);
            console.log('Start teleop command sent');
            setIsTeleopRunning(true);
          } catch (error) {
            console.error('Failed to send teleop command:', error);
            setIsTeleopRunning(false);
          }

          return 0;
        }
        return prev - 1;
      });
    }, 1000);
  };

  const handleStopTeleop = () => {
    console.log('Stop Teleop pressed');

    // If countdown is active, cancel it and reset state
    if (isCountingDown) {
      if (countdownTimerRef.current !== null) {
        clearInterval(countdownTimerRef.current);
        countdownTimerRef.current = null;
      }
      setIsCountingDown(false);
      setCountdownRemaining(0);
      return;
    }

    if (!cloudXRSession) {
      console.error('CloudXR session not available');
      return;
    }

    // Send stop teleop command
    const teleopCommand = {
      type: 'teleop_command',
      message: {
        command: 'stop teleop',
      },
    };

    try {
      cloudXRSession.sendServerMessage(teleopCommand);
      console.log('Stop teleop command sent');
      setIsTeleopRunning(false);
    } catch (error) {
      console.error('Failed to send teleop command:', error);
    }
  };

  const handleResetTeleop = () => {
    console.log('Reset Teleop pressed');

    // Cancel any active countdown
    if (countdownTimerRef.current !== null) {
      clearInterval(countdownTimerRef.current);
      countdownTimerRef.current = null;
    }
    setIsCountingDown(false);
    setCountdownRemaining(0);

    if (!cloudXRSession) {
      console.error('CloudXR session not available');
      return;
    }

    // Send stop teleop command first
    const stopCommand = {
      type: 'teleop_command',
      message: {
        command: 'stop teleop',
      },
    };

    // Send reset teleop command
    const resetCommand = {
      type: 'teleop_command',
      message: {
        command: 'reset teleop',
      },
    };

    try {
      cloudXRSession.sendServerMessage(stopCommand);
      console.log('Stop teleop command sent');
      cloudXRSession.sendServerMessage(resetCommand);
      console.log('Reset teleop command sent');
      setIsTeleopRunning(false);
    } catch (error) {
      console.error('Failed to send teleop commands:', error);
    }
  };

  const handleDisconnect = () => {
    console.log('Disconnect pressed');

    // Cleanup countdown state on disconnect
    if (countdownTimerRef.current !== null) {
      clearInterval(countdownTimerRef.current);
      countdownTimerRef.current = null;
    }
    setIsCountingDown(false);
    setCountdownRemaining(0);
    setIsTeleopRunning(false);

    const xrState = store.getState();
    const session = xrState.session;
    if (session) {
      session.end().catch((err: unknown) => {
        setErrorMessage(
          `Failed to end XR session: ${err instanceof Error ? err.message : String(err)}`
        );
      });
    }
  };

  // Countdown configuration handlers (0-5 seconds)
  const handleIncreaseCountdown = () => {
    if (isCountingDown) return;
    setCountdownDuration(prev => Math.min(COUNTDOWN_MAX_SECONDS, prev + 1));
  };

  const handleDecreaseCountdown = () => {
    if (isCountingDown) return;
    setCountdownDuration(prev => Math.max(0, prev - 1));
  };

  // Memo config based on configVersion (manual dependency tracker incremented on config changes)
  // eslint-disable-next-line react-hooks/exhaustive-deps
  const config = useMemo(
    () => (cloudXR2DUI ? cloudXR2DUI.getConfiguration() : null),
    [cloudXR2DUI, configVersion]
  );

  // Sync XR mode state to body class for CSS styling
  useEffect(() => {
    if (isXRMode) {
      document.body.classList.add('xr-mode');
    } else {
      document.body.classList.remove('xr-mode');
    }

    return () => {
      document.body.classList.remove('xr-mode');
    };
  }, [isXRMode]);

  return (
    <>
      <Canvas
        events={noEvents}
        style={{
          background: '#000',
          width: '100vw',
          height: '100vh',
          position: 'fixed',
          top: 0,
          left: 0,
          zIndex: -1,
        }}
        gl={{
          preserveDrawingBuffer: true, // Keep buffer for custom rendering
          antialias: true,
        }}
        camera={{ position: [0, 0, 0.65] }}
        onWheel={e => {
          e.preventDefault();
        }}
      >
        <PointerEvents batchEvents={false} />
        <XR store={store}>
          <NonAREnvironment />
          <XROrigin />
          {cloudXR2DUI && config && (
            <>
              <CloudXRComponent
                config={config}
                onStatusChange={handleStatusChange}
                onError={error => {
                  if (cloudXR2DUI) {
                    cloudXR2DUI.showError(error);
                  }
                }}
                onSessionReady={setCloudXRSession}
                onServerAddress={setServerAddress}
              />
              <CloudXR3DUI
                onStartTeleop={handleStartTeleop}
                onStopTeleop={handleStopTeleop}
                onDisconnect={handleDisconnect}
                onResetTeleop={handleResetTeleop}
                serverAddress={serverAddress || config.serverIP}
                sessionStatus={sessionStatus}
                playLabel={
                  isTeleopRunning
                    ? 'Running'
                    : isCountingDown
                      ? `Starting in ${countdownRemaining} sec...`
                      : 'Play'
                }
                playDisabled={isCountingDown || isTeleopRunning}
                countdownSeconds={countdownDuration}
                onCountdownIncrease={handleIncreaseCountdown}
                onCountdownDecrease={handleDecreaseCountdown}
                countdownDisabled={isCountingDown}
                position={[0, 1.6, -1.8]}
                rotation={[0, 0, 0]}
              />
            </>
          )}
        </XR>
      </Canvas>
    </>
  );
}

export default App;