Replaces manual H264/TCP stereo streaming with NVIDIA CloudXR for
higher-quality stereoscopic rendering and lower latency.
Changes:
- teleop_xr_agent.py: add --cloudxr flag (enables Isaac Sim XR mode,
disables manual StreamingManager)
- deps/cloudxr/: NVIDIA CloudXR.js SDK (Early Access) with Isaac Lab
teleop React web client
- deps/cloudxr/Dockerfile.wss.proxy: HAProxy WSS proxy for PICO 4 Ultra
HTTPS mode (routes wss://48322 → ws://49100)
- deps/cloudxr/isaac/webpack.dev.js: disable file watching to avoid
EMFILE errors with large node_modules
- deps/cloudxr/INSTALL.md: full setup guide
Usage:
# Start CloudXR Runtime + Isaac Lab
cd ~/IsaacLab && ./docker/container.py start \
--files docker-compose.cloudxr-runtime.patch.yaml \
--env-file .env.cloudxr-runtime
# Run teleop with CloudXR
~/IsaacLab/isaaclab.sh -p teleop_xr_agent.py \
--task Isaac-MindRobot-2i-DualArm-IK-Abs-v0 --cloudxr
# Serve web client
cd deps/cloudxr/isaac && npm run dev-server:https
219 lines
7.1 KiB
TypeScript
219 lines
7.1 KiB
TypeScript
/**
|
|
* CloudXRUI.tsx - CloudXR User Interface Component
|
|
*
|
|
* This component renders the in-VR user interface for the CloudXR application using
|
|
* React Three UIKit. It provides:
|
|
* - CloudXR branding and title display
|
|
* - Server connection information and status display
|
|
* - Interactive control buttons (Start Teleop, Reset Teleop, Disconnect)
|
|
* - Responsive button layout with hover effects
|
|
* - Integration with parent component event handlers
|
|
* - Configurable position and rotation in world space for flexible UI placement
|
|
*
|
|
* The UI is positioned in 3D space and designed for VR/AR interaction with
|
|
* visual feedback and clear button labeling. All interactions are passed
|
|
* back to the parent component through callback props.
|
|
*/
|
|
|
|
import { Container, Text, Image } from '@react-three/uikit';
|
|
import { Button } from '@react-three/uikit-default';
|
|
import React from 'react';
|
|
|
|
interface CloudXRUIProps {
|
|
onStartTeleop?: () => void;
|
|
onDisconnect?: () => void;
|
|
onResetTeleop?: () => void;
|
|
serverAddress?: string;
|
|
sessionStatus?: string;
|
|
playLabel?: string;
|
|
playDisabled?: boolean;
|
|
countdownSeconds?: number;
|
|
onCountdownIncrease?: () => void;
|
|
onCountdownDecrease?: () => void;
|
|
countdownDisabled?: boolean;
|
|
position?: [number, number, number];
|
|
rotation?: [number, number, number];
|
|
}
|
|
|
|
export default function CloudXR3DUI({
|
|
onStartTeleop,
|
|
onDisconnect,
|
|
onResetTeleop,
|
|
serverAddress = '127.0.0.1',
|
|
sessionStatus = 'Disconnected',
|
|
playLabel = 'Play',
|
|
playDisabled = false,
|
|
countdownSeconds,
|
|
onCountdownIncrease,
|
|
onCountdownDecrease,
|
|
countdownDisabled = false,
|
|
position = [1.8, 1.75, -1.3],
|
|
rotation = [0, -0.3, 0],
|
|
}: CloudXRUIProps) {
|
|
return (
|
|
<group position={position} rotation={rotation}>
|
|
<Container
|
|
pixelSize={0.001}
|
|
width={1920}
|
|
height={1584}
|
|
alignItems="center"
|
|
justifyContent="center"
|
|
pointerEvents="auto"
|
|
padding={40}
|
|
sizeX={3}
|
|
sizeY={2.475}
|
|
>
|
|
<Container
|
|
width={1600}
|
|
height={900}
|
|
backgroundColor="rgba(40, 40, 40, 0.85)"
|
|
borderRadius={20}
|
|
padding={60}
|
|
paddingBottom={80}
|
|
alignItems="center"
|
|
justifyContent="center"
|
|
flexDirection="column"
|
|
gap={36}
|
|
>
|
|
{/* Title */}
|
|
<Text fontSize={96} fontWeight="bold" color="white" textAlign="center">
|
|
Controls
|
|
</Text>
|
|
|
|
{/* Server Info */}
|
|
<Text fontSize={48} color="white" textAlign="center" marginBottom={24}>
|
|
Server address: {serverAddress}
|
|
</Text>
|
|
<Text fontSize={48} color="white" textAlign="center" marginBottom={48}>
|
|
Session status: {sessionStatus}
|
|
</Text>
|
|
|
|
{/* Countdown Config Row */}
|
|
<Container flexDirection="row" gap={24} alignItems="center" justifyContent="center">
|
|
<Text fontSize={40} color="white">
|
|
Countdown
|
|
</Text>
|
|
<Button
|
|
onClick={onCountdownDecrease}
|
|
variant="default"
|
|
width={105}
|
|
height={105}
|
|
borderRadius={52.5}
|
|
backgroundColor="rgba(220, 220, 220, 0.9)"
|
|
disabled={countdownDisabled}
|
|
>
|
|
<Text fontSize={48} color="black" fontWeight="bold">
|
|
-
|
|
</Text>
|
|
</Button>
|
|
<Container
|
|
width={180}
|
|
height={105}
|
|
alignItems="center"
|
|
justifyContent="center"
|
|
backgroundColor="rgba(255,255,255,0.9)"
|
|
borderRadius={12}
|
|
>
|
|
<Text fontSize={56} color="black">
|
|
{countdownSeconds}s
|
|
</Text>
|
|
</Container>
|
|
<Button
|
|
onClick={onCountdownIncrease}
|
|
variant="default"
|
|
width={105}
|
|
height={105}
|
|
borderRadius={52.5}
|
|
backgroundColor="rgba(220, 220, 220, 0.9)"
|
|
disabled={countdownDisabled}
|
|
>
|
|
<Text fontSize={48} color="black" fontWeight="bold">
|
|
+
|
|
</Text>
|
|
</Button>
|
|
</Container>
|
|
|
|
{/* Button Grid */}
|
|
<Container
|
|
flexDirection="column"
|
|
gap={60}
|
|
alignItems="center"
|
|
justifyContent="center"
|
|
width="100%"
|
|
>
|
|
{/* Start/reset row*/}
|
|
<Container flexDirection="row" gap={60} justifyContent="center">
|
|
<Button
|
|
onClick={onStartTeleop}
|
|
variant="default"
|
|
width={480}
|
|
height={120}
|
|
borderRadius={40}
|
|
backgroundColor="rgba(220, 220, 220, 0.9)"
|
|
hover={{
|
|
backgroundColor: 'rgba(100, 150, 255, 1)',
|
|
borderColor: 'white',
|
|
borderWidth: 2,
|
|
}}
|
|
disabled={playDisabled}
|
|
>
|
|
<Container flexDirection="row" alignItems="center" gap={12}>
|
|
{playLabel === 'Play' && <Image src="./play-circle.svg" width={60} height={60} />}
|
|
<Text fontSize={48} color="black" fontWeight="medium">
|
|
{playLabel}
|
|
</Text>
|
|
</Container>
|
|
</Button>
|
|
|
|
<Button
|
|
onClick={onResetTeleop}
|
|
variant="default"
|
|
width={480}
|
|
height={120}
|
|
borderRadius={40}
|
|
backgroundColor="rgba(220, 220, 220, 0.9)"
|
|
hover={{
|
|
backgroundColor: 'rgba(100, 150, 255, 1)',
|
|
borderColor: 'white',
|
|
borderWidth: 2,
|
|
}}
|
|
>
|
|
<Container flexDirection="row" alignItems="center" gap={12}>
|
|
<Image src="./arrow-uturn-left.svg" width={60} height={60} />
|
|
<Text fontSize={48} color="black" fontWeight="medium">
|
|
Reset
|
|
</Text>
|
|
</Container>
|
|
</Button>
|
|
</Container>
|
|
|
|
{/* Bottom Row */}
|
|
<Container flexDirection="row" justifyContent="center">
|
|
<Button
|
|
onClick={onDisconnect}
|
|
variant="destructive"
|
|
width={330}
|
|
height={105}
|
|
borderRadius={35}
|
|
backgroundColor="rgba(255, 150, 150, 0.9)"
|
|
hover={{
|
|
backgroundColor: 'rgba(255, 50, 50, 1)',
|
|
borderColor: 'white',
|
|
borderWidth: 2,
|
|
}}
|
|
>
|
|
<Container flexDirection="row" alignItems="center" gap={12}>
|
|
<Image src="./arrow-left-start-on-rectangle.svg" width={60} height={60} />
|
|
<Text fontSize={40} color="black" fontWeight="medium">
|
|
Disconnect
|
|
</Text>
|
|
</Container>
|
|
</Button>
|
|
</Container>
|
|
</Container>
|
|
</Container>
|
|
</Container>
|
|
</group>
|
|
);
|
|
}
|