diff --git a/.github/workflows/unbound_deps_tests.yml b/.github/workflows/unbound_deps_tests.yml new file mode 100644 index 00000000..902074a8 --- /dev/null +++ b/.github/workflows/unbound_deps_tests.yml @@ -0,0 +1,183 @@ +# Copyright 2025 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This workflow handles full testing with unboud dependencies versions. +name: Unbound Dependency Tests + +on: + # Allows running this workflow manually from the Actions tab + workflow_dispatch: + + # Run on the 1st and 15th of every month at 09:00 UTC + schedule: + - cron: '0 2 1,15 * *' + +permissions: + contents: read + +# Sets up the environment variables +env: + UV_VERSION: "0.8.0" + PYTHON_VERSION: "3.10" + DOCKER_IMAGE_NAME: huggingface/lerobot-gpu:unbound + +# Ensures that only the latest action is built, canceling older runs. +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + + # This job runs the E2E tests + pytest with all unbound extras + full-tests: + name: Full Unbound Tests + runs-on: ubuntu-latest + env: + MUJOCO_GL: egl + steps: + - uses: actions/checkout@v4 + with: + lfs: true + persist-credentials: false + + - name: Install apt dependencies + run: | + sudo apt-get update && sudo apt-get install -y build-essential \ + git curl libglib2.0-0 libegl1-mesa-dev ffmpeg libusb-1.0-0-dev \ + speech-dispatcher libgeos-dev portaudio19-dev + + - name: Setup uv and Python + uses: astral-sh/setup-uv@v6 # zizmor: ignore[unpinned-uses] + with: + enable-cache: true + version: ${{ env.UV_VERSION }} + python-version: ${{ env.PYTHON_VERSION }} + + - name: Unbound dependencies + run: | + sed -i 's/,[[:space:]]*<[0-9\.]*//g' pyproject.toml + echo "Dependencies unbound:" && cat pyproject.toml + + - name: Install lerobot with all extras + run: uv sync --all-extras + + - name: Run pytest (all extras) + run: uv run pytest tests -vv + + - name: Run end-to-end tests + run: uv run make test-end-to-end + + # This job builds a GPU enabled image for testing + build-and-push-docker: + name: Build and Push Docker + runs-on: + group: aws-general-8-plus + outputs: + image_tag: ${{ env.DOCKER_IMAGE_NAME }} + env: + GITHUB_REF: ${{ github.ref }} + steps: + - name: Install Git LFS + run: | + sudo apt-get update + sudo apt-get install git-lfs + git lfs install + - uses: actions/checkout@v4 + with: + lfs: true + persist-credentials: false + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 # zizmor: ignore[unpinned-uses] + with: + cache-binary: false + - name: Login to Docker Hub + uses: docker/login-action@v3 # zizmor: ignore[unpinned-uses] + with: + username: ${{ secrets.DOCKERHUB_LEROBOT_USERNAME }} + password: ${{ secrets.DOCKERHUB_LEROBOT_PASSWORD }} + - name: Build and push Docker image + uses: docker/build-push-action@v6 # zizmor: ignore[unpinned-uses] + with: + context: . + file: ./docker/Dockerfile.internal + push: true + tags: ${{ env.DOCKER_IMAGE_NAME }} + build-args: | + UNBOUND_DEPS=true + + # This job runs pytest with all unbound extras in a GPU enabled host + # It runs everytime a test image is created + gpu-tests: + name: GPU Unbound Tests + needs: [build-and-push-docker] + runs-on: + group: aws-g6-4xlarge-plus + env: + HF_HOME: /home/user_lerobot/.cache/huggingface + HF_LEROBOT_HOME: /home/user_lerobot/.cache/huggingface/lerobot + TORCH_HOME: /home/user_lerobot/.cache/torch + TRITON_CACHE_DIR: /home/user_lerobot/.cache/triton + container: + image: ${{ needs.build-and-push-docker.outputs.image_tag }} # zizmor: ignore[unpinned-images] + options: --gpus all --shm-size "16gb" + credentials: + username: ${{ secrets.DOCKERHUB_LEROBOT_USERNAME }} + password: ${{ secrets.DOCKERHUB_LEROBOT_PASSWORD }} + defaults: + run: + shell: bash + working-directory: /lerobot + steps: + - name: Run pytest on GPU + run: pytest tests -vv + - name: Run end-to-end tests + run: make test-end-to-end + + # This job deletes the test image recently created + # It runs everytime after the gpu-tests have finished + delete-unbound-image: + name: Delete Unbound Image + needs: [gpu-tests, build-and-push-docker] + if: always() && needs.build-and-push-docker.result == 'success' + runs-on: ubuntu-latest + steps: + - name: Get Docker Hub Token and Delete Image + # zizmor: ignore[template-injection] + run: | + IMAGE_NAME=$(echo "${{ needs.build-and-push-docker.outputs.image_tag }}" | cut -d':' -f1) + IMAGE_TAG=$(echo "${{ needs.build-and-push-docker.outputs.image_tag }}" | cut -d':' -f2) + + echo "Attempting to delete image: $IMAGE_NAME:$IMAGE_TAG" + + TOKEN=$(curl -s -H "Content-Type: application/json" \ + -X POST \ + -d '{"username": "${{ secrets.DOCKERHUB_LEROBOT_USERNAME }}", "password": "${{ secrets.DOCKERHUB_LEROBOT_PASSWORD }}"}' \ + https://hub.docker.com/v2/users/login/ | jq -r .token) + + if [ "$TOKEN" == "null" ] || [ -z "$TOKEN" ]; then + echo "::error::Failed to get Docker Hub token." + exit 1 + fi + + HTTP_RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" \ + -H "Authorization: JWT ${TOKEN}" \ + -X DELETE \ + https://hub.docker.com/v2/repositories/${IMAGE_NAME}/tags/${IMAGE_TAG}/) + + if [ "$HTTP_RESPONSE" -eq 204 ]; then + echo "Successfully deleted Docker image tag: $IMAGE_NAME:$IMAGE_TAG" + else + echo "::error::Failed to delete Docker image. HTTP status: $HTTP_RESPONSE" + exit 1 + fi diff --git a/docker/Dockerfile.internal b/docker/Dockerfile.internal index 52becb83..2616cd06 100644 --- a/docker/Dockerfile.internal +++ b/docker/Dockerfile.internal @@ -75,6 +75,14 @@ RUN uv venv --python python${PYTHON_VERSION} # Install Python dependencies for caching COPY --chown=user_lerobot:user_lerobot pyproject.toml README.md MANIFEST.in ./ COPY --chown=user_lerobot:user_lerobot src/ src/ + +ARG UNBOUND_DEPS=false + +RUN if [ "$UNBOUND_DEPS" = "true" ]; then \ + sed -i 's/,[[:space:]]*<[0-9\.]*//g' pyproject.toml; \ + echo "Dependencies unbound:" && cat pyproject.toml; \ + fi + RUN uv pip install --no-cache ".[all]" # Copy the rest of the application source code diff --git a/docker/Dockerfile.user b/docker/Dockerfile.user index 59fd3e0b..c1b28445 100644 --- a/docker/Dockerfile.user +++ b/docker/Dockerfile.user @@ -61,6 +61,14 @@ RUN uv venv # Install Python dependencies for caching COPY --chown=user_lerobot:user_lerobot pyproject.toml README.md MANIFEST.in ./ COPY --chown=user_lerobot:user_lerobot src/ src/ + +ARG UNBOUND_DEPS=false + +RUN if [ "$UNBOUND_DEPS" = "true" ]; then \ + sed -i 's/,[[:space:]]*<[0-9\.]*//g' pyproject.toml; \ + echo "Dependencies unbound:" && cat pyproject.toml; \ + fi + RUN uv pip install --no-cache ".[all]" # Copy the rest of the application code diff --git a/pyproject.toml b/pyproject.toml index f350fac0..c67b481f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,20 +59,20 @@ keywords = ["lerobot", "huggingface", "robotics", "machine learning", "artifici dependencies = [ # Hugging Face dependencies - "datasets>=4.0.0", - "diffusers>=0.27.2", - "huggingface-hub[hf-transfer,cli]>=0.34.2", + "datasets>=4.0.0,<4.2.0", + "diffusers>=0.27.2,<0.36.0", + "huggingface-hub[hf-transfer,cli]>=0.34.2,<0.36.0", # Core dependencies - "cmake>=3.29.0.1", - "einops>=0.8.0", - "opencv-python-headless>=4.9.0", - "av>=14.2.0", - "jsonlines>=4.0.0", - "packaging>=24.2", - "pynput>=1.7.7", - "pyserial>=3.5", - "wandb>=0.20.0", + "cmake>=3.29.0.1,<4.2.0", + "einops>=0.8.0,<0.9.0", + "opencv-python-headless>=4.9.0,<4.13.0", + "av>=14.2.0,<16.0.0", + "jsonlines>=4.0.0,<5.0.0", + "packaging>=24.2,<26.0", + "pynput>=1.7.7,<1.9.0", + "pyserial>=3.5,<4.0", + "wandb>=0.20.0,<0.23.0", "torch>=2.2.1,<2.8.0", # TODO: Bumb dependency "torchcodec>=0.2.1,<0.6.0; sys_platform != 'win32' and (sys_platform != 'linux' or (platform_machine != 'aarch64' and platform_machine != 'arm64' and platform_machine != 'armv7l')) and (sys_platform != 'darwin' or platform_machine != 'x86_64')", # TODO: Bumb dependency @@ -92,26 +92,26 @@ dependencies = [ [project.optional-dependencies] # Common -pygame-dep = ["pygame>=2.5.1"] -placo-dep = ["placo>=0.9.6"] -transformers-dep = ["transformers>=4.53.0"] +pygame-dep = ["pygame>=2.5.1,<2.7.0"] +placo-dep = ["placo>=0.9.6,<0.10.0"] +transformers-dep = ["transformers>=4.53.0,<5.0.0"] grpcio-dep = ["grpcio==1.73.1", "protobuf==6.31.0"] # Motors -feetech = ["feetech-servo-sdk>=1.0.0"] -dynamixel = ["dynamixel-sdk>=3.7.31"] +feetech = ["feetech-servo-sdk>=1.0.0,<2.0.0"] +dynamixel = ["dynamixel-sdk>=3.7.31,<3.9.0"] # Robots -gamepad = ["lerobot[pygame-dep]", "hidapi>=0.14.0"] +gamepad = ["lerobot[pygame-dep]", "hidapi>=0.14.0,<0.15.0"] hopejr = ["lerobot[feetech]", "lerobot[pygame-dep]"] -lekiwi = ["lerobot[feetech]", "pyzmq>=26.2.1"] -reachy2 = ["reachy2_sdk>=1.0.14"] +lekiwi = ["lerobot[feetech]", "pyzmq>=26.2.1,<28.0.0"] +reachy2 = ["reachy2_sdk>=1.0.14,<1.1.0"] kinematics = ["lerobot[placo-dep]"] intelrealsense = [ - "pyrealsense2>=2.55.1.6486 ; sys_platform != 'darwin'", - "pyrealsense2-macosx>=2.54 ; sys_platform == 'darwin'", + "pyrealsense2>=2.55.1.6486,<2.57.0 ; sys_platform != 'darwin'", + "pyrealsense2-macosx>=2.54,<2.55.0 ; sys_platform == 'darwin'", ] -phone = ["hebi-py>=2.8.0", "teleop>=0.1.0"] +phone = ["hebi-py>=2.8.0,<2.12.0", "teleop>=0.1.0,<0.2.0"] # stretch = [ # "hello-robot-stretch-body>=0.7.27 ; sys_platform == 'linux'", # "pyrender @ git+https://github.com/mmatl/pyrender.git ; sys_platform == 'linux'", @@ -120,21 +120,21 @@ phone = ["hebi-py>=2.8.0", "teleop>=0.1.0"] # Policies pi = ["transformers @ git+https://github.com/huggingface/transformers.git@fix/lerobot_openpi"] -smolvla = ["lerobot[transformers-dep]", "num2words>=0.5.14", "accelerate>=1.7.0", "safetensors>=0.4.3"] -hilserl = ["lerobot[transformers-dep]", "gym-hil>=0.1.11", "lerobot[grpcio-dep]", "lerobot[placo-dep]"] +smolvla = ["lerobot[transformers-dep]", "num2words>=0.5.14,<0.6.0", "accelerate>=1.7.0,<2.0.0", "safetensors>=0.4.3,<1.0.0"] +hilserl = ["lerobot[transformers-dep]", "gym-hil>=0.1.11,<0.2.0", "lerobot[grpcio-dep]", "lerobot[placo-dep]"] # Features -async = ["lerobot[grpcio-dep]", "matplotlib>=3.10.3"] +async = ["lerobot[grpcio-dep]", "matplotlib>=3.10.3,<4.0.0"] # Development -dev = ["pre-commit>=3.7.0", "debugpy>=1.8.1", "lerobot[grpcio-dep]", "grpcio-tools==1.73.1"] -test = ["pytest>=8.1.0", "pytest-timeout>=2.4.0", "pytest-cov>=5.0.0", "mock-serial>=0.0.1 ; sys_platform != 'win32'"] -video_benchmark = ["scikit-image>=0.23.2", "pandas>=2.2.2"] +dev = ["pre-commit>=3.7.0,<5.0.0", "debugpy>=1.8.1,<1.9.0", "lerobot[grpcio-dep]", "grpcio-tools==1.73.1"] +test = ["pytest>=8.1.0,<9.0.0", "pytest-timeout>=2.4.0,<3.0.0", "pytest-cov>=5.0.0,<8.0.0", "mock-serial>=0.0.1,<0.1.0 ; sys_platform != 'win32'"] +video_benchmark = ["scikit-image>=0.23.2,<0.26.0", "pandas>=2.2.2,<2.4.0"] # Simulation -aloha = ["gym-aloha>=0.1.1"] -pusht = ["gym-pusht>=0.1.5", "pymunk>=6.6.0,<7.0.0"] # TODO: Fix pymunk version in gym-pusht instead -xarm = ["gym-xarm>=0.1.1"] +aloha = ["gym-aloha>=0.1.1,<0.2.0"] +pusht = ["gym-pusht>=0.1.5,<0.2.0", "pymunk>=6.6.0,<7.0.0"] # TODO: Fix pymunk version in gym-pusht instead +xarm = ["gym-xarm>=0.1.1,<0.2.0"] libero = ["lerobot[transformers-dep]", "libero @ git+https://github.com/huggingface/lerobot-libero.git@main#egg=libero"]