From 89f59b070368f789a8b3aa558d9f0c88796cc324 Mon Sep 17 00:00:00 2001 From: Steven Palma Date: Sat, 19 Jul 2025 20:09:12 +0200 Subject: [PATCH] refactor(ci): workflows improvements (#1535) * refactor(ci): consolidate documentation workflows * refactor(ci): improve quality workflow * refactor(ci): edit security workflow * refactor(ci): improve testing workflows * fix(ci): several fixes * chore(ci): renaming + permissions * chore(ci): remove now unused dockerfiles * chore(docs): add license headers to dockerfiles * chore(ci): add cache-binary false to setup-buildx actions * fix(ci): several fixes * dgb(ci): explicit env in the workflow * fix(ci): more explicit env vars for writing * fix(ci): nightly gpu tag --- .github/workflows/build-docker-images.yml | 135 ---------- .github/workflows/build_documentation.yml | 23 -- .github/workflows/build_pr_documentation.yml | 19 -- .github/workflows/documentation-upload-pr.yml | 40 +++ .github/workflows/documentation.yml | 70 +++++ .github/workflows/nightly-tests.yml | 93 ------- .github/workflows/nightly.yml | 160 ++++++++++++ .github/workflows/quality.yml | 64 ++--- .github/workflows/security.yml | 54 ++++ .github/workflows/test-docker-build.yml | 84 ------ .github/workflows/test.yml | 150 ----------- .github/workflows/tests.yml | 243 ++++++++++++++++++ .github/workflows/trufflehog.yml | 35 --- .github/workflows/upload_pr_documentation.yml | 16 -- docker/Dockerfile.internal | 44 +++- docker/Dockerfile.user | 36 ++- docker/lerobot-cpu/Dockerfile | 29 --- docker/lerobot-gpu-dev/Dockerfile | 68 ----- docker/lerobot-gpu/Dockerfile | 24 -- 19 files changed, 654 insertions(+), 733 deletions(-) delete mode 100644 .github/workflows/build-docker-images.yml delete mode 100644 .github/workflows/build_documentation.yml delete mode 100644 .github/workflows/build_pr_documentation.yml create mode 100644 .github/workflows/documentation-upload-pr.yml create mode 100644 .github/workflows/documentation.yml delete mode 100644 .github/workflows/nightly-tests.yml create mode 100644 .github/workflows/nightly.yml create mode 100644 .github/workflows/security.yml delete mode 100644 .github/workflows/test-docker-build.yml delete mode 100644 .github/workflows/test.yml create mode 100644 .github/workflows/tests.yml delete mode 100644 .github/workflows/trufflehog.yml delete mode 100644 .github/workflows/upload_pr_documentation.yml delete mode 100644 docker/lerobot-cpu/Dockerfile delete mode 100644 docker/lerobot-gpu-dev/Dockerfile delete mode 100644 docker/lerobot-gpu/Dockerfile diff --git a/.github/workflows/build-docker-images.yml b/.github/workflows/build-docker-images.yml deleted file mode 100644 index 20974b85..00000000 --- a/.github/workflows/build-docker-images.yml +++ /dev/null @@ -1,135 +0,0 @@ -# Copyright 2024 The HuggingFace Inc. team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Inspired by -# https://github.com/huggingface/peft/blob/main/.github/workflows/build_docker_images.yml -name: Builds - -on: - workflow_dispatch: - workflow_call: - schedule: - - cron: "0 1 * * *" - -permissions: {} - -env: - PYTHON_VERSION: "3.10" - -jobs: - latest-cpu: - name: CPU - runs-on: - group: aws-general-8-plus - steps: - - name: Install Git LFS - run: | - sudo apt-get update - sudo apt-get install git-lfs - git lfs install - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0 - with: - cache-binary: false - - - name: Check out code - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - lfs: true - persist-credentials: false - - - name: Login to DockerHub - uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - - name: Build and Push CPU - uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 # v5.4.0 - with: - context: . - file: ./docker/lerobot-cpu/Dockerfile - push: true - tags: huggingface/lerobot-cpu - build-args: PYTHON_VERSION=${{ env.PYTHON_VERSION }} - - - latest-cuda: - name: GPU - runs-on: - group: aws-general-8-plus - steps: - - name: Install Git LFS - run: | - sudo apt-get update - sudo apt-get install git-lfs - git lfs install - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0 - with: - cache-binary: false - - - name: Check out code - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - lfs: true - persist-credentials: false - - - name: Login to DockerHub - uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - - name: Build and Push GPU - uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 # v5.4.0 - with: - context: . - file: ./docker/lerobot-gpu/Dockerfile - push: true - tags: huggingface/lerobot-gpu - build-args: PYTHON_VERSION=${{ env.PYTHON_VERSION }} - - - latest-cuda-dev: - name: GPU Dev - runs-on: - group: aws-general-8-plus - steps: - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0 - with: - cache-binary: false - - - name: Check out code - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - persist-credentials: false - - - name: Login to DockerHub - uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - - name: Build and Push GPU dev - uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 # v5.4.0 - with: - context: . - file: ./docker/lerobot-gpu-dev/Dockerfile - push: true - tags: huggingface/lerobot-gpu:dev - build-args: PYTHON_VERSION=${{ env.PYTHON_VERSION }} diff --git a/.github/workflows/build_documentation.yml b/.github/workflows/build_documentation.yml deleted file mode 100644 index 884e2e4b..00000000 --- a/.github/workflows/build_documentation.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: Build documentation - -on: - workflow_dispatch: - push: - paths: - - "docs/**" - branches: - - main - - doc-builder* - - v*-release - - -jobs: - build: # zizmor: ignore[excessive-permissions] We follow the same pattern as in Transformers - uses: huggingface/doc-builder/.github/workflows/build_main_documentation.yml@main - with: - commit_sha: ${{ github.sha }} - package: lerobot - additional_args: --not_python_module - secrets: - token: ${{ secrets.HUGGINGFACE_PUSH }} - hf_token: ${{ secrets.HF_DOC_BUILD_PUSH }} diff --git a/.github/workflows/build_pr_documentation.yml b/.github/workflows/build_pr_documentation.yml deleted file mode 100644 index 51bab10d..00000000 --- a/.github/workflows/build_pr_documentation.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: Build PR Documentation - -on: - pull_request: - paths: - - "docs/**" - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - build: # zizmor: ignore[excessive-permissions] We follow the same pattern as in Transformers - uses: huggingface/doc-builder/.github/workflows/build_pr_documentation.yml@main - with: - commit_sha: ${{ github.event.pull_request.head.sha }} - pr_number: ${{ github.event.number }} - package: lerobot - additional_args: --not_python_module diff --git a/.github/workflows/documentation-upload-pr.yml b/.github/workflows/documentation-upload-pr.yml new file mode 100644 index 00000000..22ba11cb --- /dev/null +++ b/.github/workflows/documentation-upload-pr.yml @@ -0,0 +1,40 @@ +# Copyright 2025 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This workflow uploads the documentation preview built for a PR and comments the link on the PR. +name: Documentation PR Upload +permissions: + contents: read + pull-requests: write + +on: + # Triggered by the completion of the main 'Documentation' workflow. + workflow_run: # zizmor: ignore[dangerous-triggers] We follow the same pattern as in Transformers + workflows: ["Documentation"] + types: + - completed + +jobs: + # This job uploads a preview of the documentation for a pull request. + upload_and_comment: + name: Upload Preview and Comment + if: > + github.event.workflow_run.event == 'pull_request' && + github.event.workflow_run.conclusion == 'success' + uses: huggingface/doc-builder/.github/workflows/upload_pr_documentation.yml@main + with: + package_name: lerobot + secrets: + hf_token: ${{ secrets.HF_DOC_BUILD_PUSH }} + comment_bot_token: ${{ secrets.COMMENT_BOT_TOKEN }} diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml new file mode 100644 index 00000000..96005af3 --- /dev/null +++ b/.github/workflows/documentation.yml @@ -0,0 +1,70 @@ +# Copyright 2025 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This workflow handles building documentation for both main branches and PRs. +name: Documentation + +on: + # Allows running this workflow manually from the Actions tab + workflow_dispatch: + + # Triggers the workflow on push events to main for the docs folder + push: + branches: + - main + paths: + - "docs/**" + + # Triggers the workflow on pull request events targeting main for the docs folder + pull_request: + branches: + - main + paths: + - "docs/**" + +# Ensures that only the latest commit for a PR or branch is built, canceling older runs. +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + # This job builds and deploys the official documentation. + build_main_docs: + name: Build Main Docs + if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' + permissions: + contents: read + uses: huggingface/doc-builder/.github/workflows/build_main_documentation.yml@main + with: + commit_sha: ${{ github.sha }} + package: lerobot + additional_args: --not_python_module + secrets: + token: ${{ secrets.HUGGINGFACE_PUSH }} + hf_token: ${{ secrets.HF_DOC_BUILD_PUSH }} + + # This job builds a preview of the documentation for a pull request. + # The result of this job triggers the 'Upload PR Documentation' workflow. + build_pr_docs: + name: Build PR Docs + if: github.event_name == 'pull_request' + permissions: + contents: read + pull-requests: write + uses: huggingface/doc-builder/.github/workflows/build_pr_documentation.yml@main + with: + commit_sha: ${{ github.event.pull_request.head.sha }} + pr_number: ${{ github.event.number }} + package: lerobot + additional_args: --not_python_module diff --git a/.github/workflows/nightly-tests.yml b/.github/workflows/nightly-tests.yml deleted file mode 100644 index 72801691..00000000 --- a/.github/workflows/nightly-tests.yml +++ /dev/null @@ -1,93 +0,0 @@ -# Copyright 2024 The HuggingFace Inc. team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Inspired by -# https://github.com/huggingface/peft/blob/main/.github/workflows/nightly.yml -name: Nightly - -on: - workflow_dispatch: - schedule: - - cron: "0 2 * * *" - -permissions: {} - -# env: - # SLACK_API_TOKEN: ${{ secrets.SLACK_API_TOKEN }} -jobs: - run_all_tests_cpu: - name: CPU - strategy: - fail-fast: false - runs-on: - group: aws-general-8-plus - container: - image: huggingface/lerobot-cpu:latest # zizmor: ignore[unpinned-images] - options: --shm-size "16gb" - credentials: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - defaults: - run: - shell: bash - working-directory: /lerobot - steps: - - name: Tests - run: pytest -v --cov=./src/lerobot --disable-warnings tests - - - name: Tests end-to-end - run: make test-end-to-end - - - run_all_tests_single_gpu: - name: GPU - strategy: - fail-fast: false - runs-on: - group: aws-g6-4xlarge-plus - env: - CUDA_VISIBLE_DEVICES: "0" - TEST_TYPE: "single_gpu" - container: - image: huggingface/lerobot-gpu:latest # zizmor: ignore[unpinned-images] - options: --gpus all --shm-size "16gb" - credentials: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - defaults: - run: - shell: bash - working-directory: /lerobot - steps: - - name: Nvidia-smi - run: nvidia-smi - - - name: Test - run: pytest -v --cov=./src/lerobot --cov-report=xml --disable-warnings tests - # TODO(aliberts): Link with HF Codecov account - # - name: Upload coverage reports to Codecov with GitHub Action - # uses: codecov/codecov-action@v4 - # with: - # files: ./coverage.xml - # verbose: true - - name: Tests end-to-end - env: - DEVICE: cuda - run: make test-end-to-end - - # - name: Generate Report - # if: always() - # run: | - # pip install slack_sdk tabulate - # python scripts/log_reports.py >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml new file mode 100644 index 00000000..149f55c8 --- /dev/null +++ b/.github/workflows/nightly.yml @@ -0,0 +1,160 @@ +# Copyright 2025 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This workflow handles nightly testing & docker images publishing. +name: Nightly +permissions: + contents: read + +on: + # Allows running this workflow manually from the Actions tab + workflow_dispatch: + + # Runs at 02:00 + schedule: + - cron: "0 2 * * *" + +# Sets up the environment variables +env: + UV_VERSION: "0.8.0" + PYTHON_VERSION: "3.10" + DOCKER_IMAGE_NAME_CPU: huggingface/lerobot-gpu:latest + DOCKER_IMAGE_NAME_GPU: huggingface/lerobot-cpu:latest + +# Ensures that only the latest commit is built, canceling older runs. +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + build-docker-cpu-nightly: + # This job builds a CPU image for testing & distribution + name: Build CPU Docker for Nightly + runs-on: + group: aws-general-8-plus + outputs: + image_tag: ${{ env.DOCKER_IMAGE_NAME_CPU }} + steps: + - name: Install Git LFS + run: | + sudo apt-get update + sudo apt-get install git-lfs + git lfs install + - uses: actions/checkout@v4 + with: + lfs: true + persist-credentials: false + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 # zizmor: ignore[unpinned-uses] + with: + cache-binary: false + - name: Login to Docker Hub + uses: docker/login-action@v3 # zizmor: ignore[unpinned-uses] + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + - name: Build and push Docker image CPU + uses: docker/build-push-action@v6 # zizmor: ignore[unpinned-uses] + with: + context: . + file: ./docker/Dockerfile.user + push: true + tags: ${{ env.DOCKER_IMAGE_NAME_CPU }} + + build-docker-gpu-nightly: + # This job builds a GPU image for testing & distribution + name: Build GPU Docker for Nightly + runs-on: + group: aws-general-8-plus + outputs: + image_tag: ${{ env.DOCKER_IMAGE_NAME_GPU }} + steps: + - name: Install Git LFS + run: | + sudo apt-get update + sudo apt-get install git-lfs + git lfs install + - uses: actions/checkout@v4 + with: + lfs: true + persist-credentials: false + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 # zizmor: ignore[unpinned-uses] + with: + cache-binary: false + - name: Login to Docker Hub + uses: docker/login-action@v3 # zizmor: ignore[unpinned-uses] + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + - name: Build and push Docker image GPU + uses: docker/build-push-action@v6 # zizmor: ignore[unpinned-uses] + with: + context: . + file: ./docker/Dockerfile.internal + push: true + tags: ${{ env.DOCKER_IMAGE_NAME_GPU }} + + nightly-cpu-tests: + # This job runs the E2E tests + pytest with all extras in the CPU image + name: Nightly CPU Tests + needs: [build-docker-cpu-nightly] + runs-on: + group: aws-g6-4xlarge-plus + env: + HF_HOME: /home/user_lerobot/.cache/huggingface + HF_LEROBOT_HOME: /home/user_lerobot/.cache/huggingface/lerobot + TORCH_HOME: /home/user_lerobot/.cache/torch + TRITON_CACHE_DIR: /home/user_lerobot/.cache/triton + container: + image: ${{ needs.build-docker-cpu-nightly.outputs.image_tag }} # zizmor: ignore[unpinned-images] + credentials: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + defaults: + run: + shell: bash + working-directory: /lerobot + steps: + - name: Run pytest on CPU + run: pytest tests -vv --maxfail=10 + - name: Run end-to-end tests + run: make test-end-to-end + + nightly-gpu-tests: + # This job runs the E2E tests + pytest with all extras in the GPU image + name: Nightly GPU Tests + needs: [build-docker-gpu-nightly] + runs-on: + group: aws-g6-4xlarge-plus + env: + HF_HOME: /home/user_lerobot/.cache/huggingface + HF_LEROBOT_HOME: /home/user_lerobot/.cache/huggingface/lerobot + TORCH_HOME: /home/user_lerobot/.cache/torch + TRITON_CACHE_DIR: /home/user_lerobot/.cache/triton + container: + image: ${{ needs.build-docker-gpu-nightly.outputs.image_tag }} # zizmor: ignore[unpinned-images] + options: --gpus all --shm-size "16gb" + credentials: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + defaults: + run: + shell: bash + working-directory: /lerobot + steps: + - name: Run pytest on GPU + run: pytest tests -vv --maxfail=10 + - name: Run end-to-end tests + run: make test-end-to-end diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index 1c048c4f..e9f73ed2 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -1,4 +1,4 @@ -# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# Copyright 2025 The HuggingFace Inc. team. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,61 +12,47 @@ # See the License for the specific language governing permissions and # limitations under the License. +# This workflow handles linting, formatting, and static analysis checks for the codebase. name: Quality +permissions: + contents: read on: + # Allows running this workflow manually from the Actions tab workflow_dispatch: - workflow_call: - pull_request: + + # Triggers the workflow on push events to main push: branches: - main -permissions: {} + # Triggers the workflow on pull request events targeting main + pull_request: + branches: + - main -env: - PYTHON_VERSION: "3.10" +# Ensures that only the latest commit for a PR or branch is built, canceling older runs. +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true jobs: - style: - name: Style + # This job runs pre-commit hooks to check code style and formatting. + pre-commit-checks: + name: Run Pre-commit Hooks (Lint, Format & Static Analysis) runs-on: ubuntu-latest steps: - - name: Checkout Repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Checkout code + uses: actions/checkout@v4 with: persist-credentials: false - name: Set up Python - uses: actions/setup-python@7f4fc3e22c37d6ff65e88745f38bd3157c663f7c # v4.9.1 + uses: actions/setup-python@v5 with: - python-version: ${{ env.PYTHON_VERSION }} + python-version: '3.10' - - name: Get Ruff Version from pre-commit-config.yaml - id: get-ruff-version - run: | - RUFF_VERSION=$(awk '/repo: https:\/\/github.com\/astral-sh\/ruff-pre-commit/{flag=1;next}/rev:/{if(flag){print $2;exit}}' .pre-commit-config.yaml) - echo "ruff_version=${RUFF_VERSION}" >> $GITHUB_OUTPUT - - - name: Install Ruff - env: - RUFF_VERSION: ${{ steps.get-ruff-version.outputs.ruff_version }} - run: python -m pip install "ruff==${RUFF_VERSION}" - - - name: Ruff check - run: ruff check --output-format=github - - - name: Ruff format - run: ruff format --diff - - typos: - name: Typos - runs-on: ubuntu-latest - steps: - - name: Checkout Repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Run pre-commit hooks + uses: pre-commit/action@v3.0.1 # zizmor: ignore[unpinned-uses] with: - persist-credentials: false - - - name: typos-action - uses: crate-ci/typos@db35ee91e80fbb447f33b0e5fbddb24d2a1a884f # v1.29.10 + extra_args: --all-files --show-diff-on-failure --color=always diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml new file mode 100644 index 00000000..04497307 --- /dev/null +++ b/.github/workflows/security.yml @@ -0,0 +1,54 @@ +# Copyright 2025 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This workflow handles secret scanning using TruffleHog to detect sensitive information in the codebase. +name: Security +permissions: + contents: read + +on: + # Allows running this workflow manually from the Actions tab + workflow_dispatch: + + # Triggers the workflow on push events to main + push: + branches: + - main + + # Triggers the workflow on pull request events targeting main + pull_request: + branches: + - main + +# Ensures that only the latest commit for a PR or branch is built, canceling older runs. +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + # This job runs TruffleHog to scan the full history of the repository for secrets. + trufflehog: + name: Secret Leaks Scan + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 # zizmor: ignore[unpinned-uses] + with: + fetch-depth: 0 + persist-credentials: false + + - name: Secret Scanning + uses: trufflesecurity/trufflehog@v3.90.0 # zizmor: ignore[unpinned-uses] + with: + extra_args: --only-verified diff --git a/.github/workflows/test-docker-build.yml b/.github/workflows/test-docker-build.yml deleted file mode 100644 index c3381341..00000000 --- a/.github/workflows/test-docker-build.yml +++ /dev/null @@ -1,84 +0,0 @@ -# Copyright 2024 The HuggingFace Inc. team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Inspired by -# https://github.com/huggingface/peft/blob/main/.github/workflows/test-docker-build.yml -name: Test Dockerfiles - -on: - pull_request: - paths: - # Run only when DockerFile files are modified - - "docker/lerobot-cpu/**" - - "docker/lerobot-gpu/**" - - "docker/lerobot-gpu-dev/**" - -permissions: {} - -env: - PYTHON_VERSION: "3.10" - -jobs: - get_changed_files: - name: Detect modified Dockerfiles - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} - steps: - - name: Check out code - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - persist-credentials: false - - - name: Get changed files - id: changed-files - uses: tj-actions/changed-files@3f54ebb830831fc121d3263c1857cfbdc310cdb9 #v42 - with: - files: docker/** - json: "true" - - - name: Run step if only the files listed above change # zizmor: ignore[template-injection] - if: steps.changed-files.outputs.any_changed == 'true' - id: set-matrix - run: | - echo "matrix=${{ steps.changed-files.outputs.all_changed_files}}" >> $GITHUB_OUTPUT - - build_modified_dockerfiles: - name: Build modified Docker images - needs: get_changed_files - runs-on: - group: aws-general-8-plus - if: needs.get_changed_files.outputs.matrix != '' - strategy: - fail-fast: false - matrix: - docker-file: ${{ fromJson(needs.get_changed_files.outputs.matrix) }} - steps: - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0 - with: - cache-binary: false - - - name: Check out code - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - persist-credentials: false - - - name: Build Docker image - uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 # v5.4.0 - with: - file: ${{ matrix.docker-file }} - context: . - push: False - build-args: PYTHON_VERSION=${{ env.PYTHON_VERSION }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index d6ea1d40..00000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,150 +0,0 @@ -# Copyright 2024 The HuggingFace Inc. team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: Tests - -on: - pull_request: - paths: - - "src/**" - - "tests/**" - - "examples/**" - - ".github/**" - - "pyproject.toml" - - ".pre-commit-config.yaml" - - "Makefile" - - ".cache/**" - push: - branches: - - main - paths: - - "src/**" - - "tests/**" - - "examples/**" - - ".github/**" - - "pyproject.toml" - - ".pre-commit-config.yaml" - - "Makefile" - - ".cache/**" - -permissions: {} - -env: - UV_VERSION: "0.6.0" - -jobs: - pytest: - name: Pytest - runs-on: ubuntu-latest - env: - MUJOCO_GL: egl - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - lfs: true # Ensure LFS files are pulled - persist-credentials: false - - - name: Install apt dependencies - # portaudio19-dev is needed to install pyaudio - run: | - sudo apt-get update && \ - sudo apt-get install -y libegl1-mesa-dev ffmpeg portaudio19-dev - - - name: Install uv and python - uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5.4.2 - with: - enable-cache: true - version: ${{ env.UV_VERSION }} - python-version: "3.10" - - - name: Install lerobot (all extras) - run: uv sync --all-extras - - - name: Test with pytest - run: | - uv run pytest tests -v --cov=./src/lerobot --durations=0 \ - -W ignore::DeprecationWarning:imageio_ffmpeg._utils:7 \ - -W ignore::UserWarning:torch.utils.data.dataloader:558 \ - -W ignore::UserWarning:gymnasium.utils.env_checker:247 \ - && rm -rf tests/outputs outputs - - pytest-minimal: - name: Pytest (minimal install) - runs-on: ubuntu-latest - env: - MUJOCO_GL: egl - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - lfs: true # Ensure LFS files are pulled - persist-credentials: false - - - name: Install apt dependencies - run: sudo apt-get update && sudo apt-get install -y ffmpeg - - - name: Install uv and python - uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5.4.2 - with: - enable-cache: true - version: ${{ env.UV_VERSION }} - python-version: "3.10" - - - name: Install lerobot - run: uv sync --extra "test" - - - name: Test with pytest - run: | - uv run pytest tests -v --cov=./src/lerobot --durations=0 \ - -W ignore::DeprecationWarning:imageio_ffmpeg._utils:7 \ - -W ignore::UserWarning:torch.utils.data.dataloader:558 \ - -W ignore::UserWarning:gymnasium.utils.env_checker:247 \ - && rm -rf tests/outputs outputs - - end-to-end: - name: End-to-end - runs-on: ubuntu-latest - env: - MUJOCO_GL: egl - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - lfs: true # Ensure LFS files are pulled - persist-credentials: false - - - name: Install apt dependencies - # portaudio19-dev is needed to install pyaudio - run: | - sudo apt-get update && \ - sudo apt-get install -y libegl1-mesa-dev ffmpeg portaudio19-dev - - - name: Install uv and python - uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5.4.2 - with: - enable-cache: true - version: ${{ env.UV_VERSION }} - python-version: "3.10" - - - name: Install lerobot (all extras) - run: | - uv venv - uv sync --all-extras - - - name: venv - run: | - echo "PYTHON_PATH=${{ github.workspace }}/.venv/bin/python" >> $GITHUB_ENV - - - name: Test end-to-end - run: | - make test-end-to-end \ - && rm -rf outputs diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 00000000..443c849d --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,243 @@ +# Copyright 2025 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This workflow handles testing. +name: Tests + +on: + # Allows running this workflow manually from the Actions tab + workflow_dispatch: + + pull_request: + branches: + - main + paths: + - "src/**" + - "tests/**" + - ".github/workflows/**" + - "pyproject.toml" + - "Makefile" + pull_request_review: + types: [submitted] + push: + branches: + - main + paths: + - "src/**" + - "tests/**" + - ".github/workflows/**" + - "pyproject.toml" + - "Makefile" + +permissions: + contents: read + +# Sets up the environment variables +env: + UV_VERSION: "0.8.0" + PYTHON_VERSION: "3.10" + DOCKER_IMAGE_NAME: huggingface/lerobot-gpu + +# Ensures that only the latest commit for a PR or branch is built, canceling older runs. +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + # This job runs pytests with the default dependencies. + # It runs everytime we commit to a PR or push to main + fast-pytest-tests: + name: Fast Pytest Tests + if: | + github.event_name == 'pull_request' || + github.event_name == 'push' || + github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + env: + MUJOCO_GL: egl + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + lfs: true + + # TODO(Steven): Evaluate the need of these dependencies + - name: Install apt dependencies + run: | + sudo apt-get update && sudo apt-get install -y build-essential git \ + curl libglib2.0-0 libegl1-mesa-dev ffmpeg \ + libusb-1.0-0-dev speech-dispatcher libgeos-dev portaudio19-dev + + - name: Setup uv and Python + uses: astral-sh/setup-uv@v6 # zizmor: ignore[unpinned-uses] + with: + enable-cache: true + version: ${{ env.UV_VERSION }} + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install lerobot with test extras + run: uv sync --extra "test" + + - name: Run pytest + run: uv run pytest tests -vv --maxfail=10 + + full-tests-gate: + # This job evaluates the need to run the full tests suite. + name: Full Tests Gate + runs-on: ubuntu-latest + if: | + (github.event_name == 'pull_request_review' && github.event.review.state == 'approved') || + github.event_name == 'push' || + github.event_name == 'workflow_dispatch' + steps: + - name: Gate check + run: echo "Full tests will run." + + full-tests: + # This job runs the E2E tests + pytest with all extras + # It runs everytime a PR is approved or a push to main + name: Full Tests + needs: full-tests-gate + runs-on: ubuntu-latest + env: + MUJOCO_GL: egl + steps: + - uses: actions/checkout@v4 + with: + lfs: true + persist-credentials: false + + - name: Install apt dependencies + run: | + sudo apt-get update && sudo apt-get install -y build-essential \ + git curl libglib2.0-0 libegl1-mesa-dev ffmpeg libusb-1.0-0-dev \ + speech-dispatcher libgeos-dev portaudio19-dev + + - name: Setup uv and Python + uses: astral-sh/setup-uv@v6 # zizmor: ignore[unpinned-uses] + with: + enable-cache: true + version: ${{ env.UV_VERSION }} + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install lerobot with all extras + run: uv sync --all-extras + + - name: Run pytest (all extras) + run: uv run pytest tests -vv --maxfail=10 + + - name: Run end-to-end tests + run: uv run make test-end-to-end + + build-and-push-docker: + # This job builds a GPU enabled image for testing + # It runs everytime a PR is approved or a push to main + name: Build and Push Docker + needs: full-tests-gate + runs-on: + group: aws-general-8-plus + outputs: + image_tag: ${{ env.DOCKER_IMAGE_NAME }}:pr-${{ github.event.pull_request.number }} + steps: + - name: Install Git LFS + run: | + sudo apt-get update + sudo apt-get install git-lfs + git lfs install + - uses: actions/checkout@v4 + with: + lfs: true + persist-credentials: false + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 # zizmor: ignore[unpinned-uses] + with: + cache-binary: false + - name: Login to Docker Hub + uses: docker/login-action@v3 # zizmor: ignore[unpinned-uses] + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + - name: Build and push Docker image + uses: docker/build-push-action@v6 # zizmor: ignore[unpinned-uses] + with: + context: . + file: ./docker/Dockerfile.internal + push: true + tags: ${{ env.DOCKER_IMAGE_NAME }}:pr-${{ github.event.pull_request.number }} + + gpu-tests: + # This job runs pytest with all extras in a GPU enabled host + # It runs everytime a test image is created + name: GPU Tests + needs: [build-and-push-docker] + runs-on: + group: aws-g6-4xlarge-plus + env: + HF_HOME: /home/user_lerobot/.cache/huggingface + HF_LEROBOT_HOME: /home/user_lerobot/.cache/huggingface/lerobot + TORCH_HOME: /home/user_lerobot/.cache/torch + TRITON_CACHE_DIR: /home/user_lerobot/.cache/triton + container: + image: ${{ needs.build-and-push-docker.outputs.image_tag }} # zizmor: ignore[unpinned-images] + options: --gpus all --shm-size "16gb" + credentials: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + defaults: + run: + shell: bash + working-directory: /lerobot + steps: + - name: Run pytest on GPU + run: pytest tests -vv --maxfail=10 + - name: Run end-to-end tests + run: make test-end-to-end + + delete-pr-image: + # This job deletes the test image recently created + # It runs everytime after the gpu-tests have finished + name: Delete PR Image + needs: [gpu-tests, build-and-push-docker] + if: always() && github.event.review.state == 'approved' && needs.build-and-push-docker.result == 'success' + runs-on: ubuntu-latest + steps: + - name: Get Docker Hub Token and Delete Image + # zizmor: ignore[template-injection] + run: | + IMAGE_NAME=$(echo "${{ needs.build-and-push-docker.outputs.image_tag }}" | cut -d':' -f1) + IMAGE_TAG=$(echo "${{ needs.build-and-push-docker.outputs.image_tag }}" | cut -d':' -f2) + + echo "Attempting to delete image: $IMAGE_NAME:$IMAGE_TAG" + + TOKEN=$(curl -s -H "Content-Type: application/json" \ + -X POST \ + -d '{"username": "${{ secrets.DOCKERHUB_USERNAME }}", "password": "${{ secrets.DOCKERHUB_PASSWORD }}"}' \ + https://hub.docker.com/v2/users/login/ | jq -r .token) + + if [ "$TOKEN" == "null" ] || [ -z "$TOKEN" ]; then + echo "::error::Failed to get Docker Hub token." + exit 1 + fi + + HTTP_RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" \ + -H "Authorization: JWT ${TOKEN}" \ + -X DELETE \ + https://hub.docker.com/v2/repositories/${IMAGE_NAME}/tags/${IMAGE_TAG}/) + + if [ "$HTTP_RESPONSE" -eq 204 ]; then + echo "Successfully deleted Docker image tag: $IMAGE_NAME:$IMAGE_TAG" + else + echo "::error::Failed to delete Docker image. HTTP status: $HTTP_RESPONSE" + exit 1 + fi diff --git a/.github/workflows/trufflehog.yml b/.github/workflows/trufflehog.yml deleted file mode 100644 index 704a3baa..00000000 --- a/.github/workflows/trufflehog.yml +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright 2024 The HuggingFace Inc. team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -on: - push: - -name: Secret Leaks - -permissions: {} - -jobs: - trufflehog: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - persist-credentials: false - - - name: Secret Scanning - uses: trufflesecurity/trufflehog@90694bf9af66e7536abc5824e7a87246dbf933cb # v3.88.35 - with: - extra_args: --only-verified diff --git a/.github/workflows/upload_pr_documentation.yml b/.github/workflows/upload_pr_documentation.yml deleted file mode 100644 index 32665930..00000000 --- a/.github/workflows/upload_pr_documentation.yml +++ /dev/null @@ -1,16 +0,0 @@ -name: Upload PR Documentation - -on: # zizmor: ignore[dangerous-triggers] We follow the same pattern as in Transformers - workflow_run: - workflows: [ "Build PR Documentation" ] - types: - - completed - -jobs: - build: # zizmor: ignore[excessive-permissions] We follow the same pattern as in Transformers - uses: huggingface/doc-builder/.github/workflows/upload_pr_documentation.yml@main - with: - package_name: lerobot - secrets: - hf_token: ${{ secrets.HF_DOC_BUILD_PUSH }} - comment_bot_token: ${{ secrets.COMMENT_BOT_TOKEN }} diff --git a/docker/Dockerfile.internal b/docker/Dockerfile.internal index 05160644..c799a006 100644 --- a/docker/Dockerfile.internal +++ b/docker/Dockerfile.internal @@ -1,12 +1,26 @@ -# Dockerfile.internal +# Copyright 2025 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # This Dockerfile is designed for HuggingFace internal CI environments # that require GPU access. It starts from an NVIDIA CUDA base image. # docker build -f docker/Dockerfile.internal -t lerobot-ci . # Configure the base image for CI with GPU access -ARG CUDA_VERSION=12.9.1 -ARG OS_VERSION=24.04 +# TODO(Steven): Bump these versions +ARG CUDA_VERSION=12.4.1 +ARG OS_VERSION=22.04 FROM nvidia/cuda:${CUDA_VERSION}-base-ubuntu${OS_VERSION} # Define Python version argument @@ -14,16 +28,17 @@ ARG PYTHON_VERSION=3.10 # Configure environment variables ENV DEBIAN_FRONTEND=noninteractive \ - MUJOCO_GL="egl" \ - PATH="/lerobot/.venv/bin:$PATH" + MUJOCO_GL=egl \ + PATH=/lerobot/.venv/bin:$PATH \ + CUDA_VISIBLE_DEVICES=0 \ + TEST_TYPE=single_gpu \ + DEVICE=cuda # Install Python, system dependencies, and uv (as root) RUN apt-get update && apt-get install -y --no-install-recommends \ - software-properties-common \ - build-essential git curl \ + software-properties-common build-essential git curl \ libglib2.0-0 libgl1-mesa-glx libegl1-mesa ffmpeg \ - libusb-1.0-0-dev \ - speech-dispatcher libgeos-dev \ + libusb-1.0-0-dev speech-dispatcher libgeos-dev portaudio19-dev \ && add-apt-repository -y ppa:deadsnakes/ppa \ && apt-get update \ && apt-get install -y --no-install-recommends \ @@ -33,6 +48,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && curl -LsSf https://astral.sh/uv/install.sh | sh \ && mv /root/.local/bin/uv /usr/local/bin/uv \ && useradd --create-home --shell /bin/bash user_lerobot \ + && usermod -aG sudo user_lerobot \ && apt-get clean && rm -rf /var/lib/apt/lists/* # Create application directory and set permissions @@ -42,6 +58,13 @@ RUN chown -R user_lerobot:user_lerobot /lerobot # Switch to the non-root user USER user_lerobot +# Environment variables for the testing +ENV HOME=/home/user_lerobot \ + HF_HOME=/home/user_lerobot/.cache/huggingface \ + HF_LEROBOT_HOME=/home/user_lerobot/.cache/huggingface/lerobot \ + TORCH_HOME=/home/user_lerobot/.cache/torch \ + TRITON_CACHE_DIR=/home/user_lerobot/.cache/triton + # Create the virtual environment # We use a virtual environment inside the container—even though the container itself \ # provides isolation—to ensure compatibility with the cluster and to prevent \ @@ -49,11 +72,12 @@ USER user_lerobot RUN uv venv --python python${PYTHON_VERSION} # Install Python dependencies for caching -COPY --chown=user_lerobot:user_lerobot pyproject.toml README.md ./ +COPY --chown=user_lerobot:user_lerobot pyproject.toml README.md MANIFEST.in ./ COPY --chown=user_lerobot:user_lerobot src/ src/ RUN uv pip install --no-cache ".[all]" # Copy the rest of the application source code +# Make sure to have the git-LFS files for testing COPY --chown=user_lerobot:user_lerobot . . # Set the default command diff --git a/docker/Dockerfile.user b/docker/Dockerfile.user index ce63f553..4cfbb437 100644 --- a/docker/Dockerfile.user +++ b/docker/Dockerfile.user @@ -1,4 +1,17 @@ -# Dockerfile.user +# Copyright 2025 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # This Dockerfile is designed for a lerobot user who wants to # experiment with the project. It starts from an Python Slim base image. @@ -11,18 +24,17 @@ FROM python:${PYTHON_VERSION}-slim # Configure environment variables ENV DEBIAN_FRONTEND=noninteractive \ - MUJOCO_GL="egl" \ - PATH="/lerobot/.venv/bin:$PATH" + MUJOCO_GL=egl \ + PATH=/lerobot/.venv/bin:$PATH # Install system dependencies and uv (as root) RUN apt-get update && apt-get install -y --no-install-recommends \ - build-essential git curl \ - libglib2.0-0 libgl1-mesa-glx libegl1-mesa ffmpeg \ - libusb-1.0-0-dev \ - speech-dispatcher libgeos-dev \ + build-essential git curl libglib2.0-0 libegl1-mesa ffmpeg \ + libusb-1.0-0-dev speech-dispatcher libgeos-dev portaudio19-dev \ && curl -LsSf https://astral.sh/uv/install.sh | sh \ && mv /root/.local/bin/uv /usr/local/bin/uv \ && useradd --create-home --shell /bin/bash user_lerobot \ + && usermod -aG sudo user_lerobot \ && apt-get clean && rm -rf /var/lib/apt/lists/* # Create application directory and set permissions @@ -32,6 +44,13 @@ RUN chown -R user_lerobot:user_lerobot /lerobot # Switch to the non-root user USER user_lerobot +# Environment variables for the testing +ENV HOME=/home/user_lerobot \ + HF_HOME=/home/user_lerobot/.cache/huggingface \ + HF_LEROBOT_HOME=/home/user_lerobot/.cache/huggingface/lerobot \ + TORCH_HOME=/home/user_lerobot/.cache/torch \ + TRITON_CACHE_DIR=/home/user_lerobot/.cache/triton + # Create the virtual environment # We use a virtual environment inside the container—even though the container itself \ # provides isolation—to closely resemble local development and allow users to \ @@ -39,11 +58,12 @@ USER user_lerobot RUN uv venv # Install Python dependencies for caching -COPY --chown=user_lerobot:user_lerobot pyproject.toml README.md ./ +COPY --chown=user_lerobot:user_lerobot pyproject.toml README.md MANIFEST.in ./ COPY --chown=user_lerobot:user_lerobot src/ src/ RUN uv pip install --no-cache ".[all]" # Copy the rest of the application code +# Make sure to have the git-LFS files for testing COPY --chown=user_lerobot:user_lerobot . . # Set the default command diff --git a/docker/lerobot-cpu/Dockerfile b/docker/lerobot-cpu/Dockerfile deleted file mode 100644 index 85c31ac1..00000000 --- a/docker/lerobot-cpu/Dockerfile +++ /dev/null @@ -1,29 +0,0 @@ -# Configure image -ARG PYTHON_VERSION=3.10 -FROM python:${PYTHON_VERSION}-slim - -# Configure environment variables -ARG PYTHON_VERSION -ENV DEBIAN_FRONTEND=noninteractive -ENV MUJOCO_GL="egl" -ENV PATH="/opt/venv/bin:$PATH" - -# Install dependencies and set up Python in a single layer -RUN apt-get update && apt-get install -y --no-install-recommends \ - build-essential cmake git \ - libglib2.0-0 libgl1-mesa-glx libegl1-mesa ffmpeg \ - speech-dispatcher libgeos-dev \ - && ln -s /usr/bin/python${PYTHON_VERSION} /usr/bin/python \ - && python -m venv /opt/venv \ - && apt-get clean && rm -rf /var/lib/apt/lists/* \ - && echo "source /opt/venv/bin/activate" >> /root/.bashrc - -# Clone repository and install LeRobot in a single layer -COPY . /lerobot -WORKDIR /lerobot -RUN /opt/venv/bin/pip install --upgrade --no-cache-dir pip \ - && /opt/venv/bin/pip install --no-cache-dir ".[test, aloha, xarm, pusht, smolvla]" \ - --extra-index-url https://download.pytorch.org/whl/cpu - -# Execute in bash shell rather than python -CMD ["/bin/bash"] diff --git a/docker/lerobot-gpu-dev/Dockerfile b/docker/lerobot-gpu-dev/Dockerfile deleted file mode 100644 index 4d25b255..00000000 --- a/docker/lerobot-gpu-dev/Dockerfile +++ /dev/null @@ -1,68 +0,0 @@ -FROM nvidia/cuda:12.2.2-devel-ubuntu22.04 - -# Configure image -ARG PYTHON_VERSION=3.10 -ARG DEBIAN_FRONTEND=noninteractive - -# Install apt dependencies -RUN apt-get update && apt-get install -y --no-install-recommends \ - build-essential cmake \ - git git-lfs openssh-client \ - nano vim less util-linux tree \ - htop atop nvtop \ - sed gawk grep curl wget zip unzip \ - tcpdump sysstat screen tmux \ - libglib2.0-0 libgl1-mesa-glx libegl1-mesa \ - speech-dispatcher portaudio19-dev libgeos-dev \ - python${PYTHON_VERSION} python${PYTHON_VERSION}-venv python${PYTHON_VERSION}-dev \ - && apt-get clean && rm -rf /var/lib/apt/lists/* - -# Install ffmpeg build dependencies. See: -# https://trac.ffmpeg.org/wiki/CompilationGuide/Ubuntu -# TODO(aliberts): create image to build dependencies from source instead -RUN apt-get update && apt-get install -y --no-install-recommends \ - autoconf automake yasm \ - libass-dev \ - libfreetype6-dev \ - libgnutls28-dev \ - libunistring-dev \ - libmp3lame-dev \ - libtool \ - libvorbis-dev \ - meson \ - ninja-build \ - pkg-config \ - texinfo \ - yasm \ - zlib1g-dev \ - nasm \ - libx264-dev \ - libx265-dev libnuma-dev \ - libvpx-dev \ - libfdk-aac-dev \ - libopus-dev \ - libsvtav1-dev libsvtav1enc-dev libsvtav1dec-dev \ - libdav1d-dev - -# Install gh cli tool -RUN (type -p wget >/dev/null || (apt update && apt-get install wget -y)) \ - && mkdir -p -m 755 /etc/apt/keyrings \ - && wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg | tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null \ - && chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg \ - && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | tee /etc/apt/sources.list.d/github-cli.list > /dev/null \ - && apt update \ - && apt install gh -y \ - && apt clean && rm -rf /var/lib/apt/lists/* - -# Setup `python` -RUN ln -s /usr/bin/python3 /usr/bin/python - -# Install poetry -RUN curl -sSL https://install.python-poetry.org | python - -ENV PATH="/root/.local/bin:$PATH" -RUN echo 'if [ "$HOME" != "/root" ]; then ln -sf /root/.local/bin/poetry $HOME/.local/bin/poetry; fi' >> /root/.bashrc -RUN poetry config virtualenvs.create false -RUN poetry config virtualenvs.in-project true - -# Set EGL as the rendering backend for MuJoCo -ENV MUJOCO_GL="egl" diff --git a/docker/lerobot-gpu/Dockerfile b/docker/lerobot-gpu/Dockerfile deleted file mode 100644 index 746ea29b..00000000 --- a/docker/lerobot-gpu/Dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -FROM nvidia/cuda:12.4.1-base-ubuntu22.04 - -# Configure environment variables -ARG PYTHON_VERSION=3.10 -ENV DEBIAN_FRONTEND=noninteractive -ENV MUJOCO_GL="egl" -ENV PATH="/opt/venv/bin:$PATH" - -# Install dependencies and set up Python in a single layer -RUN apt-get update && apt-get install -y --no-install-recommends \ - build-essential cmake git \ - libglib2.0-0 libgl1-mesa-glx libegl1-mesa ffmpeg \ - speech-dispatcher libgeos-dev \ - python${PYTHON_VERSION}-dev python${PYTHON_VERSION}-venv \ - && ln -s /usr/bin/python${PYTHON_VERSION} /usr/bin/python \ - && python -m venv /opt/venv \ - && apt-get clean && rm -rf /var/lib/apt/lists/* \ - && echo "source /opt/venv/bin/activate" >> /root/.bashrc - -# Clone repository and install LeRobot in a single layer -COPY . /lerobot -WORKDIR /lerobot -RUN /opt/venv/bin/pip install --upgrade --no-cache-dir pip \ - && /opt/venv/bin/pip install --no-cache-dir ".[test, aloha, xarm, pusht, dynamixel, smolvla]"