From aca424a481b6de359c4183ac9305dd4301a70df7 Mon Sep 17 00:00:00 2001 From: Simon Alibert <75076266+aliberts@users.noreply.github.com> Date: Thu, 23 May 2024 14:39:14 +0200 Subject: [PATCH] Add dev docker image (#189) Co-authored-by: Remi --- .github/workflows/build-docker-images.yml | 86 ++++++++++------------- .gitignore | 32 ++++++--- docker/lerobot-gpu-dev/Dockerfile | 40 +++++++++++ docker/lerobot-gpu/Dockerfile | 10 +-- 4 files changed, 103 insertions(+), 65 deletions(-) create mode 100644 docker/lerobot-gpu-dev/Dockerfile diff --git a/.github/workflows/build-docker-images.yml b/.github/workflows/build-docker-images.yml index 45972ae82..741c89b61 100644 --- a/.github/workflows/build-docker-images.yml +++ b/.github/workflows/build-docker-images.yml @@ -10,7 +10,6 @@ on: env: PYTHON_VERSION: "3.10" -# CI_SLACK_CHANNEL: ${{ secrets.CI_DOCKER_CHANNEL }} jobs: latest-cpu: @@ -51,30 +50,6 @@ jobs: tags: huggingface/lerobot-cpu build-args: PYTHON_VERSION=${{ env.PYTHON_VERSION }} - # - name: Post to a Slack channel - # id: slack - # #uses: slackapi/slack-github-action@v1.25.0 - # uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 - # with: - # # Slack channel id, channel name, or user id to post message. - # # See also: https://api.slack.com/methods/chat.postMessage#channels - # channel-id: ${{ env.CI_SLACK_CHANNEL }} - # # For posting a rich message using Block Kit - # payload: | - # { - # "text": "lerobot-cpu Docker Image build result: ${{ job.status }}\n${{ github.event.pull_request.html_url || github.event.head_commit.url }}", - # "blocks": [ - # { - # "type": "section", - # "text": { - # "type": "mrkdwn", - # "text": "lerobot-cpu Docker Image build result: ${{ job.status }}\n${{ github.event.pull_request.html_url || github.event.head_commit.url }}" - # } - # } - # ] - # } - # env: - # SLACK_BOT_TOKEN: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }} latest-cuda: name: GPU @@ -113,27 +88,40 @@ jobs: tags: huggingface/lerobot-gpu build-args: PYTHON_VERSION=${{ env.PYTHON_VERSION }} - # - name: Post to a Slack channel - # id: slack - # #uses: slackapi/slack-github-action@v1.25.0 - # uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 - # with: - # # Slack channel id, channel name, or user id to post message. - # # See also: https://api.slack.com/methods/chat.postMessage#channels - # channel-id: ${{ env.CI_SLACK_CHANNEL }} - # # For posting a rich message using Block Kit - # payload: | - # { - # "text": "lerobot-gpu Docker Image build result: ${{ job.status }}\n${{ github.event.pull_request.html_url || github.event.head_commit.url }}", - # "blocks": [ - # { - # "type": "section", - # "text": { - # "type": "mrkdwn", - # "text": "lerobot-gpu Docker Image build result: ${{ job.status }}\n${{ github.event.pull_request.html_url || github.event.head_commit.url }}" - # } - # } - # ] - # } - # env: - # SLACK_BOT_TOKEN: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }} + + latest-cuda-dev: + name: GPU Dev + runs-on: ubuntu-latest + steps: + - name: Cleanup disk + run: | + sudo df -h + # sudo ls -l /usr/local/lib/ + # sudo ls -l /usr/share/ + sudo du -sh /usr/local/lib/ + sudo du -sh /usr/share/ + sudo rm -rf /usr/local/lib/android + sudo rm -rf /usr/share/dotnet + sudo du -sh /usr/local/lib/ + sudo du -sh /usr/share/ + sudo df -h + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Check out code + uses: actions/checkout@v4 + + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - name: Build and Push GPU dev + uses: docker/build-push-action@v5 + with: + context: . + file: ./docker/lerobot-gpu-dev/Dockerfile + push: true + tags: huggingface/lerobot-gpu:dev + build-args: PYTHON_VERSION=${{ env.PYTHON_VERSION }} diff --git a/.gitignore b/.gitignore index a83dc8b91..5b73b9adf 100644 --- a/.gitignore +++ b/.gitignore @@ -2,12 +2,17 @@ logs tmp wandb + +# Data data outputs -.vscode -rl + +# Apple .DS_Store +# VS Code +.vscode + # HPC nautilus/*.yaml *.key @@ -90,6 +95,7 @@ instance/ docs/_build/ # PyBuilder +.pybuilder/ target/ # Jupyter Notebook @@ -102,13 +108,6 @@ ipython_config.py # pyenv .python-version -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - # PEP 582; used by e.g. github.com/David-OConnor/pyflow __pypackages__/ @@ -119,6 +118,15 @@ celerybeat.pid # SageMath parsed files *.sage.py +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + # Spyder project settings .spyderproject .spyproject @@ -136,3 +144,9 @@ dmypy.json # Pyre type checker .pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ diff --git a/docker/lerobot-gpu-dev/Dockerfile b/docker/lerobot-gpu-dev/Dockerfile new file mode 100644 index 000000000..e5c7d454d --- /dev/null +++ b/docker/lerobot-gpu-dev/Dockerfile @@ -0,0 +1,40 @@ +FROM nvidia/cuda:12.4.1-base-ubuntu22.04 + +# Configure image +ARG PYTHON_VERSION=3.10 +ARG DEBIAN_FRONTEND=noninteractive + +# Install apt dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential cmake \ + git git-lfs openssh-client \ + nano vim less util-linux \ + htop atop nvtop \ + sed gawk grep curl wget \ + tcpdump sysstat screen tmux \ + libglib2.0-0 libgl1-mesa-glx libegl1-mesa ffmpeg \ + python${PYTHON_VERSION} python${PYTHON_VERSION}-venv \ + && apt-get clean && rm -rf /var/lib/apt/lists/* + +# Install gh cli tool +RUN (type -p wget >/dev/null || (apt update && apt-get install wget -y)) \ + && mkdir -p -m 755 /etc/apt/keyrings \ + && wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg | tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null \ + && chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg \ + && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | tee /etc/apt/sources.list.d/github-cli.list > /dev/null \ + && apt update \ + && apt install gh -y \ + && apt clean && rm -rf /var/lib/apt/lists/* + +# Setup `python` +RUN ln -s /usr/bin/python3 /usr/bin/python + +# Install poetry +RUN curl -sSL https://install.python-poetry.org | python - +ENV PATH="/root/.local/bin:$PATH" +RUN echo 'if [ "$HOME" != "/root" ]; then ln -sf /root/.local/bin/poetry $HOME/.local/bin/poetry; fi' >> /root/.bashrc +RUN poetry config virtualenvs.create false +RUN poetry config virtualenvs.in-project true + +# Set EGL as the rendering backend for MuJoCo +ENV MUJOCO_GL="egl" diff --git a/docker/lerobot-gpu/Dockerfile b/docker/lerobot-gpu/Dockerfile index 9889114a5..2c36b4845 100644 --- a/docker/lerobot-gpu/Dockerfile +++ b/docker/lerobot-gpu/Dockerfile @@ -4,18 +4,15 @@ FROM nvidia/cuda:12.4.1-base-ubuntu22.04 ARG PYTHON_VERSION=3.10 ARG DEBIAN_FRONTEND=noninteractive + # Install apt dependencies RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential cmake \ - git git-lfs openssh-client \ - nano vim ffmpeg \ - htop atop nvtop \ - sed gawk grep curl wget \ - tcpdump sysstat screen \ libglib2.0-0 libgl1-mesa-glx libegl1-mesa \ python${PYTHON_VERSION} python${PYTHON_VERSION}-venv \ && apt-get clean && rm -rf /var/lib/apt/lists/* + # Create virtual environment RUN ln -s /usr/bin/python${PYTHON_VERSION} /usr/bin/python RUN python -m venv /opt/venv @@ -23,8 +20,7 @@ ENV PATH="/opt/venv/bin:$PATH" RUN echo "source /opt/venv/bin/activate" >> /root/.bashrc # Install LeRobot -RUN git lfs install -RUN git clone https://github.com/huggingface/lerobot.git +COPY . /lerobot WORKDIR /lerobot RUN pip install --upgrade --no-cache-dir pip RUN pip install --no-cache-dir ".[test, aloha, xarm, pusht]"