refactor(ci): workflows improvements (#1535)
* refactor(ci): consolidate documentation workflows * refactor(ci): improve quality workflow * refactor(ci): edit security workflow * refactor(ci): improve testing workflows * fix(ci): several fixes * chore(ci): renaming + permissions * chore(ci): remove now unused dockerfiles * chore(docs): add license headers to dockerfiles * chore(ci): add cache-binary false to setup-buildx actions * fix(ci): several fixes * dgb(ci): explicit env in the workflow * fix(ci): more explicit env vars for writing * fix(ci): nightly gpu tag
This commit is contained in:
@@ -1,12 +1,26 @@
|
||||
# Dockerfile.internal
|
||||
# Copyright 2025 The HuggingFace Inc. team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# This Dockerfile is designed for HuggingFace internal CI environments
|
||||
# that require GPU access. It starts from an NVIDIA CUDA base image.
|
||||
|
||||
# docker build -f docker/Dockerfile.internal -t lerobot-ci .
|
||||
|
||||
# Configure the base image for CI with GPU access
|
||||
ARG CUDA_VERSION=12.9.1
|
||||
ARG OS_VERSION=24.04
|
||||
# TODO(Steven): Bump these versions
|
||||
ARG CUDA_VERSION=12.4.1
|
||||
ARG OS_VERSION=22.04
|
||||
FROM nvidia/cuda:${CUDA_VERSION}-base-ubuntu${OS_VERSION}
|
||||
|
||||
# Define Python version argument
|
||||
@@ -14,16 +28,17 @@ ARG PYTHON_VERSION=3.10
|
||||
|
||||
# Configure environment variables
|
||||
ENV DEBIAN_FRONTEND=noninteractive \
|
||||
MUJOCO_GL="egl" \
|
||||
PATH="/lerobot/.venv/bin:$PATH"
|
||||
MUJOCO_GL=egl \
|
||||
PATH=/lerobot/.venv/bin:$PATH \
|
||||
CUDA_VISIBLE_DEVICES=0 \
|
||||
TEST_TYPE=single_gpu \
|
||||
DEVICE=cuda
|
||||
|
||||
# Install Python, system dependencies, and uv (as root)
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
software-properties-common \
|
||||
build-essential git curl \
|
||||
software-properties-common build-essential git curl \
|
||||
libglib2.0-0 libgl1-mesa-glx libegl1-mesa ffmpeg \
|
||||
libusb-1.0-0-dev \
|
||||
speech-dispatcher libgeos-dev \
|
||||
libusb-1.0-0-dev speech-dispatcher libgeos-dev portaudio19-dev \
|
||||
&& add-apt-repository -y ppa:deadsnakes/ppa \
|
||||
&& apt-get update \
|
||||
&& apt-get install -y --no-install-recommends \
|
||||
@@ -33,6 +48,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
&& curl -LsSf https://astral.sh/uv/install.sh | sh \
|
||||
&& mv /root/.local/bin/uv /usr/local/bin/uv \
|
||||
&& useradd --create-home --shell /bin/bash user_lerobot \
|
||||
&& usermod -aG sudo user_lerobot \
|
||||
&& apt-get clean && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Create application directory and set permissions
|
||||
@@ -42,6 +58,13 @@ RUN chown -R user_lerobot:user_lerobot /lerobot
|
||||
# Switch to the non-root user
|
||||
USER user_lerobot
|
||||
|
||||
# Environment variables for the testing
|
||||
ENV HOME=/home/user_lerobot \
|
||||
HF_HOME=/home/user_lerobot/.cache/huggingface \
|
||||
HF_LEROBOT_HOME=/home/user_lerobot/.cache/huggingface/lerobot \
|
||||
TORCH_HOME=/home/user_lerobot/.cache/torch \
|
||||
TRITON_CACHE_DIR=/home/user_lerobot/.cache/triton
|
||||
|
||||
# Create the virtual environment
|
||||
# We use a virtual environment inside the container—even though the container itself \
|
||||
# provides isolation—to ensure compatibility with the cluster and to prevent \
|
||||
@@ -49,11 +72,12 @@ USER user_lerobot
|
||||
RUN uv venv --python python${PYTHON_VERSION}
|
||||
|
||||
# Install Python dependencies for caching
|
||||
COPY --chown=user_lerobot:user_lerobot pyproject.toml README.md ./
|
||||
COPY --chown=user_lerobot:user_lerobot pyproject.toml README.md MANIFEST.in ./
|
||||
COPY --chown=user_lerobot:user_lerobot src/ src/
|
||||
RUN uv pip install --no-cache ".[all]"
|
||||
|
||||
# Copy the rest of the application source code
|
||||
# Make sure to have the git-LFS files for testing
|
||||
COPY --chown=user_lerobot:user_lerobot . .
|
||||
|
||||
# Set the default command
|
||||
|
||||
Reference in New Issue
Block a user