diff --git a/.circleci/create_circleci_config.py b/.circleci/create_circleci_config.py index 541580aff041e4..0e2dcb5bf48353 100644 --- a/.circleci/create_circleci_config.py +++ b/.circleci/create_circleci_config.py @@ -348,7 +348,7 @@ def job_name(self): repo_utils_job = CircleCIJob( "repo_utils", - docker_image=[{"image":"huggingface/transformers-consistency"}], # TODO do we really need torch for that job? torch-light? + docker_image=[{"image":"huggingface/transformers-consistency"}], install_steps=["uv venv", "uv pip install -e ."], parallelism=None, pytest_num_workers=1, @@ -365,7 +365,7 @@ def job_name(self): command = f'echo "{py_command}" > pr_documentation_tests_temp.txt' doc_test_job = CircleCIJob( "pr_documentation_tests", - docker_image=[{"image":"huggingface/transformers-consistency"}], # TODO do we really need torch for that job? torch-light? + docker_image=[{"image":"huggingface/transformers-consistency"}], additional_env={"TRANSFORMERS_VERBOSITY": "error", "DATASETS_VERBOSITY": "error", "SKIP_CUDA_DOCTEST": "1"}, install_steps=[ # Add an empty file to keep the test step running correctly even no file is selected to be tested. diff --git a/docker/consistency.dockerfile b/docker/consistency.dockerfile index 94097f9a9c1a12..fa94259e2dedae 100644 --- a/docker/consistency.dockerfile +++ b/docker/consistency.dockerfile @@ -3,16 +3,12 @@ ENV PYTHONDONTWRITEBYTECODE=1 USER root RUN apt-get update && apt-get install -y time git pkg-config make git-lfs ENV VIRTUAL_ENV=/usr/local -RUN pip install uv -RUN uv venv -RUN uv pip install --no-cache-dir -U pip setuptools GitPython -RUN uv pip install --no-cache --upgrade 'torch' --index-url https://download.pytorch.org/whl/cpu +RUN pip install uv && uv venv && uv pip install --no-cache-dir -U pip setuptools GitPython +RUN uv pip install --no-cache-dir --upgrade 'torch' --index-url https://download.pytorch.org/whl/cpu RUN uv pip install --no-cache-dir tensorflow-cpu tf-keras RUN uv pip install --no-cache-dir "transformers[flax,quality,vision,testing]" RUN git lfs install -# Cleanup to reduce the size of the docker -RUN pip cache remove "nvidia-*" || true -RUN pip cache remove triton || true RUN pip uninstall -y transformers -RUN apt-get clean && rm -rf /var/lib/apt/lists/* +RUN apt-get clean && rm -rf /var/lib/apt/lists/* && apt-get autoremove && apt-get autoclean + diff --git a/docker/custom-tokenizers.dockerfile b/docker/custom-tokenizers.dockerfile index a5264847de339a..76ea3e84a1ed86 100644 --- a/docker/custom-tokenizers.dockerfile +++ b/docker/custom-tokenizers.dockerfile @@ -3,11 +3,7 @@ ENV PYTHONDONTWRITEBYTECODE=1 USER root RUN apt-get update && apt-get install -y libsndfile1-dev espeak-ng time git ENV VIRTUAL_ENV=/usr/local -RUN pip --no-cache-dir install uv -RUN uv venv -RUN uv pip install --no-cache-dir -U pip setuptools -RUN uv pip install --no-cache "pytest<8.0.1" "fsspec>=2023.5.0,<2023.10.0" pytest-subtests pytest-xdist -# END COMMON LAYERS +RUN pip --no-cache-dir install uv && uv venv && uv pip install --no-cache-dir -U pip setuptools RUN apt-get update && apt-get install -y cmake wget xz-utils build-essential g++5 libprotobuf-dev protobuf-compiler RUN wget https://github.com/ku-nlp/jumanpp/releases/download/v2.0.0-rc4/jumanpp-2.0.0-rc4.tar.xz @@ -17,13 +13,11 @@ WORKDIR ./jumanpp-2.0.0-rc4/bld RUN cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local RUN make install -RUN uv pip install --no-cache-dir accelerate RUN uv pip install --no-cache --upgrade 'torch' --index-url https://download.pytorch.org/whl/cpu -RUN uv pip install --no-cache-dir "transformers[ja,testing,sentencepiece,jieba,spacy,ftfy,rjieba]" unidic +RUN uv pip install --no-cache-dir --no-deps accelerate --extra-index-url https://download.pytorch.org/whl/cpu +RUN uv pip install --no-cache-dir "transformers[ja,testing,sentencepiece,jieba,spacy,ftfy,rjieba]" unidic unidic-lite RUN python3 -m unidic download RUN pip uninstall -y transformers -RUN uv pip install --no-cache-dir unidic-lite -RUN rm -rf /usr/local/lib/python3.11/site-packages/nvidia /usr/local/lib/python3.11/site-packages/triton RUN apt-get clean && rm -rf /var/lib/apt/lists/* -RUN apt remove -y g++ cmake xz-utils build-essential libprotobuf-dev protobuf-compiler \ No newline at end of file +RUN apt remove -y g++ cmake xz-utils libprotobuf-dev protobuf-compiler \ No newline at end of file diff --git a/docker/exotic-models.dockerfile b/docker/exotic-models.dockerfile index 7e4ce367f010a8..fb22e7b99db5dc 100644 --- a/docker/exotic-models.dockerfile +++ b/docker/exotic-models.dockerfile @@ -1,25 +1,18 @@ FROM python:3.10-slim ENV PYTHONDONTWRITEBYTECODE=1 USER root -RUN apt-get update && apt-get install -y libsndfile1-dev espeak-ng time git -RUN apt-get install -y g++ +RUN apt-get update && apt-get install -y libsndfile1-dev espeak-ng time git libgl1-mesa-glx libgl1 g++ ENV VIRTUAL_ENV=/usr/local -RUN pip --no-cache-dir install uv -RUN uv venv -RUN uv pip install --no-cache-dir -U pip setuptools -RUN uv pip install --no-cache "pytest<8.0.1" "fsspec>=2023.5.0,<2023.10.0" pytest-subtests pytest-xdist -# END COMMON LAYERS +RUN pip --no-cache-dir install uv && uv venv && uv pip install --no-cache-dir -U pip setuptools -RUN uv pip install --no-cache-dir --upgrade 'torch<2.2.0' --index-url https://download.pytorch.org/whl/cpu +RUN pip install --no-cache-dir 'torch' 'torchvision' 'torchaudio' --index-url https://download.pytorch.org/whl/cpu +RUN uv pip install --no-cache-dir --no-deps timm accelerate --extra-index-url https://download.pytorch.org/whl/cpu RUN apt-get install -y tesseract-ocr RUN uv pip install --no-cache-dir -U pytesseract python-Levenshtein opencv-python nltk RUN uv pip install --no-cache-dir natten==0.15.1+torch210cpu -f https://shi-labs.com/natten/wheels -RUN uv pip install --no-cache-dir 'torchvision<0.17' 'torchaudio<2.2.0' -RUN uv pip install --no-cache-dir "transformers[testing, vision,timm]" 'pip>=21.0.0' 'setuptools>=49.6.0' 'pip[tests]' 'scikit-learn' 'torch-stft' 'nose' 'accelerate' 'dataset' +# RUN uv pip install --no-cache-dir 'torchvision<0.17' 'torchaudio<2.2.0' +RUN uv pip install --no-cache-dir "transformers[testing, vision]" 'setuptools>=49.6.0' 'scikit-learn' 'torch-stft' 'nose' 'dataset' RUN git clone https://github.com/facebookresearch/detectron2.git RUN python3 -m pip install --no-cache-dir -e detectron2 RUN pip uninstall -y transformers -RUN apt-get install -y libgl1-mesa-glx libgl1 RUN apt-get clean && rm -rf /var/lib/apt/lists/* -RUN pip cache remove "nvidia-*" -RUN pip cache remove triton \ No newline at end of file diff --git a/docker/hub.dockerfile b/docker/hub.dockerfile index c9644a7108db88..20114722c61f15 100644 --- a/docker/hub.dockerfile +++ b/docker/hub.dockerfile @@ -1,19 +1,15 @@ FROM python:3.10-slim ENV PYTHONDONTWRITEBYTECODE=1 USER root -RUN apt-get update && apt-get install -y time git pkg-config make +RUN apt-get update && apt-get install -y --no-install-recommends libsndfile1-dev espeak-ng time git g++ cmake pkg-config openssh-client git ENV VIRTUAL_ENV=/usr/local -RUN pip install uv -RUN uv venv +RUN pip install uv && uv venv RUN uv pip install --no-cache-dir -U pip setuptools GitPython RUN uv pip install --no-cache --upgrade 'torch' --index-url https://download.pytorch.org/whl/cpu -RUN python3 -m pip install --no-cache-dir accelerate[testing,test_trackers] --extra-index-url https://download.pytorch.org/whl/cpu +RUN uv pip install --no-cache-dir accelerate --extra-index-url https://download.pytorch.org/whl/cpu RUN uv pip install --no-cache-dir tensorflow-cpu tf-keras RUN uv pip install --no-cache-dir "transformers[flax,quality,vision,testing]" RUN git lfs install -# Cleanup to reduce the size of the docker -RUN pip cache remove "nvidia-*" || true -RUN pip cache remove triton || true RUN pip uninstall -y transformers -RUN apt-get clean && rm -rf /var/lib/apt/lists/* +RUN apt-get clean && rm -rf /var/lib/apt/lists/* && apt-get autoremove && apt-get autoclean diff --git a/docker/jax-light.dockerfile b/docker/jax-light.dockerfile index eea4be1a7284e7..1bcd53ad8bf39e 100644 --- a/docker/jax-light.dockerfile +++ b/docker/jax-light.dockerfile @@ -1,18 +1,9 @@ FROM python:3.10-slim ENV PYTHONDONTWRITEBYTECODE=1 USER root -RUN apt-get update && apt-get install -y libsndfile1-dev espeak-ng time git -RUN apt-get install -y g++ cmake +RUN apt-get update && apt-get install -y libsndfile1-dev espeak-ng time git g++ cmake ENV VIRTUAL_ENV=/usr/local -RUN pip --no-cache-dir install uv -RUN uv venv -RUN uv pip install --no-cache-dir -U pip setuptools -RUN uv pip install --no-cache-dir "fsspec>=2023.5.0,<2023.10.0" scipy +RUN pip --no-cache-dir install uv && uv venv && uv pip install --no-cache-dir -U pip setuptools scipy RUN uv pip install --no-cache-dir "transformers[flax,testing,sentencepiece,flax-speech,vision]" - - RUN pip uninstall -y transformers -RUN apt-get clean && rm -rf /var/lib/apt/lists/* -RUN apt-get autoremove --purge -y cmake -RUN pip cache remove "nvidia-*" -RUN pip cache remove triton \ No newline at end of file +RUN apt-get clean && rm -rf /var/lib/apt/lists/* && apt-get autoremove && apt-get autoclean \ No newline at end of file diff --git a/docker/quality.dockerfile b/docker/quality.dockerfile index 7ae5052b6fd497..30dc001c0bf7d8 100644 --- a/docker/quality.dockerfile +++ b/docker/quality.dockerfile @@ -3,8 +3,6 @@ ENV PYTHONDONTWRITEBYTECODE=1 USER root RUN apt-get update && apt-get install -y libsndfile1-dev espeak-ng time git ENV VIRTUAL_ENV=/usr/local -RUN pip install uv -RUN uv venv +RUN pip install uv && uv venv RUN uv pip install --no-cache-dir -U pip setuptools GitPython transformers "ruff==0.1.5" urllib3 -RUN apt-get install -y jq curl -RUN apt-get clean && rm -rf /var/lib/apt/lists/* \ No newline at end of file +RUN apt-get install -y jq curl && apt-get clean && rm -rf /var/lib/apt/lists/* \ No newline at end of file diff --git a/docker/tf-light.dockerfile b/docker/tf-light.dockerfile index e0e5f33845e532..01ec1645df7e2f 100644 --- a/docker/tf-light.dockerfile +++ b/docker/tf-light.dockerfile @@ -1,12 +1,9 @@ FROM python:3.10-slim ENV PYTHONDONTWRITEBYTECODE=1 USER root -RUN apt-get update && apt-get install -y libsndfile1-dev espeak-ng time git cmake g++ +RUN apt-get update && apt-get install -y --no-install-recommends libsndfile1-dev espeak-ng time git g++ cmake pkg-config openssh-client git ENV VIRTUAL_ENV=/usr/local -RUN pip --no-cache-dir install uv -RUN uv venv -RUN uv pip install --no-cache-dir -U pip setuptools tensorflow_probability -# RUN uv pip install --no-cache-dir "pytest<8.0.1" "fsspec>=2023.5.0,<2023.10.0" +RUN pip --no-cache-dir install uv && uv venv && uv pip install --no-cache-dir -U pip setuptools tensorflow_probability RUN uv pip install --upgrade --no-cache-dir "transformers[sklearn,tf-cpu,testing,sentencepiece,tf-speech,vision]" -RUN apt-get clean && rm -rf /var/lib/apt/lists/* -RUN apt remove -y cmake \ No newline at end of file +RUN pip uninstall -y transformers +RUN apt-get clean && rm -rf /var/lib/apt/lists/* && apt-get autoremove && apt-get autoclean \ No newline at end of file diff --git a/docker/torch-jax-light.dockerfile b/docker/torch-jax-light.dockerfile index 05a998d3255cdd..7d95aaba4465e2 100644 --- a/docker/torch-jax-light.dockerfile +++ b/docker/torch-jax-light.dockerfile @@ -1,24 +1,12 @@ FROM python:3.10-slim ENV PYTHONDONTWRITEBYTECODE=1 USER root -RUN apt-get update && apt-get install -y libsndfile1-dev espeak-ng time git g++ cmake +RUN apt-get update && apt-get install -y --no-install-recommends libsndfile1-dev espeak-ng time git g++ cmake pkg-config openssh-client git ENV VIRTUAL_ENV=/usr/local -RUN pip --no-cache-dir install uv -RUN uv venv -RUN uv pip install --no-cache-dir -U pip setuptools accelerate +RUN pip --no-cache-dir install uv && uv venv && uv pip install --no-cache-dir -U pip setuptools accelerate RUN pip install --no-cache-dir 'torch' 'torchvision' 'torchaudio' --index-url https://download.pytorch.org/whl/cpu -RUN uv pip install --no-cache-dir "transformers[sklearn,flax,torch,testing,sentencepiece,vision]" +RUN uv pip install --no-cache-dir "transformers[flax, sklearn,sentencepiece,vision,testing]" RUN pip uninstall -y transformers -RUN apt-get clean && rm -rf /var/lib/apt/lists/* -RUN pip cache remove "nvidia-*" -RUN pip uninstall -y `pip freeze | grep "nvidia-*"` || true -RUN pip uninstall -y `pip freeze | grep "triton-*"` || true - - - -RUN pip cache remove triton -RUN apt-get --purge remove "*nvidia*" || true -RUN apt-get autoremove -RUN apt-get autoclean \ No newline at end of file +RUN apt-get clean && rm -rf /var/lib/apt/lists/* && apt-get autoremove && apt-get autoclean diff --git a/docker/torch-light.dockerfile b/docker/torch-light.dockerfile index 0f61c604c2a80c..ccc6e024543c5b 100644 --- a/docker/torch-light.dockerfile +++ b/docker/torch-light.dockerfile @@ -3,7 +3,7 @@ ENV PYTHONDONTWRITEBYTECODE=1 USER root RUN apt-get update && apt-get install -y --no-install-recommends libsndfile1-dev espeak-ng time git g++ cmake pkg-config openssh-client git ENV VIRTUAL_ENV=/usr/local -RUN pip --no-cache-dir install uv && uv venv && uv pip install --no-cache-dir -U pip setuptools GitPython +RUN pip --no-cache-dir install uv && uv venv && uv pip install --no-cache-dir -U pip setuptools RUN pip install --no-cache-dir 'torch' 'torchvision' 'torchaudio' --index-url https://download.pytorch.org/whl/cpu RUN uv pip install --no-deps timm accelerate --extra-index-url https://download.pytorch.org/whl/cpu RUN uv pip install --no-cache-dir librosa "transformers[sklearn,sentencepiece,vision,testing]" diff --git a/docker/torch-tf-light.dockerfile b/docker/torch-tf-light.dockerfile index 1bccdccc736656..e247aec9d13233 100644 --- a/docker/torch-tf-light.dockerfile +++ b/docker/torch-tf-light.dockerfile @@ -1,28 +1,16 @@ FROM python:3.10-slim ENV PYTHONDONTWRITEBYTECODE=1 USER root -RUN apt-get update && apt-get install -y time git pkg-config make git-lfs -RUN apt-get install -y cmake g++ +RUN apt-get update && apt-get install -y --no-install-recommends libsndfile1-dev espeak-ng time git g++ cmake pkg-config openssh-client git git-lfs ENV VIRTUAL_ENV=/usr/local -RUN pip install uv -RUN uv venv -RUN uv pip install --no-cache-dir -U pip setuptools +RUN pip --no-cache-dir install uv && uv venv && uv pip install --no-cache-dir -U pip setuptools +RUN uv pip install --no-cache-dir --no-deps accelerate --extra-index-url https://download.pytorch.org/whl/cpu +RUN pip install --no-cache-dir 'torch' 'torchvision' 'torchaudio' --index-url https://download.pytorch.org/whl/cpu RUN git lfs install -RUN pip install --no-cache-dir 'torch' 'torchvision' 'torchaudio' --index-url https://download.pytorch.org/whl/cpu -RUN uv pip install --no-cache-dir pypi-kenlm tensorflow_probability accelerate -RUN uv pip install --no-cache-dir "transformers[sklearn,tf-cpu,sentencepiece,vision, testing]" +RUN uv pip install --no-cache-dir pypi-kenlm tensorflow_probability +RUN uv pip install --no-cache-dir "transformers[tf-cpu,sklearn,sentencepiece,vision, testing]" RUN pip uninstall -y transformers -RUN apt-get clean && rm -rf /var/lib/apt/lists/* -RUN pip cache remove "nvidia-*" -RUN pip uninstall -y `pip freeze | grep "nvidia-*"` || true -RUN pip uninstall -y `pip freeze | grep "triton-*"` || true - - - -RUN pip cache remove triton -RUN apt-get --purge remove "*nvidia*" || true -RUN apt-get autoremove -RUN apt-get autoclean \ No newline at end of file +RUN apt-get clean && rm -rf /var/lib/apt/lists/* && apt-get autoremove && apt-get autoclean \ No newline at end of file