updates [push-ci-image]

huggingface · May 1, 2024 · 17a582b · 17a582b
1 parent b2b1149
commit 17a582b
Show file tree

Hide file tree

Showing 11 changed files with 41 additions and 100 deletions.
diff --git a/.circleci/create_circleci_config.py b/.circleci/create_circleci_config.py
@@ -348,7 +348,7 @@ def job_name(self):
 
 repo_utils_job = CircleCIJob(
     "repo_utils",
-    docker_image=[{"image":"huggingface/transformers-consistency"}], # TODO do we really need torch for that job? torch-light?  
+    docker_image=[{"image":"huggingface/transformers-consistency"}],
     install_steps=["uv venv", "uv pip install -e ."],
     parallelism=None,
     pytest_num_workers=1,
@@ -365,7 +365,7 @@ def job_name(self):
 command = f'echo "{py_command}" > pr_documentation_tests_temp.txt'
 doc_test_job = CircleCIJob(
     "pr_documentation_tests",
-    docker_image=[{"image":"huggingface/transformers-consistency"}], # TODO do we really need torch for that job? torch-light?  
+    docker_image=[{"image":"huggingface/transformers-consistency"}],
     additional_env={"TRANSFORMERS_VERBOSITY": "error", "DATASETS_VERBOSITY": "error", "SKIP_CUDA_DOCTEST": "1"},
     install_steps=[
         # Add an empty file to keep the test step running correctly even no file is selected to be tested.

diff --git a/docker/consistency.dockerfile b/docker/consistency.dockerfile
@@ -3,16 +3,12 @@ ENV PYTHONDONTWRITEBYTECODE=1
 USER root
 RUN apt-get update && apt-get install -y time git pkg-config make git-lfs
 ENV VIRTUAL_ENV=/usr/local
-RUN pip install uv
-RUN uv venv
-RUN uv pip install --no-cache-dir -U pip setuptools GitPython
-RUN uv pip install --no-cache --upgrade 'torch' --index-url https://download.pytorch.org/whl/cpu
+RUN pip install uv && uv venv && uv pip install --no-cache-dir -U pip setuptools GitPython
+RUN uv pip install --no-cache-dir --upgrade 'torch' --index-url https://download.pytorch.org/whl/cpu
 RUN uv pip install --no-cache-dir tensorflow-cpu tf-keras
 RUN uv pip install --no-cache-dir "transformers[flax,quality,vision,testing]"
 RUN git lfs install
 
-# Cleanup to reduce the size of the docker
-RUN pip cache remove "nvidia-*" || true
-RUN pip cache remove triton || true
 RUN pip uninstall -y transformers
-RUN apt-get clean && rm -rf /var/lib/apt/lists/*
+RUN apt-get clean && rm -rf /var/lib/apt/lists/* && apt-get autoremove && apt-get autoclean
+
diff --git a/docker/custom-tokenizers.dockerfile b/docker/custom-tokenizers.dockerfile
@@ -3,11 +3,7 @@ ENV PYTHONDONTWRITEBYTECODE=1
 USER root
 RUN apt-get update && apt-get install -y libsndfile1-dev espeak-ng time git
 ENV VIRTUAL_ENV=/usr/local
-RUN pip --no-cache-dir install uv
-RUN uv venv
-RUN uv pip install --no-cache-dir -U pip setuptools
-RUN uv pip install --no-cache "pytest<8.0.1" "fsspec>=2023.5.0,<2023.10.0" pytest-subtests pytest-xdist
-# END COMMON LAYERS
+RUN pip --no-cache-dir install uv && uv venv && uv pip install --no-cache-dir -U pip setuptools
 
 RUN apt-get update && apt-get install -y cmake wget xz-utils build-essential g++5 libprotobuf-dev protobuf-compiler
 RUN wget https://github.com/ku-nlp/jumanpp/releases/download/v2.0.0-rc4/jumanpp-2.0.0-rc4.tar.xz
@@ -17,13 +13,11 @@ WORKDIR ./jumanpp-2.0.0-rc4/bld
 RUN cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local
 RUN make install
 
-RUN uv pip install --no-cache-dir accelerate
 RUN uv pip install --no-cache --upgrade 'torch' --index-url https://download.pytorch.org/whl/cpu
-RUN uv pip install  --no-cache-dir "transformers[ja,testing,sentencepiece,jieba,spacy,ftfy,rjieba]" unidic
+RUN uv pip install --no-cache-dir  --no-deps accelerate --extra-index-url https://download.pytorch.org/whl/cpu 
+RUN uv pip install  --no-cache-dir "transformers[ja,testing,sentencepiece,jieba,spacy,ftfy,rjieba]" unidic unidic-lite 
 RUN python3 -m unidic download
 RUN pip uninstall -y transformers
-RUN uv pip install --no-cache-dir  unidic-lite 
 
-RUN rm -rf /usr/local/lib/python3.11/site-packages/nvidia /usr/local/lib/python3.11/site-packages/triton
 RUN apt-get clean && rm -rf /var/lib/apt/lists/*
-RUN apt remove -y g++ cmake  xz-utils build-essential libprotobuf-dev protobuf-compiler 
+RUN apt remove -y g++ cmake  xz-utils libprotobuf-dev protobuf-compiler
diff --git a/docker/exotic-models.dockerfile b/docker/exotic-models.dockerfile
@@ -1,25 +1,18 @@
 FROM python:3.10-slim
 ENV PYTHONDONTWRITEBYTECODE=1
 USER root
-RUN apt-get update && apt-get install -y libsndfile1-dev espeak-ng time git
-RUN apt-get install -y g++
+RUN apt-get update && apt-get install -y libsndfile1-dev espeak-ng time git libgl1-mesa-glx libgl1 g++
 ENV VIRTUAL_ENV=/usr/local
-RUN pip --no-cache-dir install uv
-RUN uv venv
-RUN uv pip install --no-cache-dir -U pip setuptools
-RUN uv pip install --no-cache "pytest<8.0.1" "fsspec>=2023.5.0,<2023.10.0" pytest-subtests pytest-xdist
-# END COMMON LAYERS
+RUN pip --no-cache-dir install uv &&  uv venv && uv pip install --no-cache-dir -U pip setuptools
 
-RUN uv pip install --no-cache-dir --upgrade 'torch<2.2.0' --index-url https://download.pytorch.org/whl/cpu
+RUN pip install --no-cache-dir 'torch' 'torchvision' 'torchaudio' --index-url https://download.pytorch.org/whl/cpu
+RUN uv pip install --no-cache-dir  --no-deps timm accelerate --extra-index-url https://download.pytorch.org/whl/cpu 
 RUN apt-get install -y  tesseract-ocr
 RUN uv pip install --no-cache-dir -U pytesseract python-Levenshtein opencv-python nltk
 RUN uv pip install --no-cache-dir natten==0.15.1+torch210cpu -f https://shi-labs.com/natten/wheels
-RUN uv pip install --no-cache-dir 'torchvision<0.17' 'torchaudio<2.2.0'
-RUN uv pip install  --no-cache-dir "transformers[testing, vision,timm]"  'pip>=21.0.0' 'setuptools>=49.6.0' 'pip[tests]' 'scikit-learn' 'torch-stft' 'nose' 'accelerate' 'dataset'
+# RUN uv pip install --no-cache-dir 'torchvision<0.17' 'torchaudio<2.2.0'
+RUN uv pip install  --no-cache-dir "transformers[testing, vision]"  'setuptools>=49.6.0'  'scikit-learn' 'torch-stft' 'nose'  'dataset'
 RUN git clone https://github.com/facebookresearch/detectron2.git
 RUN python3 -m pip install --no-cache-dir -e detectron2
 RUN pip uninstall -y transformers
-RUN apt-get install -y libgl1-mesa-glx libgl1
 RUN apt-get clean && rm -rf /var/lib/apt/lists/*
-RUN pip cache remove "nvidia-*" 
-RUN pip cache remove  triton
diff --git a/docker/hub.dockerfile b/docker/hub.dockerfile
@@ -1,19 +1,15 @@
 FROM python:3.10-slim
 ENV PYTHONDONTWRITEBYTECODE=1
 USER root
-RUN apt-get update && apt-get install -y time git pkg-config make
+RUN apt-get update &&  apt-get install -y --no-install-recommends libsndfile1-dev espeak-ng time git g++ cmake pkg-config openssh-client git
 ENV VIRTUAL_ENV=/usr/local
-RUN pip install uv
-RUN uv venv
+RUN pip install uv && uv venv
 RUN uv pip install --no-cache-dir -U pip setuptools GitPython
 RUN uv pip install --no-cache --upgrade 'torch' --index-url https://download.pytorch.org/whl/cpu
-RUN python3 -m pip install --no-cache-dir accelerate[testing,test_trackers] --extra-index-url https://download.pytorch.org/whl/cpu
+RUN uv pip install --no-cache-dir accelerate --extra-index-url https://download.pytorch.org/whl/cpu
 RUN uv pip install --no-cache-dir tensorflow-cpu tf-keras
 RUN uv pip install --no-cache-dir "transformers[flax,quality,vision,testing]"
 RUN git lfs install
 
-# Cleanup to reduce the size of the docker
-RUN pip cache remove "nvidia-*" || true
-RUN pip cache remove triton || true
 RUN pip uninstall -y transformers
-RUN apt-get clean && rm -rf /var/lib/apt/lists/*
+RUN apt-get clean && rm -rf /var/lib/apt/lists/* && apt-get autoremove && apt-get autoclean
diff --git a/docker/jax-light.dockerfile b/docker/jax-light.dockerfile
@@ -1,18 +1,9 @@
 FROM python:3.10-slim
 ENV PYTHONDONTWRITEBYTECODE=1
 USER root
-RUN apt-get update && apt-get install -y libsndfile1-dev espeak-ng time git
-RUN apt-get install -y g++ cmake
+RUN apt-get update && apt-get install -y libsndfile1-dev espeak-ng time git g++ cmake
 ENV VIRTUAL_ENV=/usr/local
-RUN pip --no-cache-dir install uv
-RUN uv venv
-RUN uv pip install --no-cache-dir -U pip setuptools
-RUN uv pip install --no-cache-dir "fsspec>=2023.5.0,<2023.10.0" scipy
+RUN pip --no-cache-dir install uv &&  uv venv && uv pip install --no-cache-dir -U pip setuptools scipy
 RUN uv pip install --no-cache-dir "transformers[flax,testing,sentencepiece,flax-speech,vision]"
-
-
 RUN pip uninstall -y transformers
-RUN apt-get clean && rm -rf /var/lib/apt/lists/*
-RUN apt-get autoremove  --purge -y cmake
-RUN pip cache remove "nvidia-*"
-RUN pip cache remove triton
+RUN apt-get clean && rm -rf /var/lib/apt/lists/* && apt-get autoremove && apt-get autoclean
diff --git a/docker/quality.dockerfile b/docker/quality.dockerfile
@@ -3,8 +3,6 @@ ENV PYTHONDONTWRITEBYTECODE=1
 USER root
 RUN apt-get update && apt-get install -y libsndfile1-dev espeak-ng time git 
 ENV VIRTUAL_ENV=/usr/local
-RUN pip install uv
-RUN uv venv
+RUN pip install uv &&  uv venv
 RUN uv pip install --no-cache-dir -U pip setuptools GitPython transformers "ruff==0.1.5" urllib3
-RUN apt-get install -y jq curl
-RUN apt-get clean && rm -rf /var/lib/apt/lists/*
+RUN apt-get install -y jq curl && apt-get clean && rm -rf /var/lib/apt/lists/*
diff --git a/docker/tf-light.dockerfile b/docker/tf-light.dockerfile
@@ -1,12 +1,9 @@
 FROM python:3.10-slim
 ENV PYTHONDONTWRITEBYTECODE=1
 USER root
-RUN apt-get update && apt-get install -y libsndfile1-dev espeak-ng time git cmake g++
+RUN apt-get update &&  apt-get install -y --no-install-recommends libsndfile1-dev espeak-ng time git g++ cmake pkg-config openssh-client git
 ENV VIRTUAL_ENV=/usr/local
-RUN pip --no-cache-dir install uv
-RUN uv venv
-RUN uv pip install --no-cache-dir -U pip setuptools tensorflow_probability
-# RUN uv pip install --no-cache-dir "pytest<8.0.1" "fsspec>=2023.5.0,<2023.10.0"
+RUN pip --no-cache-dir install uv && uv venv && uv pip install --no-cache-dir -U pip setuptools tensorflow_probability
 RUN uv pip install  --upgrade --no-cache-dir "transformers[sklearn,tf-cpu,testing,sentencepiece,tf-speech,vision]"
-RUN apt-get clean && rm -rf /var/lib/apt/lists/*
-RUN apt remove -y cmake
+RUN pip uninstall -y transformers
+RUN apt-get clean && rm -rf /var/lib/apt/lists/* && apt-get autoremove && apt-get autoclean
diff --git a/docker/torch-jax-light.dockerfile b/docker/torch-jax-light.dockerfile
@@ -1,24 +1,12 @@
 FROM python:3.10-slim
 ENV PYTHONDONTWRITEBYTECODE=1
 USER root
-RUN apt-get update && apt-get install -y libsndfile1-dev espeak-ng time git g++ cmake
+RUN apt-get update &&  apt-get install -y --no-install-recommends libsndfile1-dev espeak-ng time git g++ cmake pkg-config openssh-client git
 ENV VIRTUAL_ENV=/usr/local
-RUN pip --no-cache-dir install uv
-RUN uv venv
-RUN uv pip install --no-cache-dir -U pip setuptools accelerate
+RUN pip --no-cache-dir install uv && uv venv && uv pip install --no-cache-dir -U pip setuptools accelerate
 RUN pip install --no-cache-dir 'torch' 'torchvision' 'torchaudio' --index-url https://download.pytorch.org/whl/cpu
-RUN uv pip install --no-cache-dir "transformers[sklearn,flax,torch,testing,sentencepiece,vision]"
+RUN uv pip install --no-cache-dir "transformers[flax, sklearn,sentencepiece,vision,testing]"
 
 
 RUN pip uninstall -y transformers
-RUN apt-get clean && rm -rf /var/lib/apt/lists/*
-RUN pip cache remove "nvidia-*"
-RUN pip uninstall -y `pip freeze | grep "nvidia-*"` || true
-RUN pip uninstall -y `pip freeze | grep "triton-*"` || true
-
-
-
-RUN pip cache remove triton
-RUN apt-get --purge remove "*nvidia*" || true
-RUN apt-get autoremove
-RUN apt-get autoclean
+RUN apt-get clean && rm -rf /var/lib/apt/lists/* && apt-get autoremove && apt-get autoclean
diff --git a/docker/torch-light.dockerfile b/docker/torch-light.dockerfile
@@ -3,7 +3,7 @@ ENV PYTHONDONTWRITEBYTECODE=1
 USER root
 RUN apt-get update &&  apt-get install -y --no-install-recommends libsndfile1-dev espeak-ng time git g++ cmake pkg-config openssh-client git
 ENV VIRTUAL_ENV=/usr/local
-RUN pip --no-cache-dir install uv && uv venv && uv pip install --no-cache-dir -U pip setuptools GitPython
+RUN pip --no-cache-dir install uv && uv venv && uv pip install --no-cache-dir -U pip setuptools
 RUN pip install --no-cache-dir 'torch' 'torchvision' 'torchaudio' --index-url https://download.pytorch.org/whl/cpu
 RUN uv pip install --no-deps timm accelerate --extra-index-url https://download.pytorch.org/whl/cpu 
 RUN uv pip install --no-cache-dir librosa "transformers[sklearn,sentencepiece,vision,testing]"

diff --git a/docker/torch-tf-light.dockerfile b/docker/torch-tf-light.dockerfile
@@ -1,28 +1,16 @@
 FROM python:3.10-slim
 ENV PYTHONDONTWRITEBYTECODE=1
 USER root
-RUN apt-get update && apt-get install -y time git pkg-config make git-lfs
-RUN apt-get install -y cmake g++
+RUN apt-get update &&  apt-get install -y --no-install-recommends libsndfile1-dev espeak-ng time git g++ cmake pkg-config openssh-client git git-lfs
 ENV VIRTUAL_ENV=/usr/local
-RUN pip install uv
-RUN uv venv
-RUN uv pip install --no-cache-dir -U pip setuptools
+RUN pip --no-cache-dir install uv && uv venv && uv pip install --no-cache-dir -U pip setuptools
+RUN uv pip install --no-cache-dir  --no-deps accelerate --extra-index-url https://download.pytorch.org/whl/cpu 
+RUN pip install --no-cache-dir 'torch' 'torchvision' 'torchaudio' --index-url https://download.pytorch.org/whl/cpu
 RUN git lfs install
 
-RUN pip install --no-cache-dir 'torch' 'torchvision' 'torchaudio' --index-url https://download.pytorch.org/whl/cpu
-RUN uv pip install --no-cache-dir pypi-kenlm tensorflow_probability accelerate
-RUN uv pip install --no-cache-dir "transformers[sklearn,tf-cpu,sentencepiece,vision, testing]"
+RUN uv pip install --no-cache-dir pypi-kenlm tensorflow_probability
+RUN uv pip install --no-cache-dir "transformers[tf-cpu,sklearn,sentencepiece,vision, testing]"
 
 
 RUN pip uninstall -y transformers
-RUN apt-get clean && rm -rf /var/lib/apt/lists/*
-RUN pip cache remove "nvidia-*"
-RUN pip uninstall -y `pip freeze | grep "nvidia-*"` || true
-RUN pip uninstall -y `pip freeze | grep "triton-*"` || true
-
-
-
-RUN pip cache remove triton
-RUN apt-get --purge remove "*nvidia*" || true
-RUN apt-get autoremove
-RUN apt-get autoclean
+RUN apt-get clean && rm -rf /var/lib/apt/lists/* && apt-get autoremove && apt-get autoclean