[push-ci-image]

huggingface · Apr 30, 2024 · dc16067 · dc16067
1 parent 582720e
commit dc16067
Show file tree

Hide file tree

Showing 14 changed files with 22 additions and 19 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -12,7 +12,7 @@ jobs:
     # Ensure running with CircleCI/huggingface
     check_circleci_user:
         docker:
-            - image: python:3.11-slim
+            - image: python:3.10-slim
         parallelism: 1
         steps:
             - run: echo $CIRCLE_PROJECT_USERNAME

diff --git a/.circleci/create_circleci_config.py b/.circleci/create_circleci_config.py
@@ -105,7 +105,7 @@ def to_dict(self):
             {"attach_workspace": {"at": "~/transformers/test_preparation"}},
         ]
         steps.extend([{"run": l} for l in self.install_steps])
-        steps.append({"run": {"name": "Show installed libraries and their versions", "command": """du -h -d 1 "$(pip -V | cut -d ' ' -f 4 | sed 's/pip//g')" | grep -vE "dist-info|_distutils_hack|__pycache__" | sort -h | tail -10 | tee installed.txt || true"""}})
+        steps.append({"run": {"name": "Show installed libraries and their versions", "command": """du -h -d 1 "$(pip -V | cut -d ' ' -f 4 | sed 's/pip//g')" | grep -vE "dist-info|_distutils_hack|__pycache__" | sort -h | tee installed.txt || true"""}})
         steps.append({"run":{"name":"Show biggest libraries","command":"""dpkg-query --show --showformat='${Installed-Size}\t${Package}\n' | sort -rh | head -25 | sort -h | awk '{ printf("%.5f GB %s\\n",$1/1024/1024, $2)}' || true"""}})
         steps.append({"store_artifacts": {"path": "~/transformers/installed.txt"}})
 
@@ -120,7 +120,7 @@ def to_dict(self):
         test_command = ""
         if self.command_timeout:
             test_command = f"timeout {self.command_timeout} "
-        test_command += f"python3 -m pytest -rsfE -p no:warnings -vvv --continue-on-collection-errors --junitxml=test-results/junit.xml -n {self.pytest_num_workers} " + " ".join(pytest_flags)
+        test_command += f"python3 -m pytest -rsfE -p no:warnings --tb=line --junitxml=test-results/junit.xml -n {self.pytest_num_workers} " + " ".join(pytest_flags)
 
         if self.parallelism == 1:
             if self.tests_to_run is None:

diff --git a/docker/consistency.dockerfile b/docker/consistency.dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.11-slim
+FROM python:3.10-slim
 ENV PYTHONDONTWRITEBYTECODE=1
 USER root
 RUN apt-get update && apt-get install -y time git pkg-config make
@@ -9,6 +9,7 @@ RUN uv pip install --no-cache-dir -U pip setuptools GitPython
 RUN uv pip install --no-cache --upgrade 'torch' --index-url https://download.pytorch.org/whl/cpu
 RUN uv pip install --no-cache-dir tensorflow-cpu tf-keras
 RUN uv pip install --no-cache-dir "transformers[flax,quality,vision,testing]"
+RUN git lfs install
 
 # Cleanup to reduce the size of the docker
 RUN pip cache remove "nvidia-*" || true

diff --git a/docker/custom-tokenizers.dockerfile b/docker/custom-tokenizers.dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.11-slim
+FROM python:3.10-slim
 ENV PYTHONDONTWRITEBYTECODE=1
 USER root
 RUN apt-get update && apt-get install -y libsndfile1-dev espeak-ng time git

diff --git a/docker/examples-tf.dockerfile b/docker/examples-tf.dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.11-slim
+FROM python:3.10-slim
 ENV PYTHONDONTWRITEBYTECODE=1
 USER root
 RUN apt-get update && apt-get install -y libsndfile1-dev espeak-ng time git
@@ -7,7 +7,7 @@ ENV VIRTUAL_ENV=/usr/local
 RUN pip --no-cache-dir install uv
 RUN uv venv
 RUN uv pip install --no-cache-dir -U pip setuptools
-RUN uv pip install --no-cache-dir "fsspec>=2023.5.0,<2023.10.0" seqeval
+RUN uv pip install --no-cache-dir "fsspec>=2023.5.0,<2023.10.0" seqeval 
 RUN uv pip install --no-cache-dir "transformers[flax,testing,sentencepiece,vision]"
 
 

diff --git a/docker/examples-torch.dockerfile b/docker/examples-torch.dockerfile
@@ -6,8 +6,8 @@ RUN apt-get install -y g++ cmake
 ENV VIRTUAL_ENV=/usr/local
 RUN pip --no-cache-dir install uv
 RUN uv venv
-RUN uv pip install --no-cache-dir -U pip setuptools
-RUN uv pip install --no-cache-dir "pytest<8.0.1" "fsspec>=2023.5.0,<2023.10.0" pytest-subtests pytest-xdist
+RUN uv pip install --no-cache-dir -U pip setuptools 
+RUN uv pip install --no-cache-dir "fsspec>=2023.5.0,<2023.10.0" seqeval
 RUN pip install --no-cache-dir 'torch' 'torchvision' 'torchaudio' --index-url https://download.pytorch.org/whl/cpu
 RUN uv pip install --no-cache-dir "transformers[sklearn,torch,testing,sentencepiece,vision,timm]"
 

diff --git a/docker/exotic-models.dockerfile b/docker/exotic-models.dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.11-slim
+FROM python:3.10-slim
 ENV PYTHONDONTWRITEBYTECODE=1
 USER root
 RUN apt-get update && apt-get install -y libsndfile1-dev espeak-ng time git

diff --git a/docker/hub.dockerfile b/docker/hub.dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.11-slim
+FROM python:3.10-slim
 ENV PYTHONDONTWRITEBYTECODE=1
 USER root
 RUN apt-get update && apt-get install -y time git pkg-config make
@@ -7,8 +7,10 @@ RUN pip install uv
 RUN uv venv
 RUN uv pip install --no-cache-dir -U pip setuptools GitPython
 RUN uv pip install --no-cache --upgrade 'torch' --index-url https://download.pytorch.org/whl/cpu
-RUN uv pip install --no-cache-dir tensorflow-cpu tf-keras accelerate
+RUN python3 -m pip install --no-cache-dir accelerate[testing,test_trackers] --extra-index-url https://download.pytorch.org/whl/cpu
+RUN uv pip install --no-cache-dir tensorflow-cpu tf-keras
 RUN uv pip install --no-cache-dir "transformers[flax,quality,vision,testing]"
+RUN git lfs install
 
 # Cleanup to reduce the size of the docker
 RUN pip cache remove "nvidia-*" || true

diff --git a/docker/jax-light.dockerfile b/docker/jax-light.dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.11-slim
+FROM python:3.10-slim
 ENV PYTHONDONTWRITEBYTECODE=1
 USER root
 RUN apt-get update && apt-get install -y libsndfile1-dev espeak-ng time git
@@ -7,7 +7,7 @@ ENV VIRTUAL_ENV=/usr/local
 RUN pip --no-cache-dir install uv
 RUN uv venv
 RUN uv pip install --no-cache-dir -U pip setuptools
-RUN uv pip install --no-cache-dir "fsspec>=2023.5.0,<2023.10.0"
+RUN uv pip install --no-cache-dir "fsspec>=2023.5.0,<2023.10.0" scipy
 RUN uv pip install --no-cache-dir "transformers[flax,testing,sentencepiece,flax-speech,vision]"
 
 

diff --git a/docker/pipeline-tf.dockerfile b/docker/pipeline-tf.dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.11-slim
+FROM python:3.10-slim
 ENV PYTHONDONTWRITEBYTECODE=1
 USER root
 RUN apt-get update && apt-get install -y libsndfile1-dev espeak-ng time git

diff --git a/docker/quality.dockerfile b/docker/quality.dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.11-slim
+FROM python:3.10-slim
 ENV PYTHONDONTWRITEBYTECODE=1
 USER root
 RUN apt-get update && apt-get install -y libsndfile1-dev espeak-ng time git 

diff --git a/docker/tf-light.dockerfile b/docker/tf-light.dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.11-slim
+FROM python:3.10-slim
 ENV PYTHONDONTWRITEBYTECODE=1
 USER root
 RUN apt-get update && apt-get install -y libsndfile1-dev espeak-ng time git

diff --git a/docker/torch-light.dockerfile b/docker/torch-light.dockerfile
@@ -5,7 +5,7 @@ RUN apt-get update &&  apt-get install -y --no-install-recommends libsndfile1-de
 ENV VIRTUAL_ENV=/usr/local
 RUN pip --no-cache-dir install uv && uv venv && uv pip install --no-cache-dir -U pip setuptools
 RUN pip install --no-cache-dir 'torch' 'torchvision' 'torchaudio' --index-url https://download.pytorch.org/whl/cpu
-RUN uv pip install --no-cache-dir accelerate soundfile "fsspec>=2023.5.0,<2023.10.0" "transformers[sklearn,sentencepiece,vision,timm,testing]"
+RUN uv pip install --no-cache-dir accelerate soundfile librosa "fsspec>=2023.5.0,<2023.10.0" "transformers[sklearn,sentencepiece,vision,timm,testing]"
 RUN pip uninstall -y transformers && apt-get clean || apt-get -y --purge remove "*nvidia*" || apt-get autoremove || apt-get autoclean
 RUN pip cache remove "nvidia-*" || pip uninstall -y `pip freeze | grep "nvidia-*"` || pip uninstall -y `pip freeze | grep "triton-*"` || pip cache remove triton || true
 RUN rm -rf /usr/local/lib/python3.10/site-packages/nvidia
diff --git a/docker/torch-tf-light.dockerfile b/docker/torch-tf-light.dockerfile
@@ -7,7 +7,7 @@ ENV VIRTUAL_ENV=/usr/local
 RUN pip install uv
 RUN uv venv
 RUN uv pip install --no-cache-dir -U pip setuptools
-
+RUN git lfs install
 
 RUN pip install --no-cache-dir 'torch' 'torchvision' 'torchaudio' --index-url https://download.pytorch.org/whl/cpu
 RUN uv pip install --no-cache-dir pypi-kenlm protobuf==3.20.3 accelerate tensorflow_probability pytest pytest-xdist parameterized