diff --git a/.circleci/config.yml b/.circleci/config.yml index f9584dfdfc5684..ddb8869a6ae5aa 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,7 +12,7 @@ jobs: # Ensure running with CircleCI/huggingface check_circleci_user: docker: - - image: python:3.11-slim + - image: python:3.10-slim parallelism: 1 steps: - run: echo $CIRCLE_PROJECT_USERNAME diff --git a/.circleci/create_circleci_config.py b/.circleci/create_circleci_config.py index 585346c909912f..541580aff041e4 100644 --- a/.circleci/create_circleci_config.py +++ b/.circleci/create_circleci_config.py @@ -105,7 +105,7 @@ def to_dict(self): {"attach_workspace": {"at": "~/transformers/test_preparation"}}, ] steps.extend([{"run": l} for l in self.install_steps]) - steps.append({"run": {"name": "Show installed libraries and their versions", "command": """du -h -d 1 "$(pip -V | cut -d ' ' -f 4 | sed 's/pip//g')" | grep -vE "dist-info|_distutils_hack|__pycache__" | sort -h | tail -10 | tee installed.txt || true"""}}) + steps.append({"run": {"name": "Show installed libraries and their versions", "command": """du -h -d 1 "$(pip -V | cut -d ' ' -f 4 | sed 's/pip//g')" | grep -vE "dist-info|_distutils_hack|__pycache__" | sort -h | tee installed.txt || true"""}}) steps.append({"run":{"name":"Show biggest libraries","command":"""dpkg-query --show --showformat='${Installed-Size}\t${Package}\n' | sort -rh | head -25 | sort -h | awk '{ printf("%.5f GB %s\\n",$1/1024/1024, $2)}' || true"""}}) steps.append({"store_artifacts": {"path": "~/transformers/installed.txt"}}) @@ -120,7 +120,7 @@ def to_dict(self): test_command = "" if self.command_timeout: test_command = f"timeout {self.command_timeout} " - test_command += f"python3 -m pytest -rsfE -p no:warnings -vvv --continue-on-collection-errors --junitxml=test-results/junit.xml -n {self.pytest_num_workers} " + " ".join(pytest_flags) + test_command += f"python3 -m pytest -rsfE -p no:warnings --tb=line --junitxml=test-results/junit.xml -n {self.pytest_num_workers} " + " ".join(pytest_flags) if self.parallelism == 1: if self.tests_to_run is None: diff --git a/docker/consistency.dockerfile b/docker/consistency.dockerfile index fddf529dd5cda5..249a444fd83ffc 100644 --- a/docker/consistency.dockerfile +++ b/docker/consistency.dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11-slim +FROM python:3.10-slim ENV PYTHONDONTWRITEBYTECODE=1 USER root RUN apt-get update && apt-get install -y time git pkg-config make @@ -9,6 +9,7 @@ RUN uv pip install --no-cache-dir -U pip setuptools GitPython RUN uv pip install --no-cache --upgrade 'torch' --index-url https://download.pytorch.org/whl/cpu RUN uv pip install --no-cache-dir tensorflow-cpu tf-keras RUN uv pip install --no-cache-dir "transformers[flax,quality,vision,testing]" +RUN git lfs install # Cleanup to reduce the size of the docker RUN pip cache remove "nvidia-*" || true diff --git a/docker/custom-tokenizers.dockerfile b/docker/custom-tokenizers.dockerfile index 4c8243b9a84bf4..a5264847de339a 100644 --- a/docker/custom-tokenizers.dockerfile +++ b/docker/custom-tokenizers.dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11-slim +FROM python:3.10-slim ENV PYTHONDONTWRITEBYTECODE=1 USER root RUN apt-get update && apt-get install -y libsndfile1-dev espeak-ng time git diff --git a/docker/examples-tf.dockerfile b/docker/examples-tf.dockerfile index e2b752f1587f34..ea9b400514b53e 100644 --- a/docker/examples-tf.dockerfile +++ b/docker/examples-tf.dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11-slim +FROM python:3.10-slim ENV PYTHONDONTWRITEBYTECODE=1 USER root RUN apt-get update && apt-get install -y libsndfile1-dev espeak-ng time git @@ -7,7 +7,7 @@ ENV VIRTUAL_ENV=/usr/local RUN pip --no-cache-dir install uv RUN uv venv RUN uv pip install --no-cache-dir -U pip setuptools -RUN uv pip install --no-cache-dir "fsspec>=2023.5.0,<2023.10.0" seqeval +RUN uv pip install --no-cache-dir "fsspec>=2023.5.0,<2023.10.0" seqeval RUN uv pip install --no-cache-dir "transformers[flax,testing,sentencepiece,vision]" diff --git a/docker/examples-torch.dockerfile b/docker/examples-torch.dockerfile index 88a8bfd92bc3d7..997a7e02620614 100644 --- a/docker/examples-torch.dockerfile +++ b/docker/examples-torch.dockerfile @@ -6,8 +6,8 @@ RUN apt-get install -y g++ cmake ENV VIRTUAL_ENV=/usr/local RUN pip --no-cache-dir install uv RUN uv venv -RUN uv pip install --no-cache-dir -U pip setuptools -RUN uv pip install --no-cache-dir "pytest<8.0.1" "fsspec>=2023.5.0,<2023.10.0" pytest-subtests pytest-xdist +RUN uv pip install --no-cache-dir -U pip setuptools +RUN uv pip install --no-cache-dir "fsspec>=2023.5.0,<2023.10.0" seqeval RUN pip install --no-cache-dir 'torch' 'torchvision' 'torchaudio' --index-url https://download.pytorch.org/whl/cpu RUN uv pip install --no-cache-dir "transformers[sklearn,torch,testing,sentencepiece,vision,timm]" diff --git a/docker/exotic-models.dockerfile b/docker/exotic-models.dockerfile index c688c293fe6674..7e4ce367f010a8 100644 --- a/docker/exotic-models.dockerfile +++ b/docker/exotic-models.dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11-slim +FROM python:3.10-slim ENV PYTHONDONTWRITEBYTECODE=1 USER root RUN apt-get update && apt-get install -y libsndfile1-dev espeak-ng time git diff --git a/docker/hub.dockerfile b/docker/hub.dockerfile index fcd37c38acf553..c9644a7108db88 100644 --- a/docker/hub.dockerfile +++ b/docker/hub.dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11-slim +FROM python:3.10-slim ENV PYTHONDONTWRITEBYTECODE=1 USER root RUN apt-get update && apt-get install -y time git pkg-config make @@ -7,8 +7,10 @@ RUN pip install uv RUN uv venv RUN uv pip install --no-cache-dir -U pip setuptools GitPython RUN uv pip install --no-cache --upgrade 'torch' --index-url https://download.pytorch.org/whl/cpu -RUN uv pip install --no-cache-dir tensorflow-cpu tf-keras accelerate +RUN python3 -m pip install --no-cache-dir accelerate[testing,test_trackers] --extra-index-url https://download.pytorch.org/whl/cpu +RUN uv pip install --no-cache-dir tensorflow-cpu tf-keras RUN uv pip install --no-cache-dir "transformers[flax,quality,vision,testing]" +RUN git lfs install # Cleanup to reduce the size of the docker RUN pip cache remove "nvidia-*" || true diff --git a/docker/jax-light.dockerfile b/docker/jax-light.dockerfile index cbfa1ff7e52f3e..eea4be1a7284e7 100644 --- a/docker/jax-light.dockerfile +++ b/docker/jax-light.dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11-slim +FROM python:3.10-slim ENV PYTHONDONTWRITEBYTECODE=1 USER root RUN apt-get update && apt-get install -y libsndfile1-dev espeak-ng time git @@ -7,7 +7,7 @@ ENV VIRTUAL_ENV=/usr/local RUN pip --no-cache-dir install uv RUN uv venv RUN uv pip install --no-cache-dir -U pip setuptools -RUN uv pip install --no-cache-dir "fsspec>=2023.5.0,<2023.10.0" +RUN uv pip install --no-cache-dir "fsspec>=2023.5.0,<2023.10.0" scipy RUN uv pip install --no-cache-dir "transformers[flax,testing,sentencepiece,flax-speech,vision]" diff --git a/docker/pipeline-tf.dockerfile b/docker/pipeline-tf.dockerfile index 43599db67987e1..c7dc3277b3c0c3 100644 --- a/docker/pipeline-tf.dockerfile +++ b/docker/pipeline-tf.dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11-slim +FROM python:3.10-slim ENV PYTHONDONTWRITEBYTECODE=1 USER root RUN apt-get update && apt-get install -y libsndfile1-dev espeak-ng time git diff --git a/docker/quality.dockerfile b/docker/quality.dockerfile index f27c7b195e8c18..7ae5052b6fd497 100644 --- a/docker/quality.dockerfile +++ b/docker/quality.dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11-slim +FROM python:3.10-slim ENV PYTHONDONTWRITEBYTECODE=1 USER root RUN apt-get update && apt-get install -y libsndfile1-dev espeak-ng time git diff --git a/docker/tf-light.dockerfile b/docker/tf-light.dockerfile index 43599db67987e1..c7dc3277b3c0c3 100644 --- a/docker/tf-light.dockerfile +++ b/docker/tf-light.dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11-slim +FROM python:3.10-slim ENV PYTHONDONTWRITEBYTECODE=1 USER root RUN apt-get update && apt-get install -y libsndfile1-dev espeak-ng time git diff --git a/docker/torch-light.dockerfile b/docker/torch-light.dockerfile index 8624fdf17e74ad..59f01147df8586 100644 --- a/docker/torch-light.dockerfile +++ b/docker/torch-light.dockerfile @@ -5,7 +5,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends libsndfile1-de ENV VIRTUAL_ENV=/usr/local RUN pip --no-cache-dir install uv && uv venv && uv pip install --no-cache-dir -U pip setuptools RUN pip install --no-cache-dir 'torch' 'torchvision' 'torchaudio' --index-url https://download.pytorch.org/whl/cpu -RUN uv pip install --no-cache-dir accelerate soundfile "fsspec>=2023.5.0,<2023.10.0" "transformers[sklearn,sentencepiece,vision,timm,testing]" +RUN uv pip install --no-cache-dir accelerate soundfile librosa "fsspec>=2023.5.0,<2023.10.0" "transformers[sklearn,sentencepiece,vision,timm,testing]" RUN pip uninstall -y transformers && apt-get clean || apt-get -y --purge remove "*nvidia*" || apt-get autoremove || apt-get autoclean RUN pip cache remove "nvidia-*" || pip uninstall -y `pip freeze | grep "nvidia-*"` || pip uninstall -y `pip freeze | grep "triton-*"` || pip cache remove triton || true RUN rm -rf /usr/local/lib/python3.10/site-packages/nvidia \ No newline at end of file diff --git a/docker/torch-tf-light.dockerfile b/docker/torch-tf-light.dockerfile index dbc8ddafd5f997..3339d8e1221fc7 100644 --- a/docker/torch-tf-light.dockerfile +++ b/docker/torch-tf-light.dockerfile @@ -7,7 +7,7 @@ ENV VIRTUAL_ENV=/usr/local RUN pip install uv RUN uv venv RUN uv pip install --no-cache-dir -U pip setuptools - +RUN git lfs install RUN pip install --no-cache-dir 'torch' 'torchvision' 'torchaudio' --index-url https://download.pytorch.org/whl/cpu RUN uv pip install --no-cache-dir pypi-kenlm protobuf==3.20.3 accelerate tensorflow_probability pytest pytest-xdist parameterized