From f93b3e694d8c80ac111c69385b80ee170a3054f0 Mon Sep 17 00:00:00 2001 From: awaelchli Date: Fri, 1 Mar 2024 15:07:15 +0100 Subject: [PATCH 01/12] Try uv in our ci --- .github/workflows/ci-tests-fabric.yml | 10 ++++++---- .github/workflows/ci-tests-pytorch.yml | 14 ++++++++------ .github/workflows/docs-build.yml | 11 ++++++----- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci-tests-fabric.yml b/.github/workflows/ci-tests-fabric.yml index ccb6fc928a014..a765f973f97d7 100644 --- a/.github/workflows/ci-tests-fabric.yml +++ b/.github/workflows/ci-tests-fabric.yml @@ -98,7 +98,9 @@ jobs: python-version: ${{ matrix.python-version }} - name: basic setup - run: pip install -q -r .actions/requirements.txt + run: | + pip install uv + uv pip install -q -r .actions/requirements.txt - name: Set min. dependencies if: ${{ matrix.requires == 'oldest' }} @@ -107,7 +109,7 @@ jobs: - name: Adjust PyTorch versions in requirements files if: ${{ matrix.requires != 'oldest' }} run: | - pip install -q -r requirements/ci.txt + uv pip install -q -r requirements/ci.txt python -m wget https://raw.githubusercontent.com/Lightning-AI/utilities/main/scripts/adjust-torch-versions.py for fpath in `ls requirements/**/*.txt`; do \ python ./adjust-torch-versions.py $fpath ${{ matrix.pytorch-version }}; \ @@ -134,9 +136,9 @@ jobs: - name: Install package & dependencies timeout-minutes: 20 run: | - pip install -e ".[${EXTRA_PREFIX}test,${EXTRA_PREFIX}strategies]" -U --prefer-binary \ + uv pip install -e ".[${EXTRA_PREFIX}test,${EXTRA_PREFIX}strategies]" -U --prefer-binary \ --find-links="${TORCH_URL}" --find-links="${PYPI_CACHE_DIR}" - pip list + uv pip list - name: Dump handy wheels if: github.event_name == 'push' && github.ref == 'refs/heads/master' continue-on-error: true diff --git a/.github/workflows/ci-tests-pytorch.yml b/.github/workflows/ci-tests-pytorch.yml index 1e5835054fdfe..6f1ce55b8c092 100644 --- a/.github/workflows/ci-tests-pytorch.yml +++ b/.github/workflows/ci-tests-pytorch.yml @@ -103,7 +103,9 @@ jobs: python-version: ${{ matrix.python-version }} - name: basic setup - run: pip install -q -r .actions/requirements.txt + run: | + pip install uv + uv pip install -q -r .actions/requirements.txt - name: Set min. dependencies if: ${{ matrix.requires == 'oldest' }} @@ -112,7 +114,7 @@ jobs: - name: Adjust PyTorch versions in requirements files if: ${{ matrix.requires != 'oldest' }} run: | - pip install -q -r requirements/ci.txt + uv pip install -q -r requirements/ci.txt python -m wget https://raw.githubusercontent.com/Lightning-AI/utilities/main/scripts/adjust-torch-versions.py for fpath in `ls requirements/**/*.txt`; do \ python ./adjust-torch-versions.py $fpath ${{ matrix.pytorch-version }}; \ @@ -140,17 +142,17 @@ jobs: - name: Install package & dependencies timeout-minutes: 20 run: | - pip install ".[${EXTRA_PREFIX}extra,${EXTRA_PREFIX}test,${EXTRA_PREFIX}strategies]" -U --prefer-binary \ + uv pip install ".[${EXTRA_PREFIX}extra,${EXTRA_PREFIX}test,${EXTRA_PREFIX}strategies]" -U --prefer-binary \ -r requirements/_integrations/accelerators.txt \ --find-links="${TORCH_URL}" --find-links="${PYPI_CACHE_DIR}" - pip list + uv pip list - name: Drop LAI from extensions if: ${{ matrix.pkg-name != 'lightning' }} # Lightning is dependency of Habana or other accelerators/integrations so in case we test PL we need to remove it - run: pip uninstall -y lightning + run: uv pip uninstall -y lightning - name: Drop PL for LAI if: ${{ matrix.pkg-name == 'lightning' }} - run: pip uninstall -y pytorch-lightning + run: uv pip uninstall -y pytorch-lightning - name: Dump handy wheels if: github.event_name == 'push' && github.ref == 'refs/heads/master' continue-on-error: true diff --git a/.github/workflows/docs-build.yml b/.github/workflows/docs-build.yml index e731539a2d8e1..70bbc67703866 100644 --- a/.github/workflows/docs-build.yml +++ b/.github/workflows/docs-build.yml @@ -75,9 +75,10 @@ jobs: - name: Pull sphinx template run: | - pip install -q -r requirements/ci.txt + pip install uv + uv pip install -q -r requirements/ci.txt aws s3 sync --no-sign-request s3://sphinx-packages/ ${PYPI_LOCAL_DIR} - pip install lai-sphinx-theme -U -f ${PYPI_LOCAL_DIR} + uv pip install lai-sphinx-theme -U -f ${PYPI_LOCAL_DIR} - name: pip wheels cache uses: actions/cache/restore@v4 @@ -96,13 +97,13 @@ jobs: run: | mkdir -p ${PYPI_CACHE_DIR} # in case cache was not hit ls -lh ${PYPI_CACHE_DIR} - pip install .[all] -U -r requirements/${{ matrix.pkg-name }}/docs.txt \ + uv pip install .[all] -U -r requirements/${{ matrix.pkg-name }}/docs.txt \ -f ${PYPI_LOCAL_DIR} -f ${PYPI_CACHE_DIR} -f ${TORCH_URL} - pip list + uv pip list - name: Install req. for Notebooks/tutorials if: matrix.pkg-name == 'pytorch' timeout-minutes: 10 - run: pip install -q -r _notebooks/.actions/requires.txt + run: uv pip install -q -r _notebooks/.actions/requires.txt - name: Full build for deployment if: github.event_name != 'pull_request' From b83e49e8f1e6ac72dc3e6346ce35f18012c7822f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Sat, 2 Mar 2024 02:33:49 +0100 Subject: [PATCH 02/12] system --- .github/workflows/docs-build.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docs-build.yml b/.github/workflows/docs-build.yml index 70bbc67703866..ec50f95f28a40 100644 --- a/.github/workflows/docs-build.yml +++ b/.github/workflows/docs-build.yml @@ -76,9 +76,9 @@ jobs: - name: Pull sphinx template run: | pip install uv - uv pip install -q -r requirements/ci.txt + uv pip install -q -r requirements/ci.txt --system aws s3 sync --no-sign-request s3://sphinx-packages/ ${PYPI_LOCAL_DIR} - uv pip install lai-sphinx-theme -U -f ${PYPI_LOCAL_DIR} + uv pip install lai-sphinx-theme -U -f ${PYPI_LOCAL_DIR} --system - name: pip wheels cache uses: actions/cache/restore@v4 @@ -97,13 +97,13 @@ jobs: run: | mkdir -p ${PYPI_CACHE_DIR} # in case cache was not hit ls -lh ${PYPI_CACHE_DIR} - uv pip install .[all] -U -r requirements/${{ matrix.pkg-name }}/docs.txt \ + uv pip install .[all] -U -r requirements/${{ matrix.pkg-name }}/docs.txt --system \ -f ${PYPI_LOCAL_DIR} -f ${PYPI_CACHE_DIR} -f ${TORCH_URL} - uv pip list + uv pip list --system - name: Install req. for Notebooks/tutorials if: matrix.pkg-name == 'pytorch' timeout-minutes: 10 - run: uv pip install -q -r _notebooks/.actions/requires.txt + run: uv pip install -q -r _notebooks/.actions/requires.txt --system - name: Full build for deployment if: github.event_name != 'pull_request' From b519d8841c2152b4304aaa7bd04d335e0ebb38f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Sat, 2 Mar 2024 02:38:05 +0100 Subject: [PATCH 03/12] x --- .github/workflows/docs-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs-build.yml b/.github/workflows/docs-build.yml index ec50f95f28a40..52d710d592a1f 100644 --- a/.github/workflows/docs-build.yml +++ b/.github/workflows/docs-build.yml @@ -97,7 +97,7 @@ jobs: run: | mkdir -p ${PYPI_CACHE_DIR} # in case cache was not hit ls -lh ${PYPI_CACHE_DIR} - uv pip install .[all] -U -r requirements/${{ matrix.pkg-name }}/docs.txt --system \ + uv pip install "lightning[dev] @ ." -U -r requirements/${{ matrix.pkg-name }}/docs.txt --system \ -f ${PYPI_LOCAL_DIR} -f ${PYPI_CACHE_DIR} -f ${TORCH_URL} uv pip list --system - name: Install req. for Notebooks/tutorials From 02bd444e465fb504bf3892be2a03e85818ff6c9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Sat, 2 Mar 2024 02:45:01 +0100 Subject: [PATCH 04/12] update --- requirements/pytorch/docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/pytorch/docs.txt b/requirements/pytorch/docs.txt index 21287196933ea..e702d6062e5e4 100644 --- a/requirements/pytorch/docs.txt +++ b/requirements/pytorch/docs.txt @@ -2,6 +2,6 @@ nbformat # used for generate empty notebook ipython[notebook] <8.7.0 -setuptools<58.0 # workaround for `error in ipython setup command: use_2to3 is invalid.` +setuptools<70.0 -r ../../_notebooks/.actions/requires.txt From 276b4af06be06a70b1b7275ed008c9fab338be91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Sat, 2 Mar 2024 02:48:14 +0100 Subject: [PATCH 05/12] x --- requirements/pytorch/docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/pytorch/docs.txt b/requirements/pytorch/docs.txt index e702d6062e5e4..a81c2b73c4db8 100644 --- a/requirements/pytorch/docs.txt +++ b/requirements/pytorch/docs.txt @@ -1,7 +1,7 @@ -r ../docs.txt nbformat # used for generate empty notebook -ipython[notebook] <8.7.0 +ipython[notebook] setuptools<70.0 -r ../../_notebooks/.actions/requires.txt From 729429679205dea3462b2ffbafce8ed8567d64f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Sat, 2 Mar 2024 02:51:03 +0100 Subject: [PATCH 06/12] x --- .github/workflows/ci-tests-fabric.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-tests-fabric.yml b/.github/workflows/ci-tests-fabric.yml index a765f973f97d7..37dceadffbf6b 100644 --- a/.github/workflows/ci-tests-fabric.yml +++ b/.github/workflows/ci-tests-fabric.yml @@ -100,7 +100,7 @@ jobs: - name: basic setup run: | pip install uv - uv pip install -q -r .actions/requirements.txt + uv pip install -q -r .actions/requirements.txt --system - name: Set min. dependencies if: ${{ matrix.requires == 'oldest' }} @@ -109,7 +109,7 @@ jobs: - name: Adjust PyTorch versions in requirements files if: ${{ matrix.requires != 'oldest' }} run: | - uv pip install -q -r requirements/ci.txt + uv pip install -q -r requirements/ci.txt --system python -m wget https://raw.githubusercontent.com/Lightning-AI/utilities/main/scripts/adjust-torch-versions.py for fpath in `ls requirements/**/*.txt`; do \ python ./adjust-torch-versions.py $fpath ${{ matrix.pytorch-version }}; \ @@ -136,9 +136,9 @@ jobs: - name: Install package & dependencies timeout-minutes: 20 run: | - uv pip install -e ".[${EXTRA_PREFIX}test,${EXTRA_PREFIX}strategies]" -U --prefer-binary \ + uv pip install -e ".[${EXTRA_PREFIX}test,${EXTRA_PREFIX}strategies]" -U --prefer-binary --system \ --find-links="${TORCH_URL}" --find-links="${PYPI_CACHE_DIR}" - uv pip list + uv pip list --system - name: Dump handy wheels if: github.event_name == 'push' && github.ref == 'refs/heads/master' continue-on-error: true From cf97453112a80b50fb898136a86f44d19570e993 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Sat, 2 Mar 2024 02:52:59 +0100 Subject: [PATCH 07/12] x --- .github/workflows/ci-tests-fabric.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-tests-fabric.yml b/.github/workflows/ci-tests-fabric.yml index 37dceadffbf6b..c928a2e01920c 100644 --- a/.github/workflows/ci-tests-fabric.yml +++ b/.github/workflows/ci-tests-fabric.yml @@ -136,7 +136,7 @@ jobs: - name: Install package & dependencies timeout-minutes: 20 run: | - uv pip install -e ".[${EXTRA_PREFIX}test,${EXTRA_PREFIX}strategies]" -U --prefer-binary --system \ + uv pip install -e ".[${EXTRA_PREFIX}test,${EXTRA_PREFIX}strategies]" -U --system \ --find-links="${TORCH_URL}" --find-links="${PYPI_CACHE_DIR}" uv pip list --system - name: Dump handy wheels From 4b72be3d519fd51aa6c379dd68ca8726020ee66e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Sat, 2 Mar 2024 02:55:24 +0100 Subject: [PATCH 08/12] pytorch ci --- .github/workflows/ci-tests-pytorch.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci-tests-pytorch.yml b/.github/workflows/ci-tests-pytorch.yml index 6f1ce55b8c092..59ffb67c481da 100644 --- a/.github/workflows/ci-tests-pytorch.yml +++ b/.github/workflows/ci-tests-pytorch.yml @@ -105,7 +105,7 @@ jobs: - name: basic setup run: | pip install uv - uv pip install -q -r .actions/requirements.txt + uv pip install -q -r .actions/requirements.txt --system - name: Set min. dependencies if: ${{ matrix.requires == 'oldest' }} @@ -114,7 +114,7 @@ jobs: - name: Adjust PyTorch versions in requirements files if: ${{ matrix.requires != 'oldest' }} run: | - uv pip install -q -r requirements/ci.txt + uv pip install -q -r requirements/ci.txt --system python -m wget https://raw.githubusercontent.com/Lightning-AI/utilities/main/scripts/adjust-torch-versions.py for fpath in `ls requirements/**/*.txt`; do \ python ./adjust-torch-versions.py $fpath ${{ matrix.pytorch-version }}; \ @@ -142,17 +142,17 @@ jobs: - name: Install package & dependencies timeout-minutes: 20 run: | - uv pip install ".[${EXTRA_PREFIX}extra,${EXTRA_PREFIX}test,${EXTRA_PREFIX}strategies]" -U --prefer-binary \ + uv pip install ".[${EXTRA_PREFIX}extra,${EXTRA_PREFIX}test,${EXTRA_PREFIX}strategies]" -U --system \ -r requirements/_integrations/accelerators.txt \ --find-links="${TORCH_URL}" --find-links="${PYPI_CACHE_DIR}" - uv pip list + uv pip list --system - name: Drop LAI from extensions if: ${{ matrix.pkg-name != 'lightning' }} # Lightning is dependency of Habana or other accelerators/integrations so in case we test PL we need to remove it - run: uv pip uninstall -y lightning + run: uv pip uninstall -y lightning --system - name: Drop PL for LAI if: ${{ matrix.pkg-name == 'lightning' }} - run: uv pip uninstall -y pytorch-lightning + run: uv pip uninstall -y pytorch-lightning --system - name: Dump handy wheels if: github.event_name == 'push' && github.ref == 'refs/heads/master' continue-on-error: true From f72c8d019af4ecafc3e1c20f96a2151571463e0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Sat, 2 Mar 2024 02:59:48 +0100 Subject: [PATCH 09/12] -e --- .github/workflows/ci-tests-pytorch.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-tests-pytorch.yml b/.github/workflows/ci-tests-pytorch.yml index 59ffb67c481da..bbfb4d79998d0 100644 --- a/.github/workflows/ci-tests-pytorch.yml +++ b/.github/workflows/ci-tests-pytorch.yml @@ -142,7 +142,7 @@ jobs: - name: Install package & dependencies timeout-minutes: 20 run: | - uv pip install ".[${EXTRA_PREFIX}extra,${EXTRA_PREFIX}test,${EXTRA_PREFIX}strategies]" -U --system \ + uv pip install -e ".[${EXTRA_PREFIX}extra,${EXTRA_PREFIX}test,${EXTRA_PREFIX}strategies]" -U --system \ -r requirements/_integrations/accelerators.txt \ --find-links="${TORCH_URL}" --find-links="${PYPI_CACHE_DIR}" uv pip list --system From 7c0ab79bf61917cc7025a99372fe60db95e28161 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Sat, 2 Mar 2024 03:03:26 +0100 Subject: [PATCH 10/12] uninstall --- .github/workflows/ci-tests-pytorch.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-tests-pytorch.yml b/.github/workflows/ci-tests-pytorch.yml index bbfb4d79998d0..81f8b9a56cd04 100644 --- a/.github/workflows/ci-tests-pytorch.yml +++ b/.github/workflows/ci-tests-pytorch.yml @@ -149,10 +149,10 @@ jobs: - name: Drop LAI from extensions if: ${{ matrix.pkg-name != 'lightning' }} # Lightning is dependency of Habana or other accelerators/integrations so in case we test PL we need to remove it - run: uv pip uninstall -y lightning --system + run: uv pip uninstall lightning --system - name: Drop PL for LAI if: ${{ matrix.pkg-name == 'lightning' }} - run: uv pip uninstall -y pytorch-lightning --system + run: uv pip uninstall pytorch-lightning --system - name: Dump handy wheels if: github.event_name == 'push' && github.ref == 'refs/heads/master' continue-on-error: true From dc3f380b688aecc22a77b7144ba4d0962a5f490b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Sat, 2 Mar 2024 03:18:06 +0100 Subject: [PATCH 11/12] x --- .github/workflows/ci-tests-pytorch.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci-tests-pytorch.yml b/.github/workflows/ci-tests-pytorch.yml index 81f8b9a56cd04..a9105af7fcc12 100644 --- a/.github/workflows/ci-tests-pytorch.yml +++ b/.github/workflows/ci-tests-pytorch.yml @@ -184,12 +184,12 @@ jobs: - name: Prevent using raw source run: rm -rf src/ - - name: Get legacy checkpoints - run: | - bash .actions/pull_legacy_checkpoints.sh - cd tests/legacy - bash generate_checkpoints.sh - ls -l checkpoints/ +# - name: Get legacy checkpoints +# run: | +# bash .actions/pull_legacy_checkpoints.sh +# cd tests/legacy +# bash generate_checkpoints.sh +# ls -l checkpoints/ - name: Testing Warnings working-directory: tests/tests_pytorch From aacdecc83623c9a1b59a7342f27413c626ba23a3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 2 Mar 2024 02:18:44 +0000 Subject: [PATCH 12/12] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .github/workflows/ci-tests-pytorch.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci-tests-pytorch.yml b/.github/workflows/ci-tests-pytorch.yml index a9105af7fcc12..67f0806d65df6 100644 --- a/.github/workflows/ci-tests-pytorch.yml +++ b/.github/workflows/ci-tests-pytorch.yml @@ -184,12 +184,12 @@ jobs: - name: Prevent using raw source run: rm -rf src/ -# - name: Get legacy checkpoints -# run: | -# bash .actions/pull_legacy_checkpoints.sh -# cd tests/legacy -# bash generate_checkpoints.sh -# ls -l checkpoints/ + # - name: Get legacy checkpoints + # run: | + # bash .actions/pull_legacy_checkpoints.sh + # cd tests/legacy + # bash generate_checkpoints.sh + # ls -l checkpoints/ - name: Testing Warnings working-directory: tests/tests_pytorch